1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21
22 import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23 import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24 import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25 import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29 import static org.junit.Assert.assertArrayEquals;
30 import static org.junit.Assert.assertEquals;
31 import static org.junit.Assert.assertFalse;
32 import static org.junit.Assert.assertNotNull;
33 import static org.junit.Assert.assertNull;
34 import static org.junit.Assert.assertTrue;
35 import static org.junit.Assert.fail;
36 import static org.mockito.Matchers.any;
37 import static org.mockito.Matchers.anyBoolean;
38 import static org.mockito.Matchers.anyLong;
39 import static org.mockito.Mockito.doThrow;
40 import static org.mockito.Mockito.mock;
41 import static org.mockito.Mockito.never;
42 import static org.mockito.Mockito.spy;
43 import static org.mockito.Mockito.times;
44 import static org.mockito.Mockito.verify;
45 import static org.mockito.Mockito.when;
46
47 import java.io.IOException;
48 import java.io.InterruptedIOException;
49 import java.security.PrivilegedExceptionAction;
50 import java.util.ArrayList;
51 import java.util.Arrays;
52 import java.util.Collection;
53 import java.util.Collections;
54 import java.util.HashMap;
55 import java.util.List;
56 import java.util.Map;
57 import java.util.NavigableMap;
58 import java.util.TreeMap;
59 import java.util.UUID;
60 import java.util.concurrent.Callable;
61 import java.util.concurrent.CountDownLatch;
62 import java.util.concurrent.ExecutorService;
63 import java.util.concurrent.Executors;
64 import java.util.concurrent.Future;
65 import java.util.concurrent.TimeUnit;
66 import java.util.concurrent.atomic.AtomicBoolean;
67 import java.util.concurrent.atomic.AtomicInteger;
68 import java.util.concurrent.atomic.AtomicReference;
69
70 import org.apache.commons.lang.RandomStringUtils;
71 import org.apache.commons.logging.Log;
72 import org.apache.commons.logging.LogFactory;
73 import org.apache.hadoop.conf.Configuration;
74 import org.apache.hadoop.fs.FSDataOutputStream;
75 import org.apache.hadoop.fs.FileStatus;
76 import org.apache.hadoop.fs.FileSystem;
77 import org.apache.hadoop.fs.Path;
78 import org.apache.hadoop.hbase.Cell;
79 import org.apache.hadoop.hbase.CellComparator;
80 import org.apache.hadoop.hbase.CellUtil;
81 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
82 import org.apache.hadoop.hbase.DroppedSnapshotException;
83 import org.apache.hadoop.hbase.HBaseConfiguration;
84 import org.apache.hadoop.hbase.HBaseTestCase;
85 import org.apache.hadoop.hbase.HBaseTestingUtility;
86 import org.apache.hadoop.hbase.HColumnDescriptor;
87 import org.apache.hadoop.hbase.HConstants;
88 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
89 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
90 import org.apache.hadoop.hbase.HRegionInfo;
91 import org.apache.hadoop.hbase.HTableDescriptor;
92 import org.apache.hadoop.hbase.KeyValue;
93 import org.apache.hadoop.hbase.MiniHBaseCluster;
94 import org.apache.hadoop.hbase.MultithreadedTestUtil;
95 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
96 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
97 import org.apache.hadoop.hbase.NotServingRegionException;
98 import org.apache.hadoop.hbase.RegionTooBusyException;
99 import org.apache.hadoop.hbase.ServerName;
100 import org.apache.hadoop.hbase.TableName;
101 import org.apache.hadoop.hbase.Tag;
102 import org.apache.hadoop.hbase.TagType;
103 import org.apache.hadoop.hbase.Waiter;
104 import org.apache.hadoop.hbase.client.Append;
105 import org.apache.hadoop.hbase.client.Delete;
106 import org.apache.hadoop.hbase.client.Durability;
107 import org.apache.hadoop.hbase.client.Get;
108 import org.apache.hadoop.hbase.client.Increment;
109 import org.apache.hadoop.hbase.client.Mutation;
110 import org.apache.hadoop.hbase.client.Put;
111 import org.apache.hadoop.hbase.client.Result;
112 import org.apache.hadoop.hbase.client.RowMutations;
113 import org.apache.hadoop.hbase.client.Scan;
114 import org.apache.hadoop.hbase.client.Table;
115 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
116 import org.apache.hadoop.hbase.filter.BinaryComparator;
117 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
118 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
119 import org.apache.hadoop.hbase.filter.Filter;
120 import org.apache.hadoop.hbase.filter.FilterBase;
121 import org.apache.hadoop.hbase.filter.FilterList;
122 import org.apache.hadoop.hbase.filter.NullComparator;
123 import org.apache.hadoop.hbase.filter.PrefixFilter;
124 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
125 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
126 import org.apache.hadoop.hbase.io.hfile.HFile;
127 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
128 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
129 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
130 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
131 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
132 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
133 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
134 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor;
135 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
136 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
137 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
138 import org.apache.hadoop.hbase.regionserver.Region.RowLock;
139 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
140 import org.apache.hadoop.hbase.regionserver.handler.FinishRegionRecoveringHandler;
141 import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
142 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
143 import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
144 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
145 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
146 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
147 import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
148 import org.apache.hadoop.hbase.security.User;
149 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
150 import org.apache.hadoop.hbase.testclassification.MediumTests;
151 import org.apache.hadoop.hbase.util.Bytes;
152 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
153 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
154 import org.apache.hadoop.hbase.util.FSUtils;
155 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
156 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
157 import org.apache.hadoop.hbase.util.PairOfSameType;
158 import org.apache.hadoop.hbase.util.Threads;
159 import org.apache.hadoop.hbase.wal.DefaultWALProvider;
160 import org.apache.hadoop.hbase.wal.FaultyFSLog;
161 import org.apache.hadoop.hbase.wal.WAL;
162 import org.apache.hadoop.hbase.wal.WALFactory;
163 import org.apache.hadoop.hbase.wal.WALKey;
164 import org.apache.hadoop.hbase.wal.WALProvider;
165 import org.apache.hadoop.hbase.wal.WALProvider.Writer;
166 import org.apache.hadoop.hbase.wal.WALSplitter;
167 import org.junit.After;
168 import org.junit.Assert;
169 import org.junit.Before;
170 import org.junit.Rule;
171 import org.junit.Test;
172 import org.junit.experimental.categories.Category;
173 import org.junit.rules.TestName;
174 import org.mockito.ArgumentCaptor;
175 import org.mockito.ArgumentMatcher;
176 import org.mockito.Mockito;
177 import org.mockito.invocation.InvocationOnMock;
178 import org.mockito.stubbing.Answer;
179
180 import com.google.common.collect.ImmutableList;
181 import com.google.common.collect.Lists;
182 import com.google.common.collect.Maps;
183 import com.google.protobuf.ByteString;
184
185
186
187
188
189
190
191 @Category(MediumTests.class)
192 @SuppressWarnings("deprecation")
193 public class TestHRegion {
194
195
196 private static final Log LOG = LogFactory.getLog(TestHRegion.class);
197 @Rule public TestName name = new TestName();
198
199 private static final String COLUMN_FAMILY = "MyCF";
200 private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
201
202 HRegion region = null;
203
204 private static HBaseTestingUtility TEST_UTIL;
205 public static Configuration CONF ;
206 private String dir;
207 private static FileSystem FILESYSTEM;
208 private final int MAX_VERSIONS = 2;
209
210
211 protected byte[] tableName;
212 protected String method;
213 protected final byte[] qual1 = Bytes.toBytes("qual1");
214 protected final byte[] qual2 = Bytes.toBytes("qual2");
215 protected final byte[] qual3 = Bytes.toBytes("qual3");
216 protected final byte[] value1 = Bytes.toBytes("value1");
217 protected final byte[] value2 = Bytes.toBytes("value2");
218 protected final byte[] row = Bytes.toBytes("rowA");
219 protected final byte[] row2 = Bytes.toBytes("rowB");
220
221 protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
222 .getInstance(MetricsAssertHelper.class);
223
224 @Before
225 public void setup() throws IOException {
226 TEST_UTIL = HBaseTestingUtility.createLocalHTU();
227 FILESYSTEM = TEST_UTIL.getTestFileSystem();
228 CONF = TEST_UTIL.getConfiguration();
229 dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
230 method = name.getMethodName();
231 tableName = Bytes.toBytes(name.getMethodName());
232 }
233
234 @After
235 public void tearDown() throws Exception {
236 EnvironmentEdgeManagerTestHelper.reset();
237 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
238 TEST_UTIL.cleanupTestDir();
239 }
240
241 String getName() {
242 return name.getMethodName();
243 }
244
245
246
247
248
249 @Test (timeout = 100000)
250 public void testSequenceId() throws IOException {
251 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
252 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId());
253
254 assertEquals(0, (long)region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES));
255 region.close();
256 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId());
257 assertEquals(0, (long)region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES));
258
259 region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
260 byte [] value = Bytes.toBytes(name.getMethodName());
261
262 Put put = new Put(value);
263 put.addColumn(COLUMN_FAMILY_BYTES, null, value);
264 region.put(put);
265
266 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId());
267 assertEquals(0, (long)region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES));
268 region.flush(true);
269 long max = region.getMaxFlushedSeqId();
270 region.close();
271 assertEquals(max, region.getMaxFlushedSeqId());
272 }
273
274
275
276
277
278
279
280
281
282
283
284 @Test (timeout=60000)
285 public void testCloseCarryingSnapshot() throws IOException {
286 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
287 Store store = region.getStore(COLUMN_FAMILY_BYTES);
288
289 byte [] value = Bytes.toBytes(name.getMethodName());
290
291 Put put = new Put(value);
292 put.add(COLUMN_FAMILY_BYTES, null, value);
293
294 region.put(put);
295 StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
296 storeFlushCtx.prepare();
297
298 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
299 region.put(put);
300
301 region.close();
302 assertEquals(0, region.getMemstoreSize());
303 HRegion.closeHRegion(region);
304 }
305
306
307
308
309
310
311
312 @Test (timeout=60000)
313 public void testMemstoreSnapshotSize() throws IOException {
314 class MyFaultyFSLog extends FaultyFSLog {
315 StoreFlushContext storeFlushCtx;
316 public MyFaultyFSLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
317 throws IOException {
318 super(fs, rootDir, logName, conf);
319 }
320
321 void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
322 this.storeFlushCtx = storeFlushCtx;
323 }
324
325 @Override
326 public void sync(long txid) throws IOException {
327 storeFlushCtx.prepare();
328 super.sync(txid);
329 }
330 }
331
332 FileSystem fs = FileSystem.get(CONF);
333 Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
334 MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
335 HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
336 CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES);
337
338 Store store = region.getStore(COLUMN_FAMILY_BYTES);
339
340 byte [] value = Bytes.toBytes(name.getMethodName());
341 faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
342
343 Put put = new Put(value);
344 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
345 faultyLog.setFailureType(FaultyFSLog.FailureType.SYNC);
346
347 boolean threwIOE = false;
348 try {
349 region.put(put);
350 } catch (IOException ioe) {
351 threwIOE = true;
352 } finally {
353 assertTrue("The regionserver should have thrown an exception", threwIOE);
354 }
355 long sz = store.getFlushableSize();
356 assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
357 HRegion.closeHRegion(region);
358 }
359
360
361
362
363 @Test
364 public void testMemstoreSizeWithFlushCanceling() throws IOException {
365 FileSystem fs = FileSystem.get(CONF);
366 Path rootDir = new Path(dir + "testMemstoreSizeWithFlushCanceling");
367 FSHLog hLog = new FSHLog(fs, rootDir, "testMemstoreSizeWithFlushCanceling", CONF);
368 HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
369 CONF, false, Durability.SYNC_WAL, hLog, COLUMN_FAMILY_BYTES);
370 Store store = region.getStore(COLUMN_FAMILY_BYTES);
371 assertEquals(0, region.getMemstoreSize());
372
373
374 byte [] value = Bytes.toBytes(name.getMethodName());
375 Put put = new Put(value);
376 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
377 region.put(put);
378 long onePutSize = region.getMemstoreSize();
379 assertTrue(onePutSize > 0);
380 region.flush(true);
381 assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize());
382 assertEquals("flushable size should be zero", 0, store.getFlushableSize());
383
384
385 RegionCoprocessorHost normalCPHost = region.getCoprocessorHost();
386 RegionCoprocessorHost mockedCPHost = Mockito.mock(RegionCoprocessorHost.class);
387 when(mockedCPHost.preFlush(Mockito.isA(HStore.class), Mockito.isA(InternalScanner.class))).
388 thenReturn(null);
389 region.setCoprocessorHost(mockedCPHost);
390 region.put(put);
391 region.flush(true);
392 assertEquals("memstoreSize should NOT be zero", onePutSize, region.getMemstoreSize());
393 assertEquals("flushable size should NOT be zero", onePutSize, store.getFlushableSize());
394
395
396 region.setCoprocessorHost(normalCPHost);
397 region.flush(true);
398 assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize());
399 assertEquals("flushable size should be zero", 0, store.getFlushableSize());
400 HRegion.closeHRegion(region);
401 }
402
403 @Test
404 public void testMemstoreSizeAccountingWithFailedPostBatchMutate() throws IOException {
405 String testName = "testMemstoreSizeAccountingWithFailedPostBatchMutate";
406 FileSystem fs = FileSystem.get(CONF);
407 Path rootDir = new Path(dir + testName);
408 FSHLog hLog = new FSHLog(fs, rootDir, testName, CONF);
409 HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
410 CONF, false, Durability.SYNC_WAL, hLog, COLUMN_FAMILY_BYTES);
411 Store store = region.getStore(COLUMN_FAMILY_BYTES);
412 assertEquals(0, region.getMemstoreSize());
413
414
415 byte [] value = Bytes.toBytes(name.getMethodName());
416 Put put = new Put(value);
417 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
418 region.put(put);
419 long onePutSize = region.getMemstoreSize();
420 assertTrue(onePutSize > 0);
421
422 RegionCoprocessorHost mockedCPHost = Mockito.mock(RegionCoprocessorHost.class);
423 doThrow(new IOException())
424 .when(mockedCPHost).postBatchMutate(Mockito.<MiniBatchOperationInProgress<Mutation>>any());
425 region.setCoprocessorHost(mockedCPHost);
426
427 put = new Put(value);
428 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("dfg"), value);
429 try {
430 region.put(put);
431 fail("Should have failed with IOException");
432 } catch (IOException expected) {
433 }
434 assertEquals("memstoreSize should be incremented", onePutSize * 2, region.getMemstoreSize());
435 assertEquals("flushable size should be incremented", onePutSize * 2, store.getFlushableSize());
436
437 region.setCoprocessorHost(null);
438 HBaseTestingUtility.closeRegionAndWAL(region);
439 }
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456 @Test (timeout=60000)
457 public void testFlushSizeAccounting() throws Exception {
458 final Configuration conf = HBaseConfiguration.create(CONF);
459
460 conf.setInt("hbase.hstore.flush.retries.number", 1);
461 final User user =
462 User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
463
464 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
465 user.runAs(new PrivilegedExceptionAction<Object>() {
466 @Override
467 public Object run() throws Exception {
468
469 FileSystem fs = FileSystem.get(conf);
470 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
471 FaultyFileSystem ffs = (FaultyFileSystem)fs;
472 HRegion region = null;
473 try {
474
475 region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
476 long size = region.getMemstoreSize();
477 Assert.assertEquals(0, size);
478
479 Put p1 = new Put(row);
480 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
481 region.put(p1);
482 final long sizeOfOnePut = region.getMemstoreSize();
483
484 try {
485 LOG.info("Flushing");
486 region.flush(true);
487 Assert.fail("Didn't bubble up IOE!");
488 } catch (DroppedSnapshotException dse) {
489
490 region.closing.set(false);
491 } catch (Exception e) {
492
493 region.closing.set(false);
494 }
495
496 ffs.fault.set(false);
497
498 try {
499 region.getWAL().rollWriter(true);
500 } catch (Exception e) {
501 int x = 0;
502 }
503
504 Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize());
505
506
507 Put p2 = new Put(row);
508 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
509 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
510 region.put(p2);
511 Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize());
512
513
514
515 region.flush(true);
516
517 Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize());
518 } catch (Exception e) {
519 int x = 0;
520 } finally {
521 HRegion.closeHRegion(region);
522 }
523 return null;
524 }
525 });
526 FileSystem.closeAllForUGI(user.getUGI());
527 }
528
529 @Test (timeout=60000)
530 public void testCloseWithFailingFlush() throws Exception {
531 final Configuration conf = HBaseConfiguration.create(CONF);
532
533 conf.setInt("hbase.hstore.flush.retries.number", 1);
534 final User user =
535 User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
536
537 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
538 user.runAs(new PrivilegedExceptionAction<Object>() {
539 @Override
540 public Object run() throws Exception {
541
542 FileSystem fs = FileSystem.get(conf);
543 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
544 FaultyFileSystem ffs = (FaultyFileSystem)fs;
545 HRegion region = null;
546 try {
547
548 region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
549 long size = region.getMemstoreSize();
550 Assert.assertEquals(0, size);
551
552 Put p1 = new Put(row);
553 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
554 region.put(p1);
555
556 Store store = region.getStore(COLUMN_FAMILY_BYTES);
557 StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
558 storeFlushCtx.prepare();
559
560 Put p2 = new Put(row);
561 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
562 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
563 region.put(p2);
564
565 region.close();
566 fail();
567 } catch (IOException dse) {
568
569 LOG.info("Expected DroppedSnapshotException");
570 } finally {
571
572 ffs.fault.set(false);
573 region.getWAL().rollWriter(true);
574 HRegion.closeHRegion(region);
575 }
576 return null;
577 }
578 });
579 FileSystem.closeAllForUGI(user.getUGI());
580 }
581
582 @Test
583 public void testCompactionAffectedByScanners() throws Exception {
584 byte[] family = Bytes.toBytes("family");
585 this.region = initHRegion(tableName, method, CONF, family);
586
587 Put put = new Put(Bytes.toBytes("r1"));
588 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
589 region.put(put);
590 region.flush(true);
591
592 Scan scan = new Scan();
593 scan.setMaxVersions(3);
594
595 RegionScanner scanner1 = region.getScanner(scan);
596
597 Delete delete = new Delete(Bytes.toBytes("r1"));
598 region.delete(delete);
599 region.flush(true);
600
601
602 RegionScanner scanner2 = region.getScanner(scan);
603
604 List<Cell> results = new ArrayList<Cell>();
605
606 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
607
608
609 region.compact(true);
610
611
612 RegionScanner scanner3 = region.getScanner(scan);
613
614
615 scanner1.next(results);
616 System.out.println(results);
617 assertEquals(1, results.size());
618
619 results.clear();
620 scanner2.next(results);
621 System.out.println(results);
622 assertEquals(0, results.size());
623
624 results.clear();
625 scanner3.next(results);
626 System.out.println(results);
627 assertEquals(0, results.size());
628 }
629
630 @Test
631 public void testToShowNPEOnRegionScannerReseek() throws Exception {
632 byte[] family = Bytes.toBytes("family");
633 this.region = initHRegion(tableName, method, CONF, family);
634
635 Put put = new Put(Bytes.toBytes("r1"));
636 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
637 region.put(put);
638 put = new Put(Bytes.toBytes("r2"));
639 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
640 region.put(put);
641 region.flush(true);
642
643 Scan scan = new Scan();
644 scan.setMaxVersions(3);
645
646 RegionScanner scanner1 = region.getScanner(scan);
647
648 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
649
650 region.compact(true);
651
652 scanner1.reseek(Bytes.toBytes("r2"));
653 List<Cell> results = new ArrayList<Cell>();
654 scanner1.next(results);
655 Cell keyValue = results.get(0);
656 Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
657 scanner1.close();
658 }
659
660 @Test
661 public void testSkipRecoveredEditsReplay() throws Exception {
662 String method = "testSkipRecoveredEditsReplay";
663 TableName tableName = TableName.valueOf(method);
664 byte[] family = Bytes.toBytes("family");
665 this.region = initHRegion(tableName, method, CONF, family);
666 final WALFactory wals = new WALFactory(CONF, null, method);
667 try {
668 Path regiondir = region.getRegionFileSystem().getRegionDir();
669 FileSystem fs = region.getRegionFileSystem().getFileSystem();
670 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
671
672 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
673
674 long maxSeqId = 1050;
675 long minSeqId = 1000;
676
677 for (long i = minSeqId; i <= maxSeqId; i += 10) {
678 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
679 fs.create(recoveredEdits);
680 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
681
682 long time = System.nanoTime();
683 WALEdit edit = new WALEdit();
684 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
685 .toBytes(i)));
686 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time,
687 HConstants.DEFAULT_CLUSTER_ID), edit));
688
689 writer.close();
690 }
691 MonitoredTask status = TaskMonitor.get().createStatus(method);
692 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
693 for (Store store : region.getStores()) {
694 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
695 }
696 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
697 assertEquals(maxSeqId, seqId);
698 region.getMVCC().advanceTo(seqId);
699 Get get = new Get(row);
700 Result result = region.get(get);
701 for (long i = minSeqId; i <= maxSeqId; i += 10) {
702 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
703 assertEquals(1, kvs.size());
704 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
705 }
706 } finally {
707 HRegion.closeHRegion(this.region);
708 this.region = null;
709 wals.close();
710 }
711 }
712
713 @Test
714 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
715 String method = "testSkipRecoveredEditsReplaySomeIgnored";
716 TableName tableName = TableName.valueOf(method);
717 byte[] family = Bytes.toBytes("family");
718 this.region = initHRegion(tableName, method, CONF, family);
719 final WALFactory wals = new WALFactory(CONF, null, method);
720 try {
721 Path regiondir = region.getRegionFileSystem().getRegionDir();
722 FileSystem fs = region.getRegionFileSystem().getFileSystem();
723 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
724
725 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
726
727 long maxSeqId = 1050;
728 long minSeqId = 1000;
729
730 for (long i = minSeqId; i <= maxSeqId; i += 10) {
731 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
732 fs.create(recoveredEdits);
733 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
734
735 long time = System.nanoTime();
736 WALEdit edit = new WALEdit();
737 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
738 .toBytes(i)));
739 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time,
740 HConstants.DEFAULT_CLUSTER_ID), edit));
741
742 writer.close();
743 }
744 long recoverSeqId = 1030;
745 MonitoredTask status = TaskMonitor.get().createStatus(method);
746 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
747 for (Store store : region.getStores()) {
748 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
749 }
750 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
751 assertEquals(maxSeqId, seqId);
752 region.getMVCC().advanceTo(seqId);
753 Get get = new Get(row);
754 Result result = region.get(get);
755 for (long i = minSeqId; i <= maxSeqId; i += 10) {
756 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
757 if (i < recoverSeqId) {
758 assertEquals(0, kvs.size());
759 } else {
760 assertEquals(1, kvs.size());
761 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
762 }
763 }
764 } finally {
765 HRegion.closeHRegion(this.region);
766 this.region = null;
767 wals.close();
768 }
769 }
770
771 @Test
772 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
773 byte[] family = Bytes.toBytes("family");
774 this.region = initHRegion(tableName, method, CONF, family);
775 try {
776 Path regiondir = region.getRegionFileSystem().getRegionDir();
777 FileSystem fs = region.getRegionFileSystem().getFileSystem();
778
779 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
780 for (int i = 1000; i < 1050; i += 10) {
781 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
782 FSDataOutputStream dos = fs.create(recoveredEdits);
783 dos.writeInt(i);
784 dos.close();
785 }
786 long minSeqId = 2000;
787 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
788 FSDataOutputStream dos = fs.create(recoveredEdits);
789 dos.close();
790
791 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
792 for (Store store : region.getStores()) {
793 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
794 }
795 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
796 assertEquals(minSeqId, seqId);
797 } finally {
798 HRegion.closeHRegion(this.region);
799 this.region = null;
800 }
801 }
802
803 @Test
804 public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
805 String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
806 TableName tableName = TableName.valueOf(method);
807 byte[] family = Bytes.toBytes("family");
808 this.region = initHRegion(tableName, method, CONF, family);
809 final WALFactory wals = new WALFactory(CONF, null, method);
810 try {
811 Path regiondir = region.getRegionFileSystem().getRegionDir();
812 FileSystem fs = region.getRegionFileSystem().getFileSystem();
813 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
814 byte[][] columns = region.getTableDesc().getFamiliesKeys().toArray(new byte[0][]);
815
816 assertEquals(0, region.getStoreFileList(columns).size());
817
818 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
819
820 long maxSeqId = 1050;
821 long minSeqId = 1000;
822
823 for (long i = minSeqId; i <= maxSeqId; i += 10) {
824 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
825 fs.create(recoveredEdits);
826 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
827
828 long time = System.nanoTime();
829 WALEdit edit = null;
830 if (i == maxSeqId) {
831 edit = WALEdit.createCompaction(region.getRegionInfo(),
832 CompactionDescriptor.newBuilder()
833 .setTableName(ByteString.copyFrom(tableName.getName()))
834 .setFamilyName(ByteString.copyFrom(regionName))
835 .setEncodedRegionName(ByteString.copyFrom(regionName))
836 .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
837 .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName()))
838 .build());
839 } else {
840 edit = new WALEdit();
841 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
842 .toBytes(i)));
843 }
844 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time,
845 HConstants.DEFAULT_CLUSTER_ID), edit));
846 writer.close();
847 }
848
849 long recoverSeqId = 1030;
850 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
851 MonitoredTask status = TaskMonitor.get().createStatus(method);
852 for (Store store : region.getStores()) {
853 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
854 }
855 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
856 assertEquals(maxSeqId, seqId);
857
858
859 assertEquals(1, region.getStoreFileList(columns).size());
860
861 } finally {
862 HRegion.closeHRegion(this.region);
863 this.region = null;
864 wals.close();
865 }
866 }
867
868 @Test
869 public void testRecoveredEditsReplayCompaction() throws Exception {
870 testRecoveredEditsReplayCompaction(false);
871 testRecoveredEditsReplayCompaction(true);
872 }
873 public void testRecoveredEditsReplayCompaction(boolean mismatchedRegionName) throws Exception {
874 String method = name.getMethodName();
875 TableName tableName = TableName.valueOf(method);
876 byte[] family = Bytes.toBytes("family");
877 this.region = initHRegion(tableName, method, CONF, family);
878 final WALFactory wals = new WALFactory(CONF, null, method);
879 try {
880 Path regiondir = region.getRegionFileSystem().getRegionDir();
881 FileSystem fs = region.getRegionFileSystem().getFileSystem();
882 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
883
884 long maxSeqId = 3;
885 long minSeqId = 0;
886
887 for (long i = minSeqId; i < maxSeqId; i++) {
888 Put put = new Put(Bytes.toBytes(i));
889 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
890 region.put(put);
891 region.flush(true);
892 }
893
894
895 assertEquals(3, region.getStore(family).getStorefilesCount());
896 List<Path> storeFiles = new ArrayList<Path>(3);
897 for (StoreFile sf : region.getStore(family).getStorefiles()) {
898 storeFiles.add(sf.getPath());
899 }
900
901
902 CONF.setBoolean("hbase.hstore.compaction.complete", false);
903 region.compactStores();
904
905
906 assertEquals(3, region.getStore(family).getStorefilesCount());
907
908
909 Path tmpDir = region.getRegionFileSystem().getTempDir();
910 FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
911 String errorMsg = "Expected to find 1 file in the region temp directory "
912 + "from the compaction, could not find any";
913 assertNotNull(errorMsg, files);
914 assertEquals(errorMsg, 1, files.length);
915
916 Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
917 files[0].getPath());
918
919 byte[] encodedNameAsBytes = this.region.getRegionInfo().getEncodedNameAsBytes();
920 byte[] fakeEncodedNameAsBytes = new byte [encodedNameAsBytes.length];
921 for (int i=0; i < encodedNameAsBytes.length; i++) {
922
923 fakeEncodedNameAsBytes[i] = (byte) (encodedNameAsBytes[i] + 1);
924 }
925
926 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
927 .getRegionInfo(), mismatchedRegionName ? fakeEncodedNameAsBytes : null, family,
928 storeFiles, Lists.newArrayList(newFile),
929 region.getRegionFileSystem().getStoreDir(Bytes.toString(family)));
930
931 WALUtil.writeCompactionMarker(region.getWAL(), this.region.getTableDesc(),
932 this.region.getRegionInfo(), compactionDescriptor, region.getMVCC());
933
934 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
935
936 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
937 fs.create(recoveredEdits);
938 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
939
940 long time = System.nanoTime();
941
942 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, 10, time,
943 HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(),
944 compactionDescriptor)));
945 writer.close();
946
947
948 region.getTableDesc();
949 region.getRegionInfo();
950 region.close();
951 try {
952 region = HRegion.openHRegion(region, null);
953 } catch (WrongRegionException wre) {
954 fail("Matching encoded region name should not have produced WrongRegionException");
955 }
956
957
958 Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
959 for (StoreFile sf : sfs) {
960 LOG.info(sf.getPath());
961 }
962 if (!mismatchedRegionName) {
963 assertEquals(1, region.getStore(family).getStorefilesCount());
964 }
965 files = FSUtils.listStatus(fs, tmpDir);
966 assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
967
968 for (long i = minSeqId; i < maxSeqId; i++) {
969 Get get = new Get(Bytes.toBytes(i));
970 Result result = region.get(get);
971 byte[] value = result.getValue(family, Bytes.toBytes(i));
972 assertArrayEquals(Bytes.toBytes(i), value);
973 }
974 } finally {
975 HRegion.closeHRegion(this.region);
976 this.region = null;
977 wals.close();
978 }
979 }
980
981 @Test
982 public void testFlushMarkers() throws Exception {
983
984 String method = name.getMethodName();
985 TableName tableName = TableName.valueOf(method);
986 byte[] family = Bytes.toBytes("family");
987 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log");
988 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
989 FSUtils.setRootDir(walConf, logDir);
990 final WALFactory wals = new WALFactory(walConf, null, method);
991 final WAL wal = wals.getWAL(tableName.getName(), tableName.getNamespace());
992
993 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
994 HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
995 try {
996 Path regiondir = region.getRegionFileSystem().getRegionDir();
997 FileSystem fs = region.getRegionFileSystem().getFileSystem();
998 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
999
1000 long maxSeqId = 3;
1001 long minSeqId = 0;
1002
1003 for (long i = minSeqId; i < maxSeqId; i++) {
1004 Put put = new Put(Bytes.toBytes(i));
1005 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
1006 region.put(put);
1007 region.flush(true);
1008 }
1009
1010
1011 assertEquals(3, region.getStore(family).getStorefilesCount());
1012 List<String> storeFiles = new ArrayList<String>(3);
1013 for (StoreFile sf : region.getStore(family).getStorefiles()) {
1014 storeFiles.add(sf.getPath().getName());
1015 }
1016
1017
1018 wal.shutdown();
1019 WAL.Reader reader = WALFactory.createReader(fs, DefaultWALProvider.getCurrentFileName(wal),
1020 TEST_UTIL.getConfiguration());
1021 try {
1022 List<WAL.Entry> flushDescriptors = new ArrayList<WAL.Entry>();
1023 long lastFlushSeqId = -1;
1024 while (true) {
1025 WAL.Entry entry = reader.next();
1026 if (entry == null) {
1027 break;
1028 }
1029 Cell cell = entry.getEdit().getCells().get(0);
1030 if (WALEdit.isMetaEditFamily(cell)) {
1031 FlushDescriptor flushDesc = WALEdit.getFlushDescriptor(cell);
1032 assertNotNull(flushDesc);
1033 assertArrayEquals(tableName.getName(), flushDesc.getTableName().toByteArray());
1034 if (flushDesc.getAction() == FlushAction.START_FLUSH) {
1035 assertTrue(flushDesc.getFlushSequenceNumber() > lastFlushSeqId);
1036 } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) {
1037 assertTrue(flushDesc.getFlushSequenceNumber() == lastFlushSeqId);
1038 }
1039 lastFlushSeqId = flushDesc.getFlushSequenceNumber();
1040 assertArrayEquals(regionName, flushDesc.getEncodedRegionName().toByteArray());
1041 assertEquals(1, flushDesc.getStoreFlushesCount());
1042 StoreFlushDescriptor storeFlushDesc = flushDesc.getStoreFlushes(0);
1043 assertArrayEquals(family, storeFlushDesc.getFamilyName().toByteArray());
1044 assertEquals("family", storeFlushDesc.getStoreHomeDir());
1045 if (flushDesc.getAction() == FlushAction.START_FLUSH) {
1046 assertEquals(0, storeFlushDesc.getFlushOutputCount());
1047 } else {
1048 assertEquals(1, storeFlushDesc.getFlushOutputCount());
1049 assertTrue(storeFiles.contains(storeFlushDesc.getFlushOutput(0)));
1050 }
1051
1052 flushDescriptors.add(entry);
1053 }
1054 }
1055
1056 assertEquals(3 * 2, flushDescriptors.size());
1057
1058
1059
1060 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
1061
1062 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
1063 fs.create(recoveredEdits);
1064 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
1065
1066 for (WAL.Entry entry : flushDescriptors) {
1067 writer.append(entry);
1068 }
1069 writer.close();
1070 } finally {
1071 if (null != reader) {
1072 try {
1073 reader.close();
1074 } catch (IOException exception) {
1075 LOG.warn("Problem closing wal: " + exception.getMessage());
1076 LOG.debug("exception details", exception);
1077 }
1078 }
1079 }
1080
1081
1082
1083 region.close();
1084 region = HRegion.openHRegion(region, null);
1085
1086
1087 for (long i = minSeqId; i < maxSeqId; i++) {
1088 Get get = new Get(Bytes.toBytes(i));
1089 Result result = region.get(get);
1090 byte[] value = result.getValue(family, Bytes.toBytes(i));
1091 assertArrayEquals(Bytes.toBytes(i), value);
1092 }
1093 } finally {
1094 HRegion.closeHRegion(this.region);
1095 this.region = null;
1096 wals.close();
1097 }
1098 }
1099
1100 class IsFlushWALMarker extends ArgumentMatcher<WALEdit> {
1101 volatile FlushAction[] actions;
1102 public IsFlushWALMarker(FlushAction... actions) {
1103 this.actions = actions;
1104 }
1105 @Override
1106 public boolean matches(Object edit) {
1107 List<Cell> cells = ((WALEdit)edit).getCells();
1108 if (cells.isEmpty()) {
1109 return false;
1110 }
1111 if (WALEdit.isMetaEditFamily(cells.get(0))) {
1112 FlushDescriptor desc = null;
1113 try {
1114 desc = WALEdit.getFlushDescriptor(cells.get(0));
1115 } catch (IOException e) {
1116 LOG.warn(e);
1117 return false;
1118 }
1119 if (desc != null) {
1120 for (FlushAction action : actions) {
1121 if (desc.getAction() == action) {
1122 return true;
1123 }
1124 }
1125 }
1126 }
1127 return false;
1128 }
1129 public IsFlushWALMarker set(FlushAction... actions) {
1130 this.actions = actions;
1131 return this;
1132 }
1133 }
1134
1135 @Test (timeout=60000)
1136 public void testFlushMarkersWALFail() throws Exception {
1137
1138 String method = name.getMethodName();
1139 TableName tableName = TableName.valueOf(method);
1140 byte[] family = Bytes.toBytes("family");
1141
1142
1143 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + "log");
1144
1145 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
1146 FSUtils.setRootDir(walConf, logDir);
1147
1148
1149 class FailAppendFlushMarkerWAL extends FSHLog {
1150 volatile FlushAction [] flushActions = null;
1151
1152 public FailAppendFlushMarkerWAL(FileSystem fs, Path root, String logDir, Configuration conf)
1153 throws IOException {
1154 super(fs, root, logDir, conf);
1155 }
1156
1157 @Override
1158 protected Writer createWriterInstance(Path path) throws IOException {
1159 final Writer w = super.createWriterInstance(path);
1160 return new Writer() {
1161 @Override
1162 public void close() throws IOException {
1163 w.close();
1164 }
1165
1166 @Override
1167 public void sync() throws IOException {
1168 w.sync();
1169 }
1170
1171 @Override
1172 public void append(Entry entry) throws IOException {
1173 List<Cell> cells = entry.getEdit().getCells();
1174 if (WALEdit.isMetaEditFamily(cells.get(0))) {
1175 FlushDescriptor desc = WALEdit.getFlushDescriptor(cells.get(0));
1176 if (desc != null) {
1177 for (FlushAction flushAction: flushActions) {
1178 if (desc.getAction().equals(flushAction)) {
1179 throw new IOException("Failed to append flush marker! " + flushAction);
1180 }
1181 }
1182 }
1183 }
1184 w.append(entry);
1185 }
1186
1187 @Override
1188 public long getLength() throws IOException {
1189 return w.getLength();
1190 }
1191 };
1192 }
1193 }
1194 FailAppendFlushMarkerWAL wal =
1195 new FailAppendFlushMarkerWAL(FileSystem.get(walConf), FSUtils.getRootDir(walConf),
1196 getName(), walConf);
1197 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
1198 HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
1199 try {
1200 int i = 0;
1201 Put put = new Put(Bytes.toBytes(i));
1202 put.setDurability(Durability.SKIP_WAL);
1203 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
1204 region.put(put);
1205
1206
1207 wal.flushActions = new FlushAction [] {FlushAction.START_FLUSH};
1208
1209
1210 try {
1211 region.flush(true);
1212 fail("This should have thrown exception");
1213 } catch (DroppedSnapshotException unexpected) {
1214
1215 throw unexpected;
1216 } catch (IOException expected) {
1217
1218 }
1219
1220
1221 region.close(true);
1222 wal.close();
1223
1224
1225 wal.flushActions = new FlushAction [] {FlushAction.COMMIT_FLUSH};
1226 wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf), FSUtils.getRootDir(walConf),
1227 getName(), walConf);
1228 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
1229 HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
1230 region.put(put);
1231
1232
1233
1234
1235 wal.flushActions = new FlushAction [] {FlushAction.COMMIT_FLUSH, FlushAction.ABORT_FLUSH};
1236
1237 try {
1238 region.flush(true);
1239 fail("This should have thrown exception");
1240 } catch (DroppedSnapshotException expected) {
1241
1242
1243 } catch (IOException unexpected) {
1244 throw unexpected;
1245 }
1246
1247 } finally {
1248 HRegion.closeHRegion(this.region);
1249 this.region = null;
1250 }
1251 }
1252
1253 @Test
1254 public void testGetWhileRegionClose() throws IOException {
1255 TableName tableName = TableName.valueOf(name.getMethodName());
1256 Configuration hc = initSplit();
1257 int numRows = 100;
1258 byte[][] families = { fam1, fam2, fam3 };
1259
1260
1261 String method = name.getMethodName();
1262 this.region = initHRegion(tableName, method, hc, families);
1263 try {
1264
1265 final int startRow = 100;
1266 putData(startRow, numRows, qual1, families);
1267 putData(startRow, numRows, qual2, families);
1268 putData(startRow, numRows, qual3, families);
1269 final AtomicBoolean done = new AtomicBoolean(false);
1270 final AtomicInteger gets = new AtomicInteger(0);
1271 GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
1272 try {
1273
1274 for (int i = 0; i < threads.length / 2; i++) {
1275 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
1276 threads[i].setDaemon(true);
1277 threads[i].start();
1278 }
1279
1280
1281 this.region.closing.set(true);
1282 for (int i = threads.length / 2; i < threads.length; i++) {
1283 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
1284 threads[i].setDaemon(true);
1285 threads[i].start();
1286 }
1287 } finally {
1288 if (this.region != null) {
1289 HRegion.closeHRegion(this.region);
1290 }
1291 }
1292 done.set(true);
1293 for (GetTillDoneOrException t : threads) {
1294 try {
1295 t.join();
1296 } catch (InterruptedException e) {
1297 e.printStackTrace();
1298 }
1299 if (t.e != null) {
1300 LOG.info("Exception=" + t.e);
1301 assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
1302 }
1303 }
1304 } finally {
1305 HRegion.closeHRegion(this.region);
1306 this.region = null;
1307 }
1308 }
1309
1310
1311
1312
1313
1314 class GetTillDoneOrException extends Thread {
1315 private final Get g;
1316 private final AtomicBoolean done;
1317 private final AtomicInteger count;
1318 private Exception e;
1319
1320 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
1321 super("getter." + i);
1322 this.g = new Get(r);
1323 this.done = d;
1324 this.count = c;
1325 }
1326
1327 @Override
1328 public void run() {
1329 while (!this.done.get()) {
1330 try {
1331 assertTrue(region.get(g).size() > 0);
1332 this.count.incrementAndGet();
1333 } catch (Exception e) {
1334 this.e = e;
1335 break;
1336 }
1337 }
1338 }
1339 }
1340
1341
1342
1343
1344 @Test
1345 public void testWeirdCacheBehaviour() throws Exception {
1346 byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
1347 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
1348 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
1349 this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
1350 try {
1351 String value = "this is the value";
1352 String value2 = "this is some other value";
1353 String keyPrefix1 = "prefix1";
1354 String keyPrefix2 = "prefix2";
1355 String keyPrefix3 = "prefix3";
1356 putRows(this.region, 3, value, keyPrefix1);
1357 putRows(this.region, 3, value, keyPrefix2);
1358 putRows(this.region, 3, value, keyPrefix3);
1359 putRows(this.region, 3, value2, keyPrefix1);
1360 putRows(this.region, 3, value2, keyPrefix2);
1361 putRows(this.region, 3, value2, keyPrefix3);
1362 System.out.println("Checking values for key: " + keyPrefix1);
1363 assertEquals("Got back incorrect number of rows from scan", 3,
1364 getNumberOfRows(keyPrefix1, value2, this.region));
1365 System.out.println("Checking values for key: " + keyPrefix2);
1366 assertEquals("Got back incorrect number of rows from scan", 3,
1367 getNumberOfRows(keyPrefix2, value2, this.region));
1368 System.out.println("Checking values for key: " + keyPrefix3);
1369 assertEquals("Got back incorrect number of rows from scan", 3,
1370 getNumberOfRows(keyPrefix3, value2, this.region));
1371 deleteColumns(this.region, value2, keyPrefix1);
1372 deleteColumns(this.region, value2, keyPrefix2);
1373 deleteColumns(this.region, value2, keyPrefix3);
1374 System.out.println("Starting important checks.....");
1375 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
1376 getNumberOfRows(keyPrefix1, value2, this.region));
1377 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
1378 getNumberOfRows(keyPrefix2, value2, this.region));
1379 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
1380 getNumberOfRows(keyPrefix3, value2, this.region));
1381 } finally {
1382 HRegion.closeHRegion(this.region);
1383 this.region = null;
1384 }
1385 }
1386
1387 @Test
1388 public void testAppendWithReadOnlyTable() throws Exception {
1389 byte[] TABLE = Bytes.toBytes("readOnlyTable");
1390 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
1391 boolean exceptionCaught = false;
1392 Append append = new Append(Bytes.toBytes("somerow"));
1393 append.setDurability(Durability.SKIP_WAL);
1394 append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
1395 Bytes.toBytes("somevalue"));
1396 try {
1397 region.append(append);
1398 } catch (IOException e) {
1399 exceptionCaught = true;
1400 } finally {
1401 HRegion.closeHRegion(this.region);
1402 this.region = null;
1403 }
1404 assertTrue(exceptionCaught == true);
1405 }
1406
1407 @Test
1408 public void testIncrWithReadOnlyTable() throws Exception {
1409 byte[] TABLE = Bytes.toBytes("readOnlyTable");
1410 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
1411 boolean exceptionCaught = false;
1412 Increment inc = new Increment(Bytes.toBytes("somerow"));
1413 inc.setDurability(Durability.SKIP_WAL);
1414 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
1415 try {
1416 region.increment(inc);
1417 } catch (IOException e) {
1418 exceptionCaught = true;
1419 } finally {
1420 HRegion.closeHRegion(this.region);
1421 this.region = null;
1422 }
1423 assertTrue(exceptionCaught == true);
1424 }
1425
1426 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
1427 InternalScanner scanner = buildScanner(keyPrefix, value, r);
1428 int count = 0;
1429 boolean more = false;
1430 List<Cell> results = new ArrayList<Cell>();
1431 do {
1432 more = scanner.next(results);
1433 if (results != null && !results.isEmpty())
1434 count++;
1435 else
1436 break;
1437 Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
1438 delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
1439 r.delete(delete);
1440 results.clear();
1441 } while (more);
1442 assertEquals("Did not perform correct number of deletes", 3, count);
1443 }
1444
1445 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
1446 InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
1447 int numberOfResults = 0;
1448 List<Cell> results = new ArrayList<Cell>();
1449 boolean more = false;
1450 do {
1451 more = resultScanner.next(results);
1452 if (results != null && !results.isEmpty())
1453 numberOfResults++;
1454 else
1455 break;
1456 for (Cell kv : results) {
1457 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
1458 }
1459 results.clear();
1460 } while (more);
1461 return numberOfResults;
1462 }
1463
1464 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
1465 throws IOException {
1466
1467 FilterList allFilters = new FilterList();
1468 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
1469
1470 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
1471 Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
1472 filter.setFilterIfMissing(true);
1473 allFilters.addFilter(filter);
1474 Scan scan = new Scan();
1475 scan.addFamily(Bytes.toBytes("trans-blob"));
1476 scan.addFamily(Bytes.toBytes("trans-type"));
1477 scan.addFamily(Bytes.toBytes("trans-date"));
1478 scan.addFamily(Bytes.toBytes("trans-tags"));
1479 scan.addFamily(Bytes.toBytes("trans-group"));
1480 scan.setFilter(allFilters);
1481 return r.getScanner(scan);
1482 }
1483
1484 private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
1485 for (int i = 0; i < numRows; i++) {
1486 String row = key + "_" + i
1487 System.out.println(String.format("Saving row: %s, with value %s", row, value));
1488 Put put = new Put(Bytes.toBytes(row));
1489 put.setDurability(Durability.SKIP_WAL);
1490 put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
1491 put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
1492 put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
1493 put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
1494 put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
1495 r.put(put);
1496 }
1497 }
1498
1499 @Test
1500 public void testFamilyWithAndWithoutColon() throws Exception {
1501 byte[] b = Bytes.toBytes(getName());
1502 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1503 this.region = initHRegion(b, getName(), CONF, cf);
1504 try {
1505 Put p = new Put(b);
1506 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
1507 p.add(cfwithcolon, cfwithcolon, cfwithcolon);
1508 boolean exception = false;
1509 try {
1510 this.region.put(p);
1511 } catch (NoSuchColumnFamilyException e) {
1512 exception = true;
1513 }
1514 assertTrue(exception);
1515 } finally {
1516 HRegion.closeHRegion(this.region);
1517 this.region = null;
1518 }
1519 }
1520
1521 @Test
1522 public void testBatchPut_whileNoRowLocksHeld() throws IOException {
1523 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1524 byte[] qual = Bytes.toBytes("qual");
1525 byte[] val = Bytes.toBytes("val");
1526 this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1527 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1528 try {
1529 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1530 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1531
1532 LOG.info("First a batch put with all valid puts");
1533 final Put[] puts = new Put[10];
1534 for (int i = 0; i < 10; i++) {
1535 puts[i] = new Put(Bytes.toBytes("row_" + i));
1536 puts[i].add(cf, qual, val);
1537 }
1538
1539 OperationStatus[] codes = this.region.batchMutate(puts);
1540 assertEquals(10, codes.length);
1541 for (int i = 0; i < 10; i++) {
1542 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1543 }
1544 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1545
1546 LOG.info("Next a batch put with one invalid family");
1547 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1548 codes = this.region.batchMutate(puts);
1549 assertEquals(10, codes.length);
1550 for (int i = 0; i < 10; i++) {
1551 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1552 codes[i].getOperationStatusCode());
1553 }
1554
1555 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1556 } finally {
1557 HRegion.closeHRegion(this.region);
1558 this.region = null;
1559 }
1560 }
1561
1562 @Test
1563 public void testBatchPut_whileMultipleRowLocksHeld() throws Exception {
1564 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1565 byte[] qual = Bytes.toBytes("qual");
1566 byte[] val = Bytes.toBytes("val");
1567 this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1568 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1569 try {
1570 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1571 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1572
1573 final Put[] puts = new Put[10];
1574 for (int i = 0; i < 10; i++) {
1575 puts[i] = new Put(Bytes.toBytes("row_" + i));
1576 puts[i].add(cf, qual, val);
1577 }
1578 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1579
1580 LOG.info("batchPut will have to break into four batches to avoid row locks");
1581 RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2"));
1582 RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_1"));
1583 RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_3"));
1584 RowLock rowLock4 = region.getRowLock(Bytes.toBytes("row_3"), true);
1585
1586
1587 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1588 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
1589 final CountDownLatch startingPuts = new CountDownLatch(1);
1590 final CountDownLatch startingClose = new CountDownLatch(1);
1591 TestThread putter = new TestThread(ctx) {
1592 @Override
1593 public void doWork() throws IOException {
1594 startingPuts.countDown();
1595 retFromThread.set(region.batchMutate(puts));
1596 }
1597 };
1598 LOG.info("...starting put thread while holding locks");
1599 ctx.addThread(putter);
1600 ctx.startThreads();
1601
1602
1603
1604
1605
1606
1607 Thread regionCloseThread = new TestThread(ctx) {
1608 @Override
1609 public void doWork() {
1610 try {
1611 startingPuts.await();
1612
1613
1614 Thread.sleep(10);
1615 startingClose.countDown();
1616 HBaseTestingUtility.closeRegionAndWAL(region);
1617 } catch (IOException e) {
1618 throw new RuntimeException(e);
1619 } catch (InterruptedException e) {
1620 throw new RuntimeException(e);
1621 }
1622 }
1623 };
1624 regionCloseThread.start();
1625
1626 startingClose.await();
1627 startingPuts.await();
1628 Thread.sleep(100);
1629 LOG.info("...releasing row lock 1, which should let put thread continue");
1630 rowLock1.release();
1631 rowLock2.release();
1632 rowLock3.release();
1633 waitForCounter(source, "syncTimeNumOps", syncs + 1);
1634
1635 LOG.info("...joining on put thread");
1636 ctx.stop();
1637 regionCloseThread.join();
1638
1639 OperationStatus[] codes = retFromThread.get();
1640 for (int i = 0; i < codes.length; i++) {
1641 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1642 codes[i].getOperationStatusCode());
1643 }
1644 rowLock4.release();
1645 } finally {
1646 HRegion.closeHRegion(this.region);
1647 this.region = null;
1648 }
1649 }
1650
1651 private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount)
1652 throws InterruptedException {
1653 long startWait = System.currentTimeMillis();
1654 long currentCount;
1655 while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) {
1656 Thread.sleep(100);
1657 if (System.currentTimeMillis() - startWait > 10000) {
1658 fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName,
1659 expectedCount, currentCount));
1660 }
1661 }
1662 }
1663
1664 @Test
1665 public void testBatchPutWithTsSlop() throws Exception {
1666 byte[] b = Bytes.toBytes(getName());
1667 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1668 byte[] qual = Bytes.toBytes("qual");
1669 byte[] val = Bytes.toBytes("val");
1670
1671
1672 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1673 this.region = initHRegion(b, getName(), CONF, cf);
1674
1675 try {
1676 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1677 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1678 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1679
1680 final Put[] puts = new Put[10];
1681 for (int i = 0; i < 10; i++) {
1682 puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1683 puts[i].add(cf, qual, val);
1684 }
1685
1686 OperationStatus[] codes = this.region.batchMutate(puts);
1687 assertEquals(10, codes.length);
1688 for (int i = 0; i < 10; i++) {
1689 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1690 }
1691 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1692
1693 } finally {
1694 HRegion.closeHRegion(this.region);
1695 this.region = null;
1696 }
1697
1698 }
1699
1700
1701
1702
1703 @Test
1704 public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1705 byte[] row1 = Bytes.toBytes("row1");
1706 byte[] fam1 = Bytes.toBytes("fam1");
1707 byte[] qf1 = Bytes.toBytes("qualifier");
1708 byte[] emptyVal = new byte[] {};
1709 byte[] val1 = Bytes.toBytes("value1");
1710 byte[] val2 = Bytes.toBytes("value2");
1711
1712
1713 String method = this.getName();
1714 this.region = initHRegion(tableName, method, CONF, fam1);
1715 try {
1716
1717 Put put = new Put(row1);
1718 put.add(fam1, qf1, emptyVal);
1719
1720
1721 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1722 emptyVal), put, true);
1723 assertTrue(res);
1724
1725
1726 put = new Put(row1);
1727 put.add(fam1, qf1, val1);
1728
1729
1730 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1731 put, true);
1732 assertTrue(res);
1733
1734
1735 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1736 put, true);
1737 assertFalse(res);
1738
1739 Delete delete = new Delete(row1);
1740 delete.deleteColumn(fam1, qf1);
1741 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1742 delete, true);
1743 assertFalse(res);
1744
1745 put = new Put(row1);
1746 put.add(fam1, qf1, val2);
1747
1748 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1749 put, true);
1750 assertTrue(res);
1751
1752
1753 delete = new Delete(row1);
1754 delete.deleteColumn(fam1, qf1);
1755 delete.deleteColumn(fam1, qf1);
1756 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1757 delete, true);
1758 assertTrue(res);
1759
1760 delete = new Delete(row1);
1761 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1762 delete, true);
1763 assertTrue(res);
1764
1765
1766 put = new Put(row1);
1767 put.add(fam1, qf1, val1);
1768
1769 res = region
1770 .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1771 assertTrue(res);
1772 } finally {
1773 HRegion.closeHRegion(this.region);
1774 this.region = null;
1775 }
1776 }
1777
1778 @Test
1779 public void testCheckAndMutate_WithWrongValue() throws IOException {
1780 byte[] row1 = Bytes.toBytes("row1");
1781 byte[] fam1 = Bytes.toBytes("fam1");
1782 byte[] qf1 = Bytes.toBytes("qualifier");
1783 byte[] val1 = Bytes.toBytes("value1");
1784 byte[] val2 = Bytes.toBytes("value2");
1785
1786
1787 String method = this.getName();
1788 this.region = initHRegion(tableName, method, CONF, fam1);
1789 try {
1790
1791 Put put = new Put(row1);
1792 put.add(fam1, qf1, val1);
1793 region.put(put);
1794
1795
1796 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1797 val2), put, true);
1798 assertEquals(false, res);
1799
1800
1801 Delete delete = new Delete(row1);
1802 delete.deleteFamily(fam1);
1803 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1804 put, true);
1805 assertEquals(false, res);
1806 } finally {
1807 HRegion.closeHRegion(this.region);
1808 this.region = null;
1809 }
1810 }
1811
1812 @Test
1813 public void testCheckAndMutate_WithCorrectValue() throws IOException {
1814 byte[] row1 = Bytes.toBytes("row1");
1815 byte[] fam1 = Bytes.toBytes("fam1");
1816 byte[] qf1 = Bytes.toBytes("qualifier");
1817 byte[] val1 = Bytes.toBytes("value1");
1818
1819
1820 String method = this.getName();
1821 this.region = initHRegion(tableName, method, CONF, fam1);
1822 try {
1823
1824 Put put = new Put(row1);
1825 put.add(fam1, qf1, val1);
1826 region.put(put);
1827
1828
1829 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1830 val1), put, true);
1831 assertEquals(true, res);
1832
1833
1834 Delete delete = new Delete(row1);
1835 delete.deleteColumn(fam1, qf1);
1836 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1837 delete, true);
1838 assertEquals(true, res);
1839 } finally {
1840 HRegion.closeHRegion(this.region);
1841 this.region = null;
1842 }
1843 }
1844
1845 @Test
1846 public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1847 byte[] row1 = Bytes.toBytes("row1");
1848 byte[] fam1 = Bytes.toBytes("fam1");
1849 byte[] qf1 = Bytes.toBytes("qualifier");
1850 byte[] val1 = Bytes.toBytes("value1");
1851 byte[] val2 = Bytes.toBytes("value2");
1852 byte[] val3 = Bytes.toBytes("value3");
1853 byte[] val4 = Bytes.toBytes("value4");
1854
1855
1856 String method = this.getName();
1857 this.region = initHRegion(tableName, method, CONF, fam1);
1858 try {
1859
1860 Put put = new Put(row1);
1861 put.add(fam1, qf1, val3);
1862 region.put(put);
1863
1864
1865 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1866 new BinaryComparator(val3), put, true);
1867 assertEquals(false, res);
1868
1869
1870 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1871 new BinaryComparator(val4), put, true);
1872 assertEquals(false, res);
1873
1874
1875
1876 put = new Put(row1);
1877 put.add(fam1, qf1, val2);
1878 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1879 new BinaryComparator(val2), put, true);
1880 assertEquals(true, res);
1881
1882
1883 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1884 new BinaryComparator(val3), put, true);
1885 assertEquals(false, res);
1886
1887
1888
1889 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1890 new BinaryComparator(val2), put, true);
1891 assertEquals(true, res);
1892
1893
1894
1895 put = new Put(row1);
1896 put.add(fam1, qf1, val3);
1897 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1898 new BinaryComparator(val1), put, true);
1899 assertEquals(true, res);
1900
1901
1902 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1903 new BinaryComparator(val3), put, true);
1904 assertEquals(false, res);
1905
1906
1907 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1908 new BinaryComparator(val2), put, true);
1909 assertEquals(false, res);
1910
1911
1912
1913 put = new Put(row1);
1914 put.add(fam1, qf1, val2);
1915 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1916 new BinaryComparator(val4), put, true);
1917 assertEquals(true, res);
1918
1919
1920 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1921 new BinaryComparator(val1), put, true);
1922 assertEquals(false, res);
1923
1924
1925
1926 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1927 new BinaryComparator(val2), put, true);
1928 assertEquals(true, res);
1929
1930
1931 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1932 new BinaryComparator(val3), put, true);
1933 assertEquals(true, res);
1934 } finally {
1935 HRegion.closeHRegion(this.region);
1936 this.region = null;
1937 }
1938 }
1939
1940 @Test
1941 public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1942 byte[] row1 = Bytes.toBytes("row1");
1943 byte[] fam1 = Bytes.toBytes("fam1");
1944 byte[] fam2 = Bytes.toBytes("fam2");
1945 byte[] qf1 = Bytes.toBytes("qualifier");
1946 byte[] val1 = Bytes.toBytes("value1");
1947 byte[] val2 = Bytes.toBytes("value2");
1948
1949 byte[][] families = { fam1, fam2 };
1950
1951
1952 String method = this.getName();
1953 this.region = initHRegion(tableName, method, CONF, families);
1954 try {
1955
1956 Put put = new Put(row1);
1957 put.add(fam1, qf1, val1);
1958 region.put(put);
1959
1960
1961 long ts = System.currentTimeMillis();
1962 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1963 put = new Put(row1);
1964 put.add(kv);
1965
1966
1967 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1968 val1), put, true);
1969 assertEquals(true, res);
1970
1971 Get get = new Get(row1);
1972 get.addColumn(fam2, qf1);
1973 Cell[] actual = region.get(get).rawCells();
1974
1975 Cell[] expected = { kv };
1976
1977 assertEquals(expected.length, actual.length);
1978 for (int i = 0; i < actual.length; i++) {
1979 assertEquals(expected[i], actual[i]);
1980 }
1981 } finally {
1982 HRegion.closeHRegion(this.region);
1983 this.region = null;
1984 }
1985 }
1986
1987 @Test
1988 public void testCheckAndPut_wrongRowInPut() throws IOException {
1989 TableName tableName = TableName.valueOf(name.getMethodName());
1990 this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1991 try {
1992 Put put = new Put(row2);
1993 put.add(fam1, qual1, value1);
1994 try {
1995 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1996 new BinaryComparator(value2), put, false);
1997 fail();
1998 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1999
2000 }
2001 } finally {
2002 HRegion.closeHRegion(this.region);
2003 this.region = null;
2004 }
2005 }
2006
2007 @Test
2008 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
2009 byte[] row1 = Bytes.toBytes("row1");
2010 byte[] fam1 = Bytes.toBytes("fam1");
2011 byte[] fam2 = Bytes.toBytes("fam2");
2012 byte[] qf1 = Bytes.toBytes("qualifier1");
2013 byte[] qf2 = Bytes.toBytes("qualifier2");
2014 byte[] qf3 = Bytes.toBytes("qualifier3");
2015 byte[] val1 = Bytes.toBytes("value1");
2016 byte[] val2 = Bytes.toBytes("value2");
2017 byte[] val3 = Bytes.toBytes("value3");
2018 byte[] emptyVal = new byte[] {};
2019
2020 byte[][] families = { fam1, fam2 };
2021
2022
2023 String method = this.getName();
2024 this.region = initHRegion(tableName, method, CONF, families);
2025 try {
2026
2027 Put put = new Put(row1);
2028 put.add(fam1, qf1, val1);
2029 region.put(put);
2030 Threads.sleep(2);
2031
2032 put = new Put(row1);
2033 put.add(fam1, qf1, val2);
2034 put.add(fam2, qf1, val3);
2035 put.add(fam2, qf2, val2);
2036 put.add(fam2, qf3, val1);
2037 put.add(fam1, qf3, val1);
2038 region.put(put);
2039
2040
2041 Delete delete = new Delete(row1);
2042 delete.deleteColumn(fam1, qf1);
2043 delete.deleteColumn(fam2, qf1);
2044 delete.deleteColumn(fam1, qf3);
2045 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
2046 val2), delete, true);
2047 assertEquals(true, res);
2048
2049 Get get = new Get(row1);
2050 get.addColumn(fam1, qf1);
2051 get.addColumn(fam1, qf3);
2052 get.addColumn(fam2, qf2);
2053 Result r = region.get(get);
2054 assertEquals(2, r.size());
2055 assertArrayEquals(val1, r.getValue(fam1, qf1));
2056 assertArrayEquals(val2, r.getValue(fam2, qf2));
2057
2058
2059 delete = new Delete(row1);
2060 delete.deleteFamily(fam2);
2061 res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
2062 delete, true);
2063 assertEquals(true, res);
2064
2065 get = new Get(row1);
2066 r = region.get(get);
2067 assertEquals(1, r.size());
2068 assertArrayEquals(val1, r.getValue(fam1, qf1));
2069
2070
2071 delete = new Delete(row1);
2072 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
2073 delete, true);
2074 assertEquals(true, res);
2075 get = new Get(row1);
2076 r = region.get(get);
2077 assertEquals(0, r.size());
2078 } finally {
2079 HRegion.closeHRegion(this.region);
2080 this.region = null;
2081 }
2082 }
2083
2084
2085
2086
2087 @Test
2088 public void testDelete_multiDeleteColumn() throws IOException {
2089 byte[] row1 = Bytes.toBytes("row1");
2090 byte[] fam1 = Bytes.toBytes("fam1");
2091 byte[] qual = Bytes.toBytes("qualifier");
2092 byte[] value = Bytes.toBytes("value");
2093
2094 Put put = new Put(row1);
2095 put.add(fam1, qual, 1, value);
2096 put.add(fam1, qual, 2, value);
2097
2098 String method = this.getName();
2099 this.region = initHRegion(tableName, method, CONF, fam1);
2100 try {
2101 region.put(put);
2102
2103
2104 Delete delete = new Delete(row1);
2105 delete.deleteColumn(fam1, qual);
2106 delete.deleteColumn(fam1, qual);
2107 region.delete(delete);
2108
2109 Get get = new Get(row1);
2110 get.addFamily(fam1);
2111 Result r = region.get(get);
2112 assertEquals(0, r.size());
2113 } finally {
2114 HRegion.closeHRegion(this.region);
2115 this.region = null;
2116 }
2117 }
2118
2119 @Test
2120 public void testDelete_CheckFamily() throws IOException {
2121 byte[] row1 = Bytes.toBytes("row1");
2122 byte[] fam1 = Bytes.toBytes("fam1");
2123 byte[] fam2 = Bytes.toBytes("fam2");
2124 byte[] fam3 = Bytes.toBytes("fam3");
2125 byte[] fam4 = Bytes.toBytes("fam4");
2126
2127
2128 String method = this.getName();
2129 this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
2130 try {
2131 List<Cell> kvs = new ArrayList<Cell>();
2132 kvs.add(new KeyValue(row1, fam4, null, null));
2133
2134
2135 byte[] family = fam2;
2136 try {
2137 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
2138 Bytes.BYTES_COMPARATOR);
2139 deleteMap.put(family, kvs);
2140 region.delete(deleteMap, Durability.SYNC_WAL);
2141 } catch (Exception e) {
2142 assertTrue("Family " + new String(family) + " does not exist", false);
2143 }
2144
2145
2146 boolean ok = false;
2147 family = fam4;
2148 try {
2149 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
2150 Bytes.BYTES_COMPARATOR);
2151 deleteMap.put(family, kvs);
2152 region.delete(deleteMap, Durability.SYNC_WAL);
2153 } catch (Exception e) {
2154 ok = true;
2155 }
2156 assertEquals("Family " + new String(family) + " does exist", true, ok);
2157 } finally {
2158 HRegion.closeHRegion(this.region);
2159 this.region = null;
2160 }
2161 }
2162
2163 @Test
2164 public void testDelete_mixed() throws IOException, InterruptedException {
2165 byte[] fam = Bytes.toBytes("info");
2166 byte[][] families = { fam };
2167 String method = this.getName();
2168 this.region = initHRegion(tableName, method, CONF, families);
2169 try {
2170 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
2171
2172 byte[] row = Bytes.toBytes("table_name");
2173
2174 byte[] serverinfo = Bytes.toBytes("serverinfo");
2175 byte[] splitA = Bytes.toBytes("splitA");
2176 byte[] splitB = Bytes.toBytes("splitB");
2177
2178
2179 Put put = new Put(row);
2180 put.add(fam, splitA, Bytes.toBytes("reference_A"));
2181 region.put(put);
2182
2183 put = new Put(row);
2184 put.add(fam, splitB, Bytes.toBytes("reference_B"));
2185 region.put(put);
2186
2187 put = new Put(row);
2188 put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
2189 region.put(put);
2190
2191
2192 Delete delete = new Delete(row);
2193 delete.deleteColumns(fam, splitA);
2194 region.delete(delete);
2195
2196
2197 Get get = new Get(row).addColumn(fam, serverinfo);
2198 Result result = region.get(get);
2199 assertEquals(1, result.size());
2200
2201 get = new Get(row).addColumn(fam, splitA);
2202 result = region.get(get);
2203 assertEquals(0, result.size());
2204
2205 get = new Get(row).addColumn(fam, splitB);
2206 result = region.get(get);
2207 assertEquals(1, result.size());
2208
2209
2210 put = new Put(row);
2211 put.add(fam, splitA, Bytes.toBytes("reference_A"));
2212 region.put(put);
2213 get = new Get(row);
2214 result = region.get(get);
2215 assertEquals(3, result.size());
2216
2217
2218 delete = new Delete(row);
2219 region.delete(delete);
2220 assertEquals(0, region.get(get).size());
2221
2222 region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
2223 result = region.get(get);
2224 assertEquals(1, result.size());
2225 } finally {
2226 HRegion.closeHRegion(this.region);
2227 this.region = null;
2228 }
2229 }
2230
2231 @Test
2232 public void testDeleteRowWithFutureTs() throws IOException {
2233 byte[] fam = Bytes.toBytes("info");
2234 byte[][] families = { fam };
2235 String method = this.getName();
2236 this.region = initHRegion(tableName, method, CONF, families);
2237 try {
2238 byte[] row = Bytes.toBytes("table_name");
2239
2240 byte[] serverinfo = Bytes.toBytes("serverinfo");
2241
2242
2243 Put put = new Put(row);
2244 put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
2245 region.put(put);
2246
2247
2248 Delete delete = new Delete(row);
2249 region.delete(delete);
2250
2251
2252 Get get = new Get(row).addColumn(fam, serverinfo);
2253 Result result = region.get(get);
2254 assertEquals(1, result.size());
2255
2256
2257 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
2258 region.delete(delete);
2259
2260
2261 get = new Get(row).addColumn(fam, serverinfo);
2262 result = region.get(get);
2263 assertEquals(0, result.size());
2264 } finally {
2265 HRegion.closeHRegion(this.region);
2266 this.region = null;
2267 }
2268 }
2269
2270
2271
2272
2273
2274 @Test
2275 public void testPutWithLatestTS() throws IOException {
2276 byte[] fam = Bytes.toBytes("info");
2277 byte[][] families = { fam };
2278 String method = this.getName();
2279 this.region = initHRegion(tableName, method, CONF, families);
2280 try {
2281 byte[] row = Bytes.toBytes("row1");
2282
2283 byte[] qual = Bytes.toBytes("qual");
2284
2285
2286 Put put = new Put(row);
2287 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
2288 region.put(put);
2289
2290
2291 Get get = new Get(row).addColumn(fam, qual);
2292 Result result = region.get(get);
2293 assertEquals(1, result.size());
2294 Cell kv = result.rawCells()[0];
2295 LOG.info("Got: " + kv);
2296 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
2297 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
2298
2299
2300
2301 row = Bytes.toBytes("row2");
2302 put = new Put(row);
2303 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
2304 region.put(put);
2305
2306
2307 get = new Get(row).addColumn(fam, qual);
2308 result = region.get(get);
2309 assertEquals(1, result.size());
2310 kv = result.rawCells()[0];
2311 LOG.info("Got: " + kv);
2312 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
2313 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
2314 } finally {
2315 HRegion.closeHRegion(this.region);
2316 this.region = null;
2317 }
2318
2319 }
2320
2321
2322
2323
2324
2325
2326 @Test
2327 public void testPutWithTsSlop() throws IOException {
2328 byte[] fam = Bytes.toBytes("info");
2329 byte[][] families = { fam };
2330 String method = this.getName();
2331
2332
2333 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
2334 this.region = initHRegion(tableName, method, CONF, families);
2335 boolean caughtExcep = false;
2336 try {
2337 try {
2338
2339 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
2340
2341 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
2342 Bytes.toBytes("value")));
2343 fail("Expected IOE for TS out of configured timerange");
2344 } catch (FailedSanityCheckException ioe) {
2345 LOG.debug("Received expected exception", ioe);
2346 caughtExcep = true;
2347 }
2348 assertTrue("Should catch FailedSanityCheckException", caughtExcep);
2349 } finally {
2350 HRegion.closeHRegion(this.region);
2351 this.region = null;
2352 }
2353 }
2354
2355 @Test
2356 public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
2357 byte[] fam1 = Bytes.toBytes("columnA");
2358 byte[] fam2 = Bytes.toBytes("columnB");
2359 this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
2360 try {
2361 byte[] rowA = Bytes.toBytes("rowA");
2362 byte[] rowB = Bytes.toBytes("rowB");
2363
2364 byte[] value = Bytes.toBytes("value");
2365
2366 Delete delete = new Delete(rowA);
2367 delete.deleteFamily(fam1);
2368
2369 region.delete(delete);
2370
2371
2372 Put put = new Put(rowA);
2373 put.add(fam2, null, value);
2374 region.put(put);
2375
2376 put = new Put(rowB);
2377 put.add(fam1, null, value);
2378 put.add(fam2, null, value);
2379 region.put(put);
2380
2381 Scan scan = new Scan();
2382 scan.addFamily(fam1).addFamily(fam2);
2383 InternalScanner s = region.getScanner(scan);
2384 List<Cell> results = new ArrayList<Cell>();
2385 s.next(results);
2386 assertTrue(CellUtil.matchingRow(results.get(0), rowA));
2387
2388 results.clear();
2389 s.next(results);
2390 assertTrue(CellUtil.matchingRow(results.get(0), rowB));
2391 } finally {
2392 HRegion.closeHRegion(this.region);
2393 this.region = null;
2394 }
2395 }
2396
2397 @Test
2398 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
2399 Delete delete = new Delete(row);
2400 delete.deleteColumns(fam1, qual1);
2401 doTestDelete_AndPostInsert(delete);
2402 }
2403
2404 @Test
2405 public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
2406 Delete delete = new Delete(row);
2407 delete.deleteFamily(fam1);
2408 doTestDelete_AndPostInsert(delete);
2409 }
2410
2411 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
2412 TableName tableName = TableName.valueOf(name.getMethodName());
2413 this.region = initHRegion(tableName, getName(), CONF, fam1);
2414 try {
2415 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
2416 Put put = new Put(row);
2417 put.add(fam1, qual1, value1);
2418 region.put(put);
2419
2420
2421 region.delete(delete);
2422
2423
2424 put = new Put(row);
2425 put.add(fam1, qual1, value2);
2426 region.put(put);
2427
2428
2429 Get get = new Get(row);
2430 get.addColumn(fam1, qual1);
2431
2432 Result r = region.get(get);
2433 assertEquals(1, r.size());
2434 assertArrayEquals(value2, r.getValue(fam1, qual1));
2435
2436
2437 Scan scan = new Scan(row);
2438 scan.addColumn(fam1, qual1);
2439 InternalScanner s = region.getScanner(scan);
2440
2441 List<Cell> results = new ArrayList<Cell>();
2442 assertEquals(false, s.next(results));
2443 assertEquals(1, results.size());
2444 Cell kv = results.get(0);
2445
2446 assertArrayEquals(value2, CellUtil.cloneValue(kv));
2447 assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
2448 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
2449 assertArrayEquals(row, CellUtil.cloneRow(kv));
2450 } finally {
2451 HRegion.closeHRegion(this.region);
2452 this.region = null;
2453 }
2454 }
2455
2456 @Test
2457 public void testDelete_CheckTimestampUpdated() throws IOException {
2458 TableName tableName = TableName.valueOf(name.getMethodName());
2459 byte[] row1 = Bytes.toBytes("row1");
2460 byte[] col1 = Bytes.toBytes("col1");
2461 byte[] col2 = Bytes.toBytes("col2");
2462 byte[] col3 = Bytes.toBytes("col3");
2463
2464
2465 String method = this.getName();
2466 this.region = initHRegion(tableName, method, CONF, fam1);
2467 try {
2468
2469 List<Cell> kvs = new ArrayList<Cell>();
2470 kvs.add(new KeyValue(row1, fam1, col1, null));
2471 kvs.add(new KeyValue(row1, fam1, col2, null));
2472 kvs.add(new KeyValue(row1, fam1, col3, null));
2473
2474 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
2475 Bytes.BYTES_COMPARATOR);
2476 deleteMap.put(fam1, kvs);
2477 region.delete(deleteMap, Durability.SYNC_WAL);
2478
2479
2480
2481 long now = System.currentTimeMillis();
2482 DefaultMemStore memstore = (DefaultMemStore) ((HStore) region.getStore(fam1)).memstore;
2483 Cell firstCell = memstore.cellSet.first();
2484 assertTrue(firstCell.getTimestamp() <= now);
2485 now = firstCell.getTimestamp();
2486 for (Cell cell : memstore.cellSet) {
2487 assertTrue(cell.getTimestamp() <= now);
2488 now = cell.getTimestamp();
2489 }
2490 } finally {
2491 HRegion.closeHRegion(this.region);
2492 this.region = null;
2493 }
2494 }
2495
2496
2497
2498
2499 @Test
2500 public void testGet_FamilyChecker() throws IOException {
2501 byte[] row1 = Bytes.toBytes("row1");
2502 byte[] fam1 = Bytes.toBytes("fam1");
2503 byte[] fam2 = Bytes.toBytes("False");
2504 byte[] col1 = Bytes.toBytes("col1");
2505
2506
2507 String method = this.getName();
2508 this.region = initHRegion(tableName, method, CONF, fam1);
2509 try {
2510 Get get = new Get(row1);
2511 get.addColumn(fam2, col1);
2512
2513
2514 try {
2515 region.get(get);
2516 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
2517 assertFalse(false);
2518 return;
2519 }
2520 assertFalse(true);
2521 } finally {
2522 HRegion.closeHRegion(this.region);
2523 this.region = null;
2524 }
2525 }
2526
2527 @Test
2528 public void testGet_Basic() throws IOException {
2529 byte[] row1 = Bytes.toBytes("row1");
2530 byte[] fam1 = Bytes.toBytes("fam1");
2531 byte[] col1 = Bytes.toBytes("col1");
2532 byte[] col2 = Bytes.toBytes("col2");
2533 byte[] col3 = Bytes.toBytes("col3");
2534 byte[] col4 = Bytes.toBytes("col4");
2535 byte[] col5 = Bytes.toBytes("col5");
2536
2537
2538 String method = this.getName();
2539 this.region = initHRegion(tableName, method, CONF, fam1);
2540 try {
2541
2542 Put put = new Put(row1);
2543 put.add(fam1, col1, null);
2544 put.add(fam1, col2, null);
2545 put.add(fam1, col3, null);
2546 put.add(fam1, col4, null);
2547 put.add(fam1, col5, null);
2548 region.put(put);
2549
2550 Get get = new Get(row1);
2551 get.addColumn(fam1, col2);
2552 get.addColumn(fam1, col4);
2553
2554 KeyValue kv1 = new KeyValue(row1, fam1, col2);
2555 KeyValue kv2 = new KeyValue(row1, fam1, col4);
2556 KeyValue[] expected = { kv1, kv2 };
2557
2558
2559 Result res = region.get(get);
2560 assertEquals(expected.length, res.size());
2561 for (int i = 0; i < res.size(); i++) {
2562 assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
2563 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
2564 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
2565 }
2566
2567
2568 Get g = new Get(row1);
2569 final int count = 2;
2570 g.setFilter(new ColumnCountGetFilter(count));
2571 res = region.get(g);
2572 assertEquals(count, res.size());
2573 } finally {
2574 HRegion.closeHRegion(this.region);
2575 this.region = null;
2576 }
2577 }
2578
2579 @Test
2580 public void testGet_Empty() throws IOException {
2581 byte[] row = Bytes.toBytes("row");
2582 byte[] fam = Bytes.toBytes("fam");
2583
2584 String method = this.getName();
2585 this.region = initHRegion(tableName, method, CONF, fam);
2586 try {
2587 Get get = new Get(row);
2588 get.addFamily(fam);
2589 Result r = region.get(get);
2590
2591 assertTrue(r.isEmpty());
2592 } finally {
2593 HRegion.closeHRegion(this.region);
2594 this.region = null;
2595 }
2596 }
2597
2598
2599
2600
2601 @Test
2602 public void testMerge() throws IOException {
2603 byte[][] families = { fam1, fam2, fam3 };
2604 Configuration hc = initSplit();
2605
2606 String method = this.getName();
2607 this.region = initHRegion(tableName, method, hc, families);
2608 try {
2609 LOG.info("" + HBaseTestCase.addContent(region, fam3));
2610 region.flush(true);
2611 region.compactStores();
2612 byte[] splitRow = region.checkSplit();
2613 assertNotNull(splitRow);
2614 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2615 HRegion[] subregions = splitRegion(region, splitRow);
2616 try {
2617
2618 for (int i = 0; i < subregions.length; i++) {
2619 HRegion.openHRegion(subregions[i], null);
2620 subregions[i].compactStores();
2621 }
2622 Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
2623 Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
2624 Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
2625 long startTime = System.currentTimeMillis();
2626 region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
2627 LOG.info("Merge regions elapsed time: "
2628 + ((System.currentTimeMillis() - startTime) / 1000.0));
2629 FILESYSTEM.delete(oldRegion1, true);
2630 FILESYSTEM.delete(oldRegion2, true);
2631 FILESYSTEM.delete(oldRegionPath, true);
2632 LOG.info("splitAndMerge completed.");
2633 } finally {
2634 for (int i = 0; i < subregions.length; i++) {
2635 try {
2636 HRegion.closeHRegion(subregions[i]);
2637 } catch (IOException e) {
2638
2639 }
2640 }
2641 }
2642 } finally {
2643 HRegion.closeHRegion(this.region);
2644 this.region = null;
2645 }
2646 }
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656 HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2657 PairOfSameType<Region> result = null;
2658 SplitTransactionImpl st = new SplitTransactionImpl(parent, midkey);
2659
2660
2661 if (!st.prepare()) {
2662 parent.clearSplit();
2663 return null;
2664 }
2665 try {
2666 result = st.execute(null, null);
2667 } catch (IOException ioe) {
2668 try {
2669 LOG.info("Running rollback of failed split of " +
2670 parent.getRegionInfo().getRegionNameAsString() + "; " + ioe.getMessage());
2671 st.rollback(null, null);
2672 LOG.info("Successful rollback of failed split of " +
2673 parent.getRegionInfo().getRegionNameAsString());
2674 return null;
2675 } catch (RuntimeException e) {
2676
2677 LOG.info("Failed rollback of failed split of " +
2678 parent.getRegionInfo().getRegionNameAsString() + " -- aborting server", e);
2679 }
2680 }
2681 finally {
2682 parent.clearSplit();
2683 }
2684 return new HRegion[] { (HRegion)result.getFirst(), (HRegion)result.getSecond() };
2685 }
2686
2687
2688
2689
2690 @Test
2691 public void testGetScanner_WithOkFamilies() throws IOException {
2692 byte[] fam1 = Bytes.toBytes("fam1");
2693 byte[] fam2 = Bytes.toBytes("fam2");
2694
2695 byte[][] families = { fam1, fam2 };
2696
2697
2698 String method = this.getName();
2699 this.region = initHRegion(tableName, method, CONF, families);
2700 try {
2701 Scan scan = new Scan();
2702 scan.addFamily(fam1);
2703 scan.addFamily(fam2);
2704 try {
2705 region.getScanner(scan);
2706 } catch (Exception e) {
2707 assertTrue("Families could not be found in Region", false);
2708 }
2709 } finally {
2710 HRegion.closeHRegion(this.region);
2711 this.region = null;
2712 }
2713 }
2714
2715 @Test
2716 public void testGetScanner_WithNotOkFamilies() throws IOException {
2717 byte[] fam1 = Bytes.toBytes("fam1");
2718 byte[] fam2 = Bytes.toBytes("fam2");
2719
2720 byte[][] families = { fam1 };
2721
2722
2723 String method = this.getName();
2724 this.region = initHRegion(tableName, method, CONF, families);
2725 try {
2726 Scan scan = new Scan();
2727 scan.addFamily(fam2);
2728 boolean ok = false;
2729 try {
2730 region.getScanner(scan);
2731 } catch (Exception e) {
2732 ok = true;
2733 }
2734 assertTrue("Families could not be found in Region", ok);
2735 } finally {
2736 HRegion.closeHRegion(this.region);
2737 this.region = null;
2738 }
2739 }
2740
2741 @Test
2742 public void testGetScanner_WithNoFamilies() throws IOException {
2743 byte[] row1 = Bytes.toBytes("row1");
2744 byte[] fam1 = Bytes.toBytes("fam1");
2745 byte[] fam2 = Bytes.toBytes("fam2");
2746 byte[] fam3 = Bytes.toBytes("fam3");
2747 byte[] fam4 = Bytes.toBytes("fam4");
2748
2749 byte[][] families = { fam1, fam2, fam3, fam4 };
2750
2751
2752 String method = this.getName();
2753 this.region = initHRegion(tableName, method, CONF, families);
2754 try {
2755
2756
2757 Put put = new Put(row1);
2758 put.add(fam1, null, null);
2759 put.add(fam2, null, null);
2760 put.add(fam3, null, null);
2761 put.add(fam4, null, null);
2762 region.put(put);
2763
2764 Scan scan = null;
2765 HRegion.RegionScannerImpl is = null;
2766
2767
2768
2769
2770 scan = new Scan();
2771 scan.addFamily(fam2);
2772 scan.addFamily(fam4);
2773 is = (RegionScannerImpl) region.getScanner(scan);
2774 assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2775
2776 scan = new Scan();
2777 is = (RegionScannerImpl) region.getScanner(scan);
2778 assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2779 } finally {
2780 HRegion.closeHRegion(this.region);
2781 this.region = null;
2782 }
2783 }
2784
2785
2786
2787
2788
2789
2790 @Test
2791 public void testGetScanner_WithRegionClosed() throws IOException {
2792 byte[] fam1 = Bytes.toBytes("fam1");
2793 byte[] fam2 = Bytes.toBytes("fam2");
2794
2795 byte[][] families = { fam1, fam2 };
2796
2797
2798 String method = this.getName();
2799 try {
2800 this.region = initHRegion(tableName, method, CONF, families);
2801 } catch (IOException e) {
2802 e.printStackTrace();
2803 fail("Got IOException during initHRegion, " + e.getMessage());
2804 }
2805 try {
2806 region.closed.set(true);
2807 try {
2808 region.getScanner(null);
2809 fail("Expected to get an exception during getScanner on a region that is closed");
2810 } catch (NotServingRegionException e) {
2811
2812 } catch (IOException e) {
2813 fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2814 + e.getMessage());
2815 }
2816 } finally {
2817 HRegion.closeHRegion(this.region);
2818 this.region = null;
2819 }
2820 }
2821
2822 @Test
2823 public void testRegionScanner_Next() throws IOException {
2824 byte[] row1 = Bytes.toBytes("row1");
2825 byte[] row2 = Bytes.toBytes("row2");
2826 byte[] fam1 = Bytes.toBytes("fam1");
2827 byte[] fam2 = Bytes.toBytes("fam2");
2828 byte[] fam3 = Bytes.toBytes("fam3");
2829 byte[] fam4 = Bytes.toBytes("fam4");
2830
2831 byte[][] families = { fam1, fam2, fam3, fam4 };
2832 long ts = System.currentTimeMillis();
2833
2834
2835 String method = this.getName();
2836 this.region = initHRegion(tableName, method, CONF, families);
2837 try {
2838
2839 Put put = null;
2840 put = new Put(row1);
2841 put.add(fam1, (byte[]) null, ts, null);
2842 put.add(fam2, (byte[]) null, ts, null);
2843 put.add(fam3, (byte[]) null, ts, null);
2844 put.add(fam4, (byte[]) null, ts, null);
2845 region.put(put);
2846
2847 put = new Put(row2);
2848 put.add(fam1, (byte[]) null, ts, null);
2849 put.add(fam2, (byte[]) null, ts, null);
2850 put.add(fam3, (byte[]) null, ts, null);
2851 put.add(fam4, (byte[]) null, ts, null);
2852 region.put(put);
2853
2854 Scan scan = new Scan();
2855 scan.addFamily(fam2);
2856 scan.addFamily(fam4);
2857 InternalScanner is = region.getScanner(scan);
2858
2859 List<Cell> res = null;
2860
2861
2862 List<Cell> expected1 = new ArrayList<Cell>();
2863 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2864 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2865
2866 res = new ArrayList<Cell>();
2867 is.next(res);
2868 for (int i = 0; i < res.size(); i++) {
2869 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2870 }
2871
2872
2873 List<Cell> expected2 = new ArrayList<Cell>();
2874 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2875 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2876
2877 res = new ArrayList<Cell>();
2878 is.next(res);
2879 for (int i = 0; i < res.size(); i++) {
2880 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2881 }
2882 } finally {
2883 HRegion.closeHRegion(this.region);
2884 this.region = null;
2885 }
2886 }
2887
2888 @Test
2889 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2890 byte[] row1 = Bytes.toBytes("row1");
2891 byte[] qf1 = Bytes.toBytes("qualifier1");
2892 byte[] qf2 = Bytes.toBytes("qualifier2");
2893 byte[] fam1 = Bytes.toBytes("fam1");
2894 byte[][] families = { fam1 };
2895
2896 long ts1 = System.currentTimeMillis();
2897 long ts2 = ts1 + 1;
2898 long ts3 = ts1 + 2;
2899
2900
2901 String method = this.getName();
2902 this.region = initHRegion(tableName, method, CONF, families);
2903 try {
2904
2905 Put put = null;
2906 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2907 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2908 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2909
2910 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2911 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2912 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2913
2914 put = new Put(row1);
2915 put.add(kv13);
2916 put.add(kv12);
2917 put.add(kv11);
2918 put.add(kv23);
2919 put.add(kv22);
2920 put.add(kv21);
2921 region.put(put);
2922
2923
2924 List<Cell> expected = new ArrayList<Cell>();
2925 expected.add(kv13);
2926 expected.add(kv12);
2927
2928 Scan scan = new Scan(row1);
2929 scan.addColumn(fam1, qf1);
2930 scan.setMaxVersions(MAX_VERSIONS);
2931 List<Cell> actual = new ArrayList<Cell>();
2932 InternalScanner scanner = region.getScanner(scan);
2933
2934 boolean hasNext = scanner.next(actual);
2935 assertEquals(false, hasNext);
2936
2937
2938 for (int i = 0; i < expected.size(); i++) {
2939 assertEquals(expected.get(i), actual.get(i));
2940 }
2941 } finally {
2942 HRegion.closeHRegion(this.region);
2943 this.region = null;
2944 }
2945 }
2946
2947 @Test
2948 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2949 byte[] row1 = Bytes.toBytes("row1");
2950 byte[] qf1 = Bytes.toBytes("qualifier1");
2951 byte[] qf2 = Bytes.toBytes("qualifier2");
2952 byte[] fam1 = Bytes.toBytes("fam1");
2953 byte[][] families = { fam1 };
2954
2955 long ts1 = 1;
2956 long ts2 = ts1 + 1;
2957 long ts3 = ts1 + 2;
2958
2959
2960 String method = this.getName();
2961 this.region = initHRegion(tableName, method, CONF, families);
2962 try {
2963
2964 Put put = null;
2965 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2966 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2967 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2968
2969 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2970 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2971 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2972
2973 put = new Put(row1);
2974 put.add(kv13);
2975 put.add(kv12);
2976 put.add(kv11);
2977 put.add(kv23);
2978 put.add(kv22);
2979 put.add(kv21);
2980 region.put(put);
2981 region.flush(true);
2982
2983
2984 List<Cell> expected = new ArrayList<Cell>();
2985 expected.add(kv13);
2986 expected.add(kv12);
2987 expected.add(kv23);
2988 expected.add(kv22);
2989
2990 Scan scan = new Scan(row1);
2991 scan.addColumn(fam1, qf1);
2992 scan.addColumn(fam1, qf2);
2993 scan.setMaxVersions(MAX_VERSIONS);
2994 List<Cell> actual = new ArrayList<Cell>();
2995 InternalScanner scanner = region.getScanner(scan);
2996
2997 boolean hasNext = scanner.next(actual);
2998 assertEquals(false, hasNext);
2999
3000
3001 for (int i = 0; i < expected.size(); i++) {
3002 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
3003 }
3004 } finally {
3005 HRegion.closeHRegion(this.region);
3006 this.region = null;
3007 }
3008 }
3009
3010 @Test
3011 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
3012 byte[] row1 = Bytes.toBytes("row1");
3013 byte[] fam1 = Bytes.toBytes("fam1");
3014 byte[][] families = { fam1 };
3015 byte[] qf1 = Bytes.toBytes("qualifier1");
3016 byte[] qf2 = Bytes.toBytes("qualifier2");
3017
3018 long ts1 = 1;
3019 long ts2 = ts1 + 1;
3020 long ts3 = ts1 + 2;
3021 long ts4 = ts1 + 3;
3022
3023
3024 String method = this.getName();
3025 this.region = initHRegion(tableName, method, CONF, families);
3026 try {
3027
3028 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
3029 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
3030 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
3031 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
3032
3033 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
3034 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
3035 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
3036 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
3037
3038 Put put = null;
3039 put = new Put(row1);
3040 put.add(kv14);
3041 put.add(kv24);
3042 region.put(put);
3043 region.flush(true);
3044
3045 put = new Put(row1);
3046 put.add(kv23);
3047 put.add(kv13);
3048 region.put(put);
3049 region.flush(true);
3050
3051 put = new Put(row1);
3052 put.add(kv22);
3053 put.add(kv12);
3054 region.put(put);
3055 region.flush(true);
3056
3057 put = new Put(row1);
3058 put.add(kv21);
3059 put.add(kv11);
3060 region.put(put);
3061
3062
3063 List<Cell> expected = new ArrayList<Cell>();
3064 expected.add(kv14);
3065 expected.add(kv13);
3066 expected.add(kv12);
3067 expected.add(kv24);
3068 expected.add(kv23);
3069 expected.add(kv22);
3070
3071 Scan scan = new Scan(row1);
3072 scan.addColumn(fam1, qf1);
3073 scan.addColumn(fam1, qf2);
3074 int versions = 3;
3075 scan.setMaxVersions(versions);
3076 List<Cell> actual = new ArrayList<Cell>();
3077 InternalScanner scanner = region.getScanner(scan);
3078
3079 boolean hasNext = scanner.next(actual);
3080 assertEquals(false, hasNext);
3081
3082
3083 for (int i = 0; i < expected.size(); i++) {
3084 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
3085 }
3086 } finally {
3087 HRegion.closeHRegion(this.region);
3088 this.region = null;
3089 }
3090 }
3091
3092 @Test
3093 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
3094 byte[] row1 = Bytes.toBytes("row1");
3095 byte[] qf1 = Bytes.toBytes("qualifier1");
3096 byte[] qf2 = Bytes.toBytes("qualifier2");
3097 byte[] fam1 = Bytes.toBytes("fam1");
3098 byte[][] families = { fam1 };
3099
3100 long ts1 = System.currentTimeMillis();
3101 long ts2 = ts1 + 1;
3102 long ts3 = ts1 + 2;
3103
3104
3105 String method = this.getName();
3106 this.region = initHRegion(tableName, method, CONF, families);
3107 try {
3108
3109 Put put = null;
3110 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
3111 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
3112 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
3113
3114 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
3115 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
3116 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
3117
3118 put = new Put(row1);
3119 put.add(kv13);
3120 put.add(kv12);
3121 put.add(kv11);
3122 put.add(kv23);
3123 put.add(kv22);
3124 put.add(kv21);
3125 region.put(put);
3126
3127
3128 List<Cell> expected = new ArrayList<Cell>();
3129 expected.add(kv13);
3130 expected.add(kv12);
3131 expected.add(kv23);
3132 expected.add(kv22);
3133
3134 Scan scan = new Scan(row1);
3135 scan.addFamily(fam1);
3136 scan.setMaxVersions(MAX_VERSIONS);
3137 List<Cell> actual = new ArrayList<Cell>();
3138 InternalScanner scanner = region.getScanner(scan);
3139
3140 boolean hasNext = scanner.next(actual);
3141 assertEquals(false, hasNext);
3142
3143
3144 for (int i = 0; i < expected.size(); i++) {
3145 assertEquals(expected.get(i), actual.get(i));
3146 }
3147 } finally {
3148 HRegion.closeHRegion(this.region);
3149 this.region = null;
3150 }
3151 }
3152
3153 @Test
3154 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
3155 byte[] row1 = Bytes.toBytes("row1");
3156 byte[] qf1 = Bytes.toBytes("qualifier1");
3157 byte[] qf2 = Bytes.toBytes("qualifier2");
3158 byte[] fam1 = Bytes.toBytes("fam1");
3159
3160 long ts1 = 1;
3161 long ts2 = ts1 + 1;
3162 long ts3 = ts1 + 2;
3163
3164
3165 String method = this.getName();
3166 this.region = initHRegion(tableName, method, CONF, fam1);
3167 try {
3168
3169 Put put = null;
3170 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
3171 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
3172 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
3173
3174 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
3175 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
3176 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
3177
3178 put = new Put(row1);
3179 put.add(kv13);
3180 put.add(kv12);
3181 put.add(kv11);
3182 put.add(kv23);
3183 put.add(kv22);
3184 put.add(kv21);
3185 region.put(put);
3186 region.flush(true);
3187
3188
3189 List<Cell> expected = new ArrayList<Cell>();
3190 expected.add(kv13);
3191 expected.add(kv12);
3192 expected.add(kv23);
3193 expected.add(kv22);
3194
3195 Scan scan = new Scan(row1);
3196 scan.addFamily(fam1);
3197 scan.setMaxVersions(MAX_VERSIONS);
3198 List<Cell> actual = new ArrayList<Cell>();
3199 InternalScanner scanner = region.getScanner(scan);
3200
3201 boolean hasNext = scanner.next(actual);
3202 assertEquals(false, hasNext);
3203
3204
3205 for (int i = 0; i < expected.size(); i++) {
3206 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
3207 }
3208 } finally {
3209 HRegion.closeHRegion(this.region);
3210 this.region = null;
3211 }
3212 }
3213
3214 @Test
3215 public void testScanner_StopRow1542() throws IOException {
3216 byte[] family = Bytes.toBytes("testFamily");
3217 this.region = initHRegion(tableName, getName(), CONF, family);
3218 try {
3219 byte[] row1 = Bytes.toBytes("row111");
3220 byte[] row2 = Bytes.toBytes("row222");
3221 byte[] row3 = Bytes.toBytes("row333");
3222 byte[] row4 = Bytes.toBytes("row444");
3223 byte[] row5 = Bytes.toBytes("row555");
3224
3225 byte[] col1 = Bytes.toBytes("Pub111");
3226 byte[] col2 = Bytes.toBytes("Pub222");
3227
3228 Put put = new Put(row1);
3229 put.add(family, col1, Bytes.toBytes(10L));
3230 region.put(put);
3231
3232 put = new Put(row2);
3233 put.add(family, col1, Bytes.toBytes(15L));
3234 region.put(put);
3235
3236 put = new Put(row3);
3237 put.add(family, col2, Bytes.toBytes(20L));
3238 region.put(put);
3239
3240 put = new Put(row4);
3241 put.add(family, col2, Bytes.toBytes(30L));
3242 region.put(put);
3243
3244 put = new Put(row5);
3245 put.add(family, col1, Bytes.toBytes(40L));
3246 region.put(put);
3247
3248 Scan scan = new Scan(row3, row4);
3249 scan.setMaxVersions();
3250 scan.addColumn(family, col1);
3251 InternalScanner s = region.getScanner(scan);
3252
3253 List<Cell> results = new ArrayList<Cell>();
3254 assertEquals(false, s.next(results));
3255 assertEquals(0, results.size());
3256 } finally {
3257 HRegion.closeHRegion(this.region);
3258 this.region = null;
3259 }
3260 }
3261
3262 @Test
3263 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
3264 byte[] row1 = Bytes.toBytes("row1");
3265 byte[] fam1 = Bytes.toBytes("fam1");
3266 byte[] qf1 = Bytes.toBytes("qualifier1");
3267 byte[] qf2 = Bytes.toBytes("quateslifier2");
3268
3269 long ts1 = 1;
3270 long ts2 = ts1 + 1;
3271 long ts3 = ts1 + 2;
3272 long ts4 = ts1 + 3;
3273
3274
3275 String method = this.getName();
3276 this.region = initHRegion(tableName, method, CONF, fam1);
3277 try {
3278
3279 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
3280 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
3281 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
3282 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
3283
3284 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
3285 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
3286 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
3287 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
3288
3289 Put put = null;
3290 put = new Put(row1);
3291 put.add(kv14);
3292 put.add(kv24);
3293 region.put(put);
3294 region.flush(true);
3295
3296 put = new Put(row1);
3297 put.add(kv23);
3298 put.add(kv13);
3299 region.put(put);
3300 region.flush(true);
3301
3302 put = new Put(row1);
3303 put.add(kv22);
3304 put.add(kv12);
3305 region.put(put);
3306 region.flush(true);
3307
3308 put = new Put(row1);
3309 put.add(kv21);
3310 put.add(kv11);
3311 region.put(put);
3312
3313
3314 List<KeyValue> expected = new ArrayList<KeyValue>();
3315 expected.add(kv14);
3316 expected.add(kv13);
3317 expected.add(kv12);
3318 expected.add(kv24);
3319 expected.add(kv23);
3320 expected.add(kv22);
3321
3322 Scan scan = new Scan(row1);
3323 int versions = 3;
3324 scan.setMaxVersions(versions);
3325 List<Cell> actual = new ArrayList<Cell>();
3326 InternalScanner scanner = region.getScanner(scan);
3327
3328 boolean hasNext = scanner.next(actual);
3329 assertEquals(false, hasNext);
3330
3331
3332 for (int i = 0; i < expected.size(); i++) {
3333 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
3334 }
3335 } finally {
3336 HRegion.closeHRegion(this.region);
3337 this.region = null;
3338 }
3339 }
3340
3341
3342
3343
3344
3345
3346
3347 @Test
3348 public void testScanner_JoinedScanners() throws IOException {
3349 byte[] cf_essential = Bytes.toBytes("essential");
3350 byte[] cf_joined = Bytes.toBytes("joined");
3351 byte[] cf_alpha = Bytes.toBytes("alpha");
3352 this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
3353 try {
3354 byte[] row1 = Bytes.toBytes("row1");
3355 byte[] row2 = Bytes.toBytes("row2");
3356 byte[] row3 = Bytes.toBytes("row3");
3357
3358 byte[] col_normal = Bytes.toBytes("d");
3359 byte[] col_alpha = Bytes.toBytes("a");
3360
3361 byte[] filtered_val = Bytes.toBytes(3);
3362
3363 Put put = new Put(row1);
3364 put.add(cf_essential, col_normal, Bytes.toBytes(1));
3365 put.add(cf_joined, col_alpha, Bytes.toBytes(1));
3366 region.put(put);
3367
3368 put = new Put(row2);
3369 put.add(cf_essential, col_alpha, Bytes.toBytes(2));
3370 put.add(cf_joined, col_normal, Bytes.toBytes(2));
3371 put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
3372 region.put(put);
3373
3374 put = new Put(row3);
3375 put.add(cf_essential, col_normal, filtered_val);
3376 put.add(cf_joined, col_normal, filtered_val);
3377 region.put(put);
3378
3379
3380
3381
3382
3383 Scan scan = new Scan();
3384 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
3385 CompareOp.NOT_EQUAL, filtered_val);
3386 scan.setFilter(filter);
3387 scan.setLoadColumnFamiliesOnDemand(true);
3388 InternalScanner s = region.getScanner(scan);
3389
3390 List<Cell> results = new ArrayList<Cell>();
3391 assertTrue(s.next(results));
3392 assertEquals(results.size(), 1);
3393 results.clear();
3394
3395 assertTrue(s.next(results));
3396 assertEquals(results.size(), 3);
3397 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
3398 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
3399 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
3400 results.clear();
3401
3402 assertFalse(s.next(results));
3403 assertEquals(results.size(), 0);
3404 } finally {
3405 HRegion.closeHRegion(this.region);
3406 this.region = null;
3407 }
3408 }
3409
3410
3411
3412
3413
3414
3415 @Test
3416 public void testScanner_JoinedScannersWithLimits() throws IOException {
3417 final byte[] cf_first = Bytes.toBytes("first");
3418 final byte[] cf_second = Bytes.toBytes("second");
3419
3420 this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
3421 try {
3422 final byte[] col_a = Bytes.toBytes("a");
3423 final byte[] col_b = Bytes.toBytes("b");
3424
3425 Put put;
3426
3427 for (int i = 0; i < 10; i++) {
3428 put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
3429 put.add(cf_first, col_a, Bytes.toBytes(i));
3430 if (i < 5) {
3431 put.add(cf_first, col_b, Bytes.toBytes(i));
3432 put.add(cf_second, col_a, Bytes.toBytes(i));
3433 put.add(cf_second, col_b, Bytes.toBytes(i));
3434 }
3435 region.put(put);
3436 }
3437
3438 Scan scan = new Scan();
3439 scan.setLoadColumnFamiliesOnDemand(true);
3440 Filter bogusFilter = new FilterBase() {
3441 @Override
3442 public ReturnCode filterKeyValue(Cell ignored) throws IOException {
3443 return ReturnCode.INCLUDE;
3444 }
3445 @Override
3446 public boolean isFamilyEssential(byte[] name) {
3447 return Bytes.equals(name, cf_first);
3448 }
3449 };
3450
3451 scan.setFilter(bogusFilter);
3452 InternalScanner s = region.getScanner(scan);
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483 List<Cell> results = new ArrayList<Cell>();
3484 int index = 0;
3485 ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(3).build();
3486 while (true) {
3487 boolean more = s.next(results, scannerContext);
3488 if ((index >> 1) < 5) {
3489 if (index % 2 == 0)
3490 assertEquals(results.size(), 3);
3491 else
3492 assertEquals(results.size(), 1);
3493 } else
3494 assertEquals(results.size(), 1);
3495 results.clear();
3496 index++;
3497 if (!more)
3498 break;
3499 }
3500 } finally {
3501 HRegion.closeHRegion(this.region);
3502 this.region = null;
3503 }
3504 }
3505
3506
3507
3508
3509
3510
3511 @Test
3512 public void testLongQualifier() throws Exception {
3513 String method = name.getMethodName();
3514 TableName tableName = TableName.valueOf(method);
3515 byte[] family = Bytes.toBytes("family");
3516 this.region = initHRegion(tableName, method, CONF, family);
3517 byte[] q = new byte[Short.MAX_VALUE+2];
3518 Arrays.fill(q, 0, q.length-1, (byte)42);
3519 for (byte i=0; i<10; i++) {
3520 Put p = new Put(Bytes.toBytes("row"));
3521
3522 q[q.length-1]=i;
3523 p.addColumn(family, q, q);
3524 region.put(p);
3525 }
3526 region.flush(false);
3527 HRegion.closeHRegion(this.region);
3528 this.region = null;
3529 }
3530
3531
3532
3533
3534
3535
3536
3537
3538 @Test
3539 public void testBasicSplit() throws Exception {
3540 byte[][] families = { fam1, fam2, fam3 };
3541
3542 Configuration hc = initSplit();
3543
3544 String method = this.getName();
3545 this.region = initHRegion(tableName, method, hc, families);
3546
3547 try {
3548 LOG.info("" + HBaseTestCase.addContent(region, fam3));
3549 region.flush(true);
3550 region.compactStores();
3551 byte[] splitRow = region.checkSplit();
3552 assertNotNull(splitRow);
3553 LOG.info("SplitRow: " + Bytes.toString(splitRow));
3554 HRegion[] regions = splitRegion(region, splitRow);
3555 try {
3556
3557
3558
3559 for (int i = 0; i < regions.length; i++) {
3560 regions[i] = HRegion.openHRegion(regions[i], null);
3561 }
3562
3563
3564 assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
3565 assertGet(regions[1], fam3, splitRow);
3566
3567 assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
3568 assertScan(regions[1], fam3, splitRow);
3569
3570 for (int i = 0; i < regions.length; i++) {
3571
3572
3573 for (int j = 0; j < 2; j++) {
3574 HBaseTestCase.addContent(regions[i], fam3);
3575 }
3576 HBaseTestCase.addContent(regions[i], fam2);
3577 HBaseTestCase.addContent(regions[i], fam1);
3578 regions[i].flush(true);
3579 }
3580
3581 byte[][] midkeys = new byte[regions.length][];
3582
3583 for (int i = 0; i < regions.length; i++) {
3584 regions[i].compactStores();
3585 midkeys[i] = regions[i].checkSplit();
3586 }
3587
3588 TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
3589
3590
3591 for (int i = 0; i < regions.length; i++) {
3592 HRegion[] rs = null;
3593 if (midkeys[i] != null) {
3594 rs = splitRegion(regions[i], midkeys[i]);
3595 for (int j = 0; j < rs.length; j++) {
3596 sortedMap.put(Bytes.toString(rs[j].getRegionInfo().getRegionName()),
3597 HRegion.openHRegion(rs[j], null));
3598 }
3599 }
3600 }
3601 LOG.info("Made 4 regions");
3602
3603
3604 int interval = (LAST_CHAR - FIRST_CHAR) / 3;
3605 byte[] b = Bytes.toBytes(START_KEY);
3606 for (HRegion r : sortedMap.values()) {
3607 assertGet(r, fam3, b);
3608 b[0] += interval;
3609 }
3610 } finally {
3611 for (int i = 0; i < regions.length; i++) {
3612 try {
3613 regions[i].close();
3614 } catch (IOException e) {
3615
3616 }
3617 }
3618 }
3619 } finally {
3620 HRegion.closeHRegion(this.region);
3621 this.region = null;
3622 }
3623 }
3624
3625 @Test
3626 public void testSplitRegion() throws IOException {
3627 byte[] qualifier = Bytes.toBytes("qualifier");
3628 Configuration hc = initSplit();
3629 int numRows = 10;
3630 byte[][] families = { fam1, fam3 };
3631
3632
3633 String method = this.getName();
3634 this.region = initHRegion(tableName, method, hc, families);
3635
3636
3637 int startRow = 100;
3638 putData(startRow, numRows, qualifier, families);
3639 int splitRow = startRow + numRows;
3640 putData(splitRow, numRows, qualifier, families);
3641 region.flush(true);
3642
3643 HRegion[] regions = null;
3644 try {
3645 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3646
3647 for (int i = 0; i < regions.length; i++) {
3648 regions[i] = HRegion.openHRegion(regions[i], null);
3649 }
3650
3651 assertEquals(2, regions.length);
3652
3653
3654
3655 verifyData(regions[0], startRow, numRows, qualifier, families);
3656 verifyData(regions[1], splitRow, numRows, qualifier, families);
3657
3658 } finally {
3659 HRegion.closeHRegion(this.region);
3660 this.region = null;
3661 }
3662 }
3663
3664 @Test
3665 public void testClearForceSplit() throws IOException {
3666 byte[] qualifier = Bytes.toBytes("qualifier");
3667 Configuration hc = initSplit();
3668 int numRows = 10;
3669 byte[][] families = { fam1, fam3 };
3670
3671
3672 String method = this.getName();
3673 this.region = initHRegion(tableName, method, hc, families);
3674
3675
3676 int startRow = 100;
3677 putData(startRow, numRows, qualifier, families);
3678 int splitRow = startRow + numRows;
3679 byte[] splitRowBytes = Bytes.toBytes("" + splitRow);
3680 putData(splitRow, numRows, qualifier, families);
3681 region.flush(true);
3682
3683 HRegion[] regions = null;
3684 try {
3685
3686 region.forceSplit(splitRowBytes);
3687 assertTrue(region.shouldForceSplit());
3688
3689 assertTrue(Bytes.equals(splitRowBytes, region.checkSplit()));
3690
3691
3692 HStore storeMock = Mockito.mock(HStore.class);
3693 when(storeMock.hasReferences()).thenReturn(true);
3694 when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
3695 when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
3696 when(storeMock.getColumnFamilyName()).thenReturn("cf");
3697 region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock);
3698 assertTrue(region.hasReferences());
3699
3700
3701 regions = splitRegion(region, splitRowBytes);
3702 assertNull(regions);
3703
3704
3705 assertFalse(region.shouldForceSplit());
3706
3707
3708 region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName()));
3709 assertFalse(region.hasReferences());
3710
3711
3712 regions = splitRegion(region, splitRowBytes);
3713
3714
3715 for (int i = 0; i < regions.length; i++) {
3716 regions[i] = HRegion.openHRegion(regions[i], null);
3717 }
3718
3719 assertEquals(2, regions.length);
3720
3721
3722
3723 verifyData(regions[0], startRow, numRows, qualifier, families);
3724 verifyData(regions[1], splitRow, numRows, qualifier, families);
3725
3726 } finally {
3727 HRegion.closeHRegion(this.region);
3728 this.region = null;
3729 }
3730 }
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742 @Test
3743 public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3744 byte[] family = Bytes.toBytes("family");
3745 int numRows = 1000;
3746 int flushAndScanInterval = 10;
3747 int compactInterval = 10 * flushAndScanInterval;
3748
3749 String method = "testFlushCacheWhileScanning";
3750 this.region = initHRegion(tableName, method, CONF, family);
3751 try {
3752 FlushThread flushThread = new FlushThread();
3753 flushThread.start();
3754
3755 Scan scan = new Scan();
3756 scan.addFamily(family);
3757 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3758 new BinaryComparator(Bytes.toBytes(5L))));
3759
3760 int expectedCount = 0;
3761 List<Cell> res = new ArrayList<Cell>();
3762
3763 boolean toggle = true;
3764 for (long i = 0; i < numRows; i++) {
3765 Put put = new Put(Bytes.toBytes(i));
3766 put.setDurability(Durability.SKIP_WAL);
3767 put.add(family, qual1, Bytes.toBytes(i % 10));
3768 region.put(put);
3769
3770 if (i != 0 && i % compactInterval == 0) {
3771
3772 region.compact(true);
3773 }
3774
3775 if (i % 10 == 5L) {
3776 expectedCount++;
3777 }
3778
3779 if (i != 0 && i % flushAndScanInterval == 0) {
3780 res.clear();
3781 InternalScanner scanner = region.getScanner(scan);
3782 if (toggle) {
3783 flushThread.flush();
3784 }
3785 while (scanner.next(res))
3786 ;
3787 if (!toggle) {
3788 flushThread.flush();
3789 }
3790 assertEquals("i=" + i, expectedCount, res.size());
3791 toggle = !toggle;
3792 }
3793 }
3794
3795 flushThread.done();
3796 flushThread.join();
3797 flushThread.checkNoError();
3798 } finally {
3799 HRegion.closeHRegion(this.region);
3800 this.region = null;
3801 }
3802 }
3803
3804 protected class FlushThread extends Thread {
3805 private volatile boolean done;
3806 private Throwable error = null;
3807
3808 FlushThread() {
3809 super("FlushThread");
3810 }
3811
3812 public void done() {
3813 done = true;
3814 synchronized (this) {
3815 interrupt();
3816 }
3817 }
3818
3819 public void checkNoError() {
3820 if (error != null) {
3821 assertNull(error);
3822 }
3823 }
3824
3825 @Override
3826 public void run() {
3827 done = false;
3828 while (!done) {
3829 synchronized (this) {
3830 try {
3831 wait();
3832 } catch (InterruptedException ignored) {
3833 if (done) {
3834 break;
3835 }
3836 }
3837 }
3838 try {
3839 region.flush(true);
3840 } catch (IOException e) {
3841 if (!done) {
3842 LOG.error("Error while flushing cache", e);
3843 error = e;
3844 }
3845 break;
3846 } catch (Throwable t) {
3847 LOG.error("Uncaught exception", t);
3848 throw t;
3849 }
3850 }
3851 }
3852
3853 public void flush() {
3854 synchronized (this) {
3855 notify();
3856 }
3857 }
3858 }
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869 @Test
3870 public void testWritesWhileScanning() throws IOException, InterruptedException {
3871 int testCount = 100;
3872 int numRows = 1;
3873 int numFamilies = 10;
3874 int numQualifiers = 100;
3875 int flushInterval = 7;
3876 int compactInterval = 5 * flushInterval;
3877 byte[][] families = new byte[numFamilies][];
3878 for (int i = 0; i < numFamilies; i++) {
3879 families[i] = Bytes.toBytes("family" + i);
3880 }
3881 byte[][] qualifiers = new byte[numQualifiers][];
3882 for (int i = 0; i < numQualifiers; i++) {
3883 qualifiers[i] = Bytes.toBytes("qual" + i);
3884 }
3885
3886 String method = "testWritesWhileScanning";
3887 this.region = initHRegion(tableName, method, CONF, families);
3888 try {
3889 PutThread putThread = new PutThread(numRows, families, qualifiers);
3890 putThread.start();
3891 putThread.waitForFirstPut();
3892
3893 FlushThread flushThread = new FlushThread();
3894 flushThread.start();
3895
3896 Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3897
3898 int expectedCount = numFamilies * numQualifiers;
3899 List<Cell> res = new ArrayList<Cell>();
3900
3901 long prevTimestamp = 0L;
3902 for (int i = 0; i < testCount; i++) {
3903
3904 if (i != 0 && i % compactInterval == 0) {
3905 region.compact(true);
3906 for (Store store : region.getStores()) {
3907 store.closeAndArchiveCompactedFiles();
3908 }
3909 }
3910
3911 if (i != 0 && i % flushInterval == 0) {
3912 flushThread.flush();
3913 }
3914
3915 boolean previousEmpty = res.isEmpty();
3916 res.clear();
3917 InternalScanner scanner = region.getScanner(scan);
3918 while (scanner.next(res))
3919 ;
3920 if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3921 assertEquals("i=" + i, expectedCount, res.size());
3922 long timestamp = res.get(0).getTimestamp();
3923 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3924 timestamp >= prevTimestamp);
3925 prevTimestamp = timestamp;
3926 }
3927 }
3928
3929 putThread.done();
3930
3931 region.flush(true);
3932
3933 putThread.join();
3934 putThread.checkNoError();
3935
3936 flushThread.done();
3937 flushThread.join();
3938 flushThread.checkNoError();
3939 } finally {
3940 try {
3941 LOG.info("Before close: " + this.region.getMVCC());
3942 HRegion.closeHRegion(this.region);
3943 } catch (DroppedSnapshotException dse) {
3944
3945
3946
3947
3948 }
3949 this.region = null;
3950 }
3951 }
3952
3953 protected class PutThread extends Thread {
3954 private volatile boolean done;
3955 private volatile int numPutsFinished = 0;
3956
3957 private Throwable error = null;
3958 private int numRows;
3959 private byte[][] families;
3960 private byte[][] qualifiers;
3961
3962 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3963 super("PutThread");
3964 this.numRows = numRows;
3965 this.families = families;
3966 this.qualifiers = qualifiers;
3967 }
3968
3969
3970
3971
3972 public void waitForFirstPut() throws InterruptedException {
3973
3974 while (numPutsFinished == 0) {
3975 checkNoError();
3976 Thread.sleep(50);
3977 }
3978 }
3979
3980 public void done() {
3981 done = true;
3982 synchronized (this) {
3983 interrupt();
3984 }
3985 }
3986
3987 public void checkNoError() {
3988 if (error != null) {
3989 assertNull(error);
3990 }
3991 }
3992
3993 @Override
3994 public void run() {
3995 done = false;
3996 while (!done) {
3997 try {
3998 for (int r = 0; r < numRows; r++) {
3999 byte[] row = Bytes.toBytes("row" + r);
4000 Put put = new Put(row);
4001 put.setDurability(Durability.SKIP_WAL);
4002 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
4003 for (byte[] family : families) {
4004 for (byte[] qualifier : qualifiers) {
4005 put.add(family, qualifier, (long) numPutsFinished, value);
4006 }
4007 }
4008 region.put(put);
4009 numPutsFinished++;
4010 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
4011 System.out.println("put iteration = " + numPutsFinished);
4012 Delete delete = new Delete(row, (long) numPutsFinished - 30);
4013 region.delete(delete);
4014 }
4015 numPutsFinished++;
4016 }
4017 } catch (InterruptedIOException e) {
4018
4019 LOG.info("Interrupted", e);
4020 } catch (IOException e) {
4021 LOG.error("Error while putting records", e);
4022 error = e;
4023 break;
4024 }
4025 }
4026
4027 }
4028
4029 }
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040 @Test
4041 public void testWritesWhileGetting() throws Exception {
4042 int testCount = 100;
4043 int numRows = 1;
4044 int numFamilies = 10;
4045 int numQualifiers = 100;
4046 int compactInterval = 100;
4047 byte[][] families = new byte[numFamilies][];
4048 for (int i = 0; i < numFamilies; i++) {
4049 families[i] = Bytes.toBytes("family" + i);
4050 }
4051 byte[][] qualifiers = new byte[numQualifiers][];
4052 for (int i = 0; i < numQualifiers; i++) {
4053 qualifiers[i] = Bytes.toBytes("qual" + i);
4054 }
4055
4056
4057 String method = "testWritesWhileGetting";
4058
4059
4060
4061
4062
4063 Configuration conf = HBaseConfiguration.create(CONF);
4064 conf.setInt("hbase.hstore.compaction.min", 1);
4065 conf.setInt("hbase.hstore.compaction.max", 1000);
4066 this.region = initHRegion(tableName, method, conf, families);
4067 PutThread putThread = null;
4068 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
4069 try {
4070 putThread = new PutThread(numRows, families, qualifiers);
4071 putThread.start();
4072 putThread.waitForFirstPut();
4073
4074
4075 ctx.addThread(new RepeatingTestThread(ctx) {
4076
4077 @Override
4078 public void doAnAction() throws Exception {
4079 region.flush(true);
4080
4081
4082 region.compact(false);
4083 for (Store store : region.getStores()) {
4084 store.closeAndArchiveCompactedFiles();
4085 }
4086 }
4087 });
4088 ctx.startThreads();
4089
4090 Get get = new Get(Bytes.toBytes("row0"));
4091 Result result = null;
4092
4093 int expectedCount = numFamilies * numQualifiers;
4094
4095 long prevTimestamp = 0L;
4096 for (int i = 0; i < testCount; i++) {
4097 LOG.info("testWritesWhileGetting verify turn " + i);
4098 boolean previousEmpty = result == null || result.isEmpty();
4099 result = region.get(get);
4100 if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
4101 assertEquals("i=" + i, expectedCount, result.size());
4102
4103
4104 long timestamp = 0;
4105 for (Cell kv : result.rawCells()) {
4106 if (CellUtil.matchingFamily(kv, families[0])
4107 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
4108 timestamp = kv.getTimestamp();
4109 }
4110 }
4111 assertTrue(timestamp >= prevTimestamp);
4112 prevTimestamp = timestamp;
4113 Cell previousKV = null;
4114
4115 for (Cell kv : result.rawCells()) {
4116 byte[] thisValue = CellUtil.cloneValue(kv);
4117 if (previousKV != null) {
4118 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
4119 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
4120 + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
4121 + "(memStoreTS:" + kv.getMvccVersion() + ")");
4122 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
4123 }
4124 }
4125 previousKV = kv;
4126 }
4127 }
4128 }
4129 } finally {
4130 if (putThread != null)
4131 putThread.done();
4132
4133 region.flush(true);
4134
4135 if (putThread != null) {
4136 putThread.join();
4137 putThread.checkNoError();
4138 }
4139
4140 ctx.stop();
4141 HRegion.closeHRegion(this.region);
4142 this.region = null;
4143 }
4144 }
4145
4146 @Test
4147 public void testHolesInMeta() throws Exception {
4148 byte[] family = Bytes.toBytes("family");
4149 this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
4150 false, family);
4151 try {
4152 byte[] rowNotServed = Bytes.toBytes("a");
4153 Get g = new Get(rowNotServed);
4154 try {
4155 region.get(g);
4156 fail();
4157 } catch (WrongRegionException x) {
4158
4159 }
4160 byte[] row = Bytes.toBytes("y");
4161 g = new Get(row);
4162 region.get(g);
4163 } finally {
4164 HRegion.closeHRegion(this.region);
4165 this.region = null;
4166 }
4167 }
4168
4169 @Test
4170 public void testIndexesScanWithOneDeletedRow() throws IOException {
4171 byte[] family = Bytes.toBytes("family");
4172
4173
4174 String method = "testIndexesScanWithOneDeletedRow";
4175 this.region = initHRegion(tableName, method, CONF, family);
4176 try {
4177 Put put = new Put(Bytes.toBytes(1L));
4178 put.add(family, qual1, 1L, Bytes.toBytes(1L));
4179 region.put(put);
4180
4181 region.flush(true);
4182
4183 Delete delete = new Delete(Bytes.toBytes(1L), 1L);
4184 region.delete(delete);
4185
4186 put = new Put(Bytes.toBytes(2L));
4187 put.add(family, qual1, 2L, Bytes.toBytes(2L));
4188 region.put(put);
4189
4190 Scan idxScan = new Scan();
4191 idxScan.addFamily(family);
4192 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
4193 new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
4194 new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
4195 CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
4196 InternalScanner scanner = region.getScanner(idxScan);
4197 List<Cell> res = new ArrayList<Cell>();
4198
4199 while (scanner.next(res))
4200 ;
4201 assertEquals(1L, res.size());
4202 } finally {
4203 HRegion.closeHRegion(this.region);
4204 this.region = null;
4205 }
4206 }
4207
4208
4209
4210
4211 @Test
4212 public void testBloomFilterSize() throws IOException {
4213 byte[] fam1 = Bytes.toBytes("fam1");
4214 byte[] qf1 = Bytes.toBytes("col");
4215 byte[] val1 = Bytes.toBytes("value1");
4216
4217 HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
4218 .setBloomFilterType(BloomType.ROWCOL);
4219
4220 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
4221 htd.addFamily(hcd);
4222 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
4223 this.region = TEST_UTIL.createLocalHRegion(info, htd);
4224 try {
4225 int num_unique_rows = 10;
4226 int duplicate_multiplier = 2;
4227 int num_storefiles = 4;
4228
4229 int version = 0;
4230 for (int f = 0; f < num_storefiles; f++) {
4231 for (int i = 0; i < duplicate_multiplier; i++) {
4232 for (int j = 0; j < num_unique_rows; j++) {
4233 Put put = new Put(Bytes.toBytes("row" + j));
4234 put.setDurability(Durability.SKIP_WAL);
4235 put.add(fam1, qf1, version++, val1);
4236 region.put(put);
4237 }
4238 }
4239 region.flush(true);
4240 }
4241
4242 HStore store = (HStore) region.getStore(fam1);
4243 Collection<StoreFile> storeFiles = store.getStorefiles();
4244 for (StoreFile storefile : storeFiles) {
4245 StoreFile.Reader reader = storefile.getReader();
4246 reader.loadFileInfo();
4247 reader.loadBloomfilter();
4248 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
4249 assertEquals(num_unique_rows, reader.getFilterEntries());
4250 }
4251
4252 region.compact(true);
4253
4254
4255 storeFiles = store.getStorefiles();
4256 for (StoreFile storefile : storeFiles) {
4257 StoreFile.Reader reader = storefile.getReader();
4258 reader.loadFileInfo();
4259 reader.loadBloomfilter();
4260 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
4261 assertEquals(num_unique_rows, reader.getFilterEntries());
4262 }
4263 } finally {
4264 HRegion.closeHRegion(this.region);
4265 this.region = null;
4266 }
4267 }
4268
4269 @Test
4270 public void testAllColumnsWithBloomFilter() throws IOException {
4271 byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
4272 byte[] FAMILY = Bytes.toBytes("family");
4273
4274
4275 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
4276 .setBloomFilterType(BloomType.ROWCOL);
4277 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
4278 htd.addFamily(hcd);
4279 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
4280 this.region = TEST_UTIL.createLocalHRegion(info, htd);
4281 try {
4282
4283 byte row[] = Bytes.toBytes("row:" + 0);
4284 byte column[] = Bytes.toBytes("column:" + 0);
4285 Put put = new Put(row);
4286 put.setDurability(Durability.SKIP_WAL);
4287 for (long idx = 1; idx <= 4; idx++) {
4288 put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
4289 }
4290 region.put(put);
4291
4292
4293 region.flush(true);
4294
4295
4296 Get get = new Get(row);
4297 get.setMaxVersions();
4298 Cell[] kvs = region.get(get).rawCells();
4299
4300
4301 assertEquals(4, kvs.length);
4302 checkOneCell(kvs[0], FAMILY, 0, 0, 4);
4303 checkOneCell(kvs[1], FAMILY, 0, 0, 3);
4304 checkOneCell(kvs[2], FAMILY, 0, 0, 2);
4305 checkOneCell(kvs[3], FAMILY, 0, 0, 1);
4306 } finally {
4307 HRegion.closeHRegion(this.region);
4308 this.region = null;
4309 }
4310 }
4311
4312
4313
4314
4315
4316
4317 @Test
4318 public void testDeleteRowWithBloomFilter() throws IOException {
4319 byte[] familyName = Bytes.toBytes("familyName");
4320
4321
4322 HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
4323 .setBloomFilterType(BloomType.ROWCOL);
4324
4325 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
4326 htd.addFamily(hcd);
4327 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
4328 this.region = TEST_UTIL.createLocalHRegion(info, htd);
4329 try {
4330
4331 byte row[] = Bytes.toBytes("row1");
4332 byte col[] = Bytes.toBytes("col1");
4333
4334 Put put = new Put(row);
4335 put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
4336 region.put(put);
4337 region.flush(true);
4338
4339 Delete del = new Delete(row);
4340 region.delete(del);
4341 region.flush(true);
4342
4343
4344 Get get = new Get(row);
4345 get.addColumn(familyName, col);
4346
4347 Cell[] keyValues = region.get(get).rawCells();
4348 assertTrue(keyValues.length == 0);
4349 } finally {
4350 HRegion.closeHRegion(this.region);
4351 this.region = null;
4352 }
4353 }
4354
4355 @Test
4356 public void testgetHDFSBlocksDistribution() throws Exception {
4357 HBaseTestingUtility htu = new HBaseTestingUtility();
4358
4359
4360
4361
4362
4363
4364
4365 htu.getConfiguration().setInt("dfs.replication", 2);
4366
4367
4368 MiniHBaseCluster cluster = null;
4369 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
4370 int regionServersCount = 3;
4371
4372 try {
4373 cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
4374 byte[][] families = { fam1, fam2 };
4375 Table ht = htu.createTable(Bytes.toBytes(this.getName()), families);
4376
4377
4378 byte row[] = Bytes.toBytes("row1");
4379 byte col[] = Bytes.toBytes("col1");
4380
4381 Put put = new Put(row);
4382 put.add(fam1, col, 1, Bytes.toBytes("test1"));
4383 put.add(fam2, col, 1, Bytes.toBytes("test2"));
4384 ht.put(put);
4385
4386 HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
4387 .get(0);
4388 firstRegion.flush(true);
4389 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
4390
4391
4392
4393
4394
4395 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
4396 StringBuilder sb = new StringBuilder();
4397 for (String host: blocksDistribution1.getTopHosts()) {
4398 if (sb.length() > 0) sb.append(", ");
4399 sb.append(host);
4400 sb.append("=");
4401 sb.append(blocksDistribution1.getWeight(host));
4402 }
4403
4404 String topHost = blocksDistribution1.getTopHosts().get(0);
4405 long topHostWeight = blocksDistribution1.getWeight(topHost);
4406 String msg = "uniqueBlocksWeight=" + uniqueBlocksWeight1 + ", topHostWeight=" +
4407 topHostWeight + ", topHost=" + topHost + "; " + sb.toString();
4408 LOG.info(msg);
4409 assertTrue(msg, uniqueBlocksWeight1 == topHostWeight);
4410
4411
4412
4413 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
4414 htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
4415 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
4416
4417 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
4418
4419 ht.close();
4420 } finally {
4421 if (cluster != null) {
4422 htu.shutdownMiniCluster();
4423 }
4424 }
4425 }
4426
4427
4428
4429
4430
4431
4432
4433 @Test
4434 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
4435 TableName tableName = TableName.valueOf(name.getMethodName());
4436 HRegionInfo info = null;
4437 try {
4438 FileSystem fs = Mockito.mock(FileSystem.class);
4439 Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
4440 HTableDescriptor htd = new HTableDescriptor(tableName);
4441 htd.addFamily(new HColumnDescriptor("cf"));
4442 info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
4443 HConstants.EMPTY_BYTE_ARRAY, false);
4444 Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
4445 region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
4446
4447 region.initialize();
4448 fail("Region initialization should fail due to IOException");
4449 } catch (IOException io) {
4450 List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
4451 for (MonitoredTask monitoredTask : tasks) {
4452 if (!(monitoredTask instanceof MonitoredRPCHandler)
4453 && monitoredTask.getDescription().contains(region.toString())) {
4454 assertTrue("Region state should be ABORTED.",
4455 monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
4456 break;
4457 }
4458 }
4459 } finally {
4460 HRegion.closeHRegion(region);
4461 }
4462 }
4463
4464
4465
4466
4467
4468 @Test
4469 public void testRegionInfoFileCreation() throws IOException {
4470 Path rootDir = new Path(dir + "testRegionInfoFileCreation");
4471
4472 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
4473 htd.addFamily(new HColumnDescriptor("cf"));
4474
4475 HRegionInfo hri = new HRegionInfo(htd.getTableName());
4476
4477
4478 HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
4479
4480 Path regionDir = region.getRegionFileSystem().getRegionDir();
4481 FileSystem fs = region.getRegionFileSystem().getFileSystem();
4482 HRegion.closeHRegion(region);
4483
4484 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
4485
4486
4487 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4488 fs.exists(regionInfoFile));
4489
4490
4491 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
4492 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
4493 HRegion.closeHRegion(region);
4494
4495
4496 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4497 fs.exists(regionInfoFile));
4498
4499
4500 fs.delete(regionInfoFile, true);
4501 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
4502 fs.exists(regionInfoFile));
4503
4504 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
4505
4506 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
4507 HRegion.closeHRegion(region);
4508
4509
4510 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4511 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
4512 }
4513
4514
4515
4516
4517 private static class Incrementer implements Runnable {
4518 private HRegion region;
4519 private final static byte[] incRow = Bytes.toBytes("incRow");
4520 private final static byte[] family = Bytes.toBytes("family");
4521 private final static byte[] qualifier = Bytes.toBytes("qualifier");
4522 private final static long ONE = 1l;
4523 private int incCounter;
4524
4525 public Incrementer(HRegion region, int incCounter) {
4526 this.region = region;
4527 this.incCounter = incCounter;
4528 }
4529
4530 @Override
4531 public void run() {
4532 int count = 0;
4533 while (count < incCounter) {
4534 Increment inc = new Increment(incRow);
4535 inc.addColumn(family, qualifier, ONE);
4536 count++;
4537 try {
4538 region.increment(inc);
4539 } catch (IOException e) {
4540 e.printStackTrace();
4541 break;
4542 }
4543 }
4544 }
4545 }
4546
4547
4548
4549
4550
4551 @Test
4552 public void testParallelIncrementWithMemStoreFlush() throws Exception {
4553 byte[] family = Incrementer.family;
4554 this.region = initHRegion(tableName, method, CONF, family);
4555 final HRegion region = this.region;
4556 final AtomicBoolean incrementDone = new AtomicBoolean(false);
4557 Runnable flusher = new Runnable() {
4558 @Override
4559 public void run() {
4560 while (!incrementDone.get()) {
4561 try {
4562 region.flush(true);
4563 } catch (Exception e) {
4564 e.printStackTrace();
4565 }
4566 }
4567 }
4568 };
4569
4570
4571 int threadNum = 20;
4572 int incCounter = 100;
4573 long expected = threadNum * incCounter;
4574 Thread[] incrementers = new Thread[threadNum];
4575 Thread flushThread = new Thread(flusher);
4576 for (int i = 0; i < threadNum; i++) {
4577 incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
4578 incrementers[i].start();
4579 }
4580 flushThread.start();
4581 for (int i = 0; i < threadNum; i++) {
4582 incrementers[i].join();
4583 }
4584
4585 incrementDone.set(true);
4586 flushThread.join();
4587
4588 Get get = new Get(Incrementer.incRow);
4589 get.addColumn(Incrementer.family, Incrementer.qualifier);
4590 get.setMaxVersions(1);
4591 Result res = this.region.get(get);
4592 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
4593
4594
4595 assertEquals(kvs.size(), 1);
4596 Cell kv = kvs.get(0);
4597 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
4598 this.region = null;
4599 }
4600
4601
4602
4603
4604 private static class Appender implements Runnable {
4605 private HRegion region;
4606 private final static byte[] appendRow = Bytes.toBytes("appendRow");
4607 private final static byte[] family = Bytes.toBytes("family");
4608 private final static byte[] qualifier = Bytes.toBytes("qualifier");
4609 private final static byte[] CHAR = Bytes.toBytes("a");
4610 private int appendCounter;
4611
4612 public Appender(HRegion region, int appendCounter) {
4613 this.region = region;
4614 this.appendCounter = appendCounter;
4615 }
4616
4617 @Override
4618 public void run() {
4619 int count = 0;
4620 while (count < appendCounter) {
4621 Append app = new Append(appendRow);
4622 app.add(family, qualifier, CHAR);
4623 count++;
4624 try {
4625 region.append(app);
4626 } catch (IOException e) {
4627 e.printStackTrace();
4628 break;
4629 }
4630 }
4631 }
4632 }
4633
4634
4635
4636
4637
4638 @Test
4639 public void testParallelAppendWithMemStoreFlush() throws Exception {
4640 byte[] family = Appender.family;
4641 this.region = initHRegion(tableName, method, CONF, family);
4642 final HRegion region = this.region;
4643 final AtomicBoolean appendDone = new AtomicBoolean(false);
4644 Runnable flusher = new Runnable() {
4645 @Override
4646 public void run() {
4647 while (!appendDone.get()) {
4648 try {
4649 region.flush(true);
4650 } catch (Exception e) {
4651 e.printStackTrace();
4652 }
4653 }
4654 }
4655 };
4656
4657
4658
4659 int threadNum = 20;
4660 int appendCounter = 100;
4661 byte[] expected = new byte[threadNum * appendCounter];
4662 for (int i = 0; i < threadNum * appendCounter; i++) {
4663 System.arraycopy(Appender.CHAR, 0, expected, i, 1);
4664 }
4665 Thread[] appenders = new Thread[threadNum];
4666 Thread flushThread = new Thread(flusher);
4667 for (int i = 0; i < threadNum; i++) {
4668 appenders[i] = new Thread(new Appender(this.region, appendCounter));
4669 appenders[i].start();
4670 }
4671 flushThread.start();
4672 for (int i = 0; i < threadNum; i++) {
4673 appenders[i].join();
4674 }
4675
4676 appendDone.set(true);
4677 flushThread.join();
4678
4679 Get get = new Get(Appender.appendRow);
4680 get.addColumn(Appender.family, Appender.qualifier);
4681 get.setMaxVersions(1);
4682 Result res = this.region.get(get);
4683 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
4684
4685
4686 assertEquals(kvs.size(), 1);
4687 Cell kv = kvs.get(0);
4688 byte[] appendResult = new byte[kv.getValueLength()];
4689 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
4690 assertArrayEquals(expected, appendResult);
4691 this.region = null;
4692 }
4693
4694
4695
4696
4697
4698 @Test
4699 public void testPutWithMemStoreFlush() throws Exception {
4700 byte[] family = Bytes.toBytes("family");
4701 ;
4702 byte[] qualifier = Bytes.toBytes("qualifier");
4703 byte[] row = Bytes.toBytes("putRow");
4704 byte[] value = null;
4705 this.region = initHRegion(tableName, method, CONF, family);
4706 Put put = null;
4707 Get get = null;
4708 List<Cell> kvs = null;
4709 Result res = null;
4710
4711 put = new Put(row);
4712 value = Bytes.toBytes("value0");
4713 put.add(family, qualifier, 1234567l, value);
4714 region.put(put);
4715 get = new Get(row);
4716 get.addColumn(family, qualifier);
4717 get.setMaxVersions();
4718 res = this.region.get(get);
4719 kvs = res.getColumnCells(family, qualifier);
4720 assertEquals(1, kvs.size());
4721 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4722
4723 region.flush(true);
4724 get = new Get(row);
4725 get.addColumn(family, qualifier);
4726 get.setMaxVersions();
4727 res = this.region.get(get);
4728 kvs = res.getColumnCells(family, qualifier);
4729 assertEquals(1, kvs.size());
4730 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4731
4732 put = new Put(row);
4733 value = Bytes.toBytes("value1");
4734 put.add(family, qualifier, 1234567l, value);
4735 region.put(put);
4736 get = new Get(row);
4737 get.addColumn(family, qualifier);
4738 get.setMaxVersions();
4739 res = this.region.get(get);
4740 kvs = res.getColumnCells(family, qualifier);
4741 assertEquals(1, kvs.size());
4742 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4743
4744 region.flush(true);
4745 get = new Get(row);
4746 get.addColumn(family, qualifier);
4747 get.setMaxVersions();
4748 res = this.region.get(get);
4749 kvs = res.getColumnCells(family, qualifier);
4750 assertEquals(1, kvs.size());
4751 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4752 }
4753
4754 @Test
4755 public void testDurability() throws Exception {
4756 String method = "testDurability";
4757
4758
4759
4760
4761
4762 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4763 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4764 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4765
4766 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4767 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4768 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4769
4770 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4771 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4772
4773 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4774 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4775
4776 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4777 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4778 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4779
4780
4781 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4782 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4783 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4784 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4785 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4786 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4787
4788 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4789 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4790 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4791 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4792 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4793 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4794
4795
4796 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, true, false, false);
4797 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, true, false, false);
4798 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, true, false, false);
4799 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, true, false, false);
4800 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, true, false, false);
4801 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4802
4803 }
4804
4805 private void durabilityTest(String method, Durability tableDurability,
4806 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4807 final boolean expectSyncFromLogSyncer) throws Exception {
4808 Configuration conf = HBaseConfiguration.create(CONF);
4809 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4810 TableName tableName = TableName.valueOf(method);
4811 byte[] family = Bytes.toBytes("family");
4812 Path logDir = new Path(new Path(dir + method), "log");
4813 final Configuration walConf = new Configuration(conf);
4814 FSUtils.setRootDir(walConf, logDir);
4815 final WALFactory wals = new WALFactory(walConf, null, UUID.randomUUID().toString());
4816 final WAL wal = spy(wals.getWAL(tableName.getName(), tableName.getNamespace()));
4817 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4818 HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, wal,
4819 new byte[][] { family });
4820
4821 Put put = new Put(Bytes.toBytes("r1"));
4822 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4823 put.setDurability(mutationDurability);
4824 region.put(put);
4825
4826
4827 verify(wal, expectAppend ? times(1) : never())
4828 .append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any(),
4829 (WALEdit)any(), Mockito.anyBoolean());
4830
4831
4832 if (expectSync || expectSyncFromLogSyncer) {
4833 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4834 @Override
4835 public boolean evaluate() throws Exception {
4836 try {
4837 if (expectSync) {
4838 verify(wal, times(1)).sync(anyLong());
4839 } else if (expectSyncFromLogSyncer) {
4840 verify(wal, times(1)).sync();
4841 }
4842 } catch (Throwable ignore) {
4843 }
4844 return true;
4845 }
4846 });
4847 } else {
4848
4849 verify(wal, never()).sync();
4850 }
4851
4852 HRegion.closeHRegion(this.region);
4853 this.region = null;
4854 }
4855
4856 @Test
4857 public void testRegionReplicaSecondary() throws IOException {
4858
4859
4860 Path rootDir = new Path(dir + "testRegionReplicaSecondary");
4861 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
4862
4863 byte[][] families = new byte[][] {
4864 Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")
4865 };
4866 byte[] cq = Bytes.toBytes("cq");
4867 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionReplicaSecondary"));
4868 for (byte[] family : families) {
4869 htd.addFamily(new HColumnDescriptor(family));
4870 }
4871
4872 long time = System.currentTimeMillis();
4873 HRegionInfo primaryHri = new HRegionInfo(htd.getTableName(),
4874 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4875 false, time, 0);
4876 HRegionInfo secondaryHri = new HRegionInfo(htd.getTableName(),
4877 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4878 false, time, 1);
4879
4880 HRegion primaryRegion = null, secondaryRegion = null;
4881
4882 try {
4883 primaryRegion = HRegion.createHRegion(primaryHri,
4884 rootDir, TEST_UTIL.getConfiguration(), htd);
4885
4886
4887 putData(primaryRegion, 0, 1000, cq, families);
4888
4889
4890 primaryRegion.flush(true);
4891
4892
4893 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, htd, null, CONF);
4894
4895 verifyData(secondaryRegion, 0, 1000, cq, families);
4896 } finally {
4897 if (primaryRegion != null) {
4898 HRegion.closeHRegion(primaryRegion);
4899 }
4900 if (secondaryRegion != null) {
4901 HRegion.closeHRegion(secondaryRegion);
4902 }
4903 }
4904 }
4905
4906 @Test
4907 public void testRegionReplicaSecondaryIsReadOnly() throws IOException {
4908
4909
4910 Path rootDir = new Path(dir + "testRegionReplicaSecondary");
4911 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
4912
4913 byte[][] families = new byte[][] {
4914 Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")
4915 };
4916 byte[] cq = Bytes.toBytes("cq");
4917 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionReplicaSecondary"));
4918 for (byte[] family : families) {
4919 htd.addFamily(new HColumnDescriptor(family));
4920 }
4921
4922 long time = System.currentTimeMillis();
4923 HRegionInfo primaryHri = new HRegionInfo(htd.getTableName(),
4924 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4925 false, time, 0);
4926 HRegionInfo secondaryHri = new HRegionInfo(htd.getTableName(),
4927 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4928 false, time, 1);
4929
4930 HRegion primaryRegion = null, secondaryRegion = null;
4931
4932 try {
4933 primaryRegion = HRegion.createHRegion(primaryHri,
4934 rootDir, TEST_UTIL.getConfiguration(), htd);
4935
4936
4937 putData(primaryRegion, 0, 1000, cq, families);
4938
4939
4940 primaryRegion.flush(true);
4941
4942
4943 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, htd, null, CONF);
4944
4945 try {
4946 putData(secondaryRegion, 0, 1000, cq, families);
4947 fail("Should have thrown exception");
4948 } catch (IOException ex) {
4949
4950 }
4951 } finally {
4952 if (primaryRegion != null) {
4953 HRegion.closeHRegion(primaryRegion);
4954 }
4955 if (secondaryRegion != null) {
4956 HRegion.closeHRegion(secondaryRegion);
4957 }
4958 }
4959 }
4960
4961 static WALFactory createWALFactory(Configuration conf, Path rootDir) throws IOException {
4962 Configuration confForWAL = new Configuration(conf);
4963 confForWAL.set(HConstants.HBASE_DIR, rootDir.toString());
4964 return new WALFactory(confForWAL,
4965 Collections.<WALActionsListener>singletonList(new MetricsWAL()),
4966 "hregion-" + RandomStringUtils.randomNumeric(8));
4967 }
4968
4969 @Test
4970 public void testCompactionFromPrimary() throws IOException {
4971 Path rootDir = new Path(dir + "testRegionReplicaSecondary");
4972 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
4973
4974 byte[][] families = new byte[][] {
4975 Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")
4976 };
4977 byte[] cq = Bytes.toBytes("cq");
4978 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionReplicaSecondary"));
4979 for (byte[] family : families) {
4980 htd.addFamily(new HColumnDescriptor(family));
4981 }
4982
4983 long time = System.currentTimeMillis();
4984 HRegionInfo primaryHri = new HRegionInfo(htd.getTableName(),
4985 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4986 false, time, 0);
4987 HRegionInfo secondaryHri = new HRegionInfo(htd.getTableName(),
4988 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4989 false, time, 1);
4990
4991 HRegion primaryRegion = null, secondaryRegion = null;
4992
4993 try {
4994 primaryRegion = HRegion.createHRegion(primaryHri,
4995 rootDir, TEST_UTIL.getConfiguration(), htd);
4996
4997
4998 putData(primaryRegion, 0, 1000, cq, families);
4999
5000
5001 primaryRegion.flush(true);
5002
5003
5004 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, htd, null, CONF);
5005
5006
5007 Collection<StoreFile> storeFiles = primaryRegion.getStore(families[0]).getStorefiles();
5008 primaryRegion.getRegionFileSystem().removeStoreFiles(Bytes.toString(families[0]), storeFiles);
5009 Collection<StoreFileInfo> storeFileInfos = primaryRegion.getRegionFileSystem().getStoreFiles(families[0]);
5010 Assert.assertTrue(storeFileInfos == null || storeFileInfos.size() == 0);
5011
5012 verifyData(secondaryRegion, 0, 1000, cq, families);
5013 } finally {
5014 if (primaryRegion != null) {
5015 HRegion.closeHRegion(primaryRegion);
5016 }
5017 if (secondaryRegion != null) {
5018 HRegion.closeHRegion(secondaryRegion);
5019 }
5020 }
5021 }
5022
5023 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
5024 putData(this.region, startRow, numRows, qf, families);
5025 }
5026
5027 private void putData(HRegion region,
5028 int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
5029 putData(region, Durability.SKIP_WAL, startRow, numRows, qf, families);
5030 }
5031
5032 static void putData(HRegion region, Durability durability,
5033 int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
5034 for (int i = startRow; i < startRow + numRows; i++) {
5035 Put put = new Put(Bytes.toBytes("" + i));
5036 put.setDurability(durability);
5037 for (byte[] family : families) {
5038 put.add(family, qf, null);
5039 }
5040 region.put(put);
5041 }
5042 }
5043
5044 static void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
5045 throws IOException {
5046 for (int i = startRow; i < startRow + numRows; i++) {
5047 byte[] row = Bytes.toBytes("" + i);
5048 Get get = new Get(row);
5049 for (byte[] family : families) {
5050 get.addColumn(family, qf);
5051 }
5052 Result result = newReg.get(get);
5053 Cell[] raw = result.rawCells();
5054 assertEquals(families.length, result.size());
5055 for (int j = 0; j < families.length; j++) {
5056 assertTrue(CellUtil.matchingRow(raw[j], row));
5057 assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
5058 assertTrue(CellUtil.matchingQualifier(raw[j], qf));
5059 }
5060 }
5061 }
5062
5063 static void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
5064
5065 Get get = new Get(k).addFamily(family).setMaxVersions();
5066 Cell[] results = r.get(get).rawCells();
5067 for (int j = 0; j < results.length; j++) {
5068 byte[] tmp = CellUtil.cloneValue(results[j]);
5069
5070 assertTrue(Bytes.equals(k, tmp));
5071 }
5072 }
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085 private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
5086 throws IOException {
5087 byte[][] families = { fs };
5088 Scan scan = new Scan();
5089 for (int i = 0; i < families.length; i++)
5090 scan.addFamily(families[i]);
5091 InternalScanner s = r.getScanner(scan);
5092 try {
5093 List<Cell> curVals = new ArrayList<Cell>();
5094 boolean first = true;
5095 OUTER_LOOP: while (s.next(curVals)) {
5096 for (Cell kv : curVals) {
5097 byte[] val = CellUtil.cloneValue(kv);
5098 byte[] curval = val;
5099 if (first) {
5100 first = false;
5101 assertTrue(Bytes.compareTo(curval, firstValue) == 0);
5102 } else {
5103
5104 break OUTER_LOOP;
5105 }
5106 }
5107 }
5108 } finally {
5109 s.close();
5110 }
5111 }
5112
5113
5114
5115
5116
5117 @Test
5118 public void testFlushResult() throws IOException {
5119 String method = name.getMethodName();
5120 byte[] tableName = Bytes.toBytes(method);
5121 byte[] family = Bytes.toBytes("family");
5122
5123 this.region = initHRegion(tableName, method, family);
5124
5125
5126 HRegion.FlushResult fr = region.flush(true);
5127 assertFalse(fr.isFlushSucceeded());
5128 assertFalse(fr.isCompactionNeeded());
5129
5130
5131 for (int i = 0; i < 2; i++) {
5132 Put put = new Put(tableName).add(family, family, tableName);
5133 region.put(put);
5134 fr = region.flush(true);
5135 assertTrue(fr.isFlushSucceeded());
5136 assertFalse(fr.isCompactionNeeded());
5137 }
5138
5139
5140 for (int i = 0; i < 2; i++) {
5141 Put put = new Put(tableName).add(family, family, tableName);
5142 region.put(put);
5143 fr = region.flush(true);
5144 assertTrue(fr.isFlushSucceeded());
5145 assertTrue(fr.isCompactionNeeded());
5146 }
5147 }
5148
5149 private Configuration initSplit() {
5150
5151 CONF.setInt("hbase.hstore.compactionThreshold", 2);
5152
5153
5154 CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
5155
5156 CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
5157
5158
5159 CONF.setLong("hbase.client.pause", 15 * 1000);
5160
5161
5162
5163 CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
5164 return CONF;
5165 }
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176 public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
5177 byte[]... families) throws IOException {
5178 return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
5179 }
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
5191 byte[]... families) throws IOException {
5192 return initHRegion(tableName, null, null, callingMethod, conf, false, families);
5193 }
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
5206 boolean isReadOnly, byte[]... families) throws IOException {
5207 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
5208 }
5209
5210 public static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
5211 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
5212 throws IOException {
5213 return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
5214 Durability.SYNC_WAL, null, families);
5215 }
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229 public static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
5230 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
5231 WAL wal, byte[]... families) throws IOException {
5232 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf,
5233 isReadOnly, durability, wal, families);
5234 }
5235
5236
5237
5238
5239
5240 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
5241 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
5242 assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
5243 Bytes.toString(CellUtil.cloneRow(kv)));
5244 assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
5245 Bytes.toString(CellUtil.cloneFamily(kv)));
5246 assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
5247 Bytes.toString(CellUtil.cloneQualifier(kv)));
5248 assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
5249 assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
5250 Bytes.toString(CellUtil.cloneValue(kv)));
5251 }
5252
5253 @Test (timeout=60000)
5254 public void testReverseScanner_FromMemStore_SingleCF_Normal()
5255 throws IOException {
5256 byte[] rowC = Bytes.toBytes("rowC");
5257 byte[] rowA = Bytes.toBytes("rowA");
5258 byte[] rowB = Bytes.toBytes("rowB");
5259 byte[] cf = Bytes.toBytes("CF");
5260 byte[][] families = { cf };
5261 byte[] col = Bytes.toBytes("C");
5262 long ts = 1;
5263 String method = this.getName();
5264 this.region = initHRegion(tableName, method, families);
5265 try {
5266 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5267 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
5268 null);
5269 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5270 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5271 Put put = null;
5272 put = new Put(rowC);
5273 put.add(kv1);
5274 put.add(kv11);
5275 region.put(put);
5276 put = new Put(rowA);
5277 put.add(kv2);
5278 region.put(put);
5279 put = new Put(rowB);
5280 put.add(kv3);
5281 region.put(put);
5282
5283 Scan scan = new Scan(rowC);
5284 scan.setMaxVersions(5);
5285 scan.setReversed(true);
5286 InternalScanner scanner = region.getScanner(scan);
5287 List<Cell> currRow = new ArrayList<Cell>();
5288 boolean hasNext = scanner.next(currRow);
5289 assertEquals(2, currRow.size());
5290 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5291 assertTrue(hasNext);
5292 currRow.clear();
5293 hasNext = scanner.next(currRow);
5294 assertEquals(1, currRow.size());
5295 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5296 assertTrue(hasNext);
5297 currRow.clear();
5298 hasNext = scanner.next(currRow);
5299 assertEquals(1, currRow.size());
5300 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
5301 assertFalse(hasNext);
5302 scanner.close();
5303 } finally {
5304 HRegion.closeHRegion(this.region);
5305 this.region = null;
5306 }
5307 }
5308
5309 @Test (timeout=60000)
5310 public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
5311 throws IOException {
5312 byte[] rowC = Bytes.toBytes("rowC");
5313 byte[] rowA = Bytes.toBytes("rowA");
5314 byte[] rowB = Bytes.toBytes("rowB");
5315 byte[] rowD = Bytes.toBytes("rowD");
5316 byte[] cf = Bytes.toBytes("CF");
5317 byte[][] families = { cf };
5318 byte[] col = Bytes.toBytes("C");
5319 long ts = 1;
5320 String method = this.getName();
5321 this.region = initHRegion(tableName, method, families);
5322 try {
5323 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5324 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
5325 null);
5326 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5327 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5328 Put put = null;
5329 put = new Put(rowC);
5330 put.add(kv1);
5331 put.add(kv11);
5332 region.put(put);
5333 put = new Put(rowA);
5334 put.add(kv2);
5335 region.put(put);
5336 put = new Put(rowB);
5337 put.add(kv3);
5338 region.put(put);
5339
5340 Scan scan = new Scan(rowD);
5341 List<Cell> currRow = new ArrayList<Cell>();
5342 scan.setReversed(true);
5343 scan.setMaxVersions(5);
5344 InternalScanner scanner = region.getScanner(scan);
5345 boolean hasNext = scanner.next(currRow);
5346 assertEquals(2, currRow.size());
5347 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5348 assertTrue(hasNext);
5349 currRow.clear();
5350 hasNext = scanner.next(currRow);
5351 assertEquals(1, currRow.size());
5352 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5353 assertTrue(hasNext);
5354 currRow.clear();
5355 hasNext = scanner.next(currRow);
5356 assertEquals(1, currRow.size());
5357 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
5358 assertFalse(hasNext);
5359 scanner.close();
5360 } finally {
5361 HRegion.closeHRegion(this.region);
5362 this.region = null;
5363 }
5364 }
5365
5366 @Test (timeout=60000)
5367 public void testReverseScanner_FromMemStore_SingleCF_FullScan()
5368 throws IOException {
5369 byte[] rowC = Bytes.toBytes("rowC");
5370 byte[] rowA = Bytes.toBytes("rowA");
5371 byte[] rowB = Bytes.toBytes("rowB");
5372 byte[] cf = Bytes.toBytes("CF");
5373 byte[][] families = { cf };
5374 byte[] col = Bytes.toBytes("C");
5375 long ts = 1;
5376 String method = this.getName();
5377 this.region = initHRegion(tableName, method, families);
5378 try {
5379 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5380 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
5381 null);
5382 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5383 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5384 Put put = null;
5385 put = new Put(rowC);
5386 put.add(kv1);
5387 put.add(kv11);
5388 region.put(put);
5389 put = new Put(rowA);
5390 put.add(kv2);
5391 region.put(put);
5392 put = new Put(rowB);
5393 put.add(kv3);
5394 region.put(put);
5395 Scan scan = new Scan();
5396 List<Cell> currRow = new ArrayList<Cell>();
5397 scan.setReversed(true);
5398 InternalScanner scanner = region.getScanner(scan);
5399 boolean hasNext = scanner.next(currRow);
5400 assertEquals(1, currRow.size());
5401 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5402 assertTrue(hasNext);
5403 currRow.clear();
5404 hasNext = scanner.next(currRow);
5405 assertEquals(1, currRow.size());
5406 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5407 assertTrue(hasNext);
5408 currRow.clear();
5409 hasNext = scanner.next(currRow);
5410 assertEquals(1, currRow.size());
5411 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
5412 assertFalse(hasNext);
5413 scanner.close();
5414 } finally {
5415 HRegion.closeHRegion(this.region);
5416 this.region = null;
5417 }
5418 }
5419
5420 @Test (timeout=60000)
5421 public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
5422
5423 byte[] rowA = Bytes.toBytes("rowA");
5424 byte[] rowB = Bytes.toBytes("rowB");
5425 byte[] rowC = Bytes.toBytes("rowC");
5426 byte[] rowD = Bytes.toBytes("rowD");
5427 byte[] rowE = Bytes.toBytes("rowE");
5428 byte[] cf = Bytes.toBytes("CF");
5429 byte[][] families = { cf };
5430 byte[] col1 = Bytes.toBytes("col1");
5431 byte[] col2 = Bytes.toBytes("col2");
5432 long ts = 1;
5433 String method = this.getName();
5434 this.region = initHRegion(tableName, method, families);
5435 try {
5436 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
5437 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
5438 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
5439 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
5440 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
5441 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
5442 Put put = null;
5443 put = new Put(rowA);
5444 put.add(kv1);
5445 region.put(put);
5446 put = new Put(rowB);
5447 put.add(kv2);
5448 region.put(put);
5449 put = new Put(rowC);
5450 put.add(kv3);
5451 region.put(put);
5452 put = new Put(rowD);
5453 put.add(kv4_1);
5454 region.put(put);
5455 put = new Put(rowD);
5456 put.add(kv4_2);
5457 region.put(put);
5458 put = new Put(rowE);
5459 put.add(kv5);
5460 region.put(put);
5461 region.flush(true);
5462 Scan scan = new Scan(rowD, rowA);
5463 scan.addColumn(families[0], col1);
5464 scan.setReversed(true);
5465 List<Cell> currRow = new ArrayList<Cell>();
5466 InternalScanner scanner = region.getScanner(scan);
5467 boolean hasNext = scanner.next(currRow);
5468 assertEquals(1, currRow.size());
5469 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5470 assertTrue(hasNext);
5471 currRow.clear();
5472 hasNext = scanner.next(currRow);
5473 assertEquals(1, currRow.size());
5474 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5475 assertTrue(hasNext);
5476 currRow.clear();
5477 hasNext = scanner.next(currRow);
5478 assertEquals(1, currRow.size());
5479 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5480 assertFalse(hasNext);
5481 scanner.close();
5482
5483 scan = new Scan(rowD, rowA);
5484 scan.addColumn(families[0], col2);
5485 scan.setReversed(true);
5486 currRow.clear();
5487 scanner = region.getScanner(scan);
5488 hasNext = scanner.next(currRow);
5489 assertEquals(1, currRow.size());
5490 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5491 scanner.close();
5492 } finally {
5493 HRegion.closeHRegion(this.region);
5494 this.region = null;
5495 }
5496 }
5497
5498 @Test (timeout=60000)
5499 public void testReverseScanner_smaller_blocksize() throws IOException {
5500
5501 byte[] rowA = Bytes.toBytes("rowA");
5502 byte[] rowB = Bytes.toBytes("rowB");
5503 byte[] rowC = Bytes.toBytes("rowC");
5504 byte[] rowD = Bytes.toBytes("rowD");
5505 byte[] rowE = Bytes.toBytes("rowE");
5506 byte[] cf = Bytes.toBytes("CF");
5507 byte[][] families = { cf };
5508 byte[] col1 = Bytes.toBytes("col1");
5509 byte[] col2 = Bytes.toBytes("col2");
5510 long ts = 1;
5511 String method = this.getName();
5512 HBaseConfiguration config = new HBaseConfiguration();
5513 config.setInt("test.block.size", 1);
5514 this.region = initHRegion(tableName, method, config, families);
5515 try {
5516 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
5517 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
5518 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
5519 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
5520 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
5521 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
5522 Put put = null;
5523 put = new Put(rowA);
5524 put.add(kv1);
5525 region.put(put);
5526 put = new Put(rowB);
5527 put.add(kv2);
5528 region.put(put);
5529 put = new Put(rowC);
5530 put.add(kv3);
5531 region.put(put);
5532 put = new Put(rowD);
5533 put.add(kv4_1);
5534 region.put(put);
5535 put = new Put(rowD);
5536 put.add(kv4_2);
5537 region.put(put);
5538 put = new Put(rowE);
5539 put.add(kv5);
5540 region.put(put);
5541 region.flush(true);
5542 Scan scan = new Scan(rowD, rowA);
5543 scan.addColumn(families[0], col1);
5544 scan.setReversed(true);
5545 List<Cell> currRow = new ArrayList<Cell>();
5546 InternalScanner scanner = region.getScanner(scan);
5547 boolean hasNext = scanner.next(currRow);
5548 assertEquals(1, currRow.size());
5549 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5550 assertTrue(hasNext);
5551 currRow.clear();
5552 hasNext = scanner.next(currRow);
5553 assertEquals(1, currRow.size());
5554 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5555 assertTrue(hasNext);
5556 currRow.clear();
5557 hasNext = scanner.next(currRow);
5558 assertEquals(1, currRow.size());
5559 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5560 assertFalse(hasNext);
5561 scanner.close();
5562
5563 scan = new Scan(rowD, rowA);
5564 scan.addColumn(families[0], col2);
5565 scan.setReversed(true);
5566 currRow.clear();
5567 scanner = region.getScanner(scan);
5568 hasNext = scanner.next(currRow);
5569 assertEquals(1, currRow.size());
5570 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5571 scanner.close();
5572 } finally {
5573 HRegion.closeHRegion(this.region);
5574 this.region = null;
5575 }
5576 }
5577
5578 @Test (timeout=60000)
5579 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
5580 throws IOException {
5581 byte[] row0 = Bytes.toBytes("row0");
5582 byte[] row1 = Bytes.toBytes("row1");
5583 byte[] row2 = Bytes.toBytes("row2");
5584 byte[] row3 = Bytes.toBytes("row3");
5585 byte[] row4 = Bytes.toBytes("row4");
5586 byte[] row5 = Bytes.toBytes("row5");
5587 byte[] cf1 = Bytes.toBytes("CF1");
5588 byte[] cf2 = Bytes.toBytes("CF2");
5589 byte[] cf3 = Bytes.toBytes("CF3");
5590 byte[][] families = { cf1, cf2, cf3 };
5591 byte[] col = Bytes.toBytes("C");
5592 long ts = 1;
5593 String method = this.getName();
5594 HBaseConfiguration conf = new HBaseConfiguration();
5595
5596 conf.setInt("hbase.hstore.compactionThreshold", 10000);
5597 this.region = initHRegion(tableName, method, conf, families);
5598 try {
5599
5600 KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
5601 null);
5602 KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
5603 null);
5604 KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
5605 KeyValue.Type.Put, null);
5606 KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
5607 null);
5608 KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
5609 null);
5610 KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
5611 null);
5612 KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
5613 KeyValue.Type.Put, null);
5614 KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
5615 null);
5616 KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
5617 KeyValue.Type.Put, null);
5618 KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
5619 null);
5620 KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
5621 null);
5622 KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
5623 KeyValue.Type.Put, null);
5624 KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
5625 null);
5626 KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
5627 KeyValue.Type.Put, null);
5628 KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
5629 null);
5630 KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
5631 null);
5632
5633 Put put = null;
5634 put = new Put(row1);
5635 put.add(kv1_2_1);
5636 region.put(put);
5637 put = new Put(row2);
5638 put.add(kv2_4_1);
5639 region.put(put);
5640 put = new Put(row4);
5641 put.add(kv4_5_4);
5642 put.add(kv4_5_5);
5643 region.put(put);
5644 region.flush(true);
5645
5646 put = new Put(row4);
5647 put.add(kv4_5_1);
5648 put.add(kv4_5_3);
5649 region.put(put);
5650 put = new Put(row1);
5651 put.add(kv1_2_2);
5652 region.put(put);
5653 put = new Put(row2);
5654 put.add(kv2_4_4);
5655 region.put(put);
5656 region.flush(true);
5657
5658 put = new Put(row4);
5659 put.add(kv4_5_2);
5660 region.put(put);
5661 put = new Put(row2);
5662 put.add(kv2_4_2);
5663 put.add(kv2_4_3);
5664 region.put(put);
5665 put = new Put(row3);
5666 put.add(kv3_2_2);
5667 region.put(put);
5668 region.flush(true);
5669
5670
5671 put = new Put(row0);
5672 put.add(kv0_1_1);
5673 region.put(put);
5674 put = new Put(row3);
5675 put.add(kv3_2_1);
5676 region.put(put);
5677 put = new Put(row5);
5678 put.add(kv5_2_1);
5679 put.add(kv5_2_2);
5680 region.put(put);
5681
5682 Scan scan = new Scan(row4);
5683 scan.setMaxVersions(5);
5684 scan.setBatch(3);
5685 scan.setReversed(true);
5686 InternalScanner scanner = region.getScanner(scan);
5687 List<Cell> currRow = new ArrayList<Cell>();
5688 boolean hasNext = false;
5689
5690
5691
5692 hasNext = scanner.next(currRow);
5693 assertEquals(3, currRow.size());
5694 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5695 assertTrue(hasNext);
5696 currRow.clear();
5697 hasNext = scanner.next(currRow);
5698 assertEquals(2, currRow.size());
5699 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5700 assertTrue(hasNext);
5701
5702 currRow.clear();
5703 hasNext = scanner.next(currRow);
5704 assertEquals(2, currRow.size());
5705 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5706 assertTrue(hasNext);
5707
5708
5709 currRow.clear();
5710 hasNext = scanner.next(currRow);
5711 assertEquals(3, currRow.size());
5712 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5713 assertTrue(hasNext);
5714 currRow.clear();
5715 hasNext = scanner.next(currRow);
5716 assertEquals(1, currRow.size());
5717 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5718 assertTrue(hasNext);
5719
5720 currRow.clear();
5721 hasNext = scanner.next(currRow);
5722 assertEquals(2, currRow.size());
5723 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5724 assertTrue(hasNext);
5725
5726 currRow.clear();
5727 hasNext = scanner.next(currRow);
5728 assertEquals(1, currRow.size());
5729 assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
5730 assertFalse(hasNext);
5731
5732 scanner.close();
5733 } finally {
5734 HRegion.closeHRegion(this.region);
5735 this.region = null;
5736 }
5737 }
5738
5739 @Test (timeout=60000)
5740 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
5741 throws IOException {
5742 byte[] row1 = Bytes.toBytes("row1");
5743 byte[] row2 = Bytes.toBytes("row2");
5744 byte[] row3 = Bytes.toBytes("row3");
5745 byte[] row4 = Bytes.toBytes("row4");
5746 byte[] cf1 = Bytes.toBytes("CF1");
5747 byte[] cf2 = Bytes.toBytes("CF2");
5748 byte[] cf3 = Bytes.toBytes("CF3");
5749 byte[] cf4 = Bytes.toBytes("CF4");
5750 byte[][] families = { cf1, cf2, cf3, cf4 };
5751 byte[] col = Bytes.toBytes("C");
5752 long ts = 1;
5753 String method = this.getName();
5754 HBaseConfiguration conf = new HBaseConfiguration();
5755
5756 conf.setInt("hbase.hstore.compactionThreshold", 10000);
5757 this.region = initHRegion(tableName, method, conf, families);
5758 try {
5759 KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
5760 KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
5761 KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
5762 KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
5763
5764 Put put = new Put(row1);
5765 put.add(kv1);
5766 region.put(put);
5767 region.flush(true);
5768
5769 put = new Put(row2);
5770 put.add(kv2);
5771 region.put(put);
5772 region.flush(true);
5773
5774 put = new Put(row3);
5775 put.add(kv3);
5776 region.put(put);
5777 region.flush(true);
5778
5779 put = new Put(row4);
5780 put.add(kv4);
5781 region.put(put);
5782
5783 Scan scan = new Scan(row4);
5784 scan.setReversed(true);
5785 scan.setBatch(10);
5786 InternalScanner scanner = region.getScanner(scan);
5787 List<Cell> currRow = new ArrayList<Cell>();
5788 boolean hasNext = scanner.next(currRow);
5789 assertEquals(1, currRow.size());
5790 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5791 assertTrue(hasNext);
5792 currRow.clear();
5793 hasNext = scanner.next(currRow);
5794 assertEquals(1, currRow.size());
5795 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5796 assertTrue(hasNext);
5797 currRow.clear();
5798 hasNext = scanner.next(currRow);
5799 assertEquals(1, currRow.size());
5800 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5801 assertTrue(hasNext);
5802 currRow.clear();
5803 hasNext = scanner.next(currRow);
5804 assertEquals(1, currRow.size());
5805 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5806 assertFalse(hasNext);
5807 } finally {
5808 HRegion.closeHRegion(this.region);
5809 this.region = null;
5810 }
5811 }
5812
5813
5814
5815
5816 @Test (timeout = 60000)
5817 public void testReverseScanner_StackOverflow() throws IOException {
5818 byte[] cf1 = Bytes.toBytes("CF1");
5819 byte[][] families = {cf1};
5820 byte[] col = Bytes.toBytes("C");
5821 String method = this.getName();
5822 HBaseConfiguration conf = new HBaseConfiguration();
5823 this.region = initHRegion(tableName, method, conf, families);
5824 try {
5825
5826 Put put = new Put(Bytes.toBytes("19998"));
5827 put.add(cf1, col, Bytes.toBytes("val"));
5828 region.put(put);
5829 region.flushcache(true, true);
5830 Put put2 = new Put(Bytes.toBytes("19997"));
5831 put2.add(cf1, col, Bytes.toBytes("val"));
5832 region.put(put2);
5833
5834 Scan scan = new Scan(Bytes.toBytes("19998"));
5835 scan.setReversed(true);
5836 InternalScanner scanner = region.getScanner(scan);
5837
5838
5839
5840 for (int i = 10000; i < 20000; i++) {
5841 Put p = new Put(Bytes.toBytes("" + i));
5842 p.add(cf1, col, Bytes.toBytes("" + i));
5843 region.put(p);
5844 }
5845 region.flushcache(true, true);
5846
5847
5848
5849 for (int i = 10000; i < 20000; i++) {
5850 Put p = new Put(Bytes.toBytes("" + i));
5851 p.add(cf1, col, Bytes.toBytes("" + i));
5852 region.put(p);
5853 }
5854
5855 List<Cell> currRow = new ArrayList<>();
5856 boolean hasNext;
5857 do {
5858 hasNext = scanner.next(currRow);
5859 } while (hasNext);
5860 assertEquals(2, currRow.size());
5861 assertArrayEquals(Bytes.toBytes("19998"), currRow.get(0).getRow());
5862 assertArrayEquals(Bytes.toBytes("19997"), currRow.get(1).getRow());
5863 } finally {
5864 HBaseTestingUtility.closeRegionAndWAL(this.region);
5865 this.region = null;
5866 }
5867 }
5868
5869 @Test (timeout=60000)
5870 public void testSplitRegionWithReverseScan() throws IOException {
5871 byte [] tableName = Bytes.toBytes("testSplitRegionWithReverseScan");
5872 byte [] qualifier = Bytes.toBytes("qualifier");
5873 Configuration hc = initSplit();
5874 int numRows = 3;
5875 byte [][] families = {fam1};
5876
5877
5878 String method = this.getName();
5879 this.region = initHRegion(tableName, method, hc, families);
5880
5881
5882 int startRow = 100;
5883 putData(startRow, numRows, qualifier, families);
5884 int splitRow = startRow + numRows;
5885 putData(splitRow, numRows, qualifier, families);
5886 region.flush(true);
5887
5888 HRegion [] regions = null;
5889 try {
5890 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
5891
5892 for (int i = 0; i < regions.length; i++) {
5893 regions[i] = HRegion.openHRegion(regions[i], null);
5894 }
5895
5896 assertEquals(2, regions.length);
5897
5898
5899
5900 verifyData(regions[0], startRow, numRows, qualifier, families);
5901 verifyData(regions[1], splitRow, numRows, qualifier, families);
5902
5903
5904 Scan scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 10 * numRows)));
5905 scan.setReversed(true);
5906 InternalScanner scanner = regions[1].getScanner(scan);
5907 List<Cell> currRow = new ArrayList<Cell>();
5908 boolean more = false;
5909 int verify = startRow + 2 * numRows - 1;
5910 do {
5911 more = scanner.next(currRow);
5912 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5913 verify--;
5914 currRow.clear();
5915 } while(more);
5916 assertEquals(verify, startRow + numRows - 1);
5917 scanner.close();
5918
5919 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 2 * numRows - 1)));
5920 scan.setReversed(true);
5921 scanner = regions[1].getScanner(scan);
5922 verify = startRow + 2 * numRows - 1;
5923 do {
5924 more = scanner.next(currRow);
5925 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5926 verify--;
5927 currRow.clear();
5928 } while(more);
5929 assertEquals(verify, startRow + numRows - 1);
5930 scanner.close();
5931
5932 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows)));
5933 scan.setReversed(true);
5934 scanner = regions[0].getScanner(scan);
5935 verify = startRow + numRows - 1;
5936 do {
5937 more = scanner.next(currRow);
5938 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5939 verify--;
5940 currRow.clear();
5941 } while(more);
5942 assertEquals(verify, 99);
5943 scanner.close();
5944
5945 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows - 1)));
5946 scan.setReversed(true);
5947 scanner = regions[0].getScanner(scan);
5948 verify = startRow + numRows - 1;
5949 do {
5950 more = scanner.next(currRow);
5951 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5952 verify--;
5953 currRow.clear();
5954 } while(more);
5955 assertEquals(verify, startRow - 1);
5956 scanner.close();
5957 } finally {
5958 HRegion.closeHRegion(this.region);
5959 this.region = null;
5960 }
5961 }
5962
5963 @Test
5964 public void testWriteRequestsCounter() throws IOException {
5965 byte[] fam = Bytes.toBytes("info");
5966 byte[][] families = { fam };
5967 this.region = initHRegion(tableName, method, CONF, families);
5968
5969 Assert.assertEquals(0L, region.getWriteRequestsCount());
5970
5971 Put put = new Put(row);
5972 put.add(fam, fam, fam);
5973
5974 Assert.assertEquals(0L, region.getWriteRequestsCount());
5975 region.put(put);
5976 Assert.assertEquals(1L, region.getWriteRequestsCount());
5977 region.put(put);
5978 Assert.assertEquals(2L, region.getWriteRequestsCount());
5979 region.put(put);
5980 Assert.assertEquals(3L, region.getWriteRequestsCount());
5981
5982 region.delete(new Delete(row));
5983 Assert.assertEquals(4L, region.getWriteRequestsCount());
5984
5985 HRegion.closeHRegion(this.region);
5986 this.region = null;
5987 }
5988
5989 @Test
5990 public void testOpenRegionWrittenToWAL() throws Exception {
5991 final ServerName serverName = ServerName.valueOf("testOpenRegionWrittenToWAL", 100, 42);
5992 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
5993
5994 HTableDescriptor htd
5995 = new HTableDescriptor(TableName.valueOf("testOpenRegionWrittenToWAL"));
5996 htd.addFamily(new HColumnDescriptor(fam1));
5997 htd.addFamily(new HColumnDescriptor(fam2));
5998
5999 HRegionInfo hri = new HRegionInfo(htd.getTableName(),
6000 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY);
6001
6002
6003 HRegion region =
6004 HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL
6005 .getConfiguration(), htd);
6006 assertNotNull(region);
6007
6008
6009 region.put(new Put(Bytes.toBytes("a")).add(fam1, fam1, fam1));
6010 region.flush(true);
6011 region.close();
6012
6013 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
6014
6015
6016 WAL wal = mockWAL();
6017 when(rss.getWAL((HRegionInfo) any())).thenReturn(wal);
6018
6019 try {
6020 region = HRegion.openHRegion(hri, htd, rss.getWAL(hri),
6021 TEST_UTIL.getConfiguration(), rss, null);
6022
6023 verify(wal, times(1)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any()
6024 , editCaptor.capture(), anyBoolean());
6025
6026 WALEdit edit = editCaptor.getValue();
6027 assertNotNull(edit);
6028 assertNotNull(edit.getCells());
6029 assertEquals(1, edit.getCells().size());
6030 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
6031 assertNotNull(desc);
6032
6033 LOG.info("RegionEventDescriptor from WAL: " + desc);
6034
6035 assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType());
6036 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName()));
6037 assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(),
6038 hri.getEncodedNameAsBytes()));
6039 assertTrue(desc.getLogSequenceNumber() > 0);
6040 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
6041 assertEquals(2, desc.getStoresCount());
6042
6043 StoreDescriptor store = desc.getStores(0);
6044 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
6045 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
6046 assertEquals(1, store.getStoreFileCount());
6047 assertFalse(store.getStoreFile(0).contains("/"));
6048
6049 store = desc.getStores(1);
6050 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
6051 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
6052 assertEquals(0, store.getStoreFileCount());
6053
6054 } finally {
6055 HRegion.closeHRegion(region);
6056 }
6057 }
6058
6059
6060 static class HRegionWithSeqId extends HRegion {
6061 public HRegionWithSeqId(final Path tableDir, final WAL wal, final FileSystem fs,
6062 final Configuration confParam, final HRegionInfo regionInfo,
6063 final HTableDescriptor htd, final RegionServerServices rsServices) {
6064 super(tableDir, wal, fs, confParam, regionInfo, htd, rsServices);
6065 }
6066 @Override
6067 protected long getNextSequenceId(WAL wal) throws IOException {
6068 return 42;
6069 }
6070 }
6071
6072 @Test
6073 public void testFlushedFileWithNoTags() throws Exception {
6074 String method = "testFlushedFileWithNoTags";
6075 HTableDescriptor htd = new HTableDescriptor(tableName);
6076 htd.addFamily(new HColumnDescriptor(fam1));
6077 region = initHRegion(Bytes.toBytes(method), method, TEST_UTIL.getConfiguration(), fam1);
6078 Put put = new Put(Bytes.toBytes("a-b-0-0"));
6079 put.addColumn(fam1, qual1, Bytes.toBytes("c1-value"));
6080 region.put(put);
6081 region.flush(true);
6082 Store store = region.getStore(fam1);
6083 Collection<StoreFile> storefiles = store.getStorefiles();
6084 for (StoreFile sf : storefiles) {
6085 assertFalse("Tags should not be present "
6086 ,sf.getReader().getHFileReader().getFileContext().isIncludesTags());
6087 }
6088 }
6089
6090 @Test
6091 public void testOpenRegionWrittenToWALForLogReplay() throws Exception {
6092
6093 final ServerName serverName = ServerName.valueOf("testOpenRegionWrittenToWALForLogReplay",
6094 100, 42);
6095 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
6096
6097 HTableDescriptor htd
6098 = new HTableDescriptor(TableName.valueOf("testOpenRegionWrittenToWALForLogReplay"));
6099 htd.addFamily(new HColumnDescriptor(fam1));
6100 htd.addFamily(new HColumnDescriptor(fam2));
6101
6102 HRegionInfo hri = new HRegionInfo(htd.getTableName(),
6103 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY);
6104
6105
6106 HRegion region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL
6107 .getConfiguration(), htd);
6108 assertNotNull(region);
6109
6110
6111 region.put(new Put(Bytes.toBytes("a")).add(fam1, fam1, fam1));
6112 region.flush(true);
6113 HRegion.closeHRegion(region);
6114
6115 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
6116
6117
6118 WAL wal = mockWAL();
6119 when(rss.getWAL((HRegionInfo) any())).thenReturn(wal);
6120
6121
6122 HashMap<String, Region> recoveringRegions = Maps.newHashMap();
6123 recoveringRegions.put(region.getRegionInfo().getEncodedName(), null);
6124 when(rss.getRecoveringRegions()).thenReturn(recoveringRegions);
6125
6126 try {
6127 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
6128 conf.set(HConstants.REGION_IMPL, HRegionWithSeqId.class.getName());
6129 region = HRegion.openHRegion(hri, htd, rss.getWAL(hri),
6130 conf, rss, null);
6131
6132
6133
6134 verify(wal, times(0)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any()
6135 , editCaptor.capture(), anyBoolean());
6136
6137
6138 new FinishRegionRecoveringHandler(rss, region.getRegionInfo().getEncodedName(), "/foo")
6139 .prepare().process();
6140
6141
6142 verify(wal, times(1)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any()
6143 , editCaptor.capture(), anyBoolean());
6144
6145 WALEdit edit = editCaptor.getValue();
6146 assertNotNull(edit);
6147 assertNotNull(edit.getCells());
6148 assertEquals(1, edit.getCells().size());
6149 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
6150 assertNotNull(desc);
6151
6152 LOG.info("RegionEventDescriptor from WAL: " + desc);
6153
6154 assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType());
6155 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName()));
6156 assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(),
6157 hri.getEncodedNameAsBytes()));
6158 assertTrue(desc.getLogSequenceNumber() > 0);
6159 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
6160 assertEquals(2, desc.getStoresCount());
6161
6162 StoreDescriptor store = desc.getStores(0);
6163 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
6164 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
6165 assertEquals(1, store.getStoreFileCount());
6166 assertFalse(store.getStoreFile(0).contains("/"));
6167
6168 store = desc.getStores(1);
6169 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
6170 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
6171 assertEquals(0, store.getStoreFileCount());
6172
6173 } finally {
6174 HRegion.closeHRegion(region);
6175 }
6176 }
6177
6178
6179
6180
6181
6182
6183
6184 private WAL mockWAL() throws IOException {
6185 WAL wal = mock(WAL.class);
6186 Mockito.when(wal.append((HTableDescriptor)Mockito.any(), (HRegionInfo)Mockito.any(),
6187 (WALKey)Mockito.any(), (WALEdit)Mockito.any(), Mockito.anyBoolean())).
6188 thenAnswer(new Answer<Long>() {
6189 @Override
6190 public Long answer(InvocationOnMock invocation) throws Throwable {
6191 WALKey key = invocation.getArgumentAt(2, WALKey.class);
6192 MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin();
6193 key.setWriteEntry(we);
6194 return 1L;
6195 }
6196
6197 });
6198 return wal;
6199 }
6200
6201 @Test
6202 public void testCloseRegionWrittenToWAL() throws Exception {
6203 final ServerName serverName = ServerName.valueOf("testCloseRegionWrittenToWAL", 100, 42);
6204 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
6205
6206 HTableDescriptor htd
6207 = new HTableDescriptor(TableName.valueOf("testOpenRegionWrittenToWAL"));
6208 htd.addFamily(new HColumnDescriptor(fam1));
6209 htd.addFamily(new HColumnDescriptor(fam2));
6210
6211 final HRegionInfo hri = new HRegionInfo(htd.getTableName(),
6212 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY);
6213
6214 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
6215
6216
6217 WAL wal = mockWAL();
6218 when(rss.getWAL((HRegionInfo) any())).thenReturn(wal);
6219
6220
6221
6222 region = HRegion.openHRegion(hri, htd, rss.getWAL(hri),
6223 TEST_UTIL.getConfiguration(), rss, null);
6224
6225
6226 region.close(false);
6227
6228
6229 verify(wal, times(2)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any(),
6230 editCaptor.capture(), anyBoolean());
6231
6232 WALEdit edit = editCaptor.getAllValues().get(1);
6233 assertNotNull(edit);
6234 assertNotNull(edit.getCells());
6235 assertEquals(1, edit.getCells().size());
6236 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
6237 assertNotNull(desc);
6238
6239 LOG.info("RegionEventDescriptor from WAL: " + desc);
6240
6241 assertEquals(RegionEventDescriptor.EventType.REGION_CLOSE, desc.getEventType());
6242 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName()));
6243 assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(),
6244 hri.getEncodedNameAsBytes()));
6245 assertTrue(desc.getLogSequenceNumber() > 0);
6246 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
6247 assertEquals(2, desc.getStoresCount());
6248
6249 StoreDescriptor store = desc.getStores(0);
6250 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
6251 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
6252 assertEquals(0, store.getStoreFileCount());
6253
6254 store = desc.getStores(1);
6255 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
6256 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
6257 assertEquals(0, store.getStoreFileCount());
6258 }
6259
6260
6261
6262
6263 @Test (timeout=24000)
6264 public void testRegionTooBusy() throws IOException {
6265 String method = "testRegionTooBusy";
6266 byte[] tableName = Bytes.toBytes(method);
6267 byte[] family = Bytes.toBytes("family");
6268 long defaultBusyWaitDuration = CONF.getLong("hbase.busy.wait.duration",
6269 HRegion.DEFAULT_BUSY_WAIT_DURATION);
6270 CONF.setLong("hbase.busy.wait.duration", 1000);
6271 region = initHRegion(tableName, method, CONF, family);
6272 final AtomicBoolean stopped = new AtomicBoolean(true);
6273 Thread t = new Thread(new Runnable() {
6274 @Override
6275 public void run() {
6276 try {
6277 region.lock.writeLock().lock();
6278 stopped.set(false);
6279 while (!stopped.get()) {
6280 Thread.sleep(100);
6281 }
6282 } catch (InterruptedException ie) {
6283 } finally {
6284 region.lock.writeLock().unlock();
6285 }
6286 }
6287 });
6288 t.start();
6289 Get get = new Get(row);
6290 try {
6291 while (stopped.get()) {
6292 Thread.sleep(100);
6293 }
6294 region.get(get);
6295 fail("Should throw RegionTooBusyException");
6296 } catch (InterruptedException ie) {
6297 fail("test interrupted");
6298 } catch (RegionTooBusyException e) {
6299
6300 } finally {
6301 stopped.set(true);
6302 try {
6303 t.join();
6304 } catch (Throwable e) {
6305 }
6306
6307 HRegion.closeHRegion(region);
6308 region = null;
6309 CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration);
6310 }
6311 }
6312
6313 @Test
6314 public void testCellTTLs() throws IOException {
6315 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
6316 EnvironmentEdgeManager.injectEdge(edge);
6317
6318 final byte[] row = Bytes.toBytes("testRow");
6319 final byte[] q1 = Bytes.toBytes("q1");
6320 final byte[] q2 = Bytes.toBytes("q2");
6321 final byte[] q3 = Bytes.toBytes("q3");
6322 final byte[] q4 = Bytes.toBytes("q4");
6323
6324 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCellTTLs"));
6325 HColumnDescriptor hcd = new HColumnDescriptor(fam1);
6326 hcd.setTimeToLive(10);
6327 htd.addFamily(hcd);
6328
6329 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
6330 conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS);
6331
6332 HRegion region = HRegion.createHRegion(new HRegionInfo(htd.getTableName(),
6333 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY),
6334 TEST_UTIL.getDataTestDir(), conf, htd);
6335 assertNotNull(region);
6336 try {
6337 long now = EnvironmentEdgeManager.currentTime();
6338
6339 region.put(new Put(row).add(new KeyValue(row, fam1, q1, now,
6340 HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
6341
6342 new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
6343
6344 region.put(new Put(row).add(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY));
6345
6346 region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1,
6347 HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
6348
6349 new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
6350
6351 region.put(new Put(row).add(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY));
6352
6353
6354 region.flush(true);
6355
6356
6357 Result r = region.get(new Get(row));
6358 assertNotNull(r.getValue(fam1, q1));
6359 assertNotNull(r.getValue(fam1, q2));
6360 assertNotNull(r.getValue(fam1, q3));
6361 assertNotNull(r.getValue(fam1, q4));
6362
6363
6364 edge.incrementTime(5000);
6365
6366 r = region.get(new Get(row));
6367 assertNull(r.getValue(fam1, q1));
6368 assertNotNull(r.getValue(fam1, q2));
6369 assertNotNull(r.getValue(fam1, q3));
6370 assertNotNull(r.getValue(fam1, q4));
6371
6372
6373 edge.incrementTime(5000);
6374
6375 r = region.get(new Get(row));
6376 assertNull(r.getValue(fam1, q1));
6377 assertNull(r.getValue(fam1, q2));
6378 assertNotNull(r.getValue(fam1, q3));
6379 assertNotNull(r.getValue(fam1, q4));
6380
6381
6382 edge.incrementTime(5000);
6383
6384 r = region.get(new Get(row));
6385 assertNull(r.getValue(fam1, q1));
6386 assertNull(r.getValue(fam1, q2));
6387 assertNull(r.getValue(fam1, q3));
6388 assertNotNull(r.getValue(fam1, q4));
6389
6390
6391 edge.incrementTime(10000);
6392
6393 r = region.get(new Get(row));
6394 assertNull(r.getValue(fam1, q1));
6395 assertNull(r.getValue(fam1, q2));
6396 assertNull(r.getValue(fam1, q3));
6397 assertNull(r.getValue(fam1, q4));
6398
6399
6400
6401
6402 region.put(new Put(row).add(fam1, q1, Bytes.toBytes(1L)));
6403 r = region.get(new Get(row));
6404 byte[] val = r.getValue(fam1, q1);
6405 assertNotNull(val);
6406 assertEquals(Bytes.toLong(val), 1L);
6407
6408
6409 Increment incr = new Increment(row).addColumn(fam1, q1, 1L);
6410 incr.setTTL(5000);
6411 region.increment(incr);
6412
6413
6414 r = region.get(new Get(row));
6415 val = r.getValue(fam1, q1);
6416 assertNotNull(val);
6417 assertEquals(Bytes.toLong(val), 2L);
6418
6419
6420 edge.incrementTime(5000);
6421
6422
6423 r = region.get(new Get(row));
6424 val = r.getValue(fam1, q1);
6425 assertNotNull(val);
6426 assertEquals(Bytes.toLong(val), 1L);
6427
6428
6429 edge.incrementTime(5000);
6430
6431
6432 r = region.get(new Get(row));
6433 assertNull(r.getValue(fam1, q1));
6434
6435 } finally {
6436 HRegion.closeHRegion(region);
6437 }
6438 }
6439
6440 @Test
6441 public void testIncrementTimestampsAreMonotonic() throws IOException {
6442 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
6443 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
6444 EnvironmentEdgeManager.injectEdge(edge);
6445
6446 edge.setValue(10);
6447 Increment inc = new Increment(row);
6448 inc.setDurability(Durability.SKIP_WAL);
6449 inc.addColumn(fam1, qual1, 1L);
6450 region.increment(inc);
6451
6452 Result result = region.get(new Get(row));
6453 Cell c = result.getColumnLatestCell(fam1, qual1);
6454 assertNotNull(c);
6455 assertEquals(c.getTimestamp(), 10L);
6456
6457 edge.setValue(1);
6458 region.increment(inc);
6459 result = region.get(new Get(row));
6460 c = result.getColumnLatestCell(fam1, qual1);
6461 assertEquals(c.getTimestamp(), 11L);
6462 assertEquals(Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength()), 2L);
6463 }
6464
6465 @Test
6466 public void testAppendTimestampsAreMonotonic() throws IOException {
6467 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
6468 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
6469 EnvironmentEdgeManager.injectEdge(edge);
6470
6471 edge.setValue(10);
6472 Append a = new Append(row);
6473 a.setDurability(Durability.SKIP_WAL);
6474 a.add(fam1, qual1, qual1);
6475 region.append(a);
6476
6477 Result result = region.get(new Get(row));
6478 Cell c = result.getColumnLatestCell(fam1, qual1);
6479 assertNotNull(c);
6480 assertEquals(c.getTimestamp(), 10L);
6481
6482 edge.setValue(1);
6483 region.append(a);
6484 result = region.get(new Get(row));
6485 c = result.getColumnLatestCell(fam1, qual1);
6486 assertEquals(c.getTimestamp(), 11L);
6487
6488 byte[] expected = new byte[qual1.length*2];
6489 System.arraycopy(qual1, 0, expected, 0, qual1.length);
6490 System.arraycopy(qual1, 0, expected, qual1.length, qual1.length);
6491
6492 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
6493 expected, 0, expected.length));
6494 }
6495
6496 @Test
6497 public void testCheckAndMutateTimestampsAreMonotonic() throws IOException {
6498 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
6499 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
6500 EnvironmentEdgeManager.injectEdge(edge);
6501
6502 edge.setValue(10);
6503 Put p = new Put(row);
6504 p.setDurability(Durability.SKIP_WAL);
6505 p.addColumn(fam1, qual1, qual1);
6506 region.put(p);
6507
6508 Result result = region.get(new Get(row));
6509 Cell c = result.getColumnLatestCell(fam1, qual1);
6510 assertNotNull(c);
6511 assertEquals(c.getTimestamp(), 10L);
6512
6513 edge.setValue(1);
6514 p = new Put(row);
6515 p.setDurability(Durability.SKIP_WAL);
6516 p.addColumn(fam1, qual1, qual2);
6517 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1), p, false);
6518 result = region.get(new Get(row));
6519 c = result.getColumnLatestCell(fam1, qual1);
6520 assertEquals(c.getTimestamp(), 10L);
6521
6522 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
6523 qual2, 0, qual2.length));
6524 }
6525
6526 @Test(timeout = 60000)
6527 public void testBatchMutateWithWrongRegionException() throws Exception {
6528 final byte[] a = Bytes.toBytes("a");
6529 final byte[] b = Bytes.toBytes("b");
6530 final byte[] c = Bytes.toBytes("c");
6531
6532 int prevLockTimeout = CONF.getInt("hbase.rowlock.wait.duration", 30000);
6533 CONF.setInt("hbase.rowlock.wait.duration", 1000);
6534 final HRegion region = initHRegion(tableName, a, c, name.getMethodName(), CONF, false, fam1);
6535
6536 Mutation[] mutations = new Mutation[] {
6537 new Put(a).addImmutable(fam1, null, null),
6538 new Put(c).addImmutable(fam1, null, null),
6539 new Put(b).addImmutable(fam1, null, null),
6540 };
6541
6542 OperationStatus[] status = region.batchMutate(mutations);
6543 assertEquals(status[0].getOperationStatusCode(), OperationStatusCode.SUCCESS);
6544 assertEquals(status[1].getOperationStatusCode(), OperationStatusCode.SANITY_CHECK_FAILURE);
6545 assertEquals(status[2].getOperationStatusCode(), OperationStatusCode.SUCCESS);
6546
6547
6548
6549 final CountDownLatch obtainedRowLock = new CountDownLatch(1);
6550 ExecutorService exec = Executors.newFixedThreadPool(2);
6551 Future<Void> f1 = exec.submit(new Callable<Void>() {
6552 @Override
6553 public Void call() throws Exception {
6554 LOG.info("Acquiring row lock");
6555 RowLock rl = region.getRowLock(b);
6556 obtainedRowLock.countDown();
6557 LOG.info("Waiting for 5 seconds before releasing lock");
6558 Threads.sleep(5000);
6559 LOG.info("Releasing row lock");
6560 rl.release();
6561 return null;
6562 }
6563 });
6564 obtainedRowLock.await(30, TimeUnit.SECONDS);
6565
6566 Future<Void> f2 = exec.submit(new Callable<Void>() {
6567 @Override
6568 public Void call() throws Exception {
6569 Mutation[] mutations = new Mutation[] {
6570 new Put(a).addImmutable(fam1, null, null),
6571 new Put(b).addImmutable(fam1, null, null),
6572 };
6573
6574
6575 OperationStatus[] status = region.batchMutate(mutations);
6576 assertEquals(status[0].getOperationStatusCode(), OperationStatusCode.SUCCESS);
6577 assertEquals(status[1].getOperationStatusCode(), OperationStatusCode.SUCCESS);
6578 return null;
6579 }
6580 });
6581
6582 f1.get();
6583 f2.get();
6584
6585 CONF.setInt("hbase.rowlock.wait.duration", prevLockTimeout);
6586 }
6587
6588 @Test
6589 public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException {
6590 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
6591 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
6592 EnvironmentEdgeManager.injectEdge(edge);
6593
6594 edge.setValue(10);
6595 Put p = new Put(row);
6596 p.setDurability(Durability.SKIP_WAL);
6597 p.addColumn(fam1, qual1, qual1);
6598 region.put(p);
6599
6600 Result result = region.get(new Get(row));
6601 Cell c = result.getColumnLatestCell(fam1, qual1);
6602 assertNotNull(c);
6603 assertEquals(c.getTimestamp(), 10L);
6604
6605 edge.setValue(1);
6606 p = new Put(row);
6607 p.setDurability(Durability.SKIP_WAL);
6608 p.addColumn(fam1, qual1, qual2);
6609 RowMutations rm = new RowMutations(row);
6610 rm.add(p);
6611 region.checkAndRowMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1),
6612 rm, false);
6613 result = region.get(new Get(row));
6614 c = result.getColumnLatestCell(fam1, qual1);
6615 assertEquals(c.getTimestamp(), 10L);
6616
6617 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
6618 qual2, 0, qual2.length));
6619 }
6620
6621 static HRegion initHRegion(byte[] tableName, String callingMethod,
6622 byte[]... families) throws IOException {
6623 return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
6624 families);
6625 }
6626
6627
6628
6629
6630
6631 @Test
6632 public void testWritesWhileRollWriter() throws IOException {
6633 int testCount = 10;
6634 int numRows = 1024;
6635 int numFamilies = 2;
6636 int numQualifiers = 2;
6637 final byte[][] families = new byte[numFamilies][];
6638 for (int i = 0; i < numFamilies; i++) {
6639 families[i] = Bytes.toBytes("family" + i);
6640 }
6641 final byte[][] qualifiers = new byte[numQualifiers][];
6642 for (int i = 0; i < numQualifiers; i++) {
6643 qualifiers[i] = Bytes.toBytes("qual" + i);
6644 }
6645
6646 String method = "testWritesWhileRollWriter";
6647 CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 2);
6648 this.region = initHRegion(tableName, method, CONF, families);
6649 try {
6650 List<Thread> threads = new ArrayList<Thread>();
6651 for (int i = 0; i < numRows; i++) {
6652 final int count = i;
6653 Thread t = new Thread(new Runnable() {
6654
6655 @Override
6656 public void run() {
6657 byte[] row = Bytes.toBytes("row" + count);
6658 Put put = new Put(row);
6659 put.setDurability(Durability.SYNC_WAL);
6660 byte[] value = Bytes.toBytes(String.valueOf(count));
6661 for (byte[] family : families) {
6662 for (byte[] qualifier : qualifiers) {
6663 put.addColumn(family, qualifier, (long) count, value);
6664 }
6665 }
6666 try {
6667 region.put(put);
6668 } catch (IOException e) {
6669 throw new RuntimeException(e);
6670 }
6671 }
6672 });
6673 threads.add(t);
6674 }
6675 for (Thread t : threads) {
6676 t.start();
6677 }
6678
6679 for (int i = 0; i < testCount; i++) {
6680 region.getWAL().rollWriter();
6681 Thread.yield();
6682 }
6683 } finally {
6684 try {
6685 HBaseTestingUtility.closeRegionAndWAL(this.region);
6686 CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 16 * 1024);
6687 } catch (DroppedSnapshotException dse) {
6688
6689
6690
6691
6692 }
6693 this.region = null;
6694 }
6695 }
6696 }