1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21
22 import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23 import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24 import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25 import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29 import static org.junit.Assert.assertArrayEquals;
30 import static org.junit.Assert.assertEquals;
31 import static org.junit.Assert.assertFalse;
32 import static org.junit.Assert.assertNotNull;
33 import static org.junit.Assert.assertNull;
34 import static org.junit.Assert.assertTrue;
35 import static org.junit.Assert.fail;
36 import static org.mockito.Matchers.any;
37 import static org.mockito.Matchers.anyBoolean;
38 import static org.mockito.Matchers.anyLong;
39 import static org.mockito.Mockito.mock;
40 import static org.mockito.Mockito.never;
41 import static org.mockito.Mockito.spy;
42 import static org.mockito.Mockito.times;
43 import static org.mockito.Mockito.verify;
44 import static org.mockito.Mockito.when;
45
46 import java.io.IOException;
47 import java.io.InterruptedIOException;
48 import java.security.PrivilegedExceptionAction;
49 import java.util.ArrayList;
50 import java.util.Arrays;
51 import java.util.Collection;
52 import java.util.Collections;
53 import java.util.HashMap;
54 import java.util.List;
55 import java.util.Map;
56 import java.util.NavigableMap;
57 import java.util.TreeMap;
58 import java.util.UUID;
59 import java.util.concurrent.Callable;
60 import java.util.concurrent.CountDownLatch;
61 import java.util.concurrent.ExecutorService;
62 import java.util.concurrent.Executors;
63 import java.util.concurrent.Future;
64 import java.util.concurrent.TimeUnit;
65 import java.util.concurrent.atomic.AtomicBoolean;
66 import java.util.concurrent.atomic.AtomicInteger;
67 import java.util.concurrent.atomic.AtomicLong;
68 import java.util.concurrent.atomic.AtomicReference;
69
70 import org.apache.commons.lang.RandomStringUtils;
71 import org.apache.commons.logging.Log;
72 import org.apache.commons.logging.LogFactory;
73 import org.apache.hadoop.conf.Configuration;
74 import org.apache.hadoop.fs.FSDataOutputStream;
75 import org.apache.hadoop.fs.FileStatus;
76 import org.apache.hadoop.fs.FileSystem;
77 import org.apache.hadoop.fs.Path;
78 import org.apache.hadoop.hbase.Cell;
79 import org.apache.hadoop.hbase.CellComparator;
80 import org.apache.hadoop.hbase.CellUtil;
81 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
82 import org.apache.hadoop.hbase.DroppedSnapshotException;
83 import org.apache.hadoop.hbase.HBaseConfiguration;
84 import org.apache.hadoop.hbase.HBaseTestCase;
85 import org.apache.hadoop.hbase.HBaseTestingUtility;
86 import org.apache.hadoop.hbase.HColumnDescriptor;
87 import org.apache.hadoop.hbase.HConstants;
88 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
89 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
90 import org.apache.hadoop.hbase.HRegionInfo;
91 import org.apache.hadoop.hbase.HTableDescriptor;
92 import org.apache.hadoop.hbase.KeyValue;
93 import org.apache.hadoop.hbase.MiniHBaseCluster;
94 import org.apache.hadoop.hbase.MultithreadedTestUtil;
95 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
96 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
97 import org.apache.hadoop.hbase.NotServingRegionException;
98 import org.apache.hadoop.hbase.RegionTooBusyException;
99 import org.apache.hadoop.hbase.ServerName;
100 import org.apache.hadoop.hbase.TableName;
101 import org.apache.hadoop.hbase.Tag;
102 import org.apache.hadoop.hbase.TagType;
103 import org.apache.hadoop.hbase.Waiter;
104 import org.apache.hadoop.hbase.client.Append;
105 import org.apache.hadoop.hbase.client.Delete;
106 import org.apache.hadoop.hbase.client.Durability;
107 import org.apache.hadoop.hbase.client.Get;
108 import org.apache.hadoop.hbase.client.Increment;
109 import org.apache.hadoop.hbase.client.Mutation;
110 import org.apache.hadoop.hbase.client.Put;
111 import org.apache.hadoop.hbase.client.Result;
112 import org.apache.hadoop.hbase.client.RowMutations;
113 import org.apache.hadoop.hbase.client.Scan;
114 import org.apache.hadoop.hbase.client.Table;
115 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
116 import org.apache.hadoop.hbase.filter.BinaryComparator;
117 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
118 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
119 import org.apache.hadoop.hbase.filter.Filter;
120 import org.apache.hadoop.hbase.filter.FilterBase;
121 import org.apache.hadoop.hbase.filter.FilterList;
122 import org.apache.hadoop.hbase.filter.NullComparator;
123 import org.apache.hadoop.hbase.filter.PrefixFilter;
124 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
125 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
126 import org.apache.hadoop.hbase.io.hfile.HFile;
127 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
128 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
129 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
130 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
131 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
132 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
133 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
134 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor;
135 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
136 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
137 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
138 import org.apache.hadoop.hbase.regionserver.Region.RowLock;
139 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
140 import org.apache.hadoop.hbase.regionserver.handler.FinishRegionRecoveringHandler;
141 import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
142 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
143 import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
144 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
145 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
146 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
147 import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
148 import org.apache.hadoop.hbase.security.User;
149 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
150 import org.apache.hadoop.hbase.testclassification.MediumTests;
151 import org.apache.hadoop.hbase.util.Bytes;
152 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
153 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
154 import org.apache.hadoop.hbase.util.FSUtils;
155 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
156 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
157 import org.apache.hadoop.hbase.util.PairOfSameType;
158 import org.apache.hadoop.hbase.util.Threads;
159 import org.apache.hadoop.hbase.wal.DefaultWALProvider;
160 import org.apache.hadoop.hbase.wal.FaultyFSLog;
161 import org.apache.hadoop.hbase.wal.WAL;
162 import org.apache.hadoop.hbase.wal.WALFactory;
163 import org.apache.hadoop.hbase.wal.WALKey;
164 import org.apache.hadoop.hbase.wal.WALProvider;
165 import org.apache.hadoop.hbase.wal.WALProvider.Writer;
166 import org.apache.hadoop.hbase.wal.WALSplitter;
167 import org.junit.After;
168 import org.junit.Assert;
169 import org.junit.Before;
170 import org.junit.Rule;
171 import org.junit.Test;
172 import org.junit.experimental.categories.Category;
173 import org.junit.rules.TestName;
174 import org.mockito.ArgumentCaptor;
175 import org.mockito.ArgumentMatcher;
176 import org.mockito.Mockito;
177
178 import com.google.common.collect.ImmutableList;
179 import com.google.common.collect.Lists;
180 import com.google.common.collect.Maps;
181 import com.google.protobuf.ByteString;
182
183
184
185
186
187
188
189 @Category(MediumTests.class)
190 @SuppressWarnings("deprecation")
191 public class TestHRegion {
192
193
194 static final Log LOG = LogFactory.getLog(TestHRegion.class);
195 @Rule public TestName name = new TestName();
196
197 private static final String COLUMN_FAMILY = "MyCF";
198 private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
199
200 HRegion region = null;
201
202 private static HBaseTestingUtility TEST_UTIL;
203 public static Configuration CONF ;
204 private String dir;
205 private static FileSystem FILESYSTEM;
206 private final int MAX_VERSIONS = 2;
207
208
209 protected byte[] tableName;
210 protected String method;
211 protected final byte[] qual1 = Bytes.toBytes("qual1");
212 protected final byte[] qual2 = Bytes.toBytes("qual2");
213 protected final byte[] qual3 = Bytes.toBytes("qual3");
214 protected final byte[] value1 = Bytes.toBytes("value1");
215 protected final byte[] value2 = Bytes.toBytes("value2");
216 protected final byte[] row = Bytes.toBytes("rowA");
217 protected final byte[] row2 = Bytes.toBytes("rowB");
218
219 protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
220 .getInstance(MetricsAssertHelper.class);
221
222 @Before
223 public void setup() throws IOException {
224 TEST_UTIL = HBaseTestingUtility.createLocalHTU();
225 FILESYSTEM = TEST_UTIL.getTestFileSystem();
226 CONF = TEST_UTIL.getConfiguration();
227 dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
228 method = name.getMethodName();
229 tableName = Bytes.toBytes(name.getMethodName());
230 }
231
232 @After
233 public void tearDown() throws Exception {
234 EnvironmentEdgeManagerTestHelper.reset();
235 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
236 TEST_UTIL.cleanupTestDir();
237 }
238
239 String getName() {
240 return name.getMethodName();
241 }
242
243
244
245
246
247
248
249
250
251
252
253 @Test (timeout=60000)
254 public void testCloseCarryingSnapshot() throws IOException {
255 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
256 Store store = region.getStore(COLUMN_FAMILY_BYTES);
257
258 byte [] value = Bytes.toBytes(name.getMethodName());
259
260 Put put = new Put(value);
261 put.add(COLUMN_FAMILY_BYTES, null, value);
262
263 region.put(put);
264 StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
265 storeFlushCtx.prepare();
266
267 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
268 region.put(put);
269
270 region.close();
271 assertEquals(0, region.getMemstoreSize());
272 HRegion.closeHRegion(region);
273 }
274
275
276
277
278
279
280
281 @Test (timeout=60000)
282 public void testMemstoreSnapshotSize() throws IOException {
283 class MyFaultyFSLog extends FaultyFSLog {
284 StoreFlushContext storeFlushCtx;
285 public MyFaultyFSLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
286 throws IOException {
287 super(fs, rootDir, logName, conf);
288 }
289
290 void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
291 this.storeFlushCtx = storeFlushCtx;
292 }
293
294 @Override
295 public void sync(long txid) throws IOException {
296 storeFlushCtx.prepare();
297 super.sync(txid);
298 }
299 }
300
301 FileSystem fs = FileSystem.get(CONF);
302 Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
303 MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
304 HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
305 CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES);
306
307 Store store = region.getStore(COLUMN_FAMILY_BYTES);
308
309 byte [] value = Bytes.toBytes(name.getMethodName());
310 faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
311
312 Put put = new Put(value);
313 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
314 faultyLog.setFailureType(FaultyFSLog.FailureType.SYNC);
315
316 boolean threwIOE = false;
317 try {
318 region.put(put);
319 } catch (IOException ioe) {
320 threwIOE = true;
321 } finally {
322 assertTrue("The regionserver should have thrown an exception", threwIOE);
323 }
324 long sz = store.getFlushableSize();
325 assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
326 HRegion.closeHRegion(region);
327 }
328
329
330
331
332 @Test
333 public void testMemstoreSizeWithFlushCanceling() throws IOException {
334 FileSystem fs = FileSystem.get(CONF);
335 Path rootDir = new Path(dir + "testMemstoreSizeWithFlushCanceling");
336 FSHLog hLog = new FSHLog(fs, rootDir, "testMemstoreSizeWithFlushCanceling", CONF);
337 HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
338 CONF, false, Durability.SYNC_WAL, hLog, COLUMN_FAMILY_BYTES);
339 Store store = region.getStore(COLUMN_FAMILY_BYTES);
340 assertEquals(0, region.getMemstoreSize());
341
342
343 byte [] value = Bytes.toBytes(name.getMethodName());
344 Put put = new Put(value);
345 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
346 region.put(put);
347 long onePutSize = region.getMemstoreSize();
348 assertTrue(onePutSize > 0);
349 region.flush(true);
350 assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize());
351 assertEquals("flushable size should be zero", 0, store.getFlushableSize());
352
353
354 RegionCoprocessorHost normalCPHost = region.getCoprocessorHost();
355 RegionCoprocessorHost mockedCPHost = Mockito.mock(RegionCoprocessorHost.class);
356 when(mockedCPHost.preFlush(Mockito.isA(HStore.class), Mockito.isA(InternalScanner.class))).
357 thenReturn(null);
358 region.setCoprocessorHost(mockedCPHost);
359 region.put(put);
360 region.flush(true);
361 assertEquals("memstoreSize should NOT be zero", onePutSize, region.getMemstoreSize());
362 assertEquals("flushable size should NOT be zero", onePutSize, store.getFlushableSize());
363
364
365 region.setCoprocessorHost(normalCPHost);
366 region.flush(true);
367 assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize());
368 assertEquals("flushable size should be zero", 0, store.getFlushableSize());
369 HRegion.closeHRegion(region);
370 }
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387 @Test (timeout=60000)
388 public void testFlushSizeAccounting() throws Exception {
389 final Configuration conf = HBaseConfiguration.create(CONF);
390
391 conf.setInt("hbase.hstore.flush.retries.number", 1);
392 final User user =
393 User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
394
395 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
396 user.runAs(new PrivilegedExceptionAction<Object>() {
397 @Override
398 public Object run() throws Exception {
399
400 FileSystem fs = FileSystem.get(conf);
401 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
402 FaultyFileSystem ffs = (FaultyFileSystem)fs;
403 HRegion region = null;
404 try {
405
406 region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
407 long size = region.getMemstoreSize();
408 Assert.assertEquals(0, size);
409
410 Put p1 = new Put(row);
411 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
412 region.put(p1);
413 final long sizeOfOnePut = region.getMemstoreSize();
414
415 try {
416 LOG.info("Flushing");
417 region.flush(true);
418 Assert.fail("Didn't bubble up IOE!");
419 } catch (DroppedSnapshotException dse) {
420
421 region.closing.set(false);
422 } catch (Exception e) {
423
424 region.closing.set(false);
425 }
426
427 ffs.fault.set(false);
428
429 try {
430 region.getWAL().rollWriter(true);
431 } catch (Exception e) {
432 int x = 0;
433 }
434
435 Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize());
436
437
438 Put p2 = new Put(row);
439 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
440 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
441 region.put(p2);
442 Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize());
443
444
445
446 region.flush(true);
447
448 Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize());
449 } catch (Exception e) {
450 int x = 0;
451 } finally {
452 HRegion.closeHRegion(region);
453 }
454 return null;
455 }
456 });
457 FileSystem.closeAllForUGI(user.getUGI());
458 }
459
460 @Test (timeout=60000)
461 public void testCloseWithFailingFlush() throws Exception {
462 final Configuration conf = HBaseConfiguration.create(CONF);
463
464 conf.setInt("hbase.hstore.flush.retries.number", 1);
465 final User user =
466 User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
467
468 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
469 user.runAs(new PrivilegedExceptionAction<Object>() {
470 @Override
471 public Object run() throws Exception {
472
473 FileSystem fs = FileSystem.get(conf);
474 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
475 FaultyFileSystem ffs = (FaultyFileSystem)fs;
476 HRegion region = null;
477 try {
478
479 region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
480 long size = region.getMemstoreSize();
481 Assert.assertEquals(0, size);
482
483 Put p1 = new Put(row);
484 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
485 region.put(p1);
486
487 Store store = region.getStore(COLUMN_FAMILY_BYTES);
488 StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
489 storeFlushCtx.prepare();
490
491 Put p2 = new Put(row);
492 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
493 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
494 region.put(p2);
495
496 region.close();
497 fail();
498 } catch (IOException dse) {
499
500 LOG.info("Expected DroppedSnapshotException");
501 } finally {
502
503 ffs.fault.set(false);
504 region.getWAL().rollWriter(true);
505 HRegion.closeHRegion(region);
506 }
507 return null;
508 }
509 });
510 FileSystem.closeAllForUGI(user.getUGI());
511 }
512
513 @Test
514 public void testCompactionAffectedByScanners() throws Exception {
515 byte[] family = Bytes.toBytes("family");
516 this.region = initHRegion(tableName, method, CONF, family);
517
518 Put put = new Put(Bytes.toBytes("r1"));
519 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
520 region.put(put);
521 region.flush(true);
522
523 Scan scan = new Scan();
524 scan.setMaxVersions(3);
525
526 RegionScanner scanner1 = region.getScanner(scan);
527
528 Delete delete = new Delete(Bytes.toBytes("r1"));
529 region.delete(delete);
530 region.flush(true);
531
532
533 RegionScanner scanner2 = region.getScanner(scan);
534
535 List<Cell> results = new ArrayList<Cell>();
536
537 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
538
539
540 region.compact(true);
541
542
543 RegionScanner scanner3 = region.getScanner(scan);
544
545
546 scanner1.next(results);
547 System.out.println(results);
548 assertEquals(1, results.size());
549
550 results.clear();
551 scanner2.next(results);
552 System.out.println(results);
553 assertEquals(0, results.size());
554
555 results.clear();
556 scanner3.next(results);
557 System.out.println(results);
558 assertEquals(0, results.size());
559 }
560
561 @Test
562 public void testToShowNPEOnRegionScannerReseek() throws Exception {
563 byte[] family = Bytes.toBytes("family");
564 this.region = initHRegion(tableName, method, CONF, family);
565
566 Put put = new Put(Bytes.toBytes("r1"));
567 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
568 region.put(put);
569 put = new Put(Bytes.toBytes("r2"));
570 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
571 region.put(put);
572 region.flush(true);
573
574 Scan scan = new Scan();
575 scan.setMaxVersions(3);
576
577 RegionScanner scanner1 = region.getScanner(scan);
578
579 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
580
581 region.compact(true);
582
583 scanner1.reseek(Bytes.toBytes("r2"));
584 List<Cell> results = new ArrayList<Cell>();
585 scanner1.next(results);
586 Cell keyValue = results.get(0);
587 Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
588 scanner1.close();
589 }
590
591 @Test
592 public void testSkipRecoveredEditsReplay() throws Exception {
593 String method = "testSkipRecoveredEditsReplay";
594 TableName tableName = TableName.valueOf(method);
595 byte[] family = Bytes.toBytes("family");
596 this.region = initHRegion(tableName, method, CONF, family);
597 final WALFactory wals = new WALFactory(CONF, null, method);
598 try {
599 Path regiondir = region.getRegionFileSystem().getRegionDir();
600 FileSystem fs = region.getRegionFileSystem().getFileSystem();
601 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
602
603 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
604
605 long maxSeqId = 1050;
606 long minSeqId = 1000;
607
608 for (long i = minSeqId; i <= maxSeqId; i += 10) {
609 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
610 fs.create(recoveredEdits);
611 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
612
613 long time = System.nanoTime();
614 WALEdit edit = new WALEdit();
615 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
616 .toBytes(i)));
617 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time,
618 HConstants.DEFAULT_CLUSTER_ID), edit));
619
620 writer.close();
621 }
622 MonitoredTask status = TaskMonitor.get().createStatus(method);
623 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
624 for (Store store : region.getStores()) {
625 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
626 }
627 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
628 assertEquals(maxSeqId, seqId);
629 region.getMVCC().initialize(seqId);
630 Get get = new Get(row);
631 Result result = region.get(get);
632 for (long i = minSeqId; i <= maxSeqId; i += 10) {
633 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
634 assertEquals(1, kvs.size());
635 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
636 }
637 } finally {
638 HRegion.closeHRegion(this.region);
639 this.region = null;
640 wals.close();
641 }
642 }
643
644 @Test
645 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
646 String method = "testSkipRecoveredEditsReplaySomeIgnored";
647 TableName tableName = TableName.valueOf(method);
648 byte[] family = Bytes.toBytes("family");
649 this.region = initHRegion(tableName, method, CONF, family);
650 final WALFactory wals = new WALFactory(CONF, null, method);
651 try {
652 Path regiondir = region.getRegionFileSystem().getRegionDir();
653 FileSystem fs = region.getRegionFileSystem().getFileSystem();
654 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
655
656 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
657
658 long maxSeqId = 1050;
659 long minSeqId = 1000;
660
661 for (long i = minSeqId; i <= maxSeqId; i += 10) {
662 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
663 fs.create(recoveredEdits);
664 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
665
666 long time = System.nanoTime();
667 WALEdit edit = new WALEdit();
668 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
669 .toBytes(i)));
670 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time,
671 HConstants.DEFAULT_CLUSTER_ID), edit));
672
673 writer.close();
674 }
675 long recoverSeqId = 1030;
676 MonitoredTask status = TaskMonitor.get().createStatus(method);
677 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
678 for (Store store : region.getStores()) {
679 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
680 }
681 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
682 assertEquals(maxSeqId, seqId);
683 region.getMVCC().initialize(seqId);
684 Get get = new Get(row);
685 Result result = region.get(get);
686 for (long i = minSeqId; i <= maxSeqId; i += 10) {
687 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
688 if (i < recoverSeqId) {
689 assertEquals(0, kvs.size());
690 } else {
691 assertEquals(1, kvs.size());
692 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
693 }
694 }
695 } finally {
696 HRegion.closeHRegion(this.region);
697 this.region = null;
698 wals.close();
699 }
700 }
701
702 @Test
703 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
704 byte[] family = Bytes.toBytes("family");
705 this.region = initHRegion(tableName, method, CONF, family);
706 try {
707 Path regiondir = region.getRegionFileSystem().getRegionDir();
708 FileSystem fs = region.getRegionFileSystem().getFileSystem();
709
710 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
711 for (int i = 1000; i < 1050; i += 10) {
712 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
713 FSDataOutputStream dos = fs.create(recoveredEdits);
714 dos.writeInt(i);
715 dos.close();
716 }
717 long minSeqId = 2000;
718 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
719 FSDataOutputStream dos = fs.create(recoveredEdits);
720 dos.close();
721
722 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
723 for (Store store : region.getStores()) {
724 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
725 }
726 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
727 assertEquals(minSeqId, seqId);
728 } finally {
729 HRegion.closeHRegion(this.region);
730 this.region = null;
731 }
732 }
733
734 @Test
735 public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
736 String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
737 TableName tableName = TableName.valueOf(method);
738 byte[] family = Bytes.toBytes("family");
739 this.region = initHRegion(tableName, method, CONF, family);
740 final WALFactory wals = new WALFactory(CONF, null, method);
741 try {
742 Path regiondir = region.getRegionFileSystem().getRegionDir();
743 FileSystem fs = region.getRegionFileSystem().getFileSystem();
744 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
745 byte[][] columns = region.getTableDesc().getFamiliesKeys().toArray(new byte[0][]);
746
747 assertEquals(0, region.getStoreFileList(columns).size());
748
749 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
750
751 long maxSeqId = 1050;
752 long minSeqId = 1000;
753
754 for (long i = minSeqId; i <= maxSeqId; i += 10) {
755 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
756 fs.create(recoveredEdits);
757 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
758
759 long time = System.nanoTime();
760 WALEdit edit = null;
761 if (i == maxSeqId) {
762 edit = WALEdit.createCompaction(region.getRegionInfo(),
763 CompactionDescriptor.newBuilder()
764 .setTableName(ByteString.copyFrom(tableName.getName()))
765 .setFamilyName(ByteString.copyFrom(regionName))
766 .setEncodedRegionName(ByteString.copyFrom(regionName))
767 .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
768 .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName()))
769 .build());
770 } else {
771 edit = new WALEdit();
772 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
773 .toBytes(i)));
774 }
775 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time,
776 HConstants.DEFAULT_CLUSTER_ID), edit));
777 writer.close();
778 }
779
780 long recoverSeqId = 1030;
781 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
782 MonitoredTask status = TaskMonitor.get().createStatus(method);
783 for (Store store : region.getStores()) {
784 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
785 }
786 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
787 assertEquals(maxSeqId, seqId);
788
789
790 assertEquals(1, region.getStoreFileList(columns).size());
791
792 } finally {
793 HRegion.closeHRegion(this.region);
794 this.region = null;
795 wals.close();
796 }
797 }
798
799 @Test
800 public void testRecoveredEditsReplayCompaction() throws Exception {
801 String method = name.getMethodName();
802 TableName tableName = TableName.valueOf(method);
803 byte[] family = Bytes.toBytes("family");
804 this.region = initHRegion(tableName, method, CONF, family);
805 final WALFactory wals = new WALFactory(CONF, null, method);
806 try {
807 Path regiondir = region.getRegionFileSystem().getRegionDir();
808 FileSystem fs = region.getRegionFileSystem().getFileSystem();
809 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
810
811 long maxSeqId = 3;
812 long minSeqId = 0;
813
814 for (long i = minSeqId; i < maxSeqId; i++) {
815 Put put = new Put(Bytes.toBytes(i));
816 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
817 region.put(put);
818 region.flush(true);
819 }
820
821
822 assertEquals(3, region.getStore(family).getStorefilesCount());
823 List<Path> storeFiles = new ArrayList<Path>(3);
824 for (StoreFile sf : region.getStore(family).getStorefiles()) {
825 storeFiles.add(sf.getPath());
826 }
827
828
829 CONF.setBoolean("hbase.hstore.compaction.complete", false);
830 region.compactStores();
831
832
833 assertEquals(3, region.getStore(family).getStorefilesCount());
834
835
836 Path tmpDir = region.getRegionFileSystem().getTempDir();
837 FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
838 String errorMsg = "Expected to find 1 file in the region temp directory "
839 + "from the compaction, could not find any";
840 assertNotNull(errorMsg, files);
841 assertEquals(errorMsg, 1, files.length);
842
843 Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
844 files[0].getPath());
845
846 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
847 .getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile), region
848 .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
849
850 WALUtil.writeCompactionMarker(region.getWAL(), this.region.getTableDesc(),
851 this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));
852
853 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
854
855 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
856 fs.create(recoveredEdits);
857 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
858
859 long time = System.nanoTime();
860
861 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, 10, time,
862 HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(),
863 compactionDescriptor)));
864 writer.close();
865
866
867 region.getTableDesc();
868 region.getRegionInfo();
869 region.close();
870 region = HRegion.openHRegion(region, null);
871
872
873 Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
874 for (StoreFile sf : sfs) {
875 LOG.info(sf.getPath());
876 }
877 assertEquals(1, region.getStore(family).getStorefilesCount());
878 files = FSUtils.listStatus(fs, tmpDir);
879 assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
880
881 for (long i = minSeqId; i < maxSeqId; i++) {
882 Get get = new Get(Bytes.toBytes(i));
883 Result result = region.get(get);
884 byte[] value = result.getValue(family, Bytes.toBytes(i));
885 assertArrayEquals(Bytes.toBytes(i), value);
886 }
887 } finally {
888 HRegion.closeHRegion(this.region);
889 this.region = null;
890 wals.close();
891 }
892 }
893
894 @Test
895 public void testFlushMarkers() throws Exception {
896
897 String method = name.getMethodName();
898 TableName tableName = TableName.valueOf(method);
899 byte[] family = Bytes.toBytes("family");
900 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log");
901 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
902 FSUtils.setRootDir(walConf, logDir);
903 final WALFactory wals = new WALFactory(walConf, null, method);
904 final WAL wal = wals.getWAL(tableName.getName());
905
906 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
907 HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
908 try {
909 Path regiondir = region.getRegionFileSystem().getRegionDir();
910 FileSystem fs = region.getRegionFileSystem().getFileSystem();
911 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
912
913 long maxSeqId = 3;
914 long minSeqId = 0;
915
916 for (long i = minSeqId; i < maxSeqId; i++) {
917 Put put = new Put(Bytes.toBytes(i));
918 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
919 region.put(put);
920 region.flush(true);
921 }
922
923
924 assertEquals(3, region.getStore(family).getStorefilesCount());
925 List<String> storeFiles = new ArrayList<String>(3);
926 for (StoreFile sf : region.getStore(family).getStorefiles()) {
927 storeFiles.add(sf.getPath().getName());
928 }
929
930
931 wal.shutdown();
932 WAL.Reader reader = WALFactory.createReader(fs, DefaultWALProvider.getCurrentFileName(wal),
933 TEST_UTIL.getConfiguration());
934 try {
935 List<WAL.Entry> flushDescriptors = new ArrayList<WAL.Entry>();
936 long lastFlushSeqId = -1;
937 while (true) {
938 WAL.Entry entry = reader.next();
939 if (entry == null) {
940 break;
941 }
942 Cell cell = entry.getEdit().getCells().get(0);
943 if (WALEdit.isMetaEditFamily(cell)) {
944 FlushDescriptor flushDesc = WALEdit.getFlushDescriptor(cell);
945 assertNotNull(flushDesc);
946 assertArrayEquals(tableName.getName(), flushDesc.getTableName().toByteArray());
947 if (flushDesc.getAction() == FlushAction.START_FLUSH) {
948 assertTrue(flushDesc.getFlushSequenceNumber() > lastFlushSeqId);
949 } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) {
950 assertTrue(flushDesc.getFlushSequenceNumber() == lastFlushSeqId);
951 }
952 lastFlushSeqId = flushDesc.getFlushSequenceNumber();
953 assertArrayEquals(regionName, flushDesc.getEncodedRegionName().toByteArray());
954 assertEquals(1, flushDesc.getStoreFlushesCount());
955 StoreFlushDescriptor storeFlushDesc = flushDesc.getStoreFlushes(0);
956 assertArrayEquals(family, storeFlushDesc.getFamilyName().toByteArray());
957 assertEquals("family", storeFlushDesc.getStoreHomeDir());
958 if (flushDesc.getAction() == FlushAction.START_FLUSH) {
959 assertEquals(0, storeFlushDesc.getFlushOutputCount());
960 } else {
961 assertEquals(1, storeFlushDesc.getFlushOutputCount());
962 assertTrue(storeFiles.contains(storeFlushDesc.getFlushOutput(0)));
963 }
964
965 flushDescriptors.add(entry);
966 }
967 }
968
969 assertEquals(3 * 2, flushDescriptors.size());
970
971
972
973 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
974
975 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
976 fs.create(recoveredEdits);
977 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
978
979 for (WAL.Entry entry : flushDescriptors) {
980 writer.append(entry);
981 }
982 writer.close();
983 } finally {
984 if (null != reader) {
985 try {
986 reader.close();
987 } catch (IOException exception) {
988 LOG.warn("Problem closing wal: " + exception.getMessage());
989 LOG.debug("exception details", exception);
990 }
991 }
992 }
993
994
995
996 region.close();
997 region = HRegion.openHRegion(region, null);
998
999
1000 for (long i = minSeqId; i < maxSeqId; i++) {
1001 Get get = new Get(Bytes.toBytes(i));
1002 Result result = region.get(get);
1003 byte[] value = result.getValue(family, Bytes.toBytes(i));
1004 assertArrayEquals(Bytes.toBytes(i), value);
1005 }
1006 } finally {
1007 HRegion.closeHRegion(this.region);
1008 this.region = null;
1009 wals.close();
1010 }
1011 }
1012
1013 class IsFlushWALMarker extends ArgumentMatcher<WALEdit> {
1014 volatile FlushAction[] actions;
1015 public IsFlushWALMarker(FlushAction... actions) {
1016 this.actions = actions;
1017 }
1018 @Override
1019 public boolean matches(Object edit) {
1020 List<Cell> cells = ((WALEdit)edit).getCells();
1021 if (cells.isEmpty()) {
1022 return false;
1023 }
1024 if (WALEdit.isMetaEditFamily(cells.get(0))) {
1025 FlushDescriptor desc = null;
1026 try {
1027 desc = WALEdit.getFlushDescriptor(cells.get(0));
1028 } catch (IOException e) {
1029 LOG.warn(e);
1030 return false;
1031 }
1032 if (desc != null) {
1033 for (FlushAction action : actions) {
1034 if (desc.getAction() == action) {
1035 return true;
1036 }
1037 }
1038 }
1039 }
1040 return false;
1041 }
1042 public IsFlushWALMarker set(FlushAction... actions) {
1043 this.actions = actions;
1044 return this;
1045 }
1046 }
1047
1048 @Test (timeout=60000)
1049 public void testFlushMarkersWALFail() throws Exception {
1050
1051 String method = name.getMethodName();
1052 TableName tableName = TableName.valueOf(method);
1053 byte[] family = Bytes.toBytes("family");
1054
1055
1056 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + "log");
1057
1058 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
1059 FSUtils.setRootDir(walConf, logDir);
1060
1061
1062 class FailAppendFlushMarkerWAL extends FSHLog {
1063 volatile FlushAction [] flushActions = null;
1064
1065 public FailAppendFlushMarkerWAL(FileSystem fs, Path root, String logDir, Configuration conf)
1066 throws IOException {
1067 super(fs, root, logDir, conf);
1068 }
1069
1070 @Override
1071 protected Writer createWriterInstance(Path path) throws IOException {
1072 final Writer w = super.createWriterInstance(path);
1073 return new Writer() {
1074 @Override
1075 public void close() throws IOException {
1076 w.close();
1077 }
1078
1079 @Override
1080 public void sync() throws IOException {
1081 w.sync();
1082 }
1083
1084 @Override
1085 public void append(Entry entry) throws IOException {
1086 List<Cell> cells = entry.getEdit().getCells();
1087 if (WALEdit.isMetaEditFamily(cells.get(0))) {
1088 FlushDescriptor desc = WALEdit.getFlushDescriptor(cells.get(0));
1089 if (desc != null) {
1090 for (FlushAction flushAction: flushActions) {
1091 if (desc.getAction().equals(flushAction)) {
1092 throw new IOException("Failed to append flush marker! " + flushAction);
1093 }
1094 }
1095 }
1096 }
1097 w.append(entry);
1098 }
1099
1100 @Override
1101 public long getLength() throws IOException {
1102 return w.getLength();
1103 }
1104 };
1105 }
1106 }
1107 FailAppendFlushMarkerWAL wal =
1108 new FailAppendFlushMarkerWAL(FileSystem.get(walConf), FSUtils.getRootDir(walConf),
1109 getName(), walConf);
1110 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
1111 HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
1112 try {
1113 int i = 0;
1114 Put put = new Put(Bytes.toBytes(i));
1115 put.setDurability(Durability.SKIP_WAL);
1116 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
1117 region.put(put);
1118
1119
1120 wal.flushActions = new FlushAction [] {FlushAction.START_FLUSH};
1121
1122
1123 try {
1124 region.flush(true);
1125 fail("This should have thrown exception");
1126 } catch (DroppedSnapshotException unexpected) {
1127
1128 throw unexpected;
1129 } catch (IOException expected) {
1130
1131 }
1132
1133
1134
1135 wal.rollWriter(true);
1136
1137
1138 wal.flushActions = new FlushAction [] {FlushAction.COMMIT_FLUSH};
1139
1140 try {
1141 region.flush(true);
1142 fail("This should have thrown exception");
1143 } catch (DroppedSnapshotException expected) {
1144
1145
1146 } catch (IOException unexpected) {
1147 throw unexpected;
1148 }
1149
1150 region.close();
1151
1152 wal.rollWriter(true);
1153 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
1154 HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
1155 region.put(put);
1156
1157
1158
1159
1160 wal.flushActions = new FlushAction [] {FlushAction.COMMIT_FLUSH, FlushAction.ABORT_FLUSH};
1161
1162 try {
1163 region.flush(true);
1164 fail("This should have thrown exception");
1165 } catch (DroppedSnapshotException expected) {
1166
1167
1168 } catch (IOException unexpected) {
1169 throw unexpected;
1170 }
1171
1172 } finally {
1173 HRegion.closeHRegion(this.region);
1174 this.region = null;
1175 }
1176 }
1177
1178 @Test
1179 public void testGetWhileRegionClose() throws IOException {
1180 TableName tableName = TableName.valueOf(name.getMethodName());
1181 Configuration hc = initSplit();
1182 int numRows = 100;
1183 byte[][] families = { fam1, fam2, fam3 };
1184
1185
1186 String method = name.getMethodName();
1187 this.region = initHRegion(tableName, method, hc, families);
1188 try {
1189
1190 final int startRow = 100;
1191 putData(startRow, numRows, qual1, families);
1192 putData(startRow, numRows, qual2, families);
1193 putData(startRow, numRows, qual3, families);
1194 final AtomicBoolean done = new AtomicBoolean(false);
1195 final AtomicInteger gets = new AtomicInteger(0);
1196 GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
1197 try {
1198
1199 for (int i = 0; i < threads.length / 2; i++) {
1200 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
1201 threads[i].setDaemon(true);
1202 threads[i].start();
1203 }
1204
1205
1206 this.region.closing.set(true);
1207 for (int i = threads.length / 2; i < threads.length; i++) {
1208 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
1209 threads[i].setDaemon(true);
1210 threads[i].start();
1211 }
1212 } finally {
1213 if (this.region != null) {
1214 HRegion.closeHRegion(this.region);
1215 }
1216 }
1217 done.set(true);
1218 for (GetTillDoneOrException t : threads) {
1219 try {
1220 t.join();
1221 } catch (InterruptedException e) {
1222 e.printStackTrace();
1223 }
1224 if (t.e != null) {
1225 LOG.info("Exception=" + t.e);
1226 assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
1227 }
1228 }
1229 } finally {
1230 HRegion.closeHRegion(this.region);
1231 this.region = null;
1232 }
1233 }
1234
1235
1236
1237
1238
1239 class GetTillDoneOrException extends Thread {
1240 private final Get g;
1241 private final AtomicBoolean done;
1242 private final AtomicInteger count;
1243 private Exception e;
1244
1245 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
1246 super("getter." + i);
1247 this.g = new Get(r);
1248 this.done = d;
1249 this.count = c;
1250 }
1251
1252 @Override
1253 public void run() {
1254 while (!this.done.get()) {
1255 try {
1256 assertTrue(region.get(g).size() > 0);
1257 this.count.incrementAndGet();
1258 } catch (Exception e) {
1259 this.e = e;
1260 break;
1261 }
1262 }
1263 }
1264 }
1265
1266
1267
1268
1269 @Test
1270 public void testWeirdCacheBehaviour() throws Exception {
1271 byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
1272 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
1273 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
1274 this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
1275 try {
1276 String value = "this is the value";
1277 String value2 = "this is some other value";
1278 String keyPrefix1 = "prefix1";
1279 String keyPrefix2 = "prefix2";
1280 String keyPrefix3 = "prefix3";
1281 putRows(this.region, 3, value, keyPrefix1);
1282 putRows(this.region, 3, value, keyPrefix2);
1283 putRows(this.region, 3, value, keyPrefix3);
1284 putRows(this.region, 3, value2, keyPrefix1);
1285 putRows(this.region, 3, value2, keyPrefix2);
1286 putRows(this.region, 3, value2, keyPrefix3);
1287 System.out.println("Checking values for key: " + keyPrefix1);
1288 assertEquals("Got back incorrect number of rows from scan", 3,
1289 getNumberOfRows(keyPrefix1, value2, this.region));
1290 System.out.println("Checking values for key: " + keyPrefix2);
1291 assertEquals("Got back incorrect number of rows from scan", 3,
1292 getNumberOfRows(keyPrefix2, value2, this.region));
1293 System.out.println("Checking values for key: " + keyPrefix3);
1294 assertEquals("Got back incorrect number of rows from scan", 3,
1295 getNumberOfRows(keyPrefix3, value2, this.region));
1296 deleteColumns(this.region, value2, keyPrefix1);
1297 deleteColumns(this.region, value2, keyPrefix2);
1298 deleteColumns(this.region, value2, keyPrefix3);
1299 System.out.println("Starting important checks.....");
1300 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
1301 getNumberOfRows(keyPrefix1, value2, this.region));
1302 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
1303 getNumberOfRows(keyPrefix2, value2, this.region));
1304 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
1305 getNumberOfRows(keyPrefix3, value2, this.region));
1306 } finally {
1307 HRegion.closeHRegion(this.region);
1308 this.region = null;
1309 }
1310 }
1311
1312 @Test
1313 public void testAppendWithReadOnlyTable() throws Exception {
1314 byte[] TABLE = Bytes.toBytes("readOnlyTable");
1315 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
1316 boolean exceptionCaught = false;
1317 Append append = new Append(Bytes.toBytes("somerow"));
1318 append.setDurability(Durability.SKIP_WAL);
1319 append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
1320 Bytes.toBytes("somevalue"));
1321 try {
1322 region.append(append);
1323 } catch (IOException e) {
1324 exceptionCaught = true;
1325 } finally {
1326 HRegion.closeHRegion(this.region);
1327 this.region = null;
1328 }
1329 assertTrue(exceptionCaught == true);
1330 }
1331
1332 @Test
1333 public void testIncrWithReadOnlyTable() throws Exception {
1334 byte[] TABLE = Bytes.toBytes("readOnlyTable");
1335 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
1336 boolean exceptionCaught = false;
1337 Increment inc = new Increment(Bytes.toBytes("somerow"));
1338 inc.setDurability(Durability.SKIP_WAL);
1339 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
1340 try {
1341 region.increment(inc);
1342 } catch (IOException e) {
1343 exceptionCaught = true;
1344 } finally {
1345 HRegion.closeHRegion(this.region);
1346 this.region = null;
1347 }
1348 assertTrue(exceptionCaught == true);
1349 }
1350
1351 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
1352 InternalScanner scanner = buildScanner(keyPrefix, value, r);
1353 int count = 0;
1354 boolean more = false;
1355 List<Cell> results = new ArrayList<Cell>();
1356 do {
1357 more = scanner.next(results);
1358 if (results != null && !results.isEmpty())
1359 count++;
1360 else
1361 break;
1362 Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
1363 delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
1364 r.delete(delete);
1365 results.clear();
1366 } while (more);
1367 assertEquals("Did not perform correct number of deletes", 3, count);
1368 }
1369
1370 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
1371 InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
1372 int numberOfResults = 0;
1373 List<Cell> results = new ArrayList<Cell>();
1374 boolean more = false;
1375 do {
1376 more = resultScanner.next(results);
1377 if (results != null && !results.isEmpty())
1378 numberOfResults++;
1379 else
1380 break;
1381 for (Cell kv : results) {
1382 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
1383 }
1384 results.clear();
1385 } while (more);
1386 return numberOfResults;
1387 }
1388
1389 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
1390 throws IOException {
1391
1392 FilterList allFilters = new FilterList();
1393 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
1394
1395 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
1396 Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
1397 filter.setFilterIfMissing(true);
1398 allFilters.addFilter(filter);
1399 Scan scan = new Scan();
1400 scan.addFamily(Bytes.toBytes("trans-blob"));
1401 scan.addFamily(Bytes.toBytes("trans-type"));
1402 scan.addFamily(Bytes.toBytes("trans-date"));
1403 scan.addFamily(Bytes.toBytes("trans-tags"));
1404 scan.addFamily(Bytes.toBytes("trans-group"));
1405 scan.setFilter(allFilters);
1406 return r.getScanner(scan);
1407 }
1408
1409 private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
1410 for (int i = 0; i < numRows; i++) {
1411 String row = key + "_" + i
1412 System.out.println(String.format("Saving row: %s, with value %s", row, value));
1413 Put put = new Put(Bytes.toBytes(row));
1414 put.setDurability(Durability.SKIP_WAL);
1415 put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
1416 put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
1417 put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
1418 put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
1419 put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
1420 r.put(put);
1421 }
1422 }
1423
1424 @Test
1425 public void testFamilyWithAndWithoutColon() throws Exception {
1426 byte[] b = Bytes.toBytes(getName());
1427 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1428 this.region = initHRegion(b, getName(), CONF, cf);
1429 try {
1430 Put p = new Put(b);
1431 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
1432 p.add(cfwithcolon, cfwithcolon, cfwithcolon);
1433 boolean exception = false;
1434 try {
1435 this.region.put(p);
1436 } catch (NoSuchColumnFamilyException e) {
1437 exception = true;
1438 }
1439 assertTrue(exception);
1440 } finally {
1441 HRegion.closeHRegion(this.region);
1442 this.region = null;
1443 }
1444 }
1445
1446 @Test
1447 public void testBatchPut_whileNoRowLocksHeld() throws IOException {
1448 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1449 byte[] qual = Bytes.toBytes("qual");
1450 byte[] val = Bytes.toBytes("val");
1451 this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1452 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1453 try {
1454 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1455 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1456
1457 LOG.info("First a batch put with all valid puts");
1458 final Put[] puts = new Put[10];
1459 for (int i = 0; i < 10; i++) {
1460 puts[i] = new Put(Bytes.toBytes("row_" + i));
1461 puts[i].add(cf, qual, val);
1462 }
1463
1464 OperationStatus[] codes = this.region.batchMutate(puts);
1465 assertEquals(10, codes.length);
1466 for (int i = 0; i < 10; i++) {
1467 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1468 }
1469 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1470
1471 LOG.info("Next a batch put with one invalid family");
1472 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1473 codes = this.region.batchMutate(puts);
1474 assertEquals(10, codes.length);
1475 for (int i = 0; i < 10; i++) {
1476 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1477 codes[i].getOperationStatusCode());
1478 }
1479
1480 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1481 } finally {
1482 HRegion.closeHRegion(this.region);
1483 this.region = null;
1484 }
1485 }
1486
1487 @Test
1488 public void testBatchPut_whileMultipleRowLocksHeld() throws Exception {
1489 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1490 byte[] qual = Bytes.toBytes("qual");
1491 byte[] val = Bytes.toBytes("val");
1492 this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1493 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1494 try {
1495 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1496 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1497
1498 final Put[] puts = new Put[10];
1499 for (int i = 0; i < 10; i++) {
1500 puts[i] = new Put(Bytes.toBytes("row_" + i));
1501 puts[i].add(cf, qual, val);
1502 }
1503 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1504
1505 LOG.info("batchPut will have to break into four batches to avoid row locks");
1506 RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2"));
1507 RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_4"));
1508 RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_6"));
1509
1510 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1511 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
1512 TestThread putter = new TestThread(ctx) {
1513 @Override
1514 public void doWork() throws IOException {
1515 retFromThread.set(region.batchMutate(puts));
1516 }
1517 };
1518 LOG.info("...starting put thread while holding locks");
1519 ctx.addThread(putter);
1520 ctx.startThreads();
1521
1522 LOG.info("...waiting for put thread to sync 1st time");
1523 waitForCounter(source, "syncTimeNumOps", syncs + 1);
1524
1525
1526
1527
1528
1529
1530 Thread regionCloseThread = new Thread() {
1531 @Override
1532 public void run() {
1533 try {
1534 HRegion.closeHRegion(region);
1535 } catch (IOException e) {
1536 throw new RuntimeException(e);
1537 }
1538 }
1539 };
1540 regionCloseThread.start();
1541
1542 LOG.info("...releasing row lock 1, which should let put thread continue");
1543 rowLock1.release();
1544
1545 LOG.info("...waiting for put thread to sync 2nd time");
1546 waitForCounter(source, "syncTimeNumOps", syncs + 2);
1547
1548 LOG.info("...releasing row lock 2, which should let put thread continue");
1549 rowLock2.release();
1550
1551 LOG.info("...waiting for put thread to sync 3rd time");
1552 waitForCounter(source, "syncTimeNumOps", syncs + 3);
1553
1554 LOG.info("...releasing row lock 3, which should let put thread continue");
1555 rowLock3.release();
1556
1557 LOG.info("...waiting for put thread to sync 4th time");
1558 waitForCounter(source, "syncTimeNumOps", syncs + 4);
1559
1560 LOG.info("...joining on put thread");
1561 ctx.stop();
1562 regionCloseThread.join();
1563
1564 OperationStatus[] codes = retFromThread.get();
1565 for (int i = 0; i < codes.length; i++) {
1566 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1567 codes[i].getOperationStatusCode());
1568 }
1569 } finally {
1570 HRegion.closeHRegion(this.region);
1571 this.region = null;
1572 }
1573 }
1574
1575 private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount)
1576 throws InterruptedException {
1577 long startWait = System.currentTimeMillis();
1578 long currentCount;
1579 while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) {
1580 Thread.sleep(100);
1581 if (System.currentTimeMillis() - startWait > 10000) {
1582 fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName,
1583 expectedCount, currentCount));
1584 }
1585 }
1586 }
1587
1588 @Test
1589 public void testBatchPutWithTsSlop() throws Exception {
1590 byte[] b = Bytes.toBytes(getName());
1591 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1592 byte[] qual = Bytes.toBytes("qual");
1593 byte[] val = Bytes.toBytes("val");
1594
1595
1596 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1597 this.region = initHRegion(b, getName(), CONF, cf);
1598
1599 try {
1600 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1601 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1602 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1603
1604 final Put[] puts = new Put[10];
1605 for (int i = 0; i < 10; i++) {
1606 puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1607 puts[i].add(cf, qual, val);
1608 }
1609
1610 OperationStatus[] codes = this.region.batchMutate(puts);
1611 assertEquals(10, codes.length);
1612 for (int i = 0; i < 10; i++) {
1613 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1614 }
1615 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1616
1617 } finally {
1618 HRegion.closeHRegion(this.region);
1619 this.region = null;
1620 }
1621
1622 }
1623
1624
1625
1626
1627 @Test
1628 public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1629 byte[] row1 = Bytes.toBytes("row1");
1630 byte[] fam1 = Bytes.toBytes("fam1");
1631 byte[] qf1 = Bytes.toBytes("qualifier");
1632 byte[] emptyVal = new byte[] {};
1633 byte[] val1 = Bytes.toBytes("value1");
1634 byte[] val2 = Bytes.toBytes("value2");
1635
1636
1637 String method = this.getName();
1638 this.region = initHRegion(tableName, method, CONF, fam1);
1639 try {
1640
1641 Put put = new Put(row1);
1642 put.add(fam1, qf1, emptyVal);
1643
1644
1645 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1646 emptyVal), put, true);
1647 assertTrue(res);
1648
1649
1650 put = new Put(row1);
1651 put.add(fam1, qf1, val1);
1652
1653
1654 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1655 put, true);
1656 assertTrue(res);
1657
1658
1659 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1660 put, true);
1661 assertFalse(res);
1662
1663 Delete delete = new Delete(row1);
1664 delete.deleteColumn(fam1, qf1);
1665 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1666 delete, true);
1667 assertFalse(res);
1668
1669 put = new Put(row1);
1670 put.add(fam1, qf1, val2);
1671
1672 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1673 put, true);
1674 assertTrue(res);
1675
1676
1677 delete = new Delete(row1);
1678 delete.deleteColumn(fam1, qf1);
1679 delete.deleteColumn(fam1, qf1);
1680 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1681 delete, true);
1682 assertTrue(res);
1683
1684 delete = new Delete(row1);
1685 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1686 delete, true);
1687 assertTrue(res);
1688
1689
1690 put = new Put(row1);
1691 put.add(fam1, qf1, val1);
1692
1693 res = region
1694 .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1695 assertTrue(res);
1696 } finally {
1697 HRegion.closeHRegion(this.region);
1698 this.region = null;
1699 }
1700 }
1701
1702 @Test
1703 public void testCheckAndMutate_WithWrongValue() throws IOException {
1704 byte[] row1 = Bytes.toBytes("row1");
1705 byte[] fam1 = Bytes.toBytes("fam1");
1706 byte[] qf1 = Bytes.toBytes("qualifier");
1707 byte[] val1 = Bytes.toBytes("value1");
1708 byte[] val2 = Bytes.toBytes("value2");
1709
1710
1711 String method = this.getName();
1712 this.region = initHRegion(tableName, method, CONF, fam1);
1713 try {
1714
1715 Put put = new Put(row1);
1716 put.add(fam1, qf1, val1);
1717 region.put(put);
1718
1719
1720 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1721 val2), put, true);
1722 assertEquals(false, res);
1723
1724
1725 Delete delete = new Delete(row1);
1726 delete.deleteFamily(fam1);
1727 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1728 put, true);
1729 assertEquals(false, res);
1730 } finally {
1731 HRegion.closeHRegion(this.region);
1732 this.region = null;
1733 }
1734 }
1735
1736 @Test
1737 public void testCheckAndMutate_WithCorrectValue() throws IOException {
1738 byte[] row1 = Bytes.toBytes("row1");
1739 byte[] fam1 = Bytes.toBytes("fam1");
1740 byte[] qf1 = Bytes.toBytes("qualifier");
1741 byte[] val1 = Bytes.toBytes("value1");
1742
1743
1744 String method = this.getName();
1745 this.region = initHRegion(tableName, method, CONF, fam1);
1746 try {
1747
1748 Put put = new Put(row1);
1749 put.add(fam1, qf1, val1);
1750 region.put(put);
1751
1752
1753 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1754 val1), put, true);
1755 assertEquals(true, res);
1756
1757
1758 Delete delete = new Delete(row1);
1759 delete.deleteColumn(fam1, qf1);
1760 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1761 delete, true);
1762 assertEquals(true, res);
1763 } finally {
1764 HRegion.closeHRegion(this.region);
1765 this.region = null;
1766 }
1767 }
1768
1769 @Test
1770 public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1771 byte[] row1 = Bytes.toBytes("row1");
1772 byte[] fam1 = Bytes.toBytes("fam1");
1773 byte[] qf1 = Bytes.toBytes("qualifier");
1774 byte[] val1 = Bytes.toBytes("value1");
1775 byte[] val2 = Bytes.toBytes("value2");
1776 byte[] val3 = Bytes.toBytes("value3");
1777 byte[] val4 = Bytes.toBytes("value4");
1778
1779
1780 String method = this.getName();
1781 this.region = initHRegion(tableName, method, CONF, fam1);
1782 try {
1783
1784 Put put = new Put(row1);
1785 put.add(fam1, qf1, val3);
1786 region.put(put);
1787
1788
1789 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1790 new BinaryComparator(val3), put, true);
1791 assertEquals(false, res);
1792
1793
1794 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1795 new BinaryComparator(val4), put, true);
1796 assertEquals(false, res);
1797
1798
1799
1800 put = new Put(row1);
1801 put.add(fam1, qf1, val2);
1802 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1803 new BinaryComparator(val2), put, true);
1804 assertEquals(true, res);
1805
1806
1807 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1808 new BinaryComparator(val3), put, true);
1809 assertEquals(false, res);
1810
1811
1812
1813 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1814 new BinaryComparator(val2), put, true);
1815 assertEquals(true, res);
1816
1817
1818
1819 put = new Put(row1);
1820 put.add(fam1, qf1, val3);
1821 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1822 new BinaryComparator(val1), put, true);
1823 assertEquals(true, res);
1824
1825
1826 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1827 new BinaryComparator(val3), put, true);
1828 assertEquals(false, res);
1829
1830
1831 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1832 new BinaryComparator(val2), put, true);
1833 assertEquals(false, res);
1834
1835
1836
1837 put = new Put(row1);
1838 put.add(fam1, qf1, val2);
1839 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1840 new BinaryComparator(val4), put, true);
1841 assertEquals(true, res);
1842
1843
1844 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1845 new BinaryComparator(val1), put, true);
1846 assertEquals(false, res);
1847
1848
1849
1850 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1851 new BinaryComparator(val2), put, true);
1852 assertEquals(true, res);
1853
1854
1855 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1856 new BinaryComparator(val3), put, true);
1857 assertEquals(true, res);
1858 } finally {
1859 HRegion.closeHRegion(this.region);
1860 this.region = null;
1861 }
1862 }
1863
1864 @Test
1865 public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1866 byte[] row1 = Bytes.toBytes("row1");
1867 byte[] fam1 = Bytes.toBytes("fam1");
1868 byte[] fam2 = Bytes.toBytes("fam2");
1869 byte[] qf1 = Bytes.toBytes("qualifier");
1870 byte[] val1 = Bytes.toBytes("value1");
1871 byte[] val2 = Bytes.toBytes("value2");
1872
1873 byte[][] families = { fam1, fam2 };
1874
1875
1876 String method = this.getName();
1877 this.region = initHRegion(tableName, method, CONF, families);
1878 try {
1879
1880 Put put = new Put(row1);
1881 put.add(fam1, qf1, val1);
1882 region.put(put);
1883
1884
1885 long ts = System.currentTimeMillis();
1886 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1887 put = new Put(row1);
1888 put.add(kv);
1889
1890
1891 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1892 val1), put, true);
1893 assertEquals(true, res);
1894
1895 Get get = new Get(row1);
1896 get.addColumn(fam2, qf1);
1897 Cell[] actual = region.get(get).rawCells();
1898
1899 Cell[] expected = { kv };
1900
1901 assertEquals(expected.length, actual.length);
1902 for (int i = 0; i < actual.length; i++) {
1903 assertEquals(expected[i], actual[i]);
1904 }
1905 } finally {
1906 HRegion.closeHRegion(this.region);
1907 this.region = null;
1908 }
1909 }
1910
1911 @Test
1912 public void testCheckAndPut_wrongRowInPut() throws IOException {
1913 TableName tableName = TableName.valueOf(name.getMethodName());
1914 this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1915 try {
1916 Put put = new Put(row2);
1917 put.add(fam1, qual1, value1);
1918 try {
1919 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1920 new BinaryComparator(value2), put, false);
1921 fail();
1922 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1923
1924 }
1925 } finally {
1926 HRegion.closeHRegion(this.region);
1927 this.region = null;
1928 }
1929 }
1930
1931 @Test
1932 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1933 byte[] row1 = Bytes.toBytes("row1");
1934 byte[] fam1 = Bytes.toBytes("fam1");
1935 byte[] fam2 = Bytes.toBytes("fam2");
1936 byte[] qf1 = Bytes.toBytes("qualifier1");
1937 byte[] qf2 = Bytes.toBytes("qualifier2");
1938 byte[] qf3 = Bytes.toBytes("qualifier3");
1939 byte[] val1 = Bytes.toBytes("value1");
1940 byte[] val2 = Bytes.toBytes("value2");
1941 byte[] val3 = Bytes.toBytes("value3");
1942 byte[] emptyVal = new byte[] {};
1943
1944 byte[][] families = { fam1, fam2 };
1945
1946
1947 String method = this.getName();
1948 this.region = initHRegion(tableName, method, CONF, families);
1949 try {
1950
1951 Put put = new Put(row1);
1952 put.add(fam1, qf1, val1);
1953 region.put(put);
1954 Threads.sleep(2);
1955
1956 put = new Put(row1);
1957 put.add(fam1, qf1, val2);
1958 put.add(fam2, qf1, val3);
1959 put.add(fam2, qf2, val2);
1960 put.add(fam2, qf3, val1);
1961 put.add(fam1, qf3, val1);
1962 region.put(put);
1963
1964
1965 Delete delete = new Delete(row1);
1966 delete.deleteColumn(fam1, qf1);
1967 delete.deleteColumn(fam2, qf1);
1968 delete.deleteColumn(fam1, qf3);
1969 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1970 val2), delete, true);
1971 assertEquals(true, res);
1972
1973 Get get = new Get(row1);
1974 get.addColumn(fam1, qf1);
1975 get.addColumn(fam1, qf3);
1976 get.addColumn(fam2, qf2);
1977 Result r = region.get(get);
1978 assertEquals(2, r.size());
1979 assertArrayEquals(val1, r.getValue(fam1, qf1));
1980 assertArrayEquals(val2, r.getValue(fam2, qf2));
1981
1982
1983 delete = new Delete(row1);
1984 delete.deleteFamily(fam2);
1985 res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1986 delete, true);
1987 assertEquals(true, res);
1988
1989 get = new Get(row1);
1990 r = region.get(get);
1991 assertEquals(1, r.size());
1992 assertArrayEquals(val1, r.getValue(fam1, qf1));
1993
1994
1995 delete = new Delete(row1);
1996 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1997 delete, true);
1998 assertEquals(true, res);
1999 get = new Get(row1);
2000 r = region.get(get);
2001 assertEquals(0, r.size());
2002 } finally {
2003 HRegion.closeHRegion(this.region);
2004 this.region = null;
2005 }
2006 }
2007
2008
2009
2010
2011 @Test
2012 public void testDelete_multiDeleteColumn() throws IOException {
2013 byte[] row1 = Bytes.toBytes("row1");
2014 byte[] fam1 = Bytes.toBytes("fam1");
2015 byte[] qual = Bytes.toBytes("qualifier");
2016 byte[] value = Bytes.toBytes("value");
2017
2018 Put put = new Put(row1);
2019 put.add(fam1, qual, 1, value);
2020 put.add(fam1, qual, 2, value);
2021
2022 String method = this.getName();
2023 this.region = initHRegion(tableName, method, CONF, fam1);
2024 try {
2025 region.put(put);
2026
2027
2028 Delete delete = new Delete(row1);
2029 delete.deleteColumn(fam1, qual);
2030 delete.deleteColumn(fam1, qual);
2031 region.delete(delete);
2032
2033 Get get = new Get(row1);
2034 get.addFamily(fam1);
2035 Result r = region.get(get);
2036 assertEquals(0, r.size());
2037 } finally {
2038 HRegion.closeHRegion(this.region);
2039 this.region = null;
2040 }
2041 }
2042
2043 @Test
2044 public void testDelete_CheckFamily() throws IOException {
2045 byte[] row1 = Bytes.toBytes("row1");
2046 byte[] fam1 = Bytes.toBytes("fam1");
2047 byte[] fam2 = Bytes.toBytes("fam2");
2048 byte[] fam3 = Bytes.toBytes("fam3");
2049 byte[] fam4 = Bytes.toBytes("fam4");
2050
2051
2052 String method = this.getName();
2053 this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
2054 try {
2055 List<Cell> kvs = new ArrayList<Cell>();
2056 kvs.add(new KeyValue(row1, fam4, null, null));
2057
2058
2059 byte[] family = fam2;
2060 try {
2061 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
2062 Bytes.BYTES_COMPARATOR);
2063 deleteMap.put(family, kvs);
2064 region.delete(deleteMap, Durability.SYNC_WAL);
2065 } catch (Exception e) {
2066 assertTrue("Family " + new String(family) + " does not exist", false);
2067 }
2068
2069
2070 boolean ok = false;
2071 family = fam4;
2072 try {
2073 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
2074 Bytes.BYTES_COMPARATOR);
2075 deleteMap.put(family, kvs);
2076 region.delete(deleteMap, Durability.SYNC_WAL);
2077 } catch (Exception e) {
2078 ok = true;
2079 }
2080 assertEquals("Family " + new String(family) + " does exist", true, ok);
2081 } finally {
2082 HRegion.closeHRegion(this.region);
2083 this.region = null;
2084 }
2085 }
2086
2087 @Test
2088 public void testDelete_mixed() throws IOException, InterruptedException {
2089 byte[] fam = Bytes.toBytes("info");
2090 byte[][] families = { fam };
2091 String method = this.getName();
2092 this.region = initHRegion(tableName, method, CONF, families);
2093 try {
2094 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
2095
2096 byte[] row = Bytes.toBytes("table_name");
2097
2098 byte[] serverinfo = Bytes.toBytes("serverinfo");
2099 byte[] splitA = Bytes.toBytes("splitA");
2100 byte[] splitB = Bytes.toBytes("splitB");
2101
2102
2103 Put put = new Put(row);
2104 put.add(fam, splitA, Bytes.toBytes("reference_A"));
2105 region.put(put);
2106
2107 put = new Put(row);
2108 put.add(fam, splitB, Bytes.toBytes("reference_B"));
2109 region.put(put);
2110
2111 put = new Put(row);
2112 put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
2113 region.put(put);
2114
2115
2116 Delete delete = new Delete(row);
2117 delete.deleteColumns(fam, splitA);
2118 region.delete(delete);
2119
2120
2121 Get get = new Get(row).addColumn(fam, serverinfo);
2122 Result result = region.get(get);
2123 assertEquals(1, result.size());
2124
2125 get = new Get(row).addColumn(fam, splitA);
2126 result = region.get(get);
2127 assertEquals(0, result.size());
2128
2129 get = new Get(row).addColumn(fam, splitB);
2130 result = region.get(get);
2131 assertEquals(1, result.size());
2132
2133
2134 put = new Put(row);
2135 put.add(fam, splitA, Bytes.toBytes("reference_A"));
2136 region.put(put);
2137 get = new Get(row);
2138 result = region.get(get);
2139 assertEquals(3, result.size());
2140
2141
2142 delete = new Delete(row);
2143 region.delete(delete);
2144 assertEquals(0, region.get(get).size());
2145
2146 region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
2147 result = region.get(get);
2148 assertEquals(1, result.size());
2149 } finally {
2150 HRegion.closeHRegion(this.region);
2151 this.region = null;
2152 }
2153 }
2154
2155 @Test
2156 public void testDeleteRowWithFutureTs() throws IOException {
2157 byte[] fam = Bytes.toBytes("info");
2158 byte[][] families = { fam };
2159 String method = this.getName();
2160 this.region = initHRegion(tableName, method, CONF, families);
2161 try {
2162 byte[] row = Bytes.toBytes("table_name");
2163
2164 byte[] serverinfo = Bytes.toBytes("serverinfo");
2165
2166
2167 Put put = new Put(row);
2168 put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
2169 region.put(put);
2170
2171
2172 Delete delete = new Delete(row);
2173 region.delete(delete);
2174
2175
2176 Get get = new Get(row).addColumn(fam, serverinfo);
2177 Result result = region.get(get);
2178 assertEquals(1, result.size());
2179
2180
2181 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
2182 region.delete(delete);
2183
2184
2185 get = new Get(row).addColumn(fam, serverinfo);
2186 result = region.get(get);
2187 assertEquals(0, result.size());
2188 } finally {
2189 HRegion.closeHRegion(this.region);
2190 this.region = null;
2191 }
2192 }
2193
2194
2195
2196
2197
2198 @Test
2199 public void testPutWithLatestTS() throws IOException {
2200 byte[] fam = Bytes.toBytes("info");
2201 byte[][] families = { fam };
2202 String method = this.getName();
2203 this.region = initHRegion(tableName, method, CONF, families);
2204 try {
2205 byte[] row = Bytes.toBytes("row1");
2206
2207 byte[] qual = Bytes.toBytes("qual");
2208
2209
2210 Put put = new Put(row);
2211 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
2212 region.put(put);
2213
2214
2215 Get get = new Get(row).addColumn(fam, qual);
2216 Result result = region.get(get);
2217 assertEquals(1, result.size());
2218 Cell kv = result.rawCells()[0];
2219 LOG.info("Got: " + kv);
2220 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
2221 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
2222
2223
2224
2225 row = Bytes.toBytes("row2");
2226 put = new Put(row);
2227 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
2228 region.put(put);
2229
2230
2231 get = new Get(row).addColumn(fam, qual);
2232 result = region.get(get);
2233 assertEquals(1, result.size());
2234 kv = result.rawCells()[0];
2235 LOG.info("Got: " + kv);
2236 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
2237 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
2238 } finally {
2239 HRegion.closeHRegion(this.region);
2240 this.region = null;
2241 }
2242
2243 }
2244
2245
2246
2247
2248
2249
2250 @Test
2251 public void testPutWithTsSlop() throws IOException {
2252 byte[] fam = Bytes.toBytes("info");
2253 byte[][] families = { fam };
2254 String method = this.getName();
2255
2256
2257 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
2258 this.region = initHRegion(tableName, method, CONF, families);
2259 boolean caughtExcep = false;
2260 try {
2261 try {
2262
2263 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
2264
2265 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
2266 Bytes.toBytes("value")));
2267 fail("Expected IOE for TS out of configured timerange");
2268 } catch (FailedSanityCheckException ioe) {
2269 LOG.debug("Received expected exception", ioe);
2270 caughtExcep = true;
2271 }
2272 assertTrue("Should catch FailedSanityCheckException", caughtExcep);
2273 } finally {
2274 HRegion.closeHRegion(this.region);
2275 this.region = null;
2276 }
2277 }
2278
2279 @Test
2280 public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
2281 byte[] fam1 = Bytes.toBytes("columnA");
2282 byte[] fam2 = Bytes.toBytes("columnB");
2283 this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
2284 try {
2285 byte[] rowA = Bytes.toBytes("rowA");
2286 byte[] rowB = Bytes.toBytes("rowB");
2287
2288 byte[] value = Bytes.toBytes("value");
2289
2290 Delete delete = new Delete(rowA);
2291 delete.deleteFamily(fam1);
2292
2293 region.delete(delete);
2294
2295
2296 Put put = new Put(rowA);
2297 put.add(fam2, null, value);
2298 region.put(put);
2299
2300 put = new Put(rowB);
2301 put.add(fam1, null, value);
2302 put.add(fam2, null, value);
2303 region.put(put);
2304
2305 Scan scan = new Scan();
2306 scan.addFamily(fam1).addFamily(fam2);
2307 InternalScanner s = region.getScanner(scan);
2308 List<Cell> results = new ArrayList<Cell>();
2309 s.next(results);
2310 assertTrue(CellUtil.matchingRow(results.get(0), rowA));
2311
2312 results.clear();
2313 s.next(results);
2314 assertTrue(CellUtil.matchingRow(results.get(0), rowB));
2315 } finally {
2316 HRegion.closeHRegion(this.region);
2317 this.region = null;
2318 }
2319 }
2320
2321 @Test
2322 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
2323 Delete delete = new Delete(row);
2324 delete.deleteColumns(fam1, qual1);
2325 doTestDelete_AndPostInsert(delete);
2326 }
2327
2328 @Test
2329 public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
2330 Delete delete = new Delete(row);
2331 delete.deleteFamily(fam1);
2332 doTestDelete_AndPostInsert(delete);
2333 }
2334
2335 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
2336 TableName tableName = TableName.valueOf(name.getMethodName());
2337 this.region = initHRegion(tableName, getName(), CONF, fam1);
2338 try {
2339 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
2340 Put put = new Put(row);
2341 put.add(fam1, qual1, value1);
2342 region.put(put);
2343
2344
2345 region.delete(delete);
2346
2347
2348 put = new Put(row);
2349 put.add(fam1, qual1, value2);
2350 region.put(put);
2351
2352
2353 Get get = new Get(row);
2354 get.addColumn(fam1, qual1);
2355
2356 Result r = region.get(get);
2357 assertEquals(1, r.size());
2358 assertArrayEquals(value2, r.getValue(fam1, qual1));
2359
2360
2361 Scan scan = new Scan(row);
2362 scan.addColumn(fam1, qual1);
2363 InternalScanner s = region.getScanner(scan);
2364
2365 List<Cell> results = new ArrayList<Cell>();
2366 assertEquals(false, s.next(results));
2367 assertEquals(1, results.size());
2368 Cell kv = results.get(0);
2369
2370 assertArrayEquals(value2, CellUtil.cloneValue(kv));
2371 assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
2372 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
2373 assertArrayEquals(row, CellUtil.cloneRow(kv));
2374 } finally {
2375 HRegion.closeHRegion(this.region);
2376 this.region = null;
2377 }
2378 }
2379
2380 @Test
2381 public void testDelete_CheckTimestampUpdated() throws IOException {
2382 TableName tableName = TableName.valueOf(name.getMethodName());
2383 byte[] row1 = Bytes.toBytes("row1");
2384 byte[] col1 = Bytes.toBytes("col1");
2385 byte[] col2 = Bytes.toBytes("col2");
2386 byte[] col3 = Bytes.toBytes("col3");
2387
2388
2389 String method = this.getName();
2390 this.region = initHRegion(tableName, method, CONF, fam1);
2391 try {
2392
2393 List<Cell> kvs = new ArrayList<Cell>();
2394 kvs.add(new KeyValue(row1, fam1, col1, null));
2395 kvs.add(new KeyValue(row1, fam1, col2, null));
2396 kvs.add(new KeyValue(row1, fam1, col3, null));
2397
2398 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
2399 Bytes.BYTES_COMPARATOR);
2400 deleteMap.put(fam1, kvs);
2401 region.delete(deleteMap, Durability.SYNC_WAL);
2402
2403
2404
2405 long now = System.currentTimeMillis();
2406 DefaultMemStore memstore = (DefaultMemStore) ((HStore) region.getStore(fam1)).memstore;
2407 Cell firstCell = memstore.cellSet.first();
2408 assertTrue(firstCell.getTimestamp() <= now);
2409 now = firstCell.getTimestamp();
2410 for (Cell cell : memstore.cellSet) {
2411 assertTrue(cell.getTimestamp() <= now);
2412 now = cell.getTimestamp();
2413 }
2414 } finally {
2415 HRegion.closeHRegion(this.region);
2416 this.region = null;
2417 }
2418 }
2419
2420
2421
2422
2423 @Test
2424 public void testGet_FamilyChecker() throws IOException {
2425 byte[] row1 = Bytes.toBytes("row1");
2426 byte[] fam1 = Bytes.toBytes("fam1");
2427 byte[] fam2 = Bytes.toBytes("False");
2428 byte[] col1 = Bytes.toBytes("col1");
2429
2430
2431 String method = this.getName();
2432 this.region = initHRegion(tableName, method, CONF, fam1);
2433 try {
2434 Get get = new Get(row1);
2435 get.addColumn(fam2, col1);
2436
2437
2438 try {
2439 region.get(get);
2440 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
2441 assertFalse(false);
2442 return;
2443 }
2444 assertFalse(true);
2445 } finally {
2446 HRegion.closeHRegion(this.region);
2447 this.region = null;
2448 }
2449 }
2450
2451 @Test
2452 public void testGet_Basic() throws IOException {
2453 byte[] row1 = Bytes.toBytes("row1");
2454 byte[] fam1 = Bytes.toBytes("fam1");
2455 byte[] col1 = Bytes.toBytes("col1");
2456 byte[] col2 = Bytes.toBytes("col2");
2457 byte[] col3 = Bytes.toBytes("col3");
2458 byte[] col4 = Bytes.toBytes("col4");
2459 byte[] col5 = Bytes.toBytes("col5");
2460
2461
2462 String method = this.getName();
2463 this.region = initHRegion(tableName, method, CONF, fam1);
2464 try {
2465
2466 Put put = new Put(row1);
2467 put.add(fam1, col1, null);
2468 put.add(fam1, col2, null);
2469 put.add(fam1, col3, null);
2470 put.add(fam1, col4, null);
2471 put.add(fam1, col5, null);
2472 region.put(put);
2473
2474 Get get = new Get(row1);
2475 get.addColumn(fam1, col2);
2476 get.addColumn(fam1, col4);
2477
2478 KeyValue kv1 = new KeyValue(row1, fam1, col2);
2479 KeyValue kv2 = new KeyValue(row1, fam1, col4);
2480 KeyValue[] expected = { kv1, kv2 };
2481
2482
2483 Result res = region.get(get);
2484 assertEquals(expected.length, res.size());
2485 for (int i = 0; i < res.size(); i++) {
2486 assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
2487 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
2488 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
2489 }
2490
2491
2492 Get g = new Get(row1);
2493 final int count = 2;
2494 g.setFilter(new ColumnCountGetFilter(count));
2495 res = region.get(g);
2496 assertEquals(count, res.size());
2497 } finally {
2498 HRegion.closeHRegion(this.region);
2499 this.region = null;
2500 }
2501 }
2502
2503 @Test
2504 public void testGet_Empty() throws IOException {
2505 byte[] row = Bytes.toBytes("row");
2506 byte[] fam = Bytes.toBytes("fam");
2507
2508 String method = this.getName();
2509 this.region = initHRegion(tableName, method, CONF, fam);
2510 try {
2511 Get get = new Get(row);
2512 get.addFamily(fam);
2513 Result r = region.get(get);
2514
2515 assertTrue(r.isEmpty());
2516 } finally {
2517 HRegion.closeHRegion(this.region);
2518 this.region = null;
2519 }
2520 }
2521
2522
2523
2524
2525 @Test
2526 public void testMerge() throws IOException {
2527 byte[][] families = { fam1, fam2, fam3 };
2528 Configuration hc = initSplit();
2529
2530 String method = this.getName();
2531 this.region = initHRegion(tableName, method, hc, families);
2532 try {
2533 LOG.info("" + HBaseTestCase.addContent(region, fam3));
2534 region.flush(true);
2535 region.compactStores();
2536 byte[] splitRow = region.checkSplit();
2537 assertNotNull(splitRow);
2538 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2539 HRegion[] subregions = splitRegion(region, splitRow);
2540 try {
2541
2542 for (int i = 0; i < subregions.length; i++) {
2543 HRegion.openHRegion(subregions[i], null);
2544 subregions[i].compactStores();
2545 }
2546 Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
2547 Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
2548 Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
2549 long startTime = System.currentTimeMillis();
2550 region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
2551 LOG.info("Merge regions elapsed time: "
2552 + ((System.currentTimeMillis() - startTime) / 1000.0));
2553 FILESYSTEM.delete(oldRegion1, true);
2554 FILESYSTEM.delete(oldRegion2, true);
2555 FILESYSTEM.delete(oldRegionPath, true);
2556 LOG.info("splitAndMerge completed.");
2557 } finally {
2558 for (int i = 0; i < subregions.length; i++) {
2559 try {
2560 HRegion.closeHRegion(subregions[i]);
2561 } catch (IOException e) {
2562
2563 }
2564 }
2565 }
2566 } finally {
2567 HRegion.closeHRegion(this.region);
2568 this.region = null;
2569 }
2570 }
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580 HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2581 PairOfSameType<Region> result = null;
2582 SplitTransactionImpl st = new SplitTransactionImpl(parent, midkey);
2583
2584
2585 if (!st.prepare()) {
2586 parent.clearSplit();
2587 return null;
2588 }
2589 try {
2590 result = st.execute(null, null);
2591 } catch (IOException ioe) {
2592 try {
2593 LOG.info("Running rollback of failed split of " +
2594 parent.getRegionInfo().getRegionNameAsString() + "; " + ioe.getMessage());
2595 st.rollback(null, null);
2596 LOG.info("Successful rollback of failed split of " +
2597 parent.getRegionInfo().getRegionNameAsString());
2598 return null;
2599 } catch (RuntimeException e) {
2600
2601 LOG.info("Failed rollback of failed split of " +
2602 parent.getRegionInfo().getRegionNameAsString() + " -- aborting server", e);
2603 }
2604 }
2605 finally {
2606 parent.clearSplit();
2607 }
2608 return new HRegion[] { (HRegion)result.getFirst(), (HRegion)result.getSecond() };
2609 }
2610
2611
2612
2613
2614 @Test
2615 public void testGetScanner_WithOkFamilies() throws IOException {
2616 byte[] fam1 = Bytes.toBytes("fam1");
2617 byte[] fam2 = Bytes.toBytes("fam2");
2618
2619 byte[][] families = { fam1, fam2 };
2620
2621
2622 String method = this.getName();
2623 this.region = initHRegion(tableName, method, CONF, families);
2624 try {
2625 Scan scan = new Scan();
2626 scan.addFamily(fam1);
2627 scan.addFamily(fam2);
2628 try {
2629 region.getScanner(scan);
2630 } catch (Exception e) {
2631 assertTrue("Families could not be found in Region", false);
2632 }
2633 } finally {
2634 HRegion.closeHRegion(this.region);
2635 this.region = null;
2636 }
2637 }
2638
2639 @Test
2640 public void testGetScanner_WithNotOkFamilies() throws IOException {
2641 byte[] fam1 = Bytes.toBytes("fam1");
2642 byte[] fam2 = Bytes.toBytes("fam2");
2643
2644 byte[][] families = { fam1 };
2645
2646
2647 String method = this.getName();
2648 this.region = initHRegion(tableName, method, CONF, families);
2649 try {
2650 Scan scan = new Scan();
2651 scan.addFamily(fam2);
2652 boolean ok = false;
2653 try {
2654 region.getScanner(scan);
2655 } catch (Exception e) {
2656 ok = true;
2657 }
2658 assertTrue("Families could not be found in Region", ok);
2659 } finally {
2660 HRegion.closeHRegion(this.region);
2661 this.region = null;
2662 }
2663 }
2664
2665 @Test
2666 public void testGetScanner_WithNoFamilies() throws IOException {
2667 byte[] row1 = Bytes.toBytes("row1");
2668 byte[] fam1 = Bytes.toBytes("fam1");
2669 byte[] fam2 = Bytes.toBytes("fam2");
2670 byte[] fam3 = Bytes.toBytes("fam3");
2671 byte[] fam4 = Bytes.toBytes("fam4");
2672
2673 byte[][] families = { fam1, fam2, fam3, fam4 };
2674
2675
2676 String method = this.getName();
2677 this.region = initHRegion(tableName, method, CONF, families);
2678 try {
2679
2680
2681 Put put = new Put(row1);
2682 put.add(fam1, null, null);
2683 put.add(fam2, null, null);
2684 put.add(fam3, null, null);
2685 put.add(fam4, null, null);
2686 region.put(put);
2687
2688 Scan scan = null;
2689 HRegion.RegionScannerImpl is = null;
2690
2691
2692
2693
2694 scan = new Scan();
2695 scan.addFamily(fam2);
2696 scan.addFamily(fam4);
2697 is = (RegionScannerImpl) region.getScanner(scan);
2698 assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2699
2700 scan = new Scan();
2701 is = (RegionScannerImpl) region.getScanner(scan);
2702 assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2703 } finally {
2704 HRegion.closeHRegion(this.region);
2705 this.region = null;
2706 }
2707 }
2708
2709
2710
2711
2712
2713
2714 @Test
2715 public void testGetScanner_WithRegionClosed() throws IOException {
2716 byte[] fam1 = Bytes.toBytes("fam1");
2717 byte[] fam2 = Bytes.toBytes("fam2");
2718
2719 byte[][] families = { fam1, fam2 };
2720
2721
2722 String method = this.getName();
2723 try {
2724 this.region = initHRegion(tableName, method, CONF, families);
2725 } catch (IOException e) {
2726 e.printStackTrace();
2727 fail("Got IOException during initHRegion, " + e.getMessage());
2728 }
2729 try {
2730 region.closed.set(true);
2731 try {
2732 region.getScanner(null);
2733 fail("Expected to get an exception during getScanner on a region that is closed");
2734 } catch (NotServingRegionException e) {
2735
2736 } catch (IOException e) {
2737 fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2738 + e.getMessage());
2739 }
2740 } finally {
2741 HRegion.closeHRegion(this.region);
2742 this.region = null;
2743 }
2744 }
2745
2746 @Test
2747 public void testRegionScanner_Next() throws IOException {
2748 byte[] row1 = Bytes.toBytes("row1");
2749 byte[] row2 = Bytes.toBytes("row2");
2750 byte[] fam1 = Bytes.toBytes("fam1");
2751 byte[] fam2 = Bytes.toBytes("fam2");
2752 byte[] fam3 = Bytes.toBytes("fam3");
2753 byte[] fam4 = Bytes.toBytes("fam4");
2754
2755 byte[][] families = { fam1, fam2, fam3, fam4 };
2756 long ts = System.currentTimeMillis();
2757
2758
2759 String method = this.getName();
2760 this.region = initHRegion(tableName, method, CONF, families);
2761 try {
2762
2763 Put put = null;
2764 put = new Put(row1);
2765 put.add(fam1, (byte[]) null, ts, null);
2766 put.add(fam2, (byte[]) null, ts, null);
2767 put.add(fam3, (byte[]) null, ts, null);
2768 put.add(fam4, (byte[]) null, ts, null);
2769 region.put(put);
2770
2771 put = new Put(row2);
2772 put.add(fam1, (byte[]) null, ts, null);
2773 put.add(fam2, (byte[]) null, ts, null);
2774 put.add(fam3, (byte[]) null, ts, null);
2775 put.add(fam4, (byte[]) null, ts, null);
2776 region.put(put);
2777
2778 Scan scan = new Scan();
2779 scan.addFamily(fam2);
2780 scan.addFamily(fam4);
2781 InternalScanner is = region.getScanner(scan);
2782
2783 List<Cell> res = null;
2784
2785
2786 List<Cell> expected1 = new ArrayList<Cell>();
2787 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2788 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2789
2790 res = new ArrayList<Cell>();
2791 is.next(res);
2792 for (int i = 0; i < res.size(); i++) {
2793 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2794 }
2795
2796
2797 List<Cell> expected2 = new ArrayList<Cell>();
2798 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2799 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2800
2801 res = new ArrayList<Cell>();
2802 is.next(res);
2803 for (int i = 0; i < res.size(); i++) {
2804 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2805 }
2806 } finally {
2807 HRegion.closeHRegion(this.region);
2808 this.region = null;
2809 }
2810 }
2811
2812 @Test
2813 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2814 byte[] row1 = Bytes.toBytes("row1");
2815 byte[] qf1 = Bytes.toBytes("qualifier1");
2816 byte[] qf2 = Bytes.toBytes("qualifier2");
2817 byte[] fam1 = Bytes.toBytes("fam1");
2818 byte[][] families = { fam1 };
2819
2820 long ts1 = System.currentTimeMillis();
2821 long ts2 = ts1 + 1;
2822 long ts3 = ts1 + 2;
2823
2824
2825 String method = this.getName();
2826 this.region = initHRegion(tableName, method, CONF, families);
2827 try {
2828
2829 Put put = null;
2830 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2831 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2832 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2833
2834 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2835 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2836 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2837
2838 put = new Put(row1);
2839 put.add(kv13);
2840 put.add(kv12);
2841 put.add(kv11);
2842 put.add(kv23);
2843 put.add(kv22);
2844 put.add(kv21);
2845 region.put(put);
2846
2847
2848 List<Cell> expected = new ArrayList<Cell>();
2849 expected.add(kv13);
2850 expected.add(kv12);
2851
2852 Scan scan = new Scan(row1);
2853 scan.addColumn(fam1, qf1);
2854 scan.setMaxVersions(MAX_VERSIONS);
2855 List<Cell> actual = new ArrayList<Cell>();
2856 InternalScanner scanner = region.getScanner(scan);
2857
2858 boolean hasNext = scanner.next(actual);
2859 assertEquals(false, hasNext);
2860
2861
2862 for (int i = 0; i < expected.size(); i++) {
2863 assertEquals(expected.get(i), actual.get(i));
2864 }
2865 } finally {
2866 HRegion.closeHRegion(this.region);
2867 this.region = null;
2868 }
2869 }
2870
2871 @Test
2872 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2873 byte[] row1 = Bytes.toBytes("row1");
2874 byte[] qf1 = Bytes.toBytes("qualifier1");
2875 byte[] qf2 = Bytes.toBytes("qualifier2");
2876 byte[] fam1 = Bytes.toBytes("fam1");
2877 byte[][] families = { fam1 };
2878
2879 long ts1 = 1;
2880 long ts2 = ts1 + 1;
2881 long ts3 = ts1 + 2;
2882
2883
2884 String method = this.getName();
2885 this.region = initHRegion(tableName, method, CONF, families);
2886 try {
2887
2888 Put put = null;
2889 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2890 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2891 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2892
2893 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2894 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2895 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2896
2897 put = new Put(row1);
2898 put.add(kv13);
2899 put.add(kv12);
2900 put.add(kv11);
2901 put.add(kv23);
2902 put.add(kv22);
2903 put.add(kv21);
2904 region.put(put);
2905 region.flush(true);
2906
2907
2908 List<Cell> expected = new ArrayList<Cell>();
2909 expected.add(kv13);
2910 expected.add(kv12);
2911 expected.add(kv23);
2912 expected.add(kv22);
2913
2914 Scan scan = new Scan(row1);
2915 scan.addColumn(fam1, qf1);
2916 scan.addColumn(fam1, qf2);
2917 scan.setMaxVersions(MAX_VERSIONS);
2918 List<Cell> actual = new ArrayList<Cell>();
2919 InternalScanner scanner = region.getScanner(scan);
2920
2921 boolean hasNext = scanner.next(actual);
2922 assertEquals(false, hasNext);
2923
2924
2925 for (int i = 0; i < expected.size(); i++) {
2926 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2927 }
2928 } finally {
2929 HRegion.closeHRegion(this.region);
2930 this.region = null;
2931 }
2932 }
2933
2934 @Test
2935 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2936 byte[] row1 = Bytes.toBytes("row1");
2937 byte[] fam1 = Bytes.toBytes("fam1");
2938 byte[][] families = { fam1 };
2939 byte[] qf1 = Bytes.toBytes("qualifier1");
2940 byte[] qf2 = Bytes.toBytes("qualifier2");
2941
2942 long ts1 = 1;
2943 long ts2 = ts1 + 1;
2944 long ts3 = ts1 + 2;
2945 long ts4 = ts1 + 3;
2946
2947
2948 String method = this.getName();
2949 this.region = initHRegion(tableName, method, CONF, families);
2950 try {
2951
2952 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2953 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2954 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2955 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2956
2957 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2958 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2959 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2960 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2961
2962 Put put = null;
2963 put = new Put(row1);
2964 put.add(kv14);
2965 put.add(kv24);
2966 region.put(put);
2967 region.flush(true);
2968
2969 put = new Put(row1);
2970 put.add(kv23);
2971 put.add(kv13);
2972 region.put(put);
2973 region.flush(true);
2974
2975 put = new Put(row1);
2976 put.add(kv22);
2977 put.add(kv12);
2978 region.put(put);
2979 region.flush(true);
2980
2981 put = new Put(row1);
2982 put.add(kv21);
2983 put.add(kv11);
2984 region.put(put);
2985
2986
2987 List<Cell> expected = new ArrayList<Cell>();
2988 expected.add(kv14);
2989 expected.add(kv13);
2990 expected.add(kv12);
2991 expected.add(kv24);
2992 expected.add(kv23);
2993 expected.add(kv22);
2994
2995 Scan scan = new Scan(row1);
2996 scan.addColumn(fam1, qf1);
2997 scan.addColumn(fam1, qf2);
2998 int versions = 3;
2999 scan.setMaxVersions(versions);
3000 List<Cell> actual = new ArrayList<Cell>();
3001 InternalScanner scanner = region.getScanner(scan);
3002
3003 boolean hasNext = scanner.next(actual);
3004 assertEquals(false, hasNext);
3005
3006
3007 for (int i = 0; i < expected.size(); i++) {
3008 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
3009 }
3010 } finally {
3011 HRegion.closeHRegion(this.region);
3012 this.region = null;
3013 }
3014 }
3015
3016 @Test
3017 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
3018 byte[] row1 = Bytes.toBytes("row1");
3019 byte[] qf1 = Bytes.toBytes("qualifier1");
3020 byte[] qf2 = Bytes.toBytes("qualifier2");
3021 byte[] fam1 = Bytes.toBytes("fam1");
3022 byte[][] families = { fam1 };
3023
3024 long ts1 = System.currentTimeMillis();
3025 long ts2 = ts1 + 1;
3026 long ts3 = ts1 + 2;
3027
3028
3029 String method = this.getName();
3030 this.region = initHRegion(tableName, method, CONF, families);
3031 try {
3032
3033 Put put = null;
3034 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
3035 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
3036 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
3037
3038 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
3039 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
3040 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
3041
3042 put = new Put(row1);
3043 put.add(kv13);
3044 put.add(kv12);
3045 put.add(kv11);
3046 put.add(kv23);
3047 put.add(kv22);
3048 put.add(kv21);
3049 region.put(put);
3050
3051
3052 List<Cell> expected = new ArrayList<Cell>();
3053 expected.add(kv13);
3054 expected.add(kv12);
3055 expected.add(kv23);
3056 expected.add(kv22);
3057
3058 Scan scan = new Scan(row1);
3059 scan.addFamily(fam1);
3060 scan.setMaxVersions(MAX_VERSIONS);
3061 List<Cell> actual = new ArrayList<Cell>();
3062 InternalScanner scanner = region.getScanner(scan);
3063
3064 boolean hasNext = scanner.next(actual);
3065 assertEquals(false, hasNext);
3066
3067
3068 for (int i = 0; i < expected.size(); i++) {
3069 assertEquals(expected.get(i), actual.get(i));
3070 }
3071 } finally {
3072 HRegion.closeHRegion(this.region);
3073 this.region = null;
3074 }
3075 }
3076
3077 @Test
3078 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
3079 byte[] row1 = Bytes.toBytes("row1");
3080 byte[] qf1 = Bytes.toBytes("qualifier1");
3081 byte[] qf2 = Bytes.toBytes("qualifier2");
3082 byte[] fam1 = Bytes.toBytes("fam1");
3083
3084 long ts1 = 1;
3085 long ts2 = ts1 + 1;
3086 long ts3 = ts1 + 2;
3087
3088
3089 String method = this.getName();
3090 this.region = initHRegion(tableName, method, CONF, fam1);
3091 try {
3092
3093 Put put = null;
3094 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
3095 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
3096 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
3097
3098 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
3099 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
3100 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
3101
3102 put = new Put(row1);
3103 put.add(kv13);
3104 put.add(kv12);
3105 put.add(kv11);
3106 put.add(kv23);
3107 put.add(kv22);
3108 put.add(kv21);
3109 region.put(put);
3110 region.flush(true);
3111
3112
3113 List<Cell> expected = new ArrayList<Cell>();
3114 expected.add(kv13);
3115 expected.add(kv12);
3116 expected.add(kv23);
3117 expected.add(kv22);
3118
3119 Scan scan = new Scan(row1);
3120 scan.addFamily(fam1);
3121 scan.setMaxVersions(MAX_VERSIONS);
3122 List<Cell> actual = new ArrayList<Cell>();
3123 InternalScanner scanner = region.getScanner(scan);
3124
3125 boolean hasNext = scanner.next(actual);
3126 assertEquals(false, hasNext);
3127
3128
3129 for (int i = 0; i < expected.size(); i++) {
3130 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
3131 }
3132 } finally {
3133 HRegion.closeHRegion(this.region);
3134 this.region = null;
3135 }
3136 }
3137
3138 @Test
3139 public void testScanner_StopRow1542() throws IOException {
3140 byte[] family = Bytes.toBytes("testFamily");
3141 this.region = initHRegion(tableName, getName(), CONF, family);
3142 try {
3143 byte[] row1 = Bytes.toBytes("row111");
3144 byte[] row2 = Bytes.toBytes("row222");
3145 byte[] row3 = Bytes.toBytes("row333");
3146 byte[] row4 = Bytes.toBytes("row444");
3147 byte[] row5 = Bytes.toBytes("row555");
3148
3149 byte[] col1 = Bytes.toBytes("Pub111");
3150 byte[] col2 = Bytes.toBytes("Pub222");
3151
3152 Put put = new Put(row1);
3153 put.add(family, col1, Bytes.toBytes(10L));
3154 region.put(put);
3155
3156 put = new Put(row2);
3157 put.add(family, col1, Bytes.toBytes(15L));
3158 region.put(put);
3159
3160 put = new Put(row3);
3161 put.add(family, col2, Bytes.toBytes(20L));
3162 region.put(put);
3163
3164 put = new Put(row4);
3165 put.add(family, col2, Bytes.toBytes(30L));
3166 region.put(put);
3167
3168 put = new Put(row5);
3169 put.add(family, col1, Bytes.toBytes(40L));
3170 region.put(put);
3171
3172 Scan scan = new Scan(row3, row4);
3173 scan.setMaxVersions();
3174 scan.addColumn(family, col1);
3175 InternalScanner s = region.getScanner(scan);
3176
3177 List<Cell> results = new ArrayList<Cell>();
3178 assertEquals(false, s.next(results));
3179 assertEquals(0, results.size());
3180 } finally {
3181 HRegion.closeHRegion(this.region);
3182 this.region = null;
3183 }
3184 }
3185
3186 @Test
3187 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
3188 byte[] row1 = Bytes.toBytes("row1");
3189 byte[] fam1 = Bytes.toBytes("fam1");
3190 byte[] qf1 = Bytes.toBytes("qualifier1");
3191 byte[] qf2 = Bytes.toBytes("quateslifier2");
3192
3193 long ts1 = 1;
3194 long ts2 = ts1 + 1;
3195 long ts3 = ts1 + 2;
3196 long ts4 = ts1 + 3;
3197
3198
3199 String method = this.getName();
3200 this.region = initHRegion(tableName, method, CONF, fam1);
3201 try {
3202
3203 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
3204 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
3205 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
3206 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
3207
3208 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
3209 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
3210 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
3211 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
3212
3213 Put put = null;
3214 put = new Put(row1);
3215 put.add(kv14);
3216 put.add(kv24);
3217 region.put(put);
3218 region.flush(true);
3219
3220 put = new Put(row1);
3221 put.add(kv23);
3222 put.add(kv13);
3223 region.put(put);
3224 region.flush(true);
3225
3226 put = new Put(row1);
3227 put.add(kv22);
3228 put.add(kv12);
3229 region.put(put);
3230 region.flush(true);
3231
3232 put = new Put(row1);
3233 put.add(kv21);
3234 put.add(kv11);
3235 region.put(put);
3236
3237
3238 List<KeyValue> expected = new ArrayList<KeyValue>();
3239 expected.add(kv14);
3240 expected.add(kv13);
3241 expected.add(kv12);
3242 expected.add(kv24);
3243 expected.add(kv23);
3244 expected.add(kv22);
3245
3246 Scan scan = new Scan(row1);
3247 int versions = 3;
3248 scan.setMaxVersions(versions);
3249 List<Cell> actual = new ArrayList<Cell>();
3250 InternalScanner scanner = region.getScanner(scan);
3251
3252 boolean hasNext = scanner.next(actual);
3253 assertEquals(false, hasNext);
3254
3255
3256 for (int i = 0; i < expected.size(); i++) {
3257 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
3258 }
3259 } finally {
3260 HRegion.closeHRegion(this.region);
3261 this.region = null;
3262 }
3263 }
3264
3265
3266
3267
3268
3269
3270
3271 @Test
3272 public void testScanner_JoinedScanners() throws IOException {
3273 byte[] cf_essential = Bytes.toBytes("essential");
3274 byte[] cf_joined = Bytes.toBytes("joined");
3275 byte[] cf_alpha = Bytes.toBytes("alpha");
3276 this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
3277 try {
3278 byte[] row1 = Bytes.toBytes("row1");
3279 byte[] row2 = Bytes.toBytes("row2");
3280 byte[] row3 = Bytes.toBytes("row3");
3281
3282 byte[] col_normal = Bytes.toBytes("d");
3283 byte[] col_alpha = Bytes.toBytes("a");
3284
3285 byte[] filtered_val = Bytes.toBytes(3);
3286
3287 Put put = new Put(row1);
3288 put.add(cf_essential, col_normal, Bytes.toBytes(1));
3289 put.add(cf_joined, col_alpha, Bytes.toBytes(1));
3290 region.put(put);
3291
3292 put = new Put(row2);
3293 put.add(cf_essential, col_alpha, Bytes.toBytes(2));
3294 put.add(cf_joined, col_normal, Bytes.toBytes(2));
3295 put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
3296 region.put(put);
3297
3298 put = new Put(row3);
3299 put.add(cf_essential, col_normal, filtered_val);
3300 put.add(cf_joined, col_normal, filtered_val);
3301 region.put(put);
3302
3303
3304
3305
3306
3307 Scan scan = new Scan();
3308 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
3309 CompareOp.NOT_EQUAL, filtered_val);
3310 scan.setFilter(filter);
3311 scan.setLoadColumnFamiliesOnDemand(true);
3312 InternalScanner s = region.getScanner(scan);
3313
3314 List<Cell> results = new ArrayList<Cell>();
3315 assertTrue(s.next(results));
3316 assertEquals(results.size(), 1);
3317 results.clear();
3318
3319 assertTrue(s.next(results));
3320 assertEquals(results.size(), 3);
3321 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
3322 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
3323 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
3324 results.clear();
3325
3326 assertFalse(s.next(results));
3327 assertEquals(results.size(), 0);
3328 } finally {
3329 HRegion.closeHRegion(this.region);
3330 this.region = null;
3331 }
3332 }
3333
3334
3335
3336
3337
3338
3339 @Test
3340 public void testScanner_JoinedScannersWithLimits() throws IOException {
3341 final byte[] cf_first = Bytes.toBytes("first");
3342 final byte[] cf_second = Bytes.toBytes("second");
3343
3344 this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
3345 try {
3346 final byte[] col_a = Bytes.toBytes("a");
3347 final byte[] col_b = Bytes.toBytes("b");
3348
3349 Put put;
3350
3351 for (int i = 0; i < 10; i++) {
3352 put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
3353 put.add(cf_first, col_a, Bytes.toBytes(i));
3354 if (i < 5) {
3355 put.add(cf_first, col_b, Bytes.toBytes(i));
3356 put.add(cf_second, col_a, Bytes.toBytes(i));
3357 put.add(cf_second, col_b, Bytes.toBytes(i));
3358 }
3359 region.put(put);
3360 }
3361
3362 Scan scan = new Scan();
3363 scan.setLoadColumnFamiliesOnDemand(true);
3364 Filter bogusFilter = new FilterBase() {
3365 @Override
3366 public ReturnCode filterKeyValue(Cell ignored) throws IOException {
3367 return ReturnCode.INCLUDE;
3368 }
3369 @Override
3370 public boolean isFamilyEssential(byte[] name) {
3371 return Bytes.equals(name, cf_first);
3372 }
3373 };
3374
3375 scan.setFilter(bogusFilter);
3376 InternalScanner s = region.getScanner(scan);
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407 List<Cell> results = new ArrayList<Cell>();
3408 int index = 0;
3409 ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(3).build();
3410 while (true) {
3411 boolean more = s.next(results, scannerContext);
3412 if ((index >> 1) < 5) {
3413 if (index % 2 == 0)
3414 assertEquals(results.size(), 3);
3415 else
3416 assertEquals(results.size(), 1);
3417 } else
3418 assertEquals(results.size(), 1);
3419 results.clear();
3420 index++;
3421 if (!more)
3422 break;
3423 }
3424 } finally {
3425 HRegion.closeHRegion(this.region);
3426 this.region = null;
3427 }
3428 }
3429
3430
3431
3432
3433
3434
3435 @Test
3436 public void testLongQualifier() throws Exception {
3437 String method = name.getMethodName();
3438 TableName tableName = TableName.valueOf(method);
3439 byte[] family = Bytes.toBytes("family");
3440 this.region = initHRegion(tableName, method, CONF, family);
3441 byte[] q = new byte[Short.MAX_VALUE+2];
3442 Arrays.fill(q, 0, q.length-1, (byte)42);
3443 for (byte i=0; i<10; i++) {
3444 Put p = new Put(Bytes.toBytes("row"));
3445
3446 q[q.length-1]=i;
3447 p.addColumn(family, q, q);
3448 region.put(p);
3449 }
3450 region.flush(false);
3451 HRegion.closeHRegion(this.region);
3452 this.region = null;
3453 }
3454
3455
3456
3457
3458
3459
3460
3461
3462 @Test
3463 public void testBasicSplit() throws Exception {
3464 byte[][] families = { fam1, fam2, fam3 };
3465
3466 Configuration hc = initSplit();
3467
3468 String method = this.getName();
3469 this.region = initHRegion(tableName, method, hc, families);
3470
3471 try {
3472 LOG.info("" + HBaseTestCase.addContent(region, fam3));
3473 region.flush(true);
3474 region.compactStores();
3475 byte[] splitRow = region.checkSplit();
3476 assertNotNull(splitRow);
3477 LOG.info("SplitRow: " + Bytes.toString(splitRow));
3478 HRegion[] regions = splitRegion(region, splitRow);
3479 try {
3480
3481
3482
3483 for (int i = 0; i < regions.length; i++) {
3484 regions[i] = HRegion.openHRegion(regions[i], null);
3485 }
3486
3487
3488 assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
3489 assertGet(regions[1], fam3, splitRow);
3490
3491 assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
3492 assertScan(regions[1], fam3, splitRow);
3493
3494 for (int i = 0; i < regions.length; i++) {
3495
3496
3497 for (int j = 0; j < 2; j++) {
3498 HBaseTestCase.addContent(regions[i], fam3);
3499 }
3500 HBaseTestCase.addContent(regions[i], fam2);
3501 HBaseTestCase.addContent(regions[i], fam1);
3502 regions[i].flush(true);
3503 }
3504
3505 byte[][] midkeys = new byte[regions.length][];
3506
3507 for (int i = 0; i < regions.length; i++) {
3508 regions[i].compactStores();
3509 midkeys[i] = regions[i].checkSplit();
3510 }
3511
3512 TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
3513
3514
3515 for (int i = 0; i < regions.length; i++) {
3516 HRegion[] rs = null;
3517 if (midkeys[i] != null) {
3518 rs = splitRegion(regions[i], midkeys[i]);
3519 for (int j = 0; j < rs.length; j++) {
3520 sortedMap.put(Bytes.toString(rs[j].getRegionInfo().getRegionName()),
3521 HRegion.openHRegion(rs[j], null));
3522 }
3523 }
3524 }
3525 LOG.info("Made 4 regions");
3526
3527
3528 int interval = (LAST_CHAR - FIRST_CHAR) / 3;
3529 byte[] b = Bytes.toBytes(START_KEY);
3530 for (HRegion r : sortedMap.values()) {
3531 assertGet(r, fam3, b);
3532 b[0] += interval;
3533 }
3534 } finally {
3535 for (int i = 0; i < regions.length; i++) {
3536 try {
3537 regions[i].close();
3538 } catch (IOException e) {
3539
3540 }
3541 }
3542 }
3543 } finally {
3544 HRegion.closeHRegion(this.region);
3545 this.region = null;
3546 }
3547 }
3548
3549 @Test
3550 public void testSplitRegion() throws IOException {
3551 byte[] qualifier = Bytes.toBytes("qualifier");
3552 Configuration hc = initSplit();
3553 int numRows = 10;
3554 byte[][] families = { fam1, fam3 };
3555
3556
3557 String method = this.getName();
3558 this.region = initHRegion(tableName, method, hc, families);
3559
3560
3561 int startRow = 100;
3562 putData(startRow, numRows, qualifier, families);
3563 int splitRow = startRow + numRows;
3564 putData(splitRow, numRows, qualifier, families);
3565 region.flush(true);
3566
3567 HRegion[] regions = null;
3568 try {
3569 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3570
3571 for (int i = 0; i < regions.length; i++) {
3572 regions[i] = HRegion.openHRegion(regions[i], null);
3573 }
3574
3575 assertEquals(2, regions.length);
3576
3577
3578
3579 verifyData(regions[0], startRow, numRows, qualifier, families);
3580 verifyData(regions[1], splitRow, numRows, qualifier, families);
3581
3582 } finally {
3583 HRegion.closeHRegion(this.region);
3584 this.region = null;
3585 }
3586 }
3587
3588 @Test
3589 public void testClearForceSplit() throws IOException {
3590 byte[] qualifier = Bytes.toBytes("qualifier");
3591 Configuration hc = initSplit();
3592 int numRows = 10;
3593 byte[][] families = { fam1, fam3 };
3594
3595
3596 String method = this.getName();
3597 this.region = initHRegion(tableName, method, hc, families);
3598
3599
3600 int startRow = 100;
3601 putData(startRow, numRows, qualifier, families);
3602 int splitRow = startRow + numRows;
3603 byte[] splitRowBytes = Bytes.toBytes("" + splitRow);
3604 putData(splitRow, numRows, qualifier, families);
3605 region.flush(true);
3606
3607 HRegion[] regions = null;
3608 try {
3609
3610 region.forceSplit(splitRowBytes);
3611 assertTrue(region.shouldForceSplit());
3612
3613 assertTrue(Bytes.equals(splitRowBytes, region.checkSplit()));
3614
3615
3616 HStore storeMock = Mockito.mock(HStore.class);
3617 when(storeMock.hasReferences()).thenReturn(true);
3618 when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
3619 when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
3620 when(storeMock.getColumnFamilyName()).thenReturn("cf");
3621 region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock);
3622 assertTrue(region.hasReferences());
3623
3624
3625 regions = splitRegion(region, splitRowBytes);
3626 assertNull(regions);
3627
3628
3629 assertFalse(region.shouldForceSplit());
3630
3631
3632 region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName()));
3633 assertFalse(region.hasReferences());
3634
3635
3636 regions = splitRegion(region, splitRowBytes);
3637
3638
3639 for (int i = 0; i < regions.length; i++) {
3640 regions[i] = HRegion.openHRegion(regions[i], null);
3641 }
3642
3643 assertEquals(2, regions.length);
3644
3645
3646
3647 verifyData(regions[0], startRow, numRows, qualifier, families);
3648 verifyData(regions[1], splitRow, numRows, qualifier, families);
3649
3650 } finally {
3651 HRegion.closeHRegion(this.region);
3652 this.region = null;
3653 }
3654 }
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666 @Test
3667 public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3668 byte[] family = Bytes.toBytes("family");
3669 int numRows = 1000;
3670 int flushAndScanInterval = 10;
3671 int compactInterval = 10 * flushAndScanInterval;
3672
3673 String method = "testFlushCacheWhileScanning";
3674 this.region = initHRegion(tableName, method, CONF, family);
3675 try {
3676 FlushThread flushThread = new FlushThread();
3677 flushThread.start();
3678
3679 Scan scan = new Scan();
3680 scan.addFamily(family);
3681 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3682 new BinaryComparator(Bytes.toBytes(5L))));
3683
3684 int expectedCount = 0;
3685 List<Cell> res = new ArrayList<Cell>();
3686
3687 boolean toggle = true;
3688 for (long i = 0; i < numRows; i++) {
3689 Put put = new Put(Bytes.toBytes(i));
3690 put.setDurability(Durability.SKIP_WAL);
3691 put.add(family, qual1, Bytes.toBytes(i % 10));
3692 region.put(put);
3693
3694 if (i != 0 && i % compactInterval == 0) {
3695
3696 region.compact(true);
3697 }
3698
3699 if (i % 10 == 5L) {
3700 expectedCount++;
3701 }
3702
3703 if (i != 0 && i % flushAndScanInterval == 0) {
3704 res.clear();
3705 InternalScanner scanner = region.getScanner(scan);
3706 if (toggle) {
3707 flushThread.flush();
3708 }
3709 while (scanner.next(res))
3710 ;
3711 if (!toggle) {
3712 flushThread.flush();
3713 }
3714 assertEquals("i=" + i, expectedCount, res.size());
3715 toggle = !toggle;
3716 }
3717 }
3718
3719 flushThread.done();
3720 flushThread.join();
3721 flushThread.checkNoError();
3722 } finally {
3723 HRegion.closeHRegion(this.region);
3724 this.region = null;
3725 }
3726 }
3727
3728 protected class FlushThread extends Thread {
3729 private volatile boolean done;
3730 private Throwable error = null;
3731
3732 public void done() {
3733 done = true;
3734 synchronized (this) {
3735 interrupt();
3736 }
3737 }
3738
3739 public void checkNoError() {
3740 if (error != null) {
3741 assertNull(error);
3742 }
3743 }
3744
3745 @Override
3746 public void run() {
3747 done = false;
3748 while (!done) {
3749 synchronized (this) {
3750 try {
3751 wait();
3752 } catch (InterruptedException ignored) {
3753 if (done) {
3754 break;
3755 }
3756 }
3757 }
3758 try {
3759 region.flush(true);
3760 } catch (IOException e) {
3761 if (!done) {
3762 LOG.error("Error while flusing cache", e);
3763 error = e;
3764 }
3765 break;
3766 }
3767 }
3768
3769 }
3770
3771 public void flush() {
3772 synchronized (this) {
3773 notify();
3774 }
3775
3776 }
3777 }
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788 @Test
3789 public void testWritesWhileScanning() throws IOException, InterruptedException {
3790 int testCount = 100;
3791 int numRows = 1;
3792 int numFamilies = 10;
3793 int numQualifiers = 100;
3794 int flushInterval = 7;
3795 int compactInterval = 5 * flushInterval;
3796 byte[][] families = new byte[numFamilies][];
3797 for (int i = 0; i < numFamilies; i++) {
3798 families[i] = Bytes.toBytes("family" + i);
3799 }
3800 byte[][] qualifiers = new byte[numQualifiers][];
3801 for (int i = 0; i < numQualifiers; i++) {
3802 qualifiers[i] = Bytes.toBytes("qual" + i);
3803 }
3804
3805 String method = "testWritesWhileScanning";
3806 this.region = initHRegion(tableName, method, CONF, families);
3807 try {
3808 PutThread putThread = new PutThread(numRows, families, qualifiers);
3809 putThread.start();
3810 putThread.waitForFirstPut();
3811
3812 FlushThread flushThread = new FlushThread();
3813 flushThread.start();
3814
3815 Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3816
3817 int expectedCount = numFamilies * numQualifiers;
3818 List<Cell> res = new ArrayList<Cell>();
3819
3820 long prevTimestamp = 0L;
3821 for (int i = 0; i < testCount; i++) {
3822
3823 if (i != 0 && i % compactInterval == 0) {
3824 region.compact(true);
3825 }
3826
3827 if (i != 0 && i % flushInterval == 0) {
3828 flushThread.flush();
3829 }
3830
3831 boolean previousEmpty = res.isEmpty();
3832 res.clear();
3833 InternalScanner scanner = region.getScanner(scan);
3834 while (scanner.next(res))
3835 ;
3836 if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3837 assertEquals("i=" + i, expectedCount, res.size());
3838 long timestamp = res.get(0).getTimestamp();
3839 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3840 timestamp >= prevTimestamp);
3841 prevTimestamp = timestamp;
3842 }
3843 }
3844
3845 putThread.done();
3846
3847 region.flush(true);
3848
3849 putThread.join();
3850 putThread.checkNoError();
3851
3852 flushThread.done();
3853 flushThread.join();
3854 flushThread.checkNoError();
3855 } finally {
3856 try {
3857 HRegion.closeHRegion(this.region);
3858 } catch (DroppedSnapshotException dse) {
3859
3860
3861
3862
3863 }
3864 this.region = null;
3865 }
3866 }
3867
3868 protected class PutThread extends Thread {
3869 private volatile boolean done;
3870 private volatile int numPutsFinished = 0;
3871
3872 private Throwable error = null;
3873 private int numRows;
3874 private byte[][] families;
3875 private byte[][] qualifiers;
3876
3877 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3878 this.numRows = numRows;
3879 this.families = families;
3880 this.qualifiers = qualifiers;
3881 }
3882
3883
3884
3885
3886 public void waitForFirstPut() throws InterruptedException {
3887
3888 while (numPutsFinished == 0) {
3889 checkNoError();
3890 Thread.sleep(50);
3891 }
3892 }
3893
3894 public void done() {
3895 done = true;
3896 synchronized (this) {
3897 interrupt();
3898 }
3899 }
3900
3901 public void checkNoError() {
3902 if (error != null) {
3903 assertNull(error);
3904 }
3905 }
3906
3907 @Override
3908 public void run() {
3909 done = false;
3910 while (!done) {
3911 try {
3912 for (int r = 0; r < numRows; r++) {
3913 byte[] row = Bytes.toBytes("row" + r);
3914 Put put = new Put(row);
3915 put.setDurability(Durability.SKIP_WAL);
3916 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3917 for (byte[] family : families) {
3918 for (byte[] qualifier : qualifiers) {
3919 put.add(family, qualifier, (long) numPutsFinished, value);
3920 }
3921 }
3922 region.put(put);
3923 numPutsFinished++;
3924 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3925 System.out.println("put iteration = " + numPutsFinished);
3926 Delete delete = new Delete(row, (long) numPutsFinished - 30);
3927 region.delete(delete);
3928 }
3929 numPutsFinished++;
3930 }
3931 } catch (InterruptedIOException e) {
3932
3933 } catch (IOException e) {
3934 LOG.error("error while putting records", e);
3935 error = e;
3936 break;
3937 }
3938 }
3939
3940 }
3941
3942 }
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953 @Test
3954 public void testWritesWhileGetting() throws Exception {
3955 int testCount = 100;
3956 int numRows = 1;
3957 int numFamilies = 10;
3958 int numQualifiers = 100;
3959 int compactInterval = 100;
3960 byte[][] families = new byte[numFamilies][];
3961 for (int i = 0; i < numFamilies; i++) {
3962 families[i] = Bytes.toBytes("family" + i);
3963 }
3964 byte[][] qualifiers = new byte[numQualifiers][];
3965 for (int i = 0; i < numQualifiers; i++) {
3966 qualifiers[i] = Bytes.toBytes("qual" + i);
3967 }
3968
3969
3970 String method = "testWritesWhileGetting";
3971
3972
3973
3974
3975
3976 Configuration conf = HBaseConfiguration.create(CONF);
3977 conf.setInt("hbase.hstore.compaction.min", 1);
3978 conf.setInt("hbase.hstore.compaction.max", 1000);
3979 this.region = initHRegion(tableName, method, conf, families);
3980 PutThread putThread = null;
3981 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3982 try {
3983 putThread = new PutThread(numRows, families, qualifiers);
3984 putThread.start();
3985 putThread.waitForFirstPut();
3986
3987
3988 ctx.addThread(new RepeatingTestThread(ctx) {
3989 private int flushesSinceCompact = 0;
3990 private final int maxFlushesSinceCompact = 20;
3991
3992 @Override
3993 public void doAnAction() throws Exception {
3994 if (region.flush(true).isCompactionNeeded()) {
3995 ++flushesSinceCompact;
3996 }
3997
3998
3999 if (flushesSinceCompact == maxFlushesSinceCompact) {
4000 region.compact(false);
4001 flushesSinceCompact = 0;
4002 }
4003 }
4004 });
4005 ctx.startThreads();
4006
4007 Get get = new Get(Bytes.toBytes("row0"));
4008 Result result = null;
4009
4010 int expectedCount = numFamilies * numQualifiers;
4011
4012 long prevTimestamp = 0L;
4013 for (int i = 0; i < testCount; i++) {
4014
4015 boolean previousEmpty = result == null || result.isEmpty();
4016 result = region.get(get);
4017 if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
4018 assertEquals("i=" + i, expectedCount, result.size());
4019
4020
4021 long timestamp = 0;
4022 for (Cell kv : result.rawCells()) {
4023 if (CellUtil.matchingFamily(kv, families[0])
4024 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
4025 timestamp = kv.getTimestamp();
4026 }
4027 }
4028 assertTrue(timestamp >= prevTimestamp);
4029 prevTimestamp = timestamp;
4030 Cell previousKV = null;
4031
4032 for (Cell kv : result.rawCells()) {
4033 byte[] thisValue = CellUtil.cloneValue(kv);
4034 if (previousKV != null) {
4035 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
4036 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
4037 + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
4038 + "(memStoreTS:" + kv.getMvccVersion() + ")");
4039 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
4040 }
4041 }
4042 previousKV = kv;
4043 }
4044 }
4045 }
4046 } finally {
4047 if (putThread != null)
4048 putThread.done();
4049
4050 region.flush(true);
4051
4052 if (putThread != null) {
4053 putThread.join();
4054 putThread.checkNoError();
4055 }
4056
4057 ctx.stop();
4058 HRegion.closeHRegion(this.region);
4059 this.region = null;
4060 }
4061 }
4062
4063 @Test
4064 public void testHolesInMeta() throws Exception {
4065 byte[] family = Bytes.toBytes("family");
4066 this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
4067 false, family);
4068 try {
4069 byte[] rowNotServed = Bytes.toBytes("a");
4070 Get g = new Get(rowNotServed);
4071 try {
4072 region.get(g);
4073 fail();
4074 } catch (WrongRegionException x) {
4075
4076 }
4077 byte[] row = Bytes.toBytes("y");
4078 g = new Get(row);
4079 region.get(g);
4080 } finally {
4081 HRegion.closeHRegion(this.region);
4082 this.region = null;
4083 }
4084 }
4085
4086 @Test
4087 public void testIndexesScanWithOneDeletedRow() throws IOException {
4088 byte[] family = Bytes.toBytes("family");
4089
4090
4091 String method = "testIndexesScanWithOneDeletedRow";
4092 this.region = initHRegion(tableName, method, CONF, family);
4093 try {
4094 Put put = new Put(Bytes.toBytes(1L));
4095 put.add(family, qual1, 1L, Bytes.toBytes(1L));
4096 region.put(put);
4097
4098 region.flush(true);
4099
4100 Delete delete = new Delete(Bytes.toBytes(1L), 1L);
4101 region.delete(delete);
4102
4103 put = new Put(Bytes.toBytes(2L));
4104 put.add(family, qual1, 2L, Bytes.toBytes(2L));
4105 region.put(put);
4106
4107 Scan idxScan = new Scan();
4108 idxScan.addFamily(family);
4109 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
4110 new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
4111 new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
4112 CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
4113 InternalScanner scanner = region.getScanner(idxScan);
4114 List<Cell> res = new ArrayList<Cell>();
4115
4116 while (scanner.next(res))
4117 ;
4118 assertEquals(1L, res.size());
4119 } finally {
4120 HRegion.closeHRegion(this.region);
4121 this.region = null;
4122 }
4123 }
4124
4125
4126
4127
4128 @Test
4129 public void testBloomFilterSize() throws IOException {
4130 byte[] fam1 = Bytes.toBytes("fam1");
4131 byte[] qf1 = Bytes.toBytes("col");
4132 byte[] val1 = Bytes.toBytes("value1");
4133
4134 HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
4135 .setBloomFilterType(BloomType.ROWCOL);
4136
4137 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
4138 htd.addFamily(hcd);
4139 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
4140 this.region = TEST_UTIL.createLocalHRegion(info, htd);
4141 try {
4142 int num_unique_rows = 10;
4143 int duplicate_multiplier = 2;
4144 int num_storefiles = 4;
4145
4146 int version = 0;
4147 for (int f = 0; f < num_storefiles; f++) {
4148 for (int i = 0; i < duplicate_multiplier; i++) {
4149 for (int j = 0; j < num_unique_rows; j++) {
4150 Put put = new Put(Bytes.toBytes("row" + j));
4151 put.setDurability(Durability.SKIP_WAL);
4152 put.add(fam1, qf1, version++, val1);
4153 region.put(put);
4154 }
4155 }
4156 region.flush(true);
4157 }
4158
4159 HStore store = (HStore) region.getStore(fam1);
4160 Collection<StoreFile> storeFiles = store.getStorefiles();
4161 for (StoreFile storefile : storeFiles) {
4162 StoreFile.Reader reader = storefile.getReader();
4163 reader.loadFileInfo();
4164 reader.loadBloomfilter();
4165 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
4166 assertEquals(num_unique_rows, reader.getFilterEntries());
4167 }
4168
4169 region.compact(true);
4170
4171
4172 storeFiles = store.getStorefiles();
4173 for (StoreFile storefile : storeFiles) {
4174 StoreFile.Reader reader = storefile.getReader();
4175 reader.loadFileInfo();
4176 reader.loadBloomfilter();
4177 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
4178 assertEquals(num_unique_rows, reader.getFilterEntries());
4179 }
4180 } finally {
4181 HRegion.closeHRegion(this.region);
4182 this.region = null;
4183 }
4184 }
4185
4186 @Test
4187 public void testAllColumnsWithBloomFilter() throws IOException {
4188 byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
4189 byte[] FAMILY = Bytes.toBytes("family");
4190
4191
4192 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
4193 .setBloomFilterType(BloomType.ROWCOL);
4194 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
4195 htd.addFamily(hcd);
4196 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
4197 this.region = TEST_UTIL.createLocalHRegion(info, htd);
4198 try {
4199
4200 byte row[] = Bytes.toBytes("row:" + 0);
4201 byte column[] = Bytes.toBytes("column:" + 0);
4202 Put put = new Put(row);
4203 put.setDurability(Durability.SKIP_WAL);
4204 for (long idx = 1; idx <= 4; idx++) {
4205 put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
4206 }
4207 region.put(put);
4208
4209
4210 region.flush(true);
4211
4212
4213 Get get = new Get(row);
4214 get.setMaxVersions();
4215 Cell[] kvs = region.get(get).rawCells();
4216
4217
4218 assertEquals(4, kvs.length);
4219 checkOneCell(kvs[0], FAMILY, 0, 0, 4);
4220 checkOneCell(kvs[1], FAMILY, 0, 0, 3);
4221 checkOneCell(kvs[2], FAMILY, 0, 0, 2);
4222 checkOneCell(kvs[3], FAMILY, 0, 0, 1);
4223 } finally {
4224 HRegion.closeHRegion(this.region);
4225 this.region = null;
4226 }
4227 }
4228
4229
4230
4231
4232
4233
4234 @Test
4235 public void testDeleteRowWithBloomFilter() throws IOException {
4236 byte[] familyName = Bytes.toBytes("familyName");
4237
4238
4239 HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
4240 .setBloomFilterType(BloomType.ROWCOL);
4241
4242 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
4243 htd.addFamily(hcd);
4244 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
4245 this.region = TEST_UTIL.createLocalHRegion(info, htd);
4246 try {
4247
4248 byte row[] = Bytes.toBytes("row1");
4249 byte col[] = Bytes.toBytes("col1");
4250
4251 Put put = new Put(row);
4252 put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
4253 region.put(put);
4254 region.flush(true);
4255
4256 Delete del = new Delete(row);
4257 region.delete(del);
4258 region.flush(true);
4259
4260
4261 Get get = new Get(row);
4262 get.addColumn(familyName, col);
4263
4264 Cell[] keyValues = region.get(get).rawCells();
4265 assertTrue(keyValues.length == 0);
4266 } finally {
4267 HRegion.closeHRegion(this.region);
4268 this.region = null;
4269 }
4270 }
4271
4272 @Test
4273 public void testgetHDFSBlocksDistribution() throws Exception {
4274 HBaseTestingUtility htu = new HBaseTestingUtility();
4275
4276
4277
4278
4279
4280
4281
4282 htu.getConfiguration().setInt("dfs.replication", 2);
4283
4284
4285 MiniHBaseCluster cluster = null;
4286 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
4287 int regionServersCount = 3;
4288
4289 try {
4290 cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
4291 byte[][] families = { fam1, fam2 };
4292 Table ht = htu.createTable(Bytes.toBytes(this.getName()), families);
4293
4294
4295 byte row[] = Bytes.toBytes("row1");
4296 byte col[] = Bytes.toBytes("col1");
4297
4298 Put put = new Put(row);
4299 put.add(fam1, col, 1, Bytes.toBytes("test1"));
4300 put.add(fam2, col, 1, Bytes.toBytes("test2"));
4301 ht.put(put);
4302
4303 HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
4304 .get(0);
4305 firstRegion.flush(true);
4306 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
4307
4308
4309
4310
4311
4312 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
4313 StringBuilder sb = new StringBuilder();
4314 for (String host: blocksDistribution1.getTopHosts()) {
4315 if (sb.length() > 0) sb.append(", ");
4316 sb.append(host);
4317 sb.append("=");
4318 sb.append(blocksDistribution1.getWeight(host));
4319 }
4320
4321 String topHost = blocksDistribution1.getTopHosts().get(0);
4322 long topHostWeight = blocksDistribution1.getWeight(topHost);
4323 String msg = "uniqueBlocksWeight=" + uniqueBlocksWeight1 + ", topHostWeight=" +
4324 topHostWeight + ", topHost=" + topHost + "; " + sb.toString();
4325 LOG.info(msg);
4326 assertTrue(msg, uniqueBlocksWeight1 == topHostWeight);
4327
4328
4329
4330 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
4331 htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
4332 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
4333
4334 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
4335
4336 ht.close();
4337 } finally {
4338 if (cluster != null) {
4339 htu.shutdownMiniCluster();
4340 }
4341 }
4342 }
4343
4344
4345
4346
4347
4348
4349
4350 @Test
4351 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
4352 TableName tableName = TableName.valueOf(name.getMethodName());
4353 HRegionInfo info = null;
4354 try {
4355 FileSystem fs = Mockito.mock(FileSystem.class);
4356 Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
4357 HTableDescriptor htd = new HTableDescriptor(tableName);
4358 htd.addFamily(new HColumnDescriptor("cf"));
4359 info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
4360 HConstants.EMPTY_BYTE_ARRAY, false);
4361 Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
4362 region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
4363
4364 region.initialize();
4365 fail("Region initialization should fail due to IOException");
4366 } catch (IOException io) {
4367 List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
4368 for (MonitoredTask monitoredTask : tasks) {
4369 if (!(monitoredTask instanceof MonitoredRPCHandler)
4370 && monitoredTask.getDescription().contains(region.toString())) {
4371 assertTrue("Region state should be ABORTED.",
4372 monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
4373 break;
4374 }
4375 }
4376 } finally {
4377 HRegion.closeHRegion(region);
4378 }
4379 }
4380
4381
4382
4383
4384
4385 @Test
4386 public void testRegionInfoFileCreation() throws IOException {
4387 Path rootDir = new Path(dir + "testRegionInfoFileCreation");
4388
4389 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
4390 htd.addFamily(new HColumnDescriptor("cf"));
4391
4392 HRegionInfo hri = new HRegionInfo(htd.getTableName());
4393
4394
4395 HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
4396
4397 Path regionDir = region.getRegionFileSystem().getRegionDir();
4398 FileSystem fs = region.getRegionFileSystem().getFileSystem();
4399 HRegion.closeHRegion(region);
4400
4401 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
4402
4403
4404 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4405 fs.exists(regionInfoFile));
4406
4407
4408 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
4409 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
4410 HRegion.closeHRegion(region);
4411
4412
4413 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4414 fs.exists(regionInfoFile));
4415
4416
4417 fs.delete(regionInfoFile, true);
4418 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
4419 fs.exists(regionInfoFile));
4420
4421 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
4422
4423 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
4424 HRegion.closeHRegion(region);
4425
4426
4427 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4428 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
4429 }
4430
4431
4432
4433
4434 private static class Incrementer implements Runnable {
4435 private HRegion region;
4436 private final static byte[] incRow = Bytes.toBytes("incRow");
4437 private final static byte[] family = Bytes.toBytes("family");
4438 private final static byte[] qualifier = Bytes.toBytes("qualifier");
4439 private final static long ONE = 1l;
4440 private int incCounter;
4441
4442 public Incrementer(HRegion region, int incCounter) {
4443 this.region = region;
4444 this.incCounter = incCounter;
4445 }
4446
4447 @Override
4448 public void run() {
4449 int count = 0;
4450 while (count < incCounter) {
4451 Increment inc = new Increment(incRow);
4452 inc.addColumn(family, qualifier, ONE);
4453 count++;
4454 try {
4455 region.increment(inc);
4456 } catch (IOException e) {
4457 e.printStackTrace();
4458 break;
4459 }
4460 }
4461 }
4462 }
4463
4464
4465
4466
4467
4468 @Test
4469 public void testParallelIncrementWithMemStoreFlush() throws Exception {
4470 byte[] family = Incrementer.family;
4471 this.region = initHRegion(tableName, method, CONF, family);
4472 final HRegion region = this.region;
4473 final AtomicBoolean incrementDone = new AtomicBoolean(false);
4474 Runnable flusher = new Runnable() {
4475 @Override
4476 public void run() {
4477 while (!incrementDone.get()) {
4478 try {
4479 region.flush(true);
4480 } catch (Exception e) {
4481 e.printStackTrace();
4482 }
4483 }
4484 }
4485 };
4486
4487
4488 int threadNum = 20;
4489 int incCounter = 100;
4490 long expected = threadNum * incCounter;
4491 Thread[] incrementers = new Thread[threadNum];
4492 Thread flushThread = new Thread(flusher);
4493 for (int i = 0; i < threadNum; i++) {
4494 incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
4495 incrementers[i].start();
4496 }
4497 flushThread.start();
4498 for (int i = 0; i < threadNum; i++) {
4499 incrementers[i].join();
4500 }
4501
4502 incrementDone.set(true);
4503 flushThread.join();
4504
4505 Get get = new Get(Incrementer.incRow);
4506 get.addColumn(Incrementer.family, Incrementer.qualifier);
4507 get.setMaxVersions(1);
4508 Result res = this.region.get(get);
4509 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
4510
4511
4512 assertEquals(kvs.size(), 1);
4513 Cell kv = kvs.get(0);
4514 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
4515 this.region = null;
4516 }
4517
4518
4519
4520
4521 private static class Appender implements Runnable {
4522 private HRegion region;
4523 private final static byte[] appendRow = Bytes.toBytes("appendRow");
4524 private final static byte[] family = Bytes.toBytes("family");
4525 private final static byte[] qualifier = Bytes.toBytes("qualifier");
4526 private final static byte[] CHAR = Bytes.toBytes("a");
4527 private int appendCounter;
4528
4529 public Appender(HRegion region, int appendCounter) {
4530 this.region = region;
4531 this.appendCounter = appendCounter;
4532 }
4533
4534 @Override
4535 public void run() {
4536 int count = 0;
4537 while (count < appendCounter) {
4538 Append app = new Append(appendRow);
4539 app.add(family, qualifier, CHAR);
4540 count++;
4541 try {
4542 region.append(app);
4543 } catch (IOException e) {
4544 e.printStackTrace();
4545 break;
4546 }
4547 }
4548 }
4549 }
4550
4551
4552
4553
4554
4555 @Test
4556 public void testParallelAppendWithMemStoreFlush() throws Exception {
4557 byte[] family = Appender.family;
4558 this.region = initHRegion(tableName, method, CONF, family);
4559 final HRegion region = this.region;
4560 final AtomicBoolean appendDone = new AtomicBoolean(false);
4561 Runnable flusher = new Runnable() {
4562 @Override
4563 public void run() {
4564 while (!appendDone.get()) {
4565 try {
4566 region.flush(true);
4567 } catch (Exception e) {
4568 e.printStackTrace();
4569 }
4570 }
4571 }
4572 };
4573
4574
4575
4576 int threadNum = 20;
4577 int appendCounter = 100;
4578 byte[] expected = new byte[threadNum * appendCounter];
4579 for (int i = 0; i < threadNum * appendCounter; i++) {
4580 System.arraycopy(Appender.CHAR, 0, expected, i, 1);
4581 }
4582 Thread[] appenders = new Thread[threadNum];
4583 Thread flushThread = new Thread(flusher);
4584 for (int i = 0; i < threadNum; i++) {
4585 appenders[i] = new Thread(new Appender(this.region, appendCounter));
4586 appenders[i].start();
4587 }
4588 flushThread.start();
4589 for (int i = 0; i < threadNum; i++) {
4590 appenders[i].join();
4591 }
4592
4593 appendDone.set(true);
4594 flushThread.join();
4595
4596 Get get = new Get(Appender.appendRow);
4597 get.addColumn(Appender.family, Appender.qualifier);
4598 get.setMaxVersions(1);
4599 Result res = this.region.get(get);
4600 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
4601
4602
4603 assertEquals(kvs.size(), 1);
4604 Cell kv = kvs.get(0);
4605 byte[] appendResult = new byte[kv.getValueLength()];
4606 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
4607 assertArrayEquals(expected, appendResult);
4608 this.region = null;
4609 }
4610
4611
4612
4613
4614
4615 @Test
4616 public void testPutWithMemStoreFlush() throws Exception {
4617 byte[] family = Bytes.toBytes("family");
4618 ;
4619 byte[] qualifier = Bytes.toBytes("qualifier");
4620 byte[] row = Bytes.toBytes("putRow");
4621 byte[] value = null;
4622 this.region = initHRegion(tableName, method, CONF, family);
4623 Put put = null;
4624 Get get = null;
4625 List<Cell> kvs = null;
4626 Result res = null;
4627
4628 put = new Put(row);
4629 value = Bytes.toBytes("value0");
4630 put.add(family, qualifier, 1234567l, value);
4631 region.put(put);
4632 get = new Get(row);
4633 get.addColumn(family, qualifier);
4634 get.setMaxVersions();
4635 res = this.region.get(get);
4636 kvs = res.getColumnCells(family, qualifier);
4637 assertEquals(1, kvs.size());
4638 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4639
4640 region.flush(true);
4641 get = new Get(row);
4642 get.addColumn(family, qualifier);
4643 get.setMaxVersions();
4644 res = this.region.get(get);
4645 kvs = res.getColumnCells(family, qualifier);
4646 assertEquals(1, kvs.size());
4647 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4648
4649 put = new Put(row);
4650 value = Bytes.toBytes("value1");
4651 put.add(family, qualifier, 1234567l, value);
4652 region.put(put);
4653 get = new Get(row);
4654 get.addColumn(family, qualifier);
4655 get.setMaxVersions();
4656 res = this.region.get(get);
4657 kvs = res.getColumnCells(family, qualifier);
4658 assertEquals(1, kvs.size());
4659 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4660
4661 region.flush(true);
4662 get = new Get(row);
4663 get.addColumn(family, qualifier);
4664 get.setMaxVersions();
4665 res = this.region.get(get);
4666 kvs = res.getColumnCells(family, qualifier);
4667 assertEquals(1, kvs.size());
4668 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4669 }
4670
4671 @Test
4672 public void testDurability() throws Exception {
4673 String method = "testDurability";
4674
4675
4676
4677
4678
4679 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4680 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4681 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4682
4683 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4684 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4685 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4686
4687 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4688 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4689
4690 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4691 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4692
4693 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4694 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4695 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4696
4697
4698 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4699 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4700 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4701 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4702 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4703 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4704
4705 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4706 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4707 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4708 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4709 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4710 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4711
4712
4713 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, true, false, false);
4714 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, true, false, false);
4715 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, true, false, false);
4716 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, true, false, false);
4717 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, true, false, false);
4718 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4719
4720 }
4721
4722 @SuppressWarnings("unchecked")
4723 private void durabilityTest(String method, Durability tableDurability,
4724 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4725 final boolean expectSyncFromLogSyncer) throws Exception {
4726 Configuration conf = HBaseConfiguration.create(CONF);
4727 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4728 TableName tableName = TableName.valueOf(method);
4729 byte[] family = Bytes.toBytes("family");
4730 Path logDir = new Path(new Path(dir + method), "log");
4731 final Configuration walConf = new Configuration(conf);
4732 FSUtils.setRootDir(walConf, logDir);
4733 final WALFactory wals = new WALFactory(walConf, null, UUID.randomUUID().toString());
4734 final WAL wal = spy(wals.getWAL(tableName.getName()));
4735 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4736 HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, wal,
4737 new byte[][] { family });
4738
4739 Put put = new Put(Bytes.toBytes("r1"));
4740 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4741 put.setDurability(mutationDurability);
4742 region.put(put);
4743
4744
4745 verify(wal, expectAppend ? times(1) : never())
4746 .append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any(),
4747 (WALEdit)any(), (AtomicLong)any(), Mockito.anyBoolean(), (List<Cell>)any());
4748
4749
4750 if (expectSync || expectSyncFromLogSyncer) {
4751 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4752 @Override
4753 public boolean evaluate() throws Exception {
4754 try {
4755 if (expectSync) {
4756 verify(wal, times(1)).sync(anyLong());
4757 } else if (expectSyncFromLogSyncer) {
4758 verify(wal, times(1)).sync();
4759 }
4760 } catch (Throwable ignore) {
4761 }
4762 return true;
4763 }
4764 });
4765 } else {
4766
4767 verify(wal, never()).sync();
4768 }
4769
4770 HRegion.closeHRegion(this.region);
4771 this.region = null;
4772 }
4773
4774 @Test
4775 public void testRegionReplicaSecondary() throws IOException {
4776
4777
4778 Path rootDir = new Path(dir + "testRegionReplicaSecondary");
4779 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
4780
4781 byte[][] families = new byte[][] {
4782 Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")
4783 };
4784 byte[] cq = Bytes.toBytes("cq");
4785 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionReplicaSecondary"));
4786 for (byte[] family : families) {
4787 htd.addFamily(new HColumnDescriptor(family));
4788 }
4789
4790 long time = System.currentTimeMillis();
4791 HRegionInfo primaryHri = new HRegionInfo(htd.getTableName(),
4792 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4793 false, time, 0);
4794 HRegionInfo secondaryHri = new HRegionInfo(htd.getTableName(),
4795 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4796 false, time, 1);
4797
4798 HRegion primaryRegion = null, secondaryRegion = null;
4799
4800 try {
4801 primaryRegion = HRegion.createHRegion(primaryHri,
4802 rootDir, TEST_UTIL.getConfiguration(), htd);
4803
4804
4805 putData(primaryRegion, 0, 1000, cq, families);
4806
4807
4808 primaryRegion.flush(true);
4809
4810
4811 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, htd, null, CONF);
4812
4813 verifyData(secondaryRegion, 0, 1000, cq, families);
4814 } finally {
4815 if (primaryRegion != null) {
4816 HRegion.closeHRegion(primaryRegion);
4817 }
4818 if (secondaryRegion != null) {
4819 HRegion.closeHRegion(secondaryRegion);
4820 }
4821 }
4822 }
4823
4824 @Test
4825 public void testRegionReplicaSecondaryIsReadOnly() throws IOException {
4826
4827
4828 Path rootDir = new Path(dir + "testRegionReplicaSecondary");
4829 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
4830
4831 byte[][] families = new byte[][] {
4832 Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")
4833 };
4834 byte[] cq = Bytes.toBytes("cq");
4835 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionReplicaSecondary"));
4836 for (byte[] family : families) {
4837 htd.addFamily(new HColumnDescriptor(family));
4838 }
4839
4840 long time = System.currentTimeMillis();
4841 HRegionInfo primaryHri = new HRegionInfo(htd.getTableName(),
4842 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4843 false, time, 0);
4844 HRegionInfo secondaryHri = new HRegionInfo(htd.getTableName(),
4845 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4846 false, time, 1);
4847
4848 HRegion primaryRegion = null, secondaryRegion = null;
4849
4850 try {
4851 primaryRegion = HRegion.createHRegion(primaryHri,
4852 rootDir, TEST_UTIL.getConfiguration(), htd);
4853
4854
4855 putData(primaryRegion, 0, 1000, cq, families);
4856
4857
4858 primaryRegion.flush(true);
4859
4860
4861 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, htd, null, CONF);
4862
4863 try {
4864 putData(secondaryRegion, 0, 1000, cq, families);
4865 fail("Should have thrown exception");
4866 } catch (IOException ex) {
4867
4868 }
4869 } finally {
4870 if (primaryRegion != null) {
4871 HRegion.closeHRegion(primaryRegion);
4872 }
4873 if (secondaryRegion != null) {
4874 HRegion.closeHRegion(secondaryRegion);
4875 }
4876 }
4877 }
4878
4879 static WALFactory createWALFactory(Configuration conf, Path rootDir) throws IOException {
4880 Configuration confForWAL = new Configuration(conf);
4881 confForWAL.set(HConstants.HBASE_DIR, rootDir.toString());
4882 return new WALFactory(confForWAL,
4883 Collections.<WALActionsListener>singletonList(new MetricsWAL()),
4884 "hregion-" + RandomStringUtils.randomNumeric(8));
4885 }
4886
4887 @Test
4888 public void testCompactionFromPrimary() throws IOException {
4889 Path rootDir = new Path(dir + "testRegionReplicaSecondary");
4890 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
4891
4892 byte[][] families = new byte[][] {
4893 Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")
4894 };
4895 byte[] cq = Bytes.toBytes("cq");
4896 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionReplicaSecondary"));
4897 for (byte[] family : families) {
4898 htd.addFamily(new HColumnDescriptor(family));
4899 }
4900
4901 long time = System.currentTimeMillis();
4902 HRegionInfo primaryHri = new HRegionInfo(htd.getTableName(),
4903 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4904 false, time, 0);
4905 HRegionInfo secondaryHri = new HRegionInfo(htd.getTableName(),
4906 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4907 false, time, 1);
4908
4909 HRegion primaryRegion = null, secondaryRegion = null;
4910
4911 try {
4912 primaryRegion = HRegion.createHRegion(primaryHri,
4913 rootDir, TEST_UTIL.getConfiguration(), htd);
4914
4915
4916 putData(primaryRegion, 0, 1000, cq, families);
4917
4918
4919 primaryRegion.flush(true);
4920
4921
4922 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, htd, null, CONF);
4923
4924
4925 Collection<StoreFile> storeFiles = primaryRegion.getStore(families[0]).getStorefiles();
4926 primaryRegion.getRegionFileSystem().removeStoreFiles(Bytes.toString(families[0]), storeFiles);
4927 Collection<StoreFileInfo> storeFileInfos = primaryRegion.getRegionFileSystem().getStoreFiles(families[0]);
4928 Assert.assertTrue(storeFileInfos == null || storeFileInfos.size() == 0);
4929
4930 verifyData(secondaryRegion, 0, 1000, cq, families);
4931 } finally {
4932 if (primaryRegion != null) {
4933 HRegion.closeHRegion(primaryRegion);
4934 }
4935 if (secondaryRegion != null) {
4936 HRegion.closeHRegion(secondaryRegion);
4937 }
4938 }
4939 }
4940
4941 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4942 putData(this.region, startRow, numRows, qf, families);
4943 }
4944
4945 private void putData(HRegion region,
4946 int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4947 putData(region, Durability.SKIP_WAL, startRow, numRows, qf, families);
4948 }
4949
4950 static void putData(HRegion region, Durability durability,
4951 int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4952 for (int i = startRow; i < startRow + numRows; i++) {
4953 Put put = new Put(Bytes.toBytes("" + i));
4954 put.setDurability(durability);
4955 for (byte[] family : families) {
4956 put.add(family, qf, null);
4957 }
4958 region.put(put);
4959 }
4960 }
4961
4962 static void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
4963 throws IOException {
4964 for (int i = startRow; i < startRow + numRows; i++) {
4965 byte[] row = Bytes.toBytes("" + i);
4966 Get get = new Get(row);
4967 for (byte[] family : families) {
4968 get.addColumn(family, qf);
4969 }
4970 Result result = newReg.get(get);
4971 Cell[] raw = result.rawCells();
4972 assertEquals(families.length, result.size());
4973 for (int j = 0; j < families.length; j++) {
4974 assertTrue(CellUtil.matchingRow(raw[j], row));
4975 assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
4976 assertTrue(CellUtil.matchingQualifier(raw[j], qf));
4977 }
4978 }
4979 }
4980
4981 static void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
4982
4983 Get get = new Get(k).addFamily(family).setMaxVersions();
4984 Cell[] results = r.get(get).rawCells();
4985 for (int j = 0; j < results.length; j++) {
4986 byte[] tmp = CellUtil.cloneValue(results[j]);
4987
4988 assertTrue(Bytes.equals(k, tmp));
4989 }
4990 }
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003 private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
5004 throws IOException {
5005 byte[][] families = { fs };
5006 Scan scan = new Scan();
5007 for (int i = 0; i < families.length; i++)
5008 scan.addFamily(families[i]);
5009 InternalScanner s = r.getScanner(scan);
5010 try {
5011 List<Cell> curVals = new ArrayList<Cell>();
5012 boolean first = true;
5013 OUTER_LOOP: while (s.next(curVals)) {
5014 for (Cell kv : curVals) {
5015 byte[] val = CellUtil.cloneValue(kv);
5016 byte[] curval = val;
5017 if (first) {
5018 first = false;
5019 assertTrue(Bytes.compareTo(curval, firstValue) == 0);
5020 } else {
5021
5022 break OUTER_LOOP;
5023 }
5024 }
5025 }
5026 } finally {
5027 s.close();
5028 }
5029 }
5030
5031
5032
5033
5034
5035 @Test
5036 public void testFlushResult() throws IOException {
5037 String method = name.getMethodName();
5038 byte[] tableName = Bytes.toBytes(method);
5039 byte[] family = Bytes.toBytes("family");
5040
5041 this.region = initHRegion(tableName, method, family);
5042
5043
5044 HRegion.FlushResult fr = region.flush(true);
5045 assertFalse(fr.isFlushSucceeded());
5046 assertFalse(fr.isCompactionNeeded());
5047
5048
5049 for (int i = 0; i < 2; i++) {
5050 Put put = new Put(tableName).add(family, family, tableName);
5051 region.put(put);
5052 fr = region.flush(true);
5053 assertTrue(fr.isFlushSucceeded());
5054 assertFalse(fr.isCompactionNeeded());
5055 }
5056
5057
5058 for (int i = 0; i < 2; i++) {
5059 Put put = new Put(tableName).add(family, family, tableName);
5060 region.put(put);
5061 fr = region.flush(true);
5062 assertTrue(fr.isFlushSucceeded());
5063 assertTrue(fr.isCompactionNeeded());
5064 }
5065 }
5066
5067 private Configuration initSplit() {
5068
5069 CONF.setInt("hbase.hstore.compactionThreshold", 2);
5070
5071
5072 CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
5073
5074 CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
5075
5076
5077 CONF.setLong("hbase.client.pause", 15 * 1000);
5078
5079
5080
5081 CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
5082 return CONF;
5083 }
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094 public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
5095 byte[]... families) throws IOException {
5096 return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
5097 }
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
5109 byte[]... families) throws IOException {
5110 return initHRegion(tableName, null, null, callingMethod, conf, false, families);
5111 }
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
5124 boolean isReadOnly, byte[]... families) throws IOException {
5125 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
5126 }
5127
5128 public static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
5129 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
5130 throws IOException {
5131 return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
5132 Durability.SYNC_WAL, null, families);
5133 }
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147 public static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
5148 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
5149 WAL wal, byte[]... families) throws IOException {
5150 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf,
5151 isReadOnly, durability, wal, families);
5152 }
5153
5154
5155
5156
5157
5158 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
5159 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
5160 assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
5161 Bytes.toString(CellUtil.cloneRow(kv)));
5162 assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
5163 Bytes.toString(CellUtil.cloneFamily(kv)));
5164 assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
5165 Bytes.toString(CellUtil.cloneQualifier(kv)));
5166 assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
5167 assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
5168 Bytes.toString(CellUtil.cloneValue(kv)));
5169 }
5170
5171 @Test (timeout=60000)
5172 public void testReverseScanner_FromMemStore_SingleCF_Normal()
5173 throws IOException {
5174 byte[] rowC = Bytes.toBytes("rowC");
5175 byte[] rowA = Bytes.toBytes("rowA");
5176 byte[] rowB = Bytes.toBytes("rowB");
5177 byte[] cf = Bytes.toBytes("CF");
5178 byte[][] families = { cf };
5179 byte[] col = Bytes.toBytes("C");
5180 long ts = 1;
5181 String method = this.getName();
5182 this.region = initHRegion(tableName, method, families);
5183 try {
5184 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5185 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
5186 null);
5187 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5188 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5189 Put put = null;
5190 put = new Put(rowC);
5191 put.add(kv1);
5192 put.add(kv11);
5193 region.put(put);
5194 put = new Put(rowA);
5195 put.add(kv2);
5196 region.put(put);
5197 put = new Put(rowB);
5198 put.add(kv3);
5199 region.put(put);
5200
5201 Scan scan = new Scan(rowC);
5202 scan.setMaxVersions(5);
5203 scan.setReversed(true);
5204 InternalScanner scanner = region.getScanner(scan);
5205 List<Cell> currRow = new ArrayList<Cell>();
5206 boolean hasNext = scanner.next(currRow);
5207 assertEquals(2, currRow.size());
5208 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5209 assertTrue(hasNext);
5210 currRow.clear();
5211 hasNext = scanner.next(currRow);
5212 assertEquals(1, currRow.size());
5213 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5214 assertTrue(hasNext);
5215 currRow.clear();
5216 hasNext = scanner.next(currRow);
5217 assertEquals(1, currRow.size());
5218 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
5219 assertFalse(hasNext);
5220 scanner.close();
5221 } finally {
5222 HRegion.closeHRegion(this.region);
5223 this.region = null;
5224 }
5225 }
5226
5227 @Test (timeout=60000)
5228 public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
5229 throws IOException {
5230 byte[] rowC = Bytes.toBytes("rowC");
5231 byte[] rowA = Bytes.toBytes("rowA");
5232 byte[] rowB = Bytes.toBytes("rowB");
5233 byte[] rowD = Bytes.toBytes("rowD");
5234 byte[] cf = Bytes.toBytes("CF");
5235 byte[][] families = { cf };
5236 byte[] col = Bytes.toBytes("C");
5237 long ts = 1;
5238 String method = this.getName();
5239 this.region = initHRegion(tableName, method, families);
5240 try {
5241 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5242 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
5243 null);
5244 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5245 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5246 Put put = null;
5247 put = new Put(rowC);
5248 put.add(kv1);
5249 put.add(kv11);
5250 region.put(put);
5251 put = new Put(rowA);
5252 put.add(kv2);
5253 region.put(put);
5254 put = new Put(rowB);
5255 put.add(kv3);
5256 region.put(put);
5257
5258 Scan scan = new Scan(rowD);
5259 List<Cell> currRow = new ArrayList<Cell>();
5260 scan.setReversed(true);
5261 scan.setMaxVersions(5);
5262 InternalScanner scanner = region.getScanner(scan);
5263 boolean hasNext = scanner.next(currRow);
5264 assertEquals(2, currRow.size());
5265 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5266 assertTrue(hasNext);
5267 currRow.clear();
5268 hasNext = scanner.next(currRow);
5269 assertEquals(1, currRow.size());
5270 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5271 assertTrue(hasNext);
5272 currRow.clear();
5273 hasNext = scanner.next(currRow);
5274 assertEquals(1, currRow.size());
5275 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
5276 assertFalse(hasNext);
5277 scanner.close();
5278 } finally {
5279 HRegion.closeHRegion(this.region);
5280 this.region = null;
5281 }
5282 }
5283
5284 @Test (timeout=60000)
5285 public void testReverseScanner_FromMemStore_SingleCF_FullScan()
5286 throws IOException {
5287 byte[] rowC = Bytes.toBytes("rowC");
5288 byte[] rowA = Bytes.toBytes("rowA");
5289 byte[] rowB = Bytes.toBytes("rowB");
5290 byte[] cf = Bytes.toBytes("CF");
5291 byte[][] families = { cf };
5292 byte[] col = Bytes.toBytes("C");
5293 long ts = 1;
5294 String method = this.getName();
5295 this.region = initHRegion(tableName, method, families);
5296 try {
5297 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5298 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
5299 null);
5300 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5301 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5302 Put put = null;
5303 put = new Put(rowC);
5304 put.add(kv1);
5305 put.add(kv11);
5306 region.put(put);
5307 put = new Put(rowA);
5308 put.add(kv2);
5309 region.put(put);
5310 put = new Put(rowB);
5311 put.add(kv3);
5312 region.put(put);
5313 Scan scan = new Scan();
5314 List<Cell> currRow = new ArrayList<Cell>();
5315 scan.setReversed(true);
5316 InternalScanner scanner = region.getScanner(scan);
5317 boolean hasNext = scanner.next(currRow);
5318 assertEquals(1, currRow.size());
5319 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5320 assertTrue(hasNext);
5321 currRow.clear();
5322 hasNext = scanner.next(currRow);
5323 assertEquals(1, currRow.size());
5324 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5325 assertTrue(hasNext);
5326 currRow.clear();
5327 hasNext = scanner.next(currRow);
5328 assertEquals(1, currRow.size());
5329 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
5330 assertFalse(hasNext);
5331 scanner.close();
5332 } finally {
5333 HRegion.closeHRegion(this.region);
5334 this.region = null;
5335 }
5336 }
5337
5338 @Test (timeout=60000)
5339 public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
5340
5341 byte[] rowA = Bytes.toBytes("rowA");
5342 byte[] rowB = Bytes.toBytes("rowB");
5343 byte[] rowC = Bytes.toBytes("rowC");
5344 byte[] rowD = Bytes.toBytes("rowD");
5345 byte[] rowE = Bytes.toBytes("rowE");
5346 byte[] cf = Bytes.toBytes("CF");
5347 byte[][] families = { cf };
5348 byte[] col1 = Bytes.toBytes("col1");
5349 byte[] col2 = Bytes.toBytes("col2");
5350 long ts = 1;
5351 String method = this.getName();
5352 this.region = initHRegion(tableName, method, families);
5353 try {
5354 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
5355 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
5356 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
5357 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
5358 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
5359 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
5360 Put put = null;
5361 put = new Put(rowA);
5362 put.add(kv1);
5363 region.put(put);
5364 put = new Put(rowB);
5365 put.add(kv2);
5366 region.put(put);
5367 put = new Put(rowC);
5368 put.add(kv3);
5369 region.put(put);
5370 put = new Put(rowD);
5371 put.add(kv4_1);
5372 region.put(put);
5373 put = new Put(rowD);
5374 put.add(kv4_2);
5375 region.put(put);
5376 put = new Put(rowE);
5377 put.add(kv5);
5378 region.put(put);
5379 region.flush(true);
5380 Scan scan = new Scan(rowD, rowA);
5381 scan.addColumn(families[0], col1);
5382 scan.setReversed(true);
5383 List<Cell> currRow = new ArrayList<Cell>();
5384 InternalScanner scanner = region.getScanner(scan);
5385 boolean hasNext = scanner.next(currRow);
5386 assertEquals(1, currRow.size());
5387 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5388 assertTrue(hasNext);
5389 currRow.clear();
5390 hasNext = scanner.next(currRow);
5391 assertEquals(1, currRow.size());
5392 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5393 assertTrue(hasNext);
5394 currRow.clear();
5395 hasNext = scanner.next(currRow);
5396 assertEquals(1, currRow.size());
5397 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5398 assertFalse(hasNext);
5399 scanner.close();
5400
5401 scan = new Scan(rowD, rowA);
5402 scan.addColumn(families[0], col2);
5403 scan.setReversed(true);
5404 currRow.clear();
5405 scanner = region.getScanner(scan);
5406 hasNext = scanner.next(currRow);
5407 assertEquals(1, currRow.size());
5408 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5409 scanner.close();
5410 } finally {
5411 HRegion.closeHRegion(this.region);
5412 this.region = null;
5413 }
5414 }
5415
5416 @Test (timeout=60000)
5417 public void testReverseScanner_smaller_blocksize() throws IOException {
5418
5419 byte[] rowA = Bytes.toBytes("rowA");
5420 byte[] rowB = Bytes.toBytes("rowB");
5421 byte[] rowC = Bytes.toBytes("rowC");
5422 byte[] rowD = Bytes.toBytes("rowD");
5423 byte[] rowE = Bytes.toBytes("rowE");
5424 byte[] cf = Bytes.toBytes("CF");
5425 byte[][] families = { cf };
5426 byte[] col1 = Bytes.toBytes("col1");
5427 byte[] col2 = Bytes.toBytes("col2");
5428 long ts = 1;
5429 String method = this.getName();
5430 HBaseConfiguration config = new HBaseConfiguration();
5431 config.setInt("test.block.size", 1);
5432 this.region = initHRegion(tableName, method, config, families);
5433 try {
5434 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
5435 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
5436 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
5437 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
5438 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
5439 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
5440 Put put = null;
5441 put = new Put(rowA);
5442 put.add(kv1);
5443 region.put(put);
5444 put = new Put(rowB);
5445 put.add(kv2);
5446 region.put(put);
5447 put = new Put(rowC);
5448 put.add(kv3);
5449 region.put(put);
5450 put = new Put(rowD);
5451 put.add(kv4_1);
5452 region.put(put);
5453 put = new Put(rowD);
5454 put.add(kv4_2);
5455 region.put(put);
5456 put = new Put(rowE);
5457 put.add(kv5);
5458 region.put(put);
5459 region.flush(true);
5460 Scan scan = new Scan(rowD, rowA);
5461 scan.addColumn(families[0], col1);
5462 scan.setReversed(true);
5463 List<Cell> currRow = new ArrayList<Cell>();
5464 InternalScanner scanner = region.getScanner(scan);
5465 boolean hasNext = scanner.next(currRow);
5466 assertEquals(1, currRow.size());
5467 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5468 assertTrue(hasNext);
5469 currRow.clear();
5470 hasNext = scanner.next(currRow);
5471 assertEquals(1, currRow.size());
5472 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5473 assertTrue(hasNext);
5474 currRow.clear();
5475 hasNext = scanner.next(currRow);
5476 assertEquals(1, currRow.size());
5477 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5478 assertFalse(hasNext);
5479 scanner.close();
5480
5481 scan = new Scan(rowD, rowA);
5482 scan.addColumn(families[0], col2);
5483 scan.setReversed(true);
5484 currRow.clear();
5485 scanner = region.getScanner(scan);
5486 hasNext = scanner.next(currRow);
5487 assertEquals(1, currRow.size());
5488 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5489 scanner.close();
5490 } finally {
5491 HRegion.closeHRegion(this.region);
5492 this.region = null;
5493 }
5494 }
5495
5496 @Test (timeout=60000)
5497 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
5498 throws IOException {
5499 byte[] row0 = Bytes.toBytes("row0");
5500 byte[] row1 = Bytes.toBytes("row1");
5501 byte[] row2 = Bytes.toBytes("row2");
5502 byte[] row3 = Bytes.toBytes("row3");
5503 byte[] row4 = Bytes.toBytes("row4");
5504 byte[] row5 = Bytes.toBytes("row5");
5505 byte[] cf1 = Bytes.toBytes("CF1");
5506 byte[] cf2 = Bytes.toBytes("CF2");
5507 byte[] cf3 = Bytes.toBytes("CF3");
5508 byte[][] families = { cf1, cf2, cf3 };
5509 byte[] col = Bytes.toBytes("C");
5510 long ts = 1;
5511 String method = this.getName();
5512 HBaseConfiguration conf = new HBaseConfiguration();
5513
5514 conf.setInt("hbase.hstore.compactionThreshold", 10000);
5515 this.region = initHRegion(tableName, method, conf, families);
5516 try {
5517
5518 KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
5519 null);
5520 KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
5521 null);
5522 KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
5523 KeyValue.Type.Put, null);
5524 KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
5525 null);
5526 KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
5527 null);
5528 KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
5529 null);
5530 KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
5531 KeyValue.Type.Put, null);
5532 KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
5533 null);
5534 KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
5535 KeyValue.Type.Put, null);
5536 KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
5537 null);
5538 KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
5539 null);
5540 KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
5541 KeyValue.Type.Put, null);
5542 KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
5543 null);
5544 KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
5545 KeyValue.Type.Put, null);
5546 KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
5547 null);
5548 KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
5549 null);
5550
5551 Put put = null;
5552 put = new Put(row1);
5553 put.add(kv1_2_1);
5554 region.put(put);
5555 put = new Put(row2);
5556 put.add(kv2_4_1);
5557 region.put(put);
5558 put = new Put(row4);
5559 put.add(kv4_5_4);
5560 put.add(kv4_5_5);
5561 region.put(put);
5562 region.flush(true);
5563
5564 put = new Put(row4);
5565 put.add(kv4_5_1);
5566 put.add(kv4_5_3);
5567 region.put(put);
5568 put = new Put(row1);
5569 put.add(kv1_2_2);
5570 region.put(put);
5571 put = new Put(row2);
5572 put.add(kv2_4_4);
5573 region.put(put);
5574 region.flush(true);
5575
5576 put = new Put(row4);
5577 put.add(kv4_5_2);
5578 region.put(put);
5579 put = new Put(row2);
5580 put.add(kv2_4_2);
5581 put.add(kv2_4_3);
5582 region.put(put);
5583 put = new Put(row3);
5584 put.add(kv3_2_2);
5585 region.put(put);
5586 region.flush(true);
5587
5588
5589 put = new Put(row0);
5590 put.add(kv0_1_1);
5591 region.put(put);
5592 put = new Put(row3);
5593 put.add(kv3_2_1);
5594 region.put(put);
5595 put = new Put(row5);
5596 put.add(kv5_2_1);
5597 put.add(kv5_2_2);
5598 region.put(put);
5599
5600 Scan scan = new Scan(row4);
5601 scan.setMaxVersions(5);
5602 scan.setBatch(3);
5603 scan.setReversed(true);
5604 InternalScanner scanner = region.getScanner(scan);
5605 List<Cell> currRow = new ArrayList<Cell>();
5606 boolean hasNext = false;
5607
5608
5609
5610 hasNext = scanner.next(currRow);
5611 assertEquals(3, currRow.size());
5612 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5613 assertTrue(hasNext);
5614 currRow.clear();
5615 hasNext = scanner.next(currRow);
5616 assertEquals(2, currRow.size());
5617 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5618 assertTrue(hasNext);
5619
5620 currRow.clear();
5621 hasNext = scanner.next(currRow);
5622 assertEquals(2, currRow.size());
5623 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5624 assertTrue(hasNext);
5625
5626
5627 currRow.clear();
5628 hasNext = scanner.next(currRow);
5629 assertEquals(3, currRow.size());
5630 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5631 assertTrue(hasNext);
5632 currRow.clear();
5633 hasNext = scanner.next(currRow);
5634 assertEquals(1, currRow.size());
5635 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5636 assertTrue(hasNext);
5637
5638 currRow.clear();
5639 hasNext = scanner.next(currRow);
5640 assertEquals(2, currRow.size());
5641 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5642 assertTrue(hasNext);
5643
5644 currRow.clear();
5645 hasNext = scanner.next(currRow);
5646 assertEquals(1, currRow.size());
5647 assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
5648 assertFalse(hasNext);
5649
5650 scanner.close();
5651 } finally {
5652 HRegion.closeHRegion(this.region);
5653 this.region = null;
5654 }
5655 }
5656
5657 @Test (timeout=60000)
5658 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
5659 throws IOException {
5660 byte[] row1 = Bytes.toBytes("row1");
5661 byte[] row2 = Bytes.toBytes("row2");
5662 byte[] row3 = Bytes.toBytes("row3");
5663 byte[] row4 = Bytes.toBytes("row4");
5664 byte[] cf1 = Bytes.toBytes("CF1");
5665 byte[] cf2 = Bytes.toBytes("CF2");
5666 byte[] cf3 = Bytes.toBytes("CF3");
5667 byte[] cf4 = Bytes.toBytes("CF4");
5668 byte[][] families = { cf1, cf2, cf3, cf4 };
5669 byte[] col = Bytes.toBytes("C");
5670 long ts = 1;
5671 String method = this.getName();
5672 HBaseConfiguration conf = new HBaseConfiguration();
5673
5674 conf.setInt("hbase.hstore.compactionThreshold", 10000);
5675 this.region = initHRegion(tableName, method, conf, families);
5676 try {
5677 KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
5678 KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
5679 KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
5680 KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
5681
5682 Put put = new Put(row1);
5683 put.add(kv1);
5684 region.put(put);
5685 region.flush(true);
5686
5687 put = new Put(row2);
5688 put.add(kv2);
5689 region.put(put);
5690 region.flush(true);
5691
5692 put = new Put(row3);
5693 put.add(kv3);
5694 region.put(put);
5695 region.flush(true);
5696
5697 put = new Put(row4);
5698 put.add(kv4);
5699 region.put(put);
5700
5701 Scan scan = new Scan(row4);
5702 scan.setReversed(true);
5703 scan.setBatch(10);
5704 InternalScanner scanner = region.getScanner(scan);
5705 List<Cell> currRow = new ArrayList<Cell>();
5706 boolean hasNext = scanner.next(currRow);
5707 assertEquals(1, currRow.size());
5708 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5709 assertTrue(hasNext);
5710 currRow.clear();
5711 hasNext = scanner.next(currRow);
5712 assertEquals(1, currRow.size());
5713 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5714 assertTrue(hasNext);
5715 currRow.clear();
5716 hasNext = scanner.next(currRow);
5717 assertEquals(1, currRow.size());
5718 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5719 assertTrue(hasNext);
5720 currRow.clear();
5721 hasNext = scanner.next(currRow);
5722 assertEquals(1, currRow.size());
5723 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5724 assertFalse(hasNext);
5725 } finally {
5726 HRegion.closeHRegion(this.region);
5727 this.region = null;
5728 }
5729 }
5730
5731 @Test (timeout=60000)
5732 public void testSplitRegionWithReverseScan() throws IOException {
5733 byte [] tableName = Bytes.toBytes("testSplitRegionWithReverseScan");
5734 byte [] qualifier = Bytes.toBytes("qualifier");
5735 Configuration hc = initSplit();
5736 int numRows = 3;
5737 byte [][] families = {fam1};
5738
5739
5740 String method = this.getName();
5741 this.region = initHRegion(tableName, method, hc, families);
5742
5743
5744 int startRow = 100;
5745 putData(startRow, numRows, qualifier, families);
5746 int splitRow = startRow + numRows;
5747 putData(splitRow, numRows, qualifier, families);
5748 region.flush(true);
5749
5750 HRegion [] regions = null;
5751 try {
5752 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
5753
5754 for (int i = 0; i < regions.length; i++) {
5755 regions[i] = HRegion.openHRegion(regions[i], null);
5756 }
5757
5758 assertEquals(2, regions.length);
5759
5760
5761
5762 verifyData(regions[0], startRow, numRows, qualifier, families);
5763 verifyData(regions[1], splitRow, numRows, qualifier, families);
5764
5765
5766 Scan scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 10 * numRows)));
5767 scan.setReversed(true);
5768 InternalScanner scanner = regions[1].getScanner(scan);
5769 List<Cell> currRow = new ArrayList<Cell>();
5770 boolean more = false;
5771 int verify = startRow + 2 * numRows - 1;
5772 do {
5773 more = scanner.next(currRow);
5774 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5775 verify--;
5776 currRow.clear();
5777 } while(more);
5778 assertEquals(verify, startRow + numRows - 1);
5779 scanner.close();
5780
5781 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 2 * numRows - 1)));
5782 scan.setReversed(true);
5783 scanner = regions[1].getScanner(scan);
5784 verify = startRow + 2 * numRows - 1;
5785 do {
5786 more = scanner.next(currRow);
5787 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5788 verify--;
5789 currRow.clear();
5790 } while(more);
5791 assertEquals(verify, startRow + numRows - 1);
5792 scanner.close();
5793
5794 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows)));
5795 scan.setReversed(true);
5796 scanner = regions[0].getScanner(scan);
5797 verify = startRow + numRows - 1;
5798 do {
5799 more = scanner.next(currRow);
5800 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5801 verify--;
5802 currRow.clear();
5803 } while(more);
5804 assertEquals(verify, 99);
5805 scanner.close();
5806
5807 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows - 1)));
5808 scan.setReversed(true);
5809 scanner = regions[0].getScanner(scan);
5810 verify = startRow + numRows - 1;
5811 do {
5812 more = scanner.next(currRow);
5813 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5814 verify--;
5815 currRow.clear();
5816 } while(more);
5817 assertEquals(verify, startRow - 1);
5818 scanner.close();
5819 } finally {
5820 HRegion.closeHRegion(this.region);
5821 this.region = null;
5822 }
5823 }
5824
5825 @Test
5826 public void testWriteRequestsCounter() throws IOException {
5827 byte[] fam = Bytes.toBytes("info");
5828 byte[][] families = { fam };
5829 this.region = initHRegion(tableName, method, CONF, families);
5830
5831 Assert.assertEquals(0L, region.getWriteRequestsCount());
5832
5833 Put put = new Put(row);
5834 put.add(fam, fam, fam);
5835
5836 Assert.assertEquals(0L, region.getWriteRequestsCount());
5837 region.put(put);
5838 Assert.assertEquals(1L, region.getWriteRequestsCount());
5839 region.put(put);
5840 Assert.assertEquals(2L, region.getWriteRequestsCount());
5841 region.put(put);
5842 Assert.assertEquals(3L, region.getWriteRequestsCount());
5843
5844 region.delete(new Delete(row));
5845 Assert.assertEquals(4L, region.getWriteRequestsCount());
5846
5847 HRegion.closeHRegion(this.region);
5848 this.region = null;
5849 }
5850
5851 @Test
5852 @SuppressWarnings("unchecked")
5853 public void testOpenRegionWrittenToWAL() throws Exception {
5854 final ServerName serverName = ServerName.valueOf("testOpenRegionWrittenToWAL", 100, 42);
5855 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
5856
5857 HTableDescriptor htd
5858 = new HTableDescriptor(TableName.valueOf("testOpenRegionWrittenToWAL"));
5859 htd.addFamily(new HColumnDescriptor(fam1));
5860 htd.addFamily(new HColumnDescriptor(fam2));
5861
5862 HRegionInfo hri = new HRegionInfo(htd.getTableName(),
5863 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY);
5864
5865
5866 HRegion region =
5867 HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL
5868 .getConfiguration(), htd);
5869 assertNotNull(region);
5870
5871
5872 region.put(new Put(Bytes.toBytes("a")).add(fam1, fam1, fam1));
5873 region.flush(true);
5874 region.close();
5875
5876 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
5877
5878
5879 WAL wal = mock(WAL.class);
5880 when(rss.getWAL((HRegionInfo) any())).thenReturn(wal);
5881
5882 try {
5883 region = HRegion.openHRegion(hri, htd, rss.getWAL(hri),
5884 TEST_UTIL.getConfiguration(), rss, null);
5885
5886 verify(wal, times(1)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any()
5887 , editCaptor.capture(), (AtomicLong)any(), anyBoolean(), (List<Cell>)any());
5888
5889 WALEdit edit = editCaptor.getValue();
5890 assertNotNull(edit);
5891 assertNotNull(edit.getCells());
5892 assertEquals(1, edit.getCells().size());
5893 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
5894 assertNotNull(desc);
5895
5896 LOG.info("RegionEventDescriptor from WAL: " + desc);
5897
5898 assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType());
5899 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName()));
5900 assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(),
5901 hri.getEncodedNameAsBytes()));
5902 assertTrue(desc.getLogSequenceNumber() > 0);
5903 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
5904 assertEquals(2, desc.getStoresCount());
5905
5906 StoreDescriptor store = desc.getStores(0);
5907 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
5908 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
5909 assertEquals(1, store.getStoreFileCount());
5910 assertFalse(store.getStoreFile(0).contains("/"));
5911
5912 store = desc.getStores(1);
5913 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
5914 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
5915 assertEquals(0, store.getStoreFileCount());
5916
5917 } finally {
5918 HRegion.closeHRegion(region);
5919 }
5920 }
5921
5922
5923 static class HRegionWithSeqId extends HRegion {
5924 public HRegionWithSeqId(final Path tableDir, final WAL wal, final FileSystem fs,
5925 final Configuration confParam, final HRegionInfo regionInfo,
5926 final HTableDescriptor htd, final RegionServerServices rsServices) {
5927 super(tableDir, wal, fs, confParam, regionInfo, htd, rsServices);
5928 }
5929 @Override
5930 protected long getNextSequenceId(WAL wal) throws IOException {
5931 return 42;
5932 }
5933 }
5934
5935 @Test
5936 @SuppressWarnings("unchecked")
5937 public void testOpenRegionWrittenToWALForLogReplay() throws Exception {
5938
5939 final ServerName serverName = ServerName.valueOf("testOpenRegionWrittenToWALForLogReplay",
5940 100, 42);
5941 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
5942
5943 HTableDescriptor htd
5944 = new HTableDescriptor(TableName.valueOf("testOpenRegionWrittenToWALForLogReplay"));
5945 htd.addFamily(new HColumnDescriptor(fam1));
5946 htd.addFamily(new HColumnDescriptor(fam2));
5947
5948 HRegionInfo hri = new HRegionInfo(htd.getTableName(),
5949 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY);
5950
5951
5952 HRegion region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL
5953 .getConfiguration(), htd);
5954 assertNotNull(region);
5955
5956
5957 region.put(new Put(Bytes.toBytes("a")).add(fam1, fam1, fam1));
5958 region.flush(true);
5959 HRegion.closeHRegion(region);
5960
5961 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
5962
5963
5964 WAL wal = mock(WAL.class);
5965 when(rss.getWAL((HRegionInfo) any())).thenReturn(wal);
5966
5967
5968 HashMap<String, Region> recoveringRegions = Maps.newHashMap();
5969 recoveringRegions.put(region.getRegionInfo().getEncodedName(), null);
5970 when(rss.getRecoveringRegions()).thenReturn(recoveringRegions);
5971
5972 try {
5973 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
5974 conf.set(HConstants.REGION_IMPL, HRegionWithSeqId.class.getName());
5975 region = HRegion.openHRegion(hri, htd, rss.getWAL(hri),
5976 conf, rss, null);
5977
5978
5979
5980 verify(wal, times(0)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any()
5981 , editCaptor.capture(), (AtomicLong)any(), anyBoolean(), (List<Cell>)any());
5982
5983
5984 new FinishRegionRecoveringHandler(rss, region.getRegionInfo().getEncodedName(), "/foo")
5985 .prepare().process();
5986
5987
5988 verify(wal, times(1)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any()
5989 , editCaptor.capture(), (AtomicLong)any(), anyBoolean(), (List<Cell>)any());
5990
5991 WALEdit edit = editCaptor.getValue();
5992 assertNotNull(edit);
5993 assertNotNull(edit.getCells());
5994 assertEquals(1, edit.getCells().size());
5995 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
5996 assertNotNull(desc);
5997
5998 LOG.info("RegionEventDescriptor from WAL: " + desc);
5999
6000 assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType());
6001 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName()));
6002 assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(),
6003 hri.getEncodedNameAsBytes()));
6004 assertTrue(desc.getLogSequenceNumber() > 0);
6005 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
6006 assertEquals(2, desc.getStoresCount());
6007
6008 StoreDescriptor store = desc.getStores(0);
6009 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
6010 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
6011 assertEquals(1, store.getStoreFileCount());
6012 assertFalse(store.getStoreFile(0).contains("/"));
6013
6014 store = desc.getStores(1);
6015 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
6016 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
6017 assertEquals(0, store.getStoreFileCount());
6018
6019 } finally {
6020 HRegion.closeHRegion(region);
6021 }
6022 }
6023
6024 @Test
6025 @SuppressWarnings("unchecked")
6026 public void testCloseRegionWrittenToWAL() throws Exception {
6027 final ServerName serverName = ServerName.valueOf("testCloseRegionWrittenToWAL", 100, 42);
6028 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
6029
6030 HTableDescriptor htd
6031 = new HTableDescriptor(TableName.valueOf("testOpenRegionWrittenToWAL"));
6032 htd.addFamily(new HColumnDescriptor(fam1));
6033 htd.addFamily(new HColumnDescriptor(fam2));
6034
6035 HRegionInfo hri = new HRegionInfo(htd.getTableName(),
6036 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY);
6037
6038 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
6039
6040
6041 WAL wal = mock(WAL.class);
6042 when(rss.getWAL((HRegionInfo) any())).thenReturn(wal);
6043
6044
6045 region = HRegion.openHRegion(hri, htd, rss.getWAL(hri),
6046 TEST_UTIL.getConfiguration(), rss, null);
6047
6048
6049 region.close(false);
6050
6051
6052 verify(wal, times(2)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any(),
6053 editCaptor.capture(), (AtomicLong)any(), anyBoolean(), (List<Cell>)any());
6054
6055 WALEdit edit = editCaptor.getAllValues().get(1);
6056 assertNotNull(edit);
6057 assertNotNull(edit.getCells());
6058 assertEquals(1, edit.getCells().size());
6059 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
6060 assertNotNull(desc);
6061
6062 LOG.info("RegionEventDescriptor from WAL: " + desc);
6063
6064 assertEquals(RegionEventDescriptor.EventType.REGION_CLOSE, desc.getEventType());
6065 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName()));
6066 assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(),
6067 hri.getEncodedNameAsBytes()));
6068 assertTrue(desc.getLogSequenceNumber() > 0);
6069 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
6070 assertEquals(2, desc.getStoresCount());
6071
6072 StoreDescriptor store = desc.getStores(0);
6073 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
6074 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
6075 assertEquals(0, store.getStoreFileCount());
6076
6077 store = desc.getStores(1);
6078 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
6079 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
6080 assertEquals(0, store.getStoreFileCount());
6081 }
6082
6083
6084
6085
6086 @Test (timeout=24000)
6087 public void testRegionTooBusy() throws IOException {
6088 String method = "testRegionTooBusy";
6089 byte[] tableName = Bytes.toBytes(method);
6090 byte[] family = Bytes.toBytes("family");
6091 long defaultBusyWaitDuration = CONF.getLong("hbase.busy.wait.duration",
6092 HRegion.DEFAULT_BUSY_WAIT_DURATION);
6093 CONF.setLong("hbase.busy.wait.duration", 1000);
6094 region = initHRegion(tableName, method, CONF, family);
6095 final AtomicBoolean stopped = new AtomicBoolean(true);
6096 Thread t = new Thread(new Runnable() {
6097 @Override
6098 public void run() {
6099 try {
6100 region.lock.writeLock().lock();
6101 stopped.set(false);
6102 while (!stopped.get()) {
6103 Thread.sleep(100);
6104 }
6105 } catch (InterruptedException ie) {
6106 } finally {
6107 region.lock.writeLock().unlock();
6108 }
6109 }
6110 });
6111 t.start();
6112 Get get = new Get(row);
6113 try {
6114 while (stopped.get()) {
6115 Thread.sleep(100);
6116 }
6117 region.get(get);
6118 fail("Should throw RegionTooBusyException");
6119 } catch (InterruptedException ie) {
6120 fail("test interrupted");
6121 } catch (RegionTooBusyException e) {
6122
6123 } finally {
6124 stopped.set(true);
6125 try {
6126 t.join();
6127 } catch (Throwable e) {
6128 }
6129
6130 HRegion.closeHRegion(region);
6131 region = null;
6132 CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration);
6133 }
6134 }
6135
6136 @Test
6137 public void testCellTTLs() throws IOException {
6138 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
6139 EnvironmentEdgeManager.injectEdge(edge);
6140
6141 final byte[] row = Bytes.toBytes("testRow");
6142 final byte[] q1 = Bytes.toBytes("q1");
6143 final byte[] q2 = Bytes.toBytes("q2");
6144 final byte[] q3 = Bytes.toBytes("q3");
6145 final byte[] q4 = Bytes.toBytes("q4");
6146
6147 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCellTTLs"));
6148 HColumnDescriptor hcd = new HColumnDescriptor(fam1);
6149 hcd.setTimeToLive(10);
6150 htd.addFamily(hcd);
6151
6152 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
6153 conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS);
6154
6155 HRegion region = HRegion.createHRegion(new HRegionInfo(htd.getTableName(),
6156 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY),
6157 TEST_UTIL.getDataTestDir(), conf, htd);
6158 assertNotNull(region);
6159 try {
6160 long now = EnvironmentEdgeManager.currentTime();
6161
6162 region.put(new Put(row).add(new KeyValue(row, fam1, q1, now,
6163 HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
6164
6165 new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
6166
6167 region.put(new Put(row).add(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY));
6168
6169 region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1,
6170 HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
6171
6172 new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
6173
6174 region.put(new Put(row).add(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY));
6175
6176
6177 region.flush(true);
6178
6179
6180 Result r = region.get(new Get(row));
6181 assertNotNull(r.getValue(fam1, q1));
6182 assertNotNull(r.getValue(fam1, q2));
6183 assertNotNull(r.getValue(fam1, q3));
6184 assertNotNull(r.getValue(fam1, q4));
6185
6186
6187 edge.incrementTime(5000);
6188
6189 r = region.get(new Get(row));
6190 assertNull(r.getValue(fam1, q1));
6191 assertNotNull(r.getValue(fam1, q2));
6192 assertNotNull(r.getValue(fam1, q3));
6193 assertNotNull(r.getValue(fam1, q4));
6194
6195
6196 edge.incrementTime(5000);
6197
6198 r = region.get(new Get(row));
6199 assertNull(r.getValue(fam1, q1));
6200 assertNull(r.getValue(fam1, q2));
6201 assertNotNull(r.getValue(fam1, q3));
6202 assertNotNull(r.getValue(fam1, q4));
6203
6204
6205 edge.incrementTime(5000);
6206
6207 r = region.get(new Get(row));
6208 assertNull(r.getValue(fam1, q1));
6209 assertNull(r.getValue(fam1, q2));
6210 assertNull(r.getValue(fam1, q3));
6211 assertNotNull(r.getValue(fam1, q4));
6212
6213
6214 edge.incrementTime(10000);
6215
6216 r = region.get(new Get(row));
6217 assertNull(r.getValue(fam1, q1));
6218 assertNull(r.getValue(fam1, q2));
6219 assertNull(r.getValue(fam1, q3));
6220 assertNull(r.getValue(fam1, q4));
6221
6222
6223
6224
6225 region.put(new Put(row).add(fam1, q1, Bytes.toBytes(1L)));
6226 r = region.get(new Get(row));
6227 byte[] val = r.getValue(fam1, q1);
6228 assertNotNull(val);
6229 assertEquals(Bytes.toLong(val), 1L);
6230
6231
6232 Increment incr = new Increment(row).addColumn(fam1, q1, 1L);
6233 incr.setTTL(5000);
6234 region.increment(incr);
6235
6236
6237 r = region.get(new Get(row));
6238 val = r.getValue(fam1, q1);
6239 assertNotNull(val);
6240 assertEquals(Bytes.toLong(val), 2L);
6241
6242
6243 edge.incrementTime(5000);
6244
6245
6246 r = region.get(new Get(row));
6247 val = r.getValue(fam1, q1);
6248 assertNotNull(val);
6249 assertEquals(Bytes.toLong(val), 1L);
6250
6251
6252 edge.incrementTime(5000);
6253
6254
6255 r = region.get(new Get(row));
6256 assertNull(r.getValue(fam1, q1));
6257
6258 } finally {
6259 HRegion.closeHRegion(region);
6260 }
6261 }
6262
6263 @Test
6264 public void testIncrementTimestampsAreMonotonic() throws IOException {
6265 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
6266 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
6267 EnvironmentEdgeManager.injectEdge(edge);
6268
6269 edge.setValue(10);
6270 Increment inc = new Increment(row);
6271 inc.setDurability(Durability.SKIP_WAL);
6272 inc.addColumn(fam1, qual1, 1L);
6273 region.increment(inc);
6274
6275 Result result = region.get(new Get(row));
6276 Cell c = result.getColumnLatestCell(fam1, qual1);
6277 assertNotNull(c);
6278 assertEquals(c.getTimestamp(), 10L);
6279
6280 edge.setValue(1);
6281 region.increment(inc);
6282 result = region.get(new Get(row));
6283 c = result.getColumnLatestCell(fam1, qual1);
6284 assertEquals(c.getTimestamp(), 10L);
6285 assertEquals(Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength()), 2L);
6286 }
6287
6288 @Test
6289 public void testAppendTimestampsAreMonotonic() throws IOException {
6290 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
6291 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
6292 EnvironmentEdgeManager.injectEdge(edge);
6293
6294 edge.setValue(10);
6295 Append a = new Append(row);
6296 a.setDurability(Durability.SKIP_WAL);
6297 a.add(fam1, qual1, qual1);
6298 region.append(a);
6299
6300 Result result = region.get(new Get(row));
6301 Cell c = result.getColumnLatestCell(fam1, qual1);
6302 assertNotNull(c);
6303 assertEquals(c.getTimestamp(), 10L);
6304
6305 edge.setValue(1);
6306 region.append(a);
6307 result = region.get(new Get(row));
6308 c = result.getColumnLatestCell(fam1, qual1);
6309 assertEquals(c.getTimestamp(), 10L);
6310
6311 byte[] expected = new byte[qual1.length*2];
6312 System.arraycopy(qual1, 0, expected, 0, qual1.length);
6313 System.arraycopy(qual1, 0, expected, qual1.length, qual1.length);
6314
6315 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
6316 expected, 0, expected.length));
6317 }
6318
6319 @Test
6320 public void testCheckAndMutateTimestampsAreMonotonic() throws IOException {
6321 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
6322 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
6323 EnvironmentEdgeManager.injectEdge(edge);
6324
6325 edge.setValue(10);
6326 Put p = new Put(row);
6327 p.setDurability(Durability.SKIP_WAL);
6328 p.addColumn(fam1, qual1, qual1);
6329 region.put(p);
6330
6331 Result result = region.get(new Get(row));
6332 Cell c = result.getColumnLatestCell(fam1, qual1);
6333 assertNotNull(c);
6334 assertEquals(c.getTimestamp(), 10L);
6335
6336 edge.setValue(1);
6337 p = new Put(row);
6338 p.setDurability(Durability.SKIP_WAL);
6339 p.addColumn(fam1, qual1, qual2);
6340 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1), p, false);
6341 result = region.get(new Get(row));
6342 c = result.getColumnLatestCell(fam1, qual1);
6343 assertEquals(c.getTimestamp(), 10L);
6344
6345 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
6346 qual2, 0, qual2.length));
6347 }
6348
6349 @Test(timeout = 60000)
6350 public void testBatchMutateWithWrongRegionException() throws Exception {
6351 final byte[] a = Bytes.toBytes("a");
6352 final byte[] b = Bytes.toBytes("b");
6353 final byte[] c = Bytes.toBytes("c");
6354
6355 int prevLockTimeout = CONF.getInt("hbase.rowlock.wait.duration", 30000);
6356 CONF.setInt("hbase.rowlock.wait.duration", 1000);
6357 final HRegion region = initHRegion(tableName, a, c, name.getMethodName(), CONF, false, fam1);
6358
6359 Mutation[] mutations = new Mutation[] {
6360 new Put(a).addImmutable(fam1, null, null),
6361 new Put(c).addImmutable(fam1, null, null),
6362 new Put(b).addImmutable(fam1, null, null),
6363 };
6364
6365 OperationStatus[] status = region.batchMutate(mutations);
6366 assertEquals(status[0].getOperationStatusCode(), OperationStatusCode.SUCCESS);
6367 assertEquals(status[1].getOperationStatusCode(), OperationStatusCode.SANITY_CHECK_FAILURE);
6368 assertEquals(status[2].getOperationStatusCode(), OperationStatusCode.SUCCESS);
6369
6370
6371
6372 final CountDownLatch obtainedRowLock = new CountDownLatch(1);
6373 ExecutorService exec = Executors.newFixedThreadPool(2);
6374 Future<Void> f1 = exec.submit(new Callable<Void>() {
6375 @Override
6376 public Void call() throws Exception {
6377 LOG.info("Acquiring row lock");
6378 RowLock rl = region.getRowLock(b);
6379 obtainedRowLock.countDown();
6380 LOG.info("Waiting for 5 seconds before releasing lock");
6381 Threads.sleep(5000);
6382 LOG.info("Releasing row lock");
6383 rl.release();
6384 return null;
6385 }
6386 });
6387 obtainedRowLock.await(30, TimeUnit.SECONDS);
6388
6389 Future<Void> f2 = exec.submit(new Callable<Void>() {
6390 @Override
6391 public Void call() throws Exception {
6392 Mutation[] mutations = new Mutation[] {
6393 new Put(a).addImmutable(fam1, null, null),
6394 new Put(b).addImmutable(fam1, null, null),
6395 };
6396
6397
6398 OperationStatus[] status = region.batchMutate(mutations);
6399 assertEquals(status[0].getOperationStatusCode(), OperationStatusCode.SUCCESS);
6400 assertEquals(status[1].getOperationStatusCode(), OperationStatusCode.SUCCESS);
6401 return null;
6402 }
6403 });
6404
6405 f1.get();
6406 f2.get();
6407
6408 CONF.setInt("hbase.rowlock.wait.duration", prevLockTimeout);
6409 }
6410
6411 @Test
6412 public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException {
6413 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
6414 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
6415 EnvironmentEdgeManager.injectEdge(edge);
6416
6417 edge.setValue(10);
6418 Put p = new Put(row);
6419 p.setDurability(Durability.SKIP_WAL);
6420 p.addColumn(fam1, qual1, qual1);
6421 region.put(p);
6422
6423 Result result = region.get(new Get(row));
6424 Cell c = result.getColumnLatestCell(fam1, qual1);
6425 assertNotNull(c);
6426 assertEquals(c.getTimestamp(), 10L);
6427
6428 edge.setValue(1);
6429 p = new Put(row);
6430 p.setDurability(Durability.SKIP_WAL);
6431 p.addColumn(fam1, qual1, qual2);
6432 RowMutations rm = new RowMutations(row);
6433 rm.add(p);
6434 region.checkAndRowMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1),
6435 rm, false);
6436 result = region.get(new Get(row));
6437 c = result.getColumnLatestCell(fam1, qual1);
6438 assertEquals(c.getTimestamp(), 10L);
6439
6440 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
6441 qual2, 0, qual2.length));
6442 }
6443
6444 static HRegion initHRegion(byte[] tableName, String callingMethod,
6445 byte[]... families) throws IOException {
6446 return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
6447 families);
6448 }
6449 }