View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  import static org.junit.Assert.assertArrayEquals;
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertNotNull;
25  import static org.junit.Assert.assertNull;
26  import static org.junit.Assert.assertSame;
27  import static org.junit.Assert.assertTrue;
28  import static org.junit.Assert.fail;
29  
30  import java.io.IOException;
31  import java.lang.reflect.Method;
32  import java.util.ArrayList;
33  import java.util.Arrays;
34  import java.util.Collections;
35  import java.util.HashSet;
36  import java.util.Iterator;
37  import java.util.List;
38  import java.util.Map;
39  import java.util.NavigableMap;
40  import java.util.UUID;
41  import java.util.concurrent.Callable;
42  import java.util.concurrent.ExecutorService;
43  import java.util.concurrent.Executors;
44  import java.util.concurrent.atomic.AtomicReference;
45  
46  import org.apache.log4j.Level;
47  import org.apache.commons.lang.ArrayUtils;
48  import org.apache.commons.logging.Log;
49  import org.apache.commons.logging.LogFactory;
50  import org.apache.hadoop.conf.Configuration;
51  import org.apache.hadoop.hbase.Abortable;
52  import org.apache.hadoop.hbase.Cell;
53  import org.apache.hadoop.hbase.CellUtil;
54  import org.apache.hadoop.hbase.HBaseTestingUtility;
55  import org.apache.hadoop.hbase.HColumnDescriptor;
56  import org.apache.hadoop.hbase.HConstants;
57  import org.apache.hadoop.hbase.HRegionInfo;
58  import org.apache.hadoop.hbase.HRegionLocation;
59  import org.apache.hadoop.hbase.HTableDescriptor;
60  import org.apache.hadoop.hbase.KeyValue;
61  import org.apache.hadoop.hbase.MiniHBaseCluster;
62  import org.apache.hadoop.hbase.RegionLocations;
63  import org.apache.hadoop.hbase.ServerName;
64  import org.apache.hadoop.hbase.TableName;
65  import org.apache.hadoop.hbase.Waiter;
66  import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
67  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
68  import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
69  import org.apache.hadoop.hbase.filter.BinaryComparator;
70  import org.apache.hadoop.hbase.filter.CompareFilter;
71  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
72  import org.apache.hadoop.hbase.filter.Filter;
73  import org.apache.hadoop.hbase.filter.FilterList;
74  import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
75  import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
76  import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
77  import org.apache.hadoop.hbase.filter.LongComparator;
78  import org.apache.hadoop.hbase.filter.PrefixFilter;
79  import org.apache.hadoop.hbase.filter.QualifierFilter;
80  import org.apache.hadoop.hbase.filter.RegexStringComparator;
81  import org.apache.hadoop.hbase.filter.RowFilter;
82  import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
83  import org.apache.hadoop.hbase.filter.WhileMatchFilter;
84  import org.apache.hadoop.hbase.io.hfile.BlockCache;
85  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
86  import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
87  import org.apache.hadoop.hbase.master.HMaster;
88  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
89  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
90  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
91  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
92  import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService;
93  import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
94  import org.apache.hadoop.hbase.regionserver.HRegion;
95  import org.apache.hadoop.hbase.regionserver.HRegionServer;
96  import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
97  import org.apache.hadoop.hbase.regionserver.Region;
98  import org.apache.hadoop.hbase.regionserver.Store;
99  import org.apache.hadoop.hbase.testclassification.LargeTests;
100 import org.apache.hadoop.hbase.util.Bytes;
101 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
102 import org.apache.hadoop.hbase.util.Pair;
103 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
104 import org.apache.log4j.AppenderSkeleton;
105 import org.apache.log4j.Logger;
106 import org.apache.log4j.spi.LoggingEvent;
107 import org.junit.After;
108 import org.junit.AfterClass;
109 import org.junit.Before;
110 import org.junit.BeforeClass;
111 import org.junit.Ignore;
112 import org.junit.Test;
113 import org.junit.experimental.categories.Category;
114 /**
115  * Run tests that use the HBase clients; {@link HTable}.
116  * Sets up the HBase mini cluster once at start and runs through all client tests.
117  * Each creates a table named for the method and does its stuff against that.
118  */
119 @Category(LargeTests.class)
120 @SuppressWarnings ("deprecation")
121 public class TestFromClientSide {
122   // NOTE: Increment tests were moved to their own class, TestIncrementsFromClientSide.
123   final Log LOG = LogFactory.getLog(getClass());
124   protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
125   private static byte [] ROW = Bytes.toBytes("testRow");
126   private static byte [] FAMILY = Bytes.toBytes("testFamily");
127   private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
128   private static byte [] VALUE = Bytes.toBytes("testValue");
129   protected static int SLAVES = 3;
130 
131   /**
132    * @throws java.lang.Exception
133    */
134   @BeforeClass
135   public static void setUpBeforeClass() throws Exception {
136     // Uncomment the following lines if more verbosity is needed for
137     // debugging (see HBASE-12285 for details).
138     //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
139     //((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
140     //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
141     Configuration conf = TEST_UTIL.getConfiguration();
142     conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
143         MultiRowMutationEndpoint.class.getName());
144     conf.setBoolean("hbase.table.sanity.checks", true); // enable for below tests
145     // We need more than one region server in this test
146     TEST_UTIL.startMiniCluster(SLAVES);
147   }
148 
149   /**
150    * @throws java.lang.Exception
151    */
152   @AfterClass
153   public static void tearDownAfterClass() throws Exception {
154     TEST_UTIL.shutdownMiniCluster();
155   }
156 
157   /**
158    * @throws java.lang.Exception
159    */
160   @Before
161   public void setUp() throws Exception {
162     // Nothing to do.
163   }
164 
165   /**
166    * @throws java.lang.Exception
167    */
168   @After
169   public void tearDown() throws Exception {
170     // Nothing to do.
171   }
172 
173   /**
174    * Basic client side validation of HBASE-4536
175    */
176    @Test
177    public void testKeepDeletedCells() throws Exception {
178      final TableName TABLENAME = TableName.valueOf("testKeepDeletesCells");
179      final byte[] FAMILY = Bytes.toBytes("family");
180      final byte[] C0 = Bytes.toBytes("c0");
181 
182      final byte[] T1 = Bytes.toBytes("T1");
183      final byte[] T2 = Bytes.toBytes("T2");
184      final byte[] T3 = Bytes.toBytes("T3");
185      HColumnDescriptor hcd = new HColumnDescriptor(FAMILY)
186          .setKeepDeletedCells(true).setMaxVersions(3);
187 
188      HTableDescriptor desc = new HTableDescriptor(TABLENAME);
189      desc.addFamily(hcd);
190      TEST_UTIL.getHBaseAdmin().createTable(desc);
191      Configuration c = TEST_UTIL.getConfiguration();
192      Table h = new HTable(c, TABLENAME);
193 
194      long ts = System.currentTimeMillis();
195      Put p = new Put(T1, ts);
196      p.add(FAMILY, C0, T1);
197      h.put(p);
198      p = new Put(T1, ts+2);
199      p.add(FAMILY, C0, T2);
200      h.put(p);
201      p = new Put(T1, ts+4);
202      p.add(FAMILY, C0, T3);
203      h.put(p);
204 
205      Delete d = new Delete(T1, ts+3);
206      h.delete(d);
207 
208      d = new Delete(T1, ts+3);
209      d.deleteColumns(FAMILY, C0, ts+3);
210      h.delete(d);
211 
212      Get g = new Get(T1);
213      // does *not* include the delete
214      g.setTimeRange(0, ts+3);
215      Result r = h.get(g);
216      assertArrayEquals(T2, r.getValue(FAMILY, C0));
217 
218      Scan s = new Scan(T1);
219      s.setTimeRange(0, ts+3);
220      s.setMaxVersions();
221      ResultScanner scanner = h.getScanner(s);
222      Cell[] kvs = scanner.next().rawCells();
223      assertArrayEquals(T2, CellUtil.cloneValue(kvs[0]));
224      assertArrayEquals(T1, CellUtil.cloneValue(kvs[1]));
225      scanner.close();
226 
227      s = new Scan(T1);
228      s.setRaw(true);
229      s.setMaxVersions();
230      scanner = h.getScanner(s);
231      kvs = scanner.next().rawCells();
232      assertTrue(CellUtil.isDeleteFamily(kvs[0]));
233      assertArrayEquals(T3, CellUtil.cloneValue(kvs[1]));
234      assertTrue(CellUtil.isDelete(kvs[2]));
235      assertArrayEquals(T2, CellUtil.cloneValue(kvs[3]));
236      assertArrayEquals(T1, CellUtil.cloneValue(kvs[4]));
237      scanner.close();
238      h.close();
239    }
240 
241     /**
242     * Basic client side validation of HBASE-10118
243     */
244    @Test
245    public void testPurgeFutureDeletes() throws Exception {
246      final TableName TABLENAME = TableName.valueOf("testPurgeFutureDeletes");
247      final byte[] ROW = Bytes.toBytes("row");
248      final byte[] FAMILY = Bytes.toBytes("family");
249      final byte[] COLUMN = Bytes.toBytes("column");
250      final byte[] VALUE = Bytes.toBytes("value");
251 
252      Table table = TEST_UTIL.createTable(TABLENAME, FAMILY);
253 
254      // future timestamp
255      long ts = System.currentTimeMillis() * 2;
256      Put put = new Put(ROW, ts);
257      put.add(FAMILY, COLUMN, VALUE);
258      table.put(put);
259 
260      Get get = new Get(ROW);
261      Result result = table.get(get);
262      assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
263 
264      Delete del = new Delete(ROW);
265      del.deleteColumn(FAMILY, COLUMN, ts);
266      table.delete(del);
267 
268      get = new Get(ROW);
269      result = table.get(get);
270      assertNull(result.getValue(FAMILY, COLUMN));
271 
272      // major compaction, purged future deletes
273      TEST_UTIL.getHBaseAdmin().flush(TABLENAME);
274      TEST_UTIL.getHBaseAdmin().majorCompact(TABLENAME);
275 
276      // waiting for the major compaction to complete
277      TEST_UTIL.waitFor(6000, new Waiter.Predicate<IOException>() {
278        @Override
279        public boolean evaluate() throws IOException {
280          return TEST_UTIL.getHBaseAdmin().getCompactionState(TABLENAME) ==
281              AdminProtos.GetRegionInfoResponse.CompactionState.NONE;
282        }
283      });
284 
285      put = new Put(ROW, ts);
286      put.add(FAMILY, COLUMN, VALUE);
287      table.put(put);
288 
289      get = new Get(ROW);
290      result = table.get(get);
291      assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
292 
293      table.close();
294    }
295 
296    /**
297     * @deprecated Tests deprecated functionality. Remove when we are past 1.0.
298     * @throws Exception
299     */
300    @Deprecated
301    @Test
302    public void testSharedZooKeeper() throws Exception {
303      Configuration newConfig = new Configuration(TEST_UTIL.getConfiguration());
304      newConfig.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "12345");
305 
306      // First with a simple ZKW
307      ZooKeeperWatcher z0 = new ZooKeeperWatcher(
308        newConfig, "hconnection", new Abortable() {
309        @Override public void abort(String why, Throwable e) {}
310        @Override public boolean isAborted() {return false;}
311      });
312      z0.getRecoverableZooKeeper().getZooKeeper().exists("/oldZooKeeperWatcher", false);
313      z0.close();
314 
315      // Then a ZooKeeperKeepAliveConnection
316      ConnectionManager.HConnectionImplementation connection1 =
317        (ConnectionManager.HConnectionImplementation)
318          HConnectionManager.getConnection(newConfig);
319 
320      ZooKeeperKeepAliveConnection z1 = connection1.getKeepAliveZooKeeperWatcher();
321      z1.getRecoverableZooKeeper().getZooKeeper().exists("/z1", false);
322 
323      z1.close();
324 
325      // will still work, because the real connection is not closed yet
326      // Not do be done in real code
327      z1.getRecoverableZooKeeper().getZooKeeper().exists("/z1afterclose", false);
328 
329 
330      ZooKeeperKeepAliveConnection z2 = connection1.getKeepAliveZooKeeperWatcher();
331      assertTrue(
332        "ZooKeeperKeepAliveConnection equals on same connection", z1 == z2);
333 
334 
335 
336      Configuration newConfig2 = new Configuration(TEST_UTIL.getConfiguration());
337      newConfig2.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "6789");
338      ConnectionManager.HConnectionImplementation connection2 =
339        (ConnectionManager.HConnectionImplementation)
340          HConnectionManager.getConnection(newConfig2);
341 
342      assertTrue("connections should be different ", connection1 != connection2);
343 
344      ZooKeeperKeepAliveConnection z3 = connection2.getKeepAliveZooKeeperWatcher();
345      assertTrue(
346        "ZooKeeperKeepAliveConnection should be different" +
347          " on different connections", z1 != z3);
348 
349      // Bypass the private access
350      Method m = ConnectionManager.HConnectionImplementation.class.
351        getDeclaredMethod("closeZooKeeperWatcher");
352      m.setAccessible(true);
353      m.invoke(connection2);
354 
355      ZooKeeperKeepAliveConnection z4 = connection2.getKeepAliveZooKeeperWatcher();
356      assertTrue(
357        "ZooKeeperKeepAliveConnection should be recreated" +
358          " when previous connections was closed"
359        , z3 != z4);
360 
361 
362      z2.getRecoverableZooKeeper().getZooKeeper().exists("/z2", false);
363      z4.getRecoverableZooKeeper().getZooKeeper().exists("/z4", false);
364 
365 
366      HConnectionManager.deleteConnection(newConfig);
367      try {
368        z2.getRecoverableZooKeeper().getZooKeeper().exists("/z2", false);
369        assertTrue("We should not have a valid connection for z2", false);
370      } catch (Exception e){
371      }
372 
373      z4.getRecoverableZooKeeper().getZooKeeper().exists("/z4", false);
374      // We expect success here.
375 
376 
377      HConnectionManager.deleteConnection(newConfig2);
378      try {
379        z4.getRecoverableZooKeeper().getZooKeeper().exists("/z4", false);
380        assertTrue("We should not have a valid connection for z4", false);
381      } catch (Exception e){
382      }
383    }
384 
385 
386   /**
387    * Verifies that getConfiguration returns the same Configuration object used
388    * to create the HTable instance.
389    */
390   @Test
391   public void testGetConfiguration() throws Exception {
392     TableName TABLE = TableName.valueOf("testGetConfiguration");
393     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
394     Configuration conf = TEST_UTIL.getConfiguration();
395     Table table = TEST_UTIL.createTable(TABLE, FAMILIES, conf);
396     assertSame(conf, table.getConfiguration());
397   }
398 
399   /**
400    * Test from client side of an involved filter against a multi family that
401    * involves deletes.
402    *
403    * @throws Exception
404    */
405   @Test
406   public void testWeirdCacheBehaviour() throws Exception {
407     TableName TABLE = TableName.valueOf("testWeirdCacheBehaviour");
408     byte [][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
409         Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
410         Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
411     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
412     String value = "this is the value";
413     String value2 = "this is some other value";
414     String keyPrefix1 = UUID.randomUUID().toString();
415     String keyPrefix2 = UUID.randomUUID().toString();
416     String keyPrefix3 = UUID.randomUUID().toString();
417     putRows(ht, 3, value, keyPrefix1);
418     putRows(ht, 3, value, keyPrefix2);
419     putRows(ht, 3, value, keyPrefix3);
420     ht.flushCommits();
421     putRows(ht, 3, value2, keyPrefix1);
422     putRows(ht, 3, value2, keyPrefix2);
423     putRows(ht, 3, value2, keyPrefix3);
424     Table table = new HTable(TEST_UTIL.getConfiguration(), TABLE);
425     System.out.println("Checking values for key: " + keyPrefix1);
426     assertEquals("Got back incorrect number of rows from scan", 3,
427         getNumberOfRows(keyPrefix1, value2, table));
428     System.out.println("Checking values for key: " + keyPrefix2);
429     assertEquals("Got back incorrect number of rows from scan", 3,
430         getNumberOfRows(keyPrefix2, value2, table));
431     System.out.println("Checking values for key: " + keyPrefix3);
432     assertEquals("Got back incorrect number of rows from scan", 3,
433         getNumberOfRows(keyPrefix3, value2, table));
434     deleteColumns(ht, value2, keyPrefix1);
435     deleteColumns(ht, value2, keyPrefix2);
436     deleteColumns(ht, value2, keyPrefix3);
437     System.out.println("Starting important checks.....");
438     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
439       0, getNumberOfRows(keyPrefix1, value2, table));
440     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
441       0, getNumberOfRows(keyPrefix2, value2, table));
442     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
443       0, getNumberOfRows(keyPrefix3, value2, table));
444     ht.setScannerCaching(0);
445     assertEquals("Got back incorrect number of rows from scan", 0,
446       getNumberOfRows(keyPrefix1, value2, table)); ht.setScannerCaching(100);
447     assertEquals("Got back incorrect number of rows from scan", 0,
448       getNumberOfRows(keyPrefix2, value2, table));
449   }
450 
451   private void deleteColumns(Table ht, String value, String keyPrefix)
452   throws IOException {
453     ResultScanner scanner = buildScanner(keyPrefix, value, ht);
454     Iterator<Result> it = scanner.iterator();
455     int count = 0;
456     while (it.hasNext()) {
457       Result result = it.next();
458       Delete delete = new Delete(result.getRow());
459       delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
460       ht.delete(delete);
461       count++;
462     }
463     assertEquals("Did not perform correct number of deletes", 3, count);
464   }
465 
466   private int getNumberOfRows(String keyPrefix, String value, Table ht)
467       throws Exception {
468     ResultScanner resultScanner = buildScanner(keyPrefix, value, ht);
469     Iterator<Result> scanner = resultScanner.iterator();
470     int numberOfResults = 0;
471     while (scanner.hasNext()) {
472       Result result = scanner.next();
473       System.out.println("Got back key: " + Bytes.toString(result.getRow()));
474       for (Cell kv : result.rawCells()) {
475         System.out.println("kv=" + kv.toString() + ", "
476             + Bytes.toString(CellUtil.cloneValue(kv)));
477       }
478       numberOfResults++;
479     }
480     return numberOfResults;
481   }
482 
483   private ResultScanner buildScanner(String keyPrefix, String value, Table ht)
484       throws IOException {
485     // OurFilterList allFilters = new OurFilterList();
486     FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */);
487     allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
488     SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes
489         .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes
490         .toBytes(value));
491     filter.setFilterIfMissing(true);
492     allFilters.addFilter(filter);
493 
494     // allFilters.addFilter(new
495     // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"),
496     // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)));
497 
498     Scan scan = new Scan();
499     scan.addFamily(Bytes.toBytes("trans-blob"));
500     scan.addFamily(Bytes.toBytes("trans-type"));
501     scan.addFamily(Bytes.toBytes("trans-date"));
502     scan.addFamily(Bytes.toBytes("trans-tags"));
503     scan.addFamily(Bytes.toBytes("trans-group"));
504     scan.setFilter(allFilters);
505 
506     return ht.getScanner(scan);
507   }
508 
509   private void putRows(Table ht, int numRows, String value, String key)
510       throws IOException {
511     for (int i = 0; i < numRows; i++) {
512       String row = key + "_" + UUID.randomUUID().toString();
513       System.out.println(String.format("Saving row: %s, with value %s", row,
514           value));
515       Put put = new Put(Bytes.toBytes(row));
516       put.setDurability(Durability.SKIP_WAL);
517       put.add(Bytes.toBytes("trans-blob"), null, Bytes
518           .toBytes("value for blob"));
519       put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
520       put.add(Bytes.toBytes("trans-date"), null, Bytes
521           .toBytes("20090921010101999"));
522       put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes
523           .toBytes(value));
524       put.add(Bytes.toBytes("trans-group"), null, Bytes
525           .toBytes("adhocTransactionGroupId"));
526       ht.put(put);
527     }
528   }
529 
530   /**
531    * Test filters when multiple regions.  It does counts.  Needs eye-balling of
532    * logs to ensure that we're not scanning more regions that we're supposed to.
533    * Related to the TestFilterAcrossRegions over in the o.a.h.h.filter package.
534    * @throws IOException
535    * @throws InterruptedException
536    */
537   @Test
538   public void testFilterAcrossMultipleRegions()
539   throws IOException, InterruptedException {
540     TableName name = TableName.valueOf("testFilterAcrossMutlipleRegions");
541     HTable t = TEST_UTIL.createTable(name, FAMILY);
542     int rowCount = TEST_UTIL.loadTable(t, FAMILY, false);
543     assertRowCount(t, rowCount);
544     // Split the table.  Should split on a reasonable key; 'lqj'
545     Map<HRegionInfo, ServerName> regions  = splitTable(t);
546     assertRowCount(t, rowCount);
547     // Get end key of first region.
548     byte [] endKey = regions.keySet().iterator().next().getEndKey();
549     // Count rows with a filter that stops us before passed 'endKey'.
550     // Should be count of rows in first region.
551     int endKeyCount = countRows(t, createScanWithRowFilter(endKey));
552     assertTrue(endKeyCount < rowCount);
553 
554     // How do I know I did not got to second region?  Thats tough.  Can't really
555     // do that in client-side region test.  I verified by tracing in debugger.
556     // I changed the messages that come out when set to DEBUG so should see
557     // when scanner is done. Says "Finished with scanning..." with region name.
558     // Check that its finished in right region.
559 
560     // New test.  Make it so scan goes into next region by one and then two.
561     // Make sure count comes out right.
562     byte [] key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 1)};
563     int plusOneCount = countRows(t, createScanWithRowFilter(key));
564     assertEquals(endKeyCount + 1, plusOneCount);
565     key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 2)};
566     int plusTwoCount = countRows(t, createScanWithRowFilter(key));
567     assertEquals(endKeyCount + 2, plusTwoCount);
568 
569     // New test.  Make it so I scan one less than endkey.
570     key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] - 1)};
571     int minusOneCount = countRows(t, createScanWithRowFilter(key));
572     assertEquals(endKeyCount - 1, minusOneCount);
573     // For above test... study logs.  Make sure we do "Finished with scanning.."
574     // in first region and that we do not fall into the next region.
575 
576     key = new byte [] {'a', 'a', 'a'};
577     int countBBB = countRows(t,
578       createScanWithRowFilter(key, null, CompareFilter.CompareOp.EQUAL));
579     assertEquals(1, countBBB);
580 
581     int countGreater = countRows(t, createScanWithRowFilter(endKey, null,
582       CompareFilter.CompareOp.GREATER_OR_EQUAL));
583     // Because started at start of table.
584     assertEquals(0, countGreater);
585     countGreater = countRows(t, createScanWithRowFilter(endKey, endKey,
586       CompareFilter.CompareOp.GREATER_OR_EQUAL));
587     assertEquals(rowCount - endKeyCount, countGreater);
588   }
589 
590   /*
591    * @param key
592    * @return Scan with RowFilter that does LESS than passed key.
593    */
594   private Scan createScanWithRowFilter(final byte [] key) {
595     return createScanWithRowFilter(key, null, CompareFilter.CompareOp.LESS);
596   }
597 
598   /*
599    * @param key
600    * @param op
601    * @param startRow
602    * @return Scan with RowFilter that does CompareOp op on passed key.
603    */
604   private Scan createScanWithRowFilter(final byte [] key,
605       final byte [] startRow, CompareFilter.CompareOp op) {
606     // Make sure key is of some substance... non-null and > than first key.
607     assertTrue(key != null && key.length > 0 &&
608       Bytes.BYTES_COMPARATOR.compare(key, new byte [] {'a', 'a', 'a'}) >= 0);
609     LOG.info("Key=" + Bytes.toString(key));
610     Scan s = startRow == null? new Scan(): new Scan(startRow);
611     Filter f = new RowFilter(op, new BinaryComparator(key));
612     f = new WhileMatchFilter(f);
613     s.setFilter(f);
614     return s;
615   }
616 
617   /*
618    * @param t
619    * @param s
620    * @return Count of rows in table.
621    * @throws IOException
622    */
623   private int countRows(final Table t, final Scan s)
624   throws IOException {
625     // Assert all rows in table.
626     ResultScanner scanner = t.getScanner(s);
627     int count = 0;
628     for (Result result: scanner) {
629       count++;
630       assertTrue(result.size() > 0);
631       // LOG.info("Count=" + count + ", row=" + Bytes.toString(result.getRow()));
632     }
633     return count;
634   }
635 
636   private void assertRowCount(final Table t, final int expected)
637   throws IOException {
638     assertEquals(expected, countRows(t, new Scan()));
639   }
640 
641   /*
642    * Split table into multiple regions.
643    * @param t Table to split.
644    * @return Map of regions to servers.
645    * @throws IOException
646    */
647   private Map<HRegionInfo, ServerName> splitTable(final HTable t)
648   throws IOException, InterruptedException {
649     // Split this table in two.
650     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
651     admin.split(t.getTableName());
652     admin.close();
653     Map<HRegionInfo, ServerName> regions = waitOnSplit(t);
654     assertTrue(regions.size() > 1);
655     return regions;
656   }
657 
658   /*
659    * Wait on table split.  May return because we waited long enough on the split
660    * and it didn't happen.  Caller should check.
661    * @param t
662    * @return Map of table regions; caller needs to check table actually split.
663    */
664   private Map<HRegionInfo, ServerName> waitOnSplit(final HTable t)
665   throws IOException {
666     Map<HRegionInfo, ServerName> regions = t.getRegionLocations();
667     int originalCount = regions.size();
668     for (int i = 0; i < TEST_UTIL.getConfiguration().getInt("hbase.test.retries", 30); i++) {
669       Thread.currentThread();
670       try {
671         Thread.sleep(1000);
672       } catch (InterruptedException e) {
673         e.printStackTrace();
674       }
675       regions = t.getRegionLocations();
676       if (regions.size() > originalCount) break;
677     }
678     return regions;
679   }
680 
681   @Test
682   public void testSuperSimple() throws Exception {
683     byte [] TABLE = Bytes.toBytes("testSuperSimple");
684     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
685     Put put = new Put(ROW);
686     put.add(FAMILY, QUALIFIER, VALUE);
687     ht.put(put);
688     Scan scan = new Scan();
689     scan.addColumn(FAMILY, TABLE);
690     ResultScanner scanner = ht.getScanner(scan);
691     Result result = scanner.next();
692     assertTrue("Expected null result", result == null);
693     scanner.close();
694   }
695 
696   @Test
697   public void testMaxKeyValueSize() throws Exception {
698     byte [] TABLE = Bytes.toBytes("testMaxKeyValueSize");
699     Configuration conf = TEST_UTIL.getConfiguration();
700     String oldMaxSize = conf.get(TableConfiguration.MAX_KEYVALUE_SIZE_KEY);
701     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
702     byte[] value = new byte[4 * 1024 * 1024];
703     Put put = new Put(ROW);
704     put.add(FAMILY, QUALIFIER, value);
705     ht.put(put);
706     try {
707       TEST_UTIL.getConfiguration().setInt(
708           TableConfiguration.MAX_KEYVALUE_SIZE_KEY, 2 * 1024 * 1024);
709       // Create new table so we pick up the change in Configuration.
710       try (Connection connection =
711           ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
712         try (Table t = connection.getTable(TableName.valueOf(FAMILY))) {
713           put = new Put(ROW);
714           put.add(FAMILY, QUALIFIER, value);
715           t.put(put);
716         }
717       }
718       fail("Inserting a too large KeyValue worked, should throw exception");
719     } catch(Exception e) {}
720     conf.set(TableConfiguration.MAX_KEYVALUE_SIZE_KEY, oldMaxSize);
721   }
722 
723   @Test
724   public void testFilters() throws Exception {
725     byte [] TABLE = Bytes.toBytes("testFilters");
726     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
727     byte [][] ROWS = makeN(ROW, 10);
728     byte [][] QUALIFIERS = {
729         Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
730         Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
731         Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
732         Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
733         Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
734     };
735     for(int i=0;i<10;i++) {
736       Put put = new Put(ROWS[i]);
737       put.setDurability(Durability.SKIP_WAL);
738       put.add(FAMILY, QUALIFIERS[i], VALUE);
739       ht.put(put);
740     }
741     Scan scan = new Scan();
742     scan.addFamily(FAMILY);
743     Filter filter = new QualifierFilter(CompareOp.EQUAL,
744       new RegexStringComparator("col[1-5]"));
745     scan.setFilter(filter);
746     ResultScanner scanner = ht.getScanner(scan);
747     int expectedIndex = 1;
748     for(Result result : ht.getScanner(scan)) {
749       assertEquals(result.size(), 1);
750       assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[expectedIndex]));
751       assertTrue(Bytes.equals(CellUtil.cloneQualifier(result.rawCells()[0]),
752           QUALIFIERS[expectedIndex]));
753       expectedIndex++;
754     }
755     assertEquals(expectedIndex, 6);
756     scanner.close();
757   }
758 
759   @Test
760   public void testFilterWithLongCompartor() throws Exception {
761     byte [] TABLE = Bytes.toBytes("testFilterWithLongCompartor");
762     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
763     byte [][] ROWS = makeN(ROW, 10);
764     byte [][] values = new byte[10][];
765     for (int i = 0; i < 10; i ++) {
766         values[i] = Bytes.toBytes(100L * i);
767     }
768     for(int i = 0; i < 10; i ++) {
769       Put put = new Put(ROWS[i]);
770       put.setDurability(Durability.SKIP_WAL);
771       put.add(FAMILY, QUALIFIER, values[i]);
772       ht.put(put);
773     }
774     Scan scan = new Scan();
775     scan.addFamily(FAMILY);
776     Filter filter = new SingleColumnValueFilter(FAMILY, QUALIFIER, CompareOp.GREATER,
777       new LongComparator(500));
778     scan.setFilter(filter);
779     ResultScanner scanner = ht.getScanner(scan);
780     int expectedIndex = 0;
781     for(Result result : ht.getScanner(scan)) {
782       assertEquals(result.size(), 1);
783       assertTrue(Bytes.toLong(result.getValue(FAMILY, QUALIFIER)) > 500);
784       expectedIndex++;
785     }
786     assertEquals(expectedIndex, 4);
787     scanner.close();
788 }
789 
790   @Test
791   public void testKeyOnlyFilter() throws Exception {
792     byte [] TABLE = Bytes.toBytes("testKeyOnlyFilter");
793     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
794     byte [][] ROWS = makeN(ROW, 10);
795     byte [][] QUALIFIERS = {
796         Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
797         Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
798         Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
799         Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
800         Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
801     };
802     for(int i=0;i<10;i++) {
803       Put put = new Put(ROWS[i]);
804       put.setDurability(Durability.SKIP_WAL);
805       put.add(FAMILY, QUALIFIERS[i], VALUE);
806       ht.put(put);
807     }
808     Scan scan = new Scan();
809     scan.addFamily(FAMILY);
810     Filter filter = new KeyOnlyFilter(true);
811     scan.setFilter(filter);
812     ResultScanner scanner = ht.getScanner(scan);
813     int count = 0;
814     for(Result result : ht.getScanner(scan)) {
815       assertEquals(result.size(), 1);
816       assertEquals(result.rawCells()[0].getValueLength(), Bytes.SIZEOF_INT);
817       assertEquals(Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])), VALUE.length);
818       count++;
819     }
820     assertEquals(count, 10);
821     scanner.close();
822   }
823 
824   /**
825    * Test simple table and non-existent row cases.
826    */
827   @Test
828   public void testSimpleMissing() throws Exception {
829     byte [] TABLE = Bytes.toBytes("testSimpleMissing");
830     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
831     byte [][] ROWS = makeN(ROW, 4);
832 
833     // Try to get a row on an empty table
834     Get get = new Get(ROWS[0]);
835     Result result = ht.get(get);
836     assertEmptyResult(result);
837 
838     get = new Get(ROWS[0]);
839     get.addFamily(FAMILY);
840     result = ht.get(get);
841     assertEmptyResult(result);
842 
843     get = new Get(ROWS[0]);
844     get.addColumn(FAMILY, QUALIFIER);
845     result = ht.get(get);
846     assertEmptyResult(result);
847 
848     Scan scan = new Scan();
849     result = getSingleScanResult(ht, scan);
850     assertNullResult(result);
851 
852 
853     scan = new Scan(ROWS[0]);
854     result = getSingleScanResult(ht, scan);
855     assertNullResult(result);
856 
857     scan = new Scan(ROWS[0],ROWS[1]);
858     result = getSingleScanResult(ht, scan);
859     assertNullResult(result);
860 
861     scan = new Scan();
862     scan.addFamily(FAMILY);
863     result = getSingleScanResult(ht, scan);
864     assertNullResult(result);
865 
866     scan = new Scan();
867     scan.addColumn(FAMILY, QUALIFIER);
868     result = getSingleScanResult(ht, scan);
869     assertNullResult(result);
870 
871     // Insert a row
872 
873     Put put = new Put(ROWS[2]);
874     put.add(FAMILY, QUALIFIER, VALUE);
875     ht.put(put);
876 
877     // Try to get empty rows around it
878 
879     get = new Get(ROWS[1]);
880     result = ht.get(get);
881     assertEmptyResult(result);
882 
883     get = new Get(ROWS[0]);
884     get.addFamily(FAMILY);
885     result = ht.get(get);
886     assertEmptyResult(result);
887 
888     get = new Get(ROWS[3]);
889     get.addColumn(FAMILY, QUALIFIER);
890     result = ht.get(get);
891     assertEmptyResult(result);
892 
893     // Try to scan empty rows around it
894 
895     scan = new Scan(ROWS[3]);
896     result = getSingleScanResult(ht, scan);
897     assertNullResult(result);
898 
899     scan = new Scan(ROWS[0],ROWS[2]);
900     result = getSingleScanResult(ht, scan);
901     assertNullResult(result);
902 
903     // Make sure we can actually get the row
904 
905     get = new Get(ROWS[2]);
906     result = ht.get(get);
907     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
908 
909     get = new Get(ROWS[2]);
910     get.addFamily(FAMILY);
911     result = ht.get(get);
912     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
913 
914     get = new Get(ROWS[2]);
915     get.addColumn(FAMILY, QUALIFIER);
916     result = ht.get(get);
917     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
918 
919     // Make sure we can scan the row
920 
921     scan = new Scan();
922     result = getSingleScanResult(ht, scan);
923     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
924 
925     scan = new Scan(ROWS[0],ROWS[3]);
926     result = getSingleScanResult(ht, scan);
927     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
928 
929     scan = new Scan(ROWS[2],ROWS[3]);
930     result = getSingleScanResult(ht, scan);
931     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
932   }
933 
934   /**
935    * Test basic puts, gets, scans, and deletes for a single row
936    * in a multiple family table.
937    */
938   @Test
939   public void testSingleRowMultipleFamily() throws Exception {
940     byte [] TABLE = Bytes.toBytes("testSingleRowMultipleFamily");
941     byte [][] ROWS = makeN(ROW, 3);
942     byte [][] FAMILIES = makeNAscii(FAMILY, 10);
943     byte [][] QUALIFIERS = makeN(QUALIFIER, 10);
944     byte [][] VALUES = makeN(VALUE, 10);
945 
946     Table ht = TEST_UTIL.createTable(TABLE, FAMILIES);
947 
948     Get get;
949     Scan scan;
950     Delete delete;
951     Put put;
952     Result result;
953 
954     ////////////////////////////////////////////////////////////////////////////
955     // Insert one column to one family
956     ////////////////////////////////////////////////////////////////////////////
957 
958     put = new Put(ROWS[0]);
959     put.add(FAMILIES[4], QUALIFIERS[0], VALUES[0]);
960     ht.put(put);
961 
962     // Get the single column
963     getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
964 
965     // Scan the single column
966     scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
967 
968     // Get empty results around inserted column
969     getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
970 
971     // Scan empty results around inserted column
972     scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
973 
974     ////////////////////////////////////////////////////////////////////////////
975     // Flush memstore and run same tests from storefiles
976     ////////////////////////////////////////////////////////////////////////////
977 
978     TEST_UTIL.flush();
979 
980     // Redo get and scan tests from storefile
981     getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
982     scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
983     getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
984     scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
985 
986     ////////////////////////////////////////////////////////////////////////////
987     // Now, Test reading from memstore and storefiles at once
988     ////////////////////////////////////////////////////////////////////////////
989 
990     // Insert multiple columns to two other families
991     put = new Put(ROWS[0]);
992     put.add(FAMILIES[2], QUALIFIERS[2], VALUES[2]);
993     put.add(FAMILIES[2], QUALIFIERS[4], VALUES[4]);
994     put.add(FAMILIES[4], QUALIFIERS[4], VALUES[4]);
995     put.add(FAMILIES[6], QUALIFIERS[6], VALUES[6]);
996     put.add(FAMILIES[6], QUALIFIERS[7], VALUES[7]);
997     put.add(FAMILIES[7], QUALIFIERS[7], VALUES[7]);
998     put.add(FAMILIES[9], QUALIFIERS[0], VALUES[0]);
999     ht.put(put);
1000 
1001     // Get multiple columns across multiple families and get empties around it
1002     singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1003 
1004     // Scan multiple columns across multiple families and scan empties around it
1005     singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1006 
1007     ////////////////////////////////////////////////////////////////////////////
1008     // Flush the table again
1009     ////////////////////////////////////////////////////////////////////////////
1010 
1011     TEST_UTIL.flush();
1012 
1013     // Redo tests again
1014     singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1015     singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1016 
1017     // Insert more data to memstore
1018     put = new Put(ROWS[0]);
1019     put.add(FAMILIES[6], QUALIFIERS[5], VALUES[5]);
1020     put.add(FAMILIES[6], QUALIFIERS[8], VALUES[8]);
1021     put.add(FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1022     put.add(FAMILIES[4], QUALIFIERS[3], VALUES[3]);
1023     ht.put(put);
1024 
1025     ////////////////////////////////////////////////////////////////////////////
1026     // Delete a storefile column
1027     ////////////////////////////////////////////////////////////////////////////
1028     delete = new Delete(ROWS[0]);
1029     delete.deleteColumns(FAMILIES[6], QUALIFIERS[7]);
1030     ht.delete(delete);
1031 
1032     // Try to get deleted column
1033     get = new Get(ROWS[0]);
1034     get.addColumn(FAMILIES[6], QUALIFIERS[7]);
1035     result = ht.get(get);
1036     assertEmptyResult(result);
1037 
1038     // Try to scan deleted column
1039     scan = new Scan();
1040     scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
1041     result = getSingleScanResult(ht, scan);
1042     assertNullResult(result);
1043 
1044     // Make sure we can still get a column before it and after it
1045     get = new Get(ROWS[0]);
1046     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
1047     result = ht.get(get);
1048     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1049 
1050     get = new Get(ROWS[0]);
1051     get.addColumn(FAMILIES[6], QUALIFIERS[8]);
1052     result = ht.get(get);
1053     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);
1054 
1055     // Make sure we can still scan a column before it and after it
1056     scan = new Scan();
1057     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1058     result = getSingleScanResult(ht, scan);
1059     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1060 
1061     scan = new Scan();
1062     scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
1063     result = getSingleScanResult(ht, scan);
1064     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);
1065 
1066     ////////////////////////////////////////////////////////////////////////////
1067     // Delete a memstore column
1068     ////////////////////////////////////////////////////////////////////////////
1069     delete = new Delete(ROWS[0]);
1070     delete.deleteColumns(FAMILIES[6], QUALIFIERS[8]);
1071     ht.delete(delete);
1072 
1073     // Try to get deleted column
1074     get = new Get(ROWS[0]);
1075     get.addColumn(FAMILIES[6], QUALIFIERS[8]);
1076     result = ht.get(get);
1077     assertEmptyResult(result);
1078 
1079     // Try to scan deleted column
1080     scan = new Scan();
1081     scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
1082     result = getSingleScanResult(ht, scan);
1083     assertNullResult(result);
1084 
1085     // Make sure we can still get a column before it and after it
1086     get = new Get(ROWS[0]);
1087     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
1088     result = ht.get(get);
1089     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1090 
1091     get = new Get(ROWS[0]);
1092     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1093     result = ht.get(get);
1094     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1095 
1096     // Make sure we can still scan a column before it and after it
1097     scan = new Scan();
1098     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1099     result = getSingleScanResult(ht, scan);
1100     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1101 
1102     scan = new Scan();
1103     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1104     result = getSingleScanResult(ht, scan);
1105     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1106 
1107     ////////////////////////////////////////////////////////////////////////////
1108     // Delete joint storefile/memstore family
1109     ////////////////////////////////////////////////////////////////////////////
1110 
1111     delete = new Delete(ROWS[0]);
1112     delete.deleteFamily(FAMILIES[4]);
1113     ht.delete(delete);
1114 
1115     // Try to get storefile column in deleted family
1116     get = new Get(ROWS[0]);
1117     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
1118     result = ht.get(get);
1119     assertEmptyResult(result);
1120 
1121     // Try to get memstore column in deleted family
1122     get = new Get(ROWS[0]);
1123     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
1124     result = ht.get(get);
1125     assertEmptyResult(result);
1126 
1127     // Try to get deleted family
1128     get = new Get(ROWS[0]);
1129     get.addFamily(FAMILIES[4]);
1130     result = ht.get(get);
1131     assertEmptyResult(result);
1132 
1133     // Try to scan storefile column in deleted family
1134     scan = new Scan();
1135     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
1136     result = getSingleScanResult(ht, scan);
1137     assertNullResult(result);
1138 
1139     // Try to scan memstore column in deleted family
1140     scan = new Scan();
1141     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
1142     result = getSingleScanResult(ht, scan);
1143     assertNullResult(result);
1144 
1145     // Try to scan deleted family
1146     scan = new Scan();
1147     scan.addFamily(FAMILIES[4]);
1148     result = getSingleScanResult(ht, scan);
1149     assertNullResult(result);
1150 
1151     // Make sure we can still get another family
1152     get = new Get(ROWS[0]);
1153     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
1154     result = ht.get(get);
1155     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
1156 
1157     get = new Get(ROWS[0]);
1158     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1159     result = ht.get(get);
1160     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1161 
1162     // Make sure we can still scan another family
1163     scan = new Scan();
1164     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1165     result = getSingleScanResult(ht, scan);
1166     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1167 
1168     scan = new Scan();
1169     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1170     result = getSingleScanResult(ht, scan);
1171     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1172 
1173     ////////////////////////////////////////////////////////////////////////////
1174     // Flush everything and rerun delete tests
1175     ////////////////////////////////////////////////////////////////////////////
1176 
1177     TEST_UTIL.flush();
1178 
1179     // Try to get storefile column in deleted family
1180     get = new Get(ROWS[0]);
1181     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
1182     result = ht.get(get);
1183     assertEmptyResult(result);
1184 
1185     // Try to get memstore column in deleted family
1186     get = new Get(ROWS[0]);
1187     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
1188     result = ht.get(get);
1189     assertEmptyResult(result);
1190 
1191     // Try to get deleted family
1192     get = new Get(ROWS[0]);
1193     get.addFamily(FAMILIES[4]);
1194     result = ht.get(get);
1195     assertEmptyResult(result);
1196 
1197     // Try to scan storefile column in deleted family
1198     scan = new Scan();
1199     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
1200     result = getSingleScanResult(ht, scan);
1201     assertNullResult(result);
1202 
1203     // Try to scan memstore column in deleted family
1204     scan = new Scan();
1205     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
1206     result = getSingleScanResult(ht, scan);
1207     assertNullResult(result);
1208 
1209     // Try to scan deleted family
1210     scan = new Scan();
1211     scan.addFamily(FAMILIES[4]);
1212     result = getSingleScanResult(ht, scan);
1213     assertNullResult(result);
1214 
1215     // Make sure we can still get another family
1216     get = new Get(ROWS[0]);
1217     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
1218     result = ht.get(get);
1219     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
1220 
1221     get = new Get(ROWS[0]);
1222     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1223     result = ht.get(get);
1224     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1225 
1226     // Make sure we can still scan another family
1227     scan = new Scan();
1228     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1229     result = getSingleScanResult(ht, scan);
1230     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1231 
1232     scan = new Scan();
1233     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1234     result = getSingleScanResult(ht, scan);
1235     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1236 
1237   }
1238 
1239   @Test
1240   public void testNull() throws Exception {
1241     byte [] TABLE = Bytes.toBytes("testNull");
1242 
1243     // Null table name (should NOT work)
1244     try {
1245       TEST_UTIL.createTable((TableName)null, FAMILY);
1246       fail("Creating a table with null name passed, should have failed");
1247     } catch(Exception e) {}
1248 
1249     // Null family (should NOT work)
1250     try {
1251       TEST_UTIL.createTable(TABLE, new byte[][]{(byte[])null});
1252       fail("Creating a table with a null family passed, should fail");
1253     } catch(Exception e) {}
1254 
1255     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
1256 
1257     // Null row (should NOT work)
1258     try {
1259       Put put = new Put((byte[])null);
1260       put.add(FAMILY, QUALIFIER, VALUE);
1261       ht.put(put);
1262       fail("Inserting a null row worked, should throw exception");
1263     } catch(Exception e) {}
1264 
1265     // Null qualifier (should work)
1266     {
1267       Put put = new Put(ROW);
1268       put.add(FAMILY, null, VALUE);
1269       ht.put(put);
1270 
1271       getTestNull(ht, ROW, FAMILY, VALUE);
1272 
1273       scanTestNull(ht, ROW, FAMILY, VALUE);
1274 
1275       Delete delete = new Delete(ROW);
1276       delete.deleteColumns(FAMILY, null);
1277       ht.delete(delete);
1278 
1279       Get get = new Get(ROW);
1280       Result result = ht.get(get);
1281       assertEmptyResult(result);
1282     }
1283 
1284     // Use a new table
1285     byte [] TABLE2 = Bytes.toBytes("testNull2");
1286     ht = TEST_UTIL.createTable(TableName.valueOf(TABLE2), FAMILY);
1287 
1288     // Empty qualifier, byte[0] instead of null (should work)
1289     try {
1290       Put put = new Put(ROW);
1291       put.add(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
1292       ht.put(put);
1293 
1294       getTestNull(ht, ROW, FAMILY, VALUE);
1295 
1296       scanTestNull(ht, ROW, FAMILY, VALUE);
1297 
1298       // Flush and try again
1299 
1300       TEST_UTIL.flush();
1301 
1302       getTestNull(ht, ROW, FAMILY, VALUE);
1303 
1304       scanTestNull(ht, ROW, FAMILY, VALUE);
1305 
1306       Delete delete = new Delete(ROW);
1307       delete.deleteColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
1308       ht.delete(delete);
1309 
1310       Get get = new Get(ROW);
1311       Result result = ht.get(get);
1312       assertEmptyResult(result);
1313 
1314     } catch(Exception e) {
1315       throw new IOException("Using a row with null qualifier threw exception, should ");
1316     }
1317 
1318     // Null value
1319     try {
1320       Put put = new Put(ROW);
1321       put.add(FAMILY, QUALIFIER, null);
1322       ht.put(put);
1323 
1324       Get get = new Get(ROW);
1325       get.addColumn(FAMILY, QUALIFIER);
1326       Result result = ht.get(get);
1327       assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
1328 
1329       Scan scan = new Scan();
1330       scan.addColumn(FAMILY, QUALIFIER);
1331       result = getSingleScanResult(ht, scan);
1332       assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
1333 
1334       Delete delete = new Delete(ROW);
1335       delete.deleteColumns(FAMILY, QUALIFIER);
1336       ht.delete(delete);
1337 
1338       get = new Get(ROW);
1339       result = ht.get(get);
1340       assertEmptyResult(result);
1341 
1342     } catch(Exception e) {
1343       throw new IOException("Null values should be allowed, but threw exception");
1344     }
1345   }
1346 
1347   @Test
1348   public void testVersions() throws Exception {
1349     byte [] TABLE = Bytes.toBytes("testVersions");
1350 
1351     long [] STAMPS = makeStamps(20);
1352     byte [][] VALUES = makeNAscii(VALUE, 20);
1353 
1354     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
1355 
1356     // Insert 4 versions of same column
1357     Put put = new Put(ROW);
1358     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1359     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1360     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1361     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1362     ht.put(put);
1363 
1364     // Verify we can get each one properly
1365     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1366     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1367     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1368     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1369     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1370     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1371     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1372     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1373 
1374     // Verify we don't accidentally get others
1375     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1376     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1377     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1378     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1379     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1380     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1381 
1382     // Ensure maxVersions in query is respected
1383     Get get = new Get(ROW);
1384     get.addColumn(FAMILY, QUALIFIER);
1385     get.setMaxVersions(2);
1386     Result result = ht.get(get);
1387     assertNResult(result, ROW, FAMILY, QUALIFIER,
1388         new long [] {STAMPS[4], STAMPS[5]},
1389         new byte[][] {VALUES[4], VALUES[5]},
1390         0, 1);
1391 
1392     Scan scan = new Scan(ROW);
1393     scan.addColumn(FAMILY, QUALIFIER);
1394     scan.setMaxVersions(2);
1395     result = getSingleScanResult(ht, scan);
1396     assertNResult(result, ROW, FAMILY, QUALIFIER,
1397         new long [] {STAMPS[4], STAMPS[5]},
1398         new byte[][] {VALUES[4], VALUES[5]},
1399         0, 1);
1400 
1401     // Flush and redo
1402 
1403     TEST_UTIL.flush();
1404 
1405     // Verify we can get each one properly
1406     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1407     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1408     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1409     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1410     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1411     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1412     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1413     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1414 
1415     // Verify we don't accidentally get others
1416     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1417     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1418     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1419     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1420     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1421     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1422 
1423     // Ensure maxVersions in query is respected
1424     get = new Get(ROW);
1425     get.addColumn(FAMILY, QUALIFIER);
1426     get.setMaxVersions(2);
1427     result = ht.get(get);
1428     assertNResult(result, ROW, FAMILY, QUALIFIER,
1429         new long [] {STAMPS[4], STAMPS[5]},
1430         new byte[][] {VALUES[4], VALUES[5]},
1431         0, 1);
1432 
1433     scan = new Scan(ROW);
1434     scan.addColumn(FAMILY, QUALIFIER);
1435     scan.setMaxVersions(2);
1436     result = getSingleScanResult(ht, scan);
1437     assertNResult(result, ROW, FAMILY, QUALIFIER,
1438         new long [] {STAMPS[4], STAMPS[5]},
1439         new byte[][] {VALUES[4], VALUES[5]},
1440         0, 1);
1441 
1442 
1443     // Add some memstore and retest
1444 
1445     // Insert 4 more versions of same column and a dupe
1446     put = new Put(ROW);
1447     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
1448     put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
1449     put.add(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1450     put.add(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
1451     ht.put(put);
1452 
1453     // Ensure maxVersions in query is respected
1454     get = new Get(ROW);
1455     get.addColumn(FAMILY, QUALIFIER);
1456     get.setMaxVersions();
1457     result = ht.get(get);
1458     assertNResult(result, ROW, FAMILY, QUALIFIER,
1459         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1460         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1461         0, 7);
1462 
1463     scan = new Scan(ROW);
1464     scan.addColumn(FAMILY, QUALIFIER);
1465     scan.setMaxVersions();
1466     result = getSingleScanResult(ht, scan);
1467     assertNResult(result, ROW, FAMILY, QUALIFIER,
1468         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1469         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1470         0, 7);
1471 
1472     get = new Get(ROW);
1473     get.setMaxVersions();
1474     result = ht.get(get);
1475     assertNResult(result, ROW, FAMILY, QUALIFIER,
1476         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1477         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1478         0, 7);
1479 
1480     scan = new Scan(ROW);
1481     scan.setMaxVersions();
1482     result = getSingleScanResult(ht, scan);
1483     assertNResult(result, ROW, FAMILY, QUALIFIER,
1484         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1485         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1486         0, 7);
1487 
1488     // Verify we can get each one properly
1489     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1490     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1491     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1492     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1493     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1494     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1495     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1496     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1497 
1498     // Verify we don't accidentally get others
1499     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1500     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
1501     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1502     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
1503 
1504     // Ensure maxVersions of table is respected
1505 
1506     TEST_UTIL.flush();
1507 
1508     // Insert 4 more versions of same column and a dupe
1509     put = new Put(ROW);
1510     put.add(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
1511     put.add(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
1512     put.add(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
1513     put.add(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
1514     ht.put(put);
1515 
1516     get = new Get(ROW);
1517     get.addColumn(FAMILY, QUALIFIER);
1518     get.setMaxVersions(Integer.MAX_VALUE);
1519     result = ht.get(get);
1520     assertNResult(result, ROW, FAMILY, QUALIFIER,
1521         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
1522         new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
1523         0, 9);
1524 
1525     scan = new Scan(ROW);
1526     scan.addColumn(FAMILY, QUALIFIER);
1527     scan.setMaxVersions(Integer.MAX_VALUE);
1528     result = getSingleScanResult(ht, scan);
1529     assertNResult(result, ROW, FAMILY, QUALIFIER,
1530         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
1531         new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
1532         0, 9);
1533 
1534     // Delete a version in the memstore and a version in a storefile
1535     Delete delete = new Delete(ROW);
1536     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[11]);
1537     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[7]);
1538     ht.delete(delete);
1539 
1540     // Test that it's gone
1541     get = new Get(ROW);
1542     get.addColumn(FAMILY, QUALIFIER);
1543     get.setMaxVersions(Integer.MAX_VALUE);
1544     result = ht.get(get);
1545     assertNResult(result, ROW, FAMILY, QUALIFIER,
1546         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
1547         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
1548         0, 9);
1549 
1550     scan = new Scan(ROW);
1551     scan.addColumn(FAMILY, QUALIFIER);
1552     scan.setMaxVersions(Integer.MAX_VALUE);
1553     result = getSingleScanResult(ht, scan);
1554     assertNResult(result, ROW, FAMILY, QUALIFIER,
1555         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
1556         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
1557         0, 9);
1558 
1559   }
1560 
1561   @Test
1562   public void testVersionLimits() throws Exception {
1563     byte [] TABLE = Bytes.toBytes("testVersionLimits");
1564     byte [][] FAMILIES = makeNAscii(FAMILY, 3);
1565     int [] LIMITS = {1,3,5};
1566     long [] STAMPS = makeStamps(10);
1567     byte [][] VALUES = makeNAscii(VALUE, 10);
1568     Table ht = TEST_UTIL.createTable(TABLE, FAMILIES, LIMITS);
1569 
1570     // Insert limit + 1 on each family
1571     Put put = new Put(ROW);
1572     put.add(FAMILIES[0], QUALIFIER, STAMPS[0], VALUES[0]);
1573     put.add(FAMILIES[0], QUALIFIER, STAMPS[1], VALUES[1]);
1574     put.add(FAMILIES[1], QUALIFIER, STAMPS[0], VALUES[0]);
1575     put.add(FAMILIES[1], QUALIFIER, STAMPS[1], VALUES[1]);
1576     put.add(FAMILIES[1], QUALIFIER, STAMPS[2], VALUES[2]);
1577     put.add(FAMILIES[1], QUALIFIER, STAMPS[3], VALUES[3]);
1578     put.add(FAMILIES[2], QUALIFIER, STAMPS[0], VALUES[0]);
1579     put.add(FAMILIES[2], QUALIFIER, STAMPS[1], VALUES[1]);
1580     put.add(FAMILIES[2], QUALIFIER, STAMPS[2], VALUES[2]);
1581     put.add(FAMILIES[2], QUALIFIER, STAMPS[3], VALUES[3]);
1582     put.add(FAMILIES[2], QUALIFIER, STAMPS[4], VALUES[4]);
1583     put.add(FAMILIES[2], QUALIFIER, STAMPS[5], VALUES[5]);
1584     put.add(FAMILIES[2], QUALIFIER, STAMPS[6], VALUES[6]);
1585     ht.put(put);
1586 
1587     // Verify we only get the right number out of each
1588 
1589     // Family0
1590 
1591     Get get = new Get(ROW);
1592     get.addColumn(FAMILIES[0], QUALIFIER);
1593     get.setMaxVersions(Integer.MAX_VALUE);
1594     Result result = ht.get(get);
1595     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1596         new long [] {STAMPS[1]},
1597         new byte[][] {VALUES[1]},
1598         0, 0);
1599 
1600     get = new Get(ROW);
1601     get.addFamily(FAMILIES[0]);
1602     get.setMaxVersions(Integer.MAX_VALUE);
1603     result = ht.get(get);
1604     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1605         new long [] {STAMPS[1]},
1606         new byte[][] {VALUES[1]},
1607         0, 0);
1608 
1609     Scan scan = new Scan(ROW);
1610     scan.addColumn(FAMILIES[0], QUALIFIER);
1611     scan.setMaxVersions(Integer.MAX_VALUE);
1612     result = getSingleScanResult(ht, scan);
1613     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1614         new long [] {STAMPS[1]},
1615         new byte[][] {VALUES[1]},
1616         0, 0);
1617 
1618     scan = new Scan(ROW);
1619     scan.addFamily(FAMILIES[0]);
1620     scan.setMaxVersions(Integer.MAX_VALUE);
1621     result = getSingleScanResult(ht, scan);
1622     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1623         new long [] {STAMPS[1]},
1624         new byte[][] {VALUES[1]},
1625         0, 0);
1626 
1627     // Family1
1628 
1629     get = new Get(ROW);
1630     get.addColumn(FAMILIES[1], QUALIFIER);
1631     get.setMaxVersions(Integer.MAX_VALUE);
1632     result = ht.get(get);
1633     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1634         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1635         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1636         0, 2);
1637 
1638     get = new Get(ROW);
1639     get.addFamily(FAMILIES[1]);
1640     get.setMaxVersions(Integer.MAX_VALUE);
1641     result = ht.get(get);
1642     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1643         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1644         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1645         0, 2);
1646 
1647     scan = new Scan(ROW);
1648     scan.addColumn(FAMILIES[1], QUALIFIER);
1649     scan.setMaxVersions(Integer.MAX_VALUE);
1650     result = getSingleScanResult(ht, scan);
1651     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1652         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1653         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1654         0, 2);
1655 
1656     scan = new Scan(ROW);
1657     scan.addFamily(FAMILIES[1]);
1658     scan.setMaxVersions(Integer.MAX_VALUE);
1659     result = getSingleScanResult(ht, scan);
1660     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1661         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1662         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1663         0, 2);
1664 
1665     // Family2
1666 
1667     get = new Get(ROW);
1668     get.addColumn(FAMILIES[2], QUALIFIER);
1669     get.setMaxVersions(Integer.MAX_VALUE);
1670     result = ht.get(get);
1671     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1672         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1673         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1674         0, 4);
1675 
1676     get = new Get(ROW);
1677     get.addFamily(FAMILIES[2]);
1678     get.setMaxVersions(Integer.MAX_VALUE);
1679     result = ht.get(get);
1680     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1681         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1682         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1683         0, 4);
1684 
1685     scan = new Scan(ROW);
1686     scan.addColumn(FAMILIES[2], QUALIFIER);
1687     scan.setMaxVersions(Integer.MAX_VALUE);
1688     result = getSingleScanResult(ht, scan);
1689     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1690         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1691         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1692         0, 4);
1693 
1694     scan = new Scan(ROW);
1695     scan.addFamily(FAMILIES[2]);
1696     scan.setMaxVersions(Integer.MAX_VALUE);
1697     result = getSingleScanResult(ht, scan);
1698     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1699         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1700         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1701         0, 4);
1702 
1703     // Try all families
1704 
1705     get = new Get(ROW);
1706     get.setMaxVersions(Integer.MAX_VALUE);
1707     result = ht.get(get);
1708     assertTrue("Expected 9 keys but received " + result.size(),
1709         result.size() == 9);
1710 
1711     get = new Get(ROW);
1712     get.addFamily(FAMILIES[0]);
1713     get.addFamily(FAMILIES[1]);
1714     get.addFamily(FAMILIES[2]);
1715     get.setMaxVersions(Integer.MAX_VALUE);
1716     result = ht.get(get);
1717     assertTrue("Expected 9 keys but received " + result.size(),
1718         result.size() == 9);
1719 
1720     get = new Get(ROW);
1721     get.addColumn(FAMILIES[0], QUALIFIER);
1722     get.addColumn(FAMILIES[1], QUALIFIER);
1723     get.addColumn(FAMILIES[2], QUALIFIER);
1724     get.setMaxVersions(Integer.MAX_VALUE);
1725     result = ht.get(get);
1726     assertTrue("Expected 9 keys but received " + result.size(),
1727         result.size() == 9);
1728 
1729     scan = new Scan(ROW);
1730     scan.setMaxVersions(Integer.MAX_VALUE);
1731     result = getSingleScanResult(ht, scan);
1732     assertTrue("Expected 9 keys but received " + result.size(),
1733         result.size() == 9);
1734 
1735     scan = new Scan(ROW);
1736     scan.setMaxVersions(Integer.MAX_VALUE);
1737     scan.addFamily(FAMILIES[0]);
1738     scan.addFamily(FAMILIES[1]);
1739     scan.addFamily(FAMILIES[2]);
1740     result = getSingleScanResult(ht, scan);
1741     assertTrue("Expected 9 keys but received " + result.size(),
1742         result.size() == 9);
1743 
1744     scan = new Scan(ROW);
1745     scan.setMaxVersions(Integer.MAX_VALUE);
1746     scan.addColumn(FAMILIES[0], QUALIFIER);
1747     scan.addColumn(FAMILIES[1], QUALIFIER);
1748     scan.addColumn(FAMILIES[2], QUALIFIER);
1749     result = getSingleScanResult(ht, scan);
1750     assertTrue("Expected 9 keys but received " + result.size(),
1751         result.size() == 9);
1752 
1753   }
1754 
1755   @Test
1756   public void testDeleteFamilyVersion() throws Exception {
1757     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
1758     byte [] TABLE = Bytes.toBytes("testDeleteFamilyVersion");
1759 
1760     byte [][] QUALIFIERS = makeNAscii(QUALIFIER, 1);
1761     byte [][] VALUES = makeN(VALUE, 5);
1762     long [] ts = {1000, 2000, 3000, 4000, 5000};
1763 
1764     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 5);
1765 
1766     Put put = new Put(ROW);
1767     for (int q = 0; q < 1; q++)
1768       for (int t = 0; t < 5; t++)
1769         put.add(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
1770     ht.put(put);
1771     admin.flush(TABLE);
1772 
1773     Delete delete = new Delete(ROW);
1774     delete.deleteFamilyVersion(FAMILY, ts[1]);  // delete version '2000'
1775     delete.deleteFamilyVersion(FAMILY, ts[3]);  // delete version '4000'
1776     ht.delete(delete);
1777     admin.flush(TABLE);
1778 
1779     for (int i = 0; i < 1; i++) {
1780       Get get = new Get(ROW);
1781       get.addColumn(FAMILY, QUALIFIERS[i]);
1782       get.setMaxVersions(Integer.MAX_VALUE);
1783       Result result = ht.get(get);
1784       // verify version '1000'/'3000'/'5000' remains for all columns
1785       assertNResult(result, ROW, FAMILY, QUALIFIERS[i],
1786           new long [] {ts[0], ts[2], ts[4]},
1787           new byte[][] {VALUES[0], VALUES[2], VALUES[4]},
1788           0, 2);
1789     }
1790     ht.close();
1791     admin.close();
1792   }
1793 
1794   @Test
1795   public void testDeleteFamilyVersionWithOtherDeletes() throws Exception {
1796     byte [] TABLE = Bytes.toBytes("testDeleteFamilyVersionWithOtherDeletes");
1797 
1798     byte [][] QUALIFIERS = makeNAscii(QUALIFIER, 5);
1799     byte [][] VALUES = makeN(VALUE, 5);
1800     long [] ts = {1000, 2000, 3000, 4000, 5000};
1801 
1802     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
1803     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 5);
1804     Put put = null;
1805     Result result = null;
1806     Get get = null;
1807     Delete delete = null;
1808 
1809     // 1. put on ROW
1810     put = new Put(ROW);
1811     for (int q = 0; q < 5; q++)
1812       for (int t = 0; t < 5; t++)
1813         put.add(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
1814     ht.put(put);
1815     admin.flush(TABLE);
1816 
1817     // 2. put on ROWS[0]
1818     byte [] ROW2 = Bytes.toBytes("myRowForTest");
1819     put = new Put(ROW2);
1820     for (int q = 0; q < 5; q++)
1821       for (int t = 0; t < 5; t++)
1822         put.add(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
1823     ht.put(put);
1824     admin.flush(TABLE);
1825 
1826     // 3. delete on ROW
1827     delete = new Delete(ROW);
1828     // delete version <= 2000 of all columns
1829     // note: deleteFamily must be the first since it will mask
1830     // the subsequent other type deletes!
1831     delete.deleteFamily(FAMILY, ts[1]);
1832     // delete version '4000' of all columns
1833     delete.deleteFamilyVersion(FAMILY, ts[3]);
1834    // delete version <= 3000 of column 0
1835     delete.deleteColumns(FAMILY, QUALIFIERS[0], ts[2]);
1836     // delete version <= 5000 of column 2
1837     delete.deleteColumns(FAMILY, QUALIFIERS[2], ts[4]);
1838     // delete version 5000 of column 4
1839     delete.deleteColumn(FAMILY, QUALIFIERS[4], ts[4]);
1840     ht.delete(delete);
1841     admin.flush(TABLE);
1842 
1843      // 4. delete on ROWS[0]
1844     delete = new Delete(ROW2);
1845     delete.deleteFamilyVersion(FAMILY, ts[1]);  // delete version '2000'
1846     delete.deleteFamilyVersion(FAMILY, ts[3]);  // delete version '4000'
1847     ht.delete(delete);
1848     admin.flush(TABLE);
1849 
1850     // 5. check ROW
1851     get = new Get(ROW);
1852     get.addColumn(FAMILY, QUALIFIERS[0]);
1853     get.setMaxVersions(Integer.MAX_VALUE);
1854     result = ht.get(get);
1855     assertNResult(result, ROW, FAMILY, QUALIFIERS[0],
1856         new long [] {ts[4]},
1857         new byte[][] {VALUES[4]},
1858         0, 0);
1859 
1860     get = new Get(ROW);
1861     get.addColumn(FAMILY, QUALIFIERS[1]);
1862     get.setMaxVersions(Integer.MAX_VALUE);
1863     result = ht.get(get);
1864     assertNResult(result, ROW, FAMILY, QUALIFIERS[1],
1865         new long [] {ts[2], ts[4]},
1866         new byte[][] {VALUES[2], VALUES[4]},
1867         0, 1);
1868 
1869     get = new Get(ROW);
1870     get.addColumn(FAMILY, QUALIFIERS[2]);
1871     get.setMaxVersions(Integer.MAX_VALUE);
1872     result = ht.get(get);
1873     assertEquals(0, result.size());
1874 
1875     get = new Get(ROW);
1876     get.addColumn(FAMILY, QUALIFIERS[3]);
1877     get.setMaxVersions(Integer.MAX_VALUE);
1878     result = ht.get(get);
1879     assertNResult(result, ROW, FAMILY, QUALIFIERS[3],
1880         new long [] {ts[2], ts[4]},
1881         new byte[][] {VALUES[2], VALUES[4]},
1882         0, 1);
1883 
1884     get = new Get(ROW);
1885     get.addColumn(FAMILY, QUALIFIERS[4]);
1886     get.setMaxVersions(Integer.MAX_VALUE);
1887     result = ht.get(get);
1888     assertNResult(result, ROW, FAMILY, QUALIFIERS[4],
1889         new long [] {ts[2]},
1890         new byte[][] {VALUES[2]},
1891         0, 0);
1892 
1893     // 6. check ROWS[0]
1894     for (int i = 0; i < 5; i++) {
1895       get = new Get(ROW2);
1896       get.addColumn(FAMILY, QUALIFIERS[i]);
1897       get.setMaxVersions(Integer.MAX_VALUE);
1898       result = ht.get(get);
1899       // verify version '1000'/'3000'/'5000' remains for all columns
1900       assertNResult(result, ROW2, FAMILY, QUALIFIERS[i],
1901           new long [] {ts[0], ts[2], ts[4]},
1902           new byte[][] {VALUES[0], VALUES[2], VALUES[4]},
1903           0, 2);
1904     }
1905     ht.close();
1906     admin.close();
1907   }
1908 
1909   @Test
1910   public void testDeletes() throws Exception {
1911     byte [] TABLE = Bytes.toBytes("testDeletes");
1912 
1913     byte [][] ROWS = makeNAscii(ROW, 6);
1914     byte [][] FAMILIES = makeNAscii(FAMILY, 3);
1915     byte [][] VALUES = makeN(VALUE, 5);
1916     long [] ts = {1000, 2000, 3000, 4000, 5000};
1917 
1918     Table ht = TEST_UTIL.createTable(TABLE, FAMILIES, 3);
1919 
1920     Put put = new Put(ROW);
1921     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
1922     put.add(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
1923     ht.put(put);
1924 
1925     Delete delete = new Delete(ROW);
1926     delete.deleteFamily(FAMILIES[0], ts[0]);
1927     ht.delete(delete);
1928 
1929     Get get = new Get(ROW);
1930     get.addFamily(FAMILIES[0]);
1931     get.setMaxVersions(Integer.MAX_VALUE);
1932     Result result = ht.get(get);
1933     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1934         new long [] {ts[1]},
1935         new byte[][] {VALUES[1]},
1936         0, 0);
1937 
1938     Scan scan = new Scan(ROW);
1939     scan.addFamily(FAMILIES[0]);
1940     scan.setMaxVersions(Integer.MAX_VALUE);
1941     result = getSingleScanResult(ht, scan);
1942     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1943         new long [] {ts[1]},
1944         new byte[][] {VALUES[1]},
1945         0, 0);
1946 
1947     // Test delete latest version
1948     put = new Put(ROW);
1949     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
1950     put.add(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
1951     put.add(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
1952     put.add(FAMILIES[0], null, ts[4], VALUES[4]);
1953     put.add(FAMILIES[0], null, ts[2], VALUES[2]);
1954     put.add(FAMILIES[0], null, ts[3], VALUES[3]);
1955     ht.put(put);
1956 
1957     delete = new Delete(ROW);
1958     delete.deleteColumn(FAMILIES[0], QUALIFIER); // ts[4]
1959     ht.delete(delete);
1960 
1961     get = new Get(ROW);
1962     get.addColumn(FAMILIES[0], QUALIFIER);
1963     get.setMaxVersions(Integer.MAX_VALUE);
1964     result = ht.get(get);
1965     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1966         new long [] {ts[1], ts[2], ts[3]},
1967         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1968         0, 2);
1969 
1970     scan = new Scan(ROW);
1971     scan.addColumn(FAMILIES[0], QUALIFIER);
1972     scan.setMaxVersions(Integer.MAX_VALUE);
1973     result = getSingleScanResult(ht, scan);
1974     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1975         new long [] {ts[1], ts[2], ts[3]},
1976         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1977         0, 2);
1978 
1979     // Test for HBASE-1847
1980     delete = new Delete(ROW);
1981     delete.deleteColumn(FAMILIES[0], null);
1982     ht.delete(delete);
1983 
1984     // Cleanup null qualifier
1985     delete = new Delete(ROW);
1986     delete.deleteColumns(FAMILIES[0], null);
1987     ht.delete(delete);
1988 
1989     // Expected client behavior might be that you can re-put deleted values
1990     // But alas, this is not to be.  We can't put them back in either case.
1991 
1992     put = new Put(ROW);
1993     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); // 1000
1994     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); // 5000
1995     ht.put(put);
1996 
1997 
1998     // It used to be due to the internal implementation of Get, that
1999     // the Get() call would return ts[4] UNLIKE the Scan below. With
2000     // the switch to using Scan for Get this is no longer the case.
2001     get = new Get(ROW);
2002     get.addFamily(FAMILIES[0]);
2003     get.setMaxVersions(Integer.MAX_VALUE);
2004     result = ht.get(get);
2005     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2006         new long [] {ts[1], ts[2], ts[3]},
2007         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
2008         0, 2);
2009 
2010     // The Scanner returns the previous values, the expected-naive-unexpected behavior
2011 
2012     scan = new Scan(ROW);
2013     scan.addFamily(FAMILIES[0]);
2014     scan.setMaxVersions(Integer.MAX_VALUE);
2015     result = getSingleScanResult(ht, scan);
2016     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2017         new long [] {ts[1], ts[2], ts[3]},
2018         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
2019         0, 2);
2020 
2021     // Test deleting an entire family from one row but not the other various ways
2022 
2023     put = new Put(ROWS[0]);
2024     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2025     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2026     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2027     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2028     ht.put(put);
2029 
2030     put = new Put(ROWS[1]);
2031     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2032     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2033     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2034     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2035     ht.put(put);
2036 
2037     put = new Put(ROWS[2]);
2038     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2039     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2040     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2041     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2042     ht.put(put);
2043 
2044     // Assert that above went in.
2045     get = new Get(ROWS[2]);
2046     get.addFamily(FAMILIES[1]);
2047     get.addFamily(FAMILIES[2]);
2048     get.setMaxVersions(Integer.MAX_VALUE);
2049     result = ht.get(get);
2050     assertTrue("Expected 4 key but received " + result.size() + ": " + result,
2051         result.size() == 4);
2052 
2053     delete = new Delete(ROWS[0]);
2054     delete.deleteFamily(FAMILIES[2]);
2055     ht.delete(delete);
2056 
2057     delete = new Delete(ROWS[1]);
2058     delete.deleteColumns(FAMILIES[1], QUALIFIER);
2059     ht.delete(delete);
2060 
2061     delete = new Delete(ROWS[2]);
2062     delete.deleteColumn(FAMILIES[1], QUALIFIER);
2063     delete.deleteColumn(FAMILIES[1], QUALIFIER);
2064     delete.deleteColumn(FAMILIES[2], QUALIFIER);
2065     ht.delete(delete);
2066 
2067     get = new Get(ROWS[0]);
2068     get.addFamily(FAMILIES[1]);
2069     get.addFamily(FAMILIES[2]);
2070     get.setMaxVersions(Integer.MAX_VALUE);
2071     result = ht.get(get);
2072     assertTrue("Expected 2 keys but received " + result.size(),
2073         result.size() == 2);
2074     assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
2075         new long [] {ts[0], ts[1]},
2076         new byte[][] {VALUES[0], VALUES[1]},
2077         0, 1);
2078 
2079     scan = new Scan(ROWS[0]);
2080     scan.addFamily(FAMILIES[1]);
2081     scan.addFamily(FAMILIES[2]);
2082     scan.setMaxVersions(Integer.MAX_VALUE);
2083     result = getSingleScanResult(ht, scan);
2084     assertTrue("Expected 2 keys but received " + result.size(),
2085         result.size() == 2);
2086     assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
2087         new long [] {ts[0], ts[1]},
2088         new byte[][] {VALUES[0], VALUES[1]},
2089         0, 1);
2090 
2091     get = new Get(ROWS[1]);
2092     get.addFamily(FAMILIES[1]);
2093     get.addFamily(FAMILIES[2]);
2094     get.setMaxVersions(Integer.MAX_VALUE);
2095     result = ht.get(get);
2096     assertTrue("Expected 2 keys but received " + result.size(),
2097         result.size() == 2);
2098 
2099     scan = new Scan(ROWS[1]);
2100     scan.addFamily(FAMILIES[1]);
2101     scan.addFamily(FAMILIES[2]);
2102     scan.setMaxVersions(Integer.MAX_VALUE);
2103     result = getSingleScanResult(ht, scan);
2104     assertTrue("Expected 2 keys but received " + result.size(),
2105         result.size() == 2);
2106 
2107     get = new Get(ROWS[2]);
2108     get.addFamily(FAMILIES[1]);
2109     get.addFamily(FAMILIES[2]);
2110     get.setMaxVersions(Integer.MAX_VALUE);
2111     result = ht.get(get);
2112     assertEquals(1, result.size());
2113     assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
2114         new long [] {ts[2]},
2115         new byte[][] {VALUES[2]},
2116         0, 0);
2117 
2118     scan = new Scan(ROWS[2]);
2119     scan.addFamily(FAMILIES[1]);
2120     scan.addFamily(FAMILIES[2]);
2121     scan.setMaxVersions(Integer.MAX_VALUE);
2122     result = getSingleScanResult(ht, scan);
2123     assertEquals(1, result.size());
2124     assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
2125         new long [] {ts[2]},
2126         new byte[][] {VALUES[2]},
2127         0, 0);
2128 
2129     // Test if we delete the family first in one row (HBASE-1541)
2130 
2131     delete = new Delete(ROWS[3]);
2132     delete.deleteFamily(FAMILIES[1]);
2133     ht.delete(delete);
2134 
2135     put = new Put(ROWS[3]);
2136     put.add(FAMILIES[2], QUALIFIER, VALUES[0]);
2137     ht.put(put);
2138 
2139     put = new Put(ROWS[4]);
2140     put.add(FAMILIES[1], QUALIFIER, VALUES[1]);
2141     put.add(FAMILIES[2], QUALIFIER, VALUES[2]);
2142     ht.put(put);
2143 
2144     get = new Get(ROWS[3]);
2145     get.addFamily(FAMILIES[1]);
2146     get.addFamily(FAMILIES[2]);
2147     get.setMaxVersions(Integer.MAX_VALUE);
2148     result = ht.get(get);
2149     assertTrue("Expected 1 key but received " + result.size(),
2150         result.size() == 1);
2151 
2152     get = new Get(ROWS[4]);
2153     get.addFamily(FAMILIES[1]);
2154     get.addFamily(FAMILIES[2]);
2155     get.setMaxVersions(Integer.MAX_VALUE);
2156     result = ht.get(get);
2157     assertTrue("Expected 2 keys but received " + result.size(),
2158         result.size() == 2);
2159 
2160     scan = new Scan(ROWS[3]);
2161     scan.addFamily(FAMILIES[1]);
2162     scan.addFamily(FAMILIES[2]);
2163     scan.setMaxVersions(Integer.MAX_VALUE);
2164     ResultScanner scanner = ht.getScanner(scan);
2165     result = scanner.next();
2166     assertTrue("Expected 1 key but received " + result.size(),
2167         result.size() == 1);
2168     assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[3]));
2169     assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[0]));
2170     result = scanner.next();
2171     assertTrue("Expected 2 keys but received " + result.size(),
2172         result.size() == 2);
2173     assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[4]));
2174     assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[1]), ROWS[4]));
2175     assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[1]));
2176     assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[1]), VALUES[2]));
2177     scanner.close();
2178 
2179     // Add test of bulk deleting.
2180     for (int i = 0; i < 10; i++) {
2181       byte [] bytes = Bytes.toBytes(i);
2182       put = new Put(bytes);
2183       put.setDurability(Durability.SKIP_WAL);
2184       put.add(FAMILIES[0], QUALIFIER, bytes);
2185       ht.put(put);
2186     }
2187     for (int i = 0; i < 10; i++) {
2188       byte [] bytes = Bytes.toBytes(i);
2189       get = new Get(bytes);
2190       get.addFamily(FAMILIES[0]);
2191       result = ht.get(get);
2192       assertTrue(result.size() == 1);
2193     }
2194     ArrayList<Delete> deletes = new ArrayList<Delete>();
2195     for (int i = 0; i < 10; i++) {
2196       byte [] bytes = Bytes.toBytes(i);
2197       delete = new Delete(bytes);
2198       delete.deleteFamily(FAMILIES[0]);
2199       deletes.add(delete);
2200     }
2201     ht.delete(deletes);
2202     for (int i = 0; i < 10; i++) {
2203       byte [] bytes = Bytes.toBytes(i);
2204       get = new Get(bytes);
2205       get.addFamily(FAMILIES[0]);
2206       result = ht.get(get);
2207       assertTrue(result.size() == 0);
2208     }
2209   }
2210 
2211   /*
2212    * Baseline "scalability" test.
2213    *
2214    * Tests one hundred families, one million columns, one million versions
2215    */
2216   @Ignore @Test
2217   public void testMillions() throws Exception {
2218 
2219     // 100 families
2220 
2221     // millions of columns
2222 
2223     // millions of versions
2224 
2225   }
2226 
2227   @Ignore @Test
2228   public void testMultipleRegionsAndBatchPuts() throws Exception {
2229     // Two family table
2230 
2231     // Insert lots of rows
2232 
2233     // Insert to the same row with batched puts
2234 
2235     // Insert to multiple rows with batched puts
2236 
2237     // Split the table
2238 
2239     // Get row from first region
2240 
2241     // Get row from second region
2242 
2243     // Scan all rows
2244 
2245     // Insert to multiple regions with batched puts
2246 
2247     // Get row from first region
2248 
2249     // Get row from second region
2250 
2251     // Scan all rows
2252 
2253 
2254   }
2255 
2256   @Ignore @Test
2257   public void testMultipleRowMultipleFamily() throws Exception {
2258 
2259   }
2260 
2261   //
2262   // JIRA Testers
2263   //
2264 
2265   /**
2266    * HBASE-867
2267    *    If millions of columns in a column family, hbase scanner won't come up
2268    *
2269    *    Test will create numRows rows, each with numColsPerRow columns
2270    *    (1 version each), and attempt to scan them all.
2271    *
2272    *    To test at scale, up numColsPerRow to the millions
2273    *    (have not gotten that to work running as junit though)
2274    */
2275   @Test
2276   public void testJiraTest867() throws Exception {
2277     int numRows = 10;
2278     int numColsPerRow = 2000;
2279 
2280     byte [] TABLE = Bytes.toBytes("testJiraTest867");
2281 
2282     byte [][] ROWS = makeN(ROW, numRows);
2283     byte [][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow);
2284 
2285     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
2286 
2287     // Insert rows
2288 
2289     for(int i=0;i<numRows;i++) {
2290       Put put = new Put(ROWS[i]);
2291       put.setDurability(Durability.SKIP_WAL);
2292       for(int j=0;j<numColsPerRow;j++) {
2293         put.add(FAMILY, QUALIFIERS[j], QUALIFIERS[j]);
2294       }
2295       assertTrue("Put expected to contain " + numColsPerRow + " columns but " +
2296           "only contains " + put.size(), put.size() == numColsPerRow);
2297       ht.put(put);
2298     }
2299 
2300     // Get a row
2301     Get get = new Get(ROWS[numRows-1]);
2302     Result result = ht.get(get);
2303     assertNumKeys(result, numColsPerRow);
2304     Cell [] keys = result.rawCells();
2305     for(int i=0;i<result.size();i++) {
2306       assertKey(keys[i], ROWS[numRows-1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2307     }
2308 
2309     // Scan the rows
2310     Scan scan = new Scan();
2311     ResultScanner scanner = ht.getScanner(scan);
2312     int rowCount = 0;
2313     while((result = scanner.next()) != null) {
2314       assertNumKeys(result, numColsPerRow);
2315       Cell [] kvs = result.rawCells();
2316       for(int i=0;i<numColsPerRow;i++) {
2317         assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2318       }
2319       rowCount++;
2320     }
2321     scanner.close();
2322     assertTrue("Expected to scan " + numRows + " rows but actually scanned "
2323         + rowCount + " rows", rowCount == numRows);
2324 
2325     // flush and try again
2326 
2327     TEST_UTIL.flush();
2328 
2329     // Get a row
2330     get = new Get(ROWS[numRows-1]);
2331     result = ht.get(get);
2332     assertNumKeys(result, numColsPerRow);
2333     keys = result.rawCells();
2334     for(int i=0;i<result.size();i++) {
2335       assertKey(keys[i], ROWS[numRows-1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2336     }
2337 
2338     // Scan the rows
2339     scan = new Scan();
2340     scanner = ht.getScanner(scan);
2341     rowCount = 0;
2342     while((result = scanner.next()) != null) {
2343       assertNumKeys(result, numColsPerRow);
2344       Cell [] kvs = result.rawCells();
2345       for(int i=0;i<numColsPerRow;i++) {
2346         assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2347       }
2348       rowCount++;
2349     }
2350     scanner.close();
2351     assertTrue("Expected to scan " + numRows + " rows but actually scanned "
2352         + rowCount + " rows", rowCount == numRows);
2353 
2354   }
2355 
2356   /**
2357    * HBASE-861
2358    *    get with timestamp will return a value if there is a version with an
2359    *    earlier timestamp
2360    */
2361   @Test
2362   public void testJiraTest861() throws Exception {
2363 
2364     byte [] TABLE = Bytes.toBytes("testJiraTest861");
2365     byte [][] VALUES = makeNAscii(VALUE, 7);
2366     long [] STAMPS = makeStamps(7);
2367 
2368     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2369 
2370     // Insert three versions
2371 
2372     Put put = new Put(ROW);
2373     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2374     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2375     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2376     ht.put(put);
2377 
2378     // Get the middle value
2379     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2380 
2381     // Try to get one version before (expect fail)
2382     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2383 
2384     // Try to get one version after (expect fail)
2385     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2386 
2387     // Try same from storefile
2388     TEST_UTIL.flush();
2389     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2390     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2391     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2392 
2393     // Insert two more versions surrounding others, into memstore
2394     put = new Put(ROW);
2395     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2396     put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2397     ht.put(put);
2398 
2399     // Check we can get everything we should and can't get what we shouldn't
2400     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2401     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2402     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2403     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2404     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2405     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2406     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2407 
2408     // Try same from two storefiles
2409     TEST_UTIL.flush();
2410     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2411     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2412     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2413     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2414     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2415     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2416     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2417 
2418   }
2419 
2420   /**
2421    * HBASE-33
2422    *    Add a HTable get/obtainScanner method that retrieves all versions of a
2423    *    particular column and row between two timestamps
2424    */
2425   @Test
2426   public void testJiraTest33() throws Exception {
2427 
2428     byte [] TABLE = Bytes.toBytes("testJiraTest33");
2429     byte [][] VALUES = makeNAscii(VALUE, 7);
2430     long [] STAMPS = makeStamps(7);
2431 
2432     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2433 
2434     // Insert lots versions
2435 
2436     Put put = new Put(ROW);
2437     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2438     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2439     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2440     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2441     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2442     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2443     ht.put(put);
2444 
2445     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2446     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2447     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2448     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2449 
2450     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2451     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2452     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2453     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2454 
2455     // Try same from storefile
2456     TEST_UTIL.flush();
2457 
2458     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2459     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2460     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2461     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2462 
2463     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2464     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2465     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2466     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2467 
2468   }
2469 
2470   /**
2471    * HBASE-1014
2472    *    commit(BatchUpdate) method should return timestamp
2473    */
2474   @Test
2475   public void testJiraTest1014() throws Exception {
2476 
2477     byte [] TABLE = Bytes.toBytes("testJiraTest1014");
2478 
2479     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2480 
2481     long manualStamp = 12345;
2482 
2483     // Insert lots versions
2484 
2485     Put put = new Put(ROW);
2486     put.add(FAMILY, QUALIFIER, manualStamp, VALUE);
2487     ht.put(put);
2488 
2489     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, manualStamp, VALUE);
2490     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp-1);
2491     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp+1);
2492 
2493   }
2494 
2495   /**
2496    * HBASE-1182
2497    *    Scan for columns > some timestamp
2498    */
2499   @Test
2500   public void testJiraTest1182() throws Exception {
2501 
2502     byte [] TABLE = Bytes.toBytes("testJiraTest1182");
2503     byte [][] VALUES = makeNAscii(VALUE, 7);
2504     long [] STAMPS = makeStamps(7);
2505 
2506     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2507 
2508     // Insert lots versions
2509 
2510     Put put = new Put(ROW);
2511     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2512     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2513     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2514     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2515     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2516     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2517     ht.put(put);
2518 
2519     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2520     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2521     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2522 
2523     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2524     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2525     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2526 
2527     // Try same from storefile
2528     TEST_UTIL.flush();
2529 
2530     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2531     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2532     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2533 
2534     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2535     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2536     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2537   }
2538 
2539   /**
2540    * HBASE-52
2541    *    Add a means of scanning over all versions
2542    */
2543   @Test
2544   public void testJiraTest52() throws Exception {
2545     byte [] TABLE = Bytes.toBytes("testJiraTest52");
2546     byte [][] VALUES = makeNAscii(VALUE, 7);
2547     long [] STAMPS = makeStamps(7);
2548 
2549     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2550 
2551     // Insert lots versions
2552 
2553     Put put = new Put(ROW);
2554     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2555     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2556     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2557     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2558     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2559     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2560     ht.put(put);
2561 
2562     getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2563 
2564     scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2565 
2566     // Try same from storefile
2567     TEST_UTIL.flush();
2568 
2569     getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2570 
2571     scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2572   }
2573 
2574   //
2575   // Bulk Testers
2576   //
2577 
2578   private void getVersionRangeAndVerifyGreaterThan(Table ht, byte [] row,
2579       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
2580       int start, int end)
2581   throws IOException {
2582     Get get = new Get(row);
2583     get.addColumn(family, qualifier);
2584     get.setMaxVersions(Integer.MAX_VALUE);
2585     get.setTimeRange(stamps[start+1], Long.MAX_VALUE);
2586     Result result = ht.get(get);
2587     assertNResult(result, row, family, qualifier, stamps, values, start+1, end);
2588   }
2589 
2590   private void getVersionRangeAndVerify(Table ht, byte [] row, byte [] family,
2591       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2592   throws IOException {
2593     Get get = new Get(row);
2594     get.addColumn(family, qualifier);
2595     get.setMaxVersions(Integer.MAX_VALUE);
2596     get.setTimeRange(stamps[start], stamps[end]+1);
2597     Result result = ht.get(get);
2598     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2599   }
2600 
2601   private void getAllVersionsAndVerify(Table ht, byte [] row, byte [] family,
2602       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2603   throws IOException {
2604     Get get = new Get(row);
2605     get.addColumn(family, qualifier);
2606     get.setMaxVersions(Integer.MAX_VALUE);
2607     Result result = ht.get(get);
2608     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2609   }
2610 
2611   private void scanVersionRangeAndVerifyGreaterThan(Table ht, byte [] row,
2612       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
2613       int start, int end)
2614   throws IOException {
2615     Scan scan = new Scan(row);
2616     scan.addColumn(family, qualifier);
2617     scan.setMaxVersions(Integer.MAX_VALUE);
2618     scan.setTimeRange(stamps[start+1], Long.MAX_VALUE);
2619     Result result = getSingleScanResult(ht, scan);
2620     assertNResult(result, row, family, qualifier, stamps, values, start+1, end);
2621   }
2622 
2623   private void scanVersionRangeAndVerify(Table ht, byte [] row, byte [] family,
2624       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2625   throws IOException {
2626     Scan scan = new Scan(row);
2627     scan.addColumn(family, qualifier);
2628     scan.setMaxVersions(Integer.MAX_VALUE);
2629     scan.setTimeRange(stamps[start], stamps[end]+1);
2630     Result result = getSingleScanResult(ht, scan);
2631     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2632   }
2633 
2634   private void scanAllVersionsAndVerify(Table ht, byte [] row, byte [] family,
2635       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2636   throws IOException {
2637     Scan scan = new Scan(row);
2638     scan.addColumn(family, qualifier);
2639     scan.setMaxVersions(Integer.MAX_VALUE);
2640     Result result = getSingleScanResult(ht, scan);
2641     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2642   }
2643 
2644   private void getVersionAndVerify(Table ht, byte [] row, byte [] family,
2645       byte [] qualifier, long stamp, byte [] value)
2646   throws Exception {
2647     Get get = new Get(row);
2648     get.addColumn(family, qualifier);
2649     get.setTimeStamp(stamp);
2650     get.setMaxVersions(Integer.MAX_VALUE);
2651     Result result = ht.get(get);
2652     assertSingleResult(result, row, family, qualifier, stamp, value);
2653   }
2654 
2655   private void getVersionAndVerifyMissing(Table ht, byte [] row, byte [] family,
2656       byte [] qualifier, long stamp)
2657   throws Exception {
2658     Get get = new Get(row);
2659     get.addColumn(family, qualifier);
2660     get.setTimeStamp(stamp);
2661     get.setMaxVersions(Integer.MAX_VALUE);
2662     Result result = ht.get(get);
2663     assertEmptyResult(result);
2664   }
2665 
2666   private void scanVersionAndVerify(Table ht, byte [] row, byte [] family,
2667       byte [] qualifier, long stamp, byte [] value)
2668   throws Exception {
2669     Scan scan = new Scan(row);
2670     scan.addColumn(family, qualifier);
2671     scan.setTimeStamp(stamp);
2672     scan.setMaxVersions(Integer.MAX_VALUE);
2673     Result result = getSingleScanResult(ht, scan);
2674     assertSingleResult(result, row, family, qualifier, stamp, value);
2675   }
2676 
2677   private void scanVersionAndVerifyMissing(Table ht, byte [] row,
2678       byte [] family, byte [] qualifier, long stamp)
2679   throws Exception {
2680     Scan scan = new Scan(row);
2681     scan.addColumn(family, qualifier);
2682     scan.setTimeStamp(stamp);
2683     scan.setMaxVersions(Integer.MAX_VALUE);
2684     Result result = getSingleScanResult(ht, scan);
2685     assertNullResult(result);
2686   }
2687 
2688   private void getTestNull(Table ht, byte [] row, byte [] family,
2689       byte [] value)
2690   throws Exception {
2691 
2692     Get get = new Get(row);
2693     get.addColumn(family, null);
2694     Result result = ht.get(get);
2695     assertSingleResult(result, row, family, null, value);
2696 
2697     get = new Get(row);
2698     get.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
2699     result = ht.get(get);
2700     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2701 
2702     get = new Get(row);
2703     get.addFamily(family);
2704     result = ht.get(get);
2705     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2706 
2707     get = new Get(row);
2708     result = ht.get(get);
2709     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2710 
2711   }
2712 
2713   private void scanTestNull(Table ht, byte[] row, byte[] family, byte[] value)
2714       throws Exception {
2715     scanTestNull(ht, row, family, value, false);
2716   }
2717 
2718   private void scanTestNull(Table ht, byte[] row, byte[] family, byte[] value,
2719       boolean isReversedScan) throws Exception {
2720 
2721     Scan scan = new Scan();
2722     scan.setReversed(isReversedScan);
2723     scan.addColumn(family, null);
2724     Result result = getSingleScanResult(ht, scan);
2725     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2726 
2727     scan = new Scan();
2728     scan.setReversed(isReversedScan);
2729     scan.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
2730     result = getSingleScanResult(ht, scan);
2731     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2732 
2733     scan = new Scan();
2734     scan.setReversed(isReversedScan);
2735     scan.addFamily(family);
2736     result = getSingleScanResult(ht, scan);
2737     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2738 
2739     scan = new Scan();
2740     scan.setReversed(isReversedScan);
2741     result = getSingleScanResult(ht, scan);
2742     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2743 
2744   }
2745 
2746   private void singleRowGetTest(Table ht, byte [][] ROWS, byte [][] FAMILIES,
2747       byte [][] QUALIFIERS, byte [][] VALUES)
2748   throws Exception {
2749 
2750     // Single column from memstore
2751     Get get = new Get(ROWS[0]);
2752     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2753     Result result = ht.get(get);
2754     assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);
2755 
2756     // Single column from storefile
2757     get = new Get(ROWS[0]);
2758     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
2759     result = ht.get(get);
2760     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
2761 
2762     // Single column from storefile, family match
2763     get = new Get(ROWS[0]);
2764     get.addFamily(FAMILIES[7]);
2765     result = ht.get(get);
2766     assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);
2767 
2768     // Two columns, one from memstore one from storefile, same family,
2769     // wildcard match
2770     get = new Get(ROWS[0]);
2771     get.addFamily(FAMILIES[4]);
2772     result = ht.get(get);
2773     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2774         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2775 
2776     // Two columns, one from memstore one from storefile, same family,
2777     // explicit match
2778     get = new Get(ROWS[0]);
2779     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2780     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
2781     result = ht.get(get);
2782     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2783         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2784 
2785     // Three column, one from memstore two from storefile, different families,
2786     // wildcard match
2787     get = new Get(ROWS[0]);
2788     get.addFamily(FAMILIES[4]);
2789     get.addFamily(FAMILIES[7]);
2790     result = ht.get(get);
2791     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2792         new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} });
2793 
2794     // Multiple columns from everywhere storefile, many family, wildcard
2795     get = new Get(ROWS[0]);
2796     get.addFamily(FAMILIES[2]);
2797     get.addFamily(FAMILIES[4]);
2798     get.addFamily(FAMILIES[6]);
2799     get.addFamily(FAMILIES[7]);
2800     result = ht.get(get);
2801     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2802         new int [][] {
2803           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2804     });
2805 
2806     // Multiple columns from everywhere storefile, many family, wildcard
2807     get = new Get(ROWS[0]);
2808     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
2809     get.addColumn(FAMILIES[2], QUALIFIERS[4]);
2810     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2811     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
2812     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
2813     get.addColumn(FAMILIES[6], QUALIFIERS[7]);
2814     get.addColumn(FAMILIES[7], QUALIFIERS[7]);
2815     get.addColumn(FAMILIES[7], QUALIFIERS[8]);
2816     result = ht.get(get);
2817     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2818         new int [][] {
2819           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2820     });
2821 
2822     // Everything
2823     get = new Get(ROWS[0]);
2824     result = ht.get(get);
2825     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2826         new int [][] {
2827           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0}
2828     });
2829 
2830     // Get around inserted columns
2831 
2832     get = new Get(ROWS[1]);
2833     result = ht.get(get);
2834     assertEmptyResult(result);
2835 
2836     get = new Get(ROWS[0]);
2837     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
2838     get.addColumn(FAMILIES[2], QUALIFIERS[3]);
2839     result = ht.get(get);
2840     assertEmptyResult(result);
2841 
2842   }
2843 
2844   private void singleRowScanTest(Table ht, byte [][] ROWS, byte [][] FAMILIES,
2845       byte [][] QUALIFIERS, byte [][] VALUES)
2846   throws Exception {
2847 
2848     // Single column from memstore
2849     Scan scan = new Scan();
2850     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
2851     Result result = getSingleScanResult(ht, scan);
2852     assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);
2853 
2854     // Single column from storefile
2855     scan = new Scan();
2856     scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
2857     result = getSingleScanResult(ht, scan);
2858     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
2859 
2860     // Single column from storefile, family match
2861     scan = new Scan();
2862     scan.addFamily(FAMILIES[7]);
2863     result = getSingleScanResult(ht, scan);
2864     assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);
2865 
2866     // Two columns, one from memstore one from storefile, same family,
2867     // wildcard match
2868     scan = new Scan();
2869     scan.addFamily(FAMILIES[4]);
2870     result = getSingleScanResult(ht, scan);
2871     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2872         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2873 
2874     // Two columns, one from memstore one from storefile, same family,
2875     // explicit match
2876     scan = new Scan();
2877     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
2878     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
2879     result = getSingleScanResult(ht, scan);
2880     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2881         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2882 
2883     // Three column, one from memstore two from storefile, different families,
2884     // wildcard match
2885     scan = new Scan();
2886     scan.addFamily(FAMILIES[4]);
2887     scan.addFamily(FAMILIES[7]);
2888     result = getSingleScanResult(ht, scan);
2889     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2890         new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} });
2891 
2892     // Multiple columns from everywhere storefile, many family, wildcard
2893     scan = new Scan();
2894     scan.addFamily(FAMILIES[2]);
2895     scan.addFamily(FAMILIES[4]);
2896     scan.addFamily(FAMILIES[6]);
2897     scan.addFamily(FAMILIES[7]);
2898     result = getSingleScanResult(ht, scan);
2899     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2900         new int [][] {
2901           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2902     });
2903 
2904     // Multiple columns from everywhere storefile, many family, wildcard
2905     scan = new Scan();
2906     scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
2907     scan.addColumn(FAMILIES[2], QUALIFIERS[4]);
2908     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
2909     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
2910     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
2911     scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
2912     scan.addColumn(FAMILIES[7], QUALIFIERS[7]);
2913     scan.addColumn(FAMILIES[7], QUALIFIERS[8]);
2914     result = getSingleScanResult(ht, scan);
2915     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2916         new int [][] {
2917           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2918     });
2919 
2920     // Everything
2921     scan = new Scan();
2922     result = getSingleScanResult(ht, scan);
2923     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2924         new int [][] {
2925           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0}
2926     });
2927 
2928     // Scan around inserted columns
2929 
2930     scan = new Scan(ROWS[1]);
2931     result = getSingleScanResult(ht, scan);
2932     assertNullResult(result);
2933 
2934     scan = new Scan();
2935     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
2936     scan.addColumn(FAMILIES[2], QUALIFIERS[3]);
2937     result = getSingleScanResult(ht, scan);
2938     assertNullResult(result);
2939   }
2940 
2941   /**
2942    * Verify a single column using gets.
2943    * Expects family and qualifier arrays to be valid for at least
2944    * the range:  idx-2 < idx < idx+2
2945    */
2946   private void getVerifySingleColumn(Table ht,
2947       byte [][] ROWS, int ROWIDX,
2948       byte [][] FAMILIES, int FAMILYIDX,
2949       byte [][] QUALIFIERS, int QUALIFIERIDX,
2950       byte [][] VALUES, int VALUEIDX)
2951   throws Exception {
2952 
2953     Get get = new Get(ROWS[ROWIDX]);
2954     Result result = ht.get(get);
2955     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2956         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2957 
2958     get = new Get(ROWS[ROWIDX]);
2959     get.addFamily(FAMILIES[FAMILYIDX]);
2960     result = ht.get(get);
2961     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2962         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2963 
2964     get = new Get(ROWS[ROWIDX]);
2965     get.addFamily(FAMILIES[FAMILYIDX-2]);
2966     get.addFamily(FAMILIES[FAMILYIDX]);
2967     get.addFamily(FAMILIES[FAMILYIDX+2]);
2968     result = ht.get(get);
2969     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2970         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2971 
2972     get = new Get(ROWS[ROWIDX]);
2973     get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[0]);
2974     result = ht.get(get);
2975     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2976         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2977 
2978     get = new Get(ROWS[ROWIDX]);
2979     get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[1]);
2980     get.addFamily(FAMILIES[FAMILYIDX]);
2981     result = ht.get(get);
2982     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2983         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2984 
2985     get = new Get(ROWS[ROWIDX]);
2986     get.addFamily(FAMILIES[FAMILYIDX]);
2987     get.addColumn(FAMILIES[FAMILYIDX+1], QUALIFIERS[1]);
2988     get.addColumn(FAMILIES[FAMILYIDX-2], QUALIFIERS[1]);
2989     get.addFamily(FAMILIES[FAMILYIDX-1]);
2990     get.addFamily(FAMILIES[FAMILYIDX+2]);
2991     result = ht.get(get);
2992     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2993         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2994 
2995   }
2996 
2997 
2998   /**
2999    * Verify a single column using scanners.
3000    * Expects family and qualifier arrays to be valid for at least
3001    * the range:  idx-2 to idx+2
3002    * Expects row array to be valid for at least idx to idx+2
3003    */
3004   private void scanVerifySingleColumn(Table ht,
3005       byte [][] ROWS, int ROWIDX,
3006       byte [][] FAMILIES, int FAMILYIDX,
3007       byte [][] QUALIFIERS, int QUALIFIERIDX,
3008       byte [][] VALUES, int VALUEIDX)
3009   throws Exception {
3010 
3011     Scan scan = new Scan();
3012     Result result = getSingleScanResult(ht, scan);
3013     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3014         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3015 
3016     scan = new Scan(ROWS[ROWIDX]);
3017     result = getSingleScanResult(ht, scan);
3018     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3019         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3020 
3021     scan = new Scan(ROWS[ROWIDX], ROWS[ROWIDX+1]);
3022     result = getSingleScanResult(ht, scan);
3023     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3024         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3025 
3026     scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX+1]);
3027     result = getSingleScanResult(ht, scan);
3028     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3029         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3030 
3031     scan = new Scan();
3032     scan.addFamily(FAMILIES[FAMILYIDX]);
3033     result = getSingleScanResult(ht, scan);
3034     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3035         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3036 
3037     scan = new Scan();
3038     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
3039     result = getSingleScanResult(ht, scan);
3040     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3041         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3042 
3043     scan = new Scan();
3044     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]);
3045     scan.addFamily(FAMILIES[FAMILYIDX]);
3046     result = getSingleScanResult(ht, scan);
3047     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3048         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3049 
3050     scan = new Scan();
3051     scan.addColumn(FAMILIES[FAMILYIDX-1], QUALIFIERS[QUALIFIERIDX+1]);
3052     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
3053     scan.addFamily(FAMILIES[FAMILYIDX+1]);
3054     result = getSingleScanResult(ht, scan);
3055     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3056         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3057 
3058   }
3059 
3060   /**
3061    * Verify we do not read any values by accident around a single column
3062    * Same requirements as getVerifySingleColumn
3063    */
3064   private void getVerifySingleEmpty(Table ht,
3065       byte [][] ROWS, int ROWIDX,
3066       byte [][] FAMILIES, int FAMILYIDX,
3067       byte [][] QUALIFIERS, int QUALIFIERIDX)
3068   throws Exception {
3069 
3070     Get get = new Get(ROWS[ROWIDX]);
3071     get.addFamily(FAMILIES[4]);
3072     get.addColumn(FAMILIES[4], QUALIFIERS[1]);
3073     Result result = ht.get(get);
3074     assertEmptyResult(result);
3075 
3076     get = new Get(ROWS[ROWIDX]);
3077     get.addFamily(FAMILIES[4]);
3078     get.addColumn(FAMILIES[4], QUALIFIERS[2]);
3079     result = ht.get(get);
3080     assertEmptyResult(result);
3081 
3082     get = new Get(ROWS[ROWIDX]);
3083     get.addFamily(FAMILIES[3]);
3084     get.addColumn(FAMILIES[4], QUALIFIERS[2]);
3085     get.addFamily(FAMILIES[5]);
3086     result = ht.get(get);
3087     assertEmptyResult(result);
3088 
3089     get = new Get(ROWS[ROWIDX+1]);
3090     result = ht.get(get);
3091     assertEmptyResult(result);
3092 
3093   }
3094 
3095   private void scanVerifySingleEmpty(Table ht,
3096       byte [][] ROWS, int ROWIDX,
3097       byte [][] FAMILIES, int FAMILYIDX,
3098       byte [][] QUALIFIERS, int QUALIFIERIDX)
3099   throws Exception {
3100 
3101     Scan scan = new Scan(ROWS[ROWIDX+1]);
3102     Result result = getSingleScanResult(ht, scan);
3103     assertNullResult(result);
3104 
3105     scan = new Scan(ROWS[ROWIDX+1],ROWS[ROWIDX+2]);
3106     result = getSingleScanResult(ht, scan);
3107     assertNullResult(result);
3108 
3109     scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX]);
3110     result = getSingleScanResult(ht, scan);
3111     assertNullResult(result);
3112 
3113     scan = new Scan();
3114     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]);
3115     scan.addFamily(FAMILIES[FAMILYIDX-1]);
3116     result = getSingleScanResult(ht, scan);
3117     assertNullResult(result);
3118 
3119   }
3120 
3121   //
3122   // Verifiers
3123   //
3124 
3125   private void assertKey(Cell key, byte [] row, byte [] family,
3126       byte [] qualifier, byte [] value)
3127   throws Exception {
3128     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3129         "Got row [" + Bytes.toString(CellUtil.cloneRow(key)) +"]",
3130         equals(row, CellUtil.cloneRow(key)));
3131     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3132         "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
3133         equals(family, CellUtil.cloneFamily(key)));
3134     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3135         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key)) + "]",
3136         equals(qualifier, CellUtil.cloneQualifier(key)));
3137     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3138         "Got value [" + Bytes.toString(CellUtil.cloneValue(key)) + "]",
3139         equals(value, CellUtil.cloneValue(key)));
3140   }
3141 
3142   static void assertIncrementKey(Cell key, byte [] row, byte [] family,
3143       byte [] qualifier, long value)
3144   throws Exception {
3145     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3146         "Got row [" + Bytes.toString(CellUtil.cloneRow(key)) +"]",
3147         equals(row, CellUtil.cloneRow(key)));
3148     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3149         "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
3150         equals(family, CellUtil.cloneFamily(key)));
3151     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3152         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key)) + "]",
3153         equals(qualifier, CellUtil.cloneQualifier(key)));
3154     assertTrue("Expected value [" + value + "] " +
3155         "Got value [" + Bytes.toLong(CellUtil.cloneValue(key)) + "]",
3156         Bytes.toLong(CellUtil.cloneValue(key)) == value);
3157   }
3158 
3159   private void assertNumKeys(Result result, int n) throws Exception {
3160     assertTrue("Expected " + n + " keys but got " + result.size(),
3161         result.size() == n);
3162   }
3163 
3164   private void assertNResult(Result result, byte [] row,
3165       byte [][] families, byte [][] qualifiers, byte [][] values,
3166       int [][] idxs)
3167   throws Exception {
3168     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3169         "Got row [" + Bytes.toString(result.getRow()) +"]",
3170         equals(row, result.getRow()));
3171     assertTrue("Expected " + idxs.length + " keys but result contains "
3172         + result.size(), result.size() == idxs.length);
3173 
3174     Cell [] keys = result.rawCells();
3175 
3176     for(int i=0;i<keys.length;i++) {
3177       byte [] family = families[idxs[i][0]];
3178       byte [] qualifier = qualifiers[idxs[i][1]];
3179       byte [] value = values[idxs[i][2]];
3180       Cell key = keys[i];
3181 
3182       byte[] famb = CellUtil.cloneFamily(key);
3183       byte[] qualb = CellUtil.cloneQualifier(key);
3184       byte[] valb = CellUtil.cloneValue(key);
3185       assertTrue("(" + i + ") Expected family [" + Bytes.toString(family)
3186           + "] " + "Got family [" + Bytes.toString(famb) + "]",
3187           equals(family, famb));
3188       assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier)
3189           + "] " + "Got qualifier [" + Bytes.toString(qualb) + "]",
3190           equals(qualifier, qualb));
3191       assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] "
3192           + "Got value [" + Bytes.toString(valb) + "]",
3193           equals(value, valb));
3194     }
3195   }
3196 
3197   private void assertNResult(Result result, byte [] row,
3198       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
3199       int start, int end)
3200   throws IOException {
3201     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3202         "Got row [" + Bytes.toString(result.getRow()) +"]",
3203         equals(row, result.getRow()));
3204     int expectedResults = end - start + 1;
3205     assertEquals(expectedResults, result.size());
3206 
3207     Cell[] keys = result.rawCells();
3208 
3209     for (int i=0; i<keys.length; i++) {
3210       byte [] value = values[end-i];
3211       long ts = stamps[end-i];
3212       Cell key = keys[i];
3213 
3214       assertTrue("(" + i + ") Expected family [" + Bytes.toString(family)
3215           + "] " + "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
3216           CellUtil.matchingFamily(key, family));
3217       assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier)
3218           + "] " + "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key))+ "]",
3219           CellUtil.matchingQualifier(key, qualifier));
3220       assertTrue("Expected ts [" + ts + "] " +
3221           "Got ts [" + key.getTimestamp() + "]", ts == key.getTimestamp());
3222       assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] "
3223           + "Got value [" + Bytes.toString(CellUtil.cloneValue(key)) + "]",
3224           CellUtil.matchingValue(key,  value));
3225     }
3226   }
3227 
3228   /**
3229    * Validate that result contains two specified keys, exactly.
3230    * It is assumed key A sorts before key B.
3231    */
3232   private void assertDoubleResult(Result result, byte [] row,
3233       byte [] familyA, byte [] qualifierA, byte [] valueA,
3234       byte [] familyB, byte [] qualifierB, byte [] valueB)
3235   throws Exception {
3236     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3237         "Got row [" + Bytes.toString(result.getRow()) +"]",
3238         equals(row, result.getRow()));
3239     assertTrue("Expected two keys but result contains " + result.size(),
3240         result.size() == 2);
3241     Cell [] kv = result.rawCells();
3242     Cell kvA = kv[0];
3243     assertTrue("(A) Expected family [" + Bytes.toString(familyA) + "] " +
3244         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kvA)) + "]",
3245         equals(familyA, CellUtil.cloneFamily(kvA)));
3246     assertTrue("(A) Expected qualifier [" + Bytes.toString(qualifierA) + "] " +
3247         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kvA)) + "]",
3248         equals(qualifierA, CellUtil.cloneQualifier(kvA)));
3249     assertTrue("(A) Expected value [" + Bytes.toString(valueA) + "] " +
3250         "Got value [" + Bytes.toString(CellUtil.cloneValue(kvA)) + "]",
3251         equals(valueA, CellUtil.cloneValue(kvA)));
3252     Cell kvB = kv[1];
3253     assertTrue("(B) Expected family [" + Bytes.toString(familyB) + "] " +
3254         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kvB)) + "]",
3255         equals(familyB, CellUtil.cloneFamily(kvB)));
3256     assertTrue("(B) Expected qualifier [" + Bytes.toString(qualifierB) + "] " +
3257         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kvB)) + "]",
3258         equals(qualifierB, CellUtil.cloneQualifier(kvB)));
3259     assertTrue("(B) Expected value [" + Bytes.toString(valueB) + "] " +
3260         "Got value [" + Bytes.toString(CellUtil.cloneValue(kvB)) + "]",
3261         equals(valueB, CellUtil.cloneValue(kvB)));
3262   }
3263 
3264   private void assertSingleResult(Result result, byte [] row, byte [] family,
3265       byte [] qualifier, byte [] value)
3266   throws Exception {
3267     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3268         "Got row [" + Bytes.toString(result.getRow()) +"]",
3269         equals(row, result.getRow()));
3270     assertTrue("Expected a single key but result contains " + result.size(),
3271         result.size() == 1);
3272     Cell kv = result.rawCells()[0];
3273     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3274         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kv)) + "]",
3275         equals(family, CellUtil.cloneFamily(kv)));
3276     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3277         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]",
3278         equals(qualifier, CellUtil.cloneQualifier(kv)));
3279     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3280         "Got value [" + Bytes.toString(CellUtil.cloneValue(kv)) + "]",
3281         equals(value, CellUtil.cloneValue(kv)));
3282   }
3283 
3284   private void assertSingleResult(Result result, byte [] row, byte [] family,
3285       byte [] qualifier, long ts, byte [] value)
3286   throws Exception {
3287     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3288         "Got row [" + Bytes.toString(result.getRow()) +"]",
3289         equals(row, result.getRow()));
3290     assertTrue("Expected a single key but result contains " + result.size(),
3291         result.size() == 1);
3292     Cell kv = result.rawCells()[0];
3293     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3294         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kv)) + "]",
3295         equals(family, CellUtil.cloneFamily(kv)));
3296     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3297         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]",
3298         equals(qualifier, CellUtil.cloneQualifier(kv)));
3299     assertTrue("Expected ts [" + ts + "] " +
3300         "Got ts [" + kv.getTimestamp() + "]", ts == kv.getTimestamp());
3301     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3302         "Got value [" + Bytes.toString(CellUtil.cloneValue(kv)) + "]",
3303         equals(value, CellUtil.cloneValue(kv)));
3304   }
3305 
3306   private void assertEmptyResult(Result result) throws Exception {
3307     assertTrue("expected an empty result but result contains " +
3308         result.size() + " keys", result.isEmpty());
3309   }
3310 
3311   private void assertNullResult(Result result) throws Exception {
3312     assertTrue("expected null result but received a non-null result",
3313         result == null);
3314   }
3315 
3316   //
3317   // Helpers
3318   //
3319 
3320   private Result getSingleScanResult(Table ht, Scan scan) throws IOException {
3321     ResultScanner scanner = ht.getScanner(scan);
3322     Result result = scanner.next();
3323     scanner.close();
3324     return result;
3325   }
3326 
3327   private byte [][] makeNAscii(byte [] base, int n) {
3328     if(n > 256) {
3329       return makeNBig(base, n);
3330     }
3331     byte [][] ret = new byte[n][];
3332     for(int i=0;i<n;i++) {
3333       byte [] tail = Bytes.toBytes(Integer.toString(i));
3334       ret[i] = Bytes.add(base, tail);
3335     }
3336     return ret;
3337   }
3338 
3339   private byte [][] makeN(byte [] base, int n) {
3340     if (n > 256) {
3341       return makeNBig(base, n);
3342     }
3343     byte [][] ret = new byte[n][];
3344     for(int i=0;i<n;i++) {
3345       ret[i] = Bytes.add(base, new byte[]{(byte)i});
3346     }
3347     return ret;
3348   }
3349 
3350   private byte [][] makeNBig(byte [] base, int n) {
3351     byte [][] ret = new byte[n][];
3352     for(int i=0;i<n;i++) {
3353       int byteA = (i % 256);
3354       int byteB = (i >> 8);
3355       ret[i] = Bytes.add(base, new byte[]{(byte)byteB,(byte)byteA});
3356     }
3357     return ret;
3358   }
3359 
3360   private long [] makeStamps(int n) {
3361     long [] stamps = new long[n];
3362     for(int i=0;i<n;i++) stamps[i] = i+1;
3363     return stamps;
3364   }
3365 
3366   static boolean equals(byte [] left, byte [] right) {
3367     if (left == null && right == null) return true;
3368     if (left == null && right.length == 0) return true;
3369     if (right == null && left.length == 0) return true;
3370     return Bytes.equals(left, right);
3371   }
3372 
3373   @Test
3374   public void testDuplicateVersions() throws Exception {
3375     byte [] TABLE = Bytes.toBytes("testDuplicateVersions");
3376 
3377     long [] STAMPS = makeStamps(20);
3378     byte [][] VALUES = makeNAscii(VALUE, 20);
3379 
3380     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3381 
3382     // Insert 4 versions of same column
3383     Put put = new Put(ROW);
3384     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3385     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3386     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3387     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3388     ht.put(put);
3389 
3390     // Verify we can get each one properly
3391     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3392     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3393     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3394     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3395     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3396     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3397     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3398     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3399 
3400     // Verify we don't accidentally get others
3401     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3402     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3403     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3404     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3405     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3406     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3407 
3408     // Ensure maxVersions in query is respected
3409     Get get = new Get(ROW);
3410     get.addColumn(FAMILY, QUALIFIER);
3411     get.setMaxVersions(2);
3412     Result result = ht.get(get);
3413     assertNResult(result, ROW, FAMILY, QUALIFIER,
3414         new long [] {STAMPS[4], STAMPS[5]},
3415         new byte[][] {VALUES[4], VALUES[5]},
3416         0, 1);
3417 
3418     Scan scan = new Scan(ROW);
3419     scan.addColumn(FAMILY, QUALIFIER);
3420     scan.setMaxVersions(2);
3421     result = getSingleScanResult(ht, scan);
3422     assertNResult(result, ROW, FAMILY, QUALIFIER,
3423         new long [] {STAMPS[4], STAMPS[5]},
3424         new byte[][] {VALUES[4], VALUES[5]},
3425         0, 1);
3426 
3427     // Flush and redo
3428 
3429     TEST_UTIL.flush();
3430 
3431     // Verify we can get each one properly
3432     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3433     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3434     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3435     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3436     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3437     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3438     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3439     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3440 
3441     // Verify we don't accidentally get others
3442     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3443     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3444     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3445     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3446     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3447     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3448 
3449     // Ensure maxVersions in query is respected
3450     get = new Get(ROW);
3451     get.addColumn(FAMILY, QUALIFIER);
3452     get.setMaxVersions(2);
3453     result = ht.get(get);
3454     assertNResult(result, ROW, FAMILY, QUALIFIER,
3455         new long [] {STAMPS[4], STAMPS[5]},
3456         new byte[][] {VALUES[4], VALUES[5]},
3457         0, 1);
3458 
3459     scan = new Scan(ROW);
3460     scan.addColumn(FAMILY, QUALIFIER);
3461     scan.setMaxVersions(2);
3462     result = getSingleScanResult(ht, scan);
3463     assertNResult(result, ROW, FAMILY, QUALIFIER,
3464         new long [] {STAMPS[4], STAMPS[5]},
3465         new byte[][] {VALUES[4], VALUES[5]},
3466         0, 1);
3467 
3468 
3469     // Add some memstore and retest
3470 
3471     // Insert 4 more versions of same column and a dupe
3472     put = new Put(ROW);
3473     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
3474     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3475     put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
3476     put.add(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3477     put.add(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
3478     ht.put(put);
3479 
3480     // Ensure maxVersions in query is respected
3481     get = new Get(ROW);
3482     get.addColumn(FAMILY, QUALIFIER);
3483     get.setMaxVersions(7);
3484     result = ht.get(get);
3485     assertNResult(result, ROW, FAMILY, QUALIFIER,
3486         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3487         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3488         0, 6);
3489 
3490     scan = new Scan(ROW);
3491     scan.addColumn(FAMILY, QUALIFIER);
3492     scan.setMaxVersions(7);
3493     result = getSingleScanResult(ht, scan);
3494     assertNResult(result, ROW, FAMILY, QUALIFIER,
3495         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3496         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3497         0, 6);
3498 
3499     get = new Get(ROW);
3500     get.setMaxVersions(7);
3501     result = ht.get(get);
3502     assertNResult(result, ROW, FAMILY, QUALIFIER,
3503         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3504         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3505         0, 6);
3506 
3507     scan = new Scan(ROW);
3508     scan.setMaxVersions(7);
3509     result = getSingleScanResult(ht, scan);
3510     assertNResult(result, ROW, FAMILY, QUALIFIER,
3511         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3512         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3513         0, 6);
3514 
3515     // Verify we can get each one properly
3516     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3517     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3518     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3519     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3520     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3521     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3522     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3523     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3524 
3525     // Verify we don't accidentally get others
3526     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3527     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
3528     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3529     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
3530 
3531     // Ensure maxVersions of table is respected
3532 
3533     TEST_UTIL.flush();
3534 
3535     // Insert 4 more versions of same column and a dupe
3536     put = new Put(ROW);
3537     put.add(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
3538     put.add(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
3539     put.add(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
3540     put.add(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
3541     ht.put(put);
3542 
3543     get = new Get(ROW);
3544     get.addColumn(FAMILY, QUALIFIER);
3545     get.setMaxVersions(Integer.MAX_VALUE);
3546     result = ht.get(get);
3547     assertNResult(result, ROW, FAMILY, QUALIFIER,
3548         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
3549         new byte[][] {VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
3550         0, 9);
3551 
3552     scan = new Scan(ROW);
3553     scan.addColumn(FAMILY, QUALIFIER);
3554     scan.setMaxVersions(Integer.MAX_VALUE);
3555     result = getSingleScanResult(ht, scan);
3556     assertNResult(result, ROW, FAMILY, QUALIFIER,
3557         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
3558         new byte[][] {VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
3559         0, 9);
3560 
3561     // Delete a version in the memstore and a version in a storefile
3562     Delete delete = new Delete(ROW);
3563     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[11]);
3564     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[7]);
3565     ht.delete(delete);
3566 
3567     // Test that it's gone
3568     get = new Get(ROW);
3569     get.addColumn(FAMILY, QUALIFIER);
3570     get.setMaxVersions(Integer.MAX_VALUE);
3571     result = ht.get(get);
3572     assertNResult(result, ROW, FAMILY, QUALIFIER,
3573         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
3574         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
3575         0, 9);
3576 
3577     scan = new Scan(ROW);
3578     scan.addColumn(FAMILY, QUALIFIER);
3579     scan.setMaxVersions(Integer.MAX_VALUE);
3580     result = getSingleScanResult(ht, scan);
3581     assertNResult(result, ROW, FAMILY, QUALIFIER,
3582         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
3583         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
3584         0, 9);
3585   }
3586 
3587   @Test
3588   public void testUpdates() throws Exception {
3589 
3590     byte [] TABLE = Bytes.toBytes("testUpdates");
3591     Table hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3592 
3593     // Write a column with values at timestamp 1, 2 and 3
3594     byte[] row = Bytes.toBytes("row1");
3595     byte[] qualifier = Bytes.toBytes("myCol");
3596     Put put = new Put(row);
3597     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3598     hTable.put(put);
3599 
3600     put = new Put(row);
3601     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3602     hTable.put(put);
3603 
3604     put = new Put(row);
3605     put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3606     hTable.put(put);
3607 
3608     Get get = new Get(row);
3609     get.addColumn(FAMILY, qualifier);
3610     get.setMaxVersions();
3611 
3612     // Check that the column indeed has the right values at timestamps 1 and
3613     // 2
3614     Result result = hTable.get(get);
3615     NavigableMap<Long, byte[]> navigableMap =
3616         result.getMap().get(FAMILY).get(qualifier);
3617     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3618     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3619 
3620     // Update the value at timestamp 1
3621     put = new Put(row);
3622     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3623     hTable.put(put);
3624 
3625     // Update the value at timestamp 2
3626     put = new Put(row);
3627     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3628     hTable.put(put);
3629 
3630     // Check that the values at timestamp 2 and 1 got updated
3631     result = hTable.get(get);
3632     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3633     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3634     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3635   }
3636 
3637   @Test
3638   public void testUpdatesWithMajorCompaction() throws Exception {
3639 
3640     TableName TABLE = TableName.valueOf("testUpdatesWithMajorCompaction");
3641     Table hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3642     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
3643 
3644     // Write a column with values at timestamp 1, 2 and 3
3645     byte[] row = Bytes.toBytes("row2");
3646     byte[] qualifier = Bytes.toBytes("myCol");
3647     Put put = new Put(row);
3648     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3649     hTable.put(put);
3650 
3651     put = new Put(row);
3652     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3653     hTable.put(put);
3654 
3655     put = new Put(row);
3656     put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3657     hTable.put(put);
3658 
3659     Get get = new Get(row);
3660     get.addColumn(FAMILY, qualifier);
3661     get.setMaxVersions();
3662 
3663     // Check that the column indeed has the right values at timestamps 1 and
3664     // 2
3665     Result result = hTable.get(get);
3666     NavigableMap<Long, byte[]> navigableMap =
3667         result.getMap().get(FAMILY).get(qualifier);
3668     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3669     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3670 
3671     // Trigger a major compaction
3672     admin.flush(TABLE);
3673     admin.majorCompact(TABLE);
3674     Thread.sleep(6000);
3675 
3676     // Update the value at timestamp 1
3677     put = new Put(row);
3678     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3679     hTable.put(put);
3680 
3681     // Update the value at timestamp 2
3682     put = new Put(row);
3683     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3684     hTable.put(put);
3685 
3686     // Trigger a major compaction
3687     admin.flush(TABLE);
3688     admin.majorCompact(TABLE);
3689     Thread.sleep(6000);
3690 
3691     // Check that the values at timestamp 2 and 1 got updated
3692     result = hTable.get(get);
3693     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3694     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3695     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3696   }
3697 
3698   @Test
3699   public void testMajorCompactionBetweenTwoUpdates() throws Exception {
3700 
3701     String tableName = "testMajorCompactionBetweenTwoUpdates";
3702     byte [] TABLE = Bytes.toBytes(tableName);
3703     Table hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3704     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
3705 
3706     // Write a column with values at timestamp 1, 2 and 3
3707     byte[] row = Bytes.toBytes("row3");
3708     byte[] qualifier = Bytes.toBytes("myCol");
3709     Put put = new Put(row);
3710     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3711     hTable.put(put);
3712 
3713     put = new Put(row);
3714     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3715     hTable.put(put);
3716 
3717     put = new Put(row);
3718     put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3719     hTable.put(put);
3720 
3721     Get get = new Get(row);
3722     get.addColumn(FAMILY, qualifier);
3723     get.setMaxVersions();
3724 
3725     // Check that the column indeed has the right values at timestamps 1 and
3726     // 2
3727     Result result = hTable.get(get);
3728     NavigableMap<Long, byte[]> navigableMap =
3729         result.getMap().get(FAMILY).get(qualifier);
3730     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3731     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3732 
3733     // Trigger a major compaction
3734     admin.flush(tableName);
3735     admin.majorCompact(tableName);
3736     Thread.sleep(6000);
3737 
3738     // Update the value at timestamp 1
3739     put = new Put(row);
3740     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3741     hTable.put(put);
3742 
3743     // Trigger a major compaction
3744     admin.flush(tableName);
3745     admin.majorCompact(tableName);
3746     Thread.sleep(6000);
3747 
3748     // Update the value at timestamp 2
3749     put = new Put(row);
3750     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3751     hTable.put(put);
3752 
3753     // Trigger a major compaction
3754     admin.flush(tableName);
3755     admin.majorCompact(tableName);
3756     Thread.sleep(6000);
3757 
3758     // Check that the values at timestamp 2 and 1 got updated
3759     result = hTable.get(get);
3760     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3761 
3762     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3763     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3764   }
3765 
3766   @Test
3767   public void testGet_EmptyTable() throws IOException {
3768     Table table = TEST_UTIL.createTable(TableName.valueOf("testGet_EmptyTable"), FAMILY);
3769     Get get = new Get(ROW);
3770     get.addFamily(FAMILY);
3771     Result r = table.get(get);
3772     assertTrue(r.isEmpty());
3773   }
3774 
3775   @Test
3776   public void testGet_NullQualifier() throws IOException {
3777     Table table = TEST_UTIL.createTable(TableName.valueOf("testGet_NullQualifier"), FAMILY);
3778     Put put = new Put(ROW);
3779     put.add(FAMILY, QUALIFIER, VALUE);
3780     table.put(put);
3781 
3782     put = new Put(ROW);
3783     put.add(FAMILY, null, VALUE);
3784     table.put(put);
3785     LOG.info("Row put");
3786 
3787     Get get = new Get(ROW);
3788     get.addColumn(FAMILY, null);
3789     Result r = table.get(get);
3790     assertEquals(1, r.size());
3791 
3792     get = new Get(ROW);
3793     get.addFamily(FAMILY);
3794     r = table.get(get);
3795     assertEquals(2, r.size());
3796   }
3797 
3798   @Test
3799   public void testGet_NonExistentRow() throws IOException {
3800     Table table = TEST_UTIL.createTable(TableName.valueOf("testGet_NonExistentRow"), FAMILY);
3801     Put put = new Put(ROW);
3802     put.add(FAMILY, QUALIFIER, VALUE);
3803     table.put(put);
3804     LOG.info("Row put");
3805 
3806     Get get = new Get(ROW);
3807     get.addFamily(FAMILY);
3808     Result r = table.get(get);
3809     assertFalse(r.isEmpty());
3810     System.out.println("Row retrieved successfully");
3811 
3812     byte [] missingrow = Bytes.toBytes("missingrow");
3813     get = new Get(missingrow);
3814     get.addFamily(FAMILY);
3815     r = table.get(get);
3816     assertTrue(r.isEmpty());
3817     LOG.info("Row missing as it should be");
3818   }
3819 
3820   @Test
3821   public void testPut() throws IOException {
3822     final byte [] CONTENTS_FAMILY = Bytes.toBytes("contents");
3823     final byte [] SMALL_FAMILY = Bytes.toBytes("smallfam");
3824     final byte [] row1 = Bytes.toBytes("row1");
3825     final byte [] row2 = Bytes.toBytes("row2");
3826     final byte [] value = Bytes.toBytes("abcd");
3827     Table table = TEST_UTIL.createTable(Bytes.toBytes("testPut"),
3828       new byte [][] {CONTENTS_FAMILY, SMALL_FAMILY});
3829     Put put = new Put(row1);
3830     put.add(CONTENTS_FAMILY, null, value);
3831     table.put(put);
3832 
3833     put = new Put(row2);
3834     put.add(CONTENTS_FAMILY, null, value);
3835 
3836     assertEquals(put.size(), 1);
3837     assertEquals(put.getFamilyCellMap().get(CONTENTS_FAMILY).size(), 1);
3838 
3839     // KeyValue v1 expectation.  Cast for now until we go all Cell all the time. TODO
3840     KeyValue kv = (KeyValue)put.getFamilyCellMap().get(CONTENTS_FAMILY).get(0);
3841 
3842     assertTrue(Bytes.equals(kv.getFamily(), CONTENTS_FAMILY));
3843     // will it return null or an empty byte array?
3844     assertTrue(Bytes.equals(kv.getQualifier(), new byte[0]));
3845 
3846     assertTrue(Bytes.equals(kv.getValue(), value));
3847 
3848     table.put(put);
3849 
3850     Scan scan = new Scan();
3851     scan.addColumn(CONTENTS_FAMILY, null);
3852     ResultScanner scanner = table.getScanner(scan);
3853     for (Result r : scanner) {
3854       for(Cell key : r.rawCells()) {
3855         System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString());
3856       }
3857     }
3858   }
3859 
3860   @Test
3861   public void testPutNoCF() throws IOException {
3862     final byte[] BAD_FAM = Bytes.toBytes("BAD_CF");
3863     final byte[] VAL = Bytes.toBytes(100);
3864     Table table = TEST_UTIL.createTable(Bytes.toBytes("testPutNoCF"), FAMILY);
3865 
3866     boolean caughtNSCFE = false;
3867 
3868     try {
3869       Put p = new Put(ROW);
3870       p.add(BAD_FAM, QUALIFIER, VAL);
3871       table.put(p);
3872     } catch (RetriesExhaustedWithDetailsException e) {
3873       caughtNSCFE = e.getCause(0) instanceof NoSuchColumnFamilyException;
3874     }
3875     assertTrue("Should throw NoSuchColumnFamilyException", caughtNSCFE);
3876 
3877   }
3878 
3879   @Test
3880   public void testRowsPut() throws IOException {
3881     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
3882     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
3883     final int NB_BATCH_ROWS = 10;
3884     final byte[] value = Bytes.toBytes("abcd");
3885     Table table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPut"),
3886       new byte[][] {CONTENTS_FAMILY, SMALL_FAMILY });
3887     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
3888     for (int i = 0; i < NB_BATCH_ROWS; i++) {
3889       byte[] row = Bytes.toBytes("row" + i);
3890       Put put = new Put(row);
3891       put.setDurability(Durability.SKIP_WAL);
3892       put.add(CONTENTS_FAMILY, null, value);
3893       rowsUpdate.add(put);
3894     }
3895     table.put(rowsUpdate);
3896     Scan scan = new Scan();
3897     scan.addFamily(CONTENTS_FAMILY);
3898     ResultScanner scanner = table.getScanner(scan);
3899     int nbRows = 0;
3900     for (@SuppressWarnings("unused")
3901     Result row : scanner)
3902       nbRows++;
3903     assertEquals(NB_BATCH_ROWS, nbRows);
3904   }
3905 
3906   @Test
3907   public void testRowsPutBufferedOneFlush() throws IOException {
3908     final byte [] CONTENTS_FAMILY = Bytes.toBytes("contents");
3909     final byte [] SMALL_FAMILY = Bytes.toBytes("smallfam");
3910     final byte [] value = Bytes.toBytes("abcd");
3911     final int NB_BATCH_ROWS = 10;
3912     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPutBufferedOneFlush"),
3913       new byte [][] {CONTENTS_FAMILY, SMALL_FAMILY});
3914     table.setAutoFlush(false);
3915     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
3916     for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
3917       byte[] row = Bytes.toBytes("row" + i);
3918       Put put = new Put(row);
3919       put.setDurability(Durability.SKIP_WAL);
3920       put.add(CONTENTS_FAMILY, null, value);
3921       rowsUpdate.add(put);
3922     }
3923     table.put(rowsUpdate);
3924 
3925     Scan scan = new Scan();
3926     scan.addFamily(CONTENTS_FAMILY);
3927     ResultScanner scanner = table.getScanner(scan);
3928     int nbRows = 0;
3929     for (@SuppressWarnings("unused")
3930     Result row : scanner)
3931       nbRows++;
3932     assertEquals(0, nbRows);
3933     scanner.close();
3934 
3935     table.flushCommits();
3936 
3937     scan = new Scan();
3938     scan.addFamily(CONTENTS_FAMILY);
3939     scanner = table.getScanner(scan);
3940     nbRows = 0;
3941     for (@SuppressWarnings("unused")
3942     Result row : scanner)
3943       nbRows++;
3944     assertEquals(NB_BATCH_ROWS * 10, nbRows);
3945     table.close();
3946   }
3947 
3948   @Test
3949   public void testRowsPutBufferedManyManyFlushes() throws IOException {
3950     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
3951     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
3952     final byte[] value = Bytes.toBytes("abcd");
3953     final int NB_BATCH_ROWS = 10;
3954     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPutBufferedManyManyFlushes"),
3955       new byte[][] {CONTENTS_FAMILY, SMALL_FAMILY });
3956     table.setWriteBufferSize(10);
3957     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
3958     for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
3959       byte[] row = Bytes.toBytes("row" + i);
3960       Put put = new Put(row);
3961       put.setDurability(Durability.SKIP_WAL);
3962       put.add(CONTENTS_FAMILY, null, value);
3963       rowsUpdate.add(put);
3964     }
3965     table.put(rowsUpdate);
3966 
3967     Scan scan = new Scan();
3968     scan.addFamily(CONTENTS_FAMILY);
3969     ResultScanner scanner = table.getScanner(scan);
3970     int nbRows = 0;
3971     for (@SuppressWarnings("unused")
3972     Result row : scanner)
3973       nbRows++;
3974     assertEquals(NB_BATCH_ROWS * 10, nbRows);
3975   }
3976 
3977   @Test
3978   public void testAddKeyValue() throws IOException {
3979     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
3980     final byte[] value = Bytes.toBytes("abcd");
3981     final byte[] row1 = Bytes.toBytes("row1");
3982     final byte[] row2 = Bytes.toBytes("row2");
3983     byte[] qualifier = Bytes.toBytes("qf1");
3984     Put put = new Put(row1);
3985 
3986     // Adding KeyValue with the same row
3987     KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value);
3988     boolean ok = true;
3989     try {
3990       put.add(kv);
3991     } catch (IOException e) {
3992       ok = false;
3993     }
3994     assertEquals(true, ok);
3995 
3996     // Adding KeyValue with the different row
3997     kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value);
3998     ok = false;
3999     try {
4000       put.add(kv);
4001     } catch (IOException e) {
4002       ok = true;
4003     }
4004     assertEquals(true, ok);
4005   }
4006 
4007   /**
4008    * test for HBASE-737
4009    * @throws IOException
4010    */
4011   @Test
4012   public void testHBase737 () throws IOException {
4013     final byte [] FAM1 = Bytes.toBytes("fam1");
4014     final byte [] FAM2 = Bytes.toBytes("fam2");
4015     // Open table
4016     Table table = TEST_UTIL.createTable(Bytes.toBytes("testHBase737"),
4017       new byte [][] {FAM1, FAM2});
4018     // Insert some values
4019     Put put = new Put(ROW);
4020     put.add(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg"));
4021     table.put(put);
4022     try {
4023       Thread.sleep(1000);
4024     } catch (InterruptedException i) {
4025       //ignore
4026     }
4027 
4028     put = new Put(ROW);
4029     put.add(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456"));
4030     table.put(put);
4031 
4032     try {
4033       Thread.sleep(1000);
4034     } catch (InterruptedException i) {
4035       //ignore
4036     }
4037 
4038     put = new Put(ROW);
4039     put.add(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop"));
4040     table.put(put);
4041 
4042     long times[] = new long[3];
4043 
4044     // First scan the memstore
4045 
4046     Scan scan = new Scan();
4047     scan.addFamily(FAM1);
4048     scan.addFamily(FAM2);
4049     ResultScanner s = table.getScanner(scan);
4050     try {
4051       int index = 0;
4052       Result r = null;
4053       while ((r = s.next()) != null) {
4054         for(Cell key : r.rawCells()) {
4055           times[index++] = key.getTimestamp();
4056         }
4057       }
4058     } finally {
4059       s.close();
4060     }
4061     for (int i = 0; i < times.length - 1; i++) {
4062       for (int j = i + 1; j < times.length; j++) {
4063         assertTrue(times[j] > times[i]);
4064       }
4065     }
4066 
4067     // Flush data to disk and try again
4068     TEST_UTIL.flush();
4069 
4070     // Reset times
4071     for(int i=0;i<times.length;i++) {
4072       times[i] = 0;
4073     }
4074 
4075     try {
4076       Thread.sleep(1000);
4077     } catch (InterruptedException i) {
4078       //ignore
4079     }
4080     scan = new Scan();
4081     scan.addFamily(FAM1);
4082     scan.addFamily(FAM2);
4083     s = table.getScanner(scan);
4084     try {
4085       int index = 0;
4086       Result r = null;
4087       while ((r = s.next()) != null) {
4088         for(Cell key : r.rawCells()) {
4089           times[index++] = key.getTimestamp();
4090         }
4091       }
4092     } finally {
4093       s.close();
4094     }
4095     for (int i = 0; i < times.length - 1; i++) {
4096       for (int j = i + 1; j < times.length; j++) {
4097         assertTrue(times[j] > times[i]);
4098       }
4099     }
4100   }
4101 
4102   @Test
4103   public void testListTables() throws IOException, InterruptedException {
4104     TableName t1 = TableName.valueOf("testListTables1");
4105     TableName t2 = TableName.valueOf("testListTables2");
4106     TableName t3 = TableName.valueOf("testListTables3");
4107     TableName [] tables = new TableName[] { t1, t2, t3 };
4108     for (int i = 0; i < tables.length; i++) {
4109       TEST_UTIL.createTable(tables[i], FAMILY);
4110     }
4111     Admin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
4112     HTableDescriptor[] ts = admin.listTables();
4113     HashSet<HTableDescriptor> result = new HashSet<HTableDescriptor>(ts.length);
4114     Collections.addAll(result, ts);
4115     int size = result.size();
4116     assertTrue(size >= tables.length);
4117     for (int i = 0; i < tables.length && i < size; i++) {
4118       boolean found = false;
4119       for (int j = 0; j < ts.length; j++) {
4120         if (ts[j].getTableName().equals(tables[i])) {
4121           found = true;
4122           break;
4123         }
4124       }
4125       assertTrue("Not found: " + tables[i], found);
4126     }
4127   }
4128 
4129   /**
4130    * creates an HTable for tableName using an unmanaged HConnection.
4131    *
4132    * @param tableName - table to create
4133    * @return the created HTable object
4134    * @throws IOException
4135    */
4136   HTable createUnmangedHConnectionHTable(final TableName tableName) throws IOException {
4137     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
4138     HConnection conn = HConnectionManager.createConnection(TEST_UTIL.getConfiguration());
4139     return (HTable)conn.getTable(tableName);
4140   }
4141 
4142   /**
4143    * simple test that just executes parts of the client
4144    * API that accept a pre-created HConnection instance
4145    *
4146    * @throws IOException
4147    */
4148   @Test
4149   public void testUnmanagedHConnection() throws IOException {
4150     final TableName tableName = TableName.valueOf("testUnmanagedHConnection");
4151     HTable t = createUnmangedHConnectionHTable(tableName);
4152     HBaseAdmin ha = new HBaseAdmin(t.getConnection());
4153     assertTrue(ha.tableExists(tableName));
4154     assertTrue(t.get(new Get(ROW)).isEmpty());
4155     ha.close();
4156   }
4157 
4158   /**
4159    * test of that unmanaged HConnections are able to reconnect
4160    * properly (see HBASE-5058)
4161    *
4162    * @throws Exception
4163    */
4164   @Test
4165   public void testUnmanagedHConnectionReconnect() throws Exception {
4166     final TableName tableName = TableName.valueOf("testUnmanagedHConnectionReconnect");
4167     HTable t = createUnmangedHConnectionHTable(tableName);
4168     Connection conn = t.getConnection();
4169     try (HBaseAdmin ha = new HBaseAdmin(conn)) {
4170       assertTrue(ha.tableExists(tableName));
4171       assertTrue(t.get(new Get(ROW)).isEmpty());
4172     }
4173 
4174     // stop the master
4175     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
4176     cluster.stopMaster(0, false);
4177     cluster.waitOnMaster(0);
4178 
4179     // start up a new master
4180     cluster.startMaster();
4181     assertTrue(cluster.waitForActiveAndReadyMaster());
4182 
4183     // test that the same unmanaged connection works with a new
4184     // HBaseAdmin and can connect to the new master;
4185     try (HBaseAdmin newAdmin = new HBaseAdmin(conn)) {
4186       assertTrue(newAdmin.tableExists(tableName));
4187       assertTrue(newAdmin.getClusterStatus().getServersSize() == SLAVES);
4188     }
4189   }
4190 
4191   @Test
4192   public void testMiscHTableStuff() throws IOException {
4193     final TableName tableAname = TableName.valueOf("testMiscHTableStuffA");
4194     final TableName tableBname = TableName.valueOf("testMiscHTableStuffB");
4195     final byte[] attrName = Bytes.toBytes("TESTATTR");
4196     final byte[] attrValue = Bytes.toBytes("somevalue");
4197     byte[] value = Bytes.toBytes("value");
4198 
4199     Table a = TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY);
4200     Table b = TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY);
4201     Put put = new Put(ROW);
4202     put.add(HConstants.CATALOG_FAMILY, null, value);
4203     a.put(put);
4204 
4205     // open a new connection to A and a connection to b
4206     Table newA = new HTable(TEST_UTIL.getConfiguration(), tableAname);
4207 
4208     // copy data from A to B
4209     Scan scan = new Scan();
4210     scan.addFamily(HConstants.CATALOG_FAMILY);
4211     ResultScanner s = newA.getScanner(scan);
4212     try {
4213       for (Result r : s) {
4214         put = new Put(r.getRow());
4215         put.setDurability(Durability.SKIP_WAL);
4216         for (Cell kv : r.rawCells()) {
4217           put.add(kv);
4218         }
4219         b.put(put);
4220       }
4221     } finally {
4222       s.close();
4223     }
4224 
4225     // Opening a new connection to A will cause the tables to be reloaded
4226     Table anotherA = new HTable(TEST_UTIL.getConfiguration(), tableAname);
4227     Get get = new Get(ROW);
4228     get.addFamily(HConstants.CATALOG_FAMILY);
4229     anotherA.get(get);
4230 
4231     // We can still access A through newA because it has the table information
4232     // cached. And if it needs to recalibrate, that will cause the information
4233     // to be reloaded.
4234 
4235     // Test user metadata
4236     Admin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
4237     // make a modifiable descriptor
4238     HTableDescriptor desc = new HTableDescriptor(a.getTableDescriptor());
4239     // offline the table
4240     admin.disableTable(tableAname);
4241     // add a user attribute to HTD
4242     desc.setValue(attrName, attrValue);
4243     // add a user attribute to HCD
4244     for (HColumnDescriptor c : desc.getFamilies())
4245       c.setValue(attrName, attrValue);
4246     // update metadata for all regions of this table
4247     admin.modifyTable(tableAname, desc);
4248     // enable the table
4249     admin.enableTable(tableAname);
4250 
4251     // Test that attribute changes were applied
4252     desc = a.getTableDescriptor();
4253     assertEquals("wrong table descriptor returned", desc.getTableName(), tableAname);
4254     // check HTD attribute
4255     value = desc.getValue(attrName);
4256     assertFalse("missing HTD attribute value", value == null);
4257     assertFalse("HTD attribute value is incorrect",
4258       Bytes.compareTo(value, attrValue) != 0);
4259     // check HCD attribute
4260     for (HColumnDescriptor c : desc.getFamilies()) {
4261       value = c.getValue(attrName);
4262       assertFalse("missing HCD attribute value", value == null);
4263       assertFalse("HCD attribute value is incorrect",
4264         Bytes.compareTo(value, attrValue) != 0);
4265     }
4266   }
4267 
4268   @Test
4269   public void testGetClosestRowBefore() throws IOException, InterruptedException {
4270     final TableName tableAname = TableName.valueOf("testGetClosestRowBefore");
4271     final byte[] firstRow = Bytes.toBytes("row111");
4272     final byte[] secondRow = Bytes.toBytes("row222");
4273     final byte[] thirdRow = Bytes.toBytes("row333");
4274     final byte[] forthRow = Bytes.toBytes("row444");
4275     final byte[] beforeFirstRow = Bytes.toBytes("row");
4276     final byte[] beforeSecondRow = Bytes.toBytes("row22");
4277     final byte[] beforeThirdRow = Bytes.toBytes("row33");
4278     final byte[] beforeForthRow = Bytes.toBytes("row44");
4279 
4280     HTable table =
4281         TEST_UTIL.createTable(tableAname,
4282           new byte[][] { HConstants.CATALOG_FAMILY, Bytes.toBytes("info2") }, 1, 1024);
4283     // set block size to 64 to making 2 kvs into one block, bypassing the walkForwardInSingleRow
4284     // in Store.rowAtOrBeforeFromStoreFile
4285     String regionName = table.getRegionLocations().firstKey().getEncodedName();
4286     Region region =
4287         TEST_UTIL.getRSForFirstRegionInTable(tableAname).getFromOnlineRegions(regionName);
4288     Put put1 = new Put(firstRow);
4289     Put put2 = new Put(secondRow);
4290     Put put3 = new Put(thirdRow);
4291     Put put4 = new Put(forthRow);
4292     byte[] one = new byte[] { 1 };
4293     byte[] two = new byte[] { 2 };
4294     byte[] three = new byte[] { 3 };
4295     byte[] four = new byte[] { 4 };
4296 
4297     put1.add(HConstants.CATALOG_FAMILY, null, one);
4298     put2.add(HConstants.CATALOG_FAMILY, null, two);
4299     put3.add(HConstants.CATALOG_FAMILY, null, three);
4300     put4.add(HConstants.CATALOG_FAMILY, null, four);
4301     table.put(put1);
4302     table.put(put2);
4303     table.put(put3);
4304     table.put(put4);
4305     region.flush(true);
4306     Result result = null;
4307 
4308     // Test before first that null is returned
4309     result = table.getRowOrBefore(beforeFirstRow, HConstants.CATALOG_FAMILY);
4310     assertTrue(result == null);
4311 
4312     // Test at first that first is returned
4313     result = table.getRowOrBefore(firstRow, HConstants.CATALOG_FAMILY);
4314     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4315     assertTrue(Bytes.equals(result.getRow(), firstRow));
4316     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
4317 
4318     // Test in between first and second that first is returned
4319     result = table.getRowOrBefore(beforeSecondRow, HConstants.CATALOG_FAMILY);
4320     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4321     assertTrue(Bytes.equals(result.getRow(), firstRow));
4322     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
4323 
4324     // Test at second make sure second is returned
4325     result = table.getRowOrBefore(secondRow, HConstants.CATALOG_FAMILY);
4326     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4327     assertTrue(Bytes.equals(result.getRow(), secondRow));
4328     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));
4329 
4330     // Test in second and third, make sure second is returned
4331     result = table.getRowOrBefore(beforeThirdRow, HConstants.CATALOG_FAMILY);
4332     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4333     assertTrue(Bytes.equals(result.getRow(), secondRow));
4334     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));
4335 
4336     // Test at third make sure third is returned
4337     result = table.getRowOrBefore(thirdRow, HConstants.CATALOG_FAMILY);
4338     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4339     assertTrue(Bytes.equals(result.getRow(), thirdRow));
4340     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));
4341 
4342     // Test in third and forth, make sure third is returned
4343     result = table.getRowOrBefore(beforeForthRow, HConstants.CATALOG_FAMILY);
4344     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4345     assertTrue(Bytes.equals(result.getRow(), thirdRow));
4346     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));
4347 
4348     // Test at forth make sure forth is returned
4349     result = table.getRowOrBefore(forthRow, HConstants.CATALOG_FAMILY);
4350     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4351     assertTrue(Bytes.equals(result.getRow(), forthRow));
4352     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));
4353 
4354     // Test after forth make sure forth is returned
4355     result = table.getRowOrBefore(Bytes.add(forthRow, one), HConstants.CATALOG_FAMILY);
4356     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4357     assertTrue(Bytes.equals(result.getRow(), forthRow));
4358     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));
4359 
4360     table.close();
4361   }
4362 
4363   /**
4364    * For HBASE-2156
4365    * @throws Exception
4366    */
4367   @Test
4368   public void testScanVariableReuse() throws Exception {
4369     Scan scan = new Scan();
4370     scan.addFamily(FAMILY);
4371     scan.addColumn(FAMILY, ROW);
4372 
4373     assertTrue(scan.getFamilyMap().get(FAMILY).size() == 1);
4374 
4375     scan = new Scan();
4376     scan.addFamily(FAMILY);
4377 
4378     assertTrue(scan.getFamilyMap().get(FAMILY) == null);
4379     assertTrue(scan.getFamilyMap().containsKey(FAMILY));
4380   }
4381 
4382   @Test
4383   public void testMultiRowMutation() throws Exception {
4384     LOG.info("Starting testMultiRowMutation");
4385     final TableName TABLENAME = TableName.valueOf("testMultiRowMutation");
4386     final byte [] ROW1 = Bytes.toBytes("testRow1");
4387 
4388     Table t = TEST_UTIL.createTable(TABLENAME, FAMILY);
4389     Put p = new Put(ROW);
4390     p.add(FAMILY, QUALIFIER, VALUE);
4391     MutationProto m1 = ProtobufUtil.toMutation(MutationType.PUT, p);
4392 
4393     p = new Put(ROW1);
4394     p.add(FAMILY, QUALIFIER, VALUE);
4395     MutationProto m2 = ProtobufUtil.toMutation(MutationType.PUT, p);
4396 
4397     MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
4398     mrmBuilder.addMutationRequest(m1);
4399     mrmBuilder.addMutationRequest(m2);
4400     MutateRowsRequest mrm = mrmBuilder.build();
4401     CoprocessorRpcChannel channel = t.coprocessorService(ROW);
4402     MultiRowMutationService.BlockingInterface service =
4403        MultiRowMutationService.newBlockingStub(channel);
4404     service.mutateRows(null, mrm);
4405     Get g = new Get(ROW);
4406     Result r = t.get(g);
4407     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
4408     g = new Get(ROW1);
4409     r = t.get(g);
4410     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
4411   }
4412 
4413   @Test
4414   public void testRowMutation() throws Exception {
4415     LOG.info("Starting testRowMutation");
4416     final TableName TABLENAME = TableName.valueOf("testRowMutation");
4417     Table t = TEST_UTIL.createTable(TABLENAME, FAMILY);
4418     byte [][] QUALIFIERS = new byte [][] {
4419         Bytes.toBytes("a"), Bytes.toBytes("b")
4420     };
4421     RowMutations arm = new RowMutations(ROW);
4422     Put p = new Put(ROW);
4423     p.add(FAMILY, QUALIFIERS[0], VALUE);
4424     arm.add(p);
4425     t.mutateRow(arm);
4426 
4427     Get g = new Get(ROW);
4428     Result r = t.get(g);
4429     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0])));
4430 
4431     arm = new RowMutations(ROW);
4432     p = new Put(ROW);
4433     p.add(FAMILY, QUALIFIERS[1], VALUE);
4434     arm.add(p);
4435     Delete d = new Delete(ROW);
4436     d.deleteColumns(FAMILY, QUALIFIERS[0]);
4437     arm.add(d);
4438     // TODO: Trying mutateRow again.  The batch was failing with a one try only.
4439     t.mutateRow(arm);
4440     r = t.get(g);
4441     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[1])));
4442     assertNull(r.getValue(FAMILY, QUALIFIERS[0]));
4443 
4444     //Test that we get a region level exception
4445     try {
4446       arm = new RowMutations(ROW);
4447       p = new Put(ROW);
4448       p.add(new byte[]{'b', 'o', 'g', 'u', 's'}, QUALIFIERS[0], VALUE);
4449       arm.add(p);
4450       t.mutateRow(arm);
4451       fail("Expected NoSuchColumnFamilyException");
4452     } catch(NoSuchColumnFamilyException e) {
4453     }
4454   }
4455 
4456   @Test
4457   public void testAppend() throws Exception {
4458     LOG.info("Starting testAppend");
4459     final TableName TABLENAME = TableName.valueOf("testAppend");
4460     Table t = TEST_UTIL.createTable(TABLENAME, FAMILY);
4461     byte[] v1 = Bytes.toBytes("42");
4462     byte[] v2 = Bytes.toBytes("23");
4463     byte [][] QUALIFIERS = new byte [][] {
4464         Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("c")
4465     };
4466     Append a = new Append(ROW);
4467     a.add(FAMILY, QUALIFIERS[0], v1);
4468     a.add(FAMILY, QUALIFIERS[1], v2);
4469     a.setReturnResults(false);
4470     assertNullResult(t.append(a));
4471 
4472     a = new Append(ROW);
4473     a.add(FAMILY, QUALIFIERS[0], v2);
4474     a.add(FAMILY, QUALIFIERS[1], v1);
4475     a.add(FAMILY, QUALIFIERS[2], v2);
4476     Result r = t.append(a);
4477     assertEquals(0, Bytes.compareTo(Bytes.add(v1,v2), r.getValue(FAMILY, QUALIFIERS[0])));
4478     assertEquals(0, Bytes.compareTo(Bytes.add(v2,v1), r.getValue(FAMILY, QUALIFIERS[1])));
4479     // QUALIFIERS[2] previously not exist, verify both value and timestamp are correct
4480     assertEquals(0, Bytes.compareTo(v2, r.getValue(FAMILY, QUALIFIERS[2])));
4481     assertEquals(r.getColumnLatest(FAMILY, QUALIFIERS[0]).getTimestamp(),
4482         r.getColumnLatest(FAMILY, QUALIFIERS[2]).getTimestamp());
4483   }
4484 
4485   @Test
4486   public void testClientPoolRoundRobin() throws IOException {
4487     final TableName tableName = TableName.valueOf("testClientPoolRoundRobin");
4488 
4489     int poolSize = 3;
4490     int numVersions = poolSize * 2;
4491     Configuration conf = TEST_UTIL.getConfiguration();
4492     conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "round-robin");
4493     conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
4494 
4495     Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, conf, Integer.MAX_VALUE);
4496 
4497     final long ts = EnvironmentEdgeManager.currentTime();
4498     Get get = new Get(ROW);
4499     get.addColumn(FAMILY, QUALIFIER);
4500     get.setMaxVersions();
4501 
4502     for (int versions = 1; versions <= numVersions; versions++) {
4503       Put put = new Put(ROW);
4504       put.add(FAMILY, QUALIFIER, ts + versions, VALUE);
4505       table.put(put);
4506 
4507       Result result = table.get(get);
4508       NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
4509           .get(QUALIFIER);
4510 
4511       assertEquals("The number of versions of '" + FAMILY + ":" + QUALIFIER
4512           + " did not match " + versions, versions, navigableMap.size());
4513       for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4514         assertTrue("The value at time " + entry.getKey()
4515             + " did not match what was put",
4516             Bytes.equals(VALUE, entry.getValue()));
4517       }
4518     }
4519   }
4520 
4521   @Ignore ("Flakey: HBASE-8989") @Test
4522   public void testClientPoolThreadLocal() throws IOException {
4523     final TableName tableName = TableName.valueOf("testClientPoolThreadLocal");
4524 
4525     int poolSize = Integer.MAX_VALUE;
4526     int numVersions = 3;
4527     Configuration conf = TEST_UTIL.getConfiguration();
4528     conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "thread-local");
4529     conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
4530 
4531     final Table table = TEST_UTIL.createTable(tableName,
4532         new byte[][] { FAMILY }, conf, 3);
4533 
4534     final long ts = EnvironmentEdgeManager.currentTime();
4535     final Get get = new Get(ROW);
4536     get.addColumn(FAMILY, QUALIFIER);
4537     get.setMaxVersions();
4538 
4539     for (int versions = 1; versions <= numVersions; versions++) {
4540       Put put = new Put(ROW);
4541       put.add(FAMILY, QUALIFIER, ts + versions, VALUE);
4542       table.put(put);
4543 
4544       Result result = table.get(get);
4545       NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
4546           .get(QUALIFIER);
4547 
4548       assertEquals("The number of versions of '" + FAMILY + ":" + QUALIFIER + " did not match " +
4549         versions + "; " + put.toString() + ", " + get.toString(), versions, navigableMap.size());
4550       for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4551         assertTrue("The value at time " + entry.getKey()
4552             + " did not match what was put",
4553             Bytes.equals(VALUE, entry.getValue()));
4554       }
4555     }
4556 
4557     final Object waitLock = new Object();
4558     ExecutorService executorService = Executors.newFixedThreadPool(numVersions);
4559     final AtomicReference<AssertionError> error = new AtomicReference<AssertionError>(null);
4560     for (int versions = numVersions; versions < numVersions * 2; versions++) {
4561       final int versionsCopy = versions;
4562       executorService.submit(new Callable<Void>() {
4563         @Override
4564         public Void call() {
4565           try {
4566             Put put = new Put(ROW);
4567             put.add(FAMILY, QUALIFIER, ts + versionsCopy, VALUE);
4568             table.put(put);
4569 
4570             Result result = table.get(get);
4571             NavigableMap<Long, byte[]> navigableMap = result.getMap()
4572                 .get(FAMILY).get(QUALIFIER);
4573 
4574             assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":"
4575                 + Bytes.toString(QUALIFIER) + " did not match " + versionsCopy, versionsCopy,
4576                 navigableMap.size());
4577             for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4578               assertTrue("The value at time " + entry.getKey()
4579                   + " did not match what was put",
4580                   Bytes.equals(VALUE, entry.getValue()));
4581             }
4582             synchronized (waitLock) {
4583               waitLock.wait();
4584             }
4585           } catch (Exception e) {
4586           } catch (AssertionError e) {
4587             // the error happens in a thread, it won't fail the test,
4588             // need to pass it to the caller for proper handling.
4589             error.set(e);
4590             LOG.error(e);
4591           }
4592 
4593           return null;
4594         }
4595       });
4596     }
4597     synchronized (waitLock) {
4598       waitLock.notifyAll();
4599     }
4600     executorService.shutdownNow();
4601     assertNull(error.get());
4602   }
4603 
4604   @Test
4605   public void testCheckAndPut() throws IOException {
4606     final byte [] anotherrow = Bytes.toBytes("anotherrow");
4607     final byte [] value2 = Bytes.toBytes("abcd");
4608 
4609     Table table = TEST_UTIL.createTable(TableName.valueOf("testCheckAndPut"), FAMILY);
4610     Put put1 = new Put(ROW);
4611     put1.add(FAMILY, QUALIFIER, VALUE);
4612 
4613     // row doesn't exist, so using non-null value should be considered "not match".
4614     boolean ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE, put1);
4615     assertEquals(ok, false);
4616 
4617     // row doesn't exist, so using "null" to check for existence should be considered "match".
4618     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1);
4619     assertEquals(ok, true);
4620 
4621     // row now exists, so using "null" to check for existence should be considered "not match".
4622     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1);
4623     assertEquals(ok, false);
4624 
4625     Put put2 = new Put(ROW);
4626     put2.add(FAMILY, QUALIFIER, value2);
4627 
4628     // row now exists, use the matching value to check
4629     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE, put2);
4630     assertEquals(ok, true);
4631 
4632     Put put3 = new Put(anotherrow);
4633     put3.add(FAMILY, QUALIFIER, VALUE);
4634 
4635     // try to do CheckAndPut on different rows
4636     try {
4637         ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, value2, put3);
4638         fail("trying to check and modify different rows should have failed.");
4639     } catch(Exception e) {}
4640 
4641   }
4642 
4643   @Test
4644   public void testCheckAndPutWithCompareOp() throws IOException {
4645     final byte [] value1 = Bytes.toBytes("aaaa");
4646     final byte [] value2 = Bytes.toBytes("bbbb");
4647     final byte [] value3 = Bytes.toBytes("cccc");
4648     final byte [] value4 = Bytes.toBytes("dddd");
4649 
4650     Table table = TEST_UTIL.createTable(TableName.valueOf("testCheckAndPutWithCompareOp"), FAMILY);
4651 
4652     Put put2 = new Put(ROW);
4653     put2.add(FAMILY, QUALIFIER, value2);
4654 
4655     Put put3 = new Put(ROW);
4656     put3.add(FAMILY, QUALIFIER, value3);
4657 
4658     // row doesn't exist, so using "null" to check for existence should be considered "match".
4659     boolean ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put2);
4660     assertEquals(ok, true);
4661 
4662     // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
4663     // turns out "match"
4664     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value1, put2);
4665     assertEquals(ok, false);
4666     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value1, put2);
4667     assertEquals(ok, false);
4668     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value1, put2);
4669     assertEquals(ok, false);
4670     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value1, put2);
4671     assertEquals(ok, true);
4672     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value1, put2);
4673     assertEquals(ok, true);
4674     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value1, put3);
4675     assertEquals(ok, true);
4676 
4677     // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
4678     // turns out "match"
4679     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value4, put3);
4680     assertEquals(ok, false);
4681     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value4, put3);
4682     assertEquals(ok, false);
4683     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value4, put3);
4684     assertEquals(ok, false);
4685     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value4, put3);
4686     assertEquals(ok, true);
4687     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value4, put3);
4688     assertEquals(ok, true);
4689     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value4, put2);
4690     assertEquals(ok, true);
4691 
4692     // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
4693     // turns out "match"
4694     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value2, put2);
4695     assertEquals(ok, false);
4696     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value2, put2);
4697     assertEquals(ok, false);
4698     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value2, put2);
4699     assertEquals(ok, false);
4700     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value2, put2);
4701     assertEquals(ok, true);
4702     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value2, put2);
4703     assertEquals(ok, true);
4704     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value2, put3);
4705     assertEquals(ok, true);
4706   }
4707 
4708   @Test
4709   public void testCheckAndDeleteWithCompareOp() throws IOException {
4710     final byte [] value1 = Bytes.toBytes("aaaa");
4711     final byte [] value2 = Bytes.toBytes("bbbb");
4712     final byte [] value3 = Bytes.toBytes("cccc");
4713     final byte [] value4 = Bytes.toBytes("dddd");
4714 
4715     Table table = TEST_UTIL.createTable(TableName.valueOf("testCheckAndDeleteWithCompareOp"),
4716         FAMILY);
4717 
4718     Put put2 = new Put(ROW);
4719     put2.add(FAMILY, QUALIFIER, value2);
4720     table.put(put2);
4721 
4722     Put put3 = new Put(ROW);
4723     put3.add(FAMILY, QUALIFIER, value3);
4724 
4725     Delete delete = new Delete(ROW);
4726     delete.deleteColumns(FAMILY, QUALIFIER);
4727 
4728     // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
4729     // turns out "match"
4730     boolean ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value1, delete);
4731     assertEquals(ok, false);
4732     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value1, delete);
4733     assertEquals(ok, false);
4734     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value1, delete);
4735     assertEquals(ok, false);
4736     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value1, delete);
4737     assertEquals(ok, true);
4738     table.put(put2);
4739     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value1, delete);
4740     assertEquals(ok, true);
4741     table.put(put2);
4742     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value1, delete);
4743     assertEquals(ok, true);
4744 
4745     // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
4746     // turns out "match"
4747     table.put(put3);
4748     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value4, delete);
4749     assertEquals(ok, false);
4750     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value4, delete);
4751     assertEquals(ok, false);
4752     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value4, delete);
4753     assertEquals(ok, false);
4754     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value4, delete);
4755     assertEquals(ok, true);
4756     table.put(put3);
4757     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value4, delete);
4758     assertEquals(ok, true);
4759     table.put(put3);
4760     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value4, delete);
4761     assertEquals(ok, true);
4762 
4763     // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
4764     // turns out "match"
4765     table.put(put2);
4766     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value2, delete);
4767     assertEquals(ok, false);
4768     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value2, delete);
4769     assertEquals(ok, false);
4770     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value2, delete);
4771     assertEquals(ok, false);
4772     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value2, delete);
4773     assertEquals(ok, true);
4774     table.put(put2);
4775     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value2, delete);
4776     assertEquals(ok, true);
4777     table.put(put2);
4778     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value2, delete);
4779     assertEquals(ok, true);
4780   }
4781 
4782   /**
4783   * Test ScanMetrics
4784   * @throws Exception
4785   */
4786   @Test
4787   @SuppressWarnings ("unused")
4788   public void testScanMetrics() throws Exception {
4789     TableName TABLENAME = TableName.valueOf("testScanMetrics");
4790 
4791     Configuration conf = TEST_UTIL.getConfiguration();
4792 
4793     // Set up test table:
4794     // Create table:
4795     HTable ht = TEST_UTIL.createMultiRegionTable(TABLENAME, FAMILY);
4796     int numOfRegions = -1;
4797     try (RegionLocator r = ht.getRegionLocator()) {
4798       numOfRegions = r.getStartKeys().length;
4799     }
4800     // Create 3 rows in the table, with rowkeys starting with "zzz*" so that
4801     // scan are forced to hit all the regions.
4802     Put put1 = new Put(Bytes.toBytes("zzz1"));
4803     put1.add(FAMILY, QUALIFIER, VALUE);
4804     Put put2 = new Put(Bytes.toBytes("zzz2"));
4805     put2.add(FAMILY, QUALIFIER, VALUE);
4806     Put put3 = new Put(Bytes.toBytes("zzz3"));
4807     put3.add(FAMILY, QUALIFIER, VALUE);
4808     ht.put(Arrays.asList(put1, put2, put3));
4809 
4810     Scan scan1 = new Scan();
4811     int numRecords = 0;
4812     ResultScanner scanner = ht.getScanner(scan1);
4813     for(Result result : scanner) {
4814       numRecords++;
4815     }
4816     scanner.close();
4817     LOG.info("test data has " + numRecords + " records.");
4818 
4819     // by default, scan metrics collection is turned off
4820     assertEquals(null, scan1.getScanMetrics());
4821 
4822     // turn on scan metrics
4823     Scan scan2 = new Scan();
4824     scan2.setScanMetricsEnabled(true);
4825     scan2.setCaching(numRecords+1);
4826     scanner = ht.getScanner(scan2);
4827     for (Result result : scanner.next(numRecords - 1)) {
4828     }
4829     scanner.close();
4830     // closing the scanner will set the metrics.
4831     assertNotNull(scan2.getScanMetrics());
4832 
4833     // set caching to 1, because metrics are collected in each roundtrip only
4834     scan2 = new Scan();
4835     scan2.setScanMetricsEnabled(true);
4836     scan2.setCaching(1);
4837     scanner = ht.getScanner(scan2);
4838     // per HBASE-5717, this should still collect even if you don't run all the way to
4839     // the end of the scanner. So this is asking for 2 of the 3 rows we inserted.
4840     for (Result result : scanner.next(numRecords - 1)) {
4841     }
4842     scanner.close();
4843 
4844     ScanMetrics scanMetrics = scan2.getScanMetrics();
4845     assertEquals("Did not access all the regions in the table", numOfRegions,
4846         scanMetrics.countOfRegions.get());
4847 
4848     // check byte counters
4849     scan2 = new Scan();
4850     scan2.setScanMetricsEnabled(true);
4851     scan2.setCaching(1);
4852     scanner = ht.getScanner(scan2);
4853     int numBytes = 0;
4854     for (Result result : scanner.next(1)) {
4855       for (Cell cell: result.listCells()) {
4856         numBytes += CellUtil.estimatedSerializedSizeOf(cell);
4857       }
4858     }
4859     scanner.close();
4860     scanMetrics = scan2.getScanMetrics();
4861     assertEquals("Did not count the result bytes", numBytes,
4862       scanMetrics.countOfBytesInResults.get());
4863 
4864     // check byte counters on a small scan
4865     scan2 = new Scan();
4866     scan2.setScanMetricsEnabled(true);
4867     scan2.setCaching(1);
4868     scan2.setSmall(true);
4869     scanner = ht.getScanner(scan2);
4870     numBytes = 0;
4871     for (Result result : scanner.next(1)) {
4872       for (Cell cell: result.listCells()) {
4873         numBytes += CellUtil.estimatedSerializedSizeOf(cell);
4874       }
4875     }
4876     scanner.close();
4877     scanMetrics = scan2.getScanMetrics();
4878     assertEquals("Did not count the result bytes", numBytes,
4879       scanMetrics.countOfBytesInResults.get());
4880 
4881     // now, test that the metrics are still collected even if you don't call close, but do
4882     // run past the end of all the records
4883     /** There seems to be a timing issue here.  Comment out for now. Fix when time.
4884     Scan scanWithoutClose = new Scan();
4885     scanWithoutClose.setCaching(1);
4886     scanWithoutClose.setScanMetricsEnabled(true);
4887     ResultScanner scannerWithoutClose = ht.getScanner(scanWithoutClose);
4888     for (Result result : scannerWithoutClose.next(numRecords + 1)) {
4889     }
4890     ScanMetrics scanMetricsWithoutClose = getScanMetrics(scanWithoutClose);
4891     assertEquals("Did not access all the regions in the table", numOfRegions,
4892         scanMetricsWithoutClose.countOfRegions.get());
4893     */
4894 
4895     // finally, test that the metrics are collected correctly if you both run past all the records,
4896     // AND close the scanner
4897     Scan scanWithClose = new Scan();
4898     // make sure we can set caching up to the number of a scanned values
4899     scanWithClose.setCaching(numRecords);
4900     scanWithClose.setScanMetricsEnabled(true);
4901     ResultScanner scannerWithClose = ht.getScanner(scanWithClose);
4902     for (Result result : scannerWithClose.next(numRecords + 1)) {
4903     }
4904     scannerWithClose.close();
4905     ScanMetrics scanMetricsWithClose = getScanMetrics(scanWithClose);
4906     assertEquals("Did not access all the regions in the table", numOfRegions,
4907         scanMetricsWithClose.countOfRegions.get());
4908   }
4909 
4910   private ScanMetrics getScanMetrics(Scan scan) throws Exception {
4911     byte[] serializedMetrics = scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA);
4912     assertTrue("Serialized metrics were not found.", serializedMetrics != null);
4913 
4914     ScanMetrics scanMetrics = ProtobufUtil.toScanMetrics(serializedMetrics);
4915 
4916     return scanMetrics;
4917   }
4918 
4919   /**
4920    * Tests that cache on write works all the way up from the client-side.
4921    *
4922    * Performs inserts, flushes, and compactions, verifying changes in the block
4923    * cache along the way.
4924    *
4925    * @throws Exception
4926    */
4927   @Test
4928   public void testCacheOnWriteEvictOnClose() throws Exception {
4929     TableName tableName = TableName.valueOf("testCOWEOCfromClient");
4930     byte [] data = Bytes.toBytes("data");
4931     HTable table = TEST_UTIL.createTable(tableName, FAMILY);
4932     // get the block cache and region
4933     String regionName = table.getRegionLocations().firstKey().getEncodedName();
4934     Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName)
4935       .getFromOnlineRegions(regionName);
4936     Store store = region.getStores().iterator().next();
4937     CacheConfig cacheConf = store.getCacheConfig();
4938     cacheConf.setCacheDataOnWrite(true);
4939     cacheConf.setEvictOnClose(true);
4940     BlockCache cache = cacheConf.getBlockCache();
4941 
4942     // establish baseline stats
4943     long startBlockCount = cache.getBlockCount();
4944     long startBlockHits = cache.getStats().getHitCount();
4945     long startBlockMiss = cache.getStats().getMissCount();
4946 
4947     // wait till baseline is stable, (minimal 500 ms)
4948     for (int i = 0; i < 5; i++) {
4949       Thread.sleep(100);
4950       if (startBlockCount != cache.getBlockCount()
4951           || startBlockHits != cache.getStats().getHitCount()
4952           || startBlockMiss != cache.getStats().getMissCount()) {
4953         startBlockCount = cache.getBlockCount();
4954         startBlockHits = cache.getStats().getHitCount();
4955         startBlockMiss = cache.getStats().getMissCount();
4956         i = -1;
4957       }
4958     }
4959 
4960     // insert data
4961     Put put = new Put(ROW);
4962     put.add(FAMILY, QUALIFIER, data);
4963     table.put(put);
4964     assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
4965     // data was in memstore so don't expect any changes
4966     assertEquals(startBlockCount, cache.getBlockCount());
4967     assertEquals(startBlockHits, cache.getStats().getHitCount());
4968     assertEquals(startBlockMiss, cache.getStats().getMissCount());
4969     // flush the data
4970     System.out.println("Flushing cache");
4971     region.flush(true);
4972     // expect one more block in cache, no change in hits/misses
4973     long expectedBlockCount = startBlockCount + 1;
4974     long expectedBlockHits = startBlockHits;
4975     long expectedBlockMiss = startBlockMiss;
4976     assertEquals(expectedBlockCount, cache.getBlockCount());
4977     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
4978     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
4979     // read the data and expect same blocks, one new hit, no misses
4980     assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
4981     assertEquals(expectedBlockCount, cache.getBlockCount());
4982     assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
4983     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
4984     // insert a second column, read the row, no new blocks, one new hit
4985     byte [] QUALIFIER2 = Bytes.add(QUALIFIER, QUALIFIER);
4986     byte [] data2 = Bytes.add(data, data);
4987     put = new Put(ROW);
4988     put.add(FAMILY, QUALIFIER2, data2);
4989     table.put(put);
4990     Result r = table.get(new Get(ROW));
4991     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
4992     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
4993     assertEquals(expectedBlockCount, cache.getBlockCount());
4994     assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
4995     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
4996     // flush, one new block
4997     System.out.println("Flushing cache");
4998     region.flush(true);
4999     assertEquals(++expectedBlockCount, cache.getBlockCount());
5000     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5001     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5002     // compact, net minus two blocks, two hits, no misses
5003     System.out.println("Compacting");
5004     assertEquals(2, store.getStorefilesCount());
5005     store.triggerMajorCompaction();
5006     region.compact(true);
5007     waitForStoreFileCount(store, 1, 10000); // wait 10 seconds max
5008     assertEquals(1, store.getStorefilesCount());
5009     expectedBlockCount -= 2; // evicted two blocks, cached none
5010     assertEquals(expectedBlockCount, cache.getBlockCount());
5011     expectedBlockHits += 2;
5012     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5013     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5014     // read the row, this should be a cache miss because we don't cache data
5015     // blocks on compaction
5016     r = table.get(new Get(ROW));
5017     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
5018     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
5019     expectedBlockCount += 1; // cached one data block
5020     assertEquals(expectedBlockCount, cache.getBlockCount());
5021     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5022     assertEquals(++expectedBlockMiss, cache.getStats().getMissCount());
5023   }
5024 
5025   private void waitForStoreFileCount(Store store, int count, int timeout)
5026   throws InterruptedException {
5027     long start = System.currentTimeMillis();
5028     while (start + timeout > System.currentTimeMillis() &&
5029         store.getStorefilesCount() != count) {
5030       Thread.sleep(100);
5031     }
5032     System.out.println("start=" + start + ", now=" +
5033         System.currentTimeMillis() + ", cur=" + store.getStorefilesCount());
5034     assertEquals(count, store.getStorefilesCount());
5035   }
5036 
5037   @Test
5038   /**
5039    * Tests the non cached version of getRegionLocator by moving a region.
5040    */
5041   public void testNonCachedGetRegionLocation() throws Exception {
5042     // Test Initialization.
5043     TableName TABLE = TableName.valueOf("testNonCachedGetRegionLocation");
5044     byte [] family1 = Bytes.toBytes("f1");
5045     byte [] family2 = Bytes.toBytes("f2");
5046     try (HTable table = TEST_UTIL.createTable(TABLE, new byte[][] {family1, family2}, 10);
5047         Admin admin = new HBaseAdmin(TEST_UTIL.getConfiguration())) {
5048       Map <HRegionInfo, ServerName> regionsMap = table.getRegionLocations();
5049       assertEquals(1, regionsMap.size());
5050       HRegionInfo regionInfo = regionsMap.keySet().iterator().next();
5051       ServerName addrBefore = regionsMap.get(regionInfo);
5052       // Verify region location before move.
5053       HRegionLocation addrCache = table.getRegionLocation(regionInfo.getStartKey(), false);
5054       HRegionLocation addrNoCache = table.getRegionLocation(regionInfo.getStartKey(),  true);
5055 
5056       assertEquals(addrBefore.getPort(), addrCache.getPort());
5057       assertEquals(addrBefore.getPort(), addrNoCache.getPort());
5058 
5059       ServerName addrAfter = null;
5060       // Now move the region to a different server.
5061       for (int i = 0; i < SLAVES; i++) {
5062         HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(i);
5063         ServerName addr = regionServer.getServerName();
5064         if (addr.getPort() != addrBefore.getPort()) {
5065           admin.move(regionInfo.getEncodedNameAsBytes(),
5066               Bytes.toBytes(addr.toString()));
5067           // Wait for the region to move.
5068           Thread.sleep(5000);
5069           addrAfter = addr;
5070           break;
5071         }
5072       }
5073 
5074       // Verify the region was moved.
5075       addrCache = table.getRegionLocation(regionInfo.getStartKey(), false);
5076       addrNoCache = table.getRegionLocation(regionInfo.getStartKey(), true);
5077       assertNotNull(addrAfter);
5078       assertTrue(addrAfter.getPort() != addrCache.getPort());
5079       assertEquals(addrAfter.getPort(), addrNoCache.getPort());
5080     }
5081   }
5082 
5083   @Test
5084   /**
5085    * Tests getRegionsInRange by creating some regions over which a range of
5086    * keys spans; then changing the key range.
5087    */
5088   public void testGetRegionsInRange() throws Exception {
5089     // Test Initialization.
5090     byte [] startKey = Bytes.toBytes("ddc");
5091     byte [] endKey = Bytes.toBytes("mmm");
5092     TableName TABLE = TableName.valueOf("testGetRegionsInRange");
5093     HTable table = TEST_UTIL.createMultiRegionTable(TABLE, new byte[][] { FAMILY }, 10);
5094     int numOfRegions = -1;
5095     try (RegionLocator r = table.getRegionLocator()) {
5096       numOfRegions = r.getStartKeys().length;
5097     }
5098     assertEquals(26, numOfRegions);
5099 
5100     // Get the regions in this range
5101     List<HRegionLocation> regionsList = table.getRegionsInRange(startKey,
5102       endKey);
5103     assertEquals(10, regionsList.size());
5104 
5105     // Change the start key
5106     startKey = Bytes.toBytes("fff");
5107     regionsList = table.getRegionsInRange(startKey, endKey);
5108     assertEquals(7, regionsList.size());
5109 
5110     // Change the end key
5111     endKey = Bytes.toBytes("nnn");
5112     regionsList = table.getRegionsInRange(startKey, endKey);
5113     assertEquals(8, regionsList.size());
5114 
5115     // Empty start key
5116     regionsList = table.getRegionsInRange(HConstants.EMPTY_START_ROW, endKey);
5117     assertEquals(13, regionsList.size());
5118 
5119     // Empty end key
5120     regionsList = table.getRegionsInRange(startKey, HConstants.EMPTY_END_ROW);
5121     assertEquals(21, regionsList.size());
5122 
5123     // Both start and end keys empty
5124     regionsList = table.getRegionsInRange(HConstants.EMPTY_START_ROW,
5125       HConstants.EMPTY_END_ROW);
5126     assertEquals(26, regionsList.size());
5127 
5128     // Change the end key to somewhere in the last block
5129     endKey = Bytes.toBytes("zzz1");
5130     regionsList = table.getRegionsInRange(startKey, endKey);
5131     assertEquals(21, regionsList.size());
5132 
5133     // Change the start key to somewhere in the first block
5134     startKey = Bytes.toBytes("aac");
5135     regionsList = table.getRegionsInRange(startKey, endKey);
5136     assertEquals(26, regionsList.size());
5137 
5138     // Make start and end key the same
5139     startKey = endKey = Bytes.toBytes("ccc");
5140     regionsList = table.getRegionsInRange(startKey, endKey);
5141     assertEquals(1, regionsList.size());
5142   }
5143 
5144   @Test
5145   public void testJira6912() throws Exception {
5146     TableName TABLE = TableName.valueOf("testJira6912");
5147     Table foo = TEST_UTIL.createTable(TABLE, new byte[][] {FAMILY}, 10);
5148 
5149     List<Put> puts = new ArrayList<Put>();
5150     for (int i=0;i !=100; i++){
5151       Put put = new Put(Bytes.toBytes(i));
5152       put.add(FAMILY, FAMILY, Bytes.toBytes(i));
5153       puts.add(put);
5154     }
5155     foo.put(puts);
5156     // If i comment this out it works
5157     TEST_UTIL.flush();
5158 
5159     Scan scan = new Scan();
5160     scan.setStartRow(Bytes.toBytes(1));
5161     scan.setStopRow(Bytes.toBytes(3));
5162     scan.addColumn(FAMILY, FAMILY);
5163     scan.setFilter(new RowFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator(Bytes.toBytes(1))));
5164 
5165     ResultScanner scanner = foo.getScanner(scan);
5166     Result[] bar = scanner.next(100);
5167     assertEquals(1, bar.length);
5168   }
5169 
5170   @Test
5171   public void testScan_NullQualifier() throws IOException {
5172     Table table = TEST_UTIL.createTable(TableName.valueOf("testScan_NullQualifier"), FAMILY);
5173     Put put = new Put(ROW);
5174     put.add(FAMILY, QUALIFIER, VALUE);
5175     table.put(put);
5176 
5177     put = new Put(ROW);
5178     put.add(FAMILY, null, VALUE);
5179     table.put(put);
5180     LOG.info("Row put");
5181 
5182     Scan scan = new Scan();
5183     scan.addColumn(FAMILY, null);
5184 
5185     ResultScanner scanner = table.getScanner(scan);
5186     Result[] bar = scanner.next(100);
5187     assertEquals(1, bar.length);
5188     assertEquals(1, bar[0].size());
5189 
5190     scan = new Scan();
5191     scan.addFamily(FAMILY);
5192 
5193     scanner = table.getScanner(scan);
5194     bar = scanner.next(100);
5195     assertEquals(1, bar.length);
5196     assertEquals(2, bar[0].size());
5197   }
5198 
5199   @Test
5200   public void testNegativeTimestamp() throws IOException {
5201     Table table = TEST_UTIL.createTable(TableName.valueOf("testNegativeTimestamp"), FAMILY);
5202 
5203     try {
5204       Put put = new Put(ROW, -1);
5205       put.add(FAMILY, QUALIFIER, VALUE);
5206       table.put(put);
5207       fail("Negative timestamps should not have been allowed");
5208     } catch (IllegalArgumentException ex) {
5209       assertTrue(ex.getMessage().contains("negative"));
5210     }
5211 
5212     try {
5213       Put put = new Put(ROW);
5214       put.add(FAMILY, QUALIFIER, -1, VALUE);
5215       table.put(put);
5216       fail("Negative timestamps should not have been allowed");
5217     } catch (IllegalArgumentException ex) {
5218       assertTrue(ex.getMessage().contains("negative"));
5219     }
5220 
5221     try {
5222       Delete delete = new Delete(ROW, -1);
5223       table.delete(delete);
5224       fail("Negative timestamps should not have been allowed");
5225     } catch (IllegalArgumentException ex) {
5226       assertTrue(ex.getMessage().contains("negative"));
5227     }
5228 
5229     try {
5230       Delete delete = new Delete(ROW);
5231       delete.deleteFamily(FAMILY, -1);
5232       table.delete(delete);
5233       fail("Negative timestamps should not have been allowed");
5234     } catch (IllegalArgumentException ex) {
5235       assertTrue(ex.getMessage().contains("negative"));
5236     }
5237 
5238     try {
5239       Scan scan = new Scan();
5240       scan.setTimeRange(-1, 1);
5241       table.getScanner(scan);
5242       fail("Negative timestamps should not have been allowed");
5243     } catch (IllegalArgumentException ex) {
5244       assertTrue(ex.getMessage().contains("negative"));
5245     }
5246 
5247     // KeyValue should allow negative timestamps for backwards compat. Otherwise, if the user
5248     // already has negative timestamps in cluster data, HBase won't be able to handle that
5249     try {
5250       new KeyValue(Bytes.toBytes(42), Bytes.toBytes(42), Bytes.toBytes(42), -1, Bytes.toBytes(42));
5251     } catch (IllegalArgumentException ex) {
5252       fail("KeyValue SHOULD allow negative timestamps");
5253     }
5254 
5255     table.close();
5256   }
5257 
5258   @Test
5259   public void testIllegalTableDescriptor() throws Exception {
5260     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testIllegalTableDescriptor"));
5261     HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
5262 
5263     // create table with 0 families
5264     checkTableIsIllegal(htd);
5265     htd.addFamily(hcd);
5266     checkTableIsLegal(htd);
5267 
5268     htd.setMaxFileSize(1024); // 1K
5269     checkTableIsIllegal(htd);
5270     htd.setMaxFileSize(0);
5271     checkTableIsIllegal(htd);
5272     htd.setMaxFileSize(1024 * 1024 * 1024); // 1G
5273     checkTableIsLegal(htd);
5274 
5275     htd.setMemStoreFlushSize(1024);
5276     checkTableIsIllegal(htd);
5277     htd.setMemStoreFlushSize(0);
5278     checkTableIsIllegal(htd);
5279     htd.setMemStoreFlushSize(128 * 1024 * 1024); // 128M
5280     checkTableIsLegal(htd);
5281 
5282     htd.setRegionSplitPolicyClassName("nonexisting.foo.class");
5283     checkTableIsIllegal(htd);
5284     htd.setRegionSplitPolicyClassName(null);
5285     checkTableIsLegal(htd);
5286 
5287     hcd.setBlocksize(0);
5288     checkTableIsIllegal(htd);
5289     hcd.setBlocksize(1024 * 1024 * 128); // 128M
5290     checkTableIsIllegal(htd);
5291     hcd.setBlocksize(1024);
5292     checkTableIsLegal(htd);
5293 
5294     hcd.setTimeToLive(0);
5295     checkTableIsIllegal(htd);
5296     hcd.setTimeToLive(-1);
5297     checkTableIsIllegal(htd);
5298     hcd.setTimeToLive(1);
5299     checkTableIsLegal(htd);
5300 
5301     hcd.setMinVersions(-1);
5302     checkTableIsIllegal(htd);
5303     hcd.setMinVersions(3);
5304     try {
5305       hcd.setMaxVersions(2);
5306       fail();
5307     } catch (IllegalArgumentException ex) {
5308       // expected
5309       hcd.setMaxVersions(10);
5310     }
5311     checkTableIsLegal(htd);
5312 
5313     // HBASE-13776 Setting illegal versions for HColumnDescriptor
5314     //  does not throw IllegalArgumentException
5315     // finally, minVersions must be less than or equal to maxVersions
5316     hcd.setMaxVersions(4);
5317     hcd.setMinVersions(5);
5318     checkTableIsIllegal(htd);
5319     hcd.setMinVersions(3);
5320 
5321     hcd.setScope(-1);
5322     checkTableIsIllegal(htd);
5323     hcd.setScope(0);
5324     checkTableIsLegal(htd);
5325 
5326     // check the conf settings to disable sanity checks
5327     htd.setMemStoreFlushSize(0);
5328 
5329     // Check that logs warn on invalid table but allow it.
5330     ListAppender listAppender = new ListAppender();
5331     Logger log = Logger.getLogger(HMaster.class);
5332     log.addAppender(listAppender);
5333     log.setLevel(Level.WARN);
5334 
5335     htd.setConfiguration("hbase.table.sanity.checks", Boolean.FALSE.toString());
5336     checkTableIsLegal(htd);
5337 
5338     assertFalse(listAppender.getMessages().isEmpty());
5339     assertTrue(listAppender.getMessages().get(0).startsWith("MEMSTORE_FLUSHSIZE for table "
5340         + "descriptor or \"hbase.hregion.memstore.flush.size\" (0) is too small, which might "
5341         + "cause very frequent flushing."));
5342 
5343     log.removeAppender(listAppender);
5344   }
5345 
5346   private static class ListAppender extends AppenderSkeleton {
5347     private final List<String> messages = new ArrayList<String>();
5348 
5349     @Override
5350     protected void append(LoggingEvent event) {
5351       messages.add(event.getMessage().toString());
5352     }
5353 
5354     @Override
5355     public void close() {
5356     }
5357 
5358     @Override
5359     public boolean requiresLayout() {
5360       return false;
5361     }
5362 
5363     public List<String> getMessages() {
5364       return messages;
5365     }
5366   }
5367 
5368   private void checkTableIsLegal(HTableDescriptor htd) throws IOException {
5369     Admin admin = TEST_UTIL.getHBaseAdmin();
5370     admin.createTable(htd);
5371     assertTrue(admin.tableExists(htd.getTableName()));
5372     admin.disableTable(htd.getTableName());
5373     admin.deleteTable(htd.getTableName());
5374   }
5375 
5376   private void checkTableIsIllegal(HTableDescriptor htd) throws IOException {
5377     Admin admin = TEST_UTIL.getHBaseAdmin();
5378     try {
5379       admin.createTable(htd);
5380       fail();
5381     } catch(Exception ex) {
5382       // should throw ex
5383     }
5384     assertFalse(admin.tableExists(htd.getTableName()));
5385   }
5386 
5387   @Test
5388   public void testRawScanRespectsVersions() throws Exception {
5389     TableName TABLE = TableName.valueOf("testRawScan");
5390     Table table = TEST_UTIL.createTable(TABLE, FAMILY);
5391     byte[] row = Bytes.toBytes("row");
5392 
5393     // put the same row 4 times, with different values
5394     Put p = new Put(row);
5395     p.add(FAMILY, QUALIFIER, 10, VALUE);
5396     table.put(p);
5397     p = new Put(row);
5398     p.add(FAMILY, QUALIFIER, 11, ArrayUtils.add(VALUE, (byte) 2));
5399     table.put(p);
5400 
5401     p = new Put(row);
5402     p.add(FAMILY, QUALIFIER, 12, ArrayUtils.add(VALUE, (byte) 3));
5403     table.put(p);
5404 
5405     p = new Put(row);
5406     p.add(FAMILY, QUALIFIER, 13, ArrayUtils.add(VALUE, (byte) 4));
5407     table.put(p);
5408 
5409     int versions = 4;
5410     Scan s = new Scan(row);
5411     // get all the possible versions
5412     s.setMaxVersions();
5413     s.setRaw(true);
5414 
5415     ResultScanner scanner = table.getScanner(s);
5416     int count = 0;
5417     for (Result r : scanner) {
5418       assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
5419       count++;
5420     }
5421     assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
5422       count);
5423     scanner.close();
5424 
5425     // then if we decrease the number of versions, but keep the scan raw, we should see exactly that
5426     // number of versions
5427     versions = 2;
5428     s.setMaxVersions(versions);
5429     scanner = table.getScanner(s);
5430     count = 0;
5431     for (Result r : scanner) {
5432       assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
5433       count++;
5434     }
5435     assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
5436       count);
5437     scanner.close();
5438 
5439     // finally, if we turn off raw scanning, but max out the number of versions, we should go back
5440     // to seeing just three
5441     versions = 3;
5442     s.setMaxVersions(versions);
5443     scanner = table.getScanner(s);
5444     count = 0;
5445     for (Result r : scanner) {
5446       assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
5447       count++;
5448     }
5449     assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
5450       count);
5451     scanner.close();
5452 
5453     table.close();
5454     TEST_UTIL.deleteTable(TABLE);
5455   }
5456 
5457   @Test
5458   public void testSmallScan() throws Exception {
5459     // Test Initialization.
5460     TableName TABLE = TableName.valueOf("testSmallScan");
5461     Table table = TEST_UTIL.createTable(TABLE, FAMILY);
5462 
5463     // Insert one row each region
5464     int insertNum = 10;
5465     for (int i = 0; i < 10; i++) {
5466       Put put = new Put(Bytes.toBytes("row" + String.format("%03d", i)));
5467       put.add(FAMILY, QUALIFIER, VALUE);
5468       table.put(put);
5469     }
5470 
5471     // nomal scan
5472     ResultScanner scanner = table.getScanner(new Scan());
5473     int count = 0;
5474     for (Result r : scanner) {
5475       assertTrue(!r.isEmpty());
5476       count++;
5477     }
5478     assertEquals(insertNum, count);
5479 
5480     // small scan
5481     Scan scan = new Scan(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
5482     scan.setSmall(true);
5483     scan.setCaching(2);
5484     scanner = table.getScanner(scan);
5485     count = 0;
5486     for (Result r : scanner) {
5487       assertTrue(!r.isEmpty());
5488       count++;
5489     }
5490     assertEquals(insertNum, count);
5491 
5492   }
5493 
5494   @Test
5495   public void testSuperSimpleWithReverseScan() throws Exception {
5496     TableName TABLE = TableName.valueOf("testSuperSimpleWithReverseScan");
5497     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
5498     Put put = new Put(Bytes.toBytes("0-b11111-0000000000000000000"));
5499     put.add(FAMILY, QUALIFIER, VALUE);
5500     ht.put(put);
5501     put = new Put(Bytes.toBytes("0-b11111-0000000000000000002"));
5502     put.add(FAMILY, QUALIFIER, VALUE);
5503     ht.put(put);
5504     put = new Put(Bytes.toBytes("0-b11111-0000000000000000004"));
5505     put.add(FAMILY, QUALIFIER, VALUE);
5506     ht.put(put);
5507     put = new Put(Bytes.toBytes("0-b11111-0000000000000000006"));
5508     put.add(FAMILY, QUALIFIER, VALUE);
5509     ht.put(put);
5510     put = new Put(Bytes.toBytes("0-b11111-0000000000000000008"));
5511     put.add(FAMILY, QUALIFIER, VALUE);
5512     ht.put(put);
5513     put = new Put(Bytes.toBytes("0-b22222-0000000000000000001"));
5514     put.add(FAMILY, QUALIFIER, VALUE);
5515     ht.put(put);
5516     put = new Put(Bytes.toBytes("0-b22222-0000000000000000003"));
5517     put.add(FAMILY, QUALIFIER, VALUE);
5518     ht.put(put);
5519     put = new Put(Bytes.toBytes("0-b22222-0000000000000000005"));
5520     put.add(FAMILY, QUALIFIER, VALUE);
5521     ht.put(put);
5522     put = new Put(Bytes.toBytes("0-b22222-0000000000000000007"));
5523     put.add(FAMILY, QUALIFIER, VALUE);
5524     ht.put(put);
5525     put = new Put(Bytes.toBytes("0-b22222-0000000000000000009"));
5526     put.add(FAMILY, QUALIFIER, VALUE);
5527     ht.put(put);
5528     Scan scan = new Scan(Bytes.toBytes("0-b11111-9223372036854775807"),
5529         Bytes.toBytes("0-b11111-0000000000000000000"));
5530     scan.setReversed(true);
5531     ResultScanner scanner = ht.getScanner(scan);
5532     Result result = scanner.next();
5533     assertTrue(Bytes.equals(result.getRow(),
5534         Bytes.toBytes("0-b11111-0000000000000000008")));
5535     scanner.close();
5536     ht.close();
5537   }
5538 
5539   @Test
5540   public void testFiltersWithReverseScan() throws Exception {
5541     TableName TABLE = TableName.valueOf("testFiltersWithReverseScan");
5542     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
5543     byte[][] ROWS = makeN(ROW, 10);
5544     byte[][] QUALIFIERS = { Bytes.toBytes("col0-<d2v1>-<d3v2>"),
5545         Bytes.toBytes("col1-<d2v1>-<d3v2>"),
5546         Bytes.toBytes("col2-<d2v1>-<d3v2>"),
5547         Bytes.toBytes("col3-<d2v1>-<d3v2>"),
5548         Bytes.toBytes("col4-<d2v1>-<d3v2>"),
5549         Bytes.toBytes("col5-<d2v1>-<d3v2>"),
5550         Bytes.toBytes("col6-<d2v1>-<d3v2>"),
5551         Bytes.toBytes("col7-<d2v1>-<d3v2>"),
5552         Bytes.toBytes("col8-<d2v1>-<d3v2>"),
5553         Bytes.toBytes("col9-<d2v1>-<d3v2>") };
5554     for (int i = 0; i < 10; i++) {
5555       Put put = new Put(ROWS[i]);
5556       put.add(FAMILY, QUALIFIERS[i], VALUE);
5557       ht.put(put);
5558     }
5559     Scan scan = new Scan();
5560     scan.setReversed(true);
5561     scan.addFamily(FAMILY);
5562     Filter filter = new QualifierFilter(CompareOp.EQUAL,
5563         new RegexStringComparator("col[1-5]"));
5564     scan.setFilter(filter);
5565     ResultScanner scanner = ht.getScanner(scan);
5566     int expectedIndex = 5;
5567     for (Result result : scanner) {
5568       assertEquals(result.size(), 1);
5569       assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[expectedIndex]));
5570       assertTrue(Bytes.equals(result.raw()[0].getQualifier(),
5571           QUALIFIERS[expectedIndex]));
5572       expectedIndex--;
5573     }
5574     assertEquals(expectedIndex, 0);
5575     scanner.close();
5576     ht.close();
5577   }
5578 
5579   @Test
5580   public void testKeyOnlyFilterWithReverseScan() throws Exception {
5581     TableName TABLE = TableName.valueOf("testKeyOnlyFilterWithReverseScan");
5582     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
5583     byte[][] ROWS = makeN(ROW, 10);
5584     byte[][] QUALIFIERS = { Bytes.toBytes("col0-<d2v1>-<d3v2>"),
5585         Bytes.toBytes("col1-<d2v1>-<d3v2>"),
5586         Bytes.toBytes("col2-<d2v1>-<d3v2>"),
5587         Bytes.toBytes("col3-<d2v1>-<d3v2>"),
5588         Bytes.toBytes("col4-<d2v1>-<d3v2>"),
5589         Bytes.toBytes("col5-<d2v1>-<d3v2>"),
5590         Bytes.toBytes("col6-<d2v1>-<d3v2>"),
5591         Bytes.toBytes("col7-<d2v1>-<d3v2>"),
5592         Bytes.toBytes("col8-<d2v1>-<d3v2>"),
5593         Bytes.toBytes("col9-<d2v1>-<d3v2>") };
5594     for (int i = 0; i < 10; i++) {
5595       Put put = new Put(ROWS[i]);
5596       put.add(FAMILY, QUALIFIERS[i], VALUE);
5597       ht.put(put);
5598     }
5599     Scan scan = new Scan();
5600     scan.setReversed(true);
5601     scan.addFamily(FAMILY);
5602     Filter filter = new KeyOnlyFilter(true);
5603     scan.setFilter(filter);
5604     ResultScanner scanner = ht.getScanner(scan);
5605     int count = 0;
5606     for (Result result : ht.getScanner(scan)) {
5607       assertEquals(result.size(), 1);
5608       assertEquals(result.raw()[0].getValueLength(), Bytes.SIZEOF_INT);
5609       assertEquals(Bytes.toInt(result.raw()[0].getValue()), VALUE.length);
5610       count++;
5611     }
5612     assertEquals(count, 10);
5613     scanner.close();
5614     ht.close();
5615   }
5616 
5617   /**
5618    * Test simple table and non-existent row cases.
5619    */
5620   @Test
5621   public void testSimpleMissingWithReverseScan() throws Exception {
5622     TableName TABLE = TableName.valueOf("testSimpleMissingWithReverseScan");
5623     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
5624     byte[][] ROWS = makeN(ROW, 4);
5625 
5626     // Try to get a row on an empty table
5627     Scan scan = new Scan();
5628     scan.setReversed(true);
5629     Result result = getSingleScanResult(ht, scan);
5630     assertNullResult(result);
5631 
5632     scan = new Scan(ROWS[0]);
5633     scan.setReversed(true);
5634     result = getSingleScanResult(ht, scan);
5635     assertNullResult(result);
5636 
5637     scan = new Scan(ROWS[0], ROWS[1]);
5638     scan.setReversed(true);
5639     result = getSingleScanResult(ht, scan);
5640     assertNullResult(result);
5641 
5642     scan = new Scan();
5643     scan.setReversed(true);
5644     scan.addFamily(FAMILY);
5645     result = getSingleScanResult(ht, scan);
5646     assertNullResult(result);
5647 
5648     scan = new Scan();
5649     scan.setReversed(true);
5650     scan.addColumn(FAMILY, QUALIFIER);
5651     result = getSingleScanResult(ht, scan);
5652     assertNullResult(result);
5653 
5654     // Insert a row
5655 
5656     Put put = new Put(ROWS[2]);
5657     put.add(FAMILY, QUALIFIER, VALUE);
5658     ht.put(put);
5659 
5660     // Make sure we can scan the row
5661     scan = new Scan();
5662     scan.setReversed(true);
5663     result = getSingleScanResult(ht, scan);
5664     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
5665 
5666     scan = new Scan(ROWS[3], ROWS[0]);
5667     scan.setReversed(true);
5668     result = getSingleScanResult(ht, scan);
5669     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
5670 
5671     scan = new Scan(ROWS[2], ROWS[1]);
5672     scan.setReversed(true);
5673     result = getSingleScanResult(ht, scan);
5674     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
5675 
5676     // Try to scan empty rows around it
5677     // Introduced MemStore#shouldSeekForReverseScan to fix the following
5678     scan = new Scan(ROWS[1]);
5679     scan.setReversed(true);
5680     result = getSingleScanResult(ht, scan);
5681     assertNullResult(result);
5682     ht.close();
5683   }
5684 
5685   @Test
5686   public void testNullWithReverseScan() throws Exception {
5687     TableName TABLE = TableName.valueOf("testNullWithReverseScan");
5688     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
5689     // Null qualifier (should work)
5690     Put put = new Put(ROW);
5691     put.add(FAMILY, null, VALUE);
5692     ht.put(put);
5693     scanTestNull(ht, ROW, FAMILY, VALUE, true);
5694     Delete delete = new Delete(ROW);
5695     delete.deleteColumns(FAMILY, null);
5696     ht.delete(delete);
5697     // Use a new table
5698     byte[] TABLE2 = Bytes.toBytes("testNull2WithReverseScan");
5699     ht = TEST_UTIL.createTable(TableName.valueOf(TABLE2), FAMILY);
5700     // Empty qualifier, byte[0] instead of null (should work)
5701     put = new Put(ROW);
5702     put.add(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
5703     ht.put(put);
5704     scanTestNull(ht, ROW, FAMILY, VALUE, true);
5705     TEST_UTIL.flush();
5706     scanTestNull(ht, ROW, FAMILY, VALUE, true);
5707     delete = new Delete(ROW);
5708     delete.deleteColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
5709     ht.delete(delete);
5710     // Null value
5711     put = new Put(ROW);
5712     put.add(FAMILY, QUALIFIER, null);
5713     ht.put(put);
5714     Scan scan = new Scan();
5715     scan.setReversed(true);
5716     scan.addColumn(FAMILY, QUALIFIER);
5717     Result result = getSingleScanResult(ht, scan);
5718     assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
5719     ht.close();
5720   }
5721 
5722   @Test
5723   public void testDeletesWithReverseScan() throws Exception {
5724     TableName TABLE = TableName.valueOf("testDeletesWithReverseScan");
5725     byte[][] ROWS = makeNAscii(ROW, 6);
5726     byte[][] FAMILIES = makeNAscii(FAMILY, 3);
5727     byte[][] VALUES = makeN(VALUE, 5);
5728     long[] ts = { 1000, 2000, 3000, 4000, 5000 };
5729     Table ht = TEST_UTIL.createTable(TABLE, FAMILIES, TEST_UTIL.getConfiguration(), 3);
5730 
5731     Put put = new Put(ROW);
5732     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
5733     put.add(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
5734     ht.put(put);
5735 
5736     Delete delete = new Delete(ROW);
5737     delete.deleteFamily(FAMILIES[0], ts[0]);
5738     ht.delete(delete);
5739 
5740     Scan scan = new Scan(ROW);
5741     scan.setReversed(true);
5742     scan.addFamily(FAMILIES[0]);
5743     scan.setMaxVersions(Integer.MAX_VALUE);
5744     Result result = getSingleScanResult(ht, scan);
5745     assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1] },
5746         new byte[][] { VALUES[1] }, 0, 0);
5747 
5748     // Test delete latest version
5749     put = new Put(ROW);
5750     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
5751     put.add(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
5752     put.add(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
5753     put.add(FAMILIES[0], null, ts[4], VALUES[4]);
5754     put.add(FAMILIES[0], null, ts[2], VALUES[2]);
5755     put.add(FAMILIES[0], null, ts[3], VALUES[3]);
5756     ht.put(put);
5757 
5758     delete = new Delete(ROW);
5759     delete.deleteColumn(FAMILIES[0], QUALIFIER); // ts[4]
5760     ht.delete(delete);
5761 
5762     scan = new Scan(ROW);
5763     scan.setReversed(true);
5764     scan.addColumn(FAMILIES[0], QUALIFIER);
5765     scan.setMaxVersions(Integer.MAX_VALUE);
5766     result = getSingleScanResult(ht, scan);
5767     assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1],
5768         ts[2], ts[3] }, new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);
5769 
5770     // Test for HBASE-1847
5771     delete = new Delete(ROW);
5772     delete.deleteColumn(FAMILIES[0], null);
5773     ht.delete(delete);
5774 
5775     // Cleanup null qualifier
5776     delete = new Delete(ROW);
5777     delete.deleteColumns(FAMILIES[0], null);
5778     ht.delete(delete);
5779 
5780     // Expected client behavior might be that you can re-put deleted values
5781     // But alas, this is not to be. We can't put them back in either case.
5782 
5783     put = new Put(ROW);
5784     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); // 1000
5785     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); // 5000
5786     ht.put(put);
5787 
5788     // The Scanner returns the previous values, the expected-naive-unexpected
5789     // behavior
5790 
5791     scan = new Scan(ROW);
5792     scan.setReversed(true);
5793     scan.addFamily(FAMILIES[0]);
5794     scan.setMaxVersions(Integer.MAX_VALUE);
5795     result = getSingleScanResult(ht, scan);
5796     assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1],
5797         ts[2], ts[3] }, new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);
5798 
5799     // Test deleting an entire family from one row but not the other various
5800     // ways
5801 
5802     put = new Put(ROWS[0]);
5803     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
5804     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
5805     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
5806     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
5807     ht.put(put);
5808 
5809     put = new Put(ROWS[1]);
5810     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
5811     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
5812     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
5813     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
5814     ht.put(put);
5815 
5816     put = new Put(ROWS[2]);
5817     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
5818     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
5819     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
5820     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
5821     ht.put(put);
5822 
5823     delete = new Delete(ROWS[0]);
5824     delete.deleteFamily(FAMILIES[2]);
5825     ht.delete(delete);
5826 
5827     delete = new Delete(ROWS[1]);
5828     delete.deleteColumns(FAMILIES[1], QUALIFIER);
5829     ht.delete(delete);
5830 
5831     delete = new Delete(ROWS[2]);
5832     delete.deleteColumn(FAMILIES[1], QUALIFIER);
5833     delete.deleteColumn(FAMILIES[1], QUALIFIER);
5834     delete.deleteColumn(FAMILIES[2], QUALIFIER);
5835     ht.delete(delete);
5836 
5837     scan = new Scan(ROWS[0]);
5838     scan.setReversed(true);
5839     scan.addFamily(FAMILIES[1]);
5840     scan.addFamily(FAMILIES[2]);
5841     scan.setMaxVersions(Integer.MAX_VALUE);
5842     result = getSingleScanResult(ht, scan);
5843     assertTrue("Expected 2 keys but received " + result.size(),
5844         result.size() == 2);
5845     assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER, new long[] { ts[0],
5846         ts[1] }, new byte[][] { VALUES[0], VALUES[1] }, 0, 1);
5847 
5848     scan = new Scan(ROWS[1]);
5849     scan.setReversed(true);
5850     scan.addFamily(FAMILIES[1]);
5851     scan.addFamily(FAMILIES[2]);
5852     scan.setMaxVersions(Integer.MAX_VALUE);
5853     result = getSingleScanResult(ht, scan);
5854     assertTrue("Expected 2 keys but received " + result.size(),
5855         result.size() == 2);
5856 
5857     scan = new Scan(ROWS[2]);
5858     scan.setReversed(true);
5859     scan.addFamily(FAMILIES[1]);
5860     scan.addFamily(FAMILIES[2]);
5861     scan.setMaxVersions(Integer.MAX_VALUE);
5862     result = getSingleScanResult(ht, scan);
5863     assertEquals(1, result.size());
5864     assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
5865         new long[] { ts[2] }, new byte[][] { VALUES[2] }, 0, 0);
5866 
5867     // Test if we delete the family first in one row (HBASE-1541)
5868 
5869     delete = new Delete(ROWS[3]);
5870     delete.deleteFamily(FAMILIES[1]);
5871     ht.delete(delete);
5872 
5873     put = new Put(ROWS[3]);
5874     put.add(FAMILIES[2], QUALIFIER, VALUES[0]);
5875     ht.put(put);
5876 
5877     put = new Put(ROWS[4]);
5878     put.add(FAMILIES[1], QUALIFIER, VALUES[1]);
5879     put.add(FAMILIES[2], QUALIFIER, VALUES[2]);
5880     ht.put(put);
5881 
5882     scan = new Scan(ROWS[4]);
5883     scan.setReversed(true);
5884     scan.addFamily(FAMILIES[1]);
5885     scan.addFamily(FAMILIES[2]);
5886     scan.setMaxVersions(Integer.MAX_VALUE);
5887     ResultScanner scanner = ht.getScanner(scan);
5888     result = scanner.next();
5889     assertTrue("Expected 2 keys but received " + result.size(),
5890         result.size() == 2);
5891     assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[4]));
5892     assertTrue(Bytes.equals(result.raw()[1].getRow(), ROWS[4]));
5893     assertTrue(Bytes.equals(result.raw()[0].getValue(), VALUES[1]));
5894     assertTrue(Bytes.equals(result.raw()[1].getValue(), VALUES[2]));
5895     result = scanner.next();
5896     assertTrue("Expected 1 key but received " + result.size(),
5897         result.size() == 1);
5898     assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[3]));
5899     assertTrue(Bytes.equals(result.raw()[0].getValue(), VALUES[0]));
5900     scanner.close();
5901     ht.close();
5902   }
5903 
5904   /**
5905    * Tests reversed scan under multi regions
5906    */
5907   @Test
5908   public void testReversedScanUnderMultiRegions() throws Exception {
5909     // Test Initialization.
5910     TableName TABLE = TableName.valueOf("testReversedScanUnderMultiRegions");
5911     byte[] maxByteArray = ReversedClientScanner.MAX_BYTE_ARRAY;
5912     byte[][] splitRows = new byte[][] { Bytes.toBytes("005"),
5913         Bytes.add(Bytes.toBytes("005"), Bytes.multiple(maxByteArray, 16)),
5914         Bytes.toBytes("006"),
5915         Bytes.add(Bytes.toBytes("006"), Bytes.multiple(maxByteArray, 8)),
5916         Bytes.toBytes("007"),
5917         Bytes.add(Bytes.toBytes("007"), Bytes.multiple(maxByteArray, 4)),
5918         Bytes.toBytes("008"), Bytes.multiple(maxByteArray, 2) };
5919     HTable table = TEST_UTIL.createTable(TABLE, FAMILY, splitRows);
5920     TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
5921 
5922     assertEquals(splitRows.length + 1, table.getRegionLocations().size());
5923     // Insert one row each region
5924     int insertNum = splitRows.length;
5925     for (int i = 0; i < insertNum; i++) {
5926       Put put = new Put(splitRows[i]);
5927       put.add(FAMILY, QUALIFIER, VALUE);
5928       table.put(put);
5929     }
5930 
5931     // scan forward
5932     ResultScanner scanner = table.getScanner(new Scan());
5933     int count = 0;
5934     for (Result r : scanner) {
5935       assertTrue(!r.isEmpty());
5936       count++;
5937     }
5938     assertEquals(insertNum, count);
5939 
5940     // scan backward
5941     Scan scan = new Scan();
5942     scan.setReversed(true);
5943     scanner = table.getScanner(scan);
5944     count = 0;
5945     byte[] lastRow = null;
5946     for (Result r : scanner) {
5947       assertTrue(!r.isEmpty());
5948       count++;
5949       byte[] thisRow = r.getRow();
5950       if (lastRow != null) {
5951         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
5952             + ",this row=" + Bytes.toString(thisRow),
5953             Bytes.compareTo(thisRow, lastRow) < 0);
5954       }
5955       lastRow = thisRow;
5956     }
5957     assertEquals(insertNum, count);
5958     table.close();
5959   }
5960 
5961   /**
5962    * Tests reversed scan under multi regions
5963    */
5964   @Test
5965   public void testSmallReversedScanUnderMultiRegions() throws Exception {
5966     // Test Initialization.
5967     TableName TABLE = TableName.valueOf("testSmallReversedScanUnderMultiRegions");
5968     byte[][] splitRows = new byte[][]{
5969         Bytes.toBytes("000"), Bytes.toBytes("002"), Bytes.toBytes("004"),
5970         Bytes.toBytes("006"), Bytes.toBytes("008"), Bytes.toBytes("010")};
5971     HTable table = TEST_UTIL.createTable(TABLE, FAMILY, splitRows);
5972     TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
5973 
5974     assertEquals(splitRows.length + 1, table.getRegionLocations().size());
5975     for (byte[] splitRow : splitRows) {
5976       Put put = new Put(splitRow);
5977       put.add(FAMILY, QUALIFIER, VALUE);
5978       table.put(put);
5979 
5980       byte[] nextRow = Bytes.copy(splitRow);
5981       nextRow[nextRow.length - 1]++;
5982 
5983       put = new Put(nextRow);
5984       put.add(FAMILY, QUALIFIER, VALUE);
5985       table.put(put);
5986     }
5987 
5988     // scan forward
5989     ResultScanner scanner = table.getScanner(new Scan());
5990     int count = 0;
5991     for (Result r : scanner) {
5992       assertTrue(!r.isEmpty());
5993       count++;
5994     }
5995     assertEquals(12, count);
5996 
5997     reverseScanTest(table, false);
5998     reverseScanTest(table, true);
5999 
6000     table.close();
6001   }
6002 
6003   private void reverseScanTest(Table table, boolean small) throws IOException {
6004     // scan backward
6005     Scan scan = new Scan();
6006     scan.setReversed(true);
6007     ResultScanner scanner = table.getScanner(scan);
6008     int count = 0;
6009     byte[] lastRow = null;
6010     for (Result r : scanner) {
6011       assertTrue(!r.isEmpty());
6012       count++;
6013       byte[] thisRow = r.getRow();
6014       if (lastRow != null) {
6015         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6016             + ",this row=" + Bytes.toString(thisRow),
6017             Bytes.compareTo(thisRow, lastRow) < 0);
6018       }
6019       lastRow = thisRow;
6020     }
6021     assertEquals(12, count);
6022 
6023     scan = new Scan();
6024     scan.setSmall(small);
6025     scan.setReversed(true);
6026     scan.setStartRow(Bytes.toBytes("002"));
6027     scanner = table.getScanner(scan);
6028     count = 0;
6029     lastRow = null;
6030     for (Result r : scanner) {
6031       assertTrue(!r.isEmpty());
6032       count++;
6033       byte[] thisRow = r.getRow();
6034       if (lastRow != null) {
6035         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6036             + ",this row=" + Bytes.toString(thisRow),
6037             Bytes.compareTo(thisRow, lastRow) < 0);
6038       }
6039       lastRow = thisRow;
6040     }
6041     assertEquals(3, count); // 000 001 002
6042 
6043     scan = new Scan();
6044     scan.setSmall(small);
6045     scan.setReversed(true);
6046     scan.setStartRow(Bytes.toBytes("002"));
6047     scan.setStopRow(Bytes.toBytes("000"));
6048     scanner = table.getScanner(scan);
6049     count = 0;
6050     lastRow = null;
6051     for (Result r : scanner) {
6052       assertTrue(!r.isEmpty());
6053       count++;
6054       byte[] thisRow = r.getRow();
6055       if (lastRow != null) {
6056         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6057             + ",this row=" + Bytes.toString(thisRow),
6058             Bytes.compareTo(thisRow, lastRow) < 0);
6059       }
6060       lastRow = thisRow;
6061     }
6062     assertEquals(2, count); // 001 002
6063 
6064     scan = new Scan();
6065     scan.setSmall(small);
6066     scan.setReversed(true);
6067     scan.setStartRow(Bytes.toBytes("001"));
6068     scanner = table.getScanner(scan);
6069     count = 0;
6070     lastRow = null;
6071     for (Result r : scanner) {
6072       assertTrue(!r.isEmpty());
6073       count++;
6074       byte[] thisRow = r.getRow();
6075       if (lastRow != null) {
6076         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6077             + ",this row=" + Bytes.toString(thisRow),
6078             Bytes.compareTo(thisRow, lastRow) < 0);
6079       }
6080       lastRow = thisRow;
6081     }
6082     assertEquals(2, count); // 000 001
6083 
6084     scan = new Scan();
6085     scan.setSmall(small);
6086     scan.setReversed(true);
6087     scan.setStartRow(Bytes.toBytes("000"));
6088     scanner = table.getScanner(scan);
6089     count = 0;
6090     lastRow = null;
6091     for (Result r : scanner) {
6092       assertTrue(!r.isEmpty());
6093       count++;
6094       byte[] thisRow = r.getRow();
6095       if (lastRow != null) {
6096         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6097             + ",this row=" + Bytes.toString(thisRow),
6098             Bytes.compareTo(thisRow, lastRow) < 0);
6099       }
6100       lastRow = thisRow;
6101     }
6102     assertEquals(1, count); // 000
6103 
6104     scan = new Scan();
6105     scan.setSmall(small);
6106     scan.setReversed(true);
6107     scan.setStartRow(Bytes.toBytes("006"));
6108     scan.setStopRow(Bytes.toBytes("002"));
6109     scanner = table.getScanner(scan);
6110     count = 0;
6111     lastRow = null;
6112     for (Result r : scanner) {
6113       assertTrue(!r.isEmpty());
6114       count++;
6115       byte[] thisRow = r.getRow();
6116       if (lastRow != null) {
6117         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6118             + ",this row=" + Bytes.toString(thisRow),
6119             Bytes.compareTo(thisRow, lastRow) < 0);
6120       }
6121       lastRow = thisRow;
6122     }
6123     assertEquals(4, count); // 003 004 005 006
6124   }
6125 
6126   @Test
6127   public void testGetStartEndKeysWithRegionReplicas() throws IOException {
6128     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testGetStartEndKeys"));
6129     HColumnDescriptor fam = new HColumnDescriptor(FAMILY);
6130     htd.addFamily(fam);
6131     byte[][] KEYS = HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE;
6132     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
6133     admin.createTable(htd, KEYS);
6134     List<HRegionInfo> regions = admin.getTableRegions(htd.getTableName());
6135 
6136     HRegionLocator locator =
6137         (HRegionLocator) admin.getConnection().getRegionLocator(htd.getTableName());
6138     for (int regionReplication = 1; regionReplication < 4; regionReplication++) {
6139       List<RegionLocations> regionLocations = new ArrayList<RegionLocations>();
6140 
6141       // mock region locations coming from meta with multiple replicas
6142       for (HRegionInfo region : regions) {
6143         HRegionLocation[] arr = new HRegionLocation[regionReplication];
6144         for (int i = 0; i < arr.length; i++) {
6145           arr[i] = new HRegionLocation(RegionReplicaUtil.getRegionInfoForReplica(region, i), null);
6146         }
6147         regionLocations.add(new RegionLocations(arr));
6148       }
6149 
6150       Pair<byte[][], byte[][]> startEndKeys = locator.getStartEndKeys(regionLocations);
6151 
6152       assertEquals(KEYS.length + 1, startEndKeys.getFirst().length);
6153 
6154       for (int i = 0; i < KEYS.length + 1; i++) {
6155         byte[] startKey = i == 0 ? HConstants.EMPTY_START_ROW : KEYS[i - 1];
6156         byte[] endKey = i == KEYS.length ? HConstants.EMPTY_END_ROW : KEYS[i];
6157         assertArrayEquals(startKey, startEndKeys.getFirst()[i]);
6158         assertArrayEquals(endKey, startEndKeys.getSecond()[i]);
6159       }
6160     }
6161   }
6162 
6163   @Test
6164   public void testFilterAllRecords() throws IOException {
6165     Scan scan = new Scan();
6166     scan.setBatch(1);
6167     scan.setCaching(1);
6168     // Filter out any records
6169     scan.setFilter(new FilterList(new FirstKeyOnlyFilter(), new InclusiveStopFilter(new byte[0])));
6170     try (Table table = TEST_UTIL.getConnection().getTable(TableName.NAMESPACE_TABLE_NAME)) {
6171       try (ResultScanner s = table.getScanner(scan)) {
6172         assertNull(s.next());
6173       }
6174     }
6175   }
6176 }