View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  import static org.junit.Assert.assertArrayEquals;
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertNotNull;
25  import static org.junit.Assert.assertNull;
26  import static org.junit.Assert.assertSame;
27  import static org.junit.Assert.assertTrue;
28  import static org.junit.Assert.fail;
29  import java.io.IOException;
30  import java.lang.reflect.Method;
31  import java.util.ArrayList;
32  import java.util.Arrays;
33  import java.util.Collections;
34  import java.util.HashMap;
35  import java.util.HashSet;
36  import java.util.Iterator;
37  import java.util.LinkedList;
38  import java.util.List;
39  import java.util.Map;
40  import java.util.NavigableMap;
41  import java.util.NavigableSet;
42  import java.util.UUID;
43  import java.util.concurrent.Callable;
44  import java.util.concurrent.ExecutorService;
45  import java.util.concurrent.Executors;
46  import java.util.concurrent.atomic.AtomicBoolean;
47  import java.util.concurrent.atomic.AtomicLong;
48  import java.util.concurrent.atomic.AtomicReference;
49  
50  import org.apache.log4j.Level;
51  import org.apache.commons.lang.ArrayUtils;
52  import org.apache.commons.logging.Log;
53  import org.apache.commons.logging.LogFactory;
54  import org.apache.hadoop.conf.Configuration;
55  import org.apache.hadoop.hbase.Abortable;
56  import org.apache.hadoop.hbase.Cell;
57  import org.apache.hadoop.hbase.CellUtil;
58  import org.apache.hadoop.hbase.DoNotRetryIOException;
59  import org.apache.hadoop.hbase.HBaseTestingUtility;
60  import org.apache.hadoop.hbase.HColumnDescriptor;
61  import org.apache.hadoop.hbase.HConstants;
62  import org.apache.hadoop.hbase.HRegionInfo;
63  import org.apache.hadoop.hbase.HRegionLocation;
64  import org.apache.hadoop.hbase.HTableDescriptor;
65  import org.apache.hadoop.hbase.KeyValue;
66  import org.apache.hadoop.hbase.MiniHBaseCluster;
67  import org.apache.hadoop.hbase.RegionLocations;
68  import org.apache.hadoop.hbase.ServerName;
69  import org.apache.hadoop.hbase.TableName;
70  import org.apache.hadoop.hbase.Waiter;
71  import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
72  import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
73  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
74  import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
75  import org.apache.hadoop.hbase.coprocessor.ObserverContext;
76  import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
77  import org.apache.hadoop.hbase.exceptions.ScannerResetException;
78  import org.apache.hadoop.hbase.filter.BinaryComparator;
79  import org.apache.hadoop.hbase.filter.CompareFilter;
80  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
81  import org.apache.hadoop.hbase.filter.Filter;
82  import org.apache.hadoop.hbase.filter.FilterList;
83  import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
84  import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
85  import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
86  import org.apache.hadoop.hbase.filter.LongComparator;
87  import org.apache.hadoop.hbase.filter.PrefixFilter;
88  import org.apache.hadoop.hbase.filter.QualifierFilter;
89  import org.apache.hadoop.hbase.filter.RegexStringComparator;
90  import org.apache.hadoop.hbase.filter.RowFilter;
91  import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
92  import org.apache.hadoop.hbase.filter.WhileMatchFilter;
93  import org.apache.hadoop.hbase.io.hfile.BlockCache;
94  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
95  import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
96  import org.apache.hadoop.hbase.master.HMaster;
97  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
98  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
99  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
100 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
101 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService;
102 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
103 import org.apache.hadoop.hbase.regionserver.DelegatingKeyValueScanner;
104 import org.apache.hadoop.hbase.regionserver.HRegionServer;
105 import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
106 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
107 import org.apache.hadoop.hbase.regionserver.Region;
108 import org.apache.hadoop.hbase.regionserver.ScanInfo;
109 import org.apache.hadoop.hbase.regionserver.Store;
110 import org.apache.hadoop.hbase.regionserver.StoreScanner;
111 import org.apache.hadoop.hbase.testclassification.LargeTests;
112 import org.apache.hadoop.hbase.util.Bytes;
113 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
114 import org.apache.hadoop.hbase.util.Pair;
115 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
116 import org.apache.log4j.AppenderSkeleton;
117 import org.apache.log4j.Logger;
118 import org.apache.log4j.spi.LoggingEvent;
119 import org.junit.After;
120 import org.junit.AfterClass;
121 import org.junit.Before;
122 import org.junit.BeforeClass;
123 import org.junit.Ignore;
124 import org.junit.Test;
125 import org.junit.experimental.categories.Category;
126 /**
127  * Run tests that use the HBase clients; {@link HTable}.
128  * Sets up the HBase mini cluster once at start and runs through all client tests.
129  * Each creates a table named for the method and does its stuff against that.
130  */
131 @Category(LargeTests.class)
132 @SuppressWarnings ("deprecation")
133 public class TestFromClientSide {
134   private static final Log LOG = LogFactory.getLog(TestFromClientSide.class);
135   protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
136   private static byte [] ROW = Bytes.toBytes("testRow");
137   private static byte [] FAMILY = Bytes.toBytes("testFamily");
138   private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
139   private static byte [] VALUE = Bytes.toBytes("testValue");
140   protected static int SLAVES = 3;
141 
142   /**
143    * @throws java.lang.Exception
144    */
145   @BeforeClass
146   public static void setUpBeforeClass() throws Exception {
147     // Uncomment the following lines if more verbosity is needed for
148     // debugging (see HBASE-12285 for details).
149     //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
150     //((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
151     //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
152     Configuration conf = TEST_UTIL.getConfiguration();
153     conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
154         MultiRowMutationEndpoint.class.getName());
155     conf.setBoolean("hbase.table.sanity.checks", true); // enable for below tests
156     // We need more than one region server in this test
157     TEST_UTIL.startMiniCluster(SLAVES);
158   }
159 
160   /**
161    * @throws java.lang.Exception
162    */
163   @AfterClass
164   public static void tearDownAfterClass() throws Exception {
165     TEST_UTIL.shutdownMiniCluster();
166   }
167 
168   /**
169    * @throws java.lang.Exception
170    */
171   @Before
172   public void setUp() throws Exception {
173     // Nothing to do.
174   }
175 
176   /**
177    * @throws java.lang.Exception
178    */
179   @After
180   public void tearDown() throws Exception {
181     // Nothing to do.
182   }
183 
184   /**
185    * Test append result when there are duplicate rpc request.
186    */
187   @Test
188   public void testDuplicateAppend() throws Exception {
189     HTableDescriptor hdt = TEST_UTIL.createTableDescriptor("HCM-testDuplicateAppend");
190     Map<String, String> kvs = new HashMap<String, String>();
191     kvs.put(HConnectionTestingUtility.SleepAtFirstRpcCall.SLEEP_TIME_CONF_KEY, "2000");
192     hdt.addCoprocessor(HConnectionTestingUtility.SleepAtFirstRpcCall.class.getName(), null, 1, kvs);
193     TEST_UTIL.createTable(hdt, new byte[][] { ROW }).close();
194     TEST_UTIL.waitTableAvailable(hdt.getTableName(), 10000);
195     Configuration c = new Configuration(TEST_UTIL.getConfiguration());
196     c.setInt(HConstants.HBASE_CLIENT_PAUSE, 50);
197     // Client will retry beacuse rpc timeout is small than the sleep time of first rpc call
198     c.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1500);
199 
200     Connection connection = ConnectionFactory.createConnection(c);
201     Table t = connection.getTable(TableName.valueOf("HCM-testDuplicateAppend"));
202     if (t instanceof HTable) {
203       HTable table = (HTable) t;
204       table.setOperationTimeout(3 * 1000);
205 
206       try {
207         Append append = new Append(ROW);
208         append.add(TEST_UTIL.fam1, QUALIFIER, VALUE);
209         Result result = table.append(append);
210 
211         // Verify expected result
212         Cell[] cells = result.rawCells();
213         assertEquals(1, cells.length);
214         assertKey(cells[0], ROW, TEST_UTIL.fam1, QUALIFIER, VALUE);
215 
216         // Verify expected result again
217         Result readResult = table.get(new Get(ROW));
218         cells = readResult.rawCells();
219         assertEquals(1, cells.length);
220         assertKey(cells[0], ROW, TEST_UTIL.fam1, QUALIFIER, VALUE);
221       } finally {
222         table.close();
223         connection.close();
224       }
225     }
226   }
227 
228   /**
229    * Basic client side validation of HBASE-4536
230    */
231    @Test
232    public void testKeepDeletedCells() throws Exception {
233      final TableName TABLENAME = TableName.valueOf("testKeepDeletesCells");
234      final byte[] FAMILY = Bytes.toBytes("family");
235      final byte[] C0 = Bytes.toBytes("c0");
236 
237      final byte[] T1 = Bytes.toBytes("T1");
238      final byte[] T2 = Bytes.toBytes("T2");
239      final byte[] T3 = Bytes.toBytes("T3");
240      HColumnDescriptor hcd = new HColumnDescriptor(FAMILY)
241          .setKeepDeletedCells(true).setMaxVersions(3);
242 
243      HTableDescriptor desc = new HTableDescriptor(TABLENAME);
244      desc.addFamily(hcd);
245      TEST_UTIL.getHBaseAdmin().createTable(desc);
246      TEST_UTIL.waitTableAvailable(desc.getTableName(), 10000);
247      Configuration c = TEST_UTIL.getConfiguration();
248      Table h = new HTable(c, TABLENAME);
249 
250      long ts = System.currentTimeMillis();
251      Put p = new Put(T1, ts);
252      p.add(FAMILY, C0, T1);
253      h.put(p);
254      p = new Put(T1, ts+2);
255      p.add(FAMILY, C0, T2);
256      h.put(p);
257      p = new Put(T1, ts+4);
258      p.add(FAMILY, C0, T3);
259      h.put(p);
260 
261      Delete d = new Delete(T1, ts+3);
262      h.delete(d);
263 
264      d = new Delete(T1, ts+3);
265      d.deleteColumns(FAMILY, C0, ts+3);
266      h.delete(d);
267 
268      Get g = new Get(T1);
269      // does *not* include the delete
270      g.setTimeRange(0, ts+3);
271      Result r = h.get(g);
272      assertArrayEquals(T2, r.getValue(FAMILY, C0));
273 
274      Scan s = new Scan(T1);
275      s.setTimeRange(0, ts+3);
276      s.setMaxVersions();
277      ResultScanner scanner = h.getScanner(s);
278      Cell[] kvs = scanner.next().rawCells();
279      assertArrayEquals(T2, CellUtil.cloneValue(kvs[0]));
280      assertArrayEquals(T1, CellUtil.cloneValue(kvs[1]));
281      scanner.close();
282 
283      s = new Scan(T1);
284      s.setRaw(true);
285      s.setMaxVersions();
286      scanner = h.getScanner(s);
287      kvs = scanner.next().rawCells();
288      assertTrue(CellUtil.isDeleteFamily(kvs[0]));
289      assertArrayEquals(T3, CellUtil.cloneValue(kvs[1]));
290      assertTrue(CellUtil.isDelete(kvs[2]));
291      assertArrayEquals(T2, CellUtil.cloneValue(kvs[3]));
292      assertArrayEquals(T1, CellUtil.cloneValue(kvs[4]));
293      scanner.close();
294      h.close();
295    }
296 
297     /**
298     * Basic client side validation of HBASE-10118
299     */
300    @Test
301    public void testPurgeFutureDeletes() throws Exception {
302      final TableName TABLENAME = TableName.valueOf("testPurgeFutureDeletes");
303      final byte[] ROW = Bytes.toBytes("row");
304      final byte[] FAMILY = Bytes.toBytes("family");
305      final byte[] COLUMN = Bytes.toBytes("column");
306      final byte[] VALUE = Bytes.toBytes("value");
307 
308      Table table = TEST_UTIL.createTable(TABLENAME, FAMILY);
309      TEST_UTIL.waitTableAvailable(TABLENAME, 10000);
310      // future timestamp
311      long ts = System.currentTimeMillis() * 2;
312      Put put = new Put(ROW, ts);
313      put.add(FAMILY, COLUMN, VALUE);
314      table.put(put);
315 
316      Get get = new Get(ROW);
317      Result result = table.get(get);
318      assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
319 
320      Delete del = new Delete(ROW);
321      del.deleteColumn(FAMILY, COLUMN, ts);
322      table.delete(del);
323 
324      get = new Get(ROW);
325      result = table.get(get);
326      assertNull(result.getValue(FAMILY, COLUMN));
327 
328      // major compaction, purged future deletes
329      TEST_UTIL.getHBaseAdmin().flush(TABLENAME);
330      TEST_UTIL.getHBaseAdmin().majorCompact(TABLENAME);
331 
332      // waiting for the major compaction to complete
333      TEST_UTIL.waitFor(6000, new Waiter.Predicate<IOException>() {
334        @Override
335        public boolean evaluate() throws IOException {
336          return TEST_UTIL.getHBaseAdmin().getCompactionState(TABLENAME) ==
337              AdminProtos.GetRegionInfoResponse.CompactionState.NONE;
338        }
339      });
340 
341      put = new Put(ROW, ts);
342      put.add(FAMILY, COLUMN, VALUE);
343      table.put(put);
344 
345      get = new Get(ROW);
346      result = table.get(get);
347      assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
348 
349      table.close();
350    }
351 
352    /**
353     * @deprecated Tests deprecated functionality. Remove when we are past 1.0.
354     * @throws Exception
355     */
356    @Deprecated
357    @Test
358    public void testSharedZooKeeper() throws Exception {
359      Configuration newConfig = new Configuration(TEST_UTIL.getConfiguration());
360      newConfig.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "12345");
361 
362      // First with a simple ZKW
363      ZooKeeperWatcher z0 = new ZooKeeperWatcher(
364        newConfig, "hconnection", new Abortable() {
365        @Override public void abort(String why, Throwable e) {}
366        @Override public boolean isAborted() {return false;}
367      });
368      z0.getRecoverableZooKeeper().getZooKeeper().exists("/oldZooKeeperWatcher", false);
369      z0.close();
370 
371      // Then a ZooKeeperKeepAliveConnection
372      ConnectionManager.HConnectionImplementation connection1 =
373        (ConnectionManager.HConnectionImplementation)
374          HConnectionManager.getConnection(newConfig);
375 
376      ZooKeeperKeepAliveConnection z1 = connection1.getKeepAliveZooKeeperWatcher();
377      z1.getRecoverableZooKeeper().getZooKeeper().exists("/z1", false);
378 
379      z1.close();
380 
381      // will still work, because the real connection is not closed yet
382      // Not do be done in real code
383      z1.getRecoverableZooKeeper().getZooKeeper().exists("/z1afterclose", false);
384 
385 
386      ZooKeeperKeepAliveConnection z2 = connection1.getKeepAliveZooKeeperWatcher();
387      assertTrue(
388        "ZooKeeperKeepAliveConnection equals on same connection", z1 == z2);
389 
390 
391 
392      Configuration newConfig2 = new Configuration(TEST_UTIL.getConfiguration());
393      newConfig2.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "6789");
394      ConnectionManager.HConnectionImplementation connection2 =
395        (ConnectionManager.HConnectionImplementation)
396          HConnectionManager.getConnection(newConfig2);
397 
398      assertTrue("connections should be different ", connection1 != connection2);
399 
400      ZooKeeperKeepAliveConnection z3 = connection2.getKeepAliveZooKeeperWatcher();
401      assertTrue(
402        "ZooKeeperKeepAliveConnection should be different" +
403          " on different connections", z1 != z3);
404 
405      // Bypass the private access
406      Method m = ConnectionManager.HConnectionImplementation.class.
407        getDeclaredMethod("closeZooKeeperWatcher");
408      m.setAccessible(true);
409      m.invoke(connection2);
410 
411      ZooKeeperKeepAliveConnection z4 = connection2.getKeepAliveZooKeeperWatcher();
412      assertTrue(
413        "ZooKeeperKeepAliveConnection should be recreated" +
414          " when previous connections was closed"
415        , z3 != z4);
416 
417 
418      z2.getRecoverableZooKeeper().getZooKeeper().exists("/z2", false);
419      z4.getRecoverableZooKeeper().getZooKeeper().exists("/z4", false);
420 
421 
422      HConnectionManager.deleteConnection(newConfig);
423      try {
424        z2.getRecoverableZooKeeper().getZooKeeper().exists("/z2", false);
425        assertTrue("We should not have a valid connection for z2", false);
426      } catch (Exception e){
427      }
428 
429      z4.getRecoverableZooKeeper().getZooKeeper().exists("/z4", false);
430      // We expect success here.
431 
432 
433      HConnectionManager.deleteConnection(newConfig2);
434      try {
435        z4.getRecoverableZooKeeper().getZooKeeper().exists("/z4", false);
436        assertTrue("We should not have a valid connection for z4", false);
437      } catch (Exception e){
438      }
439    }
440 
441 
442   /**
443    * Verifies that getConfiguration returns the same Configuration object used
444    * to create the HTable instance.
445    */
446   @Test
447   public void testGetConfiguration() throws Exception {
448     TableName TABLE = TableName.valueOf("testGetConfiguration");
449     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
450     Configuration conf = TEST_UTIL.getConfiguration();
451     Table table = TEST_UTIL.createTable(TABLE, FAMILIES, conf);
452     TEST_UTIL.waitTableAvailable(TABLE, 10000);
453     assertSame(conf, table.getConfiguration());
454   }
455 
456   /**
457    * Test from client side of an involved filter against a multi family that
458    * involves deletes.
459    *
460    * @throws Exception
461    */
462   @Test
463   public void testWeirdCacheBehaviour() throws Exception {
464     TableName TABLE = TableName.valueOf("testWeirdCacheBehaviour");
465     byte [][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
466         Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
467         Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
468     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
469     TEST_UTIL.waitTableAvailable(TABLE, 10000);
470     String value = "this is the value";
471     String value2 = "this is some other value";
472     String keyPrefix1 = UUID.randomUUID().toString();
473     String keyPrefix2 = UUID.randomUUID().toString();
474     String keyPrefix3 = UUID.randomUUID().toString();
475     putRows(ht, 3, value, keyPrefix1);
476     putRows(ht, 3, value, keyPrefix2);
477     putRows(ht, 3, value, keyPrefix3);
478     ht.flushCommits();
479     putRows(ht, 3, value2, keyPrefix1);
480     putRows(ht, 3, value2, keyPrefix2);
481     putRows(ht, 3, value2, keyPrefix3);
482     Table table = new HTable(TEST_UTIL.getConfiguration(), TABLE);
483     System.out.println("Checking values for key: " + keyPrefix1);
484     assertEquals("Got back incorrect number of rows from scan", 3,
485         getNumberOfRows(keyPrefix1, value2, table));
486     System.out.println("Checking values for key: " + keyPrefix2);
487     assertEquals("Got back incorrect number of rows from scan", 3,
488         getNumberOfRows(keyPrefix2, value2, table));
489     System.out.println("Checking values for key: " + keyPrefix3);
490     assertEquals("Got back incorrect number of rows from scan", 3,
491         getNumberOfRows(keyPrefix3, value2, table));
492     deleteColumns(ht, value2, keyPrefix1);
493     deleteColumns(ht, value2, keyPrefix2);
494     deleteColumns(ht, value2, keyPrefix3);
495     System.out.println("Starting important checks.....");
496     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
497       0, getNumberOfRows(keyPrefix1, value2, table));
498     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
499       0, getNumberOfRows(keyPrefix2, value2, table));
500     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
501       0, getNumberOfRows(keyPrefix3, value2, table));
502     ht.setScannerCaching(0);
503     assertEquals("Got back incorrect number of rows from scan", 0,
504       getNumberOfRows(keyPrefix1, value2, table)); ht.setScannerCaching(100);
505     assertEquals("Got back incorrect number of rows from scan", 0,
506       getNumberOfRows(keyPrefix2, value2, table));
507   }
508 
509   private void deleteColumns(Table ht, String value, String keyPrefix)
510   throws IOException {
511     ResultScanner scanner = buildScanner(keyPrefix, value, ht);
512     Iterator<Result> it = scanner.iterator();
513     int count = 0;
514     while (it.hasNext()) {
515       Result result = it.next();
516       Delete delete = new Delete(result.getRow());
517       delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
518       ht.delete(delete);
519       count++;
520     }
521     assertEquals("Did not perform correct number of deletes", 3, count);
522   }
523 
524   private int getNumberOfRows(String keyPrefix, String value, Table ht)
525       throws Exception {
526     ResultScanner resultScanner = buildScanner(keyPrefix, value, ht);
527     Iterator<Result> scanner = resultScanner.iterator();
528     int numberOfResults = 0;
529     while (scanner.hasNext()) {
530       Result result = scanner.next();
531       System.out.println("Got back key: " + Bytes.toString(result.getRow()));
532       for (Cell kv : result.rawCells()) {
533         System.out.println("kv=" + kv.toString() + ", "
534             + Bytes.toString(CellUtil.cloneValue(kv)));
535       }
536       numberOfResults++;
537     }
538     return numberOfResults;
539   }
540 
541   private ResultScanner buildScanner(String keyPrefix, String value, Table ht)
542       throws IOException {
543     // OurFilterList allFilters = new OurFilterList();
544     FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */);
545     allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
546     SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes
547         .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes
548         .toBytes(value));
549     filter.setFilterIfMissing(true);
550     allFilters.addFilter(filter);
551 
552     // allFilters.addFilter(new
553     // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"),
554     // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)));
555 
556     Scan scan = new Scan();
557     scan.addFamily(Bytes.toBytes("trans-blob"));
558     scan.addFamily(Bytes.toBytes("trans-type"));
559     scan.addFamily(Bytes.toBytes("trans-date"));
560     scan.addFamily(Bytes.toBytes("trans-tags"));
561     scan.addFamily(Bytes.toBytes("trans-group"));
562     scan.setFilter(allFilters);
563 
564     return ht.getScanner(scan);
565   }
566 
567   private void putRows(Table ht, int numRows, String value, String key)
568       throws IOException {
569     for (int i = 0; i < numRows; i++) {
570       String row = key + "_" + UUID.randomUUID().toString();
571       System.out.println(String.format("Saving row: %s, with value %s", row,
572           value));
573       Put put = new Put(Bytes.toBytes(row));
574       put.setDurability(Durability.SKIP_WAL);
575       put.add(Bytes.toBytes("trans-blob"), null, Bytes
576           .toBytes("value for blob"));
577       put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
578       put.add(Bytes.toBytes("trans-date"), null, Bytes
579           .toBytes("20090921010101999"));
580       put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes
581           .toBytes(value));
582       put.add(Bytes.toBytes("trans-group"), null, Bytes
583           .toBytes("adhocTransactionGroupId"));
584       ht.put(put);
585     }
586   }
587 
588   /**
589    * Test filters when multiple regions.  It does counts.  Needs eye-balling of
590    * logs to ensure that we're not scanning more regions that we're supposed to.
591    * Related to the TestFilterAcrossRegions over in the o.a.h.h.filter package.
592    * @throws IOException
593    * @throws InterruptedException
594    */
595   @Test
596   public void testFilterAcrossMultipleRegions()
597   throws IOException, InterruptedException {
598     TableName name = TableName.valueOf("testFilterAcrossMutlipleRegions");
599     HTable t = TEST_UTIL.createTable(name, FAMILY);
600     TEST_UTIL.waitTableAvailable(name, 10000);
601     int rowCount = TEST_UTIL.loadTable(t, FAMILY, false);
602     assertRowCount(t, rowCount);
603     // Split the table.  Should split on a reasonable key; 'lqj'
604     Map<HRegionInfo, ServerName> regions  = splitTable(t);
605     assertRowCount(t, rowCount);
606     // Get end key of first region.
607     byte [] endKey = regions.keySet().iterator().next().getEndKey();
608     // Count rows with a filter that stops us before passed 'endKey'.
609     // Should be count of rows in first region.
610     int endKeyCount = countRows(t, createScanWithRowFilter(endKey));
611     assertTrue(endKeyCount < rowCount);
612 
613     // How do I know I did not got to second region?  Thats tough.  Can't really
614     // do that in client-side region test.  I verified by tracing in debugger.
615     // I changed the messages that come out when set to DEBUG so should see
616     // when scanner is done. Says "Finished with scanning..." with region name.
617     // Check that its finished in right region.
618 
619     // New test.  Make it so scan goes into next region by one and then two.
620     // Make sure count comes out right.
621     byte [] key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 1)};
622     int plusOneCount = countRows(t, createScanWithRowFilter(key));
623     assertEquals(endKeyCount + 1, plusOneCount);
624     key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 2)};
625     int plusTwoCount = countRows(t, createScanWithRowFilter(key));
626     assertEquals(endKeyCount + 2, plusTwoCount);
627 
628     // New test.  Make it so I scan one less than endkey.
629     key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] - 1)};
630     int minusOneCount = countRows(t, createScanWithRowFilter(key));
631     assertEquals(endKeyCount - 1, minusOneCount);
632     // For above test... study logs.  Make sure we do "Finished with scanning.."
633     // in first region and that we do not fall into the next region.
634 
635     key = new byte [] {'a', 'a', 'a'};
636     int countBBB = countRows(t,
637       createScanWithRowFilter(key, null, CompareFilter.CompareOp.EQUAL));
638     assertEquals(1, countBBB);
639 
640     int countGreater = countRows(t, createScanWithRowFilter(endKey, null,
641       CompareFilter.CompareOp.GREATER_OR_EQUAL));
642     // Because started at start of table.
643     assertEquals(0, countGreater);
644     countGreater = countRows(t, createScanWithRowFilter(endKey, endKey,
645       CompareFilter.CompareOp.GREATER_OR_EQUAL));
646     assertEquals(rowCount - endKeyCount, countGreater);
647   }
648 
649   /**
650    * This is a coprocessor to inject a test failure so that a store scanner.reseek() call will
651    * fail with an IOException() on the first call.
652    */
653   public static class ExceptionInReseekRegionObserver extends BaseRegionObserver {
654     static AtomicLong reqCount = new AtomicLong(0);
655     static AtomicBoolean isDoNotRetry = new AtomicBoolean(false); // whether to throw DNRIOE
656     static AtomicBoolean throwOnce = new AtomicBoolean(true); // whether to only throw once
657 
658     static void reset() {
659       reqCount.set(0);
660       isDoNotRetry.set(false);
661       throwOnce.set(true);
662     }
663 
664     class MyStoreScanner extends StoreScanner {
665       public MyStoreScanner(Store store, ScanInfo scanInfo, Scan scan, NavigableSet<byte[]> columns,
666           long readPt) throws IOException {
667         super(store, scanInfo, scan, columns, readPt);
668       }
669 
670       @Override
671       protected List<KeyValueScanner> selectScannersFrom(
672           List<? extends KeyValueScanner> allScanners) {
673         List<KeyValueScanner> scanners = super.selectScannersFrom(allScanners);
674         List<KeyValueScanner> newScanners = new ArrayList<>(scanners.size());
675         for (KeyValueScanner scanner : scanners) {
676           newScanners.add(new DelegatingKeyValueScanner(scanner) {
677             @Override
678             public boolean reseek(Cell key) throws IOException {
679               reqCount.incrementAndGet();
680               if (!throwOnce.get()||  reqCount.get() == 1) {
681                 if (isDoNotRetry.get()) {
682                   throw new DoNotRetryIOException("Injected exception");
683                 } else {
684                   throw new IOException("Injected exception");
685                 }
686               }
687               return super.reseek(key);
688             }
689           });
690         }
691         return newScanners;
692       }
693     }
694 
695     @Override
696     public KeyValueScanner preStoreScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
697         Store store, Scan scan, NavigableSet<byte[]> targetCols, KeyValueScanner s)
698             throws IOException {
699       return new MyStoreScanner(store, store.getScanInfo(), scan, targetCols, Long.MAX_VALUE);
700     }
701   }
702 
703   /**
704    * Tests the case where a Scan can throw an IOException in the middle of the seek / reseek
705    * leaving the server side RegionScanner to be in dirty state. The client has to ensure that the
706    * ClientScanner does not get an exception and also sees all the data.
707    * @throws IOException
708    * @throws InterruptedException
709    */
710   @Test
711   public void testClientScannerIsResetWhenScanThrowsIOException()
712   throws IOException, InterruptedException {
713     TEST_UTIL.getConfiguration().setBoolean("hbase.client.log.scanner.activity", true);
714     TableName name = TableName.valueOf("testClientScannerIsResetWhenScanThrowsIOException");
715 
716     HTableDescriptor htd = TEST_UTIL.createTableDescriptor(name, FAMILY);
717     htd.addCoprocessor(ExceptionInReseekRegionObserver.class.getName());
718     TEST_UTIL.getHBaseAdmin().createTable(htd);
719     TEST_UTIL.waitTableAvailable(htd.getTableName(), 10000);
720     ExceptionInReseekRegionObserver.reset();
721     ExceptionInReseekRegionObserver.throwOnce.set(true); // throw exceptions only once
722     try (Table t = TEST_UTIL.getConnection().getTable(name)) {
723       int rowCount = TEST_UTIL.loadTable(t, FAMILY, false);
724       TEST_UTIL.getHBaseAdmin().flush(name);
725       int actualRowCount = countRows(t, new Scan().addColumn(FAMILY, FAMILY));
726       assertEquals(rowCount, actualRowCount);
727     }
728     assertTrue(ExceptionInReseekRegionObserver.reqCount.get() > 0);
729   }
730 
731   /**
732    * Tests the case where a coprocessor throws a DoNotRetryIOException in the scan. The expectation
733    * is that the exception will bubble up to the client scanner instead of being retried.
734    */
735   @Test (timeout = 180000)
736   public void testScannerThrowsExceptionWhenCoprocessorThrowsDNRIOE()
737       throws IOException, InterruptedException {
738     TEST_UTIL.getConfiguration().setBoolean("hbase.client.log.scanner.activity", true);
739     TableName name = TableName.valueOf("testClientScannerIsNotRetriedWhenCoprocessorThrowsDNRIOE");
740 
741     HTableDescriptor htd = TEST_UTIL.createTableDescriptor(name, FAMILY);
742     htd.addCoprocessor(ExceptionInReseekRegionObserver.class.getName());
743     TEST_UTIL.getHBaseAdmin().createTable(htd);
744     TEST_UTIL.waitTableAvailable(htd.getTableName(), 10000);
745     ExceptionInReseekRegionObserver.reset();
746     ExceptionInReseekRegionObserver.isDoNotRetry.set(true);
747     try (Table t = TEST_UTIL.getConnection().getTable(name)) {
748       TEST_UTIL.loadTable(t, FAMILY, false);
749       TEST_UTIL.getHBaseAdmin().flush(name);
750       countRows(t, new Scan().addColumn(FAMILY, FAMILY));
751       fail("Should have thrown an exception");
752     } catch (RuntimeException expected) {
753       // expected
754     }
755     assertTrue(ExceptionInReseekRegionObserver.reqCount.get() > 0);
756   }
757 
758   /**
759    * Tests the case where a coprocessor throws a regular IOException in the scan. The expectation
760    * is that the we will keep on retrying, but fail after the retries are exhausted instead of
761    * retrying indefinitely.
762    */
763   @Test (timeout = 180000)
764   public void testScannerFailsAfterRetriesWhenCoprocessorThrowsIOE()
765       throws IOException, InterruptedException {
766     TEST_UTIL.getConfiguration().setBoolean("hbase.client.log.scanner.activity", true);
767     TableName name = TableName.valueOf("testScannerFailsAfterRetriesWhenCoprocessorThrowsIOE");
768     TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
769     HTableDescriptor htd = TEST_UTIL.createTableDescriptor(name, FAMILY);
770     htd.addCoprocessor(ExceptionInReseekRegionObserver.class.getName());
771     TEST_UTIL.getHBaseAdmin().createTable(htd);
772     TEST_UTIL.waitTableAvailable(htd.getTableName(), 10000);
773     ExceptionInReseekRegionObserver.reset();
774     ExceptionInReseekRegionObserver.throwOnce.set(false); // throw exceptions in every retry
775     try (Table t = TEST_UTIL.getConnection().getTable(name)) {
776       TEST_UTIL.loadTable(t, FAMILY, false);
777       TEST_UTIL.getHBaseAdmin().flush(name);
778       countRows(t, new Scan().addColumn(FAMILY, FAMILY));
779       fail("Should have thrown an exception");
780     } catch (RuntimeException expected) {
781       // expected
782     }
783     assertTrue(ExceptionInReseekRegionObserver.reqCount.get() >= 3);
784   }
785 
786   /*
787    * @param key
788    * @return Scan with RowFilter that does LESS than passed key.
789    */
790   private Scan createScanWithRowFilter(final byte [] key) {
791     return createScanWithRowFilter(key, null, CompareFilter.CompareOp.LESS);
792   }
793 
794   /*
795    * @param key
796    * @param op
797    * @param startRow
798    * @return Scan with RowFilter that does CompareOp op on passed key.
799    */
800   private Scan createScanWithRowFilter(final byte [] key,
801       final byte [] startRow, CompareFilter.CompareOp op) {
802     // Make sure key is of some substance... non-null and > than first key.
803     assertTrue(key != null && key.length > 0 &&
804       Bytes.BYTES_COMPARATOR.compare(key, new byte [] {'a', 'a', 'a'}) >= 0);
805     LOG.info("Key=" + Bytes.toString(key));
806     Scan s = startRow == null? new Scan(): new Scan(startRow);
807     Filter f = new RowFilter(op, new BinaryComparator(key));
808     f = new WhileMatchFilter(f);
809     s.setFilter(f);
810     return s;
811   }
812 
813   /*
814    * @param t
815    * @param s
816    * @return Count of rows in table.
817    * @throws IOException
818    */
819   private int countRows(final Table t, final Scan s)
820   throws IOException {
821     // Assert all rows in table.
822     ResultScanner scanner = t.getScanner(s);
823     int count = 0;
824     for (Result result: scanner) {
825       count++;
826       assertTrue(result.size() > 0);
827       // LOG.info("Count=" + count + ", row=" + Bytes.toString(result.getRow()));
828     }
829     return count;
830   }
831 
832   private void assertRowCount(final Table t, final int expected)
833   throws IOException {
834     assertEquals(expected, countRows(t, new Scan()));
835   }
836 
837   /*
838    * Split table into multiple regions.
839    * @param t Table to split.
840    * @return Map of regions to servers.
841    * @throws IOException
842    */
843   private Map<HRegionInfo, ServerName> splitTable(final HTable t)
844   throws IOException, InterruptedException {
845     // Split this table in two.
846     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
847     admin.split(t.getTableName());
848     admin.close();
849     Map<HRegionInfo, ServerName> regions = waitOnSplit(t);
850     assertTrue(regions.size() > 1);
851     return regions;
852   }
853 
854   /*
855    * Wait on table split.  May return because we waited long enough on the split
856    * and it didn't happen.  Caller should check.
857    * @param t
858    * @return Map of table regions; caller needs to check table actually split.
859    */
860   private Map<HRegionInfo, ServerName> waitOnSplit(final HTable t)
861   throws IOException {
862     Map<HRegionInfo, ServerName> regions = t.getRegionLocations();
863     int originalCount = regions.size();
864     for (int i = 0; i < TEST_UTIL.getConfiguration().getInt("hbase.test.retries", 30); i++) {
865       Thread.currentThread();
866       try {
867         Thread.sleep(1000);
868       } catch (InterruptedException e) {
869         e.printStackTrace();
870       }
871       regions = t.getRegionLocations();
872       if (regions.size() > originalCount) break;
873     }
874     return regions;
875   }
876 
877   @Test
878   public void testSuperSimple() throws Exception {
879     byte [] TABLE = Bytes.toBytes("testSuperSimple");
880     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
881     TEST_UTIL.waitTableAvailable(TABLE, 10000);
882     Put put = new Put(ROW);
883     put.add(FAMILY, QUALIFIER, VALUE);
884     ht.put(put);
885     Scan scan = new Scan();
886     scan.addColumn(FAMILY, TABLE);
887     ResultScanner scanner = ht.getScanner(scan);
888     Result result = scanner.next();
889     assertTrue("Expected null result", result == null);
890     scanner.close();
891   }
892 
893   @Test
894   public void testMaxKeyValueSize() throws Exception {
895     byte [] TABLE = Bytes.toBytes("testMaxKeyValueSize");
896     Configuration conf = TEST_UTIL.getConfiguration();
897     String oldMaxSize = conf.get(ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY);
898     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
899     TEST_UTIL.waitTableAvailable(TABLE, 10000);
900     byte[] value = new byte[4 * 1024 * 1024];
901     Put put = new Put(ROW);
902     put.add(FAMILY, QUALIFIER, value);
903     ht.put(put);
904     try {
905       TEST_UTIL.getConfiguration().setInt(
906           ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY, 2 * 1024 * 1024);
907       // Create new table so we pick up the change in Configuration.
908       try (Connection connection =
909           ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
910         try (Table t = connection.getTable(TableName.valueOf(FAMILY))) {
911           put = new Put(ROW);
912           put.add(FAMILY, QUALIFIER, value);
913           t.put(put);
914         }
915       }
916       fail("Inserting a too large KeyValue worked, should throw exception");
917     } catch(Exception e) {}
918     conf.set(ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY, oldMaxSize);
919   }
920 
921   @Test
922   public void testFilters() throws Exception {
923     byte [] TABLE = Bytes.toBytes("testFilters");
924     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
925     TEST_UTIL.waitTableAvailable(TABLE, 10000);
926     byte [][] ROWS = makeN(ROW, 10);
927     byte [][] QUALIFIERS = {
928         Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
929         Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
930         Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
931         Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
932         Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
933     };
934     for(int i=0;i<10;i++) {
935       Put put = new Put(ROWS[i]);
936       put.setDurability(Durability.SKIP_WAL);
937       put.add(FAMILY, QUALIFIERS[i], VALUE);
938       ht.put(put);
939     }
940     Scan scan = new Scan();
941     scan.addFamily(FAMILY);
942     Filter filter = new QualifierFilter(CompareOp.EQUAL,
943       new RegexStringComparator("col[1-5]"));
944     scan.setFilter(filter);
945     ResultScanner scanner = ht.getScanner(scan);
946     int expectedIndex = 1;
947     for(Result result : ht.getScanner(scan)) {
948       assertEquals(result.size(), 1);
949       assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[expectedIndex]));
950       assertTrue(Bytes.equals(CellUtil.cloneQualifier(result.rawCells()[0]),
951           QUALIFIERS[expectedIndex]));
952       expectedIndex++;
953     }
954     assertEquals(expectedIndex, 6);
955     scanner.close();
956   }
957 
958   @Test
959   public void testFilterWithLongCompartor() throws Exception {
960     byte [] TABLE = Bytes.toBytes("testFilterWithLongCompartor");
961     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
962     TEST_UTIL.waitTableAvailable(TABLE, 10000);
963     byte [][] ROWS = makeN(ROW, 10);
964     byte [][] values = new byte[10][];
965     for (int i = 0; i < 10; i ++) {
966         values[i] = Bytes.toBytes(100L * i);
967     }
968     for(int i = 0; i < 10; i ++) {
969       Put put = new Put(ROWS[i]);
970       put.setDurability(Durability.SKIP_WAL);
971       put.add(FAMILY, QUALIFIER, values[i]);
972       ht.put(put);
973     }
974     Scan scan = new Scan();
975     scan.addFamily(FAMILY);
976     Filter filter = new SingleColumnValueFilter(FAMILY, QUALIFIER, CompareOp.GREATER,
977       new LongComparator(500));
978     scan.setFilter(filter);
979     ResultScanner scanner = ht.getScanner(scan);
980     int expectedIndex = 0;
981     for(Result result : ht.getScanner(scan)) {
982       assertEquals(result.size(), 1);
983       assertTrue(Bytes.toLong(result.getValue(FAMILY, QUALIFIER)) > 500);
984       expectedIndex++;
985     }
986     assertEquals(expectedIndex, 4);
987     scanner.close();
988 }
989 
990   @Test
991   public void testKeyOnlyFilter() throws Exception {
992     byte [] TABLE = Bytes.toBytes("testKeyOnlyFilter");
993     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
994     TEST_UTIL.waitTableAvailable(TABLE, 10000);
995     byte [][] ROWS = makeN(ROW, 10);
996     byte [][] QUALIFIERS = {
997         Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
998         Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
999         Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
1000         Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
1001         Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
1002     };
1003     for(int i=0;i<10;i++) {
1004       Put put = new Put(ROWS[i]);
1005       put.setDurability(Durability.SKIP_WAL);
1006       put.add(FAMILY, QUALIFIERS[i], VALUE);
1007       ht.put(put);
1008     }
1009     Scan scan = new Scan();
1010     scan.addFamily(FAMILY);
1011     Filter filter = new KeyOnlyFilter(true);
1012     scan.setFilter(filter);
1013     ResultScanner scanner = ht.getScanner(scan);
1014     int count = 0;
1015     for(Result result : ht.getScanner(scan)) {
1016       assertEquals(result.size(), 1);
1017       assertEquals(result.rawCells()[0].getValueLength(), Bytes.SIZEOF_INT);
1018       assertEquals(Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])), VALUE.length);
1019       count++;
1020     }
1021     assertEquals(count, 10);
1022     scanner.close();
1023   }
1024 
1025   /**
1026    * Test simple table and non-existent row cases.
1027    */
1028   @Test
1029   public void testSimpleMissing() throws Exception {
1030     byte [] TABLE = Bytes.toBytes("testSimpleMissing");
1031     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
1032     TEST_UTIL.waitTableAvailable(TABLE, 10000);
1033     byte [][] ROWS = makeN(ROW, 4);
1034 
1035     // Try to get a row on an empty table
1036     Get get = new Get(ROWS[0]);
1037     Result result = ht.get(get);
1038     assertEmptyResult(result);
1039 
1040     get = new Get(ROWS[0]);
1041     get.addFamily(FAMILY);
1042     result = ht.get(get);
1043     assertEmptyResult(result);
1044 
1045     get = new Get(ROWS[0]);
1046     get.addColumn(FAMILY, QUALIFIER);
1047     result = ht.get(get);
1048     assertEmptyResult(result);
1049 
1050     Scan scan = new Scan();
1051     result = getSingleScanResult(ht, scan);
1052     assertNullResult(result);
1053 
1054 
1055     scan = new Scan(ROWS[0]);
1056     result = getSingleScanResult(ht, scan);
1057     assertNullResult(result);
1058 
1059     scan = new Scan(ROWS[0],ROWS[1]);
1060     result = getSingleScanResult(ht, scan);
1061     assertNullResult(result);
1062 
1063     scan = new Scan();
1064     scan.addFamily(FAMILY);
1065     result = getSingleScanResult(ht, scan);
1066     assertNullResult(result);
1067 
1068     scan = new Scan();
1069     scan.addColumn(FAMILY, QUALIFIER);
1070     result = getSingleScanResult(ht, scan);
1071     assertNullResult(result);
1072 
1073     // Insert a row
1074 
1075     Put put = new Put(ROWS[2]);
1076     put.add(FAMILY, QUALIFIER, VALUE);
1077     ht.put(put);
1078 
1079     // Try to get empty rows around it
1080 
1081     get = new Get(ROWS[1]);
1082     result = ht.get(get);
1083     assertEmptyResult(result);
1084 
1085     get = new Get(ROWS[0]);
1086     get.addFamily(FAMILY);
1087     result = ht.get(get);
1088     assertEmptyResult(result);
1089 
1090     get = new Get(ROWS[3]);
1091     get.addColumn(FAMILY, QUALIFIER);
1092     result = ht.get(get);
1093     assertEmptyResult(result);
1094 
1095     // Try to scan empty rows around it
1096 
1097     scan = new Scan(ROWS[3]);
1098     result = getSingleScanResult(ht, scan);
1099     assertNullResult(result);
1100 
1101     scan = new Scan(ROWS[0],ROWS[2]);
1102     result = getSingleScanResult(ht, scan);
1103     assertNullResult(result);
1104 
1105     // Make sure we can actually get the row
1106 
1107     get = new Get(ROWS[2]);
1108     result = ht.get(get);
1109     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
1110 
1111     get = new Get(ROWS[2]);
1112     get.addFamily(FAMILY);
1113     result = ht.get(get);
1114     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
1115 
1116     get = new Get(ROWS[2]);
1117     get.addColumn(FAMILY, QUALIFIER);
1118     result = ht.get(get);
1119     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
1120 
1121     // Make sure we can scan the row
1122 
1123     scan = new Scan();
1124     result = getSingleScanResult(ht, scan);
1125     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
1126 
1127     scan = new Scan(ROWS[0],ROWS[3]);
1128     result = getSingleScanResult(ht, scan);
1129     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
1130 
1131     scan = new Scan(ROWS[2],ROWS[3]);
1132     result = getSingleScanResult(ht, scan);
1133     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
1134   }
1135 
1136   /**
1137    * Test basic puts, gets, scans, and deletes for a single row
1138    * in a multiple family table.
1139    */
1140   @Test
1141   public void testSingleRowMultipleFamily() throws Exception {
1142     byte [] TABLE = Bytes.toBytes("testSingleRowMultipleFamily");
1143     byte [][] ROWS = makeN(ROW, 3);
1144     byte [][] FAMILIES = makeNAscii(FAMILY, 10);
1145     byte [][] QUALIFIERS = makeN(QUALIFIER, 10);
1146     byte [][] VALUES = makeN(VALUE, 10);
1147 
1148     Table ht = TEST_UTIL.createTable(TABLE, FAMILIES);
1149     TEST_UTIL.waitTableAvailable(TABLE, 10000);
1150 
1151     Get get;
1152     Scan scan;
1153     Delete delete;
1154     Put put;
1155     Result result;
1156 
1157     ////////////////////////////////////////////////////////////////////////////
1158     // Insert one column to one family
1159     ////////////////////////////////////////////////////////////////////////////
1160 
1161     put = new Put(ROWS[0]);
1162     put.add(FAMILIES[4], QUALIFIERS[0], VALUES[0]);
1163     ht.put(put);
1164 
1165     // Get the single column
1166     getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
1167 
1168     // Scan the single column
1169     scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
1170 
1171     // Get empty results around inserted column
1172     getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
1173 
1174     // Scan empty results around inserted column
1175     scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
1176 
1177     ////////////////////////////////////////////////////////////////////////////
1178     // Flush memstore and run same tests from storefiles
1179     ////////////////////////////////////////////////////////////////////////////
1180 
1181     TEST_UTIL.flush();
1182 
1183     // Redo get and scan tests from storefile
1184     getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
1185     scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
1186     getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
1187     scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
1188 
1189     ////////////////////////////////////////////////////////////////////////////
1190     // Now, Test reading from memstore and storefiles at once
1191     ////////////////////////////////////////////////////////////////////////////
1192 
1193     // Insert multiple columns to two other families
1194     put = new Put(ROWS[0]);
1195     put.add(FAMILIES[2], QUALIFIERS[2], VALUES[2]);
1196     put.add(FAMILIES[2], QUALIFIERS[4], VALUES[4]);
1197     put.add(FAMILIES[4], QUALIFIERS[4], VALUES[4]);
1198     put.add(FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1199     put.add(FAMILIES[6], QUALIFIERS[7], VALUES[7]);
1200     put.add(FAMILIES[7], QUALIFIERS[7], VALUES[7]);
1201     put.add(FAMILIES[9], QUALIFIERS[0], VALUES[0]);
1202     ht.put(put);
1203 
1204     // Get multiple columns across multiple families and get empties around it
1205     singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1206 
1207     // Scan multiple columns across multiple families and scan empties around it
1208     singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1209 
1210     ////////////////////////////////////////////////////////////////////////////
1211     // Flush the table again
1212     ////////////////////////////////////////////////////////////////////////////
1213 
1214     TEST_UTIL.flush();
1215 
1216     // Redo tests again
1217     singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1218     singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1219 
1220     // Insert more data to memstore
1221     put = new Put(ROWS[0]);
1222     put.add(FAMILIES[6], QUALIFIERS[5], VALUES[5]);
1223     put.add(FAMILIES[6], QUALIFIERS[8], VALUES[8]);
1224     put.add(FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1225     put.add(FAMILIES[4], QUALIFIERS[3], VALUES[3]);
1226     ht.put(put);
1227 
1228     ////////////////////////////////////////////////////////////////////////////
1229     // Delete a storefile column
1230     ////////////////////////////////////////////////////////////////////////////
1231     delete = new Delete(ROWS[0]);
1232     delete.deleteColumns(FAMILIES[6], QUALIFIERS[7]);
1233     ht.delete(delete);
1234 
1235     // Try to get deleted column
1236     get = new Get(ROWS[0]);
1237     get.addColumn(FAMILIES[6], QUALIFIERS[7]);
1238     result = ht.get(get);
1239     assertEmptyResult(result);
1240 
1241     // Try to scan deleted column
1242     scan = new Scan();
1243     scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
1244     result = getSingleScanResult(ht, scan);
1245     assertNullResult(result);
1246 
1247     // Make sure we can still get a column before it and after it
1248     get = new Get(ROWS[0]);
1249     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
1250     result = ht.get(get);
1251     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1252 
1253     get = new Get(ROWS[0]);
1254     get.addColumn(FAMILIES[6], QUALIFIERS[8]);
1255     result = ht.get(get);
1256     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);
1257 
1258     // Make sure we can still scan a column before it and after it
1259     scan = new Scan();
1260     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1261     result = getSingleScanResult(ht, scan);
1262     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1263 
1264     scan = new Scan();
1265     scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
1266     result = getSingleScanResult(ht, scan);
1267     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);
1268 
1269     ////////////////////////////////////////////////////////////////////////////
1270     // Delete a memstore column
1271     ////////////////////////////////////////////////////////////////////////////
1272     delete = new Delete(ROWS[0]);
1273     delete.deleteColumns(FAMILIES[6], QUALIFIERS[8]);
1274     ht.delete(delete);
1275 
1276     // Try to get deleted column
1277     get = new Get(ROWS[0]);
1278     get.addColumn(FAMILIES[6], QUALIFIERS[8]);
1279     result = ht.get(get);
1280     assertEmptyResult(result);
1281 
1282     // Try to scan deleted column
1283     scan = new Scan();
1284     scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
1285     result = getSingleScanResult(ht, scan);
1286     assertNullResult(result);
1287 
1288     // Make sure we can still get a column before it and after it
1289     get = new Get(ROWS[0]);
1290     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
1291     result = ht.get(get);
1292     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1293 
1294     get = new Get(ROWS[0]);
1295     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1296     result = ht.get(get);
1297     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1298 
1299     // Make sure we can still scan a column before it and after it
1300     scan = new Scan();
1301     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1302     result = getSingleScanResult(ht, scan);
1303     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1304 
1305     scan = new Scan();
1306     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1307     result = getSingleScanResult(ht, scan);
1308     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1309 
1310     ////////////////////////////////////////////////////////////////////////////
1311     // Delete joint storefile/memstore family
1312     ////////////////////////////////////////////////////////////////////////////
1313 
1314     delete = new Delete(ROWS[0]);
1315     delete.deleteFamily(FAMILIES[4]);
1316     ht.delete(delete);
1317 
1318     // Try to get storefile column in deleted family
1319     get = new Get(ROWS[0]);
1320     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
1321     result = ht.get(get);
1322     assertEmptyResult(result);
1323 
1324     // Try to get memstore column in deleted family
1325     get = new Get(ROWS[0]);
1326     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
1327     result = ht.get(get);
1328     assertEmptyResult(result);
1329 
1330     // Try to get deleted family
1331     get = new Get(ROWS[0]);
1332     get.addFamily(FAMILIES[4]);
1333     result = ht.get(get);
1334     assertEmptyResult(result);
1335 
1336     // Try to scan storefile column in deleted family
1337     scan = new Scan();
1338     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
1339     result = getSingleScanResult(ht, scan);
1340     assertNullResult(result);
1341 
1342     // Try to scan memstore column in deleted family
1343     scan = new Scan();
1344     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
1345     result = getSingleScanResult(ht, scan);
1346     assertNullResult(result);
1347 
1348     // Try to scan deleted family
1349     scan = new Scan();
1350     scan.addFamily(FAMILIES[4]);
1351     result = getSingleScanResult(ht, scan);
1352     assertNullResult(result);
1353 
1354     // Make sure we can still get another family
1355     get = new Get(ROWS[0]);
1356     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
1357     result = ht.get(get);
1358     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
1359 
1360     get = new Get(ROWS[0]);
1361     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1362     result = ht.get(get);
1363     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1364 
1365     // Make sure we can still scan another family
1366     scan = new Scan();
1367     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1368     result = getSingleScanResult(ht, scan);
1369     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1370 
1371     scan = new Scan();
1372     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1373     result = getSingleScanResult(ht, scan);
1374     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1375 
1376     ////////////////////////////////////////////////////////////////////////////
1377     // Flush everything and rerun delete tests
1378     ////////////////////////////////////////////////////////////////////////////
1379 
1380     TEST_UTIL.flush();
1381 
1382     // Try to get storefile column in deleted family
1383     get = new Get(ROWS[0]);
1384     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
1385     result = ht.get(get);
1386     assertEmptyResult(result);
1387 
1388     // Try to get memstore column in deleted family
1389     get = new Get(ROWS[0]);
1390     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
1391     result = ht.get(get);
1392     assertEmptyResult(result);
1393 
1394     // Try to get deleted family
1395     get = new Get(ROWS[0]);
1396     get.addFamily(FAMILIES[4]);
1397     result = ht.get(get);
1398     assertEmptyResult(result);
1399 
1400     // Try to scan storefile column in deleted family
1401     scan = new Scan();
1402     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
1403     result = getSingleScanResult(ht, scan);
1404     assertNullResult(result);
1405 
1406     // Try to scan memstore column in deleted family
1407     scan = new Scan();
1408     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
1409     result = getSingleScanResult(ht, scan);
1410     assertNullResult(result);
1411 
1412     // Try to scan deleted family
1413     scan = new Scan();
1414     scan.addFamily(FAMILIES[4]);
1415     result = getSingleScanResult(ht, scan);
1416     assertNullResult(result);
1417 
1418     // Make sure we can still get another family
1419     get = new Get(ROWS[0]);
1420     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
1421     result = ht.get(get);
1422     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
1423 
1424     get = new Get(ROWS[0]);
1425     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1426     result = ht.get(get);
1427     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1428 
1429     // Make sure we can still scan another family
1430     scan = new Scan();
1431     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1432     result = getSingleScanResult(ht, scan);
1433     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1434 
1435     scan = new Scan();
1436     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1437     result = getSingleScanResult(ht, scan);
1438     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1439 
1440   }
1441 
1442   @Test
1443   public void testNull() throws Exception {
1444     byte [] TABLE = Bytes.toBytes("testNull");
1445 
1446     // Null table name (should NOT work)
1447     try {
1448       TEST_UTIL.createTable((TableName)null, FAMILY);
1449       fail("Creating a table with null name passed, should have failed");
1450     } catch(Exception e) {}
1451 
1452     // Null family (should NOT work)
1453     try {
1454       TEST_UTIL.createTable(TABLE, new byte[][]{(byte[])null});
1455       fail("Creating a table with a null family passed, should fail");
1456     } catch(Exception e) {}
1457 
1458     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
1459     TEST_UTIL.waitTableAvailable(TABLE, 10000);
1460 
1461     // Null row (should NOT work)
1462     try {
1463       Put put = new Put((byte[])null);
1464       put.add(FAMILY, QUALIFIER, VALUE);
1465       ht.put(put);
1466       fail("Inserting a null row worked, should throw exception");
1467     } catch(Exception e) {}
1468 
1469     // Null qualifier (should work)
1470     {
1471       Put put = new Put(ROW);
1472       put.add(FAMILY, null, VALUE);
1473       ht.put(put);
1474 
1475       getTestNull(ht, ROW, FAMILY, VALUE);
1476 
1477       scanTestNull(ht, ROW, FAMILY, VALUE);
1478 
1479       Delete delete = new Delete(ROW);
1480       delete.deleteColumns(FAMILY, null);
1481       ht.delete(delete);
1482 
1483       Get get = new Get(ROW);
1484       Result result = ht.get(get);
1485       assertEmptyResult(result);
1486     }
1487 
1488     // Use a new table
1489     byte [] TABLE2 = Bytes.toBytes("testNull2");
1490     ht = TEST_UTIL.createTable(TableName.valueOf(TABLE2), FAMILY);
1491     TEST_UTIL.waitTableAvailable(TableName.valueOf(TABLE2), 10000);
1492     // Empty qualifier, byte[0] instead of null (should work)
1493     try {
1494       Put put = new Put(ROW);
1495       put.add(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
1496       ht.put(put);
1497 
1498       getTestNull(ht, ROW, FAMILY, VALUE);
1499 
1500       scanTestNull(ht, ROW, FAMILY, VALUE);
1501 
1502       // Flush and try again
1503 
1504       TEST_UTIL.flush();
1505 
1506       getTestNull(ht, ROW, FAMILY, VALUE);
1507 
1508       scanTestNull(ht, ROW, FAMILY, VALUE);
1509 
1510       Delete delete = new Delete(ROW);
1511       delete.deleteColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
1512       ht.delete(delete);
1513 
1514       Get get = new Get(ROW);
1515       Result result = ht.get(get);
1516       assertEmptyResult(result);
1517 
1518     } catch(Exception e) {
1519       throw new IOException("Using a row with null qualifier threw exception, should ");
1520     }
1521 
1522     // Null value
1523     try {
1524       Put put = new Put(ROW);
1525       put.add(FAMILY, QUALIFIER, null);
1526       ht.put(put);
1527 
1528       Get get = new Get(ROW);
1529       get.addColumn(FAMILY, QUALIFIER);
1530       Result result = ht.get(get);
1531       assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
1532 
1533       Scan scan = new Scan();
1534       scan.addColumn(FAMILY, QUALIFIER);
1535       result = getSingleScanResult(ht, scan);
1536       assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
1537 
1538       Delete delete = new Delete(ROW);
1539       delete.deleteColumns(FAMILY, QUALIFIER);
1540       ht.delete(delete);
1541 
1542       get = new Get(ROW);
1543       result = ht.get(get);
1544       assertEmptyResult(result);
1545 
1546     } catch(Exception e) {
1547       throw new IOException("Null values should be allowed, but threw exception");
1548     }
1549   }
1550 
1551   @Test
1552   public void testVersions() throws Exception {
1553     byte [] TABLE = Bytes.toBytes("testVersions");
1554 
1555     long [] STAMPS = makeStamps(20);
1556     byte [][] VALUES = makeNAscii(VALUE, 20);
1557 
1558     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
1559     TEST_UTIL.waitTableAvailable(TABLE, 10000);
1560     // Insert 4 versions of same column
1561     Put put = new Put(ROW);
1562     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1563     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1564     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1565     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1566     ht.put(put);
1567 
1568     // Verify we can get each one properly
1569     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1570     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1571     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1572     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1573     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1574     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1575     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1576     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1577 
1578     // Verify we don't accidentally get others
1579     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1580     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1581     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1582     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1583     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1584     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1585 
1586     // Ensure maxVersions in query is respected
1587     Get get = new Get(ROW);
1588     get.addColumn(FAMILY, QUALIFIER);
1589     get.setMaxVersions(2);
1590     Result result = ht.get(get);
1591     assertNResult(result, ROW, FAMILY, QUALIFIER,
1592         new long [] {STAMPS[4], STAMPS[5]},
1593         new byte[][] {VALUES[4], VALUES[5]},
1594         0, 1);
1595 
1596     Scan scan = new Scan(ROW);
1597     scan.addColumn(FAMILY, QUALIFIER);
1598     scan.setMaxVersions(2);
1599     result = getSingleScanResult(ht, scan);
1600     assertNResult(result, ROW, FAMILY, QUALIFIER,
1601         new long [] {STAMPS[4], STAMPS[5]},
1602         new byte[][] {VALUES[4], VALUES[5]},
1603         0, 1);
1604 
1605     // Flush and redo
1606 
1607     TEST_UTIL.flush();
1608 
1609     // Verify we can get each one properly
1610     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1611     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1612     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1613     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1614     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1615     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1616     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1617     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1618 
1619     // Verify we don't accidentally get others
1620     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1621     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1622     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1623     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1624     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1625     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1626 
1627     // Ensure maxVersions in query is respected
1628     get = new Get(ROW);
1629     get.addColumn(FAMILY, QUALIFIER);
1630     get.setMaxVersions(2);
1631     result = ht.get(get);
1632     assertNResult(result, ROW, FAMILY, QUALIFIER,
1633         new long [] {STAMPS[4], STAMPS[5]},
1634         new byte[][] {VALUES[4], VALUES[5]},
1635         0, 1);
1636 
1637     scan = new Scan(ROW);
1638     scan.addColumn(FAMILY, QUALIFIER);
1639     scan.setMaxVersions(2);
1640     result = getSingleScanResult(ht, scan);
1641     assertNResult(result, ROW, FAMILY, QUALIFIER,
1642         new long [] {STAMPS[4], STAMPS[5]},
1643         new byte[][] {VALUES[4], VALUES[5]},
1644         0, 1);
1645 
1646 
1647     // Add some memstore and retest
1648 
1649     // Insert 4 more versions of same column and a dupe
1650     put = new Put(ROW);
1651     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
1652     put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
1653     put.add(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1654     put.add(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
1655     ht.put(put);
1656 
1657     // Ensure maxVersions in query is respected
1658     get = new Get(ROW);
1659     get.addColumn(FAMILY, QUALIFIER);
1660     get.setMaxVersions();
1661     result = ht.get(get);
1662     assertNResult(result, ROW, FAMILY, QUALIFIER,
1663         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1664         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1665         0, 7);
1666 
1667     scan = new Scan(ROW);
1668     scan.addColumn(FAMILY, QUALIFIER);
1669     scan.setMaxVersions();
1670     result = getSingleScanResult(ht, scan);
1671     assertNResult(result, ROW, FAMILY, QUALIFIER,
1672         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1673         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1674         0, 7);
1675 
1676     get = new Get(ROW);
1677     get.setMaxVersions();
1678     result = ht.get(get);
1679     assertNResult(result, ROW, FAMILY, QUALIFIER,
1680         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1681         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1682         0, 7);
1683 
1684     scan = new Scan(ROW);
1685     scan.setMaxVersions();
1686     result = getSingleScanResult(ht, scan);
1687     assertNResult(result, ROW, FAMILY, QUALIFIER,
1688         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1689         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1690         0, 7);
1691 
1692     // Verify we can get each one properly
1693     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1694     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1695     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1696     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1697     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1698     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1699     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1700     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1701 
1702     // Verify we don't accidentally get others
1703     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1704     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
1705     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1706     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
1707 
1708     // Ensure maxVersions of table is respected
1709 
1710     TEST_UTIL.flush();
1711 
1712     // Insert 4 more versions of same column and a dupe
1713     put = new Put(ROW);
1714     put.add(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
1715     put.add(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
1716     put.add(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
1717     put.add(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
1718     ht.put(put);
1719 
1720     get = new Get(ROW);
1721     get.addColumn(FAMILY, QUALIFIER);
1722     get.setMaxVersions(Integer.MAX_VALUE);
1723     result = ht.get(get);
1724     assertNResult(result, ROW, FAMILY, QUALIFIER,
1725         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
1726         new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
1727         0, 9);
1728 
1729     scan = new Scan(ROW);
1730     scan.addColumn(FAMILY, QUALIFIER);
1731     scan.setMaxVersions(Integer.MAX_VALUE);
1732     result = getSingleScanResult(ht, scan);
1733     assertNResult(result, ROW, FAMILY, QUALIFIER,
1734         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
1735         new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
1736         0, 9);
1737 
1738     // Delete a version in the memstore and a version in a storefile
1739     Delete delete = new Delete(ROW);
1740     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[11]);
1741     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[7]);
1742     ht.delete(delete);
1743 
1744     // Test that it's gone
1745     get = new Get(ROW);
1746     get.addColumn(FAMILY, QUALIFIER);
1747     get.setMaxVersions(Integer.MAX_VALUE);
1748     result = ht.get(get);
1749     assertNResult(result, ROW, FAMILY, QUALIFIER,
1750         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
1751         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
1752         0, 9);
1753 
1754     scan = new Scan(ROW);
1755     scan.addColumn(FAMILY, QUALIFIER);
1756     scan.setMaxVersions(Integer.MAX_VALUE);
1757     result = getSingleScanResult(ht, scan);
1758     assertNResult(result, ROW, FAMILY, QUALIFIER,
1759         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
1760         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
1761         0, 9);
1762 
1763   }
1764 
1765   @Test
1766   public void testVersionLimits() throws Exception {
1767     byte [] TABLE = Bytes.toBytes("testVersionLimits");
1768     byte [][] FAMILIES = makeNAscii(FAMILY, 3);
1769     int [] LIMITS = {1,3,5};
1770     long [] STAMPS = makeStamps(10);
1771     byte [][] VALUES = makeNAscii(VALUE, 10);
1772     Table ht = TEST_UTIL.createTable(TABLE, FAMILIES, LIMITS);
1773     TEST_UTIL.waitTableAvailable(TABLE, 10000);
1774     // Insert limit + 1 on each family
1775     Put put = new Put(ROW);
1776     put.add(FAMILIES[0], QUALIFIER, STAMPS[0], VALUES[0]);
1777     put.add(FAMILIES[0], QUALIFIER, STAMPS[1], VALUES[1]);
1778     put.add(FAMILIES[1], QUALIFIER, STAMPS[0], VALUES[0]);
1779     put.add(FAMILIES[1], QUALIFIER, STAMPS[1], VALUES[1]);
1780     put.add(FAMILIES[1], QUALIFIER, STAMPS[2], VALUES[2]);
1781     put.add(FAMILIES[1], QUALIFIER, STAMPS[3], VALUES[3]);
1782     put.add(FAMILIES[2], QUALIFIER, STAMPS[0], VALUES[0]);
1783     put.add(FAMILIES[2], QUALIFIER, STAMPS[1], VALUES[1]);
1784     put.add(FAMILIES[2], QUALIFIER, STAMPS[2], VALUES[2]);
1785     put.add(FAMILIES[2], QUALIFIER, STAMPS[3], VALUES[3]);
1786     put.add(FAMILIES[2], QUALIFIER, STAMPS[4], VALUES[4]);
1787     put.add(FAMILIES[2], QUALIFIER, STAMPS[5], VALUES[5]);
1788     put.add(FAMILIES[2], QUALIFIER, STAMPS[6], VALUES[6]);
1789     ht.put(put);
1790 
1791     // Verify we only get the right number out of each
1792 
1793     // Family0
1794 
1795     Get get = new Get(ROW);
1796     get.addColumn(FAMILIES[0], QUALIFIER);
1797     get.setMaxVersions(Integer.MAX_VALUE);
1798     Result result = ht.get(get);
1799     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1800         new long [] {STAMPS[1]},
1801         new byte[][] {VALUES[1]},
1802         0, 0);
1803 
1804     get = new Get(ROW);
1805     get.addFamily(FAMILIES[0]);
1806     get.setMaxVersions(Integer.MAX_VALUE);
1807     result = ht.get(get);
1808     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1809         new long [] {STAMPS[1]},
1810         new byte[][] {VALUES[1]},
1811         0, 0);
1812 
1813     Scan scan = new Scan(ROW);
1814     scan.addColumn(FAMILIES[0], QUALIFIER);
1815     scan.setMaxVersions(Integer.MAX_VALUE);
1816     result = getSingleScanResult(ht, scan);
1817     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1818         new long [] {STAMPS[1]},
1819         new byte[][] {VALUES[1]},
1820         0, 0);
1821 
1822     scan = new Scan(ROW);
1823     scan.addFamily(FAMILIES[0]);
1824     scan.setMaxVersions(Integer.MAX_VALUE);
1825     result = getSingleScanResult(ht, scan);
1826     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1827         new long [] {STAMPS[1]},
1828         new byte[][] {VALUES[1]},
1829         0, 0);
1830 
1831     // Family1
1832 
1833     get = new Get(ROW);
1834     get.addColumn(FAMILIES[1], QUALIFIER);
1835     get.setMaxVersions(Integer.MAX_VALUE);
1836     result = ht.get(get);
1837     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1838         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1839         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1840         0, 2);
1841 
1842     get = new Get(ROW);
1843     get.addFamily(FAMILIES[1]);
1844     get.setMaxVersions(Integer.MAX_VALUE);
1845     result = ht.get(get);
1846     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1847         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1848         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1849         0, 2);
1850 
1851     scan = new Scan(ROW);
1852     scan.addColumn(FAMILIES[1], QUALIFIER);
1853     scan.setMaxVersions(Integer.MAX_VALUE);
1854     result = getSingleScanResult(ht, scan);
1855     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1856         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1857         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1858         0, 2);
1859 
1860     scan = new Scan(ROW);
1861     scan.addFamily(FAMILIES[1]);
1862     scan.setMaxVersions(Integer.MAX_VALUE);
1863     result = getSingleScanResult(ht, scan);
1864     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1865         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1866         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1867         0, 2);
1868 
1869     // Family2
1870 
1871     get = new Get(ROW);
1872     get.addColumn(FAMILIES[2], QUALIFIER);
1873     get.setMaxVersions(Integer.MAX_VALUE);
1874     result = ht.get(get);
1875     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1876         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1877         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1878         0, 4);
1879 
1880     get = new Get(ROW);
1881     get.addFamily(FAMILIES[2]);
1882     get.setMaxVersions(Integer.MAX_VALUE);
1883     result = ht.get(get);
1884     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1885         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1886         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1887         0, 4);
1888 
1889     scan = new Scan(ROW);
1890     scan.addColumn(FAMILIES[2], QUALIFIER);
1891     scan.setMaxVersions(Integer.MAX_VALUE);
1892     result = getSingleScanResult(ht, scan);
1893     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1894         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1895         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1896         0, 4);
1897 
1898     scan = new Scan(ROW);
1899     scan.addFamily(FAMILIES[2]);
1900     scan.setMaxVersions(Integer.MAX_VALUE);
1901     result = getSingleScanResult(ht, scan);
1902     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1903         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1904         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1905         0, 4);
1906 
1907     // Try all families
1908 
1909     get = new Get(ROW);
1910     get.setMaxVersions(Integer.MAX_VALUE);
1911     result = ht.get(get);
1912     assertTrue("Expected 9 keys but received " + result.size(),
1913         result.size() == 9);
1914 
1915     get = new Get(ROW);
1916     get.addFamily(FAMILIES[0]);
1917     get.addFamily(FAMILIES[1]);
1918     get.addFamily(FAMILIES[2]);
1919     get.setMaxVersions(Integer.MAX_VALUE);
1920     result = ht.get(get);
1921     assertTrue("Expected 9 keys but received " + result.size(),
1922         result.size() == 9);
1923 
1924     get = new Get(ROW);
1925     get.addColumn(FAMILIES[0], QUALIFIER);
1926     get.addColumn(FAMILIES[1], QUALIFIER);
1927     get.addColumn(FAMILIES[2], QUALIFIER);
1928     get.setMaxVersions(Integer.MAX_VALUE);
1929     result = ht.get(get);
1930     assertTrue("Expected 9 keys but received " + result.size(),
1931         result.size() == 9);
1932 
1933     scan = new Scan(ROW);
1934     scan.setMaxVersions(Integer.MAX_VALUE);
1935     result = getSingleScanResult(ht, scan);
1936     assertTrue("Expected 9 keys but received " + result.size(),
1937         result.size() == 9);
1938 
1939     scan = new Scan(ROW);
1940     scan.setMaxVersions(Integer.MAX_VALUE);
1941     scan.addFamily(FAMILIES[0]);
1942     scan.addFamily(FAMILIES[1]);
1943     scan.addFamily(FAMILIES[2]);
1944     result = getSingleScanResult(ht, scan);
1945     assertTrue("Expected 9 keys but received " + result.size(),
1946         result.size() == 9);
1947 
1948     scan = new Scan(ROW);
1949     scan.setMaxVersions(Integer.MAX_VALUE);
1950     scan.addColumn(FAMILIES[0], QUALIFIER);
1951     scan.addColumn(FAMILIES[1], QUALIFIER);
1952     scan.addColumn(FAMILIES[2], QUALIFIER);
1953     result = getSingleScanResult(ht, scan);
1954     assertTrue("Expected 9 keys but received " + result.size(),
1955         result.size() == 9);
1956 
1957   }
1958 
1959   @Test
1960   public void testDeleteFamilyVersion() throws Exception {
1961     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
1962     byte [] TABLE = Bytes.toBytes("testDeleteFamilyVersion");
1963 
1964     byte [][] QUALIFIERS = makeNAscii(QUALIFIER, 1);
1965     byte [][] VALUES = makeN(VALUE, 5);
1966     long [] ts = {1000, 2000, 3000, 4000, 5000};
1967 
1968     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 5);
1969     TEST_UTIL.waitTableAvailable(TABLE, 10000);
1970 
1971     Put put = new Put(ROW);
1972     for (int q = 0; q < 1; q++)
1973       for (int t = 0; t < 5; t++)
1974         put.add(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
1975     ht.put(put);
1976     admin.flush(TABLE);
1977 
1978     Delete delete = new Delete(ROW);
1979     delete.deleteFamilyVersion(FAMILY, ts[1]);  // delete version '2000'
1980     delete.deleteFamilyVersion(FAMILY, ts[3]);  // delete version '4000'
1981     ht.delete(delete);
1982     admin.flush(TABLE);
1983 
1984     for (int i = 0; i < 1; i++) {
1985       Get get = new Get(ROW);
1986       get.addColumn(FAMILY, QUALIFIERS[i]);
1987       get.setMaxVersions(Integer.MAX_VALUE);
1988       Result result = ht.get(get);
1989       // verify version '1000'/'3000'/'5000' remains for all columns
1990       assertNResult(result, ROW, FAMILY, QUALIFIERS[i],
1991           new long [] {ts[0], ts[2], ts[4]},
1992           new byte[][] {VALUES[0], VALUES[2], VALUES[4]},
1993           0, 2);
1994     }
1995     ht.close();
1996     admin.close();
1997   }
1998 
1999   @Test
2000   public void testDeleteFamilyVersionWithOtherDeletes() throws Exception {
2001     byte [] TABLE = Bytes.toBytes("testDeleteFamilyVersionWithOtherDeletes");
2002 
2003     byte [][] QUALIFIERS = makeNAscii(QUALIFIER, 5);
2004     byte [][] VALUES = makeN(VALUE, 5);
2005     long [] ts = {1000, 2000, 3000, 4000, 5000};
2006 
2007     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
2008     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 5);
2009     TEST_UTIL.waitTableAvailable(TABLE, 10000);
2010 
2011     Put put = null;
2012     Result result = null;
2013     Get get = null;
2014     Delete delete = null;
2015 
2016     // 1. put on ROW
2017     put = new Put(ROW);
2018     for (int q = 0; q < 5; q++)
2019       for (int t = 0; t < 5; t++)
2020         put.add(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
2021     ht.put(put);
2022     admin.flush(TABLE);
2023 
2024     // 2. put on ROWS[0]
2025     byte [] ROW2 = Bytes.toBytes("myRowForTest");
2026     put = new Put(ROW2);
2027     for (int q = 0; q < 5; q++)
2028       for (int t = 0; t < 5; t++)
2029         put.add(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
2030     ht.put(put);
2031     admin.flush(TABLE);
2032 
2033     // 3. delete on ROW
2034     delete = new Delete(ROW);
2035     // delete version <= 2000 of all columns
2036     // note: deleteFamily must be the first since it will mask
2037     // the subsequent other type deletes!
2038     delete.deleteFamily(FAMILY, ts[1]);
2039     // delete version '4000' of all columns
2040     delete.deleteFamilyVersion(FAMILY, ts[3]);
2041    // delete version <= 3000 of column 0
2042     delete.deleteColumns(FAMILY, QUALIFIERS[0], ts[2]);
2043     // delete version <= 5000 of column 2
2044     delete.deleteColumns(FAMILY, QUALIFIERS[2], ts[4]);
2045     // delete version 5000 of column 4
2046     delete.deleteColumn(FAMILY, QUALIFIERS[4], ts[4]);
2047     ht.delete(delete);
2048     admin.flush(TABLE);
2049 
2050      // 4. delete on ROWS[0]
2051     delete = new Delete(ROW2);
2052     delete.deleteFamilyVersion(FAMILY, ts[1]);  // delete version '2000'
2053     delete.deleteFamilyVersion(FAMILY, ts[3]);  // delete version '4000'
2054     ht.delete(delete);
2055     admin.flush(TABLE);
2056 
2057     // 5. check ROW
2058     get = new Get(ROW);
2059     get.addColumn(FAMILY, QUALIFIERS[0]);
2060     get.setMaxVersions(Integer.MAX_VALUE);
2061     result = ht.get(get);
2062     assertNResult(result, ROW, FAMILY, QUALIFIERS[0],
2063         new long [] {ts[4]},
2064         new byte[][] {VALUES[4]},
2065         0, 0);
2066 
2067     get = new Get(ROW);
2068     get.addColumn(FAMILY, QUALIFIERS[1]);
2069     get.setMaxVersions(Integer.MAX_VALUE);
2070     result = ht.get(get);
2071     assertNResult(result, ROW, FAMILY, QUALIFIERS[1],
2072         new long [] {ts[2], ts[4]},
2073         new byte[][] {VALUES[2], VALUES[4]},
2074         0, 1);
2075 
2076     get = new Get(ROW);
2077     get.addColumn(FAMILY, QUALIFIERS[2]);
2078     get.setMaxVersions(Integer.MAX_VALUE);
2079     result = ht.get(get);
2080     assertEquals(0, result.size());
2081 
2082     get = new Get(ROW);
2083     get.addColumn(FAMILY, QUALIFIERS[3]);
2084     get.setMaxVersions(Integer.MAX_VALUE);
2085     result = ht.get(get);
2086     assertNResult(result, ROW, FAMILY, QUALIFIERS[3],
2087         new long [] {ts[2], ts[4]},
2088         new byte[][] {VALUES[2], VALUES[4]},
2089         0, 1);
2090 
2091     get = new Get(ROW);
2092     get.addColumn(FAMILY, QUALIFIERS[4]);
2093     get.setMaxVersions(Integer.MAX_VALUE);
2094     result = ht.get(get);
2095     assertNResult(result, ROW, FAMILY, QUALIFIERS[4],
2096         new long [] {ts[2]},
2097         new byte[][] {VALUES[2]},
2098         0, 0);
2099 
2100     // 6. check ROWS[0]
2101     for (int i = 0; i < 5; i++) {
2102       get = new Get(ROW2);
2103       get.addColumn(FAMILY, QUALIFIERS[i]);
2104       get.setMaxVersions(Integer.MAX_VALUE);
2105       result = ht.get(get);
2106       // verify version '1000'/'3000'/'5000' remains for all columns
2107       assertNResult(result, ROW2, FAMILY, QUALIFIERS[i],
2108           new long [] {ts[0], ts[2], ts[4]},
2109           new byte[][] {VALUES[0], VALUES[2], VALUES[4]},
2110           0, 2);
2111     }
2112     ht.close();
2113     admin.close();
2114   }
2115 
2116   @Test
2117   public void testDeletes() throws Exception {
2118     byte [] TABLE = Bytes.toBytes("testDeletes");
2119 
2120     byte [][] ROWS = makeNAscii(ROW, 6);
2121     byte [][] FAMILIES = makeNAscii(FAMILY, 3);
2122     byte [][] VALUES = makeN(VALUE, 5);
2123     long [] ts = {1000, 2000, 3000, 4000, 5000};
2124 
2125     Table ht = TEST_UTIL.createTable(TABLE, FAMILIES, 3);
2126     TEST_UTIL.waitTableAvailable(TABLE, 10000);
2127 
2128     Put put = new Put(ROW);
2129     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
2130     put.add(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
2131     ht.put(put);
2132 
2133     Delete delete = new Delete(ROW);
2134     delete.deleteFamily(FAMILIES[0], ts[0]);
2135     ht.delete(delete);
2136 
2137     Get get = new Get(ROW);
2138     get.addFamily(FAMILIES[0]);
2139     get.setMaxVersions(Integer.MAX_VALUE);
2140     Result result = ht.get(get);
2141     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2142         new long [] {ts[1]},
2143         new byte[][] {VALUES[1]},
2144         0, 0);
2145 
2146     Scan scan = new Scan(ROW);
2147     scan.addFamily(FAMILIES[0]);
2148     scan.setMaxVersions(Integer.MAX_VALUE);
2149     result = getSingleScanResult(ht, scan);
2150     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2151         new long [] {ts[1]},
2152         new byte[][] {VALUES[1]},
2153         0, 0);
2154 
2155     // Test delete latest version
2156     put = new Put(ROW);
2157     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
2158     put.add(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
2159     put.add(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
2160     put.add(FAMILIES[0], null, ts[4], VALUES[4]);
2161     put.add(FAMILIES[0], null, ts[2], VALUES[2]);
2162     put.add(FAMILIES[0], null, ts[3], VALUES[3]);
2163     ht.put(put);
2164 
2165     delete = new Delete(ROW);
2166     delete.deleteColumn(FAMILIES[0], QUALIFIER); // ts[4]
2167     ht.delete(delete);
2168 
2169     get = new Get(ROW);
2170     get.addColumn(FAMILIES[0], QUALIFIER);
2171     get.setMaxVersions(Integer.MAX_VALUE);
2172     result = ht.get(get);
2173     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2174         new long [] {ts[1], ts[2], ts[3]},
2175         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
2176         0, 2);
2177 
2178     scan = new Scan(ROW);
2179     scan.addColumn(FAMILIES[0], QUALIFIER);
2180     scan.setMaxVersions(Integer.MAX_VALUE);
2181     result = getSingleScanResult(ht, scan);
2182     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2183         new long [] {ts[1], ts[2], ts[3]},
2184         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
2185         0, 2);
2186 
2187     // Test for HBASE-1847
2188     delete = new Delete(ROW);
2189     delete.deleteColumn(FAMILIES[0], null);
2190     ht.delete(delete);
2191 
2192     // Cleanup null qualifier
2193     delete = new Delete(ROW);
2194     delete.deleteColumns(FAMILIES[0], null);
2195     ht.delete(delete);
2196 
2197     // Expected client behavior might be that you can re-put deleted values
2198     // But alas, this is not to be.  We can't put them back in either case.
2199 
2200     put = new Put(ROW);
2201     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); // 1000
2202     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); // 5000
2203     ht.put(put);
2204 
2205 
2206     // It used to be due to the internal implementation of Get, that
2207     // the Get() call would return ts[4] UNLIKE the Scan below. With
2208     // the switch to using Scan for Get this is no longer the case.
2209     get = new Get(ROW);
2210     get.addFamily(FAMILIES[0]);
2211     get.setMaxVersions(Integer.MAX_VALUE);
2212     result = ht.get(get);
2213     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2214         new long [] {ts[1], ts[2], ts[3]},
2215         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
2216         0, 2);
2217 
2218     // The Scanner returns the previous values, the expected-naive-unexpected behavior
2219 
2220     scan = new Scan(ROW);
2221     scan.addFamily(FAMILIES[0]);
2222     scan.setMaxVersions(Integer.MAX_VALUE);
2223     result = getSingleScanResult(ht, scan);
2224     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2225         new long [] {ts[1], ts[2], ts[3]},
2226         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
2227         0, 2);
2228 
2229     // Test deleting an entire family from one row but not the other various ways
2230 
2231     put = new Put(ROWS[0]);
2232     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2233     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2234     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2235     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2236     ht.put(put);
2237 
2238     put = new Put(ROWS[1]);
2239     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2240     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2241     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2242     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2243     ht.put(put);
2244 
2245     put = new Put(ROWS[2]);
2246     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2247     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2248     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2249     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2250     ht.put(put);
2251 
2252     // Assert that above went in.
2253     get = new Get(ROWS[2]);
2254     get.addFamily(FAMILIES[1]);
2255     get.addFamily(FAMILIES[2]);
2256     get.setMaxVersions(Integer.MAX_VALUE);
2257     result = ht.get(get);
2258     assertTrue("Expected 4 key but received " + result.size() + ": " + result,
2259         result.size() == 4);
2260 
2261     delete = new Delete(ROWS[0]);
2262     delete.deleteFamily(FAMILIES[2]);
2263     ht.delete(delete);
2264 
2265     delete = new Delete(ROWS[1]);
2266     delete.deleteColumns(FAMILIES[1], QUALIFIER);
2267     ht.delete(delete);
2268 
2269     delete = new Delete(ROWS[2]);
2270     delete.deleteColumn(FAMILIES[1], QUALIFIER);
2271     delete.deleteColumn(FAMILIES[1], QUALIFIER);
2272     delete.deleteColumn(FAMILIES[2], QUALIFIER);
2273     ht.delete(delete);
2274 
2275     get = new Get(ROWS[0]);
2276     get.addFamily(FAMILIES[1]);
2277     get.addFamily(FAMILIES[2]);
2278     get.setMaxVersions(Integer.MAX_VALUE);
2279     result = ht.get(get);
2280     assertTrue("Expected 2 keys but received " + result.size(),
2281         result.size() == 2);
2282     assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
2283         new long [] {ts[0], ts[1]},
2284         new byte[][] {VALUES[0], VALUES[1]},
2285         0, 1);
2286 
2287     scan = new Scan(ROWS[0]);
2288     scan.addFamily(FAMILIES[1]);
2289     scan.addFamily(FAMILIES[2]);
2290     scan.setMaxVersions(Integer.MAX_VALUE);
2291     result = getSingleScanResult(ht, scan);
2292     assertTrue("Expected 2 keys but received " + result.size(),
2293         result.size() == 2);
2294     assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
2295         new long [] {ts[0], ts[1]},
2296         new byte[][] {VALUES[0], VALUES[1]},
2297         0, 1);
2298 
2299     get = new Get(ROWS[1]);
2300     get.addFamily(FAMILIES[1]);
2301     get.addFamily(FAMILIES[2]);
2302     get.setMaxVersions(Integer.MAX_VALUE);
2303     result = ht.get(get);
2304     assertTrue("Expected 2 keys but received " + result.size(),
2305         result.size() == 2);
2306 
2307     scan = new Scan(ROWS[1]);
2308     scan.addFamily(FAMILIES[1]);
2309     scan.addFamily(FAMILIES[2]);
2310     scan.setMaxVersions(Integer.MAX_VALUE);
2311     result = getSingleScanResult(ht, scan);
2312     assertTrue("Expected 2 keys but received " + result.size(),
2313         result.size() == 2);
2314 
2315     get = new Get(ROWS[2]);
2316     get.addFamily(FAMILIES[1]);
2317     get.addFamily(FAMILIES[2]);
2318     get.setMaxVersions(Integer.MAX_VALUE);
2319     result = ht.get(get);
2320     assertEquals(1, result.size());
2321     assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
2322         new long [] {ts[2]},
2323         new byte[][] {VALUES[2]},
2324         0, 0);
2325 
2326     scan = new Scan(ROWS[2]);
2327     scan.addFamily(FAMILIES[1]);
2328     scan.addFamily(FAMILIES[2]);
2329     scan.setMaxVersions(Integer.MAX_VALUE);
2330     result = getSingleScanResult(ht, scan);
2331     assertEquals(1, result.size());
2332     assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
2333         new long [] {ts[2]},
2334         new byte[][] {VALUES[2]},
2335         0, 0);
2336 
2337     // Test if we delete the family first in one row (HBASE-1541)
2338 
2339     delete = new Delete(ROWS[3]);
2340     delete.deleteFamily(FAMILIES[1]);
2341     ht.delete(delete);
2342 
2343     put = new Put(ROWS[3]);
2344     put.add(FAMILIES[2], QUALIFIER, VALUES[0]);
2345     ht.put(put);
2346 
2347     put = new Put(ROWS[4]);
2348     put.add(FAMILIES[1], QUALIFIER, VALUES[1]);
2349     put.add(FAMILIES[2], QUALIFIER, VALUES[2]);
2350     ht.put(put);
2351 
2352     get = new Get(ROWS[3]);
2353     get.addFamily(FAMILIES[1]);
2354     get.addFamily(FAMILIES[2]);
2355     get.setMaxVersions(Integer.MAX_VALUE);
2356     result = ht.get(get);
2357     assertTrue("Expected 1 key but received " + result.size(),
2358         result.size() == 1);
2359 
2360     get = new Get(ROWS[4]);
2361     get.addFamily(FAMILIES[1]);
2362     get.addFamily(FAMILIES[2]);
2363     get.setMaxVersions(Integer.MAX_VALUE);
2364     result = ht.get(get);
2365     assertTrue("Expected 2 keys but received " + result.size(),
2366         result.size() == 2);
2367 
2368     scan = new Scan(ROWS[3]);
2369     scan.addFamily(FAMILIES[1]);
2370     scan.addFamily(FAMILIES[2]);
2371     scan.setMaxVersions(Integer.MAX_VALUE);
2372     ResultScanner scanner = ht.getScanner(scan);
2373     result = scanner.next();
2374     assertTrue("Expected 1 key but received " + result.size(),
2375         result.size() == 1);
2376     assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[3]));
2377     assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[0]));
2378     result = scanner.next();
2379     assertTrue("Expected 2 keys but received " + result.size(),
2380         result.size() == 2);
2381     assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[4]));
2382     assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[1]), ROWS[4]));
2383     assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[1]));
2384     assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[1]), VALUES[2]));
2385     scanner.close();
2386 
2387     // Add test of bulk deleting.
2388     for (int i = 0; i < 10; i++) {
2389       byte [] bytes = Bytes.toBytes(i);
2390       put = new Put(bytes);
2391       put.setDurability(Durability.SKIP_WAL);
2392       put.add(FAMILIES[0], QUALIFIER, bytes);
2393       ht.put(put);
2394     }
2395     for (int i = 0; i < 10; i++) {
2396       byte [] bytes = Bytes.toBytes(i);
2397       get = new Get(bytes);
2398       get.addFamily(FAMILIES[0]);
2399       result = ht.get(get);
2400       assertTrue(result.size() == 1);
2401     }
2402     ArrayList<Delete> deletes = new ArrayList<Delete>();
2403     for (int i = 0; i < 10; i++) {
2404       byte [] bytes = Bytes.toBytes(i);
2405       delete = new Delete(bytes);
2406       delete.deleteFamily(FAMILIES[0]);
2407       deletes.add(delete);
2408     }
2409     ht.delete(deletes);
2410     for (int i = 0; i < 10; i++) {
2411       byte [] bytes = Bytes.toBytes(i);
2412       get = new Get(bytes);
2413       get.addFamily(FAMILIES[0]);
2414       result = ht.get(get);
2415       assertTrue(result.size() == 0);
2416     }
2417   }
2418 
2419   /*
2420    * Baseline "scalability" test.
2421    *
2422    * Tests one hundred families, one million columns, one million versions
2423    */
2424   @Ignore @Test
2425   public void testMillions() throws Exception {
2426 
2427     // 100 families
2428 
2429     // millions of columns
2430 
2431     // millions of versions
2432 
2433   }
2434 
2435   @Ignore @Test
2436   public void testMultipleRegionsAndBatchPuts() throws Exception {
2437     // Two family table
2438 
2439     // Insert lots of rows
2440 
2441     // Insert to the same row with batched puts
2442 
2443     // Insert to multiple rows with batched puts
2444 
2445     // Split the table
2446 
2447     // Get row from first region
2448 
2449     // Get row from second region
2450 
2451     // Scan all rows
2452 
2453     // Insert to multiple regions with batched puts
2454 
2455     // Get row from first region
2456 
2457     // Get row from second region
2458 
2459     // Scan all rows
2460 
2461 
2462   }
2463 
2464   @Ignore @Test
2465   public void testMultipleRowMultipleFamily() throws Exception {
2466 
2467   }
2468 
2469   //
2470   // JIRA Testers
2471   //
2472 
2473   /**
2474    * HBASE-867
2475    *    If millions of columns in a column family, hbase scanner won't come up
2476    *
2477    *    Test will create numRows rows, each with numColsPerRow columns
2478    *    (1 version each), and attempt to scan them all.
2479    *
2480    *    To test at scale, up numColsPerRow to the millions
2481    *    (have not gotten that to work running as junit though)
2482    */
2483   @Test
2484   public void testJiraTest867() throws Exception {
2485     int numRows = 10;
2486     int numColsPerRow = 2000;
2487 
2488     byte [] TABLE = Bytes.toBytes("testJiraTest867");
2489 
2490     byte [][] ROWS = makeN(ROW, numRows);
2491     byte [][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow);
2492 
2493     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
2494     TEST_UTIL.waitTableAvailable(TABLE, 10000);
2495 
2496     // Insert rows
2497 
2498     for(int i=0;i<numRows;i++) {
2499       Put put = new Put(ROWS[i]);
2500       put.setDurability(Durability.SKIP_WAL);
2501       for(int j=0;j<numColsPerRow;j++) {
2502         put.add(FAMILY, QUALIFIERS[j], QUALIFIERS[j]);
2503       }
2504       assertTrue("Put expected to contain " + numColsPerRow + " columns but " +
2505           "only contains " + put.size(), put.size() == numColsPerRow);
2506       ht.put(put);
2507     }
2508 
2509     // Get a row
2510     Get get = new Get(ROWS[numRows-1]);
2511     Result result = ht.get(get);
2512     assertNumKeys(result, numColsPerRow);
2513     Cell [] keys = result.rawCells();
2514     for(int i=0;i<result.size();i++) {
2515       assertKey(keys[i], ROWS[numRows-1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2516     }
2517 
2518     // Scan the rows
2519     Scan scan = new Scan();
2520     ResultScanner scanner = ht.getScanner(scan);
2521     int rowCount = 0;
2522     while((result = scanner.next()) != null) {
2523       assertNumKeys(result, numColsPerRow);
2524       Cell [] kvs = result.rawCells();
2525       for(int i=0;i<numColsPerRow;i++) {
2526         assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2527       }
2528       rowCount++;
2529     }
2530     scanner.close();
2531     assertTrue("Expected to scan " + numRows + " rows but actually scanned "
2532         + rowCount + " rows", rowCount == numRows);
2533 
2534     // flush and try again
2535 
2536     TEST_UTIL.flush();
2537 
2538     // Get a row
2539     get = new Get(ROWS[numRows-1]);
2540     result = ht.get(get);
2541     assertNumKeys(result, numColsPerRow);
2542     keys = result.rawCells();
2543     for(int i=0;i<result.size();i++) {
2544       assertKey(keys[i], ROWS[numRows-1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2545     }
2546 
2547     // Scan the rows
2548     scan = new Scan();
2549     scanner = ht.getScanner(scan);
2550     rowCount = 0;
2551     while((result = scanner.next()) != null) {
2552       assertNumKeys(result, numColsPerRow);
2553       Cell [] kvs = result.rawCells();
2554       for(int i=0;i<numColsPerRow;i++) {
2555         assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2556       }
2557       rowCount++;
2558     }
2559     scanner.close();
2560     assertTrue("Expected to scan " + numRows + " rows but actually scanned "
2561         + rowCount + " rows", rowCount == numRows);
2562 
2563   }
2564 
2565   /**
2566    * HBASE-861
2567    *    get with timestamp will return a value if there is a version with an
2568    *    earlier timestamp
2569    */
2570   @Test
2571   public void testJiraTest861() throws Exception {
2572 
2573     byte [] TABLE = Bytes.toBytes("testJiraTest861");
2574     byte [][] VALUES = makeNAscii(VALUE, 7);
2575     long [] STAMPS = makeStamps(7);
2576 
2577     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2578     TEST_UTIL.waitTableAvailable(TABLE, 10000);
2579 
2580     // Insert three versions
2581 
2582     Put put = new Put(ROW);
2583     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2584     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2585     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2586     ht.put(put);
2587 
2588     // Get the middle value
2589     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2590 
2591     // Try to get one version before (expect fail)
2592     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2593 
2594     // Try to get one version after (expect fail)
2595     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2596 
2597     // Try same from storefile
2598     TEST_UTIL.flush();
2599     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2600     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2601     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2602 
2603     // Insert two more versions surrounding others, into memstore
2604     put = new Put(ROW);
2605     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2606     put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2607     ht.put(put);
2608 
2609     // Check we can get everything we should and can't get what we shouldn't
2610     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2611     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2612     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2613     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2614     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2615     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2616     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2617 
2618     // Try same from two storefiles
2619     TEST_UTIL.flush();
2620     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2621     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2622     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2623     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2624     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2625     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2626     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2627 
2628   }
2629 
2630   /**
2631    * HBASE-33
2632    *    Add a HTable get/obtainScanner method that retrieves all versions of a
2633    *    particular column and row between two timestamps
2634    */
2635   @Test
2636   public void testJiraTest33() throws Exception {
2637 
2638     byte [] TABLE = Bytes.toBytes("testJiraTest33");
2639     byte [][] VALUES = makeNAscii(VALUE, 7);
2640     long [] STAMPS = makeStamps(7);
2641 
2642     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2643     TEST_UTIL.waitTableAvailable(TABLE, 10000);
2644 
2645     // Insert lots versions
2646 
2647     Put put = new Put(ROW);
2648     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2649     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2650     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2651     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2652     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2653     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2654     ht.put(put);
2655 
2656     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2657     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2658     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2659     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2660 
2661     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2662     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2663     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2664     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2665 
2666     // Try same from storefile
2667     TEST_UTIL.flush();
2668 
2669     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2670     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2671     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2672     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2673 
2674     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2675     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2676     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2677     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2678 
2679   }
2680 
2681   /**
2682    * HBASE-1014
2683    *    commit(BatchUpdate) method should return timestamp
2684    */
2685   @Test
2686   public void testJiraTest1014() throws Exception {
2687 
2688     byte [] TABLE = Bytes.toBytes("testJiraTest1014");
2689 
2690     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2691     TEST_UTIL.waitTableAvailable(TABLE, 10000);
2692 
2693     long manualStamp = 12345;
2694 
2695     // Insert lots versions
2696 
2697     Put put = new Put(ROW);
2698     put.add(FAMILY, QUALIFIER, manualStamp, VALUE);
2699     ht.put(put);
2700 
2701     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, manualStamp, VALUE);
2702     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp-1);
2703     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp+1);
2704 
2705   }
2706 
2707   /**
2708    * HBASE-1182
2709    *    Scan for columns > some timestamp
2710    */
2711   @Test
2712   public void testJiraTest1182() throws Exception {
2713 
2714     byte [] TABLE = Bytes.toBytes("testJiraTest1182");
2715     byte [][] VALUES = makeNAscii(VALUE, 7);
2716     long [] STAMPS = makeStamps(7);
2717 
2718     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2719     TEST_UTIL.waitTableAvailable(TABLE, 10000);
2720 
2721     // Insert lots versions
2722 
2723     Put put = new Put(ROW);
2724     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2725     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2726     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2727     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2728     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2729     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2730     ht.put(put);
2731 
2732     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2733     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2734     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2735 
2736     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2737     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2738     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2739 
2740     // Try same from storefile
2741     TEST_UTIL.flush();
2742 
2743     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2744     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2745     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2746 
2747     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2748     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2749     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2750   }
2751 
2752   /**
2753    * HBASE-52
2754    *    Add a means of scanning over all versions
2755    */
2756   @Test
2757   public void testJiraTest52() throws Exception {
2758     byte [] TABLE = Bytes.toBytes("testJiraTest52");
2759     byte [][] VALUES = makeNAscii(VALUE, 7);
2760     long [] STAMPS = makeStamps(7);
2761 
2762     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2763     TEST_UTIL.waitTableAvailable(TABLE, 10000);
2764 
2765     // Insert lots versions
2766 
2767     Put put = new Put(ROW);
2768     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2769     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2770     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2771     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2772     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2773     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2774     ht.put(put);
2775 
2776     getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2777 
2778     scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2779 
2780     // Try same from storefile
2781     TEST_UTIL.flush();
2782 
2783     getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2784 
2785     scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2786   }
2787 
2788   //
2789   // Bulk Testers
2790   //
2791 
2792   private void getVersionRangeAndVerifyGreaterThan(Table ht, byte [] row,
2793       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
2794       int start, int end)
2795   throws IOException {
2796     Get get = new Get(row);
2797     get.addColumn(family, qualifier);
2798     get.setMaxVersions(Integer.MAX_VALUE);
2799     get.setTimeRange(stamps[start+1], Long.MAX_VALUE);
2800     Result result = ht.get(get);
2801     assertNResult(result, row, family, qualifier, stamps, values, start+1, end);
2802   }
2803 
2804   private void getVersionRangeAndVerify(Table ht, byte [] row, byte [] family,
2805       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2806   throws IOException {
2807     Get get = new Get(row);
2808     get.addColumn(family, qualifier);
2809     get.setMaxVersions(Integer.MAX_VALUE);
2810     get.setTimeRange(stamps[start], stamps[end]+1);
2811     Result result = ht.get(get);
2812     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2813   }
2814 
2815   private void getAllVersionsAndVerify(Table ht, byte [] row, byte [] family,
2816       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2817   throws IOException {
2818     Get get = new Get(row);
2819     get.addColumn(family, qualifier);
2820     get.setMaxVersions(Integer.MAX_VALUE);
2821     Result result = ht.get(get);
2822     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2823   }
2824 
2825   private void scanVersionRangeAndVerifyGreaterThan(Table ht, byte [] row,
2826       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
2827       int start, int end)
2828   throws IOException {
2829     Scan scan = new Scan(row);
2830     scan.addColumn(family, qualifier);
2831     scan.setMaxVersions(Integer.MAX_VALUE);
2832     scan.setTimeRange(stamps[start+1], Long.MAX_VALUE);
2833     Result result = getSingleScanResult(ht, scan);
2834     assertNResult(result, row, family, qualifier, stamps, values, start+1, end);
2835   }
2836 
2837   private void scanVersionRangeAndVerify(Table ht, byte [] row, byte [] family,
2838       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2839   throws IOException {
2840     Scan scan = new Scan(row);
2841     scan.addColumn(family, qualifier);
2842     scan.setMaxVersions(Integer.MAX_VALUE);
2843     scan.setTimeRange(stamps[start], stamps[end]+1);
2844     Result result = getSingleScanResult(ht, scan);
2845     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2846   }
2847 
2848   private void scanAllVersionsAndVerify(Table ht, byte [] row, byte [] family,
2849       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2850   throws IOException {
2851     Scan scan = new Scan(row);
2852     scan.addColumn(family, qualifier);
2853     scan.setMaxVersions(Integer.MAX_VALUE);
2854     Result result = getSingleScanResult(ht, scan);
2855     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2856   }
2857 
2858   private void getVersionAndVerify(Table ht, byte [] row, byte [] family,
2859       byte [] qualifier, long stamp, byte [] value)
2860   throws Exception {
2861     Get get = new Get(row);
2862     get.addColumn(family, qualifier);
2863     get.setTimeStamp(stamp);
2864     get.setMaxVersions(Integer.MAX_VALUE);
2865     Result result = ht.get(get);
2866     assertSingleResult(result, row, family, qualifier, stamp, value);
2867   }
2868 
2869   private void getVersionAndVerifyMissing(Table ht, byte [] row, byte [] family,
2870       byte [] qualifier, long stamp)
2871   throws Exception {
2872     Get get = new Get(row);
2873     get.addColumn(family, qualifier);
2874     get.setTimeStamp(stamp);
2875     get.setMaxVersions(Integer.MAX_VALUE);
2876     Result result = ht.get(get);
2877     assertEmptyResult(result);
2878   }
2879 
2880   private void scanVersionAndVerify(Table ht, byte [] row, byte [] family,
2881       byte [] qualifier, long stamp, byte [] value)
2882   throws Exception {
2883     Scan scan = new Scan(row);
2884     scan.addColumn(family, qualifier);
2885     scan.setTimeStamp(stamp);
2886     scan.setMaxVersions(Integer.MAX_VALUE);
2887     Result result = getSingleScanResult(ht, scan);
2888     assertSingleResult(result, row, family, qualifier, stamp, value);
2889   }
2890 
2891   private void scanVersionAndVerifyMissing(Table ht, byte [] row,
2892       byte [] family, byte [] qualifier, long stamp)
2893   throws Exception {
2894     Scan scan = new Scan(row);
2895     scan.addColumn(family, qualifier);
2896     scan.setTimeStamp(stamp);
2897     scan.setMaxVersions(Integer.MAX_VALUE);
2898     Result result = getSingleScanResult(ht, scan);
2899     assertNullResult(result);
2900   }
2901 
2902   private void getTestNull(Table ht, byte [] row, byte [] family,
2903       byte [] value)
2904   throws Exception {
2905 
2906     Get get = new Get(row);
2907     get.addColumn(family, null);
2908     Result result = ht.get(get);
2909     assertSingleResult(result, row, family, null, value);
2910 
2911     get = new Get(row);
2912     get.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
2913     result = ht.get(get);
2914     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2915 
2916     get = new Get(row);
2917     get.addFamily(family);
2918     result = ht.get(get);
2919     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2920 
2921     get = new Get(row);
2922     result = ht.get(get);
2923     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2924 
2925   }
2926 
2927   private void scanTestNull(Table ht, byte[] row, byte[] family, byte[] value)
2928       throws Exception {
2929     scanTestNull(ht, row, family, value, false);
2930   }
2931 
2932   private void scanTestNull(Table ht, byte[] row, byte[] family, byte[] value,
2933       boolean isReversedScan) throws Exception {
2934 
2935     Scan scan = new Scan();
2936     scan.setReversed(isReversedScan);
2937     scan.addColumn(family, null);
2938     Result result = getSingleScanResult(ht, scan);
2939     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2940 
2941     scan = new Scan();
2942     scan.setReversed(isReversedScan);
2943     scan.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
2944     result = getSingleScanResult(ht, scan);
2945     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2946 
2947     scan = new Scan();
2948     scan.setReversed(isReversedScan);
2949     scan.addFamily(family);
2950     result = getSingleScanResult(ht, scan);
2951     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2952 
2953     scan = new Scan();
2954     scan.setReversed(isReversedScan);
2955     result = getSingleScanResult(ht, scan);
2956     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2957 
2958   }
2959 
2960   private void singleRowGetTest(Table ht, byte [][] ROWS, byte [][] FAMILIES,
2961       byte [][] QUALIFIERS, byte [][] VALUES)
2962   throws Exception {
2963 
2964     // Single column from memstore
2965     Get get = new Get(ROWS[0]);
2966     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2967     Result result = ht.get(get);
2968     assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);
2969 
2970     // Single column from storefile
2971     get = new Get(ROWS[0]);
2972     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
2973     result = ht.get(get);
2974     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
2975 
2976     // Single column from storefile, family match
2977     get = new Get(ROWS[0]);
2978     get.addFamily(FAMILIES[7]);
2979     result = ht.get(get);
2980     assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);
2981 
2982     // Two columns, one from memstore one from storefile, same family,
2983     // wildcard match
2984     get = new Get(ROWS[0]);
2985     get.addFamily(FAMILIES[4]);
2986     result = ht.get(get);
2987     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2988         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2989 
2990     // Two columns, one from memstore one from storefile, same family,
2991     // explicit match
2992     get = new Get(ROWS[0]);
2993     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2994     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
2995     result = ht.get(get);
2996     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2997         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2998 
2999     // Three column, one from memstore two from storefile, different families,
3000     // wildcard match
3001     get = new Get(ROWS[0]);
3002     get.addFamily(FAMILIES[4]);
3003     get.addFamily(FAMILIES[7]);
3004     result = ht.get(get);
3005     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
3006         new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} });
3007 
3008     // Multiple columns from everywhere storefile, many family, wildcard
3009     get = new Get(ROWS[0]);
3010     get.addFamily(FAMILIES[2]);
3011     get.addFamily(FAMILIES[4]);
3012     get.addFamily(FAMILIES[6]);
3013     get.addFamily(FAMILIES[7]);
3014     result = ht.get(get);
3015     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
3016         new int [][] {
3017           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
3018     });
3019 
3020     // Multiple columns from everywhere storefile, many family, wildcard
3021     get = new Get(ROWS[0]);
3022     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
3023     get.addColumn(FAMILIES[2], QUALIFIERS[4]);
3024     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
3025     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
3026     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
3027     get.addColumn(FAMILIES[6], QUALIFIERS[7]);
3028     get.addColumn(FAMILIES[7], QUALIFIERS[7]);
3029     get.addColumn(FAMILIES[7], QUALIFIERS[8]);
3030     result = ht.get(get);
3031     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
3032         new int [][] {
3033           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
3034     });
3035 
3036     // Everything
3037     get = new Get(ROWS[0]);
3038     result = ht.get(get);
3039     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
3040         new int [][] {
3041           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0}
3042     });
3043 
3044     // Get around inserted columns
3045 
3046     get = new Get(ROWS[1]);
3047     result = ht.get(get);
3048     assertEmptyResult(result);
3049 
3050     get = new Get(ROWS[0]);
3051     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
3052     get.addColumn(FAMILIES[2], QUALIFIERS[3]);
3053     result = ht.get(get);
3054     assertEmptyResult(result);
3055 
3056   }
3057 
3058   private void singleRowScanTest(Table ht, byte [][] ROWS, byte [][] FAMILIES,
3059       byte [][] QUALIFIERS, byte [][] VALUES)
3060   throws Exception {
3061 
3062     // Single column from memstore
3063     Scan scan = new Scan();
3064     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
3065     Result result = getSingleScanResult(ht, scan);
3066     assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);
3067 
3068     // Single column from storefile
3069     scan = new Scan();
3070     scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
3071     result = getSingleScanResult(ht, scan);
3072     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
3073 
3074     // Single column from storefile, family match
3075     scan = new Scan();
3076     scan.addFamily(FAMILIES[7]);
3077     result = getSingleScanResult(ht, scan);
3078     assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);
3079 
3080     // Two columns, one from memstore one from storefile, same family,
3081     // wildcard match
3082     scan = new Scan();
3083     scan.addFamily(FAMILIES[4]);
3084     result = getSingleScanResult(ht, scan);
3085     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
3086         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
3087 
3088     // Two columns, one from memstore one from storefile, same family,
3089     // explicit match
3090     scan = new Scan();
3091     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
3092     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
3093     result = getSingleScanResult(ht, scan);
3094     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
3095         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
3096 
3097     // Three column, one from memstore two from storefile, different families,
3098     // wildcard match
3099     scan = new Scan();
3100     scan.addFamily(FAMILIES[4]);
3101     scan.addFamily(FAMILIES[7]);
3102     result = getSingleScanResult(ht, scan);
3103     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
3104         new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} });
3105 
3106     // Multiple columns from everywhere storefile, many family, wildcard
3107     scan = new Scan();
3108     scan.addFamily(FAMILIES[2]);
3109     scan.addFamily(FAMILIES[4]);
3110     scan.addFamily(FAMILIES[6]);
3111     scan.addFamily(FAMILIES[7]);
3112     result = getSingleScanResult(ht, scan);
3113     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
3114         new int [][] {
3115           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
3116     });
3117 
3118     // Multiple columns from everywhere storefile, many family, wildcard
3119     scan = new Scan();
3120     scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
3121     scan.addColumn(FAMILIES[2], QUALIFIERS[4]);
3122     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
3123     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
3124     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
3125     scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
3126     scan.addColumn(FAMILIES[7], QUALIFIERS[7]);
3127     scan.addColumn(FAMILIES[7], QUALIFIERS[8]);
3128     result = getSingleScanResult(ht, scan);
3129     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
3130         new int [][] {
3131           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
3132     });
3133 
3134     // Everything
3135     scan = new Scan();
3136     result = getSingleScanResult(ht, scan);
3137     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
3138         new int [][] {
3139           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0}
3140     });
3141 
3142     // Scan around inserted columns
3143 
3144     scan = new Scan(ROWS[1]);
3145     result = getSingleScanResult(ht, scan);
3146     assertNullResult(result);
3147 
3148     scan = new Scan();
3149     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
3150     scan.addColumn(FAMILIES[2], QUALIFIERS[3]);
3151     result = getSingleScanResult(ht, scan);
3152     assertNullResult(result);
3153   }
3154 
3155   /**
3156    * Verify a single column using gets.
3157    * Expects family and qualifier arrays to be valid for at least
3158    * the range:  idx-2 < idx < idx+2
3159    */
3160   private void getVerifySingleColumn(Table ht,
3161       byte [][] ROWS, int ROWIDX,
3162       byte [][] FAMILIES, int FAMILYIDX,
3163       byte [][] QUALIFIERS, int QUALIFIERIDX,
3164       byte [][] VALUES, int VALUEIDX)
3165   throws Exception {
3166 
3167     Get get = new Get(ROWS[ROWIDX]);
3168     Result result = ht.get(get);
3169     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3170         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3171 
3172     get = new Get(ROWS[ROWIDX]);
3173     get.addFamily(FAMILIES[FAMILYIDX]);
3174     result = ht.get(get);
3175     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3176         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3177 
3178     get = new Get(ROWS[ROWIDX]);
3179     get.addFamily(FAMILIES[FAMILYIDX-2]);
3180     get.addFamily(FAMILIES[FAMILYIDX]);
3181     get.addFamily(FAMILIES[FAMILYIDX+2]);
3182     result = ht.get(get);
3183     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3184         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3185 
3186     get = new Get(ROWS[ROWIDX]);
3187     get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[0]);
3188     result = ht.get(get);
3189     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3190         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3191 
3192     get = new Get(ROWS[ROWIDX]);
3193     get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[1]);
3194     get.addFamily(FAMILIES[FAMILYIDX]);
3195     result = ht.get(get);
3196     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3197         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3198 
3199     get = new Get(ROWS[ROWIDX]);
3200     get.addFamily(FAMILIES[FAMILYIDX]);
3201     get.addColumn(FAMILIES[FAMILYIDX+1], QUALIFIERS[1]);
3202     get.addColumn(FAMILIES[FAMILYIDX-2], QUALIFIERS[1]);
3203     get.addFamily(FAMILIES[FAMILYIDX-1]);
3204     get.addFamily(FAMILIES[FAMILYIDX+2]);
3205     result = ht.get(get);
3206     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3207         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3208 
3209   }
3210 
3211 
3212   /**
3213    * Verify a single column using scanners.
3214    * Expects family and qualifier arrays to be valid for at least
3215    * the range:  idx-2 to idx+2
3216    * Expects row array to be valid for at least idx to idx+2
3217    */
3218   private void scanVerifySingleColumn(Table ht,
3219       byte [][] ROWS, int ROWIDX,
3220       byte [][] FAMILIES, int FAMILYIDX,
3221       byte [][] QUALIFIERS, int QUALIFIERIDX,
3222       byte [][] VALUES, int VALUEIDX)
3223   throws Exception {
3224 
3225     Scan scan = new Scan();
3226     Result result = getSingleScanResult(ht, scan);
3227     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3228         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3229 
3230     scan = new Scan(ROWS[ROWIDX]);
3231     result = getSingleScanResult(ht, scan);
3232     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3233         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3234 
3235     scan = new Scan(ROWS[ROWIDX], ROWS[ROWIDX+1]);
3236     result = getSingleScanResult(ht, scan);
3237     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3238         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3239 
3240     scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX+1]);
3241     result = getSingleScanResult(ht, scan);
3242     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3243         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3244 
3245     scan = new Scan();
3246     scan.addFamily(FAMILIES[FAMILYIDX]);
3247     result = getSingleScanResult(ht, scan);
3248     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3249         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3250 
3251     scan = new Scan();
3252     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
3253     result = getSingleScanResult(ht, scan);
3254     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3255         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3256 
3257     scan = new Scan();
3258     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]);
3259     scan.addFamily(FAMILIES[FAMILYIDX]);
3260     result = getSingleScanResult(ht, scan);
3261     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3262         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3263 
3264     scan = new Scan();
3265     scan.addColumn(FAMILIES[FAMILYIDX-1], QUALIFIERS[QUALIFIERIDX+1]);
3266     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
3267     scan.addFamily(FAMILIES[FAMILYIDX+1]);
3268     result = getSingleScanResult(ht, scan);
3269     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3270         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3271 
3272   }
3273 
3274   /**
3275    * Verify we do not read any values by accident around a single column
3276    * Same requirements as getVerifySingleColumn
3277    */
3278   private void getVerifySingleEmpty(Table ht,
3279       byte [][] ROWS, int ROWIDX,
3280       byte [][] FAMILIES, int FAMILYIDX,
3281       byte [][] QUALIFIERS, int QUALIFIERIDX)
3282   throws Exception {
3283 
3284     Get get = new Get(ROWS[ROWIDX]);
3285     get.addFamily(FAMILIES[4]);
3286     get.addColumn(FAMILIES[4], QUALIFIERS[1]);
3287     Result result = ht.get(get);
3288     assertEmptyResult(result);
3289 
3290     get = new Get(ROWS[ROWIDX]);
3291     get.addFamily(FAMILIES[4]);
3292     get.addColumn(FAMILIES[4], QUALIFIERS[2]);
3293     result = ht.get(get);
3294     assertEmptyResult(result);
3295 
3296     get = new Get(ROWS[ROWIDX]);
3297     get.addFamily(FAMILIES[3]);
3298     get.addColumn(FAMILIES[4], QUALIFIERS[2]);
3299     get.addFamily(FAMILIES[5]);
3300     result = ht.get(get);
3301     assertEmptyResult(result);
3302 
3303     get = new Get(ROWS[ROWIDX+1]);
3304     result = ht.get(get);
3305     assertEmptyResult(result);
3306 
3307   }
3308 
3309   private void scanVerifySingleEmpty(Table ht,
3310       byte [][] ROWS, int ROWIDX,
3311       byte [][] FAMILIES, int FAMILYIDX,
3312       byte [][] QUALIFIERS, int QUALIFIERIDX)
3313   throws Exception {
3314 
3315     Scan scan = new Scan(ROWS[ROWIDX+1]);
3316     Result result = getSingleScanResult(ht, scan);
3317     assertNullResult(result);
3318 
3319     scan = new Scan(ROWS[ROWIDX+1],ROWS[ROWIDX+2]);
3320     result = getSingleScanResult(ht, scan);
3321     assertNullResult(result);
3322 
3323     scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX]);
3324     result = getSingleScanResult(ht, scan);
3325     assertNullResult(result);
3326 
3327     scan = new Scan();
3328     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]);
3329     scan.addFamily(FAMILIES[FAMILYIDX-1]);
3330     result = getSingleScanResult(ht, scan);
3331     assertNullResult(result);
3332 
3333   }
3334 
3335   //
3336   // Verifiers
3337   //
3338 
3339   private void assertKey(Cell key, byte [] row, byte [] family,
3340       byte [] qualifier, byte [] value)
3341   throws Exception {
3342     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3343         "Got row [" + Bytes.toString(CellUtil.cloneRow(key)) +"]",
3344         equals(row, CellUtil.cloneRow(key)));
3345     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3346         "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
3347         equals(family, CellUtil.cloneFamily(key)));
3348     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3349         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key)) + "]",
3350         equals(qualifier, CellUtil.cloneQualifier(key)));
3351     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3352         "Got value [" + Bytes.toString(CellUtil.cloneValue(key)) + "]",
3353         equals(value, CellUtil.cloneValue(key)));
3354   }
3355 
3356   static void assertIncrementKey(Cell key, byte [] row, byte [] family,
3357       byte [] qualifier, long value)
3358   throws Exception {
3359     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3360         "Got row [" + Bytes.toString(CellUtil.cloneRow(key)) +"]",
3361         equals(row, CellUtil.cloneRow(key)));
3362     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3363         "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
3364         equals(family, CellUtil.cloneFamily(key)));
3365     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3366         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key)) + "]",
3367         equals(qualifier, CellUtil.cloneQualifier(key)));
3368     assertTrue("Expected value [" + value + "] " +
3369         "Got value [" + Bytes.toLong(CellUtil.cloneValue(key)) + "]",
3370         Bytes.toLong(CellUtil.cloneValue(key)) == value);
3371   }
3372 
3373   private void assertNumKeys(Result result, int n) throws Exception {
3374     assertTrue("Expected " + n + " keys but got " + result.size(),
3375         result.size() == n);
3376   }
3377 
3378   private void assertNResult(Result result, byte [] row,
3379       byte [][] families, byte [][] qualifiers, byte [][] values,
3380       int [][] idxs)
3381   throws Exception {
3382     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3383         "Got row [" + Bytes.toString(result.getRow()) +"]",
3384         equals(row, result.getRow()));
3385     assertTrue("Expected " + idxs.length + " keys but result contains "
3386         + result.size(), result.size() == idxs.length);
3387 
3388     Cell [] keys = result.rawCells();
3389 
3390     for(int i=0;i<keys.length;i++) {
3391       byte [] family = families[idxs[i][0]];
3392       byte [] qualifier = qualifiers[idxs[i][1]];
3393       byte [] value = values[idxs[i][2]];
3394       Cell key = keys[i];
3395 
3396       byte[] famb = CellUtil.cloneFamily(key);
3397       byte[] qualb = CellUtil.cloneQualifier(key);
3398       byte[] valb = CellUtil.cloneValue(key);
3399       assertTrue("(" + i + ") Expected family [" + Bytes.toString(family)
3400           + "] " + "Got family [" + Bytes.toString(famb) + "]",
3401           equals(family, famb));
3402       assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier)
3403           + "] " + "Got qualifier [" + Bytes.toString(qualb) + "]",
3404           equals(qualifier, qualb));
3405       assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] "
3406           + "Got value [" + Bytes.toString(valb) + "]",
3407           equals(value, valb));
3408     }
3409   }
3410 
3411   private void assertNResult(Result result, byte [] row,
3412       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
3413       int start, int end)
3414   throws IOException {
3415     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3416         "Got row [" + Bytes.toString(result.getRow()) +"]",
3417         equals(row, result.getRow()));
3418     int expectedResults = end - start + 1;
3419     assertEquals(expectedResults, result.size());
3420 
3421     Cell[] keys = result.rawCells();
3422 
3423     for (int i=0; i<keys.length; i++) {
3424       byte [] value = values[end-i];
3425       long ts = stamps[end-i];
3426       Cell key = keys[i];
3427 
3428       assertTrue("(" + i + ") Expected family [" + Bytes.toString(family)
3429           + "] " + "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
3430           CellUtil.matchingFamily(key, family));
3431       assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier)
3432           + "] " + "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key))+ "]",
3433           CellUtil.matchingQualifier(key, qualifier));
3434       assertTrue("Expected ts [" + ts + "] " +
3435           "Got ts [" + key.getTimestamp() + "]", ts == key.getTimestamp());
3436       assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] "
3437           + "Got value [" + Bytes.toString(CellUtil.cloneValue(key)) + "]",
3438           CellUtil.matchingValue(key,  value));
3439     }
3440   }
3441 
3442   /**
3443    * Validate that result contains two specified keys, exactly.
3444    * It is assumed key A sorts before key B.
3445    */
3446   private void assertDoubleResult(Result result, byte [] row,
3447       byte [] familyA, byte [] qualifierA, byte [] valueA,
3448       byte [] familyB, byte [] qualifierB, byte [] valueB)
3449   throws Exception {
3450     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3451         "Got row [" + Bytes.toString(result.getRow()) +"]",
3452         equals(row, result.getRow()));
3453     assertTrue("Expected two keys but result contains " + result.size(),
3454         result.size() == 2);
3455     Cell [] kv = result.rawCells();
3456     Cell kvA = kv[0];
3457     assertTrue("(A) Expected family [" + Bytes.toString(familyA) + "] " +
3458         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kvA)) + "]",
3459         equals(familyA, CellUtil.cloneFamily(kvA)));
3460     assertTrue("(A) Expected qualifier [" + Bytes.toString(qualifierA) + "] " +
3461         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kvA)) + "]",
3462         equals(qualifierA, CellUtil.cloneQualifier(kvA)));
3463     assertTrue("(A) Expected value [" + Bytes.toString(valueA) + "] " +
3464         "Got value [" + Bytes.toString(CellUtil.cloneValue(kvA)) + "]",
3465         equals(valueA, CellUtil.cloneValue(kvA)));
3466     Cell kvB = kv[1];
3467     assertTrue("(B) Expected family [" + Bytes.toString(familyB) + "] " +
3468         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kvB)) + "]",
3469         equals(familyB, CellUtil.cloneFamily(kvB)));
3470     assertTrue("(B) Expected qualifier [" + Bytes.toString(qualifierB) + "] " +
3471         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kvB)) + "]",
3472         equals(qualifierB, CellUtil.cloneQualifier(kvB)));
3473     assertTrue("(B) Expected value [" + Bytes.toString(valueB) + "] " +
3474         "Got value [" + Bytes.toString(CellUtil.cloneValue(kvB)) + "]",
3475         equals(valueB, CellUtil.cloneValue(kvB)));
3476   }
3477 
3478   private void assertSingleResult(Result result, byte [] row, byte [] family,
3479       byte [] qualifier, byte [] value)
3480   throws Exception {
3481     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3482         "Got row [" + Bytes.toString(result.getRow()) +"]",
3483         equals(row, result.getRow()));
3484     assertTrue("Expected a single key but result contains " + result.size(),
3485         result.size() == 1);
3486     Cell kv = result.rawCells()[0];
3487     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3488         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kv)) + "]",
3489         equals(family, CellUtil.cloneFamily(kv)));
3490     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3491         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]",
3492         equals(qualifier, CellUtil.cloneQualifier(kv)));
3493     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3494         "Got value [" + Bytes.toString(CellUtil.cloneValue(kv)) + "]",
3495         equals(value, CellUtil.cloneValue(kv)));
3496   }
3497 
3498   private void assertSingleResult(Result result, byte [] row, byte [] family,
3499       byte [] qualifier, long ts, byte [] value)
3500   throws Exception {
3501     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3502         "Got row [" + Bytes.toString(result.getRow()) +"]",
3503         equals(row, result.getRow()));
3504     assertTrue("Expected a single key but result contains " + result.size(),
3505         result.size() == 1);
3506     Cell kv = result.rawCells()[0];
3507     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3508         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kv)) + "]",
3509         equals(family, CellUtil.cloneFamily(kv)));
3510     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3511         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]",
3512         equals(qualifier, CellUtil.cloneQualifier(kv)));
3513     assertTrue("Expected ts [" + ts + "] " +
3514         "Got ts [" + kv.getTimestamp() + "]", ts == kv.getTimestamp());
3515     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3516         "Got value [" + Bytes.toString(CellUtil.cloneValue(kv)) + "]",
3517         equals(value, CellUtil.cloneValue(kv)));
3518   }
3519 
3520   private void assertEmptyResult(Result result) throws Exception {
3521     assertTrue("expected an empty result but result contains " +
3522         result.size() + " keys", result.isEmpty());
3523   }
3524 
3525   private void assertNullResult(Result result) throws Exception {
3526     assertTrue("expected null result but received a non-null result",
3527         result == null);
3528   }
3529 
3530   //
3531   // Helpers
3532   //
3533 
3534   private Result getSingleScanResult(Table ht, Scan scan) throws IOException {
3535     ResultScanner scanner = ht.getScanner(scan);
3536     Result result = scanner.next();
3537     scanner.close();
3538     return result;
3539   }
3540 
3541   private byte [][] makeNAscii(byte [] base, int n) {
3542     if(n > 256) {
3543       return makeNBig(base, n);
3544     }
3545     byte [][] ret = new byte[n][];
3546     for(int i=0;i<n;i++) {
3547       byte [] tail = Bytes.toBytes(Integer.toString(i));
3548       ret[i] = Bytes.add(base, tail);
3549     }
3550     return ret;
3551   }
3552 
3553   private byte [][] makeN(byte [] base, int n) {
3554     if (n > 256) {
3555       return makeNBig(base, n);
3556     }
3557     byte [][] ret = new byte[n][];
3558     for(int i=0;i<n;i++) {
3559       ret[i] = Bytes.add(base, new byte[]{(byte)i});
3560     }
3561     return ret;
3562   }
3563 
3564   private byte [][] makeNBig(byte [] base, int n) {
3565     byte [][] ret = new byte[n][];
3566     for(int i=0;i<n;i++) {
3567       int byteA = (i % 256);
3568       int byteB = (i >> 8);
3569       ret[i] = Bytes.add(base, new byte[]{(byte)byteB,(byte)byteA});
3570     }
3571     return ret;
3572   }
3573 
3574   private long [] makeStamps(int n) {
3575     long [] stamps = new long[n];
3576     for(int i=0;i<n;i++) stamps[i] = i+1;
3577     return stamps;
3578   }
3579 
3580   static boolean equals(byte [] left, byte [] right) {
3581     if (left == null && right == null) return true;
3582     if (left == null && right.length == 0) return true;
3583     if (right == null && left.length == 0) return true;
3584     return Bytes.equals(left, right);
3585   }
3586 
3587   @Test
3588   public void testDuplicateVersions() throws Exception {
3589     byte [] TABLE = Bytes.toBytes("testDuplicateVersions");
3590 
3591     long [] STAMPS = makeStamps(20);
3592     byte [][] VALUES = makeNAscii(VALUE, 20);
3593 
3594     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3595     TEST_UTIL.waitTableAvailable(TABLE, 10000);
3596 
3597     // Insert 4 versions of same column
3598     Put put = new Put(ROW);
3599     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3600     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3601     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3602     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3603     ht.put(put);
3604 
3605     // Verify we can get each one properly
3606     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3607     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3608     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3609     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3610     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3611     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3612     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3613     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3614 
3615     // Verify we don't accidentally get others
3616     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3617     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3618     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3619     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3620     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3621     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3622 
3623     // Ensure maxVersions in query is respected
3624     Get get = new Get(ROW);
3625     get.addColumn(FAMILY, QUALIFIER);
3626     get.setMaxVersions(2);
3627     Result result = ht.get(get);
3628     assertNResult(result, ROW, FAMILY, QUALIFIER,
3629         new long [] {STAMPS[4], STAMPS[5]},
3630         new byte[][] {VALUES[4], VALUES[5]},
3631         0, 1);
3632 
3633     Scan scan = new Scan(ROW);
3634     scan.addColumn(FAMILY, QUALIFIER);
3635     scan.setMaxVersions(2);
3636     result = getSingleScanResult(ht, scan);
3637     assertNResult(result, ROW, FAMILY, QUALIFIER,
3638         new long [] {STAMPS[4], STAMPS[5]},
3639         new byte[][] {VALUES[4], VALUES[5]},
3640         0, 1);
3641 
3642     // Flush and redo
3643 
3644     TEST_UTIL.flush();
3645 
3646     // Verify we can get each one properly
3647     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3648     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3649     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3650     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3651     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3652     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3653     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3654     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3655 
3656     // Verify we don't accidentally get others
3657     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3658     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3659     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3660     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3661     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3662     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3663 
3664     // Ensure maxVersions in query is respected
3665     get = new Get(ROW);
3666     get.addColumn(FAMILY, QUALIFIER);
3667     get.setMaxVersions(2);
3668     result = ht.get(get);
3669     assertNResult(result, ROW, FAMILY, QUALIFIER,
3670         new long [] {STAMPS[4], STAMPS[5]},
3671         new byte[][] {VALUES[4], VALUES[5]},
3672         0, 1);
3673 
3674     scan = new Scan(ROW);
3675     scan.addColumn(FAMILY, QUALIFIER);
3676     scan.setMaxVersions(2);
3677     result = getSingleScanResult(ht, scan);
3678     assertNResult(result, ROW, FAMILY, QUALIFIER,
3679         new long [] {STAMPS[4], STAMPS[5]},
3680         new byte[][] {VALUES[4], VALUES[5]},
3681         0, 1);
3682 
3683 
3684     // Add some memstore and retest
3685 
3686     // Insert 4 more versions of same column and a dupe
3687     put = new Put(ROW);
3688     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
3689     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3690     put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
3691     put.add(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3692     put.add(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
3693     ht.put(put);
3694 
3695     // Ensure maxVersions in query is respected
3696     get = new Get(ROW);
3697     get.addColumn(FAMILY, QUALIFIER);
3698     get.setMaxVersions(7);
3699     result = ht.get(get);
3700     assertNResult(result, ROW, FAMILY, QUALIFIER,
3701         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3702         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3703         0, 6);
3704 
3705     scan = new Scan(ROW);
3706     scan.addColumn(FAMILY, QUALIFIER);
3707     scan.setMaxVersions(7);
3708     result = getSingleScanResult(ht, scan);
3709     assertNResult(result, ROW, FAMILY, QUALIFIER,
3710         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3711         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3712         0, 6);
3713 
3714     get = new Get(ROW);
3715     get.setMaxVersions(7);
3716     result = ht.get(get);
3717     assertNResult(result, ROW, FAMILY, QUALIFIER,
3718         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3719         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3720         0, 6);
3721 
3722     scan = new Scan(ROW);
3723     scan.setMaxVersions(7);
3724     result = getSingleScanResult(ht, scan);
3725     assertNResult(result, ROW, FAMILY, QUALIFIER,
3726         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3727         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3728         0, 6);
3729 
3730     // Verify we can get each one properly
3731     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3732     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3733     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3734     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3735     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3736     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3737     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3738     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3739 
3740     // Verify we don't accidentally get others
3741     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3742     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
3743     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3744     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
3745 
3746     // Ensure maxVersions of table is respected
3747 
3748     TEST_UTIL.flush();
3749 
3750     // Insert 4 more versions of same column and a dupe
3751     put = new Put(ROW);
3752     put.add(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
3753     put.add(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
3754     put.add(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
3755     put.add(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
3756     ht.put(put);
3757 
3758     get = new Get(ROW);
3759     get.addColumn(FAMILY, QUALIFIER);
3760     get.setMaxVersions(Integer.MAX_VALUE);
3761     result = ht.get(get);
3762     assertNResult(result, ROW, FAMILY, QUALIFIER,
3763         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
3764         new byte[][] {VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
3765         0, 9);
3766 
3767     scan = new Scan(ROW);
3768     scan.addColumn(FAMILY, QUALIFIER);
3769     scan.setMaxVersions(Integer.MAX_VALUE);
3770     result = getSingleScanResult(ht, scan);
3771     assertNResult(result, ROW, FAMILY, QUALIFIER,
3772         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
3773         new byte[][] {VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
3774         0, 9);
3775 
3776     // Delete a version in the memstore and a version in a storefile
3777     Delete delete = new Delete(ROW);
3778     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[11]);
3779     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[7]);
3780     ht.delete(delete);
3781 
3782     // Test that it's gone
3783     get = new Get(ROW);
3784     get.addColumn(FAMILY, QUALIFIER);
3785     get.setMaxVersions(Integer.MAX_VALUE);
3786     result = ht.get(get);
3787     assertNResult(result, ROW, FAMILY, QUALIFIER,
3788         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
3789         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
3790         0, 9);
3791 
3792     scan = new Scan(ROW);
3793     scan.addColumn(FAMILY, QUALIFIER);
3794     scan.setMaxVersions(Integer.MAX_VALUE);
3795     result = getSingleScanResult(ht, scan);
3796     assertNResult(result, ROW, FAMILY, QUALIFIER,
3797         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
3798         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
3799         0, 9);
3800   }
3801 
3802   @Test
3803   public void testUpdates() throws Exception {
3804 
3805     byte [] TABLE = Bytes.toBytes("testUpdates");
3806     Table hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3807     TEST_UTIL.waitTableAvailable(TABLE, 10000);
3808 
3809     // Write a column with values at timestamp 1, 2 and 3
3810     byte[] row = Bytes.toBytes("row1");
3811     byte[] qualifier = Bytes.toBytes("myCol");
3812     Put put = new Put(row);
3813     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3814     hTable.put(put);
3815 
3816     put = new Put(row);
3817     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3818     hTable.put(put);
3819 
3820     put = new Put(row);
3821     put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3822     hTable.put(put);
3823 
3824     Get get = new Get(row);
3825     get.addColumn(FAMILY, qualifier);
3826     get.setMaxVersions();
3827 
3828     // Check that the column indeed has the right values at timestamps 1 and
3829     // 2
3830     Result result = hTable.get(get);
3831     NavigableMap<Long, byte[]> navigableMap =
3832         result.getMap().get(FAMILY).get(qualifier);
3833     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3834     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3835 
3836     // Update the value at timestamp 1
3837     put = new Put(row);
3838     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3839     hTable.put(put);
3840 
3841     // Update the value at timestamp 2
3842     put = new Put(row);
3843     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3844     hTable.put(put);
3845 
3846     // Check that the values at timestamp 2 and 1 got updated
3847     result = hTable.get(get);
3848     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3849     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3850     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3851   }
3852 
3853   @Test
3854   public void testUpdatesWithMajorCompaction() throws Exception {
3855 
3856     TableName TABLE = TableName.valueOf("testUpdatesWithMajorCompaction");
3857     Table hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3858     TEST_UTIL.waitTableAvailable(TABLE, 10000);
3859 
3860     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
3861 
3862     // Write a column with values at timestamp 1, 2 and 3
3863     byte[] row = Bytes.toBytes("row2");
3864     byte[] qualifier = Bytes.toBytes("myCol");
3865     Put put = new Put(row);
3866     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3867     hTable.put(put);
3868 
3869     put = new Put(row);
3870     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3871     hTable.put(put);
3872 
3873     put = new Put(row);
3874     put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3875     hTable.put(put);
3876 
3877     Get get = new Get(row);
3878     get.addColumn(FAMILY, qualifier);
3879     get.setMaxVersions();
3880 
3881     // Check that the column indeed has the right values at timestamps 1 and
3882     // 2
3883     Result result = hTable.get(get);
3884     NavigableMap<Long, byte[]> navigableMap =
3885         result.getMap().get(FAMILY).get(qualifier);
3886     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3887     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3888 
3889     // Trigger a major compaction
3890     admin.flush(TABLE);
3891     admin.majorCompact(TABLE);
3892     Thread.sleep(6000);
3893 
3894     // Update the value at timestamp 1
3895     put = new Put(row);
3896     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3897     hTable.put(put);
3898 
3899     // Update the value at timestamp 2
3900     put = new Put(row);
3901     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3902     hTable.put(put);
3903 
3904     // Trigger a major compaction
3905     admin.flush(TABLE);
3906     admin.majorCompact(TABLE);
3907     Thread.sleep(6000);
3908 
3909     // Check that the values at timestamp 2 and 1 got updated
3910     result = hTable.get(get);
3911     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3912     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3913     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3914   }
3915 
3916   @Test
3917   public void testMajorCompactionBetweenTwoUpdates() throws Exception {
3918 
3919     String tableName = "testMajorCompactionBetweenTwoUpdates";
3920     byte [] TABLE = Bytes.toBytes(tableName);
3921     Table hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3922     TEST_UTIL.waitTableAvailable(TABLE, 10000);
3923     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
3924 
3925     // Write a column with values at timestamp 1, 2 and 3
3926     byte[] row = Bytes.toBytes("row3");
3927     byte[] qualifier = Bytes.toBytes("myCol");
3928     Put put = new Put(row);
3929     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3930     hTable.put(put);
3931 
3932     put = new Put(row);
3933     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3934     hTable.put(put);
3935 
3936     put = new Put(row);
3937     put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3938     hTable.put(put);
3939 
3940     Get get = new Get(row);
3941     get.addColumn(FAMILY, qualifier);
3942     get.setMaxVersions();
3943 
3944     // Check that the column indeed has the right values at timestamps 1 and
3945     // 2
3946     Result result = hTable.get(get);
3947     NavigableMap<Long, byte[]> navigableMap =
3948         result.getMap().get(FAMILY).get(qualifier);
3949     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3950     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3951 
3952     // Trigger a major compaction
3953     admin.flush(tableName);
3954     admin.majorCompact(tableName);
3955     Thread.sleep(6000);
3956 
3957     // Update the value at timestamp 1
3958     put = new Put(row);
3959     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3960     hTable.put(put);
3961 
3962     // Trigger a major compaction
3963     admin.flush(tableName);
3964     admin.majorCompact(tableName);
3965     Thread.sleep(6000);
3966 
3967     // Update the value at timestamp 2
3968     put = new Put(row);
3969     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3970     hTable.put(put);
3971 
3972     // Trigger a major compaction
3973     admin.flush(tableName);
3974     admin.majorCompact(tableName);
3975     Thread.sleep(6000);
3976 
3977     // Check that the values at timestamp 2 and 1 got updated
3978     result = hTable.get(get);
3979     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3980 
3981     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3982     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3983   }
3984 
3985   @Test
3986   public void testGet_EmptyTable() throws IOException, InterruptedException {
3987     Table table = TEST_UTIL.createTable(TableName.valueOf("testGet_EmptyTable"), FAMILY);
3988     TEST_UTIL.waitTableAvailable(TableName.valueOf("testGet_EmptyTable"), 10000);
3989     Get get = new Get(ROW);
3990     get.addFamily(FAMILY);
3991     Result r = table.get(get);
3992     assertTrue(r.isEmpty());
3993   }
3994 
3995   @Test
3996   public void testGet_NullQualifier() throws IOException, InterruptedException {
3997     Table table = TEST_UTIL.createTable(TableName.valueOf("testGet_NullQualifier"), FAMILY);
3998     TEST_UTIL.waitTableAvailable(TableName.valueOf("testGet_NullQualifier"), 10000);
3999     Put put = new Put(ROW);
4000     put.add(FAMILY, QUALIFIER, VALUE);
4001     table.put(put);
4002 
4003     put = new Put(ROW);
4004     put.add(FAMILY, null, VALUE);
4005     table.put(put);
4006     LOG.info("Row put");
4007 
4008     Get get = new Get(ROW);
4009     get.addColumn(FAMILY, null);
4010     Result r = table.get(get);
4011     assertEquals(1, r.size());
4012 
4013     get = new Get(ROW);
4014     get.addFamily(FAMILY);
4015     r = table.get(get);
4016     assertEquals(2, r.size());
4017   }
4018 
4019   @Test
4020   public void testGet_NonExistentRow() throws IOException, InterruptedException {
4021     Table table = TEST_UTIL.createTable(TableName.valueOf("testGet_NonExistentRow"), FAMILY);
4022     TEST_UTIL.waitTableAvailable(TableName.valueOf("testGet_NonExistentRow"), 10000);
4023     Put put = new Put(ROW);
4024     put.add(FAMILY, QUALIFIER, VALUE);
4025     table.put(put);
4026     LOG.info("Row put");
4027 
4028     Get get = new Get(ROW);
4029     get.addFamily(FAMILY);
4030     Result r = table.get(get);
4031     assertFalse(r.isEmpty());
4032     System.out.println("Row retrieved successfully");
4033 
4034     byte [] missingrow = Bytes.toBytes("missingrow");
4035     get = new Get(missingrow);
4036     get.addFamily(FAMILY);
4037     r = table.get(get);
4038     assertTrue(r.isEmpty());
4039     LOG.info("Row missing as it should be");
4040   }
4041 
4042   @Test
4043   public void testPut() throws IOException, InterruptedException {
4044     final byte [] CONTENTS_FAMILY = Bytes.toBytes("contents");
4045     final byte [] SMALL_FAMILY = Bytes.toBytes("smallfam");
4046     final byte [] row1 = Bytes.toBytes("row1");
4047     final byte [] row2 = Bytes.toBytes("row2");
4048     final byte [] value = Bytes.toBytes("abcd");
4049     Table table = TEST_UTIL.createTable(Bytes.toBytes("testPut"),
4050       new byte [][] {CONTENTS_FAMILY, SMALL_FAMILY});
4051     TEST_UTIL.waitTableAvailable(Bytes.toBytes("testPut"), 10000);
4052     Put put = new Put(row1);
4053     put.add(CONTENTS_FAMILY, null, value);
4054     table.put(put);
4055 
4056     put = new Put(row2);
4057     put.add(CONTENTS_FAMILY, null, value);
4058 
4059     assertEquals(put.size(), 1);
4060     assertEquals(put.getFamilyCellMap().get(CONTENTS_FAMILY).size(), 1);
4061 
4062     // KeyValue v1 expectation.  Cast for now until we go all Cell all the time. TODO
4063     KeyValue kv = (KeyValue)put.getFamilyCellMap().get(CONTENTS_FAMILY).get(0);
4064 
4065     assertTrue(Bytes.equals(kv.getFamily(), CONTENTS_FAMILY));
4066     // will it return null or an empty byte array?
4067     assertTrue(Bytes.equals(kv.getQualifier(), new byte[0]));
4068 
4069     assertTrue(Bytes.equals(kv.getValue(), value));
4070 
4071     table.put(put);
4072 
4073     Scan scan = new Scan();
4074     scan.addColumn(CONTENTS_FAMILY, null);
4075     ResultScanner scanner = table.getScanner(scan);
4076     for (Result r : scanner) {
4077       for(Cell key : r.rawCells()) {
4078         System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString());
4079       }
4080     }
4081   }
4082 
4083   @Test
4084   public void testPutNoCF() throws IOException, InterruptedException {
4085     final byte[] BAD_FAM = Bytes.toBytes("BAD_CF");
4086     final byte[] VAL = Bytes.toBytes(100);
4087     Table table = TEST_UTIL.createTable(Bytes.toBytes("testPutNoCF"), FAMILY);
4088     TEST_UTIL.waitTableAvailable(Bytes.toBytes("testPutNoCF"), 10000);
4089 
4090     boolean caughtNSCFE = false;
4091 
4092     try {
4093       Put p = new Put(ROW);
4094       p.add(BAD_FAM, QUALIFIER, VAL);
4095       table.put(p);
4096     } catch (RetriesExhaustedWithDetailsException e) {
4097       caughtNSCFE = e.getCause(0) instanceof NoSuchColumnFamilyException;
4098     }
4099     assertTrue("Should throw NoSuchColumnFamilyException", caughtNSCFE);
4100 
4101   }
4102 
4103   @Test
4104   public void testRowsPut() throws IOException, InterruptedException {
4105     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
4106     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
4107     final int NB_BATCH_ROWS = 10;
4108     final byte[] value = Bytes.toBytes("abcd");
4109     Table table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPut"),
4110       new byte[][] {CONTENTS_FAMILY, SMALL_FAMILY });
4111     TEST_UTIL.waitTableAvailable(Bytes.toBytes("testRowsPut"), 10000);
4112     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
4113     for (int i = 0; i < NB_BATCH_ROWS; i++) {
4114       byte[] row = Bytes.toBytes("row" + i);
4115       Put put = new Put(row);
4116       put.setDurability(Durability.SKIP_WAL);
4117       put.add(CONTENTS_FAMILY, null, value);
4118       rowsUpdate.add(put);
4119     }
4120     table.put(rowsUpdate);
4121     Scan scan = new Scan();
4122     scan.addFamily(CONTENTS_FAMILY);
4123     ResultScanner scanner = table.getScanner(scan);
4124     int nbRows = 0;
4125     for (@SuppressWarnings("unused")
4126     Result row : scanner)
4127       nbRows++;
4128     assertEquals(NB_BATCH_ROWS, nbRows);
4129   }
4130 
4131   @Test
4132   public void testRowsPutBufferedOneFlush() throws IOException, InterruptedException {
4133     final byte [] CONTENTS_FAMILY = Bytes.toBytes("contents");
4134     final byte [] SMALL_FAMILY = Bytes.toBytes("smallfam");
4135     final byte [] value = Bytes.toBytes("abcd");
4136     final int NB_BATCH_ROWS = 10;
4137     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPutBufferedOneFlush"),
4138       new byte [][] {CONTENTS_FAMILY, SMALL_FAMILY});
4139     TEST_UTIL.waitTableAvailable(Bytes.toBytes("testRowsPutBufferedOneFlush"),
4140         10000);
4141     table.setAutoFlush(false);
4142     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
4143     for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
4144       byte[] row = Bytes.toBytes("row" + i);
4145       Put put = new Put(row);
4146       put.setDurability(Durability.SKIP_WAL);
4147       put.add(CONTENTS_FAMILY, null, value);
4148       rowsUpdate.add(put);
4149     }
4150     table.put(rowsUpdate);
4151 
4152     Scan scan = new Scan();
4153     scan.addFamily(CONTENTS_FAMILY);
4154     ResultScanner scanner = table.getScanner(scan);
4155     int nbRows = 0;
4156     for (@SuppressWarnings("unused")
4157     Result row : scanner)
4158       nbRows++;
4159     assertEquals(0, nbRows);
4160     scanner.close();
4161 
4162     table.flushCommits();
4163 
4164     scan = new Scan();
4165     scan.addFamily(CONTENTS_FAMILY);
4166     scanner = table.getScanner(scan);
4167     nbRows = 0;
4168     for (@SuppressWarnings("unused")
4169     Result row : scanner)
4170       nbRows++;
4171     assertEquals(NB_BATCH_ROWS * 10, nbRows);
4172     table.close();
4173   }
4174 
4175   @Test
4176   public void testRowsPutBufferedManyManyFlushes() throws IOException, InterruptedException {
4177     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
4178     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
4179     final byte[] value = Bytes.toBytes("abcd");
4180     final int NB_BATCH_ROWS = 10;
4181     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPutBufferedManyManyFlushes"),
4182       new byte[][] {CONTENTS_FAMILY, SMALL_FAMILY });
4183     TEST_UTIL.waitTableAvailable(Bytes.toBytes("testRowsPutBufferedManyManyFlushes"),
4184         10000);
4185     table.setWriteBufferSize(10);
4186     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
4187     for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
4188       byte[] row = Bytes.toBytes("row" + i);
4189       Put put = new Put(row);
4190       put.setDurability(Durability.SKIP_WAL);
4191       put.add(CONTENTS_FAMILY, null, value);
4192       rowsUpdate.add(put);
4193     }
4194     table.put(rowsUpdate);
4195 
4196     Scan scan = new Scan();
4197     scan.addFamily(CONTENTS_FAMILY);
4198     ResultScanner scanner = table.getScanner(scan);
4199     int nbRows = 0;
4200     for (@SuppressWarnings("unused")
4201     Result row : scanner)
4202       nbRows++;
4203     assertEquals(NB_BATCH_ROWS * 10, nbRows);
4204   }
4205 
4206   @Test
4207   public void testAddKeyValue() throws IOException {
4208     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
4209     final byte[] value = Bytes.toBytes("abcd");
4210     final byte[] row1 = Bytes.toBytes("row1");
4211     final byte[] row2 = Bytes.toBytes("row2");
4212     byte[] qualifier = Bytes.toBytes("qf1");
4213     Put put = new Put(row1);
4214 
4215     // Adding KeyValue with the same row
4216     KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value);
4217     boolean ok = true;
4218     try {
4219       put.add(kv);
4220     } catch (IOException e) {
4221       ok = false;
4222     }
4223     assertEquals(true, ok);
4224 
4225     // Adding KeyValue with the different row
4226     kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value);
4227     ok = false;
4228     try {
4229       put.add(kv);
4230     } catch (IOException e) {
4231       ok = true;
4232     }
4233     assertEquals(true, ok);
4234   }
4235 
4236   /**
4237    * test for HBASE-737
4238    * @throws IOException
4239    */
4240   @Test
4241   public void testHBase737() throws IOException, InterruptedException {
4242     final byte [] FAM1 = Bytes.toBytes("fam1");
4243     final byte [] FAM2 = Bytes.toBytes("fam2");
4244     // Open table
4245     Table table = TEST_UTIL.createTable(Bytes.toBytes("testHBase737"),
4246       new byte [][] {FAM1, FAM2});
4247     TEST_UTIL.waitTableAvailable(Bytes.toBytes("testHBase737"), 10000);
4248     // Insert some values
4249     Put put = new Put(ROW);
4250     put.add(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg"));
4251     table.put(put);
4252     try {
4253       Thread.sleep(1000);
4254     } catch (InterruptedException i) {
4255       //ignore
4256     }
4257 
4258     put = new Put(ROW);
4259     put.add(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456"));
4260     table.put(put);
4261 
4262     try {
4263       Thread.sleep(1000);
4264     } catch (InterruptedException i) {
4265       //ignore
4266     }
4267 
4268     put = new Put(ROW);
4269     put.add(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop"));
4270     table.put(put);
4271 
4272     long times[] = new long[3];
4273 
4274     // First scan the memstore
4275 
4276     Scan scan = new Scan();
4277     scan.addFamily(FAM1);
4278     scan.addFamily(FAM2);
4279     ResultScanner s = table.getScanner(scan);
4280     try {
4281       int index = 0;
4282       Result r = null;
4283       while ((r = s.next()) != null) {
4284         for(Cell key : r.rawCells()) {
4285           times[index++] = key.getTimestamp();
4286         }
4287       }
4288     } finally {
4289       s.close();
4290     }
4291     for (int i = 0; i < times.length - 1; i++) {
4292       for (int j = i + 1; j < times.length; j++) {
4293         assertTrue(times[j] > times[i]);
4294       }
4295     }
4296 
4297     // Flush data to disk and try again
4298     TEST_UTIL.flush();
4299 
4300     // Reset times
4301     for(int i=0;i<times.length;i++) {
4302       times[i] = 0;
4303     }
4304 
4305     try {
4306       Thread.sleep(1000);
4307     } catch (InterruptedException i) {
4308       //ignore
4309     }
4310     scan = new Scan();
4311     scan.addFamily(FAM1);
4312     scan.addFamily(FAM2);
4313     s = table.getScanner(scan);
4314     try {
4315       int index = 0;
4316       Result r = null;
4317       while ((r = s.next()) != null) {
4318         for(Cell key : r.rawCells()) {
4319           times[index++] = key.getTimestamp();
4320         }
4321       }
4322     } finally {
4323       s.close();
4324     }
4325     for (int i = 0; i < times.length - 1; i++) {
4326       for (int j = i + 1; j < times.length; j++) {
4327         assertTrue(times[j] > times[i]);
4328       }
4329     }
4330   }
4331 
4332   @Test
4333   public void testListTables() throws IOException, InterruptedException {
4334     TableName t1 = TableName.valueOf("testListTables1");
4335     TableName t2 = TableName.valueOf("testListTables2");
4336     TableName t3 = TableName.valueOf("testListTables3");
4337     TableName [] tables = new TableName[] { t1, t2, t3 };
4338     for (int i = 0; i < tables.length; i++) {
4339       TEST_UTIL.createTable(tables[i], FAMILY);
4340       TEST_UTIL.waitTableAvailable(tables[i], 10000);
4341     }
4342     Admin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
4343     HTableDescriptor[] ts = admin.listTables();
4344     HashSet<HTableDescriptor> result = new HashSet<HTableDescriptor>(ts.length);
4345     Collections.addAll(result, ts);
4346     int size = result.size();
4347     assertTrue(size >= tables.length);
4348     for (int i = 0; i < tables.length && i < size; i++) {
4349       boolean found = false;
4350       for (int j = 0; j < ts.length; j++) {
4351         if (ts[j].getTableName().equals(tables[i])) {
4352           found = true;
4353           break;
4354         }
4355       }
4356       assertTrue("Not found: " + tables[i], found);
4357     }
4358   }
4359 
4360   /**
4361    * creates an HTable for tableName using an unmanaged HConnection.
4362    *
4363    * @param tableName - table to create
4364    * @return the created HTable object
4365    * @throws IOException
4366    */
4367   HTable createUnmangedHConnectionHTable(final TableName tableName) throws IOException,
4368       InterruptedException {
4369     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
4370     TEST_UTIL.waitTableAvailable(tableName, 10000);
4371     HConnection conn = HConnectionManager.createConnection(TEST_UTIL.getConfiguration());
4372     return (HTable)conn.getTable(tableName);
4373   }
4374 
4375   /**
4376    * simple test that just executes parts of the client
4377    * API that accept a pre-created HConnection instance
4378    *
4379    * @throws IOException
4380    */
4381   @Test
4382   public void testUnmanagedHConnection() throws IOException, InterruptedException {
4383     final TableName tableName = TableName.valueOf("testUnmanagedHConnection");
4384     HTable t = createUnmangedHConnectionHTable(tableName);
4385     HBaseAdmin ha = new HBaseAdmin(t.getConnection());
4386     assertTrue(ha.tableExists(tableName));
4387     assertTrue(t.get(new Get(ROW)).isEmpty());
4388     ha.close();
4389   }
4390 
4391   /**
4392    * test of that unmanaged HConnections are able to reconnect
4393    * properly (see HBASE-5058)
4394    *
4395    * @throws Exception
4396    */
4397   @Test
4398   public void testUnmanagedHConnectionReconnect() throws Exception {
4399     final TableName tableName = TableName.valueOf("testUnmanagedHConnectionReconnect");
4400     HTable t = createUnmangedHConnectionHTable(tableName);
4401     Connection conn = t.getConnection();
4402     try (HBaseAdmin ha = new HBaseAdmin(conn)) {
4403       assertTrue(ha.tableExists(tableName));
4404       assertTrue(t.get(new Get(ROW)).isEmpty());
4405     }
4406 
4407     // stop the master
4408     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
4409     cluster.stopMaster(0, false);
4410     cluster.waitOnMaster(0);
4411 
4412     // start up a new master
4413     cluster.startMaster();
4414     assertTrue(cluster.waitForActiveAndReadyMaster());
4415 
4416     // test that the same unmanaged connection works with a new
4417     // HBaseAdmin and can connect to the new master;
4418     try (HBaseAdmin newAdmin = new HBaseAdmin(conn)) {
4419       assertTrue(newAdmin.tableExists(tableName));
4420       assertTrue(newAdmin.getClusterStatus().getServersSize() == SLAVES);
4421     }
4422   }
4423 
4424   @Test
4425   public void testMiscHTableStuff() throws IOException, InterruptedException {
4426     final TableName tableAname = TableName.valueOf("testMiscHTableStuffA");
4427     final TableName tableBname = TableName.valueOf("testMiscHTableStuffB");
4428     final byte[] attrName = Bytes.toBytes("TESTATTR");
4429     final byte[] attrValue = Bytes.toBytes("somevalue");
4430     byte[] value = Bytes.toBytes("value");
4431 
4432     Table a = TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY);
4433     TEST_UTIL.waitTableAvailable(tableAname, 10000);
4434     Table b = TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY);
4435     TEST_UTIL.waitTableAvailable(tableBname, 10000);
4436     Put put = new Put(ROW);
4437     put.add(HConstants.CATALOG_FAMILY, null, value);
4438     a.put(put);
4439 
4440     // open a new connection to A and a connection to b
4441     Table newA = new HTable(TEST_UTIL.getConfiguration(), tableAname);
4442 
4443     // copy data from A to B
4444     Scan scan = new Scan();
4445     scan.addFamily(HConstants.CATALOG_FAMILY);
4446     ResultScanner s = newA.getScanner(scan);
4447     try {
4448       for (Result r : s) {
4449         put = new Put(r.getRow());
4450         put.setDurability(Durability.SKIP_WAL);
4451         for (Cell kv : r.rawCells()) {
4452           put.add(kv);
4453         }
4454         b.put(put);
4455       }
4456     } finally {
4457       s.close();
4458     }
4459 
4460     // Opening a new connection to A will cause the tables to be reloaded
4461     Table anotherA = new HTable(TEST_UTIL.getConfiguration(), tableAname);
4462     Get get = new Get(ROW);
4463     get.addFamily(HConstants.CATALOG_FAMILY);
4464     anotherA.get(get);
4465 
4466     // We can still access A through newA because it has the table information
4467     // cached. And if it needs to recalibrate, that will cause the information
4468     // to be reloaded.
4469 
4470     // Test user metadata
4471     Admin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
4472     // make a modifiable descriptor
4473     HTableDescriptor desc = new HTableDescriptor(a.getTableDescriptor());
4474     // offline the table
4475     admin.disableTable(tableAname);
4476     // add a user attribute to HTD
4477     desc.setValue(attrName, attrValue);
4478     // add a user attribute to HCD
4479     for (HColumnDescriptor c : desc.getFamilies())
4480       c.setValue(attrName, attrValue);
4481     // update metadata for all regions of this table
4482     admin.modifyTable(tableAname, desc);
4483     // enable the table
4484     admin.enableTable(tableAname);
4485 
4486     // Test that attribute changes were applied
4487     desc = a.getTableDescriptor();
4488     assertEquals("wrong table descriptor returned", desc.getTableName(), tableAname);
4489     // check HTD attribute
4490     value = desc.getValue(attrName);
4491     assertFalse("missing HTD attribute value", value == null);
4492     assertFalse("HTD attribute value is incorrect",
4493       Bytes.compareTo(value, attrValue) != 0);
4494     // check HCD attribute
4495     for (HColumnDescriptor c : desc.getFamilies()) {
4496       value = c.getValue(attrName);
4497       assertFalse("missing HCD attribute value", value == null);
4498       assertFalse("HCD attribute value is incorrect",
4499         Bytes.compareTo(value, attrValue) != 0);
4500     }
4501   }
4502 
4503   @Test
4504   public void testGetClosestRowBefore() throws IOException, InterruptedException {
4505     final TableName tableAname = TableName.valueOf("testGetClosestRowBefore");
4506     final byte[] firstRow = Bytes.toBytes("row111");
4507     final byte[] secondRow = Bytes.toBytes("row222");
4508     final byte[] thirdRow = Bytes.toBytes("row333");
4509     final byte[] forthRow = Bytes.toBytes("row444");
4510     final byte[] beforeFirstRow = Bytes.toBytes("row");
4511     final byte[] beforeSecondRow = Bytes.toBytes("row22");
4512     final byte[] beforeThirdRow = Bytes.toBytes("row33");
4513     final byte[] beforeForthRow = Bytes.toBytes("row44");
4514 
4515     HTable table =
4516         TEST_UTIL.createTable(tableAname,
4517           new byte[][] { HConstants.CATALOG_FAMILY, Bytes.toBytes("info2") }, 1,
4518             1024);
4519     TEST_UTIL.waitTableAvailable(tableAname, 10000);
4520     // set block size to 64 to making 2 kvs into one block, bypassing the walkForwardInSingleRow
4521     // in Store.rowAtOrBeforeFromStoreFile
4522     String regionName = table.getRegionLocations().firstKey().getEncodedName();
4523     Region region =
4524         TEST_UTIL.getRSForFirstRegionInTable(tableAname).getFromOnlineRegions(regionName);
4525     Put put1 = new Put(firstRow);
4526     Put put2 = new Put(secondRow);
4527     Put put3 = new Put(thirdRow);
4528     Put put4 = new Put(forthRow);
4529     byte[] one = new byte[] { 1 };
4530     byte[] two = new byte[] { 2 };
4531     byte[] three = new byte[] { 3 };
4532     byte[] four = new byte[] { 4 };
4533 
4534     put1.add(HConstants.CATALOG_FAMILY, null, one);
4535     put2.add(HConstants.CATALOG_FAMILY, null, two);
4536     put3.add(HConstants.CATALOG_FAMILY, null, three);
4537     put4.add(HConstants.CATALOG_FAMILY, null, four);
4538     table.put(put1);
4539     table.put(put2);
4540     table.put(put3);
4541     table.put(put4);
4542     region.flush(true);
4543     Result result = null;
4544 
4545     // Test before first that null is returned
4546     result = table.getRowOrBefore(beforeFirstRow, HConstants.CATALOG_FAMILY);
4547     assertTrue(result == null);
4548 
4549     // Test at first that first is returned
4550     result = table.getRowOrBefore(firstRow, HConstants.CATALOG_FAMILY);
4551     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4552     assertTrue(Bytes.equals(result.getRow(), firstRow));
4553     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
4554 
4555     // Test in between first and second that first is returned
4556     result = table.getRowOrBefore(beforeSecondRow, HConstants.CATALOG_FAMILY);
4557     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4558     assertTrue(Bytes.equals(result.getRow(), firstRow));
4559     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
4560 
4561     // Test at second make sure second is returned
4562     result = table.getRowOrBefore(secondRow, HConstants.CATALOG_FAMILY);
4563     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4564     assertTrue(Bytes.equals(result.getRow(), secondRow));
4565     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));
4566 
4567     // Test in second and third, make sure second is returned
4568     result = table.getRowOrBefore(beforeThirdRow, HConstants.CATALOG_FAMILY);
4569     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4570     assertTrue(Bytes.equals(result.getRow(), secondRow));
4571     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));
4572 
4573     // Test at third make sure third is returned
4574     result = table.getRowOrBefore(thirdRow, HConstants.CATALOG_FAMILY);
4575     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4576     assertTrue(Bytes.equals(result.getRow(), thirdRow));
4577     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));
4578 
4579     // Test in third and forth, make sure third is returned
4580     result = table.getRowOrBefore(beforeForthRow, HConstants.CATALOG_FAMILY);
4581     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4582     assertTrue(Bytes.equals(result.getRow(), thirdRow));
4583     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));
4584 
4585     // Test at forth make sure forth is returned
4586     result = table.getRowOrBefore(forthRow, HConstants.CATALOG_FAMILY);
4587     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4588     assertTrue(Bytes.equals(result.getRow(), forthRow));
4589     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));
4590 
4591     // Test after forth make sure forth is returned
4592     result = table.getRowOrBefore(Bytes.add(forthRow, one), HConstants.CATALOG_FAMILY);
4593     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4594     assertTrue(Bytes.equals(result.getRow(), forthRow));
4595     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));
4596 
4597     table.close();
4598   }
4599 
4600   /**
4601    * For HBASE-2156
4602    * @throws Exception
4603    */
4604   @Test
4605   public void testScanVariableReuse() throws Exception {
4606     Scan scan = new Scan();
4607     scan.addFamily(FAMILY);
4608     scan.addColumn(FAMILY, ROW);
4609 
4610     assertTrue(scan.getFamilyMap().get(FAMILY).size() == 1);
4611 
4612     scan = new Scan();
4613     scan.addFamily(FAMILY);
4614 
4615     assertTrue(scan.getFamilyMap().get(FAMILY) == null);
4616     assertTrue(scan.getFamilyMap().containsKey(FAMILY));
4617   }
4618 
4619   @Test
4620   public void testMultiRowMutation() throws Exception {
4621     LOG.info("Starting testMultiRowMutation");
4622     final TableName TABLENAME = TableName.valueOf("testMultiRowMutation");
4623     final byte [] ROW1 = Bytes.toBytes("testRow1");
4624 
4625     Table t = TEST_UTIL.createTable(TABLENAME, FAMILY);
4626     TEST_UTIL.waitTableAvailable(TABLENAME, 10000);
4627     Put p = new Put(ROW);
4628     p.add(FAMILY, QUALIFIER, VALUE);
4629     MutationProto m1 = ProtobufUtil.toMutation(MutationType.PUT, p);
4630 
4631     p = new Put(ROW1);
4632     p.add(FAMILY, QUALIFIER, VALUE);
4633     MutationProto m2 = ProtobufUtil.toMutation(MutationType.PUT, p);
4634 
4635     MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
4636     mrmBuilder.addMutationRequest(m1);
4637     mrmBuilder.addMutationRequest(m2);
4638     MutateRowsRequest mrm = mrmBuilder.build();
4639     CoprocessorRpcChannel channel = t.coprocessorService(ROW);
4640     MultiRowMutationService.BlockingInterface service =
4641        MultiRowMutationService.newBlockingStub(channel);
4642     service.mutateRows(null, mrm);
4643     Get g = new Get(ROW);
4644     Result r = t.get(g);
4645     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
4646     g = new Get(ROW1);
4647     r = t.get(g);
4648     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
4649   }
4650 
4651   @Test
4652   public void testRowMutation() throws Exception {
4653     LOG.info("Starting testRowMutation");
4654     final TableName TABLENAME = TableName.valueOf("testRowMutation");
4655     Table t = TEST_UTIL.createTable(TABLENAME, FAMILY);
4656     TEST_UTIL.waitTableAvailable(TABLENAME, 10000);
4657     byte [][] QUALIFIERS = new byte [][] {
4658         Bytes.toBytes("a"), Bytes.toBytes("b")
4659     };
4660     RowMutations arm = new RowMutations(ROW);
4661     Put p = new Put(ROW);
4662     p.add(FAMILY, QUALIFIERS[0], VALUE);
4663     arm.add(p);
4664     t.mutateRow(arm);
4665 
4666     Get g = new Get(ROW);
4667     Result r = t.get(g);
4668     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0])));
4669 
4670     arm = new RowMutations(ROW);
4671     p = new Put(ROW);
4672     p.add(FAMILY, QUALIFIERS[1], VALUE);
4673     arm.add(p);
4674     Delete d = new Delete(ROW);
4675     d.deleteColumns(FAMILY, QUALIFIERS[0]);
4676     arm.add(d);
4677     // TODO: Trying mutateRow again.  The batch was failing with a one try only.
4678     t.mutateRow(arm);
4679     r = t.get(g);
4680     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[1])));
4681     assertNull(r.getValue(FAMILY, QUALIFIERS[0]));
4682 
4683     //Test that we get a region level exception
4684     try {
4685       arm = new RowMutations(ROW);
4686       p = new Put(ROW);
4687       p.add(new byte[]{'b', 'o', 'g', 'u', 's'}, QUALIFIERS[0], VALUE);
4688       arm.add(p);
4689       t.mutateRow(arm);
4690       fail("Expected NoSuchColumnFamilyException");
4691     } catch(RetriesExhaustedWithDetailsException e) {
4692       for(Throwable rootCause: e.getCauses()){
4693         if(rootCause instanceof NoSuchColumnFamilyException){
4694           return;
4695         }
4696       }
4697       throw e;
4698     }
4699   }
4700 
4701   @Test
4702   public void testAppend() throws Exception {
4703     LOG.info("Starting testAppend");
4704     final TableName TABLENAME = TableName.valueOf("testAppend");
4705     Table t = TEST_UTIL.createTable(TABLENAME, FAMILY);
4706     TEST_UTIL.waitTableAvailable(TABLENAME, 10000);
4707     byte[] v1 = Bytes.toBytes("42");
4708     byte[] v2 = Bytes.toBytes("23");
4709     byte [][] QUALIFIERS = new byte [][] {
4710         Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("c")
4711     };
4712     Append a = new Append(ROW);
4713     a.add(FAMILY, QUALIFIERS[0], v1);
4714     a.add(FAMILY, QUALIFIERS[1], v2);
4715     a.setReturnResults(false);
4716     assertNullResult(t.append(a));
4717 
4718     a = new Append(ROW);
4719     a.add(FAMILY, QUALIFIERS[0], v2);
4720     a.add(FAMILY, QUALIFIERS[1], v1);
4721     a.add(FAMILY, QUALIFIERS[2], v2);
4722     Result r = t.append(a);
4723     assertEquals(0, Bytes.compareTo(Bytes.add(v1,v2), r.getValue(FAMILY, QUALIFIERS[0])));
4724     assertEquals(0, Bytes.compareTo(Bytes.add(v2,v1), r.getValue(FAMILY, QUALIFIERS[1])));
4725     // QUALIFIERS[2] previously not exist, verify both value and timestamp are correct
4726     assertEquals(0, Bytes.compareTo(v2, r.getValue(FAMILY, QUALIFIERS[2])));
4727     assertEquals(r.getColumnLatest(FAMILY, QUALIFIERS[0]).getTimestamp(),
4728         r.getColumnLatest(FAMILY, QUALIFIERS[2]).getTimestamp());
4729   }
4730 
4731   @Test
4732   public void testClientPoolRoundRobin() throws IOException, InterruptedException {
4733     final TableName tableName = TableName.valueOf("testClientPoolRoundRobin");
4734 
4735     int poolSize = 3;
4736     int numVersions = poolSize * 2;
4737     Configuration conf = TEST_UTIL.getConfiguration();
4738     conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "round-robin");
4739     conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
4740 
4741     Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, conf, Integer.MAX_VALUE);
4742     TEST_UTIL.waitTableAvailable(tableName, 10000);
4743 
4744     final long ts = EnvironmentEdgeManager.currentTime();
4745     Get get = new Get(ROW);
4746     get.addColumn(FAMILY, QUALIFIER);
4747     get.setMaxVersions();
4748 
4749     for (int versions = 1; versions <= numVersions; versions++) {
4750       Put put = new Put(ROW);
4751       put.add(FAMILY, QUALIFIER, ts + versions, VALUE);
4752       table.put(put);
4753 
4754       Result result = table.get(get);
4755       NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
4756           .get(QUALIFIER);
4757 
4758       assertEquals("The number of versions of '" + FAMILY + ":" + QUALIFIER
4759           + " did not match " + versions, versions, navigableMap.size());
4760       for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4761         assertTrue("The value at time " + entry.getKey()
4762             + " did not match what was put",
4763             Bytes.equals(VALUE, entry.getValue()));
4764       }
4765     }
4766   }
4767 
4768   @Ignore ("Flakey: HBASE-8989") @Test
4769   public void testClientPoolThreadLocal() throws IOException, InterruptedException {
4770     final TableName tableName = TableName.valueOf("testClientPoolThreadLocal");
4771 
4772     int poolSize = Integer.MAX_VALUE;
4773     int numVersions = 3;
4774     Configuration conf = TEST_UTIL.getConfiguration();
4775     conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "thread-local");
4776     conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
4777 
4778     final Table table = TEST_UTIL.createTable(tableName,
4779         new byte[][] { FAMILY }, conf, 3);
4780     TEST_UTIL.waitTableAvailable(tableName, 10000);
4781     final long ts = EnvironmentEdgeManager.currentTime();
4782     final Get get = new Get(ROW);
4783     get.addColumn(FAMILY, QUALIFIER);
4784     get.setMaxVersions();
4785 
4786     for (int versions = 1; versions <= numVersions; versions++) {
4787       Put put = new Put(ROW);
4788       put.add(FAMILY, QUALIFIER, ts + versions, VALUE);
4789       table.put(put);
4790 
4791       Result result = table.get(get);
4792       NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
4793           .get(QUALIFIER);
4794 
4795       assertEquals("The number of versions of '" + FAMILY + ":" + QUALIFIER + " did not match " +
4796         versions + "; " + put.toString() + ", " + get.toString(), versions, navigableMap.size());
4797       for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4798         assertTrue("The value at time " + entry.getKey()
4799             + " did not match what was put",
4800             Bytes.equals(VALUE, entry.getValue()));
4801       }
4802     }
4803 
4804     final Object waitLock = new Object();
4805     ExecutorService executorService = Executors.newFixedThreadPool(numVersions);
4806     final AtomicReference<AssertionError> error = new AtomicReference<AssertionError>(null);
4807     for (int versions = numVersions; versions < numVersions * 2; versions++) {
4808       final int versionsCopy = versions;
4809       executorService.submit(new Callable<Void>() {
4810         @Override
4811         public Void call() {
4812           try {
4813             Put put = new Put(ROW);
4814             put.add(FAMILY, QUALIFIER, ts + versionsCopy, VALUE);
4815             table.put(put);
4816 
4817             Result result = table.get(get);
4818             NavigableMap<Long, byte[]> navigableMap = result.getMap()
4819                 .get(FAMILY).get(QUALIFIER);
4820 
4821             assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":"
4822                 + Bytes.toString(QUALIFIER) + " did not match " + versionsCopy, versionsCopy,
4823                 navigableMap.size());
4824             for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4825               assertTrue("The value at time " + entry.getKey()
4826                   + " did not match what was put",
4827                   Bytes.equals(VALUE, entry.getValue()));
4828             }
4829             synchronized (waitLock) {
4830               waitLock.wait();
4831             }
4832           } catch (Exception e) {
4833           } catch (AssertionError e) {
4834             // the error happens in a thread, it won't fail the test,
4835             // need to pass it to the caller for proper handling.
4836             error.set(e);
4837             LOG.error(e);
4838           }
4839 
4840           return null;
4841         }
4842       });
4843     }
4844     synchronized (waitLock) {
4845       waitLock.notifyAll();
4846     }
4847     executorService.shutdownNow();
4848     assertNull(error.get());
4849   }
4850 
4851   @Test
4852   public void testCheckAndPut() throws IOException, InterruptedException {
4853     final byte [] anotherrow = Bytes.toBytes("anotherrow");
4854     final byte [] value2 = Bytes.toBytes("abcd");
4855 
4856     Table table = TEST_UTIL.createTable(TableName.valueOf("testCheckAndPut"), FAMILY);
4857     TEST_UTIL.waitTableAvailable(TableName.valueOf("testCheckAndPut"), 10000);
4858     Put put1 = new Put(ROW);
4859     put1.add(FAMILY, QUALIFIER, VALUE);
4860 
4861     // row doesn't exist, so using non-null value should be considered "not match".
4862     boolean ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE, put1);
4863     assertEquals(ok, false);
4864 
4865     // row doesn't exist, so using "null" to check for existence should be considered "match".
4866     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1);
4867     assertEquals(ok, true);
4868 
4869     // row now exists, so using "null" to check for existence should be considered "not match".
4870     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1);
4871     assertEquals(ok, false);
4872 
4873     Put put2 = new Put(ROW);
4874     put2.add(FAMILY, QUALIFIER, value2);
4875 
4876     // row now exists, use the matching value to check
4877     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE, put2);
4878     assertEquals(ok, true);
4879 
4880     Put put3 = new Put(anotherrow);
4881     put3.add(FAMILY, QUALIFIER, VALUE);
4882 
4883     // try to do CheckAndPut on different rows
4884     try {
4885         ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, value2, put3);
4886         fail("trying to check and modify different rows should have failed.");
4887     } catch(Exception e) {}
4888   }
4889 
4890   @Test
4891   public void testCheckAndPutWithCompareOp() throws IOException, InterruptedException {
4892     final byte [] value1 = Bytes.toBytes("aaaa");
4893     final byte [] value2 = Bytes.toBytes("bbbb");
4894     final byte [] value3 = Bytes.toBytes("cccc");
4895     final byte [] value4 = Bytes.toBytes("dddd");
4896 
4897     Table table = TEST_UTIL.createTable(TableName.valueOf("testCheckAndPutWithCompareOp"), FAMILY);
4898     TEST_UTIL.waitTableAvailable(TableName.valueOf("testCheckAndPutWithCompareOp"), 10000);
4899 
4900     Put put2 = new Put(ROW);
4901     put2.add(FAMILY, QUALIFIER, value2);
4902 
4903     Put put3 = new Put(ROW);
4904     put3.add(FAMILY, QUALIFIER, value3);
4905 
4906     // row doesn't exist, so using "null" to check for existence should be considered "match".
4907     boolean ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put2);
4908     assertEquals(ok, true);
4909 
4910     // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
4911     // turns out "match"
4912     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value1, put2);
4913     assertEquals(ok, false);
4914     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value1, put2);
4915     assertEquals(ok, false);
4916     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value1, put2);
4917     assertEquals(ok, false);
4918     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value1, put2);
4919     assertEquals(ok, true);
4920     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value1, put2);
4921     assertEquals(ok, true);
4922     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value1, put3);
4923     assertEquals(ok, true);
4924 
4925     // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
4926     // turns out "match"
4927     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value4, put3);
4928     assertEquals(ok, false);
4929     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value4, put3);
4930     assertEquals(ok, false);
4931     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value4, put3);
4932     assertEquals(ok, false);
4933     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value4, put3);
4934     assertEquals(ok, true);
4935     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value4, put3);
4936     assertEquals(ok, true);
4937     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value4, put2);
4938     assertEquals(ok, true);
4939 
4940     // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
4941     // turns out "match"
4942     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value2, put2);
4943     assertEquals(ok, false);
4944     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value2, put2);
4945     assertEquals(ok, false);
4946     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value2, put2);
4947     assertEquals(ok, false);
4948     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value2, put2);
4949     assertEquals(ok, true);
4950     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value2, put2);
4951     assertEquals(ok, true);
4952     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value2, put3);
4953     assertEquals(ok, true);
4954   }
4955 
4956   @Test
4957   public void testCheckAndDeleteWithCompareOp() throws IOException, InterruptedException {
4958     final byte [] value1 = Bytes.toBytes("aaaa");
4959     final byte [] value2 = Bytes.toBytes("bbbb");
4960     final byte [] value3 = Bytes.toBytes("cccc");
4961     final byte [] value4 = Bytes.toBytes("dddd");
4962     Table table = TEST_UTIL.createTable(TableName.valueOf("testCheckAndDeleteWithCompareOp"),
4963         FAMILY);
4964     TEST_UTIL.waitTableAvailable(TableName.valueOf("testCheckAndDeleteWithCompareOp"), 10000);
4965 
4966     Put put2 = new Put(ROW);
4967     put2.add(FAMILY, QUALIFIER, value2);
4968     table.put(put2);
4969 
4970     Put put3 = new Put(ROW);
4971     put3.add(FAMILY, QUALIFIER, value3);
4972 
4973     Delete delete = new Delete(ROW);
4974     delete.deleteColumns(FAMILY, QUALIFIER);
4975 
4976     // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
4977     // turns out "match"
4978     boolean ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value1, delete);
4979     assertEquals(ok, false);
4980     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value1, delete);
4981     assertEquals(ok, false);
4982     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value1, delete);
4983     assertEquals(ok, false);
4984     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value1, delete);
4985     assertEquals(ok, true);
4986     table.put(put2);
4987     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value1, delete);
4988     assertEquals(ok, true);
4989     table.put(put2);
4990 
4991     assertEquals(ok, true);
4992 
4993     // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
4994     // turns out "match"
4995     table.put(put3);
4996     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value4, delete);
4997 
4998     assertEquals(ok, false);
4999     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value4, delete);
5000 
5001     assertEquals(ok, false);
5002     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value4, delete);
5003 
5004     assertEquals(ok, false);
5005     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value4, delete);
5006 
5007     assertEquals(ok, true);
5008     table.put(put3);
5009     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value4, delete);
5010     assertEquals(ok, true);
5011     table.put(put3);
5012     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value4, delete);
5013 
5014     assertEquals(ok, true);
5015 
5016     // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
5017     // turns out "match"
5018     table.put(put2);
5019     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value2, delete);
5020     assertEquals(ok, false);
5021     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value2, delete);
5022     assertEquals(ok, false);
5023     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value2, delete);
5024     assertEquals(ok, false);
5025     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value2, delete);
5026     assertEquals(ok, true);
5027     table.put(put2);
5028     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value2, delete);
5029     assertEquals(ok, true);
5030     table.put(put2);
5031     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value2, delete);
5032     assertEquals(ok, true);
5033   }
5034 
5035   /**
5036   * Test ScanMetrics
5037   * @throws Exception
5038   */
5039   @Test
5040   @SuppressWarnings ("unused")
5041   public void testScanMetrics() throws Exception {
5042     TableName TABLENAME = TableName.valueOf("testScanMetrics");
5043 
5044     Configuration conf = TEST_UTIL.getConfiguration();
5045 
5046     // Set up test table:
5047     // Create table:
5048     HTable ht = TEST_UTIL.createMultiRegionTable(TABLENAME, FAMILY);
5049     int numOfRegions = -1;
5050     try (RegionLocator r = ht.getRegionLocator()) {
5051       numOfRegions = r.getStartKeys().length;
5052     }
5053     // Create 3 rows in the table, with rowkeys starting with "zzz*" so that
5054     // scan are forced to hit all the regions.
5055     Put put1 = new Put(Bytes.toBytes("zzz1"));
5056     put1.add(FAMILY, QUALIFIER, VALUE);
5057     Put put2 = new Put(Bytes.toBytes("zzz2"));
5058     put2.add(FAMILY, QUALIFIER, VALUE);
5059     Put put3 = new Put(Bytes.toBytes("zzz3"));
5060     put3.add(FAMILY, QUALIFIER, VALUE);
5061     ht.put(Arrays.asList(put1, put2, put3));
5062 
5063     Scan scan1 = new Scan();
5064     int numRecords = 0;
5065     ResultScanner scanner = ht.getScanner(scan1);
5066     for(Result result : scanner) {
5067       numRecords++;
5068     }
5069     scanner.close();
5070     LOG.info("test data has " + numRecords + " records.");
5071 
5072     // by default, scan metrics collection is turned off
5073     assertEquals(null, scan1.getScanMetrics());
5074 
5075     // turn on scan metrics
5076     Scan scan2 = new Scan();
5077     scan2.setScanMetricsEnabled(true);
5078     scan2.setCaching(numRecords+1);
5079     scanner = ht.getScanner(scan2);
5080     for (Result result : scanner.next(numRecords - 1)) {
5081     }
5082     scanner.close();
5083     // closing the scanner will set the metrics.
5084     assertNotNull(scan2.getScanMetrics());
5085 
5086     // set caching to 1, because metrics are collected in each roundtrip only
5087     scan2 = new Scan();
5088     scan2.setScanMetricsEnabled(true);
5089     scan2.setCaching(1);
5090     scanner = ht.getScanner(scan2);
5091     // per HBASE-5717, this should still collect even if you don't run all the way to
5092     // the end of the scanner. So this is asking for 2 of the 3 rows we inserted.
5093     for (Result result : scanner.next(numRecords - 1)) {
5094     }
5095     scanner.close();
5096 
5097     ScanMetrics scanMetrics = scan2.getScanMetrics();
5098     assertEquals("Did not access all the regions in the table", numOfRegions,
5099         scanMetrics.countOfRegions.get());
5100 
5101     // check byte counters
5102     scan2 = new Scan();
5103     scan2.setScanMetricsEnabled(true);
5104     scan2.setCaching(1);
5105     scanner = ht.getScanner(scan2);
5106     int numBytes = 0;
5107     for (Result result : scanner.next(1)) {
5108       for (Cell cell: result.listCells()) {
5109         numBytes += CellUtil.estimatedSerializedSizeOf(cell);
5110       }
5111     }
5112     scanner.close();
5113     scanMetrics = scan2.getScanMetrics();
5114     assertEquals("Did not count the result bytes", numBytes,
5115       scanMetrics.countOfBytesInResults.get());
5116 
5117     // check byte counters on a small scan
5118     scan2 = new Scan();
5119     scan2.setScanMetricsEnabled(true);
5120     scan2.setCaching(1);
5121     scan2.setSmall(true);
5122     scanner = ht.getScanner(scan2);
5123     numBytes = 0;
5124     for (Result result : scanner.next(1)) {
5125       for (Cell cell: result.listCells()) {
5126         numBytes += CellUtil.estimatedSerializedSizeOf(cell);
5127       }
5128     }
5129     scanner.close();
5130     scanMetrics = scan2.getScanMetrics();
5131     assertEquals("Did not count the result bytes", numBytes,
5132       scanMetrics.countOfBytesInResults.get());
5133 
5134     // now, test that the metrics are still collected even if you don't call close, but do
5135     // run past the end of all the records
5136     /** There seems to be a timing issue here.  Comment out for now. Fix when time.
5137     Scan scanWithoutClose = new Scan();
5138     scanWithoutClose.setCaching(1);
5139     scanWithoutClose.setScanMetricsEnabled(true);
5140     ResultScanner scannerWithoutClose = ht.getScanner(scanWithoutClose);
5141     for (Result result : scannerWithoutClose.next(numRecords + 1)) {
5142     }
5143     ScanMetrics scanMetricsWithoutClose = getScanMetrics(scanWithoutClose);
5144     assertEquals("Did not access all the regions in the table", numOfRegions,
5145         scanMetricsWithoutClose.countOfRegions.get());
5146     */
5147 
5148     // finally, test that the metrics are collected correctly if you both run past all the records,
5149     // AND close the scanner
5150     Scan scanWithClose = new Scan();
5151     // make sure we can set caching up to the number of a scanned values
5152     scanWithClose.setCaching(numRecords);
5153     scanWithClose.setScanMetricsEnabled(true);
5154     ResultScanner scannerWithClose = ht.getScanner(scanWithClose);
5155     for (Result result : scannerWithClose.next(numRecords + 1)) {
5156     }
5157     scannerWithClose.close();
5158     ScanMetrics scanMetricsWithClose = getScanMetrics(scanWithClose);
5159     assertEquals("Did not access all the regions in the table", numOfRegions,
5160         scanMetricsWithClose.countOfRegions.get());
5161   }
5162 
5163   private ScanMetrics getScanMetrics(Scan scan) throws Exception {
5164     byte[] serializedMetrics = scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA);
5165     assertTrue("Serialized metrics were not found.", serializedMetrics != null);
5166 
5167     ScanMetrics scanMetrics = ProtobufUtil.toScanMetrics(serializedMetrics);
5168 
5169     return scanMetrics;
5170   }
5171 
5172   /**
5173    * Tests that cache on write works all the way up from the client-side.
5174    *
5175    * Performs inserts, flushes, and compactions, verifying changes in the block
5176    * cache along the way.
5177    *
5178    * @throws Exception
5179    */
5180   @Test
5181   public void testCacheOnWriteEvictOnClose() throws Exception {
5182     TableName tableName = TableName.valueOf("testCOWEOCfromClient");
5183     byte [] data = Bytes.toBytes("data");
5184     HTable table = TEST_UTIL.createTable(tableName, FAMILY);
5185     TEST_UTIL.waitTableAvailable(tableName, 10000);
5186     // get the block cache and region
5187     String regionName = table.getRegionLocations().firstKey().getEncodedName();
5188     Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName)
5189       .getFromOnlineRegions(regionName);
5190     Store store = region.getStores().iterator().next();
5191     CacheConfig cacheConf = store.getCacheConfig();
5192     cacheConf.setCacheDataOnWrite(true);
5193     cacheConf.setEvictOnClose(true);
5194     BlockCache cache = cacheConf.getBlockCache();
5195 
5196     // establish baseline stats
5197     long startBlockCount = cache.getBlockCount();
5198     long startBlockHits = cache.getStats().getHitCount();
5199     long startBlockMiss = cache.getStats().getMissCount();
5200 
5201     // wait till baseline is stable, (minimal 500 ms)
5202     for (int i = 0; i < 5; i++) {
5203       Thread.sleep(100);
5204       if (startBlockCount != cache.getBlockCount()
5205           || startBlockHits != cache.getStats().getHitCount()
5206           || startBlockMiss != cache.getStats().getMissCount()) {
5207         startBlockCount = cache.getBlockCount();
5208         startBlockHits = cache.getStats().getHitCount();
5209         startBlockMiss = cache.getStats().getMissCount();
5210         i = -1;
5211       }
5212     }
5213 
5214     // insert data
5215     Put put = new Put(ROW);
5216     put.add(FAMILY, QUALIFIER, data);
5217     table.put(put);
5218     assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
5219     // data was in memstore so don't expect any changes
5220     assertEquals(startBlockCount, cache.getBlockCount());
5221     assertEquals(startBlockHits, cache.getStats().getHitCount());
5222     assertEquals(startBlockMiss, cache.getStats().getMissCount());
5223     // flush the data
5224     System.out.println("Flushing cache");
5225     region.flush(true);
5226     // expect one more block in cache, no change in hits/misses
5227     long expectedBlockCount = startBlockCount + 1;
5228     long expectedBlockHits = startBlockHits;
5229     long expectedBlockMiss = startBlockMiss;
5230     assertEquals(expectedBlockCount, cache.getBlockCount());
5231     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5232     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5233     // read the data and expect same blocks, one new hit, no misses
5234     assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
5235     assertEquals(expectedBlockCount, cache.getBlockCount());
5236     assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
5237     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5238     // insert a second column, read the row, no new blocks, one new hit
5239     byte [] QUALIFIER2 = Bytes.add(QUALIFIER, QUALIFIER);
5240     byte [] data2 = Bytes.add(data, data);
5241     put = new Put(ROW);
5242     put.add(FAMILY, QUALIFIER2, data2);
5243     table.put(put);
5244     Result r = table.get(new Get(ROW));
5245     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
5246     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
5247     assertEquals(expectedBlockCount, cache.getBlockCount());
5248     assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
5249     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5250     // flush, one new block
5251     System.out.println("Flushing cache");
5252     region.flush(true);
5253     assertEquals(++expectedBlockCount, cache.getBlockCount());
5254     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5255     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5256     // compact, net minus two blocks, two hits, no misses
5257     System.out.println("Compacting");
5258     assertEquals(2, store.getStorefilesCount());
5259     store.triggerMajorCompaction();
5260     region.compact(true);
5261     waitForStoreFileCount(store, 1, 10000); // wait 10 seconds max
5262     assertEquals(1, store.getStorefilesCount());
5263     expectedBlockCount -= 2; // evicted two blocks, cached none
5264     assertEquals(expectedBlockCount, cache.getBlockCount());
5265     expectedBlockHits += 2;
5266     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5267     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5268     // read the row, this should be a cache miss because we don't cache data
5269     // blocks on compaction
5270     r = table.get(new Get(ROW));
5271     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
5272     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
5273     expectedBlockCount += 1; // cached one data block
5274     assertEquals(expectedBlockCount, cache.getBlockCount());
5275     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5276     assertEquals(++expectedBlockMiss, cache.getStats().getMissCount());
5277   }
5278 
5279   private void waitForStoreFileCount(Store store, int count, int timeout)
5280   throws InterruptedException {
5281     long start = System.currentTimeMillis();
5282     while (start + timeout > System.currentTimeMillis() &&
5283         store.getStorefilesCount() != count) {
5284       Thread.sleep(100);
5285     }
5286     System.out.println("start=" + start + ", now=" +
5287         System.currentTimeMillis() + ", cur=" + store.getStorefilesCount());
5288     assertEquals(count, store.getStorefilesCount());
5289   }
5290 
5291   @Test
5292   /**
5293    * Tests the non cached version of getRegionLocator by moving a region.
5294    */
5295   public void testNonCachedGetRegionLocation() throws Exception {
5296     // Test Initialization.
5297     TableName TABLE = TableName.valueOf("testNonCachedGetRegionLocation");
5298     byte [] family1 = Bytes.toBytes("f1");
5299     byte [] family2 = Bytes.toBytes("f2");
5300     try (HTable table = TEST_UTIL.createTable(TABLE, new byte[][] {family1, family2}, 10);
5301         Admin admin = new HBaseAdmin(TEST_UTIL.getConfiguration())) {
5302       TEST_UTIL.waitTableAvailable(TABLE, 10000);
5303       Map <HRegionInfo, ServerName> regionsMap = table.getRegionLocations();
5304       assertEquals(1, regionsMap.size());
5305       HRegionInfo regionInfo = regionsMap.keySet().iterator().next();
5306       ServerName addrBefore = regionsMap.get(regionInfo);
5307       // Verify region location before move.
5308       HRegionLocation addrCache = table.getRegionLocation(regionInfo.getStartKey(), false);
5309       HRegionLocation addrNoCache = table.getRegionLocation(regionInfo.getStartKey(),  true);
5310 
5311       assertEquals(addrBefore.getPort(), addrCache.getPort());
5312       assertEquals(addrBefore.getPort(), addrNoCache.getPort());
5313 
5314       ServerName addrAfter = null;
5315       // Now move the region to a different server.
5316       for (int i = 0; i < SLAVES; i++) {
5317         HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(i);
5318         ServerName addr = regionServer.getServerName();
5319         if (addr.getPort() != addrBefore.getPort()) {
5320           admin.move(regionInfo.getEncodedNameAsBytes(),
5321               Bytes.toBytes(addr.toString()));
5322           // Wait for the region to move.
5323           Thread.sleep(5000);
5324           addrAfter = addr;
5325           break;
5326         }
5327       }
5328 
5329       // Verify the region was moved.
5330       addrCache = table.getRegionLocation(regionInfo.getStartKey(), false);
5331       addrNoCache = table.getRegionLocation(regionInfo.getStartKey(), true);
5332       assertNotNull(addrAfter);
5333       assertTrue(addrAfter.getPort() != addrCache.getPort());
5334       assertEquals(addrAfter.getPort(), addrNoCache.getPort());
5335     }
5336   }
5337 
5338   @Test
5339   /**
5340    * Tests getRegionsInRange by creating some regions over which a range of
5341    * keys spans; then changing the key range.
5342    */
5343   public void testGetRegionsInRange() throws Exception {
5344     // Test Initialization.
5345     byte [] startKey = Bytes.toBytes("ddc");
5346     byte [] endKey = Bytes.toBytes("mmm");
5347     TableName TABLE = TableName.valueOf("testGetRegionsInRange");
5348     HTable table = TEST_UTIL.createMultiRegionTable(TABLE, new byte[][] { FAMILY }, 10);
5349     int numOfRegions = -1;
5350     try (RegionLocator r = table.getRegionLocator()) {
5351       numOfRegions = r.getStartKeys().length;
5352     }
5353     assertEquals(26, numOfRegions);
5354 
5355     // Get the regions in this range
5356     List<HRegionLocation> regionsList = table.getRegionsInRange(startKey,
5357       endKey);
5358     assertEquals(10, regionsList.size());
5359 
5360     // Change the start key
5361     startKey = Bytes.toBytes("fff");
5362     regionsList = table.getRegionsInRange(startKey, endKey);
5363     assertEquals(7, regionsList.size());
5364 
5365     // Change the end key
5366     endKey = Bytes.toBytes("nnn");
5367     regionsList = table.getRegionsInRange(startKey, endKey);
5368     assertEquals(8, regionsList.size());
5369 
5370     // Empty start key
5371     regionsList = table.getRegionsInRange(HConstants.EMPTY_START_ROW, endKey);
5372     assertEquals(13, regionsList.size());
5373 
5374     // Empty end key
5375     regionsList = table.getRegionsInRange(startKey, HConstants.EMPTY_END_ROW);
5376     assertEquals(21, regionsList.size());
5377 
5378     // Both start and end keys empty
5379     regionsList = table.getRegionsInRange(HConstants.EMPTY_START_ROW,
5380       HConstants.EMPTY_END_ROW);
5381     assertEquals(26, regionsList.size());
5382 
5383     // Change the end key to somewhere in the last block
5384     endKey = Bytes.toBytes("zzz1");
5385     regionsList = table.getRegionsInRange(startKey, endKey);
5386     assertEquals(21, regionsList.size());
5387 
5388     // Change the start key to somewhere in the first block
5389     startKey = Bytes.toBytes("aac");
5390     regionsList = table.getRegionsInRange(startKey, endKey);
5391     assertEquals(26, regionsList.size());
5392 
5393     // Make start and end key the same
5394     startKey = endKey = Bytes.toBytes("ccc");
5395     regionsList = table.getRegionsInRange(startKey, endKey);
5396     assertEquals(1, regionsList.size());
5397   }
5398 
5399   @Test
5400   public void testJira6912() throws Exception {
5401     TableName TABLE = TableName.valueOf("testJira6912");
5402     Table foo = TEST_UTIL.createTable(TABLE, new byte[][] {FAMILY}, 10);
5403     TEST_UTIL.waitTableAvailable(TABLE, 10000);
5404     List<Put> puts = new ArrayList<Put>();
5405     for (int i=0;i !=100; i++){
5406       Put put = new Put(Bytes.toBytes(i));
5407       put.add(FAMILY, FAMILY, Bytes.toBytes(i));
5408       puts.add(put);
5409     }
5410     foo.put(puts);
5411     // If i comment this out it works
5412     TEST_UTIL.flush();
5413 
5414     Scan scan = new Scan();
5415     scan.setStartRow(Bytes.toBytes(1));
5416     scan.setStopRow(Bytes.toBytes(3));
5417     scan.addColumn(FAMILY, FAMILY);
5418     scan.setFilter(new RowFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator(Bytes.toBytes(1))));
5419 
5420     ResultScanner scanner = foo.getScanner(scan);
5421     Result[] bar = scanner.next(100);
5422     assertEquals(1, bar.length);
5423   }
5424 
5425   @Test
5426   public void testScan_NullQualifier() throws IOException, InterruptedException {
5427     Table table = TEST_UTIL.createTable(TableName.valueOf("testScan_NullQualifier"), FAMILY);
5428     TEST_UTIL.waitTableAvailable(TableName.valueOf("testScan_NullQualifier"), 10000);
5429 
5430     Put put = new Put(ROW);
5431     put.add(FAMILY, QUALIFIER, VALUE);
5432     table.put(put);
5433 
5434     put = new Put(ROW);
5435     put.add(FAMILY, null, VALUE);
5436     table.put(put);
5437     LOG.info("Row put");
5438 
5439     Scan scan = new Scan();
5440     scan.addColumn(FAMILY, null);
5441 
5442     ResultScanner scanner = table.getScanner(scan);
5443     Result[] bar = scanner.next(100);
5444     assertEquals(1, bar.length);
5445     assertEquals(1, bar[0].size());
5446 
5447     scan = new Scan();
5448     scan.addFamily(FAMILY);
5449 
5450     scanner = table.getScanner(scan);
5451     bar = scanner.next(100);
5452     assertEquals(1, bar.length);
5453     assertEquals(2, bar[0].size());
5454   }
5455 
5456   @Test
5457   public void testNegativeTimestamp() throws IOException, InterruptedException {
5458     Table table = TEST_UTIL.createTable(TableName.valueOf("testNegativeTimestamp"), FAMILY);
5459     TEST_UTIL.waitTableAvailable(TableName.valueOf("testNegativeTimestamp"), 10000);
5460     try {
5461       Put put = new Put(ROW, -1);
5462       put.add(FAMILY, QUALIFIER, VALUE);
5463       table.put(put);
5464       fail("Negative timestamps should not have been allowed");
5465     } catch (IllegalArgumentException ex) {
5466       assertTrue(ex.getMessage().contains("negative"));
5467     }
5468 
5469     try {
5470       Put put = new Put(ROW);
5471       put.add(FAMILY, QUALIFIER, -1, VALUE);
5472       table.put(put);
5473       fail("Negative timestamps should not have been allowed");
5474     } catch (IllegalArgumentException ex) {
5475       assertTrue(ex.getMessage().contains("negative"));
5476     }
5477 
5478     try {
5479       Delete delete = new Delete(ROW, -1);
5480       table.delete(delete);
5481       fail("Negative timestamps should not have been allowed");
5482     } catch (IllegalArgumentException ex) {
5483       assertTrue(ex.getMessage().contains("negative"));
5484     }
5485 
5486     try {
5487       Delete delete = new Delete(ROW);
5488       delete.deleteFamily(FAMILY, -1);
5489       table.delete(delete);
5490       fail("Negative timestamps should not have been allowed");
5491     } catch (IllegalArgumentException ex) {
5492       assertTrue(ex.getMessage().contains("negative"));
5493     }
5494 
5495     try {
5496       Scan scan = new Scan();
5497       scan.setTimeRange(-1, 1);
5498       table.getScanner(scan);
5499       fail("Negative timestamps should not have been allowed");
5500     } catch (IllegalArgumentException ex) {
5501       assertTrue(ex.getMessage().contains("negative"));
5502     }
5503 
5504     // KeyValue should allow negative timestamps for backwards compat. Otherwise, if the user
5505     // already has negative timestamps in cluster data, HBase won't be able to handle that
5506     try {
5507       new KeyValue(Bytes.toBytes(42), Bytes.toBytes(42), Bytes.toBytes(42), -1, Bytes.toBytes(42));
5508     } catch (IllegalArgumentException ex) {
5509       fail("KeyValue SHOULD allow negative timestamps");
5510     }
5511 
5512     table.close();
5513   }
5514 
5515   @Test
5516   public void testIllegalTableDescriptor() throws Exception {
5517     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testIllegalTableDescriptor"));
5518     HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
5519 
5520     // create table with 0 families
5521     checkTableIsIllegal(htd);
5522     htd.addFamily(hcd);
5523     checkTableIsLegal(htd);
5524 
5525     htd.setMaxFileSize(1024); // 1K
5526     checkTableIsIllegal(htd);
5527     htd.setMaxFileSize(0);
5528     checkTableIsIllegal(htd);
5529     htd.setMaxFileSize(1024 * 1024 * 1024); // 1G
5530     checkTableIsLegal(htd);
5531 
5532     htd.setMemStoreFlushSize(1024);
5533     checkTableIsIllegal(htd);
5534     htd.setMemStoreFlushSize(0);
5535     checkTableIsIllegal(htd);
5536     htd.setMemStoreFlushSize(128 * 1024 * 1024); // 128M
5537     checkTableIsLegal(htd);
5538 
5539     htd.setRegionSplitPolicyClassName("nonexisting.foo.class");
5540     checkTableIsIllegal(htd);
5541     htd.setRegionSplitPolicyClassName(null);
5542     checkTableIsLegal(htd);
5543 
5544     hcd.setBlocksize(0);
5545     checkTableIsIllegal(htd);
5546     hcd.setBlocksize(1024 * 1024 * 128); // 128M
5547     checkTableIsIllegal(htd);
5548     hcd.setBlocksize(1024);
5549     checkTableIsLegal(htd);
5550 
5551     hcd.setTimeToLive(0);
5552     checkTableIsIllegal(htd);
5553     hcd.setTimeToLive(-1);
5554     checkTableIsIllegal(htd);
5555     hcd.setTimeToLive(1);
5556     checkTableIsLegal(htd);
5557 
5558     hcd.setMinVersions(-1);
5559     checkTableIsIllegal(htd);
5560     hcd.setMinVersions(3);
5561     try {
5562       hcd.setMaxVersions(2);
5563       fail();
5564     } catch (IllegalArgumentException ex) {
5565       // expected
5566       hcd.setMaxVersions(10);
5567     }
5568     checkTableIsLegal(htd);
5569 
5570     // HBASE-13776 Setting illegal versions for HColumnDescriptor
5571     //  does not throw IllegalArgumentException
5572     // finally, minVersions must be less than or equal to maxVersions
5573     hcd.setMaxVersions(4);
5574     hcd.setMinVersions(5);
5575     checkTableIsIllegal(htd);
5576     hcd.setMinVersions(3);
5577 
5578     try {
5579       hcd.setScope(-1);
5580       fail("Illegal value for setScope did not throw");
5581     } catch (IllegalArgumentException e) {
5582       // expected
5583       hcd.setScope(0);
5584     }
5585     checkTableIsLegal(htd);
5586 
5587     try {
5588       hcd.setDFSReplication((short) -1);
5589       fail("Illegal value for setDFSReplication did not throw");
5590     } catch (IllegalArgumentException e) {
5591       // pass
5592     }
5593     // set an illegal DFS replication value by hand
5594     hcd.setValue(HColumnDescriptor.DFS_REPLICATION, "-1");
5595     checkTableIsIllegal(htd);
5596     try {
5597       hcd.setDFSReplication((short) -1);
5598       fail("Should throw exception if an illegal value is explicitly being set");
5599     } catch (IllegalArgumentException e) {
5600       // pass
5601     }
5602 
5603     // check the conf settings to disable sanity checks
5604     htd.setMemStoreFlushSize(0);
5605 
5606     // Check that logs warn on invalid table but allow it.
5607     ListAppender listAppender = new ListAppender();
5608     Logger log = Logger.getLogger(HMaster.class);
5609     log.addAppender(listAppender);
5610     log.setLevel(Level.WARN);
5611 
5612     htd.setConfiguration("hbase.table.sanity.checks", Boolean.FALSE.toString());
5613     checkTableIsLegal(htd);
5614 
5615     assertFalse(listAppender.getMessages().isEmpty());
5616     assertTrue(listAppender.getMessages().get(0).startsWith("MEMSTORE_FLUSHSIZE for table "
5617         + "descriptor or \"hbase.hregion.memstore.flush.size\" (0) is too small, which might "
5618         + "cause very frequent flushing."));
5619 
5620     log.removeAppender(listAppender);
5621   }
5622 
5623   private static class ListAppender extends AppenderSkeleton {
5624     private final List<String> messages = new ArrayList<String>();
5625 
5626     @Override
5627     protected void append(LoggingEvent event) {
5628       messages.add(event.getMessage().toString());
5629     }
5630 
5631     @Override
5632     public void close() {
5633     }
5634 
5635     @Override
5636     public boolean requiresLayout() {
5637       return false;
5638     }
5639 
5640     public List<String> getMessages() {
5641       return messages;
5642     }
5643   }
5644 
5645   private void checkTableIsLegal(HTableDescriptor htd) throws IOException {
5646     Admin admin = TEST_UTIL.getHBaseAdmin();
5647     admin.createTable(htd);
5648     assertTrue(admin.tableExists(htd.getTableName()));
5649     admin.disableTable(htd.getTableName());
5650     admin.deleteTable(htd.getTableName());
5651   }
5652 
5653   private void checkTableIsIllegal(HTableDescriptor htd) throws IOException {
5654     Admin admin = TEST_UTIL.getHBaseAdmin();
5655     try {
5656       admin.createTable(htd);
5657       fail();
5658     } catch(Exception ex) {
5659       // should throw ex
5660     }
5661     assertFalse(admin.tableExists(htd.getTableName()));
5662   }
5663 
5664   @Test
5665   public void testRawScanRespectsVersions() throws Exception {
5666     TableName TABLE = TableName.valueOf("testRawScan");
5667     Table table = TEST_UTIL.createTable(TABLE, FAMILY);
5668     TEST_UTIL.waitTableAvailable(TABLE, 10000);
5669     byte[] row = Bytes.toBytes("row");
5670 
5671     // put the same row 4 times, with different values
5672     Put p = new Put(row);
5673     p.add(FAMILY, QUALIFIER, 10, VALUE);
5674     table.put(p);
5675     p = new Put(row);
5676     p.add(FAMILY, QUALIFIER, 11, ArrayUtils.add(VALUE, (byte) 2));
5677     table.put(p);
5678 
5679     p = new Put(row);
5680     p.add(FAMILY, QUALIFIER, 12, ArrayUtils.add(VALUE, (byte) 3));
5681     table.put(p);
5682 
5683     p = new Put(row);
5684     p.add(FAMILY, QUALIFIER, 13, ArrayUtils.add(VALUE, (byte) 4));
5685     table.put(p);
5686 
5687     int versions = 4;
5688     Scan s = new Scan(row);
5689     // get all the possible versions
5690     s.setMaxVersions();
5691     s.setRaw(true);
5692 
5693     ResultScanner scanner = table.getScanner(s);
5694     int count = 0;
5695     for (Result r : scanner) {
5696       assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
5697       count++;
5698     }
5699     assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
5700       count);
5701     scanner.close();
5702 
5703     // then if we decrease the number of versions, but keep the scan raw, we should see exactly that
5704     // number of versions
5705     versions = 2;
5706     s.setMaxVersions(versions);
5707     scanner = table.getScanner(s);
5708     count = 0;
5709     for (Result r : scanner) {
5710       assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
5711       count++;
5712     }
5713     assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
5714       count);
5715     scanner.close();
5716 
5717     // finally, if we turn off raw scanning, but max out the number of versions, we should go back
5718     // to seeing just three
5719     versions = 3;
5720     s.setMaxVersions(versions);
5721     scanner = table.getScanner(s);
5722     count = 0;
5723     for (Result r : scanner) {
5724       assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
5725       count++;
5726     }
5727     assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
5728       count);
5729     scanner.close();
5730 
5731     table.close();
5732     TEST_UTIL.deleteTable(TABLE);
5733   }
5734 
5735   @Test
5736   public void testEmptyFilterList() throws Exception {
5737     // Test Initialization.
5738     TableName TABLE = TableName.valueOf("testEmptyFilterList");
5739     Table table = TEST_UTIL.createTable(TABLE, FAMILY);
5740     TEST_UTIL.waitTableAvailable(TABLE, 10000);
5741 
5742     // Insert one row each region
5743     Put put = new Put(Bytes.toBytes("row"));
5744     put.addColumn(FAMILY, QUALIFIER, VALUE);
5745     table.put(put);
5746 
5747     List<Result> scanResults = new LinkedList<>();
5748     Scan scan = new Scan();
5749     scan.setFilter(new FilterList());
5750     try (ResultScanner scanner = table.getScanner(scan)) {
5751       for (Result r : scanner) {
5752         scanResults.add(r);
5753       }
5754     }
5755 
5756     Get g = new Get(Bytes.toBytes("row"));
5757     g.setFilter(new FilterList());
5758     Result getResult = table.get(g);
5759     if (scanResults.isEmpty()) {
5760       assertTrue(getResult.isEmpty());
5761     } else if (scanResults.size() == 1) {
5762       Result scanResult = scanResults.get(0);
5763       assertEquals(scanResult.rawCells().length, getResult.rawCells().length);
5764       for (int i = 0; i != scanResult.rawCells().length; ++i) {
5765         Cell scanCell = scanResult.rawCells()[i];
5766         Cell getCell = getResult.rawCells()[i];
5767         assertEquals(0, Bytes.compareTo(CellUtil.cloneRow(scanCell), CellUtil.cloneRow(getCell)));
5768         assertEquals(0,
5769           Bytes.compareTo(CellUtil.cloneFamily(scanCell), CellUtil.cloneFamily(getCell)));
5770         assertEquals(0,
5771           Bytes.compareTo(CellUtil.cloneQualifier(scanCell), CellUtil.cloneQualifier(getCell)));
5772         assertEquals(0,
5773           Bytes.compareTo(CellUtil.cloneValue(scanCell), CellUtil.cloneValue(getCell)));
5774       }
5775     } else {
5776       fail("The result retrieved from SCAN and Get should be same");
5777     }
5778   }
5779 
5780   @Test
5781   public void testSmallScan() throws Exception {
5782     // Test Initialization.
5783     TableName TABLE = TableName.valueOf("testSmallScan");
5784     Table table = TEST_UTIL.createTable(TABLE, FAMILY);
5785     TEST_UTIL.waitTableAvailable(TABLE, 10000);
5786 
5787     // Insert one row each region
5788     int insertNum = 10;
5789     for (int i = 0; i < 10; i++) {
5790       Put put = new Put(Bytes.toBytes("row" + String.format("%03d", i)));
5791       put.add(FAMILY, QUALIFIER, VALUE);
5792       table.put(put);
5793     }
5794 
5795     // nomal scan
5796     ResultScanner scanner = table.getScanner(new Scan());
5797     int count = 0;
5798     for (Result r : scanner) {
5799       assertTrue(!r.isEmpty());
5800       count++;
5801     }
5802     assertEquals(insertNum, count);
5803 
5804     // small scan
5805     Scan scan = new Scan(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
5806     scan.setSmall(true);
5807     scan.setCaching(2);
5808     scanner = table.getScanner(scan);
5809     count = 0;
5810     for (Result r : scanner) {
5811       assertTrue(!r.isEmpty());
5812       count++;
5813     }
5814     assertEquals(insertNum, count);
5815 
5816   }
5817 
5818   @Test
5819   public void testSuperSimpleWithReverseScan() throws Exception {
5820     TableName TABLE = TableName.valueOf("testSuperSimpleWithReverseScan");
5821     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
5822     TEST_UTIL.waitTableAvailable(TABLE, 10000);
5823 
5824     Put put = new Put(Bytes.toBytes("0-b11111-0000000000000000000"));
5825     put.add(FAMILY, QUALIFIER, VALUE);
5826     ht.put(put);
5827     put = new Put(Bytes.toBytes("0-b11111-0000000000000000002"));
5828     put.add(FAMILY, QUALIFIER, VALUE);
5829     ht.put(put);
5830     put = new Put(Bytes.toBytes("0-b11111-0000000000000000004"));
5831     put.add(FAMILY, QUALIFIER, VALUE);
5832     ht.put(put);
5833     put = new Put(Bytes.toBytes("0-b11111-0000000000000000006"));
5834     put.add(FAMILY, QUALIFIER, VALUE);
5835     ht.put(put);
5836     put = new Put(Bytes.toBytes("0-b11111-0000000000000000008"));
5837     put.add(FAMILY, QUALIFIER, VALUE);
5838     ht.put(put);
5839     put = new Put(Bytes.toBytes("0-b22222-0000000000000000001"));
5840     put.add(FAMILY, QUALIFIER, VALUE);
5841     ht.put(put);
5842     put = new Put(Bytes.toBytes("0-b22222-0000000000000000003"));
5843     put.add(FAMILY, QUALIFIER, VALUE);
5844     ht.put(put);
5845     put = new Put(Bytes.toBytes("0-b22222-0000000000000000005"));
5846     put.add(FAMILY, QUALIFIER, VALUE);
5847     ht.put(put);
5848     put = new Put(Bytes.toBytes("0-b22222-0000000000000000007"));
5849     put.add(FAMILY, QUALIFIER, VALUE);
5850     ht.put(put);
5851     put = new Put(Bytes.toBytes("0-b22222-0000000000000000009"));
5852     put.add(FAMILY, QUALIFIER, VALUE);
5853     ht.put(put);
5854     Scan scan = new Scan(Bytes.toBytes("0-b11111-9223372036854775807"),
5855         Bytes.toBytes("0-b11111-0000000000000000000"));
5856     scan.setReversed(true);
5857     ResultScanner scanner = ht.getScanner(scan);
5858     Result result = scanner.next();
5859     assertTrue(Bytes.equals(result.getRow(),
5860         Bytes.toBytes("0-b11111-0000000000000000008")));
5861     scanner.close();
5862     ht.close();
5863   }
5864 
5865   @Test
5866   public void testFiltersWithReverseScan() throws Exception {
5867     TableName TABLE = TableName.valueOf("testFiltersWithReverseScan");
5868     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
5869     TEST_UTIL.waitTableAvailable(TABLE, 10000);
5870 
5871     byte[][] ROWS = makeN(ROW, 10);
5872     byte[][] QUALIFIERS = { Bytes.toBytes("col0-<d2v1>-<d3v2>"),
5873         Bytes.toBytes("col1-<d2v1>-<d3v2>"),
5874         Bytes.toBytes("col2-<d2v1>-<d3v2>"),
5875         Bytes.toBytes("col3-<d2v1>-<d3v2>"),
5876         Bytes.toBytes("col4-<d2v1>-<d3v2>"),
5877         Bytes.toBytes("col5-<d2v1>-<d3v2>"),
5878         Bytes.toBytes("col6-<d2v1>-<d3v2>"),
5879         Bytes.toBytes("col7-<d2v1>-<d3v2>"),
5880         Bytes.toBytes("col8-<d2v1>-<d3v2>"),
5881         Bytes.toBytes("col9-<d2v1>-<d3v2>") };
5882     for (int i = 0; i < 10; i++) {
5883       Put put = new Put(ROWS[i]);
5884       put.add(FAMILY, QUALIFIERS[i], VALUE);
5885       ht.put(put);
5886     }
5887     Scan scan = new Scan();
5888     scan.setReversed(true);
5889     scan.addFamily(FAMILY);
5890     Filter filter = new QualifierFilter(CompareOp.EQUAL,
5891         new RegexStringComparator("col[1-5]"));
5892     scan.setFilter(filter);
5893     ResultScanner scanner = ht.getScanner(scan);
5894     int expectedIndex = 5;
5895     for (Result result : scanner) {
5896       assertEquals(result.size(), 1);
5897       assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[expectedIndex]));
5898       assertTrue(Bytes.equals(result.raw()[0].getQualifier(),
5899           QUALIFIERS[expectedIndex]));
5900       expectedIndex--;
5901     }
5902     assertEquals(expectedIndex, 0);
5903     scanner.close();
5904     ht.close();
5905   }
5906 
5907   @Test
5908   public void testKeyOnlyFilterWithReverseScan() throws Exception {
5909     TableName TABLE = TableName.valueOf("testKeyOnlyFilterWithReverseScan");
5910     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
5911     TEST_UTIL.waitTableAvailable(TABLE, 10000);
5912 
5913     byte[][] ROWS = makeN(ROW, 10);
5914     byte[][] QUALIFIERS = { Bytes.toBytes("col0-<d2v1>-<d3v2>"),
5915         Bytes.toBytes("col1-<d2v1>-<d3v2>"),
5916         Bytes.toBytes("col2-<d2v1>-<d3v2>"),
5917         Bytes.toBytes("col3-<d2v1>-<d3v2>"),
5918         Bytes.toBytes("col4-<d2v1>-<d3v2>"),
5919         Bytes.toBytes("col5-<d2v1>-<d3v2>"),
5920         Bytes.toBytes("col6-<d2v1>-<d3v2>"),
5921         Bytes.toBytes("col7-<d2v1>-<d3v2>"),
5922         Bytes.toBytes("col8-<d2v1>-<d3v2>"),
5923         Bytes.toBytes("col9-<d2v1>-<d3v2>") };
5924     for (int i = 0; i < 10; i++) {
5925       Put put = new Put(ROWS[i]);
5926       put.add(FAMILY, QUALIFIERS[i], VALUE);
5927       ht.put(put);
5928     }
5929     Scan scan = new Scan();
5930     scan.setReversed(true);
5931     scan.addFamily(FAMILY);
5932     Filter filter = new KeyOnlyFilter(true);
5933     scan.setFilter(filter);
5934     ResultScanner scanner = ht.getScanner(scan);
5935     int count = 0;
5936     for (Result result : ht.getScanner(scan)) {
5937       assertEquals(result.size(), 1);
5938       assertEquals(result.raw()[0].getValueLength(), Bytes.SIZEOF_INT);
5939       assertEquals(Bytes.toInt(result.raw()[0].getValue()), VALUE.length);
5940       count++;
5941     }
5942     assertEquals(count, 10);
5943     scanner.close();
5944     ht.close();
5945   }
5946 
5947   /**
5948    * Test simple table and non-existent row cases.
5949    */
5950   @Test
5951   public void testSimpleMissingWithReverseScan() throws Exception {
5952     TableName TABLE = TableName.valueOf("testSimpleMissingWithReverseScan");
5953     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
5954     TEST_UTIL.waitTableAvailable(TABLE, 10000);
5955 
5956     byte[][] ROWS = makeN(ROW, 4);
5957 
5958     // Try to get a row on an empty table
5959     Scan scan = new Scan();
5960     scan.setReversed(true);
5961     Result result = getSingleScanResult(ht, scan);
5962     assertNullResult(result);
5963 
5964     scan = new Scan(ROWS[0]);
5965     scan.setReversed(true);
5966     result = getSingleScanResult(ht, scan);
5967     assertNullResult(result);
5968 
5969     scan = new Scan(ROWS[0], ROWS[1]);
5970     scan.setReversed(true);
5971     result = getSingleScanResult(ht, scan);
5972     assertNullResult(result);
5973 
5974     scan = new Scan();
5975     scan.setReversed(true);
5976     scan.addFamily(FAMILY);
5977     result = getSingleScanResult(ht, scan);
5978     assertNullResult(result);
5979 
5980     scan = new Scan();
5981     scan.setReversed(true);
5982     scan.addColumn(FAMILY, QUALIFIER);
5983     result = getSingleScanResult(ht, scan);
5984     assertNullResult(result);
5985 
5986     // Insert a row
5987 
5988     Put put = new Put(ROWS[2]);
5989     put.add(FAMILY, QUALIFIER, VALUE);
5990     ht.put(put);
5991 
5992     // Make sure we can scan the row
5993     scan = new Scan();
5994     scan.setReversed(true);
5995     result = getSingleScanResult(ht, scan);
5996     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
5997 
5998     scan = new Scan(ROWS[3], ROWS[0]);
5999     scan.setReversed(true);
6000     result = getSingleScanResult(ht, scan);
6001     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
6002 
6003     scan = new Scan(ROWS[2], ROWS[1]);
6004     scan.setReversed(true);
6005     result = getSingleScanResult(ht, scan);
6006     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
6007 
6008     // Try to scan empty rows around it
6009     // Introduced MemStore#shouldSeekForReverseScan to fix the following
6010     scan = new Scan(ROWS[1]);
6011     scan.setReversed(true);
6012     result = getSingleScanResult(ht, scan);
6013     assertNullResult(result);
6014     ht.close();
6015   }
6016 
6017   @Test
6018   public void testNullWithReverseScan() throws Exception {
6019     TableName TABLE = TableName.valueOf("testNullWithReverseScan");
6020     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
6021     TEST_UTIL.waitTableAvailable(TABLE, 10000);
6022 
6023     // Null qualifier (should work)
6024     Put put = new Put(ROW);
6025     put.add(FAMILY, null, VALUE);
6026     ht.put(put);
6027     scanTestNull(ht, ROW, FAMILY, VALUE, true);
6028     Delete delete = new Delete(ROW);
6029     delete.deleteColumns(FAMILY, null);
6030     ht.delete(delete);
6031     // Use a new table
6032     byte[] TABLE2 = Bytes.toBytes("testNull2WithReverseScan");
6033     ht = TEST_UTIL.createTable(TableName.valueOf(TABLE2), FAMILY);
6034     TEST_UTIL.waitTableAvailable(TableName.valueOf(TABLE2), 10000);
6035 
6036     // Empty qualifier, byte[0] instead of null (should work)
6037     put = new Put(ROW);
6038     put.add(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
6039     ht.put(put);
6040     scanTestNull(ht, ROW, FAMILY, VALUE, true);
6041     TEST_UTIL.flush();
6042     scanTestNull(ht, ROW, FAMILY, VALUE, true);
6043     delete = new Delete(ROW);
6044     delete.deleteColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
6045     ht.delete(delete);
6046     // Null value
6047     put = new Put(ROW);
6048     put.add(FAMILY, QUALIFIER, null);
6049     ht.put(put);
6050     Scan scan = new Scan();
6051     scan.setReversed(true);
6052     scan.addColumn(FAMILY, QUALIFIER);
6053     Result result = getSingleScanResult(ht, scan);
6054     assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
6055     ht.close();
6056   }
6057 
6058   @Test
6059   public void testDeletesWithReverseScan() throws Exception {
6060     TableName TABLE = TableName.valueOf("testDeletesWithReverseScan");
6061     byte[][] ROWS = makeNAscii(ROW, 6);
6062     byte[][] FAMILIES = makeNAscii(FAMILY, 3);
6063     byte[][] VALUES = makeN(VALUE, 5);
6064     long[] ts = { 1000, 2000, 3000, 4000, 5000 };
6065     Table ht = TEST_UTIL.createTable(TABLE, FAMILIES, TEST_UTIL.getConfiguration(), 3);
6066     TEST_UTIL.waitTableAvailable(TABLE, 10000);
6067 
6068     Put put = new Put(ROW);
6069     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
6070     put.add(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
6071     ht.put(put);
6072 
6073     Delete delete = new Delete(ROW);
6074     delete.deleteFamily(FAMILIES[0], ts[0]);
6075     ht.delete(delete);
6076 
6077     Scan scan = new Scan(ROW);
6078     scan.setReversed(true);
6079     scan.addFamily(FAMILIES[0]);
6080     scan.setMaxVersions(Integer.MAX_VALUE);
6081     Result result = getSingleScanResult(ht, scan);
6082     assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1] },
6083         new byte[][] { VALUES[1] }, 0, 0);
6084 
6085     // Test delete latest version
6086     put = new Put(ROW);
6087     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
6088     put.add(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
6089     put.add(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
6090     put.add(FAMILIES[0], null, ts[4], VALUES[4]);
6091     put.add(FAMILIES[0], null, ts[2], VALUES[2]);
6092     put.add(FAMILIES[0], null, ts[3], VALUES[3]);
6093     ht.put(put);
6094 
6095     delete = new Delete(ROW);
6096     delete.deleteColumn(FAMILIES[0], QUALIFIER); // ts[4]
6097     ht.delete(delete);
6098 
6099     scan = new Scan(ROW);
6100     scan.setReversed(true);
6101     scan.addColumn(FAMILIES[0], QUALIFIER);
6102     scan.setMaxVersions(Integer.MAX_VALUE);
6103     result = getSingleScanResult(ht, scan);
6104     assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1],
6105         ts[2], ts[3] }, new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);
6106 
6107     // Test for HBASE-1847
6108     delete = new Delete(ROW);
6109     delete.deleteColumn(FAMILIES[0], null);
6110     ht.delete(delete);
6111 
6112     // Cleanup null qualifier
6113     delete = new Delete(ROW);
6114     delete.deleteColumns(FAMILIES[0], null);
6115     ht.delete(delete);
6116 
6117     // Expected client behavior might be that you can re-put deleted values
6118     // But alas, this is not to be. We can't put them back in either case.
6119 
6120     put = new Put(ROW);
6121     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); // 1000
6122     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); // 5000
6123     ht.put(put);
6124 
6125     // The Scanner returns the previous values, the expected-naive-unexpected
6126     // behavior
6127 
6128     scan = new Scan(ROW);
6129     scan.setReversed(true);
6130     scan.addFamily(FAMILIES[0]);
6131     scan.setMaxVersions(Integer.MAX_VALUE);
6132     result = getSingleScanResult(ht, scan);
6133     assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1],
6134         ts[2], ts[3] }, new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);
6135 
6136     // Test deleting an entire family from one row but not the other various
6137     // ways
6138 
6139     put = new Put(ROWS[0]);
6140     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
6141     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
6142     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
6143     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
6144     ht.put(put);
6145 
6146     put = new Put(ROWS[1]);
6147     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
6148     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
6149     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
6150     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
6151     ht.put(put);
6152 
6153     put = new Put(ROWS[2]);
6154     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
6155     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
6156     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
6157     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
6158     ht.put(put);
6159 
6160     delete = new Delete(ROWS[0]);
6161     delete.deleteFamily(FAMILIES[2]);
6162     ht.delete(delete);
6163 
6164     delete = new Delete(ROWS[1]);
6165     delete.deleteColumns(FAMILIES[1], QUALIFIER);
6166     ht.delete(delete);
6167 
6168     delete = new Delete(ROWS[2]);
6169     delete.deleteColumn(FAMILIES[1], QUALIFIER);
6170     delete.deleteColumn(FAMILIES[1], QUALIFIER);
6171     delete.deleteColumn(FAMILIES[2], QUALIFIER);
6172     ht.delete(delete);
6173 
6174     scan = new Scan(ROWS[0]);
6175     scan.setReversed(true);
6176     scan.addFamily(FAMILIES[1]);
6177     scan.addFamily(FAMILIES[2]);
6178     scan.setMaxVersions(Integer.MAX_VALUE);
6179     result = getSingleScanResult(ht, scan);
6180     assertTrue("Expected 2 keys but received " + result.size(),
6181         result.size() == 2);
6182     assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER, new long[] { ts[0],
6183         ts[1] }, new byte[][] { VALUES[0], VALUES[1] }, 0, 1);
6184 
6185     scan = new Scan(ROWS[1]);
6186     scan.setReversed(true);
6187     scan.addFamily(FAMILIES[1]);
6188     scan.addFamily(FAMILIES[2]);
6189     scan.setMaxVersions(Integer.MAX_VALUE);
6190     result = getSingleScanResult(ht, scan);
6191     assertTrue("Expected 2 keys but received " + result.size(),
6192         result.size() == 2);
6193 
6194     scan = new Scan(ROWS[2]);
6195     scan.setReversed(true);
6196     scan.addFamily(FAMILIES[1]);
6197     scan.addFamily(FAMILIES[2]);
6198     scan.setMaxVersions(Integer.MAX_VALUE);
6199     result = getSingleScanResult(ht, scan);
6200     assertEquals(1, result.size());
6201     assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
6202         new long[] { ts[2] }, new byte[][] { VALUES[2] }, 0, 0);
6203 
6204     // Test if we delete the family first in one row (HBASE-1541)
6205 
6206     delete = new Delete(ROWS[3]);
6207     delete.deleteFamily(FAMILIES[1]);
6208     ht.delete(delete);
6209 
6210     put = new Put(ROWS[3]);
6211     put.add(FAMILIES[2], QUALIFIER, VALUES[0]);
6212     ht.put(put);
6213 
6214     put = new Put(ROWS[4]);
6215     put.add(FAMILIES[1], QUALIFIER, VALUES[1]);
6216     put.add(FAMILIES[2], QUALIFIER, VALUES[2]);
6217     ht.put(put);
6218 
6219     scan = new Scan(ROWS[4]);
6220     scan.setReversed(true);
6221     scan.addFamily(FAMILIES[1]);
6222     scan.addFamily(FAMILIES[2]);
6223     scan.setMaxVersions(Integer.MAX_VALUE);
6224     ResultScanner scanner = ht.getScanner(scan);
6225     result = scanner.next();
6226     assertTrue("Expected 2 keys but received " + result.size(),
6227         result.size() == 2);
6228     assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[4]));
6229     assertTrue(Bytes.equals(result.raw()[1].getRow(), ROWS[4]));
6230     assertTrue(Bytes.equals(result.raw()[0].getValue(), VALUES[1]));
6231     assertTrue(Bytes.equals(result.raw()[1].getValue(), VALUES[2]));
6232     result = scanner.next();
6233     assertTrue("Expected 1 key but received " + result.size(),
6234         result.size() == 1);
6235     assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[3]));
6236     assertTrue(Bytes.equals(result.raw()[0].getValue(), VALUES[0]));
6237     scanner.close();
6238     ht.close();
6239   }
6240 
6241   /**
6242    * Tests reversed scan under multi regions
6243    */
6244   @Test
6245   public void testReversedScanUnderMultiRegions() throws Exception {
6246     // Test Initialization.
6247     TableName TABLE = TableName.valueOf("testReversedScanUnderMultiRegions");
6248     byte[] maxByteArray = ReversedClientScanner.MAX_BYTE_ARRAY;
6249     byte[][] splitRows = new byte[][] { Bytes.toBytes("005"),
6250         Bytes.add(Bytes.toBytes("005"), Bytes.multiple(maxByteArray, 16)),
6251         Bytes.toBytes("006"),
6252         Bytes.add(Bytes.toBytes("006"), Bytes.multiple(maxByteArray, 8)),
6253         Bytes.toBytes("007"),
6254         Bytes.add(Bytes.toBytes("007"), Bytes.multiple(maxByteArray, 4)),
6255         Bytes.toBytes("008"), Bytes.multiple(maxByteArray, 2) };
6256     HTable table = TEST_UTIL.createTable(TABLE, FAMILY, splitRows);
6257     TEST_UTIL.waitTableAvailable(TABLE, 10000);
6258 
6259     TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
6260 
6261     assertEquals(splitRows.length + 1, table.getRegionLocations().size());
6262     // Insert one row each region
6263     int insertNum = splitRows.length;
6264     for (int i = 0; i < insertNum; i++) {
6265       Put put = new Put(splitRows[i]);
6266       put.add(FAMILY, QUALIFIER, VALUE);
6267       table.put(put);
6268     }
6269 
6270     // scan forward
6271     ResultScanner scanner = table.getScanner(new Scan());
6272     int count = 0;
6273     for (Result r : scanner) {
6274       assertTrue(!r.isEmpty());
6275       count++;
6276     }
6277     assertEquals(insertNum, count);
6278 
6279     // scan backward
6280     Scan scan = new Scan();
6281     scan.setReversed(true);
6282     scanner = table.getScanner(scan);
6283     count = 0;
6284     byte[] lastRow = null;
6285     for (Result r : scanner) {
6286       assertTrue(!r.isEmpty());
6287       count++;
6288       byte[] thisRow = r.getRow();
6289       if (lastRow != null) {
6290         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6291             + ",this row=" + Bytes.toString(thisRow),
6292             Bytes.compareTo(thisRow, lastRow) < 0);
6293       }
6294       lastRow = thisRow;
6295     }
6296     assertEquals(insertNum, count);
6297     table.close();
6298   }
6299 
6300   /**
6301    * Tests reversed scan under multi regions
6302    */
6303   @Test
6304   public void testSmallReversedScanUnderMultiRegions() throws Exception {
6305     // Test Initialization.
6306     TableName TABLE = TableName.valueOf("testSmallReversedScanUnderMultiRegions");
6307     byte[][] splitRows = new byte[][]{
6308         Bytes.toBytes("000"), Bytes.toBytes("002"), Bytes.toBytes("004"),
6309         Bytes.toBytes("006"), Bytes.toBytes("008"), Bytes.toBytes("010")};
6310     HTable table = TEST_UTIL.createTable(TABLE, FAMILY, splitRows);
6311     TEST_UTIL.waitTableAvailable(TABLE, 10000);
6312 
6313     TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
6314 
6315     assertEquals(splitRows.length + 1, table.getRegionLocations().size());
6316     for (byte[] splitRow : splitRows) {
6317       Put put = new Put(splitRow);
6318       put.add(FAMILY, QUALIFIER, VALUE);
6319       table.put(put);
6320 
6321       byte[] nextRow = Bytes.copy(splitRow);
6322       nextRow[nextRow.length - 1]++;
6323 
6324       put = new Put(nextRow);
6325       put.add(FAMILY, QUALIFIER, VALUE);
6326       table.put(put);
6327     }
6328 
6329     // scan forward
6330     ResultScanner scanner = table.getScanner(new Scan());
6331     int count = 0;
6332     for (Result r : scanner) {
6333       assertTrue(!r.isEmpty());
6334       count++;
6335     }
6336     assertEquals(12, count);
6337 
6338     reverseScanTest(table, false);
6339     reverseScanTest(table, true);
6340 
6341     table.close();
6342   }
6343 
6344   private void reverseScanTest(Table table, boolean small) throws IOException {
6345     // scan backward
6346     Scan scan = new Scan();
6347     scan.setReversed(true);
6348     ResultScanner scanner = table.getScanner(scan);
6349     int count = 0;
6350     byte[] lastRow = null;
6351     for (Result r : scanner) {
6352       assertTrue(!r.isEmpty());
6353       count++;
6354       byte[] thisRow = r.getRow();
6355       if (lastRow != null) {
6356         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6357             + ",this row=" + Bytes.toString(thisRow),
6358             Bytes.compareTo(thisRow, lastRow) < 0);
6359       }
6360       lastRow = thisRow;
6361     }
6362     assertEquals(12, count);
6363 
6364     scan = new Scan();
6365     scan.setSmall(small);
6366     scan.setReversed(true);
6367     scan.setStartRow(Bytes.toBytes("002"));
6368     scanner = table.getScanner(scan);
6369     count = 0;
6370     lastRow = null;
6371     for (Result r : scanner) {
6372       assertTrue(!r.isEmpty());
6373       count++;
6374       byte[] thisRow = r.getRow();
6375       if (lastRow != null) {
6376         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6377             + ",this row=" + Bytes.toString(thisRow),
6378             Bytes.compareTo(thisRow, lastRow) < 0);
6379       }
6380       lastRow = thisRow;
6381     }
6382     assertEquals(3, count); // 000 001 002
6383 
6384     scan = new Scan();
6385     scan.setSmall(small);
6386     scan.setReversed(true);
6387     scan.setStartRow(Bytes.toBytes("002"));
6388     scan.setStopRow(Bytes.toBytes("000"));
6389     scanner = table.getScanner(scan);
6390     count = 0;
6391     lastRow = null;
6392     for (Result r : scanner) {
6393       assertTrue(!r.isEmpty());
6394       count++;
6395       byte[] thisRow = r.getRow();
6396       if (lastRow != null) {
6397         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6398             + ",this row=" + Bytes.toString(thisRow),
6399             Bytes.compareTo(thisRow, lastRow) < 0);
6400       }
6401       lastRow = thisRow;
6402     }
6403     assertEquals(2, count); // 001 002
6404 
6405     scan = new Scan();
6406     scan.setSmall(small);
6407     scan.setReversed(true);
6408     scan.setStartRow(Bytes.toBytes("001"));
6409     scanner = table.getScanner(scan);
6410     count = 0;
6411     lastRow = null;
6412     for (Result r : scanner) {
6413       assertTrue(!r.isEmpty());
6414       count++;
6415       byte[] thisRow = r.getRow();
6416       if (lastRow != null) {
6417         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6418             + ",this row=" + Bytes.toString(thisRow),
6419             Bytes.compareTo(thisRow, lastRow) < 0);
6420       }
6421       lastRow = thisRow;
6422     }
6423     assertEquals(2, count); // 000 001
6424 
6425     scan = new Scan();
6426     scan.setSmall(small);
6427     scan.setReversed(true);
6428     scan.setStartRow(Bytes.toBytes("000"));
6429     scanner = table.getScanner(scan);
6430     count = 0;
6431     lastRow = null;
6432     for (Result r : scanner) {
6433       assertTrue(!r.isEmpty());
6434       count++;
6435       byte[] thisRow = r.getRow();
6436       if (lastRow != null) {
6437         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6438             + ",this row=" + Bytes.toString(thisRow),
6439             Bytes.compareTo(thisRow, lastRow) < 0);
6440       }
6441       lastRow = thisRow;
6442     }
6443     assertEquals(1, count); // 000
6444 
6445     scan = new Scan();
6446     scan.setSmall(small);
6447     scan.setReversed(true);
6448     scan.setStartRow(Bytes.toBytes("006"));
6449     scan.setStopRow(Bytes.toBytes("002"));
6450     scanner = table.getScanner(scan);
6451     count = 0;
6452     lastRow = null;
6453     for (Result r : scanner) {
6454       assertTrue(!r.isEmpty());
6455       count++;
6456       byte[] thisRow = r.getRow();
6457       if (lastRow != null) {
6458         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6459             + ",this row=" + Bytes.toString(thisRow),
6460             Bytes.compareTo(thisRow, lastRow) < 0);
6461       }
6462       lastRow = thisRow;
6463     }
6464     assertEquals(4, count); // 003 004 005 006
6465   }
6466 
6467   @Test
6468   public void testGetStartEndKeysWithRegionReplicas() throws IOException, InterruptedException {
6469     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testGetStartEndKeys"));
6470     HColumnDescriptor fam = new HColumnDescriptor(FAMILY);
6471     htd.addFamily(fam);
6472     byte[][] KEYS = HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE;
6473     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
6474     admin.createTable(htd, KEYS);
6475     TEST_UTIL.waitTableAvailable(htd.getTableName(), 10000);
6476     List<HRegionInfo> regions = admin.getTableRegions(htd.getTableName());
6477 
6478     HRegionLocator locator =
6479         (HRegionLocator) admin.getConnection().getRegionLocator(htd.getTableName());
6480     for (int regionReplication = 1; regionReplication < 4; regionReplication++) {
6481       List<RegionLocations> regionLocations = new ArrayList<RegionLocations>();
6482 
6483       // mock region locations coming from meta with multiple replicas
6484       for (HRegionInfo region : regions) {
6485         HRegionLocation[] arr = new HRegionLocation[regionReplication];
6486         for (int i = 0; i < arr.length; i++) {
6487           arr[i] = new HRegionLocation(RegionReplicaUtil.getRegionInfoForReplica(region, i), null);
6488         }
6489         regionLocations.add(new RegionLocations(arr));
6490       }
6491 
6492       Pair<byte[][], byte[][]> startEndKeys = locator.getStartEndKeys(regionLocations);
6493 
6494       assertEquals(KEYS.length + 1, startEndKeys.getFirst().length);
6495 
6496       for (int i = 0; i < KEYS.length + 1; i++) {
6497         byte[] startKey = i == 0 ? HConstants.EMPTY_START_ROW : KEYS[i - 1];
6498         byte[] endKey = i == KEYS.length ? HConstants.EMPTY_END_ROW : KEYS[i];
6499         assertArrayEquals(startKey, startEndKeys.getFirst()[i]);
6500         assertArrayEquals(endKey, startEndKeys.getSecond()[i]);
6501       }
6502     }
6503   }
6504 
6505   @Test
6506   public void testFilterAllRecords() throws IOException {
6507     Scan scan = new Scan();
6508     scan.setBatch(1);
6509     scan.setCaching(1);
6510     // Filter out any records
6511     scan.setFilter(new FilterList(new FirstKeyOnlyFilter(), new InclusiveStopFilter(new byte[0])));
6512     Table table = TEST_UTIL.getConnection().getTable(TableName.NAMESPACE_TABLE_NAME);
6513     ResultScanner s = table.getScanner(scan);
6514     assertNull(s.next());
6515     table.close();
6516   }
6517 
6518   @Test
6519   public void testRegionCache() throws IOException, InterruptedException {
6520     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionCache"));
6521     HColumnDescriptor fam = new HColumnDescriptor(FAMILY);
6522     htd.addFamily(fam);
6523     byte[][] KEYS = HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE;
6524     Admin admin = TEST_UTIL.getHBaseAdmin();
6525     admin.createTable(htd, KEYS);
6526     TEST_UTIL.waitTableAvailable(htd.getTableName(), 10000);
6527 
6528     HRegionLocator locator =
6529       (HRegionLocator) admin.getConnection().getRegionLocator(htd.getTableName());
6530     List<HRegionLocation> results = locator.getAllRegionLocations();
6531     int number = ((ConnectionManager.HConnectionImplementation)admin.getConnection())
6532       .getNumberOfCachedRegionLocations(htd.getTableName());
6533     assertEquals(results.size(), number);
6534   }
6535 }