View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.omid.transaction;
19  
20  import static org.mockito.Matchers.any;
21  import static org.mockito.Mockito.doAnswer;
22  import static org.mockito.Mockito.doReturn;
23  import static org.mockito.Mockito.doThrow;
24  import static org.mockito.Mockito.spy;
25  import static org.testng.Assert.assertEquals;
26  import static org.testng.Assert.assertFalse;
27  import static org.testng.Assert.assertNull;
28  import static org.testng.Assert.assertTrue;
29  import static org.testng.Assert.fail;
30  
31  import java.io.IOException;
32  import java.util.ArrayList;
33  import java.util.List;
34  import java.util.Random;
35  import java.util.Set;
36  import java.util.concurrent.TimeUnit;
37  import java.util.concurrent.atomic.AtomicBoolean;
38  
39  import org.apache.hadoop.conf.Configuration;
40  import org.apache.hadoop.hbase.Cell;
41  import org.apache.hadoop.hbase.CellUtil;
42  import org.apache.hadoop.hbase.Coprocessor;
43  import org.apache.hadoop.hbase.HBaseTestingUtility;
44  import org.apache.hadoop.hbase.HColumnDescriptor;
45  import org.apache.hadoop.hbase.HTableDescriptor;
46  import org.apache.hadoop.hbase.MiniHBaseCluster;
47  import org.apache.hadoop.hbase.TableName;
48  import org.apache.hadoop.hbase.client.Admin;
49  import org.apache.hadoop.hbase.client.Connection;
50  import org.apache.hadoop.hbase.client.ConnectionFactory;
51  import org.apache.hadoop.hbase.client.Delete;
52  import org.apache.hadoop.hbase.client.Get;
53  import org.apache.hadoop.hbase.client.Put;
54  import org.apache.hadoop.hbase.client.Result;
55  import org.apache.hadoop.hbase.client.ResultScanner;
56  import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
57  import org.apache.hadoop.hbase.client.Row;
58  import org.apache.hadoop.hbase.client.Scan;
59  import org.apache.hadoop.hbase.client.Table;
60  import org.apache.hadoop.hbase.util.Bytes;
61  import org.apache.omid.HBaseShims;
62  import org.apache.omid.TestUtils;
63  import org.apache.omid.committable.CommitTable;
64  import org.apache.omid.committable.hbase.HBaseCommitTableConfig;
65  import org.apache.omid.metrics.NullMetricsProvider;
66  import org.apache.omid.timestamp.storage.HBaseTimestampStorageConfig;
67  import org.apache.omid.tso.TSOServer;
68  import org.apache.omid.tso.TSOServerConfig;
69  import org.apache.omid.tso.client.OmidClientConfiguration;
70  import org.mockito.invocation.InvocationOnMock;
71  import org.mockito.stubbing.Answer;
72  import org.slf4j.Logger;
73  import org.slf4j.LoggerFactory;
74  import org.testng.annotations.AfterClass;
75  import org.testng.annotations.BeforeClass;
76  import org.testng.annotations.BeforeMethod;
77  import org.testng.annotations.Test;
78  
79  import org.apache.phoenix.thirdparty.com.google.common.util.concurrent.SettableFuture;
80  import com.google.inject.Guice;
81  import com.google.inject.Injector;
82  
83  public class TestCompaction {
84  
85      private static final Logger LOG = LoggerFactory.getLogger(TestCompaction.class);
86  
87      private static final String TEST_FAMILY = "test-fam";
88      private static final String TEST_QUALIFIER = "test-qual";
89  
90      private final byte[] fam = Bytes.toBytes(TEST_FAMILY);
91      private final byte[] qual = Bytes.toBytes(TEST_QUALIFIER);
92      private final byte[] data = Bytes.toBytes("testWrite-1");
93  
94      private static final int MAX_VERSIONS = 3;
95  
96      private Random randomGenerator;
97      private AbstractTransactionManager tm;
98  
99      private Injector injector;
100 
101     private Admin admin;
102     private Configuration hbaseConf;
103     private HBaseTestingUtility hbaseTestUtil;
104     private MiniHBaseCluster hbaseCluster;
105 
106     private TSOServer tso;
107 
108 
109     private CommitTable commitTable;
110     private PostCommitActions syncPostCommitter;
111     private static Connection connection;
112 
113     @BeforeClass
114     public void setupTestCompation() throws Exception {
115         TSOServerConfig tsoConfig = new TSOServerConfig();
116         tsoConfig.setPort(1234);
117         tsoConfig.setConflictMapSize(1);
118         tsoConfig.setWaitStrategy("LOW_CPU");
119         injector = Guice.createInjector(new TSOForHBaseCompactorTestModule(tsoConfig));
120         hbaseConf = injector.getInstance(Configuration.class);
121         HBaseCommitTableConfig hBaseCommitTableConfig = injector.getInstance(HBaseCommitTableConfig.class);
122         HBaseTimestampStorageConfig hBaseTimestampStorageConfig = injector.getInstance(HBaseTimestampStorageConfig.class);
123 
124         // settings required for #testDuplicateDeletes()
125         hbaseConf.setInt("hbase.hstore.compaction.min", 2);
126         hbaseConf.setInt("hbase.hstore.compaction.max", 2);
127         setupHBase();
128         connection = ConnectionFactory.createConnection(hbaseConf);
129         admin = connection.getAdmin();
130         createRequiredHBaseTables(hBaseTimestampStorageConfig, hBaseCommitTableConfig);
131         setupTSO();
132 
133         commitTable = injector.getInstance(CommitTable.class);
134     }
135 
136     private void setupHBase() throws Exception {
137         LOG.info("--------------------------------------------------------------------------------------------------");
138         LOG.info("Setting up HBase");
139         LOG.info("--------------------------------------------------------------------------------------------------");
140         hbaseTestUtil = new HBaseTestingUtility(hbaseConf);
141         LOG.info("--------------------------------------------------------------------------------------------------");
142         LOG.info("Creating HBase MiniCluster");
143         LOG.info("--------------------------------------------------------------------------------------------------");
144         hbaseCluster = hbaseTestUtil.startMiniCluster(1);
145     }
146 
147     private void createRequiredHBaseTables(HBaseTimestampStorageConfig timestampStorageConfig,
148                                            HBaseCommitTableConfig hBaseCommitTableConfig) throws IOException {
149         createTableIfNotExists(timestampStorageConfig.getTableName(), timestampStorageConfig.getFamilyName().getBytes());
150 
151         createTableIfNotExists(hBaseCommitTableConfig.getTableName(), hBaseCommitTableConfig.getCommitTableFamily(), hBaseCommitTableConfig.getLowWatermarkFamily());
152     }
153 
154     private void createTableIfNotExists(String tableName, byte[]... families) throws IOException {
155         if (!admin.tableExists(TableName.valueOf(tableName))) {
156             LOG.info("Creating {} table...", tableName);
157             HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
158 
159             for (byte[] family : families) {
160                 HColumnDescriptor datafam = new HColumnDescriptor(family);
161                 datafam.setMaxVersions(MAX_VERSIONS);
162                 desc.addFamily(datafam);
163             }
164 
165             desc.addCoprocessor("org.apache.hadoop.hbase.coprocessor.AggregateImplementation",null,Coprocessor.PRIORITY_HIGHEST,null);
166             admin.createTable(desc);
167             for (byte[] family : families) {
168                 CompactorUtil.enableOmidCompaction(connection, TableName.valueOf(tableName), family);
169             }
170         }
171 
172     }
173 
174     private void setupTSO() throws IOException, InterruptedException {
175         tso = injector.getInstance(TSOServer.class);
176         tso.startAsync();
177         tso.awaitRunning();
178         TestUtils.waitForSocketListening("localhost", 1234, 100);
179         Thread.currentThread().setName("UnitTest(s) thread");
180     }
181 
182     @AfterClass
183     public void cleanupTestCompation() throws Exception {
184         teardownTSO();
185         hbaseCluster.shutdown();
186     }
187 
188     private void teardownTSO() throws IOException, InterruptedException {
189         tso.stopAsync();
190         tso.awaitTerminated();
191         TestUtils.waitForSocketNotListening("localhost", 1234, 1000);
192     }
193 
194     @BeforeMethod
195     public void setupTestCompactionIndividualTest() throws Exception {
196         randomGenerator = new Random(0xfeedcafeL);
197         tm = spy((AbstractTransactionManager) newTransactionManager());
198     }
199 
200     private TransactionManager newTransactionManager() throws Exception {
201         HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
202         hbaseOmidClientConf.setConnectionString("localhost:1234");
203         hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
204         CommitTable.Client commitTableClient = commitTable.getClient();
205         syncPostCommitter =
206                 spy(new HBaseSyncPostCommitter(new NullMetricsProvider(),commitTableClient, connection));
207         return HBaseTransactionManager.builder(hbaseOmidClientConf)
208                 .postCommitter(syncPostCommitter)
209                 .commitTableClient(commitTableClient)
210                 .build();
211     }
212 
213 
214     @Test
215     public void testShadowCellsAboveLWMSurviveCompaction() throws Exception {
216         String TEST_TABLE = "testShadowCellsAboveLWMSurviveCompaction";
217         createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
218         TTable txTable = new TTable(connection, TEST_TABLE);
219 
220         byte[] rowId = Bytes.toBytes("row");
221 
222         // Create 3 transactions modifying the same cell in a particular row
223         HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
224         Put put1 = new Put(rowId);
225         put1.addColumn(fam, qual, Bytes.toBytes("testValue 1"));
226         txTable.put(tx1, put1);
227         tm.commit(tx1);
228 
229         HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
230         Put put2 = new Put(rowId);
231         put2.addColumn(fam, qual, Bytes.toBytes("testValue 2"));
232         txTable.put(tx2, put2);
233         tm.commit(tx2);
234 
235         HBaseTransaction tx3 = (HBaseTransaction) tm.begin();
236         Put put3 = new Put(rowId);
237         put3.addColumn(fam, qual, Bytes.toBytes("testValue 3"));
238         txTable.put(tx3, put3);
239         tm.commit(tx3);
240 
241         // Before compaction, the three timestamped values for the cell should be there
242         TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
243         assertTrue(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
244                 "Put cell of Tx1 should be there");
245         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
246                 "Put shadow cell of Tx1 should be there");
247         assertTrue(CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
248                 "Put cell of Tx2 cell should be there");
249         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
250                 "Put shadow cell of Tx2 should be there");
251         assertTrue(CellUtils.hasCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
252                 "Put cell of Tx3 cell should be there");
253         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
254                 "Put shadow cell of Tx3 should be there");
255 
256         // Compact
257         compactWithLWM(0, TEST_TABLE);
258 
259         // After compaction, the three timestamped values for the cell should be there
260         assertTrue(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
261                 "Put cell of Tx1 should be there");
262         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
263                 "Put shadow cell of Tx1 should be there");
264         assertTrue(CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
265                 "Put cell of Tx2 cell should be there");
266         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
267                 "Put shadow cell of Tx2 should be there");
268         assertTrue(CellUtils.hasCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
269                 "Put cell of Tx3 cell should be there");
270         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
271                 "Put shadow cell of Tx3 should be there");
272     }
273 
274     @Test(timeOut = 60_000)
275     public void testStandardTXsWithShadowCellsAndWithSTBelowAndAboveLWMArePresevedAfterCompaction() throws Throwable {
276         String TEST_TABLE = "testStandardTXsWithShadowCellsAndWithSTBelowAndAboveLWMArePresevedAfterCompaction";
277         createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
278         TTable txTable = new TTable(connection, TEST_TABLE);
279 
280         final int ROWS_TO_ADD = 5;
281 
282         long fakeAssignedLowWatermark = 0L;
283         for (int i = 0; i < ROWS_TO_ADD; ++i) {
284             long rowId = randomGenerator.nextLong();
285             Transaction tx = tm.begin();
286             if (i == (ROWS_TO_ADD / 2)) {
287                 fakeAssignedLowWatermark = tx.getTransactionId();
288                 LOG.info("AssignedLowWatermark " + fakeAssignedLowWatermark);
289             }
290             Put put = new Put(Bytes.toBytes(rowId));
291             put.addColumn(fam, qual, data);
292             txTable.put(tx, put);
293             tm.commit(tx);
294         }
295 
296         LOG.info("Flushing table {}", TEST_TABLE);
297         admin.flush(TableName.valueOf(TEST_TABLE));
298 
299         // Return a LWM that triggers compaction & stays between 1 and the max start timestamp assigned to previous TXs
300         LOG.info("Regions in table {}: {}", TEST_TABLE, hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).size());
301         OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).get(0)
302                 .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
303         CommitTable commitTable = injector.getInstance(CommitTable.class);
304         CommitTable.Client commitTableClient = spy(commitTable.getClient());
305         SettableFuture<Long> f = SettableFuture.create();
306         f.set(fakeAssignedLowWatermark);
307         doReturn(f).when(commitTableClient).readLowWatermark();
308         omidCompactor.commitTableClient = commitTableClient;
309         LOG.info("Compacting table {}", TEST_TABLE);
310         admin.majorCompact(TableName.valueOf(TEST_TABLE));
311 
312         LOG.info("Sleeping for 3 secs");
313         Thread.sleep(3000);
314         LOG.info("Waking up after 3 secs");
315 
316         // No rows should have been discarded after compacting
317         assertEquals(rowCount(TEST_TABLE, fam), ROWS_TO_ADD, "Rows in table after compacting should be " + ROWS_TO_ADD);
318     }
319 
320     @Test(timeOut = 60_000)
321     public void testTXWithoutShadowCellsAndWithSTBelowLWMGetsShadowCellHealedAfterCompaction() throws Exception {
322         String TEST_TABLE = "testTXWithoutShadowCellsAndWithSTBelowLWMGetsShadowCellHealedAfterCompaction";
323         createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
324         TTable txTable = new TTable(connection, TEST_TABLE);
325 
326         // The following line emulates a crash after commit that is observed in (*) below
327         doThrow(new RuntimeException()).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
328 
329         HBaseTransaction problematicTx = (HBaseTransaction) tm.begin();
330 
331         long row = randomGenerator.nextLong();
332 
333         // Test shadow cell are created properly
334         Put put = new Put(Bytes.toBytes(row));
335         put.addColumn(fam, qual, data);
336         txTable.put(problematicTx, put);
337         try {
338             tm.commit(problematicTx);
339         } catch (Exception e) { // (*) Crash
340             // Do nothing
341         }
342 
343         assertTrue(CellUtils.hasCell(Bytes.toBytes(row), fam, qual, problematicTx.getStartTimestamp(),
344                                      new TTableCellGetterAdapter(txTable)),
345                    "Cell should be there");
346         assertFalse(CellUtils.hasShadowCell(Bytes.toBytes(row), fam, qual, problematicTx.getStartTimestamp(),
347                                             new TTableCellGetterAdapter(txTable)),
348                     "Shadow cell should not be there");
349 
350         // Return a LWM that triggers compaction and has all the possible start timestamps below it
351         LOG.info("Regions in table {}: {}", TEST_TABLE, hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).size());
352         OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).get(0)
353                 .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
354         CommitTable commitTable = injector.getInstance(CommitTable.class);
355         CommitTable.Client commitTableClient = spy(commitTable.getClient());
356         SettableFuture<Long> f = SettableFuture.create();
357         f.set(Long.MAX_VALUE);
358         doReturn(f).when(commitTableClient).readLowWatermark();
359         omidCompactor.commitTableClient = commitTableClient;
360 
361         LOG.info("Flushing table {}", TEST_TABLE);
362         admin.flush(TableName.valueOf(TEST_TABLE));
363 
364         LOG.info("Compacting table {}", TEST_TABLE);
365         admin.majorCompact(TableName.valueOf(TEST_TABLE));
366 
367         LOG.info("Sleeping for 3 secs");
368         Thread.sleep(3000);
369         LOG.info("Waking up after 3 secs");
370 
371         assertTrue(CellUtils.hasCell(Bytes.toBytes(row), fam, qual, problematicTx.getStartTimestamp(),
372                                      new TTableCellGetterAdapter(txTable)),
373                    "Cell should be there");
374         assertTrue(CellUtils.hasShadowCell(Bytes.toBytes(row), fam, qual, problematicTx.getStartTimestamp(),
375                                            new TTableCellGetterAdapter(txTable)),
376                    "Shadow cell should not be there");
377     }
378 
379     @Test(timeOut = 60_000)
380     public void testNeverendingTXsWithSTBelowAndAboveLWMAreDiscardedAndPreservedRespectivelyAfterCompaction()
381             throws Throwable {
382         String
383                 TEST_TABLE =
384                 "testNeverendingTXsWithSTBelowAndAboveLWMAreDiscardedAndPreservedRespectivelyAfterCompaction";
385         createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
386         TTable txTable = new TTable(connection, TEST_TABLE);
387 
388         // The KV in this transaction should be discarded
389         HBaseTransaction neverendingTxBelowLowWatermark = (HBaseTransaction) tm.begin();
390         long rowId = randomGenerator.nextLong();
391         Put put = new Put(Bytes.toBytes(rowId));
392         put.addColumn(fam, qual, data);
393         txTable.put(neverendingTxBelowLowWatermark, put);
394         assertTrue(CellUtils.hasCell(Bytes.toBytes(rowId), fam, qual, neverendingTxBelowLowWatermark.getStartTimestamp(),
395                                      new TTableCellGetterAdapter(txTable)),
396                    "Cell should be there");
397         assertFalse(CellUtils.hasShadowCell(Bytes.toBytes(rowId), fam, qual, neverendingTxBelowLowWatermark.getStartTimestamp(),
398                                             new TTableCellGetterAdapter(txTable)),
399                     "Shadow cell should not be there");
400 
401         // The KV in this transaction should be added without the shadow cells
402         HBaseTransaction neverendingTxAboveLowWatermark = (HBaseTransaction) tm.begin();
403         long anotherRowId = randomGenerator.nextLong();
404         put = new Put(Bytes.toBytes(anotherRowId));
405         put.addColumn(fam, qual, data);
406         txTable.put(neverendingTxAboveLowWatermark, put);
407         assertTrue(CellUtils.hasCell(Bytes.toBytes(anotherRowId), fam, qual, neverendingTxAboveLowWatermark.getStartTimestamp(),
408                                      new TTableCellGetterAdapter(txTable)),
409                    "Cell should be there");
410         assertFalse(CellUtils.hasShadowCell(Bytes.toBytes(anotherRowId), fam, qual, neverendingTxAboveLowWatermark.getStartTimestamp(),
411                                             new TTableCellGetterAdapter(txTable)),
412                     "Shadow cell should not be there");
413 
414         assertEquals(rowCount(TEST_TABLE, fam), 2, "Rows in table before flushing should be 2");
415         LOG.info("Flushing table {}", TEST_TABLE);
416         admin.flush(TableName.valueOf(TEST_TABLE));
417         assertEquals(rowCount(TEST_TABLE, fam), 2, "Rows in table after flushing should be 2");
418 
419         // Return a LWM that triggers compaction and stays between both ST of TXs, so assign 1st TX's start timestamp
420         LOG.info("Regions in table {}: {}", TEST_TABLE, hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).size());
421         OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).get(0)
422                 .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
423         CommitTable commitTable = injector.getInstance(CommitTable.class);
424         CommitTable.Client commitTableClient = spy(commitTable.getClient());
425         SettableFuture<Long> f = SettableFuture.create();
426         f.set(neverendingTxBelowLowWatermark.getStartTimestamp());
427         doReturn(f).when(commitTableClient).readLowWatermark();
428         omidCompactor.commitTableClient = commitTableClient;
429         LOG.info("Compacting table {}", TEST_TABLE);
430         admin.majorCompact(TableName.valueOf(TEST_TABLE));
431 
432         LOG.info("Sleeping for 3 secs");
433         Thread.sleep(3000);
434         LOG.info("Waking up after 3 secs");
435 
436         // One row should have been discarded after compacting
437         assertEquals(rowCount(TEST_TABLE, fam), 1, "There should be only one row in table after compacting");
438         // The row from the TX below the LWM should not be there (nor its Shadow Cell)
439         assertFalse(CellUtils.hasCell(Bytes.toBytes(rowId), fam, qual, neverendingTxBelowLowWatermark.getStartTimestamp(),
440                                       new TTableCellGetterAdapter(txTable)),
441                     "Cell should not be there");
442         assertFalse(CellUtils.hasShadowCell(Bytes.toBytes(rowId), fam, qual, neverendingTxBelowLowWatermark.getStartTimestamp(),
443                                             new TTableCellGetterAdapter(txTable)),
444                     "Shadow cell should not be there");
445         // The row from the TX above the LWM should be there without the Shadow Cell
446         assertTrue(CellUtils.hasCell(Bytes.toBytes(anotherRowId), fam, qual, neverendingTxAboveLowWatermark.getStartTimestamp(),
447                                      new TTableCellGetterAdapter(txTable)),
448                    "Cell should be there");
449         assertFalse(CellUtils.hasShadowCell(Bytes.toBytes(anotherRowId), fam, qual, neverendingTxAboveLowWatermark.getStartTimestamp(),
450                                             new TTableCellGetterAdapter(txTable)),
451                     "Shadow cell should not be there");
452 
453     }
454 
455     @Test(timeOut = 60_000)
456     public void testRowsUnalteredWhenCommitTableCannotBeReached() throws Throwable {
457         String TEST_TABLE = "testRowsUnalteredWhenCommitTableCannotBeReached";
458         createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
459         TTable txTable = new TTable(connection, TEST_TABLE);
460 
461         // The KV in this transaction should be discarded but in the end should remain there because
462         // the commit table won't be accessed (simulating an error on access)
463         HBaseTransaction neverendingTx = (HBaseTransaction) tm.begin();
464         long rowId = randomGenerator.nextLong();
465         Put put = new Put(Bytes.toBytes(rowId));
466         put.addColumn(fam, qual, data);
467         txTable.put(neverendingTx, put);
468         assertTrue(CellUtils.hasCell(Bytes.toBytes(rowId), fam, qual, neverendingTx.getStartTimestamp(),
469                                      new TTableCellGetterAdapter(txTable)),
470                    "Cell should be there");
471         assertFalse(CellUtils.hasShadowCell(Bytes.toBytes(rowId), fam, qual, neverendingTx.getStartTimestamp(),
472                                             new TTableCellGetterAdapter(txTable)),
473                     "Shadow cell should not be there");
474 
475         assertEquals(rowCount(TEST_TABLE, fam), 1, "There should be only one rows in table before flushing");
476         LOG.info("Flushing table {}", TEST_TABLE);
477         admin.flush(TableName.valueOf(TEST_TABLE));
478         assertEquals(rowCount(TEST_TABLE, fam), 1, "There should be only one rows in table after flushing");
479 
480         // Break access to CommitTable functionality in Compactor
481         LOG.info("Regions in table {}: {}", TEST_TABLE, hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).size());
482         OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).get(0)
483                 .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
484         CommitTable commitTable = injector.getInstance(CommitTable.class);
485         CommitTable.Client commitTableClient = spy(commitTable.getClient());
486         SettableFuture<Long> f = SettableFuture.create();
487         f.setException(new IOException("Unable to read"));
488         doReturn(f).when(commitTableClient).readLowWatermark();
489         omidCompactor.commitTableClient = commitTableClient;
490         LOG.info("Compacting table {}", TEST_TABLE);
491         admin.majorCompact(TableName.valueOf(TEST_TABLE)); // Should trigger the error when accessing CommitTable funct.
492 
493         LOG.info("Sleeping for 3 secs");
494         Thread.sleep(3000);
495         LOG.info("Waking up after 3 secs");
496 
497         // All rows should be there after the failed compaction
498         assertEquals(rowCount(TEST_TABLE, fam), 1, "There should be only one row in table after compacting");
499         assertTrue(CellUtils.hasCell(Bytes.toBytes(rowId), fam, qual, neverendingTx.getStartTimestamp(),
500                                      new TTableCellGetterAdapter(txTable)),
501                    "Cell should be there");
502         assertFalse(CellUtils.hasShadowCell(Bytes.toBytes(rowId), fam, qual, neverendingTx.getStartTimestamp(),
503                                             new TTableCellGetterAdapter(txTable)),
504                     "Shadow cell should not be there");
505     }
506 
507     @Test(timeOut = 60_000)
508     public void testOriginalTableParametersAreAvoidedAlsoWhenCompacting() throws Throwable {
509         String TEST_TABLE = "testOriginalTableParametersAreAvoidedAlsoWhenCompacting";
510         createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
511         TTable txTable = new TTable(connection, TEST_TABLE);
512 
513         long rowId = randomGenerator.nextLong();
514         for (int versionCount = 0; versionCount <= (2 * MAX_VERSIONS); versionCount++) {
515             Transaction tx = tm.begin();
516             Put put = new Put(Bytes.toBytes(rowId));
517             put.addColumn(fam, qual, Bytes.toBytes("testWrite-" + versionCount));
518             txTable.put(tx, put);
519             tm.commit(tx);
520         }
521 
522         Transaction tx = tm.begin();
523         Get get = new Get(Bytes.toBytes(rowId));
524         get.setMaxVersions(2 * MAX_VERSIONS);
525         assertEquals(get.getMaxVersions(), (2 * MAX_VERSIONS), "Max versions should be set to " + (2 * MAX_VERSIONS));
526         get.addColumn(fam, qual);
527         Result result = txTable.get(tx, get);
528         tm.commit(tx);
529         List<Cell> column = result.getColumnCells(fam, qual);
530         assertEquals(column.size(), 1, "There should be only one version in the result");
531 
532         assertEquals(rowCount(TEST_TABLE, fam), 1, "There should be only one row in table before flushing");
533         LOG.info("Flushing table {}", TEST_TABLE);
534         admin.flush(TableName.valueOf(TEST_TABLE));
535         assertEquals(rowCount(TEST_TABLE, fam), 1, "There should be only one row in table after flushing");
536 
537         // Return a LWM that triggers compaction
538         compactEverything(TEST_TABLE);
539 
540         // One row should have been discarded after compacting
541         assertEquals(rowCount(TEST_TABLE, fam), 1, "There should be only one row in table after compacting");
542 
543         tx = tm.begin();
544         get = new Get(Bytes.toBytes(rowId));
545         get.setMaxVersions(2 * MAX_VERSIONS);
546         assertEquals(get.getMaxVersions(), (2 * MAX_VERSIONS), "Max versions should be set to " + (2 * MAX_VERSIONS));
547         get.addColumn(fam, qual);
548         result = txTable.get(tx, get);
549         tm.commit(tx);
550         column = result.getColumnCells(fam, qual);
551         assertEquals(column.size(), 1, "There should be only one version in the result");
552         assertEquals(Bytes.toString(CellUtil.cloneValue(column.get(0))), "testWrite-" + (2 * MAX_VERSIONS),
553                      "Values don't match");
554     }
555 
556     // manually flush the regions on the region server.
557     // flushing like this prevents compaction running
558     // directly after the flush, which we want to avoid.
559     private void manualFlush(String tableName) throws Throwable {
560         LOG.info("Manually flushing all regions and waiting 2 secs");
561         HBaseShims.flushAllOnlineRegions(hbaseTestUtil.getHBaseCluster().getRegionServer(0),
562                                          TableName.valueOf(tableName));
563         TimeUnit.SECONDS.sleep(2);
564     }
565 
566     @Test(timeOut = 60_000)
567     public void testOldCellsAreDiscardedAfterCompaction() throws Exception {
568         String TEST_TABLE = "testOldCellsAreDiscardedAfterCompaction";
569         createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
570         TTable txTable = new TTable(connection, TEST_TABLE);
571 
572         byte[] rowId = Bytes.toBytes("row");
573 
574         // Create 3 transactions modifying the same cell in a particular row
575         HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
576         Put put1 = new Put(rowId);
577         put1.addColumn(fam, qual, Bytes.toBytes("testValue 1"));
578         txTable.put(tx1, put1);
579         tm.commit(tx1);
580 
581         HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
582         Put put2 = new Put(rowId);
583         put2.addColumn(fam, qual, Bytes.toBytes("testValue 2"));
584         txTable.put(tx2, put2);
585         tm.commit(tx2);
586 
587         HBaseTransaction tx3 = (HBaseTransaction) tm.begin();
588         Put put3 = new Put(rowId);
589         put3.addColumn(fam, qual, Bytes.toBytes("testValue 3"));
590         txTable.put(tx3, put3);
591         tm.commit(tx3);
592 
593         // Before compaction, the three timestamped values for the cell should be there
594         TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
595         assertTrue(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
596                    "Put cell of Tx1 should be there");
597         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
598                    "Put shadow cell of Tx1 should be there");
599         assertTrue(CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
600                    "Put cell of Tx2 cell should be there");
601         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
602                    "Put shadow cell of Tx2 should be there");
603         assertTrue(CellUtils.hasCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
604                    "Put cell of Tx3 cell should be there");
605         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
606                    "Put shadow cell of Tx3 should be there");
607 
608         // Compact
609         HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
610         compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
611 
612         // After compaction, only the last value for the cell should have survived
613         assertFalse(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
614                     "Put cell of Tx1 should not be there");
615         assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
616                     "Put shadow cell of Tx1 should not be there");
617         assertFalse(CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
618                     "Put cell of Tx2 should not be there");
619         assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
620                     "Put shadow cell of Tx2 should not be there");
621         assertTrue(CellUtils.hasCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
622                    "Put cell of Tx3 cell should be there");
623         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
624                    "Put shadow cell of Tx3 should be there");
625 
626         // A new transaction after compaction should read the last value written
627         HBaseTransaction newTx1 = (HBaseTransaction) tm.begin();
628         Get newGet1 = new Get(rowId);
629         newGet1.addColumn(fam, qual);
630         Result result = txTable.get(newTx1, newGet1);
631         assertEquals(Bytes.toBytes("testValue 3"), result.getValue(fam, qual));
632         // Write a new value
633         Put newPut1 = new Put(rowId);
634         newPut1.addColumn(fam, qual, Bytes.toBytes("new testValue 1"));
635         txTable.put(newTx1, newPut1);
636 
637         // Start a second new transaction
638         HBaseTransaction newTx2 = (HBaseTransaction) tm.begin();
639         // Commit first of the new tx
640         tm.commit(newTx1);
641 
642         // The second transaction should still read the previous value
643         Get newGet2 = new Get(rowId);
644         newGet2.addColumn(fam, qual);
645         result = txTable.get(newTx2, newGet2);
646         assertEquals(Bytes.toBytes("testValue 3"), result.getValue(fam, qual));
647         tm.commit(newTx2);
648 
649         // Only two values -the new written by newTx1 and the last value
650         // for the cell after compaction- should have survived
651         assertFalse(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
652                     "Put cell of Tx1 should not be there");
653         assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
654                     "Put shadow cell of Tx1 should not be there");
655         assertFalse(CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
656                     "Put cell of Tx2 should not be there");
657         assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
658                     "Put shadow cell of Tx2 should not be there");
659         assertTrue(CellUtils.hasCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
660                    "Put cell of Tx3 cell should be there");
661         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
662                    "Put shadow cell of Tx3 should be there");
663         assertTrue(CellUtils.hasCell(rowId, fam, qual, newTx1.getStartTimestamp(), getter),
664                    "Put cell of NewTx1 cell should be there");
665         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, newTx1.getStartTimestamp(), getter),
666                    "Put shadow cell of NewTx1 should be there");
667     }
668 
669     /**
670      * Tests a case where a temporary failure to flush causes the compactor to crash
671      */
672     @Test(timeOut = 60_000)
673     public void testDuplicateDeletes() throws Throwable {
674         String TEST_TABLE = "testDuplicateDeletes";
675         createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
676         TTable txTable = new TTable(connection, TEST_TABLE);
677 
678         // jump through hoops to trigger a minor compaction.
679         // a minor compaction will only run if there are enough
680         // files to be compacted, but that is less than the number
681         // of total files, in which case it will run a major
682         // compaction. The issue this is testing only shows up
683         // with minor compaction, as only Deletes can be duplicate
684         // and major compactions filter them out.
685         byte[] firstRow = "FirstRow".getBytes();
686         HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
687         Put put0 = new Put(firstRow);
688         put0.addColumn(fam, qual, Bytes.toBytes("testWrite-1"));
689         txTable.put(tx0, put0);
690         tm.commit(tx0);
691 
692         // create the first hfile
693         manualFlush(TEST_TABLE);
694 
695         // write a row, it won't be committed
696         byte[] rowToBeCompactedAway = "compactMe".getBytes();
697         HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
698         Put put1 = new Put(rowToBeCompactedAway);
699         put1.addColumn(fam, qual, Bytes.toBytes("testWrite-1"));
700         txTable.put(tx1, put1);
701         txTable.flushCommits();
702 
703         // write a row to trigger the double delete problem
704         byte[] row = "iCauseErrors".getBytes();
705         HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
706         Put put2 = new Put(row);
707         put2.addColumn(fam, qual, Bytes.toBytes("testWrite-1"));
708         txTable.put(tx2, put2);
709         tm.commit(tx2);
710 
711         HBaseTransaction tx3 = (HBaseTransaction) tm.begin();
712         Put put3 = new Put(row);
713         put3.addColumn(fam, qual, Bytes.toBytes("testWrite-1"));
714         txTable.put(tx3, put3);
715         txTable.flushCommits();
716 
717         // cause a failure on HBaseTM#preCommit();
718         Set<HBaseCellId> writeSet = tx3.getWriteSet();
719         assertEquals(1, writeSet.size());
720         List<HBaseCellId> newWriteSet = new ArrayList<>();
721         final AtomicBoolean flushFailing = new AtomicBoolean(true);
722         for (HBaseCellId id : writeSet) {
723             TTable failableHTable = spy(id.getTable());
724             doAnswer(new Answer<Void>() {
725                 @Override
726                 public Void answer(InvocationOnMock invocation)
727                         throws Throwable {
728                     if (flushFailing.get()) {
729                         throw new RetriesExhaustedWithDetailsException(new ArrayList<Throwable>(),
730                                                                        new ArrayList<Row>(), new ArrayList<String>());
731                     } else {
732                         invocation.callRealMethod();
733                     }
734                     return null;
735                 }
736             }).when(failableHTable).flushCommits();
737 
738             newWriteSet.add(new HBaseCellId(failableHTable,
739                                             id.getRow(), id.getFamily(),
740                                             id.getQualifier(), id.getTimestamp()));
741         }
742         writeSet.clear();
743         writeSet.addAll(newWriteSet);
744 
745         try {
746             tm.commit(tx3);
747             fail("Shouldn't succeed");
748         } catch (TransactionException tme) {
749             flushFailing.set(false);
750             tm.rollback(tx3);
751         }
752 
753         // create second hfile,
754         // it should contain multiple deletes
755         manualFlush(TEST_TABLE);
756 
757         // create loads of files
758         byte[] anotherRow = "someotherrow".getBytes();
759         HBaseTransaction tx4 = (HBaseTransaction) tm.begin();
760         Put put4 = new Put(anotherRow);
761         put4.addColumn(fam, qual, Bytes.toBytes("testWrite-1"));
762         txTable.put(tx4, put4);
763         tm.commit(tx4);
764 
765         // create third hfile
766         manualFlush(TEST_TABLE);
767 
768         // trigger minor compaction and give it time to run
769         setCompactorLWM(tx4.getStartTimestamp(), TEST_TABLE);
770         admin.compact(TableName.valueOf(TEST_TABLE));
771         Thread.sleep(3000);
772 
773         // check if the cell that should be compacted, is compacted
774         assertFalse(CellUtils.hasCell(rowToBeCompactedAway, fam, qual, tx1.getStartTimestamp(),
775                                       new TTableCellGetterAdapter(txTable)),
776                     "Cell should not be be there");
777     }
778 
779     @Test(timeOut = 60_000)
780     public void testNonOmidCFIsUntouched() throws Throwable {
781         String TEST_TABLE = "testNonOmidCFIsUntouched";
782         createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
783         TTable txTable = new TTable(connection, TEST_TABLE);
784 
785         admin.disableTable(TableName.valueOf(TEST_TABLE));
786         byte[] nonOmidCF = Bytes.toBytes("nonOmidCF");
787         byte[] nonOmidQual = Bytes.toBytes("nonOmidCol");
788         HColumnDescriptor nonomidfam = new HColumnDescriptor(nonOmidCF);
789         nonomidfam.setMaxVersions(MAX_VERSIONS);
790         admin.addColumn(TableName.valueOf(TEST_TABLE), nonomidfam);
791         admin.enableTable(TableName.valueOf(TEST_TABLE));
792 
793         byte[] rowId = Bytes.toBytes("testRow");
794         Transaction tx = tm.begin();
795         Put put = new Put(rowId);
796         put.addColumn(fam, qual, Bytes.toBytes("testValue"));
797         txTable.put(tx, put);
798 
799         Put nonTxPut = new Put(rowId);
800         nonTxPut.addColumn(nonOmidCF, nonOmidQual, Bytes.toBytes("nonTxVal"));
801         txTable.getHTable().put(nonTxPut);
802         txTable.flushCommits(); // to make sure it left the client
803 
804         Get g = new Get(rowId);
805         Result result = txTable.getHTable().get(g);
806         assertEquals(result.getColumnCells(nonOmidCF, nonOmidQual).size(), 1, "Should be there, precompact");
807         assertEquals(result.getColumnCells(fam, qual).size(), 1, "Should be there, precompact");
808 
809         compactEverything(TEST_TABLE);
810 
811         result = txTable.getHTable().get(g);
812         assertEquals(result.getColumnCells(nonOmidCF, nonOmidQual).size(), 1, "Should be there, postcompact");
813         assertEquals(result.getColumnCells(fam, qual).size(), 0, "Should not be there, postcompact");
814     }
815 
816     // ----------------------------------------------------------------------------------------------------------------
817     // Tests on tombstones and non-transactional Deletes
818     // ----------------------------------------------------------------------------------------------------------------
819 
820     /**
821      * Test that when a major compaction runs, cells that were deleted non-transactionally dissapear
822      */
823     @Test(timeOut = 60_000)
824     public void testACellDeletedNonTransactionallyDoesNotAppearWhenAMajorCompactionOccurs() throws Throwable {
825         String TEST_TABLE = "testACellDeletedNonTransactionallyDoesNotAppearWhenAMajorCompactionOccurs";
826         createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
827         TTable txTable = new TTable(connection, TEST_TABLE);
828 
829         Table table = txTable.getHTable();
830 
831         // Write first a value transactionally
832         HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
833         byte[] rowId = Bytes.toBytes("row1");
834         Put p0 = new Put(rowId);
835         p0.addColumn(fam, qual, Bytes.toBytes("testValue-0"));
836         txTable.put(tx0, p0);
837         tm.commit(tx0);
838 
839         // Then perform a non-transactional Delete
840         Delete d = new Delete(rowId);
841         d.addColumn(fam, qual);
842         table.delete(d);
843 
844         // Trigger a major compaction
845         HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
846         compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
847 
848         // Then perform a non-tx (raw) scan...
849         Scan scan = new Scan();
850         scan.setRaw(true);
851         ResultScanner scannerResults = table.getScanner(scan);
852 
853         // ...and test the deleted cell is not there anymore
854         assertNull(scannerResults.next(), "There should be no results in scan results");
855 
856         table.close();
857 
858     }
859 
860     /**
861      * Test that when a minor compaction runs, cells that were deleted non-transactionally are preserved. This is to
862      * allow users still access the cells when doing "improper" operations on a transactional table
863      */
864     @Test(timeOut = 60_000)
865     public void testACellDeletedNonTransactionallyIsPreservedWhenMinorCompactionOccurs() throws Throwable {
866         String TEST_TABLE = "testACellDeletedNonTransactionallyIsPreservedWhenMinorCompactionOccurs";
867         createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
868         TTable txTable = new TTable(connection, TEST_TABLE);
869 
870         Table table = txTable.getHTable();
871 
872         // Configure the environment to create a minor compaction
873 
874         // Write first a value transactionally
875         HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
876         byte[] rowId = Bytes.toBytes("row1");
877         Put p0 = new Put(rowId);
878         p0.addColumn(fam, qual, Bytes.toBytes("testValue-0"));
879         txTable.put(tx0, p0);
880         tm.commit(tx0);
881 
882         // create the first hfile
883         manualFlush(TEST_TABLE);
884 
885         // Write another value transactionally
886         HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
887         Put p1 = new Put(rowId);
888         p1.addColumn(fam, qual, Bytes.toBytes("testValue-1"));
889         txTable.put(tx1, p1);
890         tm.commit(tx1);
891 
892         // create the second hfile
893         manualFlush(TEST_TABLE);
894 
895         // Write yet another value transactionally
896         HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
897         Put p2 = new Put(rowId);
898         p2.addColumn(fam, qual, Bytes.toBytes("testValue-2"));
899         txTable.put(tx2, p2);
900         tm.commit(tx2);
901 
902         // create a third hfile
903         manualFlush(TEST_TABLE);
904 
905         // Then perform a non-transactional Delete
906         Delete d = new Delete(rowId);
907         d.addColumn(fam, qual);
908         table.delete(d);
909 
910         // create the fourth hfile
911         manualFlush(TEST_TABLE);
912 
913         // Trigger the minor compaction
914         HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
915         setCompactorLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
916         admin.compact(TableName.valueOf(TEST_TABLE));
917         Thread.sleep(5000);
918 
919         // Then perform a non-tx (raw) scan...
920         Scan scan = new Scan();
921         scan.setRaw(true);
922         ResultScanner scannerResults = table.getScanner(scan);
923 
924         // ...and test the deleted cell is still there
925         int count = 0;
926         Result scanResult;
927         List<Cell> listOfCellsScanned = new ArrayList<>();
928         while ((scanResult = scannerResults.next()) != null) {
929             listOfCellsScanned = scanResult.listCells(); // equivalent to rawCells()
930             count++;
931         }
932         assertEquals(count, 1, "There should be only one result in scan results");
933         assertEquals(listOfCellsScanned.size(), 3, "There should be 3 cell entries in scan results (2 puts, 1 del)");
934         boolean wasDeletedCellFound = false;
935         int numberOfDeletedCellsFound = 0;
936         for (Cell cell : listOfCellsScanned) {
937             if (CellUtil.isDelete(cell)) {
938                 wasDeletedCellFound = true;
939                 numberOfDeletedCellsFound++;
940             }
941         }
942         assertTrue(wasDeletedCellFound, "We should have found a non-transactionally deleted cell");
943         assertEquals(numberOfDeletedCellsFound, 1, "There should be only only one deleted cell");
944 
945         table.close();
946     }
947 
948     /**
949      * Test that when a minor compaction runs, tombstones are not cleaned up
950      */
951     @Test(timeOut = 60_000)
952     public void testTombstonesAreNotCleanedUpWhenMinorCompactionOccurs() throws Throwable {
953         String TEST_TABLE = "testTombstonesAreNotCleanedUpWhenMinorCompactionOccurs";
954         createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
955         TTable txTable = new TTable(connection, TEST_TABLE);
956 
957         // Configure the environment to create a minor compaction
958 
959         HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
960         byte[] rowId = Bytes.toBytes("case1");
961         Put p = new Put(rowId);
962         p.addColumn(fam, qual, Bytes.toBytes("testValue-0"));
963         txTable.put(tx0, p);
964         tm.commit(tx0);
965 
966         // create the first hfile
967         manualFlush(TEST_TABLE);
968 
969         // Create the tombstone
970         HBaseTransaction deleteTx = (HBaseTransaction) tm.begin();
971         Delete d = new Delete(rowId);
972         d.addColumn(fam, qual);
973         txTable.delete(deleteTx, d);
974         tm.commit(deleteTx);
975 
976         // create the second hfile
977         manualFlush(TEST_TABLE);
978 
979         HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
980         Put p1 = new Put(rowId);
981         p1.addColumn(fam, qual, Bytes.toBytes("testValue-11"));
982         txTable.put(tx1, p1);
983         tm.commit(tx1);
984 
985         // create the third hfile
986         manualFlush(TEST_TABLE);
987 
988         HBaseTransaction lastTx = (HBaseTransaction) tm.begin();
989         Put p2 = new Put(rowId);
990         p2.addColumn(fam, qual, Bytes.toBytes("testValue-222"));
991         txTable.put(lastTx, p2);
992         tm.commit(lastTx);
993 
994         // Trigger the minor compaction
995         HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
996         setCompactorLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
997         admin.compact(TableName.valueOf(TEST_TABLE));
998         Thread.sleep(5000);
999 
1000         // Checks on results after compaction
1001         TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
1002         assertFalse(CellUtils.hasCell(rowId, fam, qual, tx0.getStartTimestamp(), getter), "Put cell should be there");
1003         assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx0.getStartTimestamp(), getter),
1004                     "Put shadow cell should be there");
1005         assertTrue(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter), "Put cell should be there");
1006         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1007                    "Put shadow cell should be there");
1008         assertTrue(CellUtils.hasCell(rowId, fam, qual, deleteTx.getStartTimestamp(), getter),
1009                    "Delete cell should be there");
1010         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, deleteTx.getStartTimestamp(), getter),
1011                    "Delete shadow cell should be there");
1012         assertTrue(CellUtils.hasCell(rowId, fam, qual, lastTx.getStartTimestamp(), getter),
1013                    "Put cell should be there");
1014         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, lastTx.getStartTimestamp(), getter),
1015                    "Put shadow cell should be there");
1016     }
1017 
1018 
1019     /**
1020      * Test that when compaction runs, tombstones are cleaned up case1: 1 put (ts < lwm) then tombstone (ts > lwm)
1021      */
1022     @Test(timeOut = 60_000)
1023     public void testTombstonesAreCleanedUpCase1() throws Exception {
1024         String TEST_TABLE = "testTombstonesAreCleanedUpCase1";
1025         createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
1026         TTable txTable = new TTable(connection, TEST_TABLE);
1027 
1028         HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
1029         byte[] rowId = Bytes.toBytes("case1");
1030         Put p = new Put(rowId);
1031         p.addColumn(fam, qual, Bytes.toBytes("testValue"));
1032         txTable.put(tx1, p);
1033         tm.commit(tx1);
1034 
1035         HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
1036         setCompactorLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
1037 
1038         HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
1039         Delete d = new Delete(rowId);
1040         d.addColumn(fam, qual);
1041         txTable.delete(tx2, d);
1042         tm.commit(tx2);
1043 
1044         TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
1045         assertTrue(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1046                    "Put cell should be there");
1047         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1048                    "Put shadow cell should be there");
1049         assertTrue(CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
1050                    "Delete cell should be there");
1051         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
1052                    "Delete shadow cell should be there");
1053     }
1054 
1055     /**
1056      * Test that when compaction runs, tombstones are cleaned up case2: 1 put (ts < lwm) then tombstone (ts < lwm)
1057      */
1058     @Test(timeOut = 60_000)
1059     public void testTombstonesAreCleanedUpCase2() throws Exception {
1060         String TEST_TABLE = "testTombstonesAreCleanedUpCase2";
1061         createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
1062         TTable txTable = new TTable(connection, TEST_TABLE);
1063 
1064         HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
1065         byte[] rowId = Bytes.toBytes("case2");
1066         Put p = new Put(rowId);
1067         p.addColumn(fam, qual, Bytes.toBytes("testValue"));
1068         txTable.put(tx1, p);
1069         tm.commit(tx1);
1070 
1071         HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
1072         Delete d = new Delete(rowId);
1073         d.addColumn(fam, qual);
1074         txTable.delete(tx2, d);
1075         tm.commit(tx2);
1076 
1077         HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
1078         compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
1079 
1080         TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
1081         assertFalse(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1082                     "Put cell shouldn't be there");
1083         assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1084                     "Put shadow cell shouldn't be there");
1085         assertFalse(CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
1086                     "Delete cell shouldn't be there");
1087         assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
1088                     "Delete shadow cell shouldn't be there");
1089     }
1090 
1091 
1092     //Cell level conflict detection
1093     @Test(timeOut = 60_000)
1094     public void testFamiliyDeleteTombstonesAreCleanedUpCellCF() throws Exception {
1095         String TEST_TABLE = "testFamiliyDeleteTombstonesAreCleanedUpCellCF";
1096         byte[] fam2 = Bytes.toBytes("2");
1097         byte[] fam3 = Bytes.toBytes("3");
1098         byte[] fam4 = Bytes.toBytes("4");
1099         createTableIfNotExists(TEST_TABLE, fam2, fam3, fam4);
1100         TTable txTable = new TTable(connection, TEST_TABLE);
1101 
1102         HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
1103         byte[] rowId = Bytes.toBytes("case2");
1104         byte[] qual2 = Bytes.toBytes("qual2");
1105 
1106         Put p = new Put(rowId);
1107         p.addColumn(fam2, qual, Bytes.toBytes("testValue"));
1108         p.addColumn(fam2, qual2 , Bytes.toBytes("testValue"));
1109 
1110         p.addColumn(fam3, qual, Bytes.toBytes("testValue"));
1111         p.addColumn(fam3, qual2 , Bytes.toBytes("testValue"));
1112 
1113         p.addColumn(fam4, qual, Bytes.toBytes("testValue"));
1114         p.addColumn(fam4, qual2 , Bytes.toBytes("testValue"));
1115 
1116 
1117         txTable.put(tx1, p);
1118 
1119         byte[] rowId2 = Bytes.toBytes("case22");
1120         Put p2 = new Put(rowId2);
1121         p2.addColumn(fam3, qual, Bytes.toBytes("testValue2"));
1122         txTable.put(tx1, p2);
1123 
1124         tm.commit(tx1);
1125 
1126         HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
1127         Delete d = new Delete(rowId);
1128         d.addFamily(fam3);
1129         txTable.delete(tx2, d);
1130         tm.commit(tx2);
1131 
1132         HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
1133 
1134         TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
1135         assertTrue(CellUtils.hasCell(rowId, fam2, qual, tx1.getStartTimestamp(), getter),
1136                 "Put cell should be there");
1137         assertTrue(CellUtils.hasShadowCell(rowId, fam2, qual, tx1.getStartTimestamp(), getter),
1138                 "Put shadow cell should be there");
1139         assertTrue(CellUtils.hasCell(rowId, fam2, qual2, tx1.getStartTimestamp(), getter),
1140                 "Put cell should be there");
1141         assertTrue(CellUtils.hasShadowCell(rowId, fam2, qual2, tx1.getStartTimestamp(), getter),
1142                 "Put shadow cell should be there");
1143 
1144         assertTrue(CellUtils.hasCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1145                 "Put cell should be there");
1146         assertTrue(CellUtils.hasShadowCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1147                 "Put shadow cell should be there");
1148         assertTrue(CellUtils.hasCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1149                 "Put cell should be there");
1150         assertTrue(CellUtils.hasShadowCell(rowId, fam3, qual2, tx1.getStartTimestamp(), getter),
1151                 "Put shadow cell should be there");
1152 
1153         assertTrue(CellUtils.hasCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1154                 "Put cell should be there");
1155         assertTrue(CellUtils.hasShadowCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1156                 "Put shadow cell should be there");
1157         assertTrue(CellUtils.hasCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1158                 "Put cell should be there");
1159         assertTrue(CellUtils.hasShadowCell(rowId, fam4, qual2, tx1.getStartTimestamp(), getter),
1160                 "Put shadow cell should be there");
1161 
1162 
1163         assertTrue(CellUtils.hasCell(rowId, fam3, CellUtils.FAMILY_DELETE_QUALIFIER, tx2.getStartTimestamp(), getter),
1164                 "Delete cell should be there");
1165         assertTrue(CellUtils.hasShadowCell(rowId, fam3, CellUtils.FAMILY_DELETE_QUALIFIER, tx2.getStartTimestamp(), getter),
1166                 "Delete shadow cell should be there");
1167 
1168         //Do major compaction
1169         compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
1170 
1171 
1172 
1173         assertTrue(CellUtils.hasCell(rowId, fam2, qual, tx1.getStartTimestamp(), getter),
1174                 "Put cell should be there");
1175         assertTrue(CellUtils.hasShadowCell(rowId, fam2, qual, tx1.getStartTimestamp(), getter),
1176                 "Put shadow cell should be there");
1177         assertTrue(CellUtils.hasCell(rowId, fam2, qual2, tx1.getStartTimestamp(), getter),
1178                 "Put cell should be there");
1179         assertTrue(CellUtils.hasShadowCell(rowId, fam2, qual2, tx1.getStartTimestamp(), getter),
1180                 "Put shadow cell should be there");
1181 
1182 
1183         assertFalse(CellUtils.hasCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1184                 "Put cell shouldn't be there");
1185         assertFalse(CellUtils.hasShadowCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1186                 "Put shadow cell shouldn't be there");
1187         assertFalse(CellUtils.hasCell(rowId, fam3, qual2, tx1.getStartTimestamp(), getter),
1188                 "Put cell shouldn't be there");
1189         assertFalse(CellUtils.hasShadowCell(rowId, fam3, qual2, tx1.getStartTimestamp(), getter),
1190                 "Put shadow cell shouldn't be there");
1191 
1192 
1193         assertTrue(CellUtils.hasCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1194                 "Put cell should be there");
1195         assertTrue(CellUtils.hasShadowCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1196                 "Put shadow cell should be there");
1197         assertTrue(CellUtils.hasCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1198                 "Put cell should be there");
1199         assertTrue(CellUtils.hasShadowCell(rowId, fam4, qual2, tx1.getStartTimestamp(), getter),
1200                 "Put shadow cell should be there");
1201 
1202 
1203 
1204         assertFalse(CellUtils.hasCell(rowId, fam3, CellUtils.FAMILY_DELETE_QUALIFIER, tx2.getStartTimestamp(), getter),
1205                 "Delete cell shouldn't be there");
1206         assertFalse(CellUtils.hasShadowCell(rowId, fam3, CellUtils.FAMILY_DELETE_QUALIFIER, tx2.getStartTimestamp(), getter),
1207                 "Delete shadow cell shouldn't be there");
1208     }
1209 
1210 
1211     //Row level conflict detection
1212     @Test(timeOut = 60_000)
1213     public void testFamiliyDeleteTombstonesAreCleanedUpRowCF() throws Exception {
1214         String TEST_TABLE = "testFamiliyDeleteTombstonesAreCleanedUpRowCF";
1215         ((HBaseTransactionManager) tm).setConflictDetectionLevel(OmidClientConfiguration.ConflictDetectionLevel.ROW);
1216 
1217         byte[] fam2 = Bytes.toBytes("2");
1218         byte[] fam3 = Bytes.toBytes("3");
1219         byte[] fam4 = Bytes.toBytes("4");
1220         createTableIfNotExists(TEST_TABLE, fam2, fam3, fam4);
1221         TTable txTable = new TTable(connection, TEST_TABLE);
1222 
1223         HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
1224         byte[] rowId = Bytes.toBytes("case2");
1225         byte[] qual2 = Bytes.toBytes("qual2");
1226 
1227         Put p = new Put(rowId);
1228         p.addColumn(fam2, qual, Bytes.toBytes("testValue"));
1229         p.addColumn(fam2, qual2 , Bytes.toBytes("testValue"));
1230 
1231         p.addColumn(fam3, qual, Bytes.toBytes("testValue"));
1232         p.addColumn(fam3, qual2 , Bytes.toBytes("testValue"));
1233 
1234         p.addColumn(fam4, qual, Bytes.toBytes("testValue"));
1235         p.addColumn(fam4, qual2 , Bytes.toBytes("testValue"));
1236 
1237 
1238         txTable.put(tx1, p);
1239 
1240         byte[] rowId2 = Bytes.toBytes("case22");
1241         Put p2 = new Put(rowId2);
1242         p2.addColumn(fam3, qual, Bytes.toBytes("testValue2"));
1243         txTable.put(tx1, p2);
1244 
1245         tm.commit(tx1);
1246 
1247         HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
1248         Delete d = new Delete(rowId);
1249         d.addFamily(fam3);
1250         txTable.delete(tx2, d);
1251         tm.commit(tx2);
1252 
1253         HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
1254 
1255         TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
1256         assertTrue(CellUtils.hasCell(rowId, fam2, qual, tx1.getStartTimestamp(), getter),
1257                 "Put cell should be there");
1258         assertTrue(CellUtils.hasShadowCell(rowId, fam2, qual, tx1.getStartTimestamp(), getter),
1259                 "Put shadow cell should be there");
1260         assertTrue(CellUtils.hasCell(rowId, fam2, qual2, tx1.getStartTimestamp(), getter),
1261                 "Put cell should be there");
1262         assertTrue(CellUtils.hasShadowCell(rowId, fam2, qual2, tx1.getStartTimestamp(), getter),
1263                 "Put shadow cell should be there");
1264 
1265         assertTrue(CellUtils.hasCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1266                 "Put cell should be there");
1267         assertTrue(CellUtils.hasShadowCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1268                 "Put shadow cell should be there");
1269         assertTrue(CellUtils.hasCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1270                 "Put cell should be there");
1271         assertTrue(CellUtils.hasShadowCell(rowId, fam3, qual2, tx1.getStartTimestamp(), getter),
1272                 "Put shadow cell should be there");
1273 
1274         assertTrue(CellUtils.hasCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1275                 "Put cell should be there");
1276         assertTrue(CellUtils.hasShadowCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1277                 "Put shadow cell should be there");
1278         assertTrue(CellUtils.hasCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1279                 "Put cell should be there");
1280         assertTrue(CellUtils.hasShadowCell(rowId, fam4, qual2, tx1.getStartTimestamp(), getter),
1281                 "Put shadow cell should be there");
1282 
1283 
1284         assertTrue(CellUtils.hasCell(rowId, fam3, CellUtils.FAMILY_DELETE_QUALIFIER, tx2.getStartTimestamp(), getter),
1285                 "Delete cell should be there");
1286         assertTrue(CellUtils.hasShadowCell(rowId, fam3, CellUtils.FAMILY_DELETE_QUALIFIER, tx2.getStartTimestamp(), getter),
1287                 "Delete shadow cell should be there");
1288 
1289         //Do major compaction
1290         compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
1291 
1292 
1293 
1294         assertTrue(CellUtils.hasCell(rowId, fam2, qual, tx1.getStartTimestamp(), getter),
1295                 "Put cell should be there");
1296         assertTrue(CellUtils.hasShadowCell(rowId, fam2, qual, tx1.getStartTimestamp(), getter),
1297                 "Put shadow cell should be there");
1298         assertTrue(CellUtils.hasCell(rowId, fam2, qual2, tx1.getStartTimestamp(), getter),
1299                 "Put cell should be there");
1300         assertTrue(CellUtils.hasShadowCell(rowId, fam2, qual2, tx1.getStartTimestamp(), getter),
1301                 "Put shadow cell should be there");
1302 
1303 
1304         assertFalse(CellUtils.hasCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1305                 "Put cell shouldn't be there");
1306         assertFalse(CellUtils.hasShadowCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1307                 "Put shadow cell shouldn't be there");
1308         assertFalse(CellUtils.hasCell(rowId, fam3, qual2, tx1.getStartTimestamp(), getter),
1309                 "Put cell shouldn't be there");
1310         assertFalse(CellUtils.hasShadowCell(rowId, fam3, qual2, tx1.getStartTimestamp(), getter),
1311                 "Put shadow cell shouldn't be there");
1312 
1313 
1314         assertTrue(CellUtils.hasCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1315                 "Put cell should be there");
1316         assertTrue(CellUtils.hasShadowCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1317                 "Put shadow cell should be there");
1318         assertTrue(CellUtils.hasCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1319                 "Put cell should be there");
1320         assertTrue(CellUtils.hasShadowCell(rowId, fam4, qual2, tx1.getStartTimestamp(), getter),
1321                 "Put shadow cell should be there");
1322 
1323 
1324 
1325         assertFalse(CellUtils.hasCell(rowId, fam3, CellUtils.FAMILY_DELETE_QUALIFIER, tx2.getStartTimestamp(), getter),
1326                 "Delete cell shouldn't be there");
1327         assertFalse(CellUtils.hasShadowCell(rowId, fam3, CellUtils.FAMILY_DELETE_QUALIFIER, tx2.getStartTimestamp(), getter),
1328                 "Delete shadow cell shouldn't be there");
1329     }
1330 
1331 
1332 
1333     /**
1334      * Test that when compaction runs, tombstones are cleaned up case3: 1 put (ts < lwm) then tombstone (ts < lwm) not
1335      * committed
1336      */
1337     @Test(timeOut = 60_000)
1338     public void testTombstonesAreCleanedUpCase3() throws Exception {
1339         String TEST_TABLE = "testTombstonesAreCleanedUpCase3";
1340         createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
1341         TTable txTable = new TTable(connection, TEST_TABLE);
1342 
1343         HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
1344         byte[] rowId = Bytes.toBytes("case3");
1345         Put p = new Put(rowId);
1346         p.addColumn(fam, qual, Bytes.toBytes("testValue"));
1347         txTable.put(tx1, p);
1348         tm.commit(tx1);
1349 
1350         HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
1351         Delete d = new Delete(rowId);
1352         d.addColumn(fam, qual);
1353         txTable.delete(tx2, d);
1354 
1355         HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
1356         compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
1357 
1358         TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
1359         assertTrue(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1360                    "Put cell should be there");
1361         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1362                    "Put shadow cell shouldn't be there");
1363         assertFalse(CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
1364                     "Delete cell shouldn't be there");
1365         assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
1366                     "Delete shadow cell shouldn't be there");
1367     }
1368 
1369     /**
1370      * Test that when compaction runs, tombstones are cleaned up case4: 1 put (ts < lwm) then tombstone (ts > lwm) not
1371      * committed
1372      */
1373     @Test(timeOut = 60_000)
1374     public void testTombstonesAreCleanedUpCase4() throws Exception {
1375         String TEST_TABLE = "testTombstonesAreCleanedUpCase4";
1376         createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
1377         TTable txTable = new TTable(connection, TEST_TABLE);
1378 
1379         HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
1380         byte[] rowId = Bytes.toBytes("case4");
1381         Put p = new Put(rowId);
1382         p.addColumn(fam, qual, Bytes.toBytes("testValue"));
1383         txTable.put(tx1, p);
1384         tm.commit(tx1);
1385 
1386         HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
1387 
1388         HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
1389         Delete d = new Delete(rowId);
1390         d.addColumn(fam, qual);
1391         txTable.delete(tx2, d);
1392         compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
1393 
1394         TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
1395         assertTrue(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1396                    "Put cell should be there");
1397         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1398                    "Put shadow cell shouldn't be there");
1399         assertTrue(CellUtils.hasCell(rowId, fam, qual,tx2.getStartTimestamp(), getter),
1400                    "Delete cell should be there");
1401         assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
1402                     "Delete shadow cell shouldn't be there");
1403     }
1404 
1405     /**
1406      * Test that when compaction runs, tombstones are cleaned up case5: tombstone (ts < lwm)
1407      */
1408     @Test(timeOut = 60_000)
1409     public void testTombstonesAreCleanedUpCase5() throws Exception {
1410         String TEST_TABLE = "testTombstonesAreCleanedUpCase5";
1411         createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
1412         TTable txTable = new TTable(connection, TEST_TABLE);
1413 
1414         HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
1415         byte[] rowId = Bytes.toBytes("case5");
1416         Delete d = new Delete(rowId);
1417         d.addColumn(fam, qual);
1418         txTable.delete(tx1, d);
1419         tm.commit(tx1);
1420 
1421         HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
1422         compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
1423 
1424         TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
1425         assertFalse(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1426                     "Delete cell shouldn't be there");
1427         assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1428                     "Delete shadow cell shouldn't be there");
1429     }
1430 
1431     /**
1432      * Test that when compaction runs, tombstones are cleaned up case6: tombstone (ts < lwm), then put (ts < lwm)
1433      */
1434     @Test(timeOut = 60_000)
1435     public void testTombstonesAreCleanedUpCase6() throws Exception {
1436         String TEST_TABLE = "testTombstonesAreCleanedUpCase6";
1437         createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
1438         TTable txTable = new TTable(connection, TEST_TABLE);
1439         byte[] rowId = Bytes.toBytes("case6");
1440 
1441         HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
1442         Delete d = new Delete(rowId);
1443         d.addColumn(fam, qual);
1444         txTable.delete(tx1, d);
1445         tm.commit(tx1);
1446 
1447         HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
1448         Put p = new Put(rowId);
1449         p.addColumn(fam, qual, Bytes.toBytes("testValue"));
1450         txTable.put(tx2, p);
1451         tm.commit(tx2);
1452 
1453         HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
1454         compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
1455 
1456         TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
1457         assertFalse(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1458                     "Delete cell shouldn't be there");
1459         assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1460                     "Delete shadow cell shouldn't be there");
1461         assertTrue(CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
1462                    "Put cell should be there");
1463         assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
1464                    "Put shadow cell shouldn't be there");
1465     }
1466 
1467     @Test(timeOut = 60_000)
1468     public void testCommitTableNoInvalidation() throws Exception {
1469         String TEST_TABLE = "testCommitTableInvalidation";
1470         createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
1471         TTable txTable = new TTable(connection, TEST_TABLE);
1472         byte[] rowId = Bytes.toBytes("row");
1473 
1474         HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
1475         Put p = new Put(rowId);
1476         p.addColumn(fam, qual, Bytes.toBytes("testValue"));
1477         txTable.put(tx1, p);
1478 
1479         HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
1480         compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
1481 
1482         try {
1483             //give compaction time to invalidate
1484             Thread.sleep(1000);
1485 
1486             tm.commit(tx1);
1487 
1488         } catch (RollbackException e) {
1489             fail(" Should have not been invalidated");
1490         }
1491     }
1492 
1493 
1494     private void setCompactorLWM(long lwm, String tableName) throws Exception {
1495         OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(tableName)).get(0)
1496                 .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
1497         CommitTable commitTable = injector.getInstance(CommitTable.class);
1498         CommitTable.Client commitTableClient = spy(commitTable.getClient());
1499         SettableFuture<Long> f = SettableFuture.create();
1500         f.set(lwm);
1501         doReturn(f).when(commitTableClient).readLowWatermark();
1502         omidCompactor.commitTableClient = commitTableClient;
1503     }
1504 
1505     private void compactEverything(String tableName) throws Exception {
1506         compactWithLWM(Long.MAX_VALUE, tableName);
1507     }
1508 
1509     private void compactWithLWM(long lwm, String tableName) throws Exception {
1510         admin.flush(TableName.valueOf(tableName));
1511 
1512         LOG.info("Regions in table {}: {}", tableName, hbaseCluster.getRegions(Bytes.toBytes(tableName)).size());
1513         setCompactorLWM(lwm, tableName);
1514         LOG.info("Compacting table {}", tableName);
1515         admin.majorCompact(TableName.valueOf(tableName));
1516 
1517         LOG.info("Sleeping for 3 secs");
1518         Thread.sleep(3000);
1519         LOG.info("Waking up after 3 secs");
1520     }
1521 
1522     private static long rowCount(String tableName, byte[] family) throws Throwable {
1523         Scan scan = new Scan();
1524         scan.addFamily(family);
1525         Table table = connection.getTable(TableName.valueOf(tableName));
1526         try (ResultScanner scanner = table.getScanner(scan)) {
1527             int count = 0;
1528             while (scanner.next() != null) {
1529                 count++;
1530             }
1531             return count;
1532         }
1533     }
1534 }