1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.omid.transaction;
19
20 import static org.mockito.Matchers.any;
21 import static org.mockito.Mockito.doAnswer;
22 import static org.mockito.Mockito.doReturn;
23 import static org.mockito.Mockito.doThrow;
24 import static org.mockito.Mockito.spy;
25 import static org.testng.Assert.assertEquals;
26 import static org.testng.Assert.assertFalse;
27 import static org.testng.Assert.assertNull;
28 import static org.testng.Assert.assertTrue;
29 import static org.testng.Assert.fail;
30
31 import java.io.IOException;
32 import java.util.ArrayList;
33 import java.util.List;
34 import java.util.Random;
35 import java.util.Set;
36 import java.util.concurrent.TimeUnit;
37 import java.util.concurrent.atomic.AtomicBoolean;
38
39 import org.apache.hadoop.conf.Configuration;
40 import org.apache.hadoop.hbase.Cell;
41 import org.apache.hadoop.hbase.CellUtil;
42 import org.apache.hadoop.hbase.Coprocessor;
43 import org.apache.hadoop.hbase.HBaseTestingUtility;
44 import org.apache.hadoop.hbase.HColumnDescriptor;
45 import org.apache.hadoop.hbase.HTableDescriptor;
46 import org.apache.hadoop.hbase.MiniHBaseCluster;
47 import org.apache.hadoop.hbase.TableName;
48 import org.apache.hadoop.hbase.client.Admin;
49 import org.apache.hadoop.hbase.client.Connection;
50 import org.apache.hadoop.hbase.client.ConnectionFactory;
51 import org.apache.hadoop.hbase.client.Delete;
52 import org.apache.hadoop.hbase.client.Get;
53 import org.apache.hadoop.hbase.client.Put;
54 import org.apache.hadoop.hbase.client.Result;
55 import org.apache.hadoop.hbase.client.ResultScanner;
56 import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
57 import org.apache.hadoop.hbase.client.Row;
58 import org.apache.hadoop.hbase.client.Scan;
59 import org.apache.hadoop.hbase.client.Table;
60 import org.apache.hadoop.hbase.util.Bytes;
61 import org.apache.omid.HBaseShims;
62 import org.apache.omid.TestUtils;
63 import org.apache.omid.committable.CommitTable;
64 import org.apache.omid.committable.hbase.HBaseCommitTableConfig;
65 import org.apache.omid.metrics.NullMetricsProvider;
66 import org.apache.omid.timestamp.storage.HBaseTimestampStorageConfig;
67 import org.apache.omid.tso.TSOServer;
68 import org.apache.omid.tso.TSOServerConfig;
69 import org.apache.omid.tso.client.OmidClientConfiguration;
70 import org.mockito.invocation.InvocationOnMock;
71 import org.mockito.stubbing.Answer;
72 import org.slf4j.Logger;
73 import org.slf4j.LoggerFactory;
74 import org.testng.annotations.AfterClass;
75 import org.testng.annotations.BeforeClass;
76 import org.testng.annotations.BeforeMethod;
77 import org.testng.annotations.Test;
78
79 import org.apache.phoenix.thirdparty.com.google.common.util.concurrent.SettableFuture;
80 import com.google.inject.Guice;
81 import com.google.inject.Injector;
82
83 public class TestCompaction {
84
85 private static final Logger LOG = LoggerFactory.getLogger(TestCompaction.class);
86
87 private static final String TEST_FAMILY = "test-fam";
88 private static final String TEST_QUALIFIER = "test-qual";
89
90 private final byte[] fam = Bytes.toBytes(TEST_FAMILY);
91 private final byte[] qual = Bytes.toBytes(TEST_QUALIFIER);
92 private final byte[] data = Bytes.toBytes("testWrite-1");
93
94 private static final int MAX_VERSIONS = 3;
95
96 private Random randomGenerator;
97 private AbstractTransactionManager tm;
98
99 private Injector injector;
100
101 private Admin admin;
102 private Configuration hbaseConf;
103 private HBaseTestingUtility hbaseTestUtil;
104 private MiniHBaseCluster hbaseCluster;
105
106 private TSOServer tso;
107
108
109 private CommitTable commitTable;
110 private PostCommitActions syncPostCommitter;
111 private static Connection connection;
112
113 @BeforeClass
114 public void setupTestCompation() throws Exception {
115 TSOServerConfig tsoConfig = new TSOServerConfig();
116 tsoConfig.setPort(1234);
117 tsoConfig.setConflictMapSize(1);
118 tsoConfig.setWaitStrategy("LOW_CPU");
119 injector = Guice.createInjector(new TSOForHBaseCompactorTestModule(tsoConfig));
120 hbaseConf = injector.getInstance(Configuration.class);
121 HBaseCommitTableConfig hBaseCommitTableConfig = injector.getInstance(HBaseCommitTableConfig.class);
122 HBaseTimestampStorageConfig hBaseTimestampStorageConfig = injector.getInstance(HBaseTimestampStorageConfig.class);
123
124
125 hbaseConf.setInt("hbase.hstore.compaction.min", 2);
126 hbaseConf.setInt("hbase.hstore.compaction.max", 2);
127 setupHBase();
128 connection = ConnectionFactory.createConnection(hbaseConf);
129 admin = connection.getAdmin();
130 createRequiredHBaseTables(hBaseTimestampStorageConfig, hBaseCommitTableConfig);
131 setupTSO();
132
133 commitTable = injector.getInstance(CommitTable.class);
134 }
135
136 private void setupHBase() throws Exception {
137 LOG.info("--------------------------------------------------------------------------------------------------");
138 LOG.info("Setting up HBase");
139 LOG.info("--------------------------------------------------------------------------------------------------");
140 hbaseTestUtil = new HBaseTestingUtility(hbaseConf);
141 LOG.info("--------------------------------------------------------------------------------------------------");
142 LOG.info("Creating HBase MiniCluster");
143 LOG.info("--------------------------------------------------------------------------------------------------");
144 hbaseCluster = hbaseTestUtil.startMiniCluster(1);
145 }
146
147 private void createRequiredHBaseTables(HBaseTimestampStorageConfig timestampStorageConfig,
148 HBaseCommitTableConfig hBaseCommitTableConfig) throws IOException {
149 createTableIfNotExists(timestampStorageConfig.getTableName(), timestampStorageConfig.getFamilyName().getBytes());
150
151 createTableIfNotExists(hBaseCommitTableConfig.getTableName(), hBaseCommitTableConfig.getCommitTableFamily(), hBaseCommitTableConfig.getLowWatermarkFamily());
152 }
153
154 private void createTableIfNotExists(String tableName, byte[]... families) throws IOException {
155 if (!admin.tableExists(TableName.valueOf(tableName))) {
156 LOG.info("Creating {} table...", tableName);
157 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
158
159 for (byte[] family : families) {
160 HColumnDescriptor datafam = new HColumnDescriptor(family);
161 datafam.setMaxVersions(MAX_VERSIONS);
162 desc.addFamily(datafam);
163 }
164
165 desc.addCoprocessor("org.apache.hadoop.hbase.coprocessor.AggregateImplementation",null,Coprocessor.PRIORITY_HIGHEST,null);
166 admin.createTable(desc);
167 for (byte[] family : families) {
168 CompactorUtil.enableOmidCompaction(connection, TableName.valueOf(tableName), family);
169 }
170 }
171
172 }
173
174 private void setupTSO() throws IOException, InterruptedException {
175 tso = injector.getInstance(TSOServer.class);
176 tso.startAsync();
177 tso.awaitRunning();
178 TestUtils.waitForSocketListening("localhost", 1234, 100);
179 Thread.currentThread().setName("UnitTest(s) thread");
180 }
181
182 @AfterClass
183 public void cleanupTestCompation() throws Exception {
184 teardownTSO();
185 hbaseCluster.shutdown();
186 }
187
188 private void teardownTSO() throws IOException, InterruptedException {
189 tso.stopAsync();
190 tso.awaitTerminated();
191 TestUtils.waitForSocketNotListening("localhost", 1234, 1000);
192 }
193
194 @BeforeMethod
195 public void setupTestCompactionIndividualTest() throws Exception {
196 randomGenerator = new Random(0xfeedcafeL);
197 tm = spy((AbstractTransactionManager) newTransactionManager());
198 }
199
200 private TransactionManager newTransactionManager() throws Exception {
201 HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
202 hbaseOmidClientConf.setConnectionString("localhost:1234");
203 hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
204 CommitTable.Client commitTableClient = commitTable.getClient();
205 syncPostCommitter =
206 spy(new HBaseSyncPostCommitter(new NullMetricsProvider(),commitTableClient, connection));
207 return HBaseTransactionManager.builder(hbaseOmidClientConf)
208 .postCommitter(syncPostCommitter)
209 .commitTableClient(commitTableClient)
210 .build();
211 }
212
213
214 @Test
215 public void testShadowCellsAboveLWMSurviveCompaction() throws Exception {
216 String TEST_TABLE = "testShadowCellsAboveLWMSurviveCompaction";
217 createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
218 TTable txTable = new TTable(connection, TEST_TABLE);
219
220 byte[] rowId = Bytes.toBytes("row");
221
222
223 HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
224 Put put1 = new Put(rowId);
225 put1.addColumn(fam, qual, Bytes.toBytes("testValue 1"));
226 txTable.put(tx1, put1);
227 tm.commit(tx1);
228
229 HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
230 Put put2 = new Put(rowId);
231 put2.addColumn(fam, qual, Bytes.toBytes("testValue 2"));
232 txTable.put(tx2, put2);
233 tm.commit(tx2);
234
235 HBaseTransaction tx3 = (HBaseTransaction) tm.begin();
236 Put put3 = new Put(rowId);
237 put3.addColumn(fam, qual, Bytes.toBytes("testValue 3"));
238 txTable.put(tx3, put3);
239 tm.commit(tx3);
240
241
242 TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
243 assertTrue(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
244 "Put cell of Tx1 should be there");
245 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
246 "Put shadow cell of Tx1 should be there");
247 assertTrue(CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
248 "Put cell of Tx2 cell should be there");
249 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
250 "Put shadow cell of Tx2 should be there");
251 assertTrue(CellUtils.hasCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
252 "Put cell of Tx3 cell should be there");
253 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
254 "Put shadow cell of Tx3 should be there");
255
256
257 compactWithLWM(0, TEST_TABLE);
258
259
260 assertTrue(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
261 "Put cell of Tx1 should be there");
262 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
263 "Put shadow cell of Tx1 should be there");
264 assertTrue(CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
265 "Put cell of Tx2 cell should be there");
266 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
267 "Put shadow cell of Tx2 should be there");
268 assertTrue(CellUtils.hasCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
269 "Put cell of Tx3 cell should be there");
270 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
271 "Put shadow cell of Tx3 should be there");
272 }
273
274 @Test(timeOut = 60_000)
275 public void testStandardTXsWithShadowCellsAndWithSTBelowAndAboveLWMArePresevedAfterCompaction() throws Throwable {
276 String TEST_TABLE = "testStandardTXsWithShadowCellsAndWithSTBelowAndAboveLWMArePresevedAfterCompaction";
277 createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
278 TTable txTable = new TTable(connection, TEST_TABLE);
279
280 final int ROWS_TO_ADD = 5;
281
282 long fakeAssignedLowWatermark = 0L;
283 for (int i = 0; i < ROWS_TO_ADD; ++i) {
284 long rowId = randomGenerator.nextLong();
285 Transaction tx = tm.begin();
286 if (i == (ROWS_TO_ADD / 2)) {
287 fakeAssignedLowWatermark = tx.getTransactionId();
288 LOG.info("AssignedLowWatermark " + fakeAssignedLowWatermark);
289 }
290 Put put = new Put(Bytes.toBytes(rowId));
291 put.addColumn(fam, qual, data);
292 txTable.put(tx, put);
293 tm.commit(tx);
294 }
295
296 LOG.info("Flushing table {}", TEST_TABLE);
297 admin.flush(TableName.valueOf(TEST_TABLE));
298
299
300 LOG.info("Regions in table {}: {}", TEST_TABLE, hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).size());
301 OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).get(0)
302 .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
303 CommitTable commitTable = injector.getInstance(CommitTable.class);
304 CommitTable.Client commitTableClient = spy(commitTable.getClient());
305 SettableFuture<Long> f = SettableFuture.create();
306 f.set(fakeAssignedLowWatermark);
307 doReturn(f).when(commitTableClient).readLowWatermark();
308 omidCompactor.commitTableClient = commitTableClient;
309 LOG.info("Compacting table {}", TEST_TABLE);
310 admin.majorCompact(TableName.valueOf(TEST_TABLE));
311
312 LOG.info("Sleeping for 3 secs");
313 Thread.sleep(3000);
314 LOG.info("Waking up after 3 secs");
315
316
317 assertEquals(rowCount(TEST_TABLE, fam), ROWS_TO_ADD, "Rows in table after compacting should be " + ROWS_TO_ADD);
318 }
319
320 @Test(timeOut = 60_000)
321 public void testTXWithoutShadowCellsAndWithSTBelowLWMGetsShadowCellHealedAfterCompaction() throws Exception {
322 String TEST_TABLE = "testTXWithoutShadowCellsAndWithSTBelowLWMGetsShadowCellHealedAfterCompaction";
323 createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
324 TTable txTable = new TTable(connection, TEST_TABLE);
325
326
327 doThrow(new RuntimeException()).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
328
329 HBaseTransaction problematicTx = (HBaseTransaction) tm.begin();
330
331 long row = randomGenerator.nextLong();
332
333
334 Put put = new Put(Bytes.toBytes(row));
335 put.addColumn(fam, qual, data);
336 txTable.put(problematicTx, put);
337 try {
338 tm.commit(problematicTx);
339 } catch (Exception e) {
340
341 }
342
343 assertTrue(CellUtils.hasCell(Bytes.toBytes(row), fam, qual, problematicTx.getStartTimestamp(),
344 new TTableCellGetterAdapter(txTable)),
345 "Cell should be there");
346 assertFalse(CellUtils.hasShadowCell(Bytes.toBytes(row), fam, qual, problematicTx.getStartTimestamp(),
347 new TTableCellGetterAdapter(txTable)),
348 "Shadow cell should not be there");
349
350
351 LOG.info("Regions in table {}: {}", TEST_TABLE, hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).size());
352 OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).get(0)
353 .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
354 CommitTable commitTable = injector.getInstance(CommitTable.class);
355 CommitTable.Client commitTableClient = spy(commitTable.getClient());
356 SettableFuture<Long> f = SettableFuture.create();
357 f.set(Long.MAX_VALUE);
358 doReturn(f).when(commitTableClient).readLowWatermark();
359 omidCompactor.commitTableClient = commitTableClient;
360
361 LOG.info("Flushing table {}", TEST_TABLE);
362 admin.flush(TableName.valueOf(TEST_TABLE));
363
364 LOG.info("Compacting table {}", TEST_TABLE);
365 admin.majorCompact(TableName.valueOf(TEST_TABLE));
366
367 LOG.info("Sleeping for 3 secs");
368 Thread.sleep(3000);
369 LOG.info("Waking up after 3 secs");
370
371 assertTrue(CellUtils.hasCell(Bytes.toBytes(row), fam, qual, problematicTx.getStartTimestamp(),
372 new TTableCellGetterAdapter(txTable)),
373 "Cell should be there");
374 assertTrue(CellUtils.hasShadowCell(Bytes.toBytes(row), fam, qual, problematicTx.getStartTimestamp(),
375 new TTableCellGetterAdapter(txTable)),
376 "Shadow cell should not be there");
377 }
378
379 @Test(timeOut = 60_000)
380 public void testNeverendingTXsWithSTBelowAndAboveLWMAreDiscardedAndPreservedRespectivelyAfterCompaction()
381 throws Throwable {
382 String
383 TEST_TABLE =
384 "testNeverendingTXsWithSTBelowAndAboveLWMAreDiscardedAndPreservedRespectivelyAfterCompaction";
385 createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
386 TTable txTable = new TTable(connection, TEST_TABLE);
387
388
389 HBaseTransaction neverendingTxBelowLowWatermark = (HBaseTransaction) tm.begin();
390 long rowId = randomGenerator.nextLong();
391 Put put = new Put(Bytes.toBytes(rowId));
392 put.addColumn(fam, qual, data);
393 txTable.put(neverendingTxBelowLowWatermark, put);
394 assertTrue(CellUtils.hasCell(Bytes.toBytes(rowId), fam, qual, neverendingTxBelowLowWatermark.getStartTimestamp(),
395 new TTableCellGetterAdapter(txTable)),
396 "Cell should be there");
397 assertFalse(CellUtils.hasShadowCell(Bytes.toBytes(rowId), fam, qual, neverendingTxBelowLowWatermark.getStartTimestamp(),
398 new TTableCellGetterAdapter(txTable)),
399 "Shadow cell should not be there");
400
401
402 HBaseTransaction neverendingTxAboveLowWatermark = (HBaseTransaction) tm.begin();
403 long anotherRowId = randomGenerator.nextLong();
404 put = new Put(Bytes.toBytes(anotherRowId));
405 put.addColumn(fam, qual, data);
406 txTable.put(neverendingTxAboveLowWatermark, put);
407 assertTrue(CellUtils.hasCell(Bytes.toBytes(anotherRowId), fam, qual, neverendingTxAboveLowWatermark.getStartTimestamp(),
408 new TTableCellGetterAdapter(txTable)),
409 "Cell should be there");
410 assertFalse(CellUtils.hasShadowCell(Bytes.toBytes(anotherRowId), fam, qual, neverendingTxAboveLowWatermark.getStartTimestamp(),
411 new TTableCellGetterAdapter(txTable)),
412 "Shadow cell should not be there");
413
414 assertEquals(rowCount(TEST_TABLE, fam), 2, "Rows in table before flushing should be 2");
415 LOG.info("Flushing table {}", TEST_TABLE);
416 admin.flush(TableName.valueOf(TEST_TABLE));
417 assertEquals(rowCount(TEST_TABLE, fam), 2, "Rows in table after flushing should be 2");
418
419
420 LOG.info("Regions in table {}: {}", TEST_TABLE, hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).size());
421 OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).get(0)
422 .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
423 CommitTable commitTable = injector.getInstance(CommitTable.class);
424 CommitTable.Client commitTableClient = spy(commitTable.getClient());
425 SettableFuture<Long> f = SettableFuture.create();
426 f.set(neverendingTxBelowLowWatermark.getStartTimestamp());
427 doReturn(f).when(commitTableClient).readLowWatermark();
428 omidCompactor.commitTableClient = commitTableClient;
429 LOG.info("Compacting table {}", TEST_TABLE);
430 admin.majorCompact(TableName.valueOf(TEST_TABLE));
431
432 LOG.info("Sleeping for 3 secs");
433 Thread.sleep(3000);
434 LOG.info("Waking up after 3 secs");
435
436
437 assertEquals(rowCount(TEST_TABLE, fam), 1, "There should be only one row in table after compacting");
438
439 assertFalse(CellUtils.hasCell(Bytes.toBytes(rowId), fam, qual, neverendingTxBelowLowWatermark.getStartTimestamp(),
440 new TTableCellGetterAdapter(txTable)),
441 "Cell should not be there");
442 assertFalse(CellUtils.hasShadowCell(Bytes.toBytes(rowId), fam, qual, neverendingTxBelowLowWatermark.getStartTimestamp(),
443 new TTableCellGetterAdapter(txTable)),
444 "Shadow cell should not be there");
445
446 assertTrue(CellUtils.hasCell(Bytes.toBytes(anotherRowId), fam, qual, neverendingTxAboveLowWatermark.getStartTimestamp(),
447 new TTableCellGetterAdapter(txTable)),
448 "Cell should be there");
449 assertFalse(CellUtils.hasShadowCell(Bytes.toBytes(anotherRowId), fam, qual, neverendingTxAboveLowWatermark.getStartTimestamp(),
450 new TTableCellGetterAdapter(txTable)),
451 "Shadow cell should not be there");
452
453 }
454
455 @Test(timeOut = 60_000)
456 public void testRowsUnalteredWhenCommitTableCannotBeReached() throws Throwable {
457 String TEST_TABLE = "testRowsUnalteredWhenCommitTableCannotBeReached";
458 createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
459 TTable txTable = new TTable(connection, TEST_TABLE);
460
461
462
463 HBaseTransaction neverendingTx = (HBaseTransaction) tm.begin();
464 long rowId = randomGenerator.nextLong();
465 Put put = new Put(Bytes.toBytes(rowId));
466 put.addColumn(fam, qual, data);
467 txTable.put(neverendingTx, put);
468 assertTrue(CellUtils.hasCell(Bytes.toBytes(rowId), fam, qual, neverendingTx.getStartTimestamp(),
469 new TTableCellGetterAdapter(txTable)),
470 "Cell should be there");
471 assertFalse(CellUtils.hasShadowCell(Bytes.toBytes(rowId), fam, qual, neverendingTx.getStartTimestamp(),
472 new TTableCellGetterAdapter(txTable)),
473 "Shadow cell should not be there");
474
475 assertEquals(rowCount(TEST_TABLE, fam), 1, "There should be only one rows in table before flushing");
476 LOG.info("Flushing table {}", TEST_TABLE);
477 admin.flush(TableName.valueOf(TEST_TABLE));
478 assertEquals(rowCount(TEST_TABLE, fam), 1, "There should be only one rows in table after flushing");
479
480
481 LOG.info("Regions in table {}: {}", TEST_TABLE, hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).size());
482 OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).get(0)
483 .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
484 CommitTable commitTable = injector.getInstance(CommitTable.class);
485 CommitTable.Client commitTableClient = spy(commitTable.getClient());
486 SettableFuture<Long> f = SettableFuture.create();
487 f.setException(new IOException("Unable to read"));
488 doReturn(f).when(commitTableClient).readLowWatermark();
489 omidCompactor.commitTableClient = commitTableClient;
490 LOG.info("Compacting table {}", TEST_TABLE);
491 admin.majorCompact(TableName.valueOf(TEST_TABLE));
492
493 LOG.info("Sleeping for 3 secs");
494 Thread.sleep(3000);
495 LOG.info("Waking up after 3 secs");
496
497
498 assertEquals(rowCount(TEST_TABLE, fam), 1, "There should be only one row in table after compacting");
499 assertTrue(CellUtils.hasCell(Bytes.toBytes(rowId), fam, qual, neverendingTx.getStartTimestamp(),
500 new TTableCellGetterAdapter(txTable)),
501 "Cell should be there");
502 assertFalse(CellUtils.hasShadowCell(Bytes.toBytes(rowId), fam, qual, neverendingTx.getStartTimestamp(),
503 new TTableCellGetterAdapter(txTable)),
504 "Shadow cell should not be there");
505 }
506
507 @Test(timeOut = 60_000)
508 public void testOriginalTableParametersAreAvoidedAlsoWhenCompacting() throws Throwable {
509 String TEST_TABLE = "testOriginalTableParametersAreAvoidedAlsoWhenCompacting";
510 createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
511 TTable txTable = new TTable(connection, TEST_TABLE);
512
513 long rowId = randomGenerator.nextLong();
514 for (int versionCount = 0; versionCount <= (2 * MAX_VERSIONS); versionCount++) {
515 Transaction tx = tm.begin();
516 Put put = new Put(Bytes.toBytes(rowId));
517 put.addColumn(fam, qual, Bytes.toBytes("testWrite-" + versionCount));
518 txTable.put(tx, put);
519 tm.commit(tx);
520 }
521
522 Transaction tx = tm.begin();
523 Get get = new Get(Bytes.toBytes(rowId));
524 get.setMaxVersions(2 * MAX_VERSIONS);
525 assertEquals(get.getMaxVersions(), (2 * MAX_VERSIONS), "Max versions should be set to " + (2 * MAX_VERSIONS));
526 get.addColumn(fam, qual);
527 Result result = txTable.get(tx, get);
528 tm.commit(tx);
529 List<Cell> column = result.getColumnCells(fam, qual);
530 assertEquals(column.size(), 1, "There should be only one version in the result");
531
532 assertEquals(rowCount(TEST_TABLE, fam), 1, "There should be only one row in table before flushing");
533 LOG.info("Flushing table {}", TEST_TABLE);
534 admin.flush(TableName.valueOf(TEST_TABLE));
535 assertEquals(rowCount(TEST_TABLE, fam), 1, "There should be only one row in table after flushing");
536
537
538 compactEverything(TEST_TABLE);
539
540
541 assertEquals(rowCount(TEST_TABLE, fam), 1, "There should be only one row in table after compacting");
542
543 tx = tm.begin();
544 get = new Get(Bytes.toBytes(rowId));
545 get.setMaxVersions(2 * MAX_VERSIONS);
546 assertEquals(get.getMaxVersions(), (2 * MAX_VERSIONS), "Max versions should be set to " + (2 * MAX_VERSIONS));
547 get.addColumn(fam, qual);
548 result = txTable.get(tx, get);
549 tm.commit(tx);
550 column = result.getColumnCells(fam, qual);
551 assertEquals(column.size(), 1, "There should be only one version in the result");
552 assertEquals(Bytes.toString(CellUtil.cloneValue(column.get(0))), "testWrite-" + (2 * MAX_VERSIONS),
553 "Values don't match");
554 }
555
556
557
558
559 private void manualFlush(String tableName) throws Throwable {
560 LOG.info("Manually flushing all regions and waiting 2 secs");
561 HBaseShims.flushAllOnlineRegions(hbaseTestUtil.getHBaseCluster().getRegionServer(0),
562 TableName.valueOf(tableName));
563 TimeUnit.SECONDS.sleep(2);
564 }
565
566 @Test(timeOut = 60_000)
567 public void testOldCellsAreDiscardedAfterCompaction() throws Exception {
568 String TEST_TABLE = "testOldCellsAreDiscardedAfterCompaction";
569 createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
570 TTable txTable = new TTable(connection, TEST_TABLE);
571
572 byte[] rowId = Bytes.toBytes("row");
573
574
575 HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
576 Put put1 = new Put(rowId);
577 put1.addColumn(fam, qual, Bytes.toBytes("testValue 1"));
578 txTable.put(tx1, put1);
579 tm.commit(tx1);
580
581 HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
582 Put put2 = new Put(rowId);
583 put2.addColumn(fam, qual, Bytes.toBytes("testValue 2"));
584 txTable.put(tx2, put2);
585 tm.commit(tx2);
586
587 HBaseTransaction tx3 = (HBaseTransaction) tm.begin();
588 Put put3 = new Put(rowId);
589 put3.addColumn(fam, qual, Bytes.toBytes("testValue 3"));
590 txTable.put(tx3, put3);
591 tm.commit(tx3);
592
593
594 TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
595 assertTrue(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
596 "Put cell of Tx1 should be there");
597 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
598 "Put shadow cell of Tx1 should be there");
599 assertTrue(CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
600 "Put cell of Tx2 cell should be there");
601 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
602 "Put shadow cell of Tx2 should be there");
603 assertTrue(CellUtils.hasCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
604 "Put cell of Tx3 cell should be there");
605 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
606 "Put shadow cell of Tx3 should be there");
607
608
609 HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
610 compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
611
612
613 assertFalse(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
614 "Put cell of Tx1 should not be there");
615 assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
616 "Put shadow cell of Tx1 should not be there");
617 assertFalse(CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
618 "Put cell of Tx2 should not be there");
619 assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
620 "Put shadow cell of Tx2 should not be there");
621 assertTrue(CellUtils.hasCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
622 "Put cell of Tx3 cell should be there");
623 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
624 "Put shadow cell of Tx3 should be there");
625
626
627 HBaseTransaction newTx1 = (HBaseTransaction) tm.begin();
628 Get newGet1 = new Get(rowId);
629 newGet1.addColumn(fam, qual);
630 Result result = txTable.get(newTx1, newGet1);
631 assertEquals(Bytes.toBytes("testValue 3"), result.getValue(fam, qual));
632
633 Put newPut1 = new Put(rowId);
634 newPut1.addColumn(fam, qual, Bytes.toBytes("new testValue 1"));
635 txTable.put(newTx1, newPut1);
636
637
638 HBaseTransaction newTx2 = (HBaseTransaction) tm.begin();
639
640 tm.commit(newTx1);
641
642
643 Get newGet2 = new Get(rowId);
644 newGet2.addColumn(fam, qual);
645 result = txTable.get(newTx2, newGet2);
646 assertEquals(Bytes.toBytes("testValue 3"), result.getValue(fam, qual));
647 tm.commit(newTx2);
648
649
650
651 assertFalse(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
652 "Put cell of Tx1 should not be there");
653 assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
654 "Put shadow cell of Tx1 should not be there");
655 assertFalse(CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
656 "Put cell of Tx2 should not be there");
657 assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
658 "Put shadow cell of Tx2 should not be there");
659 assertTrue(CellUtils.hasCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
660 "Put cell of Tx3 cell should be there");
661 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx3.getStartTimestamp(), getter),
662 "Put shadow cell of Tx3 should be there");
663 assertTrue(CellUtils.hasCell(rowId, fam, qual, newTx1.getStartTimestamp(), getter),
664 "Put cell of NewTx1 cell should be there");
665 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, newTx1.getStartTimestamp(), getter),
666 "Put shadow cell of NewTx1 should be there");
667 }
668
669
670
671
672 @Test(timeOut = 60_000)
673 public void testDuplicateDeletes() throws Throwable {
674 String TEST_TABLE = "testDuplicateDeletes";
675 createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
676 TTable txTable = new TTable(connection, TEST_TABLE);
677
678
679
680
681
682
683
684
685 byte[] firstRow = "FirstRow".getBytes();
686 HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
687 Put put0 = new Put(firstRow);
688 put0.addColumn(fam, qual, Bytes.toBytes("testWrite-1"));
689 txTable.put(tx0, put0);
690 tm.commit(tx0);
691
692
693 manualFlush(TEST_TABLE);
694
695
696 byte[] rowToBeCompactedAway = "compactMe".getBytes();
697 HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
698 Put put1 = new Put(rowToBeCompactedAway);
699 put1.addColumn(fam, qual, Bytes.toBytes("testWrite-1"));
700 txTable.put(tx1, put1);
701 txTable.flushCommits();
702
703
704 byte[] row = "iCauseErrors".getBytes();
705 HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
706 Put put2 = new Put(row);
707 put2.addColumn(fam, qual, Bytes.toBytes("testWrite-1"));
708 txTable.put(tx2, put2);
709 tm.commit(tx2);
710
711 HBaseTransaction tx3 = (HBaseTransaction) tm.begin();
712 Put put3 = new Put(row);
713 put3.addColumn(fam, qual, Bytes.toBytes("testWrite-1"));
714 txTable.put(tx3, put3);
715 txTable.flushCommits();
716
717
718 Set<HBaseCellId> writeSet = tx3.getWriteSet();
719 assertEquals(1, writeSet.size());
720 List<HBaseCellId> newWriteSet = new ArrayList<>();
721 final AtomicBoolean flushFailing = new AtomicBoolean(true);
722 for (HBaseCellId id : writeSet) {
723 TTable failableHTable = spy(id.getTable());
724 doAnswer(new Answer<Void>() {
725 @Override
726 public Void answer(InvocationOnMock invocation)
727 throws Throwable {
728 if (flushFailing.get()) {
729 throw new RetriesExhaustedWithDetailsException(new ArrayList<Throwable>(),
730 new ArrayList<Row>(), new ArrayList<String>());
731 } else {
732 invocation.callRealMethod();
733 }
734 return null;
735 }
736 }).when(failableHTable).flushCommits();
737
738 newWriteSet.add(new HBaseCellId(failableHTable,
739 id.getRow(), id.getFamily(),
740 id.getQualifier(), id.getTimestamp()));
741 }
742 writeSet.clear();
743 writeSet.addAll(newWriteSet);
744
745 try {
746 tm.commit(tx3);
747 fail("Shouldn't succeed");
748 } catch (TransactionException tme) {
749 flushFailing.set(false);
750 tm.rollback(tx3);
751 }
752
753
754
755 manualFlush(TEST_TABLE);
756
757
758 byte[] anotherRow = "someotherrow".getBytes();
759 HBaseTransaction tx4 = (HBaseTransaction) tm.begin();
760 Put put4 = new Put(anotherRow);
761 put4.addColumn(fam, qual, Bytes.toBytes("testWrite-1"));
762 txTable.put(tx4, put4);
763 tm.commit(tx4);
764
765
766 manualFlush(TEST_TABLE);
767
768
769 setCompactorLWM(tx4.getStartTimestamp(), TEST_TABLE);
770 admin.compact(TableName.valueOf(TEST_TABLE));
771 Thread.sleep(3000);
772
773
774 assertFalse(CellUtils.hasCell(rowToBeCompactedAway, fam, qual, tx1.getStartTimestamp(),
775 new TTableCellGetterAdapter(txTable)),
776 "Cell should not be be there");
777 }
778
779 @Test(timeOut = 60_000)
780 public void testNonOmidCFIsUntouched() throws Throwable {
781 String TEST_TABLE = "testNonOmidCFIsUntouched";
782 createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
783 TTable txTable = new TTable(connection, TEST_TABLE);
784
785 admin.disableTable(TableName.valueOf(TEST_TABLE));
786 byte[] nonOmidCF = Bytes.toBytes("nonOmidCF");
787 byte[] nonOmidQual = Bytes.toBytes("nonOmidCol");
788 HColumnDescriptor nonomidfam = new HColumnDescriptor(nonOmidCF);
789 nonomidfam.setMaxVersions(MAX_VERSIONS);
790 admin.addColumn(TableName.valueOf(TEST_TABLE), nonomidfam);
791 admin.enableTable(TableName.valueOf(TEST_TABLE));
792
793 byte[] rowId = Bytes.toBytes("testRow");
794 Transaction tx = tm.begin();
795 Put put = new Put(rowId);
796 put.addColumn(fam, qual, Bytes.toBytes("testValue"));
797 txTable.put(tx, put);
798
799 Put nonTxPut = new Put(rowId);
800 nonTxPut.addColumn(nonOmidCF, nonOmidQual, Bytes.toBytes("nonTxVal"));
801 txTable.getHTable().put(nonTxPut);
802 txTable.flushCommits();
803
804 Get g = new Get(rowId);
805 Result result = txTable.getHTable().get(g);
806 assertEquals(result.getColumnCells(nonOmidCF, nonOmidQual).size(), 1, "Should be there, precompact");
807 assertEquals(result.getColumnCells(fam, qual).size(), 1, "Should be there, precompact");
808
809 compactEverything(TEST_TABLE);
810
811 result = txTable.getHTable().get(g);
812 assertEquals(result.getColumnCells(nonOmidCF, nonOmidQual).size(), 1, "Should be there, postcompact");
813 assertEquals(result.getColumnCells(fam, qual).size(), 0, "Should not be there, postcompact");
814 }
815
816
817
818
819
820
821
822
823 @Test(timeOut = 60_000)
824 public void testACellDeletedNonTransactionallyDoesNotAppearWhenAMajorCompactionOccurs() throws Throwable {
825 String TEST_TABLE = "testACellDeletedNonTransactionallyDoesNotAppearWhenAMajorCompactionOccurs";
826 createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
827 TTable txTable = new TTable(connection, TEST_TABLE);
828
829 Table table = txTable.getHTable();
830
831
832 HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
833 byte[] rowId = Bytes.toBytes("row1");
834 Put p0 = new Put(rowId);
835 p0.addColumn(fam, qual, Bytes.toBytes("testValue-0"));
836 txTable.put(tx0, p0);
837 tm.commit(tx0);
838
839
840 Delete d = new Delete(rowId);
841 d.addColumn(fam, qual);
842 table.delete(d);
843
844
845 HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
846 compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
847
848
849 Scan scan = new Scan();
850 scan.setRaw(true);
851 ResultScanner scannerResults = table.getScanner(scan);
852
853
854 assertNull(scannerResults.next(), "There should be no results in scan results");
855
856 table.close();
857
858 }
859
860
861
862
863
864 @Test(timeOut = 60_000)
865 public void testACellDeletedNonTransactionallyIsPreservedWhenMinorCompactionOccurs() throws Throwable {
866 String TEST_TABLE = "testACellDeletedNonTransactionallyIsPreservedWhenMinorCompactionOccurs";
867 createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
868 TTable txTable = new TTable(connection, TEST_TABLE);
869
870 Table table = txTable.getHTable();
871
872
873
874
875 HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
876 byte[] rowId = Bytes.toBytes("row1");
877 Put p0 = new Put(rowId);
878 p0.addColumn(fam, qual, Bytes.toBytes("testValue-0"));
879 txTable.put(tx0, p0);
880 tm.commit(tx0);
881
882
883 manualFlush(TEST_TABLE);
884
885
886 HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
887 Put p1 = new Put(rowId);
888 p1.addColumn(fam, qual, Bytes.toBytes("testValue-1"));
889 txTable.put(tx1, p1);
890 tm.commit(tx1);
891
892
893 manualFlush(TEST_TABLE);
894
895
896 HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
897 Put p2 = new Put(rowId);
898 p2.addColumn(fam, qual, Bytes.toBytes("testValue-2"));
899 txTable.put(tx2, p2);
900 tm.commit(tx2);
901
902
903 manualFlush(TEST_TABLE);
904
905
906 Delete d = new Delete(rowId);
907 d.addColumn(fam, qual);
908 table.delete(d);
909
910
911 manualFlush(TEST_TABLE);
912
913
914 HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
915 setCompactorLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
916 admin.compact(TableName.valueOf(TEST_TABLE));
917 Thread.sleep(5000);
918
919
920 Scan scan = new Scan();
921 scan.setRaw(true);
922 ResultScanner scannerResults = table.getScanner(scan);
923
924
925 int count = 0;
926 Result scanResult;
927 List<Cell> listOfCellsScanned = new ArrayList<>();
928 while ((scanResult = scannerResults.next()) != null) {
929 listOfCellsScanned = scanResult.listCells();
930 count++;
931 }
932 assertEquals(count, 1, "There should be only one result in scan results");
933 assertEquals(listOfCellsScanned.size(), 3, "There should be 3 cell entries in scan results (2 puts, 1 del)");
934 boolean wasDeletedCellFound = false;
935 int numberOfDeletedCellsFound = 0;
936 for (Cell cell : listOfCellsScanned) {
937 if (CellUtil.isDelete(cell)) {
938 wasDeletedCellFound = true;
939 numberOfDeletedCellsFound++;
940 }
941 }
942 assertTrue(wasDeletedCellFound, "We should have found a non-transactionally deleted cell");
943 assertEquals(numberOfDeletedCellsFound, 1, "There should be only only one deleted cell");
944
945 table.close();
946 }
947
948
949
950
951 @Test(timeOut = 60_000)
952 public void testTombstonesAreNotCleanedUpWhenMinorCompactionOccurs() throws Throwable {
953 String TEST_TABLE = "testTombstonesAreNotCleanedUpWhenMinorCompactionOccurs";
954 createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
955 TTable txTable = new TTable(connection, TEST_TABLE);
956
957
958
959 HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
960 byte[] rowId = Bytes.toBytes("case1");
961 Put p = new Put(rowId);
962 p.addColumn(fam, qual, Bytes.toBytes("testValue-0"));
963 txTable.put(tx0, p);
964 tm.commit(tx0);
965
966
967 manualFlush(TEST_TABLE);
968
969
970 HBaseTransaction deleteTx = (HBaseTransaction) tm.begin();
971 Delete d = new Delete(rowId);
972 d.addColumn(fam, qual);
973 txTable.delete(deleteTx, d);
974 tm.commit(deleteTx);
975
976
977 manualFlush(TEST_TABLE);
978
979 HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
980 Put p1 = new Put(rowId);
981 p1.addColumn(fam, qual, Bytes.toBytes("testValue-11"));
982 txTable.put(tx1, p1);
983 tm.commit(tx1);
984
985
986 manualFlush(TEST_TABLE);
987
988 HBaseTransaction lastTx = (HBaseTransaction) tm.begin();
989 Put p2 = new Put(rowId);
990 p2.addColumn(fam, qual, Bytes.toBytes("testValue-222"));
991 txTable.put(lastTx, p2);
992 tm.commit(lastTx);
993
994
995 HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
996 setCompactorLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
997 admin.compact(TableName.valueOf(TEST_TABLE));
998 Thread.sleep(5000);
999
1000
1001 TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
1002 assertFalse(CellUtils.hasCell(rowId, fam, qual, tx0.getStartTimestamp(), getter), "Put cell should be there");
1003 assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx0.getStartTimestamp(), getter),
1004 "Put shadow cell should be there");
1005 assertTrue(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter), "Put cell should be there");
1006 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1007 "Put shadow cell should be there");
1008 assertTrue(CellUtils.hasCell(rowId, fam, qual, deleteTx.getStartTimestamp(), getter),
1009 "Delete cell should be there");
1010 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, deleteTx.getStartTimestamp(), getter),
1011 "Delete shadow cell should be there");
1012 assertTrue(CellUtils.hasCell(rowId, fam, qual, lastTx.getStartTimestamp(), getter),
1013 "Put cell should be there");
1014 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, lastTx.getStartTimestamp(), getter),
1015 "Put shadow cell should be there");
1016 }
1017
1018
1019
1020
1021
1022 @Test(timeOut = 60_000)
1023 public void testTombstonesAreCleanedUpCase1() throws Exception {
1024 String TEST_TABLE = "testTombstonesAreCleanedUpCase1";
1025 createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
1026 TTable txTable = new TTable(connection, TEST_TABLE);
1027
1028 HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
1029 byte[] rowId = Bytes.toBytes("case1");
1030 Put p = new Put(rowId);
1031 p.addColumn(fam, qual, Bytes.toBytes("testValue"));
1032 txTable.put(tx1, p);
1033 tm.commit(tx1);
1034
1035 HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
1036 setCompactorLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
1037
1038 HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
1039 Delete d = new Delete(rowId);
1040 d.addColumn(fam, qual);
1041 txTable.delete(tx2, d);
1042 tm.commit(tx2);
1043
1044 TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
1045 assertTrue(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1046 "Put cell should be there");
1047 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1048 "Put shadow cell should be there");
1049 assertTrue(CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
1050 "Delete cell should be there");
1051 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
1052 "Delete shadow cell should be there");
1053 }
1054
1055
1056
1057
1058 @Test(timeOut = 60_000)
1059 public void testTombstonesAreCleanedUpCase2() throws Exception {
1060 String TEST_TABLE = "testTombstonesAreCleanedUpCase2";
1061 createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
1062 TTable txTable = new TTable(connection, TEST_TABLE);
1063
1064 HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
1065 byte[] rowId = Bytes.toBytes("case2");
1066 Put p = new Put(rowId);
1067 p.addColumn(fam, qual, Bytes.toBytes("testValue"));
1068 txTable.put(tx1, p);
1069 tm.commit(tx1);
1070
1071 HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
1072 Delete d = new Delete(rowId);
1073 d.addColumn(fam, qual);
1074 txTable.delete(tx2, d);
1075 tm.commit(tx2);
1076
1077 HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
1078 compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
1079
1080 TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
1081 assertFalse(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1082 "Put cell shouldn't be there");
1083 assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1084 "Put shadow cell shouldn't be there");
1085 assertFalse(CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
1086 "Delete cell shouldn't be there");
1087 assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
1088 "Delete shadow cell shouldn't be there");
1089 }
1090
1091
1092
1093 @Test(timeOut = 60_000)
1094 public void testFamiliyDeleteTombstonesAreCleanedUpCellCF() throws Exception {
1095 String TEST_TABLE = "testFamiliyDeleteTombstonesAreCleanedUpCellCF";
1096 byte[] fam2 = Bytes.toBytes("2");
1097 byte[] fam3 = Bytes.toBytes("3");
1098 byte[] fam4 = Bytes.toBytes("4");
1099 createTableIfNotExists(TEST_TABLE, fam2, fam3, fam4);
1100 TTable txTable = new TTable(connection, TEST_TABLE);
1101
1102 HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
1103 byte[] rowId = Bytes.toBytes("case2");
1104 byte[] qual2 = Bytes.toBytes("qual2");
1105
1106 Put p = new Put(rowId);
1107 p.addColumn(fam2, qual, Bytes.toBytes("testValue"));
1108 p.addColumn(fam2, qual2 , Bytes.toBytes("testValue"));
1109
1110 p.addColumn(fam3, qual, Bytes.toBytes("testValue"));
1111 p.addColumn(fam3, qual2 , Bytes.toBytes("testValue"));
1112
1113 p.addColumn(fam4, qual, Bytes.toBytes("testValue"));
1114 p.addColumn(fam4, qual2 , Bytes.toBytes("testValue"));
1115
1116
1117 txTable.put(tx1, p);
1118
1119 byte[] rowId2 = Bytes.toBytes("case22");
1120 Put p2 = new Put(rowId2);
1121 p2.addColumn(fam3, qual, Bytes.toBytes("testValue2"));
1122 txTable.put(tx1, p2);
1123
1124 tm.commit(tx1);
1125
1126 HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
1127 Delete d = new Delete(rowId);
1128 d.addFamily(fam3);
1129 txTable.delete(tx2, d);
1130 tm.commit(tx2);
1131
1132 HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
1133
1134 TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
1135 assertTrue(CellUtils.hasCell(rowId, fam2, qual, tx1.getStartTimestamp(), getter),
1136 "Put cell should be there");
1137 assertTrue(CellUtils.hasShadowCell(rowId, fam2, qual, tx1.getStartTimestamp(), getter),
1138 "Put shadow cell should be there");
1139 assertTrue(CellUtils.hasCell(rowId, fam2, qual2, tx1.getStartTimestamp(), getter),
1140 "Put cell should be there");
1141 assertTrue(CellUtils.hasShadowCell(rowId, fam2, qual2, tx1.getStartTimestamp(), getter),
1142 "Put shadow cell should be there");
1143
1144 assertTrue(CellUtils.hasCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1145 "Put cell should be there");
1146 assertTrue(CellUtils.hasShadowCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1147 "Put shadow cell should be there");
1148 assertTrue(CellUtils.hasCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1149 "Put cell should be there");
1150 assertTrue(CellUtils.hasShadowCell(rowId, fam3, qual2, tx1.getStartTimestamp(), getter),
1151 "Put shadow cell should be there");
1152
1153 assertTrue(CellUtils.hasCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1154 "Put cell should be there");
1155 assertTrue(CellUtils.hasShadowCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1156 "Put shadow cell should be there");
1157 assertTrue(CellUtils.hasCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1158 "Put cell should be there");
1159 assertTrue(CellUtils.hasShadowCell(rowId, fam4, qual2, tx1.getStartTimestamp(), getter),
1160 "Put shadow cell should be there");
1161
1162
1163 assertTrue(CellUtils.hasCell(rowId, fam3, CellUtils.FAMILY_DELETE_QUALIFIER, tx2.getStartTimestamp(), getter),
1164 "Delete cell should be there");
1165 assertTrue(CellUtils.hasShadowCell(rowId, fam3, CellUtils.FAMILY_DELETE_QUALIFIER, tx2.getStartTimestamp(), getter),
1166 "Delete shadow cell should be there");
1167
1168
1169 compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
1170
1171
1172
1173 assertTrue(CellUtils.hasCell(rowId, fam2, qual, tx1.getStartTimestamp(), getter),
1174 "Put cell should be there");
1175 assertTrue(CellUtils.hasShadowCell(rowId, fam2, qual, tx1.getStartTimestamp(), getter),
1176 "Put shadow cell should be there");
1177 assertTrue(CellUtils.hasCell(rowId, fam2, qual2, tx1.getStartTimestamp(), getter),
1178 "Put cell should be there");
1179 assertTrue(CellUtils.hasShadowCell(rowId, fam2, qual2, tx1.getStartTimestamp(), getter),
1180 "Put shadow cell should be there");
1181
1182
1183 assertFalse(CellUtils.hasCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1184 "Put cell shouldn't be there");
1185 assertFalse(CellUtils.hasShadowCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1186 "Put shadow cell shouldn't be there");
1187 assertFalse(CellUtils.hasCell(rowId, fam3, qual2, tx1.getStartTimestamp(), getter),
1188 "Put cell shouldn't be there");
1189 assertFalse(CellUtils.hasShadowCell(rowId, fam3, qual2, tx1.getStartTimestamp(), getter),
1190 "Put shadow cell shouldn't be there");
1191
1192
1193 assertTrue(CellUtils.hasCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1194 "Put cell should be there");
1195 assertTrue(CellUtils.hasShadowCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1196 "Put shadow cell should be there");
1197 assertTrue(CellUtils.hasCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1198 "Put cell should be there");
1199 assertTrue(CellUtils.hasShadowCell(rowId, fam4, qual2, tx1.getStartTimestamp(), getter),
1200 "Put shadow cell should be there");
1201
1202
1203
1204 assertFalse(CellUtils.hasCell(rowId, fam3, CellUtils.FAMILY_DELETE_QUALIFIER, tx2.getStartTimestamp(), getter),
1205 "Delete cell shouldn't be there");
1206 assertFalse(CellUtils.hasShadowCell(rowId, fam3, CellUtils.FAMILY_DELETE_QUALIFIER, tx2.getStartTimestamp(), getter),
1207 "Delete shadow cell shouldn't be there");
1208 }
1209
1210
1211
1212 @Test(timeOut = 60_000)
1213 public void testFamiliyDeleteTombstonesAreCleanedUpRowCF() throws Exception {
1214 String TEST_TABLE = "testFamiliyDeleteTombstonesAreCleanedUpRowCF";
1215 ((HBaseTransactionManager) tm).setConflictDetectionLevel(OmidClientConfiguration.ConflictDetectionLevel.ROW);
1216
1217 byte[] fam2 = Bytes.toBytes("2");
1218 byte[] fam3 = Bytes.toBytes("3");
1219 byte[] fam4 = Bytes.toBytes("4");
1220 createTableIfNotExists(TEST_TABLE, fam2, fam3, fam4);
1221 TTable txTable = new TTable(connection, TEST_TABLE);
1222
1223 HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
1224 byte[] rowId = Bytes.toBytes("case2");
1225 byte[] qual2 = Bytes.toBytes("qual2");
1226
1227 Put p = new Put(rowId);
1228 p.addColumn(fam2, qual, Bytes.toBytes("testValue"));
1229 p.addColumn(fam2, qual2 , Bytes.toBytes("testValue"));
1230
1231 p.addColumn(fam3, qual, Bytes.toBytes("testValue"));
1232 p.addColumn(fam3, qual2 , Bytes.toBytes("testValue"));
1233
1234 p.addColumn(fam4, qual, Bytes.toBytes("testValue"));
1235 p.addColumn(fam4, qual2 , Bytes.toBytes("testValue"));
1236
1237
1238 txTable.put(tx1, p);
1239
1240 byte[] rowId2 = Bytes.toBytes("case22");
1241 Put p2 = new Put(rowId2);
1242 p2.addColumn(fam3, qual, Bytes.toBytes("testValue2"));
1243 txTable.put(tx1, p2);
1244
1245 tm.commit(tx1);
1246
1247 HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
1248 Delete d = new Delete(rowId);
1249 d.addFamily(fam3);
1250 txTable.delete(tx2, d);
1251 tm.commit(tx2);
1252
1253 HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
1254
1255 TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
1256 assertTrue(CellUtils.hasCell(rowId, fam2, qual, tx1.getStartTimestamp(), getter),
1257 "Put cell should be there");
1258 assertTrue(CellUtils.hasShadowCell(rowId, fam2, qual, tx1.getStartTimestamp(), getter),
1259 "Put shadow cell should be there");
1260 assertTrue(CellUtils.hasCell(rowId, fam2, qual2, tx1.getStartTimestamp(), getter),
1261 "Put cell should be there");
1262 assertTrue(CellUtils.hasShadowCell(rowId, fam2, qual2, tx1.getStartTimestamp(), getter),
1263 "Put shadow cell should be there");
1264
1265 assertTrue(CellUtils.hasCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1266 "Put cell should be there");
1267 assertTrue(CellUtils.hasShadowCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1268 "Put shadow cell should be there");
1269 assertTrue(CellUtils.hasCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1270 "Put cell should be there");
1271 assertTrue(CellUtils.hasShadowCell(rowId, fam3, qual2, tx1.getStartTimestamp(), getter),
1272 "Put shadow cell should be there");
1273
1274 assertTrue(CellUtils.hasCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1275 "Put cell should be there");
1276 assertTrue(CellUtils.hasShadowCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1277 "Put shadow cell should be there");
1278 assertTrue(CellUtils.hasCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1279 "Put cell should be there");
1280 assertTrue(CellUtils.hasShadowCell(rowId, fam4, qual2, tx1.getStartTimestamp(), getter),
1281 "Put shadow cell should be there");
1282
1283
1284 assertTrue(CellUtils.hasCell(rowId, fam3, CellUtils.FAMILY_DELETE_QUALIFIER, tx2.getStartTimestamp(), getter),
1285 "Delete cell should be there");
1286 assertTrue(CellUtils.hasShadowCell(rowId, fam3, CellUtils.FAMILY_DELETE_QUALIFIER, tx2.getStartTimestamp(), getter),
1287 "Delete shadow cell should be there");
1288
1289
1290 compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
1291
1292
1293
1294 assertTrue(CellUtils.hasCell(rowId, fam2, qual, tx1.getStartTimestamp(), getter),
1295 "Put cell should be there");
1296 assertTrue(CellUtils.hasShadowCell(rowId, fam2, qual, tx1.getStartTimestamp(), getter),
1297 "Put shadow cell should be there");
1298 assertTrue(CellUtils.hasCell(rowId, fam2, qual2, tx1.getStartTimestamp(), getter),
1299 "Put cell should be there");
1300 assertTrue(CellUtils.hasShadowCell(rowId, fam2, qual2, tx1.getStartTimestamp(), getter),
1301 "Put shadow cell should be there");
1302
1303
1304 assertFalse(CellUtils.hasCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1305 "Put cell shouldn't be there");
1306 assertFalse(CellUtils.hasShadowCell(rowId, fam3, qual, tx1.getStartTimestamp(), getter),
1307 "Put shadow cell shouldn't be there");
1308 assertFalse(CellUtils.hasCell(rowId, fam3, qual2, tx1.getStartTimestamp(), getter),
1309 "Put cell shouldn't be there");
1310 assertFalse(CellUtils.hasShadowCell(rowId, fam3, qual2, tx1.getStartTimestamp(), getter),
1311 "Put shadow cell shouldn't be there");
1312
1313
1314 assertTrue(CellUtils.hasCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1315 "Put cell should be there");
1316 assertTrue(CellUtils.hasShadowCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1317 "Put shadow cell should be there");
1318 assertTrue(CellUtils.hasCell(rowId, fam4, qual, tx1.getStartTimestamp(), getter),
1319 "Put cell should be there");
1320 assertTrue(CellUtils.hasShadowCell(rowId, fam4, qual2, tx1.getStartTimestamp(), getter),
1321 "Put shadow cell should be there");
1322
1323
1324
1325 assertFalse(CellUtils.hasCell(rowId, fam3, CellUtils.FAMILY_DELETE_QUALIFIER, tx2.getStartTimestamp(), getter),
1326 "Delete cell shouldn't be there");
1327 assertFalse(CellUtils.hasShadowCell(rowId, fam3, CellUtils.FAMILY_DELETE_QUALIFIER, tx2.getStartTimestamp(), getter),
1328 "Delete shadow cell shouldn't be there");
1329 }
1330
1331
1332
1333
1334
1335
1336
1337 @Test(timeOut = 60_000)
1338 public void testTombstonesAreCleanedUpCase3() throws Exception {
1339 String TEST_TABLE = "testTombstonesAreCleanedUpCase3";
1340 createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
1341 TTable txTable = new TTable(connection, TEST_TABLE);
1342
1343 HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
1344 byte[] rowId = Bytes.toBytes("case3");
1345 Put p = new Put(rowId);
1346 p.addColumn(fam, qual, Bytes.toBytes("testValue"));
1347 txTable.put(tx1, p);
1348 tm.commit(tx1);
1349
1350 HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
1351 Delete d = new Delete(rowId);
1352 d.addColumn(fam, qual);
1353 txTable.delete(tx2, d);
1354
1355 HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
1356 compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
1357
1358 TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
1359 assertTrue(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1360 "Put cell should be there");
1361 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1362 "Put shadow cell shouldn't be there");
1363 assertFalse(CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
1364 "Delete cell shouldn't be there");
1365 assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
1366 "Delete shadow cell shouldn't be there");
1367 }
1368
1369
1370
1371
1372
1373 @Test(timeOut = 60_000)
1374 public void testTombstonesAreCleanedUpCase4() throws Exception {
1375 String TEST_TABLE = "testTombstonesAreCleanedUpCase4";
1376 createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
1377 TTable txTable = new TTable(connection, TEST_TABLE);
1378
1379 HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
1380 byte[] rowId = Bytes.toBytes("case4");
1381 Put p = new Put(rowId);
1382 p.addColumn(fam, qual, Bytes.toBytes("testValue"));
1383 txTable.put(tx1, p);
1384 tm.commit(tx1);
1385
1386 HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
1387
1388 HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
1389 Delete d = new Delete(rowId);
1390 d.addColumn(fam, qual);
1391 txTable.delete(tx2, d);
1392 compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
1393
1394 TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
1395 assertTrue(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1396 "Put cell should be there");
1397 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1398 "Put shadow cell shouldn't be there");
1399 assertTrue(CellUtils.hasCell(rowId, fam, qual,tx2.getStartTimestamp(), getter),
1400 "Delete cell should be there");
1401 assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
1402 "Delete shadow cell shouldn't be there");
1403 }
1404
1405
1406
1407
1408 @Test(timeOut = 60_000)
1409 public void testTombstonesAreCleanedUpCase5() throws Exception {
1410 String TEST_TABLE = "testTombstonesAreCleanedUpCase5";
1411 createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
1412 TTable txTable = new TTable(connection, TEST_TABLE);
1413
1414 HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
1415 byte[] rowId = Bytes.toBytes("case5");
1416 Delete d = new Delete(rowId);
1417 d.addColumn(fam, qual);
1418 txTable.delete(tx1, d);
1419 tm.commit(tx1);
1420
1421 HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
1422 compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
1423
1424 TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
1425 assertFalse(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1426 "Delete cell shouldn't be there");
1427 assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1428 "Delete shadow cell shouldn't be there");
1429 }
1430
1431
1432
1433
1434 @Test(timeOut = 60_000)
1435 public void testTombstonesAreCleanedUpCase6() throws Exception {
1436 String TEST_TABLE = "testTombstonesAreCleanedUpCase6";
1437 createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
1438 TTable txTable = new TTable(connection, TEST_TABLE);
1439 byte[] rowId = Bytes.toBytes("case6");
1440
1441 HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
1442 Delete d = new Delete(rowId);
1443 d.addColumn(fam, qual);
1444 txTable.delete(tx1, d);
1445 tm.commit(tx1);
1446
1447 HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
1448 Put p = new Put(rowId);
1449 p.addColumn(fam, qual, Bytes.toBytes("testValue"));
1450 txTable.put(tx2, p);
1451 tm.commit(tx2);
1452
1453 HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
1454 compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
1455
1456 TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
1457 assertFalse(CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1458 "Delete cell shouldn't be there");
1459 assertFalse(CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter),
1460 "Delete shadow cell shouldn't be there");
1461 assertTrue(CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
1462 "Put cell should be there");
1463 assertTrue(CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter),
1464 "Put shadow cell shouldn't be there");
1465 }
1466
1467 @Test(timeOut = 60_000)
1468 public void testCommitTableNoInvalidation() throws Exception {
1469 String TEST_TABLE = "testCommitTableInvalidation";
1470 createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
1471 TTable txTable = new TTable(connection, TEST_TABLE);
1472 byte[] rowId = Bytes.toBytes("row");
1473
1474 HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
1475 Put p = new Put(rowId);
1476 p.addColumn(fam, qual, Bytes.toBytes("testValue"));
1477 txTable.put(tx1, p);
1478
1479 HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
1480 compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
1481
1482 try {
1483
1484 Thread.sleep(1000);
1485
1486 tm.commit(tx1);
1487
1488 } catch (RollbackException e) {
1489 fail(" Should have not been invalidated");
1490 }
1491 }
1492
1493
1494 private void setCompactorLWM(long lwm, String tableName) throws Exception {
1495 OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(tableName)).get(0)
1496 .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
1497 CommitTable commitTable = injector.getInstance(CommitTable.class);
1498 CommitTable.Client commitTableClient = spy(commitTable.getClient());
1499 SettableFuture<Long> f = SettableFuture.create();
1500 f.set(lwm);
1501 doReturn(f).when(commitTableClient).readLowWatermark();
1502 omidCompactor.commitTableClient = commitTableClient;
1503 }
1504
1505 private void compactEverything(String tableName) throws Exception {
1506 compactWithLWM(Long.MAX_VALUE, tableName);
1507 }
1508
1509 private void compactWithLWM(long lwm, String tableName) throws Exception {
1510 admin.flush(TableName.valueOf(tableName));
1511
1512 LOG.info("Regions in table {}: {}", tableName, hbaseCluster.getRegions(Bytes.toBytes(tableName)).size());
1513 setCompactorLWM(lwm, tableName);
1514 LOG.info("Compacting table {}", tableName);
1515 admin.majorCompact(TableName.valueOf(tableName));
1516
1517 LOG.info("Sleeping for 3 secs");
1518 Thread.sleep(3000);
1519 LOG.info("Waking up after 3 secs");
1520 }
1521
1522 private static long rowCount(String tableName, byte[] family) throws Throwable {
1523 Scan scan = new Scan();
1524 scan.addFamily(family);
1525 Table table = connection.getTable(TableName.valueOf(tableName));
1526 try (ResultScanner scanner = table.getScanner(scan)) {
1527 int count = 0;
1528 while (scanner.next() != null) {
1529 count++;
1530 }
1531 return count;
1532 }
1533 }
1534 }