1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.omid.transaction;
19
20 import static org.testng.Assert.assertEquals;
21 import static org.testng.Assert.assertTrue;
22
23 import java.io.IOException;
24 import java.util.HashMap;
25 import java.util.Map;
26
27 import org.apache.hadoop.hbase.client.Delete;
28 import org.apache.hadoop.hbase.client.Get;
29 import org.apache.hadoop.hbase.client.Put;
30 import org.apache.hadoop.hbase.client.Result;
31 import org.apache.hadoop.hbase.client.ResultScanner;
32 import org.apache.hadoop.hbase.client.Scan;
33 import org.apache.hadoop.hbase.util.Bytes;
34 import org.apache.omid.tso.client.OmidClientConfiguration.ConflictDetectionLevel;
35 import org.slf4j.Logger;
36 import org.slf4j.LoggerFactory;
37 import org.testng.ITestContext;
38 import org.testng.annotations.Test;
39
40 @Test(groups = "sharedHBase")
41 public class TestDeletion extends OmidTestBase {
42
43 private static final Logger LOG = LoggerFactory.getLogger(TestDeletion.class);
44
45 private byte[] famA = Bytes.toBytes(TEST_FAMILY);
46 private byte[] famB = Bytes.toBytes(TEST_FAMILY2);
47 private byte[] colA = Bytes.toBytes("testdataA");
48 private byte[] colB = Bytes.toBytes("testdataB");
49 private byte[] data1 = Bytes.toBytes("testWrite-1");
50 private byte[] modrow = Bytes.toBytes("test-del" + 0);
51
52 private static class FamCol {
53
54 final byte[] fam;
55 final byte[] col;
56
57 FamCol(byte[] fam, byte[] col) {
58 this.fam = fam;
59 this.col = col;
60 }
61
62 }
63
64 @Test(timeOut = 10_000)
65 public void runTestDeleteFamilyRow(ITestContext context) throws Exception {
66
67 TransactionManager tm = newTransactionManager(context);
68 TTable tt = new TTable(connection, TEST_TABLE);
69
70 ((HBaseTransactionManager) tm).setConflictDetectionLevel(ConflictDetectionLevel.ROW);
71
72 Transaction t1 = tm.begin();
73 LOG.info("Transaction created " + t1);
74
75 int rowsWritten = 1;
76 FamCol famColA = new FamCol(famA, colA);
77 writeRows(tt, t1, rowsWritten, famColA);
78 tm.commit(t1);
79
80 Transaction t2 = tm.begin();
81 Delete d = new Delete(modrow);
82 d.addFamily(famA);
83 tt.delete(t2, d);
84
85 Transaction tscan = tm.begin();
86 ResultScanner rs = tt.getScanner(tscan, new Scan());
87
88 Map<FamCol, Integer> count = countColsInRows(rs, famColA);
89 assertEquals((int) count.get(famColA), rowsWritten, "ColA count should be equal to rowsWritten");
90 if (getClient(context).isLowLatency()) {
91 return;
92 }
93 tm.commit(t2);
94
95 tscan = tm.begin();
96 rs = tt.getScanner(tscan, new Scan());
97
98 count = countColsInRows(rs, famColA);
99 Integer countFamColA = count.get(famColA);
100 assertEquals(countFamColA, null);
101
102 Transaction t3 = tm.begin();
103 d.addFamily(famA);
104 tt.delete(t3, d);
105
106 tscan = tm.begin();
107 rs = tt.getScanner(tscan, new Scan());
108
109 count = countColsInRows(rs, famColA);
110 countFamColA = count.get(famColA);
111 assertEquals(countFamColA, null);
112
113 ((HBaseTransactionManager) tm).setConflictDetectionLevel(ConflictDetectionLevel.CELL);
114
115 }
116
117 @Test(timeOut = 10_000)
118 public void runTestDeleteFamilyCell(ITestContext context) throws Exception {
119
120 TransactionManager tm = newTransactionManager(context);
121 TTable tt = new TTable(connection, TEST_TABLE);
122
123 Transaction t1 = tm.begin();
124 LOG.info("Transaction created " + t1);
125
126 int rowsWritten = 1;
127 FamCol famColA = new FamCol(famA, colA);
128 writeRows(tt, t1, rowsWritten, famColA);
129 tm.commit(t1);
130
131 Transaction t2 = tm.begin();
132 Delete d = new Delete(modrow);
133 d.addFamily(famA);
134 tt.delete(t2, d);
135
136 Transaction tscan = tm.begin();
137 ResultScanner rs = tt.getScanner(tscan, new Scan());
138
139 Map<FamCol, Integer> count = countColsInRows(rs, famColA);
140 assertEquals((int) count.get(famColA), rowsWritten, "ColA count should be equal to rowsWritten");
141 if (getClient(context).isLowLatency()) {
142 return;
143 }
144 tm.commit(t2);
145
146 tscan = tm.begin();
147 rs = tt.getScanner(tscan, new Scan());
148
149 count = countColsInRows(rs, famColA);
150 Integer countFamColA = count.get(famColA);
151 assertEquals(countFamColA, null);
152
153 Transaction t3 = tm.begin();
154 d.addFamily(famA);
155 tt.delete(t3, d);
156
157 tscan = tm.begin();
158 rs = tt.getScanner(tscan, new Scan());
159
160 count = countColsInRows(rs, famColA);
161 countFamColA = count.get(famColA);
162 assertEquals(countFamColA, null);
163
164 }
165
166 @Test(timeOut = 10_000)
167 public void runTestDeleteFamily(ITestContext context) throws Exception {
168
169 TransactionManager tm = newTransactionManager(context);
170 TTable tt = new TTable(connection, TEST_TABLE);
171
172 Transaction t1 = tm.begin();
173 LOG.info("Transaction created " + t1);
174
175 int rowsWritten = 10;
176 FamCol famColA = new FamCol(famA, colA);
177 FamCol famColB = new FamCol(famB, colB);
178 writeRows(tt, t1, rowsWritten, famColA, famColB);
179 tm.commit(t1);
180
181 Transaction t2 = tm.begin();
182 Delete d = new Delete(modrow);
183 d.addFamily(famA);
184 tt.delete(t2, d);
185
186 Transaction tscan = tm.begin();
187 ResultScanner rs = tt.getScanner(tscan, new Scan());
188
189 Map<FamCol, Integer> count = countColsInRows(rs, famColA, famColB);
190 assertEquals((int) count.get(famColA), rowsWritten, "ColA count should be equal to rowsWritten");
191 assertEquals((int) count.get(famColB), rowsWritten, "ColB count should be equal to rowsWritten");
192 if (getClient(context).isLowLatency()) {
193 return;
194 }
195 tm.commit(t2);
196
197 tscan = tm.begin();
198 rs = tt.getScanner(tscan, new Scan());
199
200 count = countColsInRows(rs, famColA, famColB);
201 assertEquals((int) count.get(famColA), (rowsWritten - 1), "ColA count should be equal to rowsWritten - 1");
202 assertEquals((int) count.get(famColB), rowsWritten, "ColB count should be equal to rowsWritten");
203 }
204
205 @Test(timeOut = 10_000)
206 public void runTestDeleteFamilyRowLevelCA(ITestContext context) throws Exception {
207
208 TransactionManager tm = newTransactionManager(context);
209 TTable tt = new TTable(connection, TEST_TABLE);
210
211 ((HBaseTransactionManager) tm).setConflictDetectionLevel(ConflictDetectionLevel.ROW);
212
213 Transaction t1 = tm.begin();
214 LOG.info("Transaction created " + t1);
215
216 int rowsWritten = 10;
217 FamCol famColA = new FamCol(famA, colA);
218 FamCol famColB = new FamCol(famB, colB);
219 writeRows(tt, t1, rowsWritten, famColA, famColB);
220 tm.commit(t1);
221
222 Transaction t2 = tm.begin();
223 Delete d = new Delete(modrow);
224 d.addFamily(famA);
225 tt.delete(t2, d);
226
227 Transaction tscan = tm.begin();
228 ResultScanner rs = tt.getScanner(tscan, new Scan());
229
230 Map<FamCol, Integer> count = countColsInRows(rs, famColA, famColB);
231 assertEquals((int) count.get(famColA), rowsWritten, "ColA count should be equal to rowsWritten");
232 assertEquals((int) count.get(famColB), rowsWritten, "ColB count should be equal to rowsWritten");
233 if (getClient(context).isLowLatency()) {
234 return;
235 }
236 tm.commit(t2);
237
238 tscan = tm.begin();
239 rs = tt.getScanner(tscan, new Scan());
240
241 count = countColsInRows(rs, famColA, famColB);
242 assertEquals((int) count.get(famColA), (rowsWritten - 1), "ColA count should be equal to rowsWritten - 1");
243 assertEquals((int) count.get(famColB), rowsWritten, "ColB count should be equal to rowsWritten");
244
245 ((HBaseTransactionManager) tm).setConflictDetectionLevel(ConflictDetectionLevel.CELL);
246 }
247
248 @Test(timeOut = 10_000)
249 public void runTestDeleteFamilyAborts(ITestContext context) throws Exception {
250
251 TransactionManager tm = newTransactionManager(context);
252 TTable tt = new TTable(connection, TEST_TABLE);
253
254 ((HBaseTransactionManager) tm).setConflictDetectionLevel(ConflictDetectionLevel.ROW);
255
256 Transaction t1 = tm.begin();
257 LOG.info("Transaction created " + t1);
258
259 int rowsWritten = 10;
260 FamCol famColA = new FamCol(famA, colA);
261 FamCol famColB = new FamCol(famB, colB);
262 writeRows(tt, t1, rowsWritten, famColA, famColB);
263
264 Transaction t2 = tm.begin();
265
266 tm.commit(t1);
267
268 Delete d = new Delete(modrow);
269 d.addFamily(famA);
270 tt.delete(t2, d);
271
272 try {
273 tm.commit(t2);
274 } catch(RollbackException e) {
275 System.out.println("Rollback");
276 System.out.flush();
277 }
278
279 Transaction tscan = tm.begin();
280 ResultScanner rs = tt.getScanner(tscan, new Scan());
281
282 Map<FamCol, Integer> count = countColsInRows(rs, famColA, famColB);
283 assertEquals((int) count.get(famColA), rowsWritten, "ColA count should be equal to rowsWritten");
284 assertEquals((int) count.get(famColB), rowsWritten, "ColB count should be equal to rowsWritten");
285
286 ((HBaseTransactionManager) tm).setConflictDetectionLevel(ConflictDetectionLevel.CELL);
287 }
288
289 @Test(timeOut = 10_000)
290 public void runTestDeleteColumn(ITestContext context) throws Exception {
291
292 TransactionManager tm = newTransactionManager(context);
293 TTable tt = new TTable(connection, TEST_TABLE);
294
295 Transaction t1 = tm.begin();
296 LOG.info("Transaction created " + t1);
297
298 int rowsWritten = 10;
299
300 FamCol famColA = new FamCol(famA, colA);
301 FamCol famColB = new FamCol(famA, colB);
302 writeRows(tt, t1, rowsWritten, famColA, famColB);
303 tm.commit(t1);
304
305 Transaction t2 = tm.begin();
306 Delete d = new Delete(modrow);
307 d.addColumn(famA, colA);
308 tt.delete(t2, d);
309
310 Transaction tscan = tm.begin();
311 ResultScanner rs = tt.getScanner(tscan, new Scan());
312
313 Map<FamCol, Integer> count = countColsInRows(rs, famColA, famColB);
314 assertEquals((int) count.get(famColA), rowsWritten, "ColA count should be equal to rowsWritten");
315 assertEquals((int) count.get(famColB), rowsWritten, "ColB count should be equal to rowsWritten");
316
317 if (getClient(context).isLowLatency()) {
318 return;
319 }
320
321 tm.commit(t2);
322
323 tscan = tm.begin();
324 rs = tt.getScanner(tscan, new Scan());
325
326 count = countColsInRows(rs, famColA, famColB);
327 assertEquals((int) count.get(famColA), (rowsWritten - 1), "ColA count should be equal to rowsWritten - 1");
328 assertEquals((int) count.get(famColB), rowsWritten, "ColB count should be equal to rowsWritten");
329
330 }
331
332
333
334
335 @Test(timeOut = 10_000)
336 public void runTestDeleteColumns(ITestContext context) throws Exception {
337
338 TransactionManager tm = newTransactionManager(context);
339 TTable tt = new TTable(connection, TEST_TABLE);
340
341 Transaction t1 = tm.begin();
342 LOG.info("Transaction created " + t1);
343
344 int rowsWritten = 10;
345
346 FamCol famColA = new FamCol(famA, colA);
347 FamCol famColB = new FamCol(famA, colB);
348 writeRows(tt, t1, rowsWritten, famColA, famColB);
349 tm.commit(t1);
350
351 Transaction t2 = tm.begin();
352 Delete d = new Delete(modrow);
353 d.addColumns(famA, colA);
354 tt.delete(t2, d);
355
356 Transaction tscan = tm.begin();
357 ResultScanner rs = tt.getScanner(tscan, new Scan());
358
359 Map<FamCol, Integer> count = countColsInRows(rs, famColA, famColB);
360 assertEquals((int) count.get(famColA), rowsWritten, "ColA count should be equal to rowsWritten");
361 assertEquals((int) count.get(famColB), rowsWritten, "ColB count should be equal to rowsWritten");
362 if (getClient(context).isLowLatency()) {
363 return;
364 }
365 tm.commit(t2);
366
367 tscan = tm.begin();
368 rs = tt.getScanner(tscan, new Scan());
369
370 count = countColsInRows(rs, famColA, famColB);
371
372 assertEquals((int) count.get(famColA), (rowsWritten - 1), "ColA count should be equal to rowsWritten - 1");
373 assertEquals((int) count.get(famColB), rowsWritten, "ColB count should be equal to rowsWritten");
374
375 }
376
377 @Test(timeOut = 10_000)
378 public void runTestDeleteRow(ITestContext context) throws Exception {
379 TransactionManager tm = newTransactionManager(context);
380 TTable tt = new TTable(connection, TEST_TABLE);
381
382 Transaction t1 = tm.begin();
383 LOG.info("Transaction created " + t1);
384
385 int rowsWritten = 10;
386
387 FamCol famColA = new FamCol(famA, colA);
388 writeRows(tt, t1, rowsWritten, famColA);
389
390 tm.commit(t1);
391
392 Transaction t2 = tm.begin();
393 Delete d = new Delete(modrow);
394 tt.delete(t2, d);
395
396 Transaction tscan = tm.begin();
397 ResultScanner rs = tt.getScanner(tscan, new Scan());
398
399 int rowsRead = countRows(rs);
400 assertTrue(rowsRead == rowsWritten, "Expected " + rowsWritten + " rows but " + rowsRead + " found");
401 if (getClient(context).isLowLatency()) {
402 return;
403 }
404 tm.commit(t2);
405
406 tscan = tm.begin();
407 rs = tt.getScanner(tscan, new Scan());
408
409 rowsRead = countRows(rs);
410 assertTrue(rowsRead == (rowsWritten - 1), "Expected " + (rowsWritten - 1) + " rows but " + rowsRead + " found");
411
412 }
413
414 @Test(timeOut = 10_000)
415 public void testDeletionOfNonExistingColumnFamilyDoesNotWriteToHBase(ITestContext context) throws Exception {
416
417 if (getClient(context).isLowLatency())
418 return;
419
420
421
422 TransactionManager tm = newTransactionManager(context);
423 TTable txTable = new TTable(connection, TEST_TABLE);
424
425 Transaction tx1 = tm.begin();
426 LOG.info("{} writing initial data created ", tx1);
427 Put p = new Put(Bytes.toBytes("row1"));
428 p.addColumn(famA, colA, data1);
429 txTable.put(tx1, p);
430 tm.commit(tx1);
431
432
433
434
435 Transaction deleteTx = tm.begin();
436 LOG.info("{} trying to delete a non-existing family created ", deleteTx);
437 Delete del = new Delete(Bytes.toBytes("row1"));
438 del.addFamily(famB);
439
440 txTable.delete(deleteTx, del);
441
442
443
444
445 Get get = new Get(Bytes.toBytes("row1"));
446 get.setTimeStamp(deleteTx.getTransactionId());
447 Result result = txTable.getHTable().get(get);
448 assertTrue(result.isEmpty());
449
450 }
451
452 private int countRows(ResultScanner rs) throws IOException {
453 int count;
454 Result r = rs.next();
455 count = 0;
456 while (r != null) {
457 count++;
458 LOG.trace("row: " + Bytes.toString(r.getRow()) + " count: " + count);
459 r = rs.next();
460 }
461 return count;
462 }
463
464 private void writeRows(TTable tt, Transaction t1, int rowcount, FamCol... famCols) throws IOException {
465 for (int i = 0; i < rowcount; i++) {
466 byte[] row = Bytes.toBytes("test-del" + i);
467
468 Put p = new Put(row);
469 for (FamCol col : famCols) {
470 p.addColumn(col.fam, col.col, data1);
471 }
472 tt.put(t1, p);
473 }
474 }
475
476 private Map<FamCol, Integer> countColsInRows(ResultScanner rs, FamCol... famCols) throws IOException {
477 Map<FamCol, Integer> colCount = new HashMap<>();
478 Result r = rs.next();
479 while (r != null) {
480 for (FamCol col : famCols) {
481 if (r.containsColumn(col.fam, col.col)) {
482 Integer c = colCount.get(col);
483
484 if (c == null) {
485 colCount.put(col, 1);
486 } else {
487 colCount.put(col, c + 1);
488 }
489 }
490 }
491 r = rs.next();
492 }
493 return colCount;
494 }
495
496 }