1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.omid.tools.hbase;
19
20 import java.io.IOException;
21
22 import org.apache.hadoop.conf.Configuration;
23 import org.apache.hadoop.hbase.HBaseConfiguration;
24 import org.apache.hadoop.hbase.HColumnDescriptor;
25 import org.apache.hadoop.hbase.HTableDescriptor;
26 import org.apache.hadoop.hbase.TableName;
27 import org.apache.hadoop.hbase.client.Admin;
28 import org.apache.hadoop.hbase.client.Connection;
29 import org.apache.hadoop.hbase.client.ConnectionFactory;
30 import org.apache.hadoop.hbase.util.Bytes;
31 import org.apache.omid.HBaseShims;
32 import org.apache.omid.committable.hbase.HBaseCommitTableConfig;
33 import org.apache.omid.committable.hbase.KeyGenerator;
34 import org.apache.omid.committable.hbase.KeyGeneratorImplementations;
35 import org.apache.omid.committable.hbase.RegionSplitter;
36 import org.apache.omid.timestamp.storage.HBaseTimestampStorageConfig;
37 import org.slf4j.Logger;
38 import org.slf4j.LoggerFactory;
39
40 import com.beust.jcommander.IParameterValidator;
41 import com.beust.jcommander.JCommander;
42 import com.beust.jcommander.Parameter;
43 import com.beust.jcommander.ParameterException;
44 import com.beust.jcommander.Parameters;
45 import com.beust.jcommander.ParametersDelegate;
46
47
48
49
50 public class OmidTableManager {
51
52 private static final Logger LOG = LoggerFactory.getLogger(OmidTableManager.class);
53
54 public static final String COMMIT_TABLE_COMMAND_NAME = "commit-table";
55 static final String TIMESTAMP_TABLE_COMMAND_NAME = "timestamp-table";
56
57 private static final byte[][] commitTableFamilies = new byte[][]{
58 Bytes.toBytes(HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_CF_NAME),
59 Bytes.toBytes(HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_LWM_CF_NAME)};
60 private static final byte[][] timestampTableFamilies = new byte[][]{
61 Bytes.toBytes(HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_CF_NAME)};
62
63 private JCommander commandLine;
64 private MainConfig mainConfig = new MainConfig();
65 private CommitTableCommand commitTableCommand = new CommitTableCommand();
66 private TimestampTableCommand timestampTableCommand = new TimestampTableCommand();
67
68 public OmidTableManager(String... args) {
69 commandLine = new JCommander(mainConfig);
70 commandLine.addCommand(COMMIT_TABLE_COMMAND_NAME, commitTableCommand);
71 commandLine.addCommand(TIMESTAMP_TABLE_COMMAND_NAME, timestampTableCommand);
72 try {
73 commandLine.parse(args);
74 } catch (ParameterException ex) {
75 commandLine.usage();
76 throw new IllegalArgumentException(ex.getMessage());
77 }
78 }
79
80 public void executeActionsOnHBase(Configuration hbaseConf) throws IOException {
81
82 HBaseLogin.loginIfNeeded(mainConfig.loginFlags);
83
84 try (Connection conn = ConnectionFactory.createConnection(hbaseConf);
85 Admin hBaseAdmin = conn.getAdmin()) {
86 byte[][] tableFamilies;
87 byte[][] splitKeys = new byte[0][0];
88 String tableName;
89
90 LOG.info("----------------------------------------------------------------------------------------------");
91 switch (commandLine.getParsedCommand()) {
92 case COMMIT_TABLE_COMMAND_NAME:
93 LOG.info("Performing actions related to COMMIT TABLE");
94 tableName = commitTableCommand.tableName;
95 tableFamilies = commitTableFamilies;
96 if (commitTableCommand.numRegions > 1) {
97 splitKeys = splitInUniformRegions(hbaseConf, commitTableCommand.numRegions);
98 }
99 break;
100 case TIMESTAMP_TABLE_COMMAND_NAME:
101 LOG.info("Performing actions related to TIMESTAMP TABLE");
102 tableName = timestampTableCommand.tableName;
103 tableFamilies = timestampTableFamilies;
104 break;
105 default:
106 LOG.error("Unknown command: {}", commandLine.getParsedCommand());
107 commandLine.usage();
108 return;
109 }
110
111 createTable(hBaseAdmin, tableName, tableFamilies, splitKeys, 1);
112 LOG.info("----------------------------------------------------------------------------------------------");
113
114 }
115 }
116
117 public static void main(String... args) throws Exception {
118
119 OmidTableManager tableManager = new OmidTableManager(args);
120 tableManager.executeActionsOnHBase(HBaseConfiguration.create());
121
122 }
123
124
125
126
127
128 private static byte[][] splitInUniformRegions(Configuration hBaseConf, int numRegions) throws IOException {
129
130 KeyGenerator keyGen = KeyGeneratorImplementations.defaultKeyGenerator();
131 RegionSplitter.SplitAlgorithm algo =
132 RegionSplitter.newSplitAlgoInstance(hBaseConf, RegionSplitter.UniformSplit.class.getName());
133 algo.setFirstRow(algo.rowToStr(keyGen.startTimestampToKey(0)));
134 algo.setLastRow(algo.rowToStr(keyGen.startTimestampToKey(Long.MAX_VALUE)));
135
136
137 return algo.split(numRegions);
138
139 }
140
141 private static void createTable(Admin admin, String tableName, byte[][] families, byte[][] splitKeys,
142 int maxVersions)
143 throws IOException {
144
145 LOG.info("About to create Table named {} with {} splits", tableName, splitKeys.length);
146
147 TableName hTableName = TableName.valueOf(tableName);
148 if (admin.tableExists(hTableName)) {
149 LOG.error("Table {} already exists. Table creation cancelled", tableName);
150 return;
151 }
152
153 HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(tableName));
154
155 for (byte[] family : families) {
156 HColumnDescriptor colDescriptor = new HColumnDescriptor(family);
157 colDescriptor.setMaxVersions(maxVersions);
158 HBaseShims.addFamilyToHTableDescriptor(tableDescriptor, colDescriptor);
159 LOG.info("\tAdding Family {}", colDescriptor);
160 }
161
162 admin.createTable(tableDescriptor, splitKeys);
163
164 LOG.info("Table {} created. Regions: {}", tableName, admin.getTableRegions(hTableName).size());
165
166 }
167
168
169
170 static class MainConfig {
171
172 @ParametersDelegate
173 SecureHBaseConfig loginFlags = new SecureHBaseConfig();
174
175 }
176
177 @Parameters(commandDescription = "Specifies configuration for the Commit Table")
178 static class CommitTableCommand {
179
180 @Parameter(names = "-tableName", description = "Table name where to stores the commits", required = false)
181 String tableName = HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_NAME;
182
183 @Parameter(names = "-numRegions", description = "Number of splits (to pre-split tableName)", required = false,
184 validateWith = IntegerGreaterThanZero.class)
185 int numRegions = 16;
186
187 }
188
189 @Parameters(commandDescription = "Specifies configuration for the Timestamp Table")
190 static class TimestampTableCommand {
191
192 @Parameter(names = "-tableName", description = "Table name where to store timestamps")
193 String tableName = HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME;
194
195 }
196
197 public static class IntegerGreaterThanZero implements IParameterValidator {
198
199 @Override
200 public void validate(String name, String value) throws ParameterException {
201 int n = Integer.parseInt(value);
202 if (n <= 0) {
203 throw new ParameterException("Parameter " + name + " should be > 0 (found " + value + ")");
204 }
205 }
206
207 }
208
209 }