diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java index aac55367adc..df95f0dd71f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java @@ -26,9 +26,12 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.db.DBDefinition; +import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.hdds.utils.db.FixedLengthStringCodec; +import org.apache.hadoop.hdds.utils.db.StringCodec; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneConsts; @@ -80,6 +83,8 @@ public class TestLDBCli { private static final String KEY_TABLE = "keyTable"; private static final String BLOCK_DATA = "block_data"; public static final String PIPELINES = "pipelines"; + public static final String DUMMY_DB = "anotherKeyTable"; + public static final String ANOTHER_KEY_TABLE_NAME = "anotherKeyTable"; private static final ObjectMapper MAPPER = new ObjectMapper(); private OzoneConfiguration conf; private DBStore dbStore; @@ -410,6 +415,40 @@ void testSchemaCommand() throws IOException { assertEquals("", stderr.toString()); } + @Test + void testCommandsWithDBDefOverride() throws IOException { + // Prepare dummy table + prepareTable(DUMMY_DB, true); + + // Prepare args for value-schema command + List completeScanArgs = new ArrayList<>(Arrays.asList( + "--db", dbStore.getDbLocation().getAbsolutePath(), + "--schema", DummyDBDefinition.class.getName(), + "value-schema", + "--column-family", ANOTHER_KEY_TABLE_NAME)); + + int exitCode = cmd.execute(completeScanArgs.toArray(new String[0])); + assertEquals(0, exitCode, stderr.toString()); + Pattern p = Pattern.compile(".*String.*String.*", Pattern.MULTILINE); + Matcher m = p.matcher(stdout.toString()); + assertTrue(m.find()); + assertEquals("", stderr.toString()); + + // Prepare args for scan command + completeScanArgs = new ArrayList<>(Arrays.asList( + "--db", dbStore.getDbLocation().getAbsolutePath(), + "--schema", DummyDBDefinition.class.getName(), + "scan", + "--column-family", ANOTHER_KEY_TABLE_NAME)); + + exitCode = cmd.execute(completeScanArgs.toArray(new String[0])); + assertEquals(0, exitCode, stderr.toString()); + p = Pattern.compile(".*random-key.*random-value.*", Pattern.MULTILINE); + m = p.matcher(stdout.toString()); + assertTrue(m.find()); + assertEquals("", stderr.toString()); + } + /** * Converts String input to a Map and compares to the given Map input. * @param expected expected result Map @@ -473,11 +512,23 @@ private void prepareTable(String tableName, boolean schemaV3) } } break; + case PIPELINES: // Empty table dbStore = DBStoreBuilder.newBuilder(conf).setName("scm.db") .setPath(tempDir.toPath()).addTable(PIPELINES).build(); break; + + case DUMMY_DB: + dbStore = DBStoreBuilder.newBuilder(new OzoneConfiguration()) + .setName("another.db") + .setPath(tempDir.toPath()) + .addTable(ANOTHER_KEY_TABLE_NAME) + .build(); + dbStore.getTable(ANOTHER_KEY_TABLE_NAME, String.class, String.class) + .put("random-key", "random-value"); + break; + default: throw new IllegalArgumentException("Unsupported table: " + tableName); } @@ -512,4 +563,25 @@ private static Map toMap(Object obj) throws IOException { return MAPPER.readValue(json, new TypeReference>() { }); } + /** + * New DBDefinition to test arbitrary schemas for ldb commands. + */ + public static class DummyDBDefinition extends DBDefinition.WithMap { + public static final DBColumnFamilyDefinition ANOTHER_KEY_TABLE + = new DBColumnFamilyDefinition<>(ANOTHER_KEY_TABLE_NAME, StringCodec.get(), StringCodec.get()); + private static final Map> + COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(ANOTHER_KEY_TABLE); + protected DummyDBDefinition() { + super(COLUMN_FAMILIES); + } + @Override + public String getName() { + return "another.db"; + } + @Override + public String getLocationConfigKey() { + return ""; + } + } + } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java index ca79aa41fa4..b53021fb77a 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java @@ -18,10 +18,13 @@ package org.apache.hadoop.ozone.debug; +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; @@ -39,6 +42,7 @@ import com.amazonaws.services.kms.model.InvalidArnException; import com.google.common.base.Preconditions; +import org.apache.ratis.util.function.CheckedSupplier; import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_CONTAINER_KEY_DB; @@ -98,6 +102,56 @@ public static DBDefinition getDefinition(Path dbPath, return getDefinition(dbName); } + static List> getFactories( + Class clazz, String dbPathString, ConfigurationSource config) { + return Arrays.asList( + () -> { + Constructor constructor = clazz.getDeclaredConstructor(String.class, ConfigurationSource.class); + constructor.setAccessible(true); + return (DBDefinition) constructor.newInstance(dbPathString, config); + }, + () -> { + Constructor constructor = clazz.getDeclaredConstructor(String.class); + constructor.setAccessible(true); + return (DBDefinition) constructor.newInstance(dbPathString); + }, + () -> { + Method factoryMethod = clazz.getDeclaredMethod("get"); + factoryMethod.setAccessible(true); + return (DBDefinition) factoryMethod.invoke(clazz); + }, + () -> { + Constructor constructor = clazz.getDeclaredConstructor(); + constructor.setAccessible(true); + return (DBDefinition) constructor.newInstance(); + } + ); + } + + public static DBDefinition getDefinition(Path dbPath, ConfigurationSource config, String overrideDBDef) { + if (overrideDBDef == null) { + return getDefinition(dbPath, config); + } + try { + Class clazz = Class.forName(overrideDBDef); + if (DBDefinition.class.isAssignableFrom(clazz)) { + String dbPathString = dbPath.toAbsolutePath().toString(); + for (CheckedSupplier factory : getFactories(clazz, dbPathString, config)) { + try { + return factory.get(); + } catch (Exception ignored) { + } + } + throw new IllegalArgumentException("Could not get instance of " + overrideDBDef); + } else { + System.err.println("Class does not implement DBDefinition: " + overrideDBDef); + } + } catch (ClassNotFoundException e) { + System.err.println("Class not found: " + overrideDBDef); + } + throw new IllegalArgumentException("Incorrect DBDefinition class."); + } + private static DBDefinition getReconDBDefinition(String dbName) { if (dbName.startsWith(RECON_CONTAINER_KEY_DB)) { return new ReconDBDefinition(dbName); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ldb/DBScanner.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ldb/DBScanner.java index 6fbbd1a3083..96d8b68347f 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ldb/DBScanner.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ldb/DBScanner.java @@ -581,7 +581,7 @@ private boolean printTable(List columnFamilyHandleList, dbPath = removeTrailingSlashIfNeeded(dbPath); DBDefinitionFactory.setDnDBSchemaVersion(dnDBSchemaVersion); DBDefinition dbDefinition = DBDefinitionFactory.getDefinition( - Paths.get(dbPath), new OzoneConfiguration()); + Paths.get(dbPath), new OzoneConfiguration(), parent.getDbDefinition()); if (dbDefinition == null) { err().println("Error: Incorrect DB Path"); return false; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ldb/RDBParser.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ldb/RDBParser.java index f07e8f35fba..38adc94c5a7 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ldb/RDBParser.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ldb/RDBParser.java @@ -43,6 +43,10 @@ public class RDBParser implements DebugSubcommand { description = "Database File Path") private String dbPath; + @CommandLine.Option(names = {"--schema"}, + description = "DBDefinition of the database") + private String dbDefinition; + public String getDbPath() { return dbPath; } @@ -50,4 +54,8 @@ public String getDbPath() { public void setDbPath(String dbPath) { this.dbPath = dbPath; } + + public String getDbDefinition() { + return dbDefinition; + } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ldb/ValueSchema.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ldb/ValueSchema.java index 4b8eb3b3208..d5f6ffd33fe 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ldb/ValueSchema.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ldb/ValueSchema.java @@ -86,7 +86,7 @@ public Void call() throws Exception { String dbPath = parent.getDbPath(); Map fields = new HashMap<>(); - success = getValueFields(dbPath, fields, depth, tableName, dnDBSchemaVersion); + success = getValueFields(dbPath, fields, depth, tableName, dnDBSchemaVersion, parent.getDbDefinition()); out().println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(fields)); @@ -100,11 +100,11 @@ public Void call() throws Exception { } public static boolean getValueFields(String dbPath, Map valueSchema, int d, String table, - String dnDBSchemaVersion) { + String dnDBSchemaVersion, String dbDef) { dbPath = removeTrailingSlashIfNeeded(dbPath); DBDefinitionFactory.setDnDBSchemaVersion(dnDBSchemaVersion); - DBDefinition dbDefinition = DBDefinitionFactory.getDefinition(Paths.get(dbPath), new OzoneConfiguration()); + DBDefinition dbDefinition = DBDefinitionFactory.getDefinition(Paths.get(dbPath), new OzoneConfiguration(), dbDef); if (dbDefinition == null) { err().println("Error: Incorrect DB Path"); return false; diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestDBDefinitionFactory.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestDBDefinitionFactory.java index 5f0be7859d4..dbde2c79feb 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestDBDefinitionFactory.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestDBDefinitionFactory.java @@ -21,9 +21,14 @@ import java.nio.file.Path; import java.nio.file.Paths; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; import org.apache.hadoop.hdds.utils.db.DBDefinition; +import org.apache.hadoop.ozone.container.metadata.AbstractDatanodeDBDefinition; import org.apache.hadoop.ozone.container.metadata.DatanodeSchemaOneDBDefinition; import org.apache.hadoop.ozone.container.metadata.DatanodeSchemaThreeDBDefinition; import org.apache.hadoop.ozone.container.metadata.DatanodeSchemaTwoDBDefinition; @@ -34,8 +39,13 @@ import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_CONTAINER_KEY_DB; import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_OM_SNAPSHOT_DB; import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import org.apache.ratis.util.function.CheckedSupplier; import org.junit.jupiter.api.Test; +import org.reflections.Reflections; /** * Simple factory unit test. @@ -75,4 +85,47 @@ public void testGetDefinition() { definition = DBDefinitionFactory.getDefinition(dbPath, conf); assertInstanceOf(DatanodeSchemaThreeDBDefinition.class, definition); } + + @Test + public void testGetDefinitionWithOverride() { + final OzoneConfiguration conf = new OzoneConfiguration(); + Path dbPath = Paths.get("another.db"); + DBDefinition definition = DBDefinitionFactory.getDefinition(dbPath, conf, OMDBDefinition.class.getName()); + assertInstanceOf(OMDBDefinition.class, definition); + } + + /* + * Test to ensure that any DBDefinition has a default constructor or a constructor with 1 parameter. + * This is needed for ldb tools to run with arbitrary DB definitions. + */ + @Test + public void testAllDBDefinitionsHaveCorrectConstructor() { + Set> subclasses = new HashSet<>(); + try { + Reflections reflections = new Reflections("org.apache"); + subclasses.addAll(reflections.getSubTypesOf(DBDefinition.class)); + subclasses.remove(DBDefinition.WithMap.class); + subclasses.remove(DBDefinition.WithMapInterface.class); + subclasses.remove(AbstractDatanodeDBDefinition.class); + } catch (Exception e) { + fail("Error while finding subclasses: " + e.getMessage()); + } + assertFalse(subclasses.isEmpty(), "No classes found extending DBDefinition"); + + for (Class clazz : subclasses) { + List> factories = + DBDefinitionFactory.getFactories(clazz, "testDbPath", new OzoneConfiguration()); + boolean hasValidFactory = factories.stream().anyMatch(factory -> { + try { + factory.get(); + return true; + } catch (Exception e) { + return false; + } + }); + + assertTrue(hasValidFactory, + "Class " + clazz.getName() + " does not have a valid constructor or factory method."); + } + } }