diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java index aac55367adc..1871c46b1a6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java @@ -26,9 +26,12 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.db.DBDefinition; +import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.hdds.utils.db.FixedLengthStringCodec; +import org.apache.hadoop.hdds.utils.db.StringCodec; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneConsts; @@ -80,6 +83,7 @@ public class TestLDBCli { private static final String KEY_TABLE = "keyTable"; private static final String BLOCK_DATA = "block_data"; public static final String PIPELINES = "pipelines"; + public static final String DUMMY_DB = "anotherKeyTable"; private static final ObjectMapper MAPPER = new ObjectMapper(); private OzoneConfiguration conf; private DBStore dbStore; @@ -410,6 +414,40 @@ void testSchemaCommand() throws IOException { assertEquals("", stderr.toString()); } + @Test + void testCommandsWithDBDefOverride() throws IOException { + // Prepare dummy table + prepareTable(DUMMY_DB, true); + + // Prepare args for value-schema command + List completeScanArgs = new ArrayList<>(Arrays.asList( + "--db", dbStore.getDbLocation().getAbsolutePath(), + "--schema", DummyDBDefinition.class.getName(), + "value-schema", + "--column-family", DummyDBDefinition.ANOTHER_KEY_TABLE_NAME)); + + int exitCode = cmd.execute(completeScanArgs.toArray(new String[0])); + assertEquals(0, exitCode, stderr.toString()); + Pattern p = Pattern.compile(".*String.*String.*", Pattern.MULTILINE); + Matcher m = p.matcher(stdout.toString()); + assertTrue(m.find()); + assertEquals("", stderr.toString()); + + // Prepare args for scan command + completeScanArgs = new ArrayList<>(Arrays.asList( + "--db", dbStore.getDbLocation().getAbsolutePath(), + "--schema", DummyDBDefinition.class.getName(), + "scan", + "--column-family", DummyDBDefinition.ANOTHER_KEY_TABLE_NAME)); + + exitCode = cmd.execute(completeScanArgs.toArray(new String[0])); + assertEquals(0, exitCode, stderr.toString()); + p = Pattern.compile(".*random-key.*random-value.*", Pattern.MULTILINE); + m = p.matcher(stdout.toString()); + assertTrue(m.find()); + assertEquals("", stderr.toString()); + } + /** * Converts String input to a Map and compares to the given Map input. * @param expected expected result Map @@ -473,11 +511,23 @@ private void prepareTable(String tableName, boolean schemaV3) } } break; + case PIPELINES: // Empty table dbStore = DBStoreBuilder.newBuilder(conf).setName("scm.db") .setPath(tempDir.toPath()).addTable(PIPELINES).build(); break; + + case DUMMY_DB: + dbStore = DBStoreBuilder.newBuilder(new OzoneConfiguration()) + .setName("another.db") + .setPath(tempDir.toPath()) + .addTable(DummyDBDefinition.ANOTHER_KEY_TABLE_NAME) + .build(); + dbStore.getTable(DummyDBDefinition.ANOTHER_KEY_TABLE_NAME, String.class, String.class) + .put("random-key", "random-value"); + break; + default: throw new IllegalArgumentException("Unsupported table: " + tableName); } @@ -512,4 +562,23 @@ private static Map toMap(Object obj) throws IOException { return MAPPER.readValue(json, new TypeReference>() { }); } + public static class DummyDBDefinition extends DBDefinition.WithMap { + public static String ANOTHER_KEY_TABLE_NAME = "anotherKeyTable"; + public static final DBColumnFamilyDefinition ANOTHER_KEY_TABLE + = new DBColumnFamilyDefinition<>(ANOTHER_KEY_TABLE_NAME, StringCodec.get(), StringCodec.get()); + private static final Map> + COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(ANOTHER_KEY_TABLE); + protected DummyDBDefinition() { + super(COLUMN_FAMILIES); + } + @Override + public String getName() { + return "another.db"; + } + @Override + public String getLocationConfigKey() { + return ""; + } + } + } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java index 7eb70b3279f..adcedd78800 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java @@ -99,9 +99,9 @@ public static DBDefinition getDefinition(Path dbPath, return getDefinition(dbName); } - public static DBDefinition getDefinition(Path dbName, ConfigurationSource config, String overrideDBDef) { + public static DBDefinition getDefinition(Path dbPath, ConfigurationSource config, String overrideDBDef) { if (overrideDBDef == null) { - return getDefinition(dbName, config); + return getDefinition(dbPath, config); } try { Class clazz = Class.forName(overrideDBDef); @@ -114,7 +114,7 @@ public static DBDefinition getDefinition(Path dbName, ConfigurationSource config } catch (NoSuchMethodException e) { Constructor stringParamConstructor = clazz.getDeclaredConstructor(String.class); stringParamConstructor.setAccessible(true); - instance = (DBDefinition) stringParamConstructor.newInstance(dbName.toAbsolutePath().toString()); + instance = (DBDefinition) stringParamConstructor.newInstance(dbPath.toAbsolutePath().toString()); } return instance; } else { diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestDBDefinitionFactory.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestDBDefinitionFactory.java index 5f0be7859d4..617b222e2a0 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestDBDefinitionFactory.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestDBDefinitionFactory.java @@ -18,9 +18,13 @@ package org.apache.hadoop.ozone.debug; +import java.lang.reflect.Constructor; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.HashSet; +import java.util.Set; + import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; import org.apache.hadoop.hdds.utils.db.DBDefinition; @@ -34,8 +38,12 @@ import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_CONTAINER_KEY_DB; import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_OM_SNAPSHOT_DB; import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.Test; +import org.reflections.Reflections; /** * Simple factory unit test. @@ -75,4 +83,46 @@ public void testGetDefinition() { definition = DBDefinitionFactory.getDefinition(dbPath, conf); assertInstanceOf(DatanodeSchemaThreeDBDefinition.class, definition); } + + @Test + public void testGetDefinitionWithOverride() { + final OzoneConfiguration conf = new OzoneConfiguration(); + Path dbPath = Paths.get("another.db"); + DBDefinition definition = DBDefinitionFactory.getDefinition(dbPath, conf, OMDBDefinition.class.getName()); + assertInstanceOf(OMDBDefinition.class, definition); + } + + /* + * Test to ensure that any DBDefinition has a default constructor or a constructor with 1 parameter. + * This is needed for ldb tools to run with arbitrary DB definitions. + */ + @Test + public void testAllDBDefinitionsHaveCorrectConstructor() { + Set> withMapSubclasses = new HashSet<>(); + try { + Reflections reflections = new Reflections("org.apache.hadoop"); + withMapSubclasses = reflections.getSubTypesOf(DBDefinition.WithMap.class); + } catch (Exception e) { + fail("Error while finding subclasses: " + e.getMessage()); + } + assertFalse(withMapSubclasses.isEmpty(), "No classes found extending DBDefinition.WithMap"); + System.out.println(withMapSubclasses); + + // Check constructors for each subclass + for (Class clazz : withMapSubclasses) { + Constructor[] constructors = clazz.getDeclaredConstructors(); + boolean hasValidConstructor = false; + + for (Constructor constructor : constructors) { + int paramCount = constructor.getParameterCount(); + if (paramCount == 0 || paramCount == 1) { + hasValidConstructor = true; + break; + } + } + + assertTrue(hasValidConstructor, + "Class " + clazz.getName() + " does not have a valid constructor (default or single parameter)"); + } + } }