Skip to content

Commit

Permalink
Add tests to check arbitrary schema option
Browse files Browse the repository at this point in the history
  • Loading branch information
tejaskriya committed Jan 6, 2025
1 parent a890c9c commit 8e43cec
Show file tree
Hide file tree
Showing 3 changed files with 122 additions and 3 deletions.
69 changes: 69 additions & 0 deletions hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,12 @@
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.utils.db.DBDefinition;
import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
import org.apache.hadoop.hdds.utils.db.FixedLengthStringCodec;
import org.apache.hadoop.hdds.utils.db.StringCodec;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.ClientVersion;
import org.apache.hadoop.ozone.OzoneConsts;
Expand Down Expand Up @@ -80,6 +83,7 @@ public class TestLDBCli {
private static final String KEY_TABLE = "keyTable";
private static final String BLOCK_DATA = "block_data";
public static final String PIPELINES = "pipelines";
public static final String DUMMY_DB = "anotherKeyTable";
private static final ObjectMapper MAPPER = new ObjectMapper();
private OzoneConfiguration conf;
private DBStore dbStore;
Expand Down Expand Up @@ -410,6 +414,40 @@ void testSchemaCommand() throws IOException {
assertEquals("", stderr.toString());
}

@Test
void testCommandsWithDBDefOverride() throws IOException {
// Prepare dummy table
prepareTable(DUMMY_DB, true);

// Prepare args for value-schema command
List<String> completeScanArgs = new ArrayList<>(Arrays.asList(
"--db", dbStore.getDbLocation().getAbsolutePath(),
"--schema", DummyDBDefinition.class.getName(),
"value-schema",
"--column-family", DummyDBDefinition.ANOTHER_KEY_TABLE_NAME));

int exitCode = cmd.execute(completeScanArgs.toArray(new String[0]));
assertEquals(0, exitCode, stderr.toString());
Pattern p = Pattern.compile(".*String.*String.*", Pattern.MULTILINE);
Matcher m = p.matcher(stdout.toString());
assertTrue(m.find());
assertEquals("", stderr.toString());

// Prepare args for scan command
completeScanArgs = new ArrayList<>(Arrays.asList(
"--db", dbStore.getDbLocation().getAbsolutePath(),
"--schema", DummyDBDefinition.class.getName(),
"scan",
"--column-family", DummyDBDefinition.ANOTHER_KEY_TABLE_NAME));

exitCode = cmd.execute(completeScanArgs.toArray(new String[0]));
assertEquals(0, exitCode, stderr.toString());
p = Pattern.compile(".*random-key.*random-value.*", Pattern.MULTILINE);
m = p.matcher(stdout.toString());
assertTrue(m.find());
assertEquals("", stderr.toString());
}

/**
* Converts String input to a Map and compares to the given Map input.
* @param expected expected result Map
Expand Down Expand Up @@ -473,11 +511,23 @@ private void prepareTable(String tableName, boolean schemaV3)
}
}
break;

case PIPELINES:
// Empty table
dbStore = DBStoreBuilder.newBuilder(conf).setName("scm.db")
.setPath(tempDir.toPath()).addTable(PIPELINES).build();
break;

case DUMMY_DB:
dbStore = DBStoreBuilder.newBuilder(new OzoneConfiguration())
.setName("another.db")
.setPath(tempDir.toPath())
.addTable(DummyDBDefinition.ANOTHER_KEY_TABLE_NAME)
.build();
dbStore.getTable(DummyDBDefinition.ANOTHER_KEY_TABLE_NAME, String.class, String.class)
.put("random-key", "random-value");
break;

default:
throw new IllegalArgumentException("Unsupported table: " + tableName);
}
Expand Down Expand Up @@ -512,4 +562,23 @@ private static Map<String, Object> toMap(Object obj) throws IOException {
return MAPPER.readValue(json, new TypeReference<Map<String, Object>>() { });
}

public static class DummyDBDefinition extends DBDefinition.WithMap {
public static String ANOTHER_KEY_TABLE_NAME = "anotherKeyTable";
public static final DBColumnFamilyDefinition<String, String> ANOTHER_KEY_TABLE
= new DBColumnFamilyDefinition<>(ANOTHER_KEY_TABLE_NAME, StringCodec.get(), StringCodec.get());
private static final Map<String, DBColumnFamilyDefinition<?, ?>>
COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(ANOTHER_KEY_TABLE);
protected DummyDBDefinition() {
super(COLUMN_FAMILIES);
}
@Override
public String getName() {
return "another.db";
}
@Override
public String getLocationConfigKey() {
return "";
}
}

}
6 changes: 3 additions & 3 deletions hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,9 +99,9 @@ public static DBDefinition getDefinition(Path dbPath,
return getDefinition(dbName);
}

public static DBDefinition getDefinition(Path dbName, ConfigurationSource config, String overrideDBDef) {
public static DBDefinition getDefinition(Path dbPath, ConfigurationSource config, String overrideDBDef) {
if (overrideDBDef == null) {
return getDefinition(dbName, config);
return getDefinition(dbPath, config);
}
try {
Class<?> clazz = Class.forName(overrideDBDef);
Expand All @@ -114,7 +114,7 @@ public static DBDefinition getDefinition(Path dbName, ConfigurationSource config
} catch (NoSuchMethodException e) {
Constructor<?> stringParamConstructor = clazz.getDeclaredConstructor(String.class);
stringParamConstructor.setAccessible(true);
instance = (DBDefinition) stringParamConstructor.newInstance(dbName.toAbsolutePath().toString());
instance = (DBDefinition) stringParamConstructor.newInstance(dbPath.toAbsolutePath().toString());
}
return instance;
} else {
Expand Down
50 changes: 50 additions & 0 deletions hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestDBDefinitionFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,13 @@

package org.apache.hadoop.ozone.debug;

import java.lang.reflect.Constructor;
import java.nio.file.Path;
import java.nio.file.Paths;

import java.util.HashSet;
import java.util.Set;

import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
import org.apache.hadoop.hdds.utils.db.DBDefinition;
Expand All @@ -34,8 +38,12 @@
import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_CONTAINER_KEY_DB;
import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_OM_SNAPSHOT_DB;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.fail;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

import org.junit.jupiter.api.Test;
import org.reflections.Reflections;

/**
* Simple factory unit test.
Expand Down Expand Up @@ -75,4 +83,46 @@ public void testGetDefinition() {
definition = DBDefinitionFactory.getDefinition(dbPath, conf);
assertInstanceOf(DatanodeSchemaThreeDBDefinition.class, definition);
}

@Test
public void testGetDefinitionWithOverride() {
final OzoneConfiguration conf = new OzoneConfiguration();
Path dbPath = Paths.get("another.db");
DBDefinition definition = DBDefinitionFactory.getDefinition(dbPath, conf, OMDBDefinition.class.getName());
assertInstanceOf(OMDBDefinition.class, definition);
}

/*
* Test to ensure that any DBDefinition has a default constructor or a constructor with 1 parameter.
* This is needed for ldb tools to run with arbitrary DB definitions.
*/
@Test
public void testAllDBDefinitionsHaveCorrectConstructor() {
Set<Class<? extends DBDefinition.WithMap>> withMapSubclasses = new HashSet<>();
try {
Reflections reflections = new Reflections("org.apache.hadoop");
withMapSubclasses = reflections.getSubTypesOf(DBDefinition.WithMap.class);
} catch (Exception e) {
fail("Error while finding subclasses: " + e.getMessage());
}
assertFalse(withMapSubclasses.isEmpty(), "No classes found extending DBDefinition.WithMap");
System.out.println(withMapSubclasses);

// Check constructors for each subclass
for (Class<?> clazz : withMapSubclasses) {
Constructor<?>[] constructors = clazz.getDeclaredConstructors();
boolean hasValidConstructor = false;

for (Constructor<?> constructor : constructors) {
int paramCount = constructor.getParameterCount();
if (paramCount == 0 || paramCount == 1) {
hasValidConstructor = true;
break;
}
}

assertTrue(hasValidConstructor,
"Class " + clazz.getName() + " does not have a valid constructor (default or single parameter)");
}
}
}

0 comments on commit 8e43cec

Please sign in to comment.