Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(interactive): Convert Schema from Groot Spec to Flex Spec #4447

Merged
merged 4 commits into from
Jan 24, 2025
Merged

fix(interactive): Convert Schema from Groot Spec to Flex Spec #4447

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2026317
convert groot schema to flex spec
shirly121 Jan 24, 2025
601926b
add tests
shirly121 Jan 24, 2025
c3ac2c6
format java codes
shirly121 Jan 24, 2025
e03e196
Merge remote-tracking branch 'origin/main' into ir_groot_schema
shirly121 Jan 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions ...on/src/main/java/com/alibaba/graphscope/groot/common/schema/impl/DefaultEdgeRelation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import com.alibaba.graphscope.groot.common.schema.api.EdgeRelation;
import com.alibaba.graphscope.groot.common.schema.api.GraphVertex;
import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;

public class DefaultEdgeRelation implements EdgeRelation {
private final GraphVertex source;
Expand Down Expand Up @@ -57,4 +58,19 @@ public String toString() {
public long getTableId() {
return this.tableId;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
DefaultEdgeRelation that = (DefaultEdgeRelation) o;
return tableId == that.tableId
&& Objects.equal(source, that.source)
&& Objects.equal(target, that.target);
}

@Override
public int hashCode() {
return Objects.hashCode(source, target, tableId);
}
}
17 changes: 17 additions & 0 deletions ...n/src/main/java/com/alibaba/graphscope/groot/common/schema/impl/DefaultGraphProperty.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import com.alibaba.graphscope.groot.common.schema.api.GraphProperty;
import com.alibaba.graphscope.groot.common.schema.wrapper.DataType;
import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;

public class DefaultGraphProperty implements GraphProperty {
private final int id;
Expand Down Expand Up @@ -71,4 +72,20 @@ public String toString() {
.add("dataType", dataType)
.toString();
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
DefaultGraphProperty that = (DefaultGraphProperty) o;
return id == that.id
&& Objects.equal(name, that.name)
&& dataType == that.dataType
&& Objects.equal(defaultValue, that.defaultValue);
}

@Override
public int hashCode() {
return Objects.hashCode(id, name, dataType, defaultValue);
}
}
3 changes: 2 additions & 1 deletion ...ne/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GraphOptTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,8 @@ private RelDataType deriveType(GraphProperty property) {
}
RelDataTypeFactory typeFactory = this.schema.getTypeFactory();
requireNonNull(typeFactory, "typeFactory");
IrDataTypeConvertor<DataType> typeConvertor = new IrDataTypeConvertor.Groot(typeFactory);
IrDataTypeConvertor<DataType> typeConvertor =
new IrDataTypeConvertor.Groot(typeFactory, true);
return typeConvertor.convert(property.getDataType());
}

Expand Down
22 changes: 17 additions & 5 deletions ...piler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,143 +51,155 @@

class Groot implements IrDataTypeConvertor<DataType> {
private final RelDataTypeFactory typeFactory;
private final boolean throwsOnFail;

public Groot(RelDataTypeFactory typeFactory) {
public Groot(RelDataTypeFactory typeFactory, boolean throwsOnFail) {
this.typeFactory = typeFactory;
this.throwsOnFail = throwsOnFail;
}

@Override
public RelDataType convert(DataType from) {
requireNonNull(typeFactory, "typeFactory should not be null");
switch (from) {
case BOOL:
return typeFactory.createSqlType(SqlTypeName.BOOLEAN);
case CHAR:
// single character
return typeFactory.createSqlType(SqlTypeName.CHAR, 1);
case STRING:
// string with unlimited length
return typeFactory.createSqlType(
SqlTypeName.VARCHAR, RelDataType.PRECISION_NOT_SPECIFIED);
case SHORT:
// 2-bytes signed integer
return typeFactory.createSqlType(SqlTypeName.SMALLINT);
case INT:
// 4-bytes signed integer
return typeFactory.createSqlType(SqlTypeName.INTEGER);
case LONG:
// 8-bytes signed integer
return typeFactory.createSqlType(SqlTypeName.BIGINT);
case FLOAT:
// single precision floating point, 4 bytes
return typeFactory.createSqlType(SqlTypeName.FLOAT);
case DOUBLE:
// double precision floating point, 8 bytes
return typeFactory.createSqlType(SqlTypeName.DOUBLE);
case DATE:
// int32 days since 1970-01-01
return typeFactory.createSqlType(SqlTypeName.DATE);
case TIME32:
// int32 milliseconds past midnight
return typeFactory.createSqlType(SqlTypeName.TIME);
case TIMESTAMP:
// int64 milliseconds since 1970-01-01 00:00:00.000000
return typeFactory.createSqlType(SqlTypeName.TIMESTAMP);
case INT_LIST:
// array of 4-bytes signed integer, unlimited size
return typeFactory.createArrayType(
convert(DataType.INT), RelDataType.PRECISION_NOT_SPECIFIED);
case LONG_LIST:
// array of 8-bytes signed integer, unlimited size
return typeFactory.createArrayType(
convert(DataType.LONG), RelDataType.PRECISION_NOT_SPECIFIED);
case FLOAT_LIST:
// array of single precision floating point, unlimited size
return typeFactory.createArrayType(
convert(DataType.FLOAT), RelDataType.PRECISION_NOT_SPECIFIED);
case DOUBLE_LIST:
// array of double precision floating point, unlimited size
return typeFactory.createArrayType(
convert(DataType.DOUBLE), RelDataType.PRECISION_NOT_SPECIFIED);
case STRING_LIST:
// array of string, unlimited size
return typeFactory.createArrayType(
convert(DataType.STRING), RelDataType.PRECISION_NOT_SPECIFIED);
case UNKNOWN:
return typeFactory.createSqlType(SqlTypeName.UNKNOWN);
case BYTES:
case BYTES_LIST:
default:
throw new UnsupportedOperationException(
"convert GrootDataType ["
+ from.name()
+ "] to RelDataType is unsupported yet");
if (throwsOnFail) {
throw new UnsupportedOperationException(
"convert GrootDataType ["
+ from.name()
+ "] to RelDataType is unsupported yet");
}
return typeFactory.createSqlType(SqlTypeName.ANY);
}
}

Check notice on line 132 in interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java

View check run for this annotation

codefactor.io / CodeFactor

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java#L61-L132 

Complex Method
@Override
public DataType convert(RelDataType dataFrom) {
SqlTypeName typeName = dataFrom.getSqlTypeName();
switch (typeName) {
case BOOLEAN:
return DataType.BOOL;
case CHAR:
if (dataFrom.getPrecision() == 1) {
return DataType.CHAR;
}
break;
case VARCHAR:
if (dataFrom.getPrecision() == RelDataType.PRECISION_NOT_SPECIFIED) {
return DataType.STRING;
}
break;
case SMALLINT:
return DataType.SHORT;
case INTEGER:
return DataType.INT;
case BIGINT:
return DataType.LONG;
case FLOAT:
return DataType.FLOAT;
case DOUBLE:
return DataType.DOUBLE;
case DATE:
return DataType.DATE;
case TIME:
return DataType.TIME32;
case TIMESTAMP:
return DataType.TIMESTAMP;
case MULTISET:
case ARRAY:
RelDataType componentType = dataFrom.getComponentType();
// check the array or set is an unlimited size list of primitive type
if (componentType != null
&& dataFrom.getPrecision() == RelDataType.PRECISION_NOT_SPECIFIED) {
switch (componentType.getSqlTypeName()) {
case INTEGER:
return DataType.INT_LIST;
case BIGINT:
return DataType.LONG_LIST;
case FLOAT:
return DataType.FLOAT_LIST;
case DOUBLE:
return DataType.DOUBLE_LIST;
case VARCHAR:
if (componentType.getPrecision()
== RelDataType.PRECISION_NOT_SPECIFIED) {
return DataType.STRING_LIST;
}
}
}
break;
case UNKNOWN:
return DataType.UNKNOWN;
default:
}
if (throwsOnFail) {
throw new UnsupportedOperationException(
"convert RelDataType ["
+ dataFrom
+ "] to GrootDataType is unsupported yet");
}
return DataType.UNKNOWN;
}
}

class Flex implements IrDataTypeConvertor<GSDataTypeDesc> {

Check notice on line 202 in interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java

View check run for this annotation

codefactor.io / CodeFactor

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java#L133-L202 

Complex Method
private final RelDataTypeFactory typeFactory;
private final boolean throwsOnFail;

Expand All @@ -196,252 +208,252 @@
this.throwsOnFail = throwsOnFail;
}

@Override
public RelDataType convert(GSDataTypeDesc from) {
Objects.requireNonNull(typeFactory, "typeFactory should not be null");
Map<String, Object> typeMap = from.getYamlDesc();
Object value;
if ((value = typeMap.get("primitive_type")) != null) {
switch (value.toString()) {
case "DT_NULL":
return typeFactory.createSqlType(SqlTypeName.NULL);
case "DT_ANY":
// any type
return typeFactory.createSqlType(SqlTypeName.ANY);
case "DT_SIGNED_INT32":
// 4-bytes signed integer
return typeFactory.createSqlType(SqlTypeName.INTEGER);
case "DT_SIGNED_INT64":
// 8-bytes signed integer
return typeFactory.createSqlType(SqlTypeName.BIGINT);
case "DT_BOOL":
// boolean type
return typeFactory.createSqlType(SqlTypeName.BOOLEAN);
case "DT_FLOAT":
// single precision floating point, 4 bytes
return typeFactory.createSqlType(SqlTypeName.FLOAT);
case "DT_DOUBLE":
// double precision floating point, 8 bytes
return typeFactory.createSqlType(SqlTypeName.DOUBLE);
}
} else if ((value = typeMap.get("string")) != null) {
Map<String, Object> strType = (Map<String, Object>) value;
if (strType.containsKey("long_text")) {
// string with unlimited length
return typeFactory.createSqlType(
SqlTypeName.VARCHAR, RelDataType.PRECISION_NOT_SPECIFIED);
} else if (strType.containsKey("char")) {
Object charValue = strType.get("char");
Integer fixedLen = getIntValue(charValue, "fixed_length");
if (fixedLen == null) {
fixedLen =
typeFactory.getTypeSystem().getDefaultPrecision(SqlTypeName.CHAR);
logger.warn(
"can not convert {} to a valid fixed length, use default"
+ " length {} instead",
charValue,
fixedLen);
}
// string with fixed length
return typeFactory.createSqlType(SqlTypeName.CHAR, fixedLen);
} else if (strType.containsKey("var_char")) {
Object varCharValue = strType.get("var_char");
Integer maxLen = getIntValue(varCharValue, "max_length");
if (maxLen == null) {
maxLen =
typeFactory
.getTypeSystem()
.getDefaultPrecision(SqlTypeName.VARCHAR);
logger.warn(
"can not convert {} to a valid max length, use default"
+ " length {} instead",
varCharValue,
maxLen);
}
// string with variable length, bound by max length
return typeFactory.createSqlType(SqlTypeName.VARCHAR, maxLen);
}
} else if ((value = typeMap.get("temporal")) != null) {
Map<String, Object> temporalType = (Map<String, Object>) value;
if (temporalType.containsKey("date32")) {
// int32 days since 1970-01-01
return typeFactory.createSqlType(SqlTypeName.DATE);
} else if (temporalType.containsKey("time32")) {
// int32 milliseconds past midnight
return typeFactory.createSqlType(SqlTypeName.TIME);
} else if (temporalType.containsKey("timestamp")) {
// int64 milliseconds since 1970-01-01 00:00:00.000000
return typeFactory.createSqlType(SqlTypeName.TIMESTAMP);
}
} else if ((value = typeMap.get("array")) != null) {
Map<String, Object> arrayType = (Map<String, Object>) value;
Map<String, Object> componentType =
(Map<String, Object>) arrayType.get("component_type");
Preconditions.checkArgument(
componentType != null, "field 'component_type' is required in array type");
// array of component type, unlimited size
return typeFactory.createArrayType(
convert(new GSDataTypeDesc(componentType)),
RelDataType.PRECISION_NOT_SPECIFIED);
} else if ((value = typeMap.get("map")) != null) {
Map<String, Object> mapType = (Map<String, Object>) value;
Map<String, Object> keyType = (Map<String, Object>) mapType.get("key_type");
Preconditions.checkArgument(
keyType != null, "field 'key_type' is required in map type");
Map<String, Object> valueType = (Map<String, Object>) mapType.get("value_type");
Preconditions.checkArgument(
valueType != null, "field 'value_type' is required in map type");
// map of key type to value type
return typeFactory.createMapType(
convert(new GSDataTypeDesc(keyType)),
convert(new GSDataTypeDesc(valueType)));
} else if ((value = typeMap.get("decimal")) != null) {
Integer precision = getIntValue(value, "precision");
if (precision == null) {
precision =
typeFactory.getTypeSystem().getDefaultPrecision(SqlTypeName.DECIMAL);
logger.warn(
"can not convert {} to a valid precision, use default"
+ " precision {} instead",
value,
precision);
}
Integer scale = getIntValue(value, "scale");
if (scale == null) {
scale = typeFactory.getTypeSystem().getMaxScale(SqlTypeName.DECIMAL);
logger.warn(
"can not convert {} to a valid scale, use max" + " scale {} instead",
value,
scale);
}
// decimal type with precision and scale
return typeFactory.createSqlType(SqlTypeName.DECIMAL, precision, scale);
}
if (throwsOnFail) {
throw new UnsupportedOperationException(
"convert GSDataTypeDesc [" + from + "] to RelDataType is unsupported yet");
}
return typeFactory.createSqlType(SqlTypeName.ANY);
}

Check notice on line 338 in interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java

View check run for this annotation

codefactor.io / CodeFactor

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java#L211-L338 

Complex Method
@Override
public GSDataTypeDesc convert(RelDataType from) {
if (from instanceof IntervalSqlType) {
return new GSDataTypeDesc(ImmutableMap.of("primitive_type", "DT_SIGNED_INT64"));
} else if (from instanceof GraphLabelType) {
return new GSDataTypeDesc(ImmutableMap.of("primitive_type", "DT_SIGNED_INT32"));
}
SqlTypeName typeName = from.getSqlTypeName();
Map<String, Object> yamlDesc;
switch (typeName) {
case ANY:
yamlDesc = ImmutableMap.of("primitive_type", "DT_ANY");
break;
case NULL:
yamlDesc = ImmutableMap.of("primitive_type", "DT_NULL");
break;
case INTEGER:
yamlDesc = ImmutableMap.of("primitive_type", "DT_SIGNED_INT32");
break;
case BIGINT:
yamlDesc = ImmutableMap.of("primitive_type", "DT_SIGNED_INT64");
break;
case BOOLEAN:
yamlDesc = ImmutableMap.of("primitive_type", "DT_BOOL");
break;
case FLOAT:
yamlDesc = ImmutableMap.of("primitive_type", "DT_FLOAT");
break;
case DOUBLE:
yamlDesc = ImmutableMap.of("primitive_type", "DT_DOUBLE");
break;
case CHAR:
Map charMap = Maps.newHashMap();
charMap.put("char", ImmutableMap.of("fixed_length", from.getPrecision()));
yamlDesc = ImmutableMap.of("string", charMap);
break;
case VARCHAR:
if (from.getPrecision() == RelDataType.PRECISION_NOT_SPECIFIED) {
Map longTextMap = Maps.newHashMap();
longTextMap.put("long_text", ImmutableMap.of());
yamlDesc = ImmutableMap.of("string", longTextMap);
} else {
Map varCharMap = Maps.newHashMap();
varCharMap.put(
"var_char", ImmutableMap.of("max_length", from.getPrecision()));
yamlDesc = ImmutableMap.of("string", varCharMap);
}
break;
case DATE:
Map dateMap = Maps.newHashMap();
dateMap.put("date32", ImmutableMap.of());
yamlDesc = ImmutableMap.of("temporal", dateMap);
break;
case TIME:
Map timeMap = Maps.newHashMap();
timeMap.put("time32", ImmutableMap.of());
yamlDesc = ImmutableMap.of("temporal", timeMap);
break;
case TIMESTAMP:
Map timestampMap = Maps.newHashMap();
timestampMap.put("timestamp", ImmutableMap.of());
yamlDesc = ImmutableMap.of("temporal", timestampMap);
break;
case ARRAY:
case MULTISET:
Map<String, Object> componentType;
if (from instanceof ArbitraryArrayType) {
componentType = ImmutableMap.of("primitive_type", "DT_ANY");
} else {
componentType = convert(from.getComponentType()).getYamlDesc();
}
yamlDesc =
ImmutableMap.of(
"array",
ImmutableMap.of(
"component_type",
componentType,
"max_length",
from.getPrecision()
== RelDataType.PRECISION_NOT_SPECIFIED
? Integer.MAX_VALUE
: from.getPrecision()));
break;
case MAP:
Map<String, Object> keyType;
Map<String, Object> valueType;
if (from instanceof ArbitraryMapType) {
keyType = ImmutableMap.of("primitive_type", "DT_ANY");
valueType = ImmutableMap.of("primitive_type", "DT_ANY");
} else {
keyType = convert(from.getKeyType()).getYamlDesc();
valueType = convert(from.getValueType()).getYamlDesc();
}
yamlDesc =
ImmutableMap.of(
"map",
ImmutableMap.of("key_type", keyType, "value_type", valueType));
break;
case DECIMAL:
yamlDesc =
ImmutableMap.of(
"decimal",
ImmutableMap.of(
"precision", from.getPrecision(),
"scale", from.getScale()));
break;
default:
if (throwsOnFail) {
throw new UnsupportedOperationException(
"convert RelDataType ["
+ from
+ "] to GSDataTypeDesc is unsupported yet");
}
yamlDesc = ImmutableMap.of("primitive_type", "DT_ANY");
}
return new GSDataTypeDesc(yamlDesc);
}

Check notice on line 456 in interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java

View check run for this annotation

codefactor.io / CodeFactor

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java#L339-L456 

Complex Method
private @Nullable Integer getIntValue(Object valueMap, String key) {
if (valueMap instanceof Map) {
Object value = ((Map) valueMap).get(key);
Expand Down
17 changes: 16 additions & 1 deletion .../compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrGraphProperty.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,33 @@
package com.alibaba.graphscope.common.ir.meta.schema;

import com.alibaba.graphscope.groot.common.schema.impl.DefaultGraphProperty;
import com.google.common.base.Objects;

import org.apache.calcite.rel.type.RelDataType;

public class IrGraphProperty extends DefaultGraphProperty {
private final RelDataType relDataType;

public IrGraphProperty(int id, String name, RelDataType relDataType) {
super(id, name, new IrDataTypeConvertor.Groot(null).convert(relDataType));
super(id, name, new IrDataTypeConvertor.Groot(null, false).convert(relDataType));
this.relDataType = relDataType;
}

public RelDataType getRelDataType() {
return this.relDataType;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
if (!super.equals(o)) return false;
IrGraphProperty that = (IrGraphProperty) o;
return Objects.equal(relDataType, that.relDataType);
}

@Override
public int hashCode() {
return Objects.hashCode(super.hashCode(), relDataType);
}
}
14 changes: 11 additions & 3 deletions ...ne/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrGraphSchema.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@
import com.alibaba.graphscope.groot.common.exception.PropertyNotFoundException;
import com.alibaba.graphscope.groot.common.exception.TypeNotFoundException;
import com.alibaba.graphscope.groot.common.schema.api.*;
import com.google.common.collect.ImmutableMap;

import org.apache.calcite.rel.type.RelDataTypeFactory;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
Expand All @@ -42,14 +45,19 @@ public IrGraphSchema(Configs configs, SchemaInputStream schemaInputStream) throw
schemaInputStream.getInputStream().readAllBytes(), StandardCharsets.UTF_8);
schemaInputStream.getInputStream().close();
SchemaSpec spec = new SchemaSpec(schemaInputStream.getType(), content);
this.graphSchema = spec.convert(new GraphTypeFactoryImpl(configs));
this.specManager = new SchemaSpecManager(this, spec);
RelDataTypeFactory typeFactory = new GraphTypeFactoryImpl(configs);
this.graphSchema = spec.convert(typeFactory);
this.specManager = new SchemaSpecManager(this.graphSchema, false, typeFactory, spec);
}

public IrGraphSchema(GraphSchema graphSchema, boolean isColumnId) {
this.graphSchema = graphSchema;
this.isColumnId = isColumnId;
this.specManager = new SchemaSpecManager(this);
this.specManager =
new SchemaSpecManager(
this.graphSchema,
this.isColumnId,
new GraphTypeFactoryImpl(new Configs(ImmutableMap.of())));
}

public boolean isColumnId() {
Expand Down
177 changes: 117 additions & 60 deletions ...ompiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/SchemaSpecManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,31 +18,47 @@

package com.alibaba.graphscope.common.ir.meta.schema;

import com.alibaba.graphscope.groot.common.schema.api.*;
import com.alibaba.graphscope.groot.common.util.IrSchemaParser;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;

import org.checkerframework.checker.nullness.qual.Nullable;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;

import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

public class SchemaSpecManager {
private static final Logger logger = LoggerFactory.getLogger(SchemaSpecManager.class);
private final IrGraphSchema parent;
private final GraphSchema rootSchema;
private final boolean isColumnId;
private final RelDataTypeFactory typeFactory;
private final List<SchemaSpec> specifications;

public SchemaSpecManager(IrGraphSchema parent) {
this.parent = parent;
this.specifications = Lists.newArrayList(convert(null, SchemaSpec.Type.IR_CORE_IN_JSON));
public SchemaSpecManager(
GraphSchema rootSchema, boolean isColumnId, RelDataTypeFactory typeFactory) {
this.rootSchema = rootSchema;
this.isColumnId = isColumnId;
this.typeFactory = typeFactory;
this.specifications = Lists.newArrayList();
}

public SchemaSpecManager(IrGraphSchema parent, SchemaSpec input) {
this.parent = parent;
public SchemaSpecManager(
GraphSchema rootSchema,
boolean isColumnId,
RelDataTypeFactory typeFactory,
SchemaSpec input) {
this.rootSchema = rootSchema;
this.isColumnId = isColumnId;
this.typeFactory = typeFactory;
this.specifications = Lists.newArrayList(input);
}

Expand All @@ -52,62 +68,103 @@ public SchemaSpec getSpec(SchemaSpec.Type type) {
return spec;
}
}
// if not exist, try to create a new JsonSpecification with content converted from others
SchemaSpec newSpec;
List<SchemaSpec.Type> existing = Lists.newArrayList();
for (SchemaSpec spec : specifications) {
if ((newSpec = convert(spec, type)) != null) {
specifications.add(newSpec);
return newSpec;
}
existing.add(spec.getType());
switch (type) {
case IR_CORE_IN_JSON:
newSpec =
new SchemaSpec(
type, IrSchemaParser.getInstance().parse(rootSchema, isColumnId));
break;
case FLEX_IN_JSON:
SchemaSpec yamlSpec = getSpec(SchemaSpec.Type.FLEX_IN_YAML);
Preconditions.checkArgument(
yamlSpec != null,
"cannot get schema specification of type " + SchemaSpec.Type.FLEX_IN_YAML);
Yaml yaml = new Yaml();
Map yamlMap = yaml.load(yamlSpec.getContent());
ObjectMapper mapper = new ObjectMapper();
try {
newSpec = new SchemaSpec(type, mapper.writeValueAsString(yamlMap));
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
break;
case FLEX_IN_YAML:
default:
newSpec = convertToFlex(rootSchema);
break;
}
throw new IllegalArgumentException(
"spec type ["
+ type
+ "] cannot be converted from any existing spec types "
+ existing);
this.specifications.add(newSpec);
return newSpec;
}

private @Nullable SchemaSpec convert(@Nullable SchemaSpec source, SchemaSpec.Type target) {
try {
if (source != null && source.getType() == target) {
return source;
}
switch (target) {
case IR_CORE_IN_JSON:
return new SchemaSpec(
target,
IrSchemaParser.getInstance()
.parse(parent.getGraphSchema(), parent.isColumnId()));
case FLEX_IN_JSON:
if (source != null && source.getType() == SchemaSpec.Type.FLEX_IN_YAML) {
Yaml yaml = new Yaml();
Map rootMap = yaml.load(source.getContent());
ObjectMapper mapper = new ObjectMapper();
return new SchemaSpec(target, mapper.writeValueAsString(rootMap));
}
// todo: convert from JSON in IR_CORE to JSON in FLEX
return null;
case FLEX_IN_YAML:
default:
if (source != null && source.getType() == SchemaSpec.Type.FLEX_IN_JSON) {
ObjectMapper mapper = new ObjectMapper();
JsonNode rootNode = mapper.readTree(source.getContent());
Map rootMap = mapper.convertValue(rootNode, Map.class);
Yaml yaml = new Yaml();
return new SchemaSpec(target, yaml.dump(rootMap));
}
// todo: convert from JSON in IR_CORE to YAML in FlEX
return null;
}
} catch (Exception e) {
logger.warn(
"can not convert from {} to {} due to some unexpected exception:",
source == null ? null : source.getType(),
target,
e);
return null;
private SchemaSpec convertToFlex(GraphSchema schema) {
List<Map> vertices =
schema.getVertexList().stream()
.map(this::convertVertex)
.collect(Collectors.toList());
List<Map> edges =
schema.getEdgeList().stream().map(this::convertEdge).collect(Collectors.toList());
Map<String, Object> flexMap =
ImmutableMap.of(
"schema", ImmutableMap.of("vertex_types", vertices, "edge_types", edges));
Yaml yaml = new Yaml();
return new SchemaSpec(SchemaSpec.Type.FLEX_IN_YAML, yaml.dump(flexMap));
}

private Map<String, Object> convertVertex(GraphVertex vertex) {
return ImmutableMap.of(
"type_name", vertex.getLabel(),
"type_id", vertex.getLabelId(),
"properties",
vertex.getPropertyList().stream()
.map(this::convertProperty)
.collect(Collectors.toList()),
"primary_keys",
vertex.getPrimaryKeyList().stream()
.map(GraphProperty::getName)
.collect(Collectors.toList()));
}

private Map<String, Object> convertEdge(GraphEdge edge) {
return ImmutableMap.of(
"type_name", edge.getLabel(),
"type_id", edge.getLabelId(),
"vertex_type_pair_relations",
edge.getRelationList().stream()
.map(this::convertRelation)
.collect(Collectors.toList()),
"properties",
edge.getPropertyList().stream()
.map(this::convertProperty)
.collect(Collectors.toList()),
"primary_keys",
edge.getPrimaryKeyList().stream()
.map(GraphProperty::getName)
.collect(Collectors.toList()));
}

private Map<String, Object> convertRelation(EdgeRelation relation) {
return ImmutableMap.of(
"source_vertex", relation.getSource().getLabel(),
"destination_vertex", relation.getTarget().getLabel());
}

private Map<String, Object> convertProperty(GraphProperty property) {
RelDataType propertyType;
if (property instanceof IrGraphProperty) {
propertyType = ((IrGraphProperty) property).getRelDataType();
} else {
propertyType =
(new IrDataTypeConvertor.Groot(typeFactory, false))
.convert(property.getDataType());
}
// convert property type to flex format
IrDataTypeConvertor.Flex flexConvertor = new IrDataTypeConvertor.Flex(typeFactory, false);
GSDataTypeDesc typeDesc = flexConvertor.convert(propertyType);
return ImmutableMap.of(
"property_id", property.getId(),
"property_name", property.getName(),
"property_type", typeDesc.getYamlDesc());
}
}
Loading
Loading