From 73ee336359285d88e786919ab693c016e7cc2935 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 1 Oct 2024 03:40:42 -0400 Subject: [PATCH] feat(schema): add API descriptors, struct, oneof & list types, and wire encoding spec (#21482) Co-authored-by: marbar3778 Co-authored-by: Marko --- schema/api.go | 98 +++++++++++++++++++++++++++++++++ schema/field.go | 15 +++++- schema/kind.go | 120 +++++++++++++++++++++++++++++++++++++++-- schema/oneof.go | 43 +++++++++++++++ schema/state_object.go | 13 +++-- schema/struct.go | 21 ++++++++ 6 files changed, 302 insertions(+), 8 deletions(-) create mode 100644 schema/api.go create mode 100644 schema/oneof.go create mode 100644 schema/struct.go diff --git a/schema/api.go b/schema/api.go new file mode 100644 index 000000000000..3d8224a9a6d9 --- /dev/null +++ b/schema/api.go @@ -0,0 +1,98 @@ +package schema + +// APIDescriptor is a public versioned descriptor of an API. +// +// An APIDescriptor can be used as a native descriptor of an API's encoding. +// The native binary encoding of API requests and responses is to encode the input and output +// fields using value binary encoding. +// The native JSON encoding would be to encode the fields as a JSON object, canonically +// sorted by field name with no extra whitespace. +// Thus, APIDefinitions have deterministic binary and JSON encodings. +// +// APIDefinitions have a strong definition of compatibility between different versions +// of the same API. +// It is an INCOMPATIBLE change to add new input fields to existing methods or to remove or modify +// existing input or output fields. +// Input fields also cannot reference any unsealed structs, directly or transitively, +// because these types allow adding new fields. +// Adding new input fields to a method introduces the possibility that a newer client +// will send an incomprehensible message to an older server. +// The only safe ways that input field schemas can be extended are by adding +// new values to EnumType's and new cases to OneOfType's. +// It is a COMPATIBLE change to add new methods to an API and to add new output fields +// to existing methods. +// Output fields can reference any sealed or unsealed StructType, directly or transitively. +// +// Existing protobuf APIs could also be mapped into APIDefinitions, and used in the following ways: +// - to produce, user-friendly deterministic JSON +// - to produce a deterministic binary encoding +// - to check for compatibility in a way that is more appropriate to blockchain applications +// - to use any code generators designed to support this spec as an alternative to protobuf +// Also, a standardized way of serializing schema types as protobuf could be defined which +// maps to the original protobuf encoding, so that schemas can be used as an interop +// layer between different less expressive encoding systems. +// +// Existing EVM contract APIs expressed in Solidity could be mapped into APIDefinitions, and +// a mapping of all schema values to ABI encoding could be defined which preserves the +// original ABI encoding. +// +// In this way, we can define an interop layer between contracts in the EVM world, +// SDK modules accepting protobuf types, and any API using this schema system natively. +type APIDescriptor struct { + // Name is the versioned name of the API. + Name string + + // Methods is the list of methods in the API. + // It is a COMPATIBLE change to add new methods to an API. + // If a newer client tries to call a method that an older server does not recognize it, + // an error will simply be returned. + Methods []MethodDescriptor +} + +// MethodDescriptor describes a method in the API. +type MethodDescriptor struct { + // Name is the name of the method. + Name string + + // InputFields is the list of input fields for the method. + // + // It is an INCOMPATIBLE change to add, remove or update input fields to a method. + // The addition of new fields introduces the possibility that a newer client + // will send an incomprehensible message to an older server. + // InputFields can only reference sealed StructTypes, either directly and transitively. + // + // As a special case to represent protobuf service definitions, there can be a single + // unnamed struct input field that code generators can choose to either reference + // as a named struct or to expand inline as function arguments. + InputFields []Field + + // OutputFields is the list of output fields for the method. + // + // It is a COMPATIBLE change to add new output fields to a method, + // but existing output fields should not be removed or modified. + // OutputFields can reference any sealed or unsealed StructType, directly or transitively. + // If a newer client tries to call a method on an older server, the newer expected result output + // fields will simply be populated with the default values for that field kind. + // + // As a special case to represent protobuf service definitions, there can be a single + // unnamed struct output field. + // In this case, adding new output fields is an INCOMPATIBLE change (because protobuf service definitions + // don't allow this), but new fields can be added to the referenced struct if it is unsealed. + OutputFields []Field + + // Volatility is the volatility of the method. + Volatility Volatility +} + +// Volatility is the volatility of a method. +type Volatility int + +const ( + // PureVolatility indicates that the method can neither read nor write state. + PureVolatility Volatility = iota + // ReadonlyVolatility indicates that the method can read state but not write state. + ReadonlyVolatility + + // VolatileVolatility indicates that the method can read and write state. + VolatileVolatility +) diff --git a/schema/field.go b/schema/field.go index dfbae6ed1b97..b2bc76a2e3b6 100644 --- a/schema/field.go +++ b/schema/field.go @@ -15,8 +15,21 @@ type Field struct { // Nullable indicates whether null values are accepted for the field. Key fields CANNOT be nullable. Nullable bool `json:"nullable,omitempty"` - // ReferencedType is the referenced type name when Kind is EnumKind. + // ReferencedType is the referenced type name when Kind is EnumKind, StructKind or OneOfKind. ReferencedType string `json:"referenced_type,omitempty"` + + // ElementKind is the element type when Kind is ListKind. + // Support for this is currently UNIMPLEMENTED, this notice will be removed when it is added. + ElementKind Kind `json:"element_kind,omitempty"` + + // Size specifies the size or max-size of a field. + // Support for this is currently UNIMPLEMENTED, this notice will be removed when it is added. + // Its specific meaning may vary depending on the field kind. + // For IntNKind and UintNKind fields, it specifies the bit width of the field. + // For StringKind, BytesKind, AddressKind, and JSONKind, fields it specifies the maximum length rather than a fixed length. + // If it is 0, such fields have no maximum length. + // It is invalid to have a non-zero Size for other kinds. + Size uint32 `json:"size,omitempty"` } // Validate validates the field. diff --git a/schema/kind.go b/schema/kind.go index 5eedee68c244..0992b297baa7 100644 --- a/schema/kind.go +++ b/schema/kind.go @@ -10,12 +10,25 @@ import ( // Kind represents the basic type of a field in an object. // Each kind defines the following encodings: -// Go Encoding: the golang type which should be accepted by listeners and +// +// - Go Encoding: the golang type which should be accepted by listeners and // generated by decoders when providing entity updates. -// JSON Encoding: the JSON encoding which should be used when encoding the field to JSON. +// - JSON Encoding: the JSON encoding which should be used when encoding the field to JSON. +// - Key Binary Encoding: the encoding which should be used when encoding the field +// as a key in binary messages. Some encodings specify a terminal and non-terminal form +// depending on whether or not the field is the last field in the key. +// - Value Binary Encoding: the encoding which should be used when encoding the field +// as a value in binary messages. +// // When there is some non-determinism in an encoding, kinds should specify what // values they accept and also what is the canonical, deterministic encoding which // should be preferably emitted by serializers. +// +// Binary encodings were chosen based on what is likely to be the most convenient default binary encoding +// for state management implementations. This encoding allows for sorted keys whenever it is possible for a kind +// and is deterministic. +// Modules that use the specified encoding natively will have a trivial decoder implementation because the +// encoding is already in the correct format after any initial prefix bytes are stripped. type Kind int const ( @@ -25,54 +38,82 @@ const ( // StringKind is a string type. // Go Encoding: UTF-8 string with no null characters. // JSON Encoding: string + // Key Binary Encoding: + // non-terminal: UTF-8 string with no null characters suffixed with a null character + // terminal: UTF-8 string with no null characters + // Value Binary Encoding: the same value binary encoding as BytesKind. StringKind - // BytesKind is a bytes type. + // BytesKind represents a byte array. // Go Encoding: []byte // JSON Encoding: base64 encoded string, canonical values should be encoded with standard encoding and padding. // Either standard or URL encoding with or without padding should be accepted. + // Key Binary Encoding: + // non-terminal: length prefixed bytes where the width of the length prefix is 1, 2, 3 or 4 bytes depending on + // the field's MaxLength (defaulting to 4 bytes). + // Length prefixes should be big-endian encoded. + // Values larger than 2^32 bytes are not supported (likely key-value stores impose a lower limit). + // terminal: raw bytes with no length prefix + // Value Binary Encoding: two 32-bit unsigned little-endian integers, the first one representing the offset of the + // value in the buffer and the second one representing the length of the value. BytesKind // Int8Kind represents an 8-bit signed integer. // Go Encoding: int8 // JSON Encoding: number + // Key Binary Encoding: 1-byte two's complement encoding, with the first bit inverted for sorting. + // Value Binary Encoding: 1-byte two's complement encoding. Int8Kind // Uint8Kind represents an 8-bit unsigned integer. // Go Encoding: uint8 // JSON Encoding: number + // Key Binary Encoding: 1-byte unsigned encoding. + // Value Binary Encoding: 1-byte unsigned encoding. Uint8Kind // Int16Kind represents a 16-bit signed integer. // Go Encoding: int16 // JSON Encoding: number + // Key Binary Encoding: 2-byte two's complement big-endian encoding, with the first bit inverted for sorting. + // Value Binary Encoding: 2 byte two's complement little-endian encoding. Int16Kind // Uint16Kind represents a 16-bit unsigned integer. // Go Encoding: uint16 // JSON Encoding: number + // Key Binary Encoding: 2-byte unsigned big-endian encoding. + // Value Binary Encoding: 2-byte unsigned little-endian encoding. Uint16Kind // Int32Kind represents a 32-bit signed integer. // Go Encoding: int32 // JSON Encoding: number + // Key Binary Encoding: 4-byte two's complement big-endian encoding, with the first bit inverted for sorting. + // Value Binary Encoding: 4-byte two's complement little-endian encoding. Int32Kind // Uint32Kind represents a 32-bit unsigned integer. // Go Encoding: uint32 // JSON Encoding: number + // Key Binary Encoding: 4-byte unsigned big-endian encoding. + // Value Binary Encoding: 4-byte unsigned little-endian encoding. Uint32Kind // Int64Kind represents a 64-bit signed integer. // Go Encoding: int64 // JSON Encoding: base10 integer string which matches the IntegerFormat regex // The canonical encoding should include no leading zeros. + // Key Binary Encoding: 8-byte two's complement big-endian encoding, with the first bit inverted for sorting. + // Value Binary Encoding: 8-byte two's complement little-endian encoding. Int64Kind // Uint64Kind represents a 64-bit unsigned integer. // Go Encoding: uint64 // JSON Encoding: base10 integer string which matches the IntegerFormat regex // Canonically encoded values should include no leading zeros. + // Key Binary Encoding: 8-byte unsigned big-endian encoding. + // Value Binary Encoding: 8-byte unsigned little-endian encoding. Uint64Kind // IntegerKind represents an arbitrary precision integer number. @@ -98,6 +139,8 @@ const ( // BoolKind represents a boolean true or false value. // Go Encoding: bool // JSON Encoding: boolean + // Key Binary Encoding: 1-byte encoding where 0 is false and 1 is true. + // Value Binary Encoding: 1-byte encoding where 0 is false and 1 is true. BoolKind // TimeKind represents a nanosecond precision UNIX time value (with zero representing January 1, 1970 UTC). @@ -107,6 +150,8 @@ const ( // Canonical values should be encoded with UTC time zone Z, nanoseconds should // be encoded with no trailing zeros, and T time values should always be present // even at 00:00:00. + // Key Binary Encoding: 8-byte two's complement big-endian encoding, with the first bit inverted for sorting. + // Value Binary Encoding: 8-byte two's complement little-endian encoding. TimeKind // DurationKind represents the elapsed time between two nanosecond precision time values. @@ -114,24 +159,35 @@ const ( // Go Encoding: time.Duration // JSON Encoding: the number of seconds as a decimal string with no trailing zeros followed by // a lowercase 's' character to represent seconds. + // Key Binary Encoding: 8-byte two's complement big-endian encoding, with the first bit inverted for sorting. + // Value Binary Encoding: 8-byte two's complement little-endian encoding. DurationKind // Float32Kind represents an IEEE-754 32-bit floating point number. // Go Encoding: float32 // JSON Encoding: number + // Key Binary Encoding: 4-byte IEEE-754 encoding. + // Value Binary Encoding: 4-byte IEEE-754 encoding. Float32Kind // Float64Kind represents an IEEE-754 64-bit floating point number. // Go Encoding: float64 // JSON Encoding: number + // Key Binary Encoding: 8-byte IEEE-754 encoding. + // Value Binary Encoding: 8-byte IEEE-754 encoding. Float64Kind // AddressKind represents an account address which is represented by a variable length array of bytes. // Addresses usually have a human-readable rendering, such as bech32, and tooling should provide - // a way for apps to define a string encoder for friendly user-facing display. + // a way for apps to define a string encoder for friendly user-facing display. Addresses have a maximum + // supported length of 63 bytes. // Go Encoding: []byte // JSON Encoding: addresses should be encoded as strings using the human-readable address renderer // provided to the JSON encoder. + // Key Binary Encoding: + // non-terminal: bytes prefixed with 1-byte length prefix + // terminal: raw bytes with no length prefix + // Value Binary Encoding: bytes prefixed with 1-byte length prefix. AddressKind // EnumKind represents a value of an enum type. @@ -139,12 +195,68 @@ const ( // definition. // Go Encoding: string // JSON Encoding: string + // Key Binary Encoding: the same binary encoding as the EnumType's numeric kind. + // Value Binary Encoding: the same binary encoding as the EnumType's numeric kind. EnumKind // JSONKind represents arbitrary JSON data. // Go Encoding: json.RawMessage // JSON Encoding: any valid JSON value + // Key Binary Encoding: string encoding + // Value Binary Encoding: string encoding JSONKind + + // UIntNKind represents a signed integer type with a width in bits specified by the Size field in the + // field definition. + // Support for this is currently UNIMPLEMENTED, this notice will be removed when it is added. + // N must be a multiple of 8, and it is invalid for N to equal 8, 16, 32, 64 as there are more specific + // types for these widths. + // Go Encoding: []byte where len([]byte) == Size / 8, little-endian encoded. + // JSON Encoding: base10 integer string matching the IntegerFormat regex, canonically with no leading zeros. + // Key Binary Encoding: N / 8 bytes big-endian encoded + // Value Binary Encoding: N / 8 bytes little-endian encoded + UIntNKind + + // IntNKind represents an unsigned integer type with a width in bits specified by the Size field in the + // field definition. N must be a multiple of 8. + // Support for this is currently UNIMPLEMENTED, this notice will be removed when it is added. + // N must be a multiple of 8, and it is invalid for N to equal 8, 16, 32, 64 as there are more specific + // types for these widths. + // Go Encoding: []byte where len([]byte) == Size / 8, two's complement little-endian encoded. + // JSON Encoding: base10 integer string matching the IntegerFormat regex, canonically with no leading zeros. + // Key Binary Encoding: N / 8 bytes big-endian two's complement encoded with the first bit inverted for sorting. + // Value Binary Encoding: N / 8 bytes little-endian two's complement encoded. + IntNKind + + // StructKind represents a struct object. + // Support for this is currently UNIMPLEMENTED, this notice will be removed when it is added. + // Go Encoding: an array of type []interface{} where each element is of the respective field's kind type. + // JSON Encoding: an object where each key is the field name and the value is the field value. + // Canonically, keys are in alphabetical order with no extra whitespace. + // Key Binary Encoding: not valid as a key field. + // Value Binary Encoding: 32-bit unsigned little-endian length prefix, + // followed by the value binary encoding of each field in order. + StructKind + + // OneOfKind represents a field that can be one of a set of types. + // Support for this is currently UNIMPLEMENTED, this notice will be removed when it is added. + // Go Encoding: the anonymous struct { Case string; Value interface{} }, aliased as OneOfValue. + // JSON Encoding: same as the case's struct encoding with "@type" set to the case name. + // Key Binary Encoding: not valid as a key field. + // Value Binary Encoding: the oneof's discriminant numeric value encoded as its discriminant kind + // followed by the encoded value. + OneOfKind + + // ListKind represents a list of elements. + // Support for this is currently UNIMPLEMENTED, this notice will be removed when it is added. + // Go Encoding: an array of type []interface{} where each element is of the respective field's kind type. + // JSON Encoding: an array of values where each element is the field value. + // Canonically, there is no extra whitespace. + // Key Binary Encoding: not valid as a key field. + // Value Binary Encoding: 32-bit unsigned little-endian size prefix indicating the size of the encoded data in bytes, + // followed by a 32-bit unsigned little-endian count of the number of elements in the list, + // followed by each element encoded with value binary encoding. + ListKind ) // MAX_VALID_KIND is the maximum valid kind value. diff --git a/schema/oneof.go b/schema/oneof.go new file mode 100644 index 000000000000..ec0cd6d649ae --- /dev/null +++ b/schema/oneof.go @@ -0,0 +1,43 @@ +package schema + +// OneOfType represents a oneof type. +// Support for this is currently UNIMPLEMENTED, this notice will be removed when it is added. +type OneOfType struct { + // Name is the name of the oneof type. It must conform to the NameFormat regular expression. + Name string + + // Cases is a list of cases in the oneof type. + // It is a COMPATIBLE change to add new cases to a oneof type. + // If a newer client tries to send a message with a case that an older server does not recognize, + // the older server will simply reject it in a switch statement. + // It is INCOMPATIBLE to remove existing cases from a oneof type. + Cases []OneOfCase + + // DiscriminantKind is the kind of the discriminant field. + // It must be Uint8Kind, Int8Kind, Uint16Kind, Int16Kind, or Int32Kind. + DiscriminantKind Kind +} + +// OneOfCase represents a case in a oneof type. It is represented by a struct type internally with a discriminant value. +type OneOfCase struct { + // Name is the name of the case. It must conform to the NameFormat regular expression. + Name string + + // Discriminant is the discriminant value for the case. + Discriminant int32 + + // Kind is the kind of the case. ListKind is not allowed. + Kind Kind + + // Reference is the referenced type if Kind is EnumKind, StructKind, or OneOfKind. + ReferencedType string +} + +// OneOfValue is the golang runtime representation of a oneof value. +type OneOfValue = struct { + // Case is the name of the case. + Case string + + // Value is the value of the case. + Value interface{} +} diff --git a/schema/state_object.go b/schema/state_object.go index a4825726f685..eff8d1f98459 100644 --- a/schema/state_object.go +++ b/schema/state_object.go @@ -9,16 +9,23 @@ type StateObjectType struct { Name string `json:"name"` // KeyFields is a list of fields that make up the primary key of the object. - // It can be empty in which case indexers should assume that this object is + // It can be empty, in which case, indexers should assume that this object is // a singleton and only has one value. Field names must be unique within the // object between both key and value fields. - // Key fields CANNOT be nullable and Float32Kind, Float64Kind, and JSONKind types - // are not allowed. + // Key fields CANNOT be nullable and Float32Kind, Float64Kind, JSONKind, StructKind, + // OneOfKind, RepeatedKind, ListKind or ObjectKind + // are NOT ALLOWED. + // It is an INCOMPATIBLE change to add, remove or change fields in the key as this + // changes the underlying primary key of the object. KeyFields []Field `json:"key_fields,omitempty"` // ValueFields is a list of fields that are not part of the primary key of the object. // It can be empty in the case where all fields are part of the primary key. // Field names must be unique within the object between both key and value fields. + // ObjectKind fields are not allowed. + // It is a COMPATIBLE change to add new value fields to an object type because + // this does not affect the primary key of the object. + // Existing value fields should not be removed or modified. ValueFields []Field `json:"value_fields,omitempty"` // RetainDeletions is a flag that indicates whether the indexer should retain diff --git a/schema/struct.go b/schema/struct.go new file mode 100644 index 000000000000..20676a743ade --- /dev/null +++ b/schema/struct.go @@ -0,0 +1,21 @@ +package schema + +// StructType represents a struct type. +// Support for this is currently UNIMPLEMENTED, this notice will be removed when it is added. +type StructType struct { + // Name is the name of the struct type. + Name string + + // Fields is the list of fields in the struct. + // It is a COMPATIBLE change to add new fields to an unsealed struct, + // but it is an INCOMPATIBLE change to add new fields to a sealed struct. + // + // A sealed struct cannot reference any unsealed structs directly or + // transitively because these types allow adding new fields. + Fields []Field + + // Sealed is true if it is an INCOMPATIBLE change to add new fields to the struct. + // It is a COMPATIBLE change to change an unsealed struct to sealed, but it is + // an INCOMPATIBLE change to change a sealed struct to unsealed. + Sealed bool +}