Fix scvals to print in json string

Cargo.toml

@@ -0,0 +1,45 @@
+[workspace]
+resolver = "2"
+members = [
+    "lib/ffi",
+    "lib/xdr2json"
+]
+
+[workspace.package]
+rust-version = "1.79.0"
+
+[workspace.dependencies.soroban-env-host-prev]
+package = "soroban-env-host"
+version = "=21.2.1"
+
+[workspace.dependencies.soroban-env-host-curr]
+package = "soroban-env-host"
+version = "=22.0.0"
+#git = "https://github.com/stellar/rs-soroban-env"
+#rev = "0497816694bef2b103494c8c61b7c8a06a72c7d3"
+
+[workspace.dependencies.soroban-simulation-prev]
+package = "soroban-simulation"
+version = "=21.2.1"
+
+[workspace.dependencies.soroban-simulation-curr]
+package = "soroban-simulation"
+version = "=22.0.0"
+# git = "https://github.com/stellar/rs-soroban-env"
+# rev = "0497816694bef2b103494c8c61b7c8a06a72c7d3"
+
+[workspace.dependencies.stellar-xdr]
+version = "=22.0.0"
+features = [ "serde" ]
+
+[workspace.dependencies]
+base64 = "0.22.0"
+sha2 = "0.10.7"
+libc = "0.2.147"
+anyhow = "1.0.75"
+rand = { version = "0.8.5", features = [] }
+serde_json = "1.0"
+
+[profile.release-with-panic-unwind]
+inherits = 'release'
+panic = 'unwind'

Makefile

@@ -7,6 +7,17 @@ BUILD_DATE := $(shell date -u +%FT%TZ)
 DEFAULT_ETLHASH := stellar/stellar-etl:$(shell git rev-parse --short=9 HEAD)
 ETLHASH ?= $(DEFAULT_ETLHASH)
 
+# Build rust xdr2json
+CARGO_BUILD_TARGET ?= $(shell rustc -vV | sed -n 's|host: ||p')
+
+# update the Cargo.lock every time the Cargo.toml changes.
+Cargo.lock: Cargo.toml
+	cargo update --workspace
+
+build-libs: Cargo.lock
+	cd lib/xdr2json && \
+	cargo build --target $(CARGO_BUILD_TARGET) --profile release-with-panic-unwind
+
 docker-build:
 	$(SUDO) docker build --platform linux/amd64 --pull --no-cache --label org.opencontainers.image.created="$(BUILD_DATE)" \
 	-t $(ETLHASH) -t stellar/stellar-etl:latest -f ./docker/Dockerfile .

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.17.0
-	github.com/stellar/go v0.0.0-20240905180041-acfaa0686213
+	github.com/stellar/go v0.0.0-20240924182550-69667b25baf4
 	github.com/stretchr/testify v1.9.0
 	github.com/xitongsys/parquet-go v1.6.2
 	github.com/xitongsys/parquet-go-source v0.0.0-20240122235623-d6294584ab18

go.sum

@@ -641,8 +641,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.17.0 h1:I5txKw7MJasPL/BrfkbA0Jyo/oELqVmux4pR/UxOMfI=
 github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0+yVI=
-github.com/stellar/go v0.0.0-20240905180041-acfaa0686213 h1:224VUCwV1xmmeTru1zCmTHxvi2RECoHdfdWgd9ni518=
-github.com/stellar/go v0.0.0-20240905180041-acfaa0686213/go.mod h1:rrFK7a8i2h9xad9HTfnSN/dTNEqXVHKAbkFeR7UxAgs=
+github.com/stellar/go v0.0.0-20240924182550-69667b25baf4 h1:Kd4ivg3hCG8AfFQpxjUjhEXKc40Ux+piUWL03dBB/sw=
+github.com/stellar/go v0.0.0-20240924182550-69667b25baf4/go.mod h1:rrFK7a8i2h9xad9HTfnSN/dTNEqXVHKAbkFeR7UxAgs=
 github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2 h1:OzCVd0SV5qE3ZcDeSFCmOWLZfEWZ3Oe8KtmSOYKEVWE=
 github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2/go.mod h1:yoxyU/M8nl9LKeWIoBrbDPQ7Cy+4jxRcWcOayZ4BMps=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

internal/transform/contract_data.go

@@ -104,8 +104,14 @@ func (t *TransformContractDataStruct) TransformContractData(ledgerChange ingest.
 
 	ledgerSequence := header.Header.LedgerSeq
 
-	outputKey, outputKeyDecoded := serializeScVal(contractData.Key)
-	outputVal, outputValDecoded := serializeScVal(contractData.Val)
+	outputKey, outputKeyDecoded, err := serializeScVal(contractData.Key)
+	if err != nil {
+		return ContractDataOutput{}, err, false
+	}
+	outputVal, outputValDecoded, err := serializeScVal(contractData.Val)
+	if err != nil {
+		return ContractDataOutput{}, err, false
+	}
 
 	outputContractDataXDR, err := xdr.MarshalBase64(contractData)
 	if err != nil {

internal/transform/contract_data_test.go

@@ -135,7 +135,7 @@ func makeContractDataTestOutput() []ContractDataOutput {
 
 	keyDecoded := map[string]string{
 		"type":  "Instance",
-		"value": "0000000000000000000000000000000000000000000000000000000000000000: [{a a}]",
+		"value": "{\"Instance\":{\"Executable\":{\"Type\":0,\"WasmHash\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},\"Storage\":[{\"Key\":{\"Str\":\"a\",\"Type\":14},\"Val\":{\"Str\":\"a\",\"Type\":14}}]},\"Type\":19}",
 	}
 
 	val := map[string]string{
@@ -145,7 +145,7 @@ func makeContractDataTestOutput() []ContractDataOutput {
 
 	valDecoded := map[string]string{
 		"type":  "B",
-		"value": "true",
+		"value": "{\"B\":true,\"Type\":0}",
 	}
 
 	return []ContractDataOutput{

internal/transform/contract_events.go

@@ -2,10 +2,12 @@ package transform
 
 import (
 	"encoding/base64"
+	"encoding/json"
 	"fmt"
 
 	"github.com/stellar/stellar-etl/internal/toid"
 	"github.com/stellar/stellar-etl/internal/utils"
+	"github.com/stellar/stellar-etl/internal/xdr2json"
 
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/strkey"
@@ -48,12 +50,18 @@ func TransformContractEvent(transaction ingest.LedgerTransaction, lhe xdr.Ledger
 		outputTypeString := event.Type.String()
 
 		eventTopics := getEventTopics(event.Body)
-		outputTopics, outputTopicsDecoded := serializeScValArray(eventTopics)
+		outputTopics, outputTopicsDecoded, err := serializeScValArray(eventTopics)
+		if err != nil {
+			return []ContractEventOutput{}, err
+		}
 		outputTopicsJson["topics"] = outputTopics
 		outputTopicsDecodedJson["topics_decoded"] = outputTopicsDecoded
 
 		eventData := getEventData(event.Body)
-		outputData, outputDataDecoded := serializeScVal(eventData)
+		outputData, outputDataDecoded, err := serializeScVal(eventData)
+		if err != nil {
+			return []ContractEventOutput{}, err
+		}
 
 		// Convert the xdrContactId to string
 		// TODO: https://stellarorg.atlassian.net/browse/HUBBLE-386 this should be a stellar/go/xdr function
@@ -117,7 +125,7 @@ func getEventData(eventBody xdr.ContractEventBody) xdr.ScVal {
 }
 
 // TODO this should also be used in the operations processor
-func serializeScVal(scVal xdr.ScVal) (map[string]string, map[string]string) {
+func serializeScVal(scVal xdr.ScVal) (map[string]string, map[string]string, error) {
 	serializedData := map[string]string{}
 	serializedData["value"] = "n/a"
 	serializedData["type"] = "n/a"
@@ -127,27 +135,41 @@ func serializeScVal(scVal xdr.ScVal) (map[string]string, map[string]string) {
 	serializedDataDecoded["type"] = "n/a"
 
 	if scValTypeName, ok := scVal.ArmForSwitch(int32(scVal.Type)); ok {
+		var err error
+		var raw []byte
+		var jsonMessage json.RawMessage
 		serializedData["type"] = scValTypeName
 		serializedDataDecoded["type"] = scValTypeName
-		if raw, err := scVal.MarshalBinary(); err == nil {
-			serializedData["value"] = base64.StdEncoding.EncodeToString(raw)
-			serializedDataDecoded["value"] = scVal.String()
+		raw, err = scVal.MarshalBinary()
+		if err != nil {
+			return nil, nil, err
 		}
+
+		serializedData["value"] = base64.StdEncoding.EncodeToString(raw)
+		jsonMessage, err = xdr2json.ConvertBytes(xdr.ScVal{}, raw)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		serializedDataDecoded["value"] = string(jsonMessage)
 	}
 
-	return serializedData, serializedDataDecoded
+	return serializedData, serializedDataDecoded, nil
 }
 
 // TODO this should also be used in the operations processor
-func serializeScValArray(scVals []xdr.ScVal) ([]map[string]string, []map[string]string) {
+func serializeScValArray(scVals []xdr.ScVal) ([]map[string]string, []map[string]string, error) {
 	data := make([]map[string]string, 0, len(scVals))
 	dataDecoded := make([]map[string]string, 0, len(scVals))
 
 	for _, scVal := range scVals {
-		serializedData, serializedDataDecoded := serializeScVal(scVal)
+		serializedData, serializedDataDecoded, err := serializeScVal(scVal)
+		if err != nil {
+			return nil, nil, err
+		}
 		data = append(data, serializedData)
 		dataDecoded = append(dataDecoded, serializedDataDecoded)
 	}
 
-	return data, dataDecoded
+	return data, dataDecoded, nil
 }

internal/transform/contract_events_test.go

@@ -57,7 +57,7 @@ func makeContractEventTestOutput() (output [][]ContractEventOutput, err error) {
 	topicsDecoded["topics_decoded"] = []map[string]string{
 		{
 			"type":  "B",
-			"value": "true",
+			"value": "{\"B\":true,\"Type\":0}",
 		},
 	}
 
@@ -68,7 +68,7 @@ func makeContractEventTestOutput() (output [][]ContractEventOutput, err error) {
 
 	dataDecoded := map[string]string{
 		"type":  "B",
-		"value": "true",
+		"value": "{\"B\":true,\"Type\":0}",
 	}
 
 	output = [][]ContractEventOutput{{

internal/xdr2json/conversion.go

@@ -0,0 +1,90 @@
+//nolint:lll
+package xdr2json
+
+/*
+// See preflight.go for add'l explanations:
+// Note: no blank lines allowed.
+#include <stdlib.h>
+#include "../../lib/xdr2json.h"
+#cgo windows,amd64 LDFLAGS: -L${SRCDIR}/../../target/x86_64-pc-windows-gnu/release-with-panic-unwind/ -lxdr2json -lntdll -static -lws2_32 -lbcrypt -luserenv
+#cgo darwin,amd64  LDFLAGS: -L${SRCDIR}/../../target/x86_64-apple-darwin/release-with-panic-unwind/ -lxdr2json -ldl -lm
+#cgo darwin,arm64  LDFLAGS: -L${SRCDIR}/../../target/aarch64-apple-darwin/release-with-panic-unwind/ -lxdr2json -ldl -lm
+#cgo linux,amd64   LDFLAGS: -L${SRCDIR}/../../target/x86_64-unknown-linux-gnu/release-with-panic-unwind/ -lxdr2json -ldl -lm
+#cgo linux,arm64   LDFLAGS: -L${SRCDIR}/../../target/aarch64-unknown-linux-gnu/release-with-panic-unwind/ -lxdr2json -ldl -lm
+*/
+import "C"
+
+import (
+	"encoding"
+	"encoding/json"
+	"reflect"
+	"unsafe"
+
+	"github.com/pkg/errors"
+)
+
+// ConvertBytes takes an XDR object (`xdr`) and its serialized bytes (`field`)
+// and returns the raw JSON-formatted serialization of that object.
+// It can be unmarshalled to a proper JSON structure, but the raw bytes are
+// returned to avoid unnecessary round-trips. If there is an
+// error, it returns an empty string.
+//
+// The `xdr` object does not need to actually be initialized/valid:
+// we only use it to determine the name of the structure. We could just
+// accept a string, but that would make mistakes likelier than passing the
+// structure itself (by reference).
+func ConvertBytes(xdr interface{}, field []byte) (json.RawMessage, error) {
+	if len(field) == 0 {
+		return []byte(""), nil
+	}
+
+	xdrTypeName := reflect.TypeOf(xdr).Name()
+	return convertAnyBytes(xdrTypeName, field)
+}
+
+// ConvertInterface takes a valid XDR object (`xdr`) and returns
+// the raw JSON-formatted serialization of that object. If there is an
+// error, it returns an empty string.
+//
+// Unlike `ConvertBytes`, the value here needs to be valid and
+// serializable.
+func ConvertInterface(xdr encoding.BinaryMarshaler) (json.RawMessage, error) {
+	xdrTypeName := reflect.TypeOf(xdr).Name()
+	data, err := xdr.MarshalBinary()
+	if err != nil {
+		return []byte(""), errors.Wrapf(err, "failed to serialize XDR type '%s'", xdrTypeName)
+	}
+
+	return convertAnyBytes(xdrTypeName, data)
+}
+
+func convertAnyBytes(xdrTypeName string, field []byte) (json.RawMessage, error) {
+	var jsonStr, errStr string
+	// scope just added to show matching alloc/frees
+	{
+		goRawXdr := CXDR(field)
+		b := C.CString(xdrTypeName)
+
+		result := C.xdr_to_json(b, goRawXdr)
+		C.free(unsafe.Pointer(b))
+
+		jsonStr = C.GoString(result.json)
+		errStr = C.GoString(result.error)
+
+		C.free_conversion_result(result)
+	}
+
+	if errStr != "" {
+		return json.RawMessage(jsonStr), errors.New(errStr)
+	}
+
+	return json.RawMessage(jsonStr), nil
+}
+
+// CXDR is ripped directly from preflight.go to avoid a dependency.
+func CXDR(xdr []byte) C.xdr_t {
+	return C.xdr_t{
+		xdr: (*C.uchar)(C.CBytes(xdr)),
+		len: C.size_t(len(xdr)),
+	}
+}

internal/xdr2json/conversion_test.go

@@ -0,0 +1,47 @@
+package xdr2json
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/stellar/go/keypair"
+	"github.com/stellar/go/xdr"
+)
+
+func TestConversion(t *testing.T) {
+	// Make a structure to encode
+	pubkey := keypair.MustRandom()
+	asset := xdr.MustNewCreditAsset("ABCD", pubkey.Address())
+
+	// Try the all-inclusive version
+	jsi, err := ConvertInterface(asset)
+	require.NoError(t, err)
+
+	// Try the byte-and-interface version
+	rawBytes, err := asset.MarshalBinary()
+	require.NoError(t, err)
+	jsb, err := ConvertBytes(xdr.Asset{}, rawBytes)
+	require.NoError(t, err)
+
+	for _, rawJs := range []json.RawMessage{jsi, jsb} {
+		var dest map[string]interface{}
+		require.NoError(t, json.Unmarshal(rawJs, &dest))
+
+		require.Contains(t, dest, "credit_alphanum4")
+		require.Contains(t, dest["credit_alphanum4"], "asset_code")
+		require.Contains(t, dest["credit_alphanum4"], "issuer")
+		require.IsType(t, map[string]interface{}{}, dest["credit_alphanum4"])
+		if converted, ok := dest["credit_alphanum4"].(map[string]interface{}); assert.True(t, ok) {
+			require.Equal(t, pubkey.Address(), converted["issuer"])
+		}
+	}
+}
+
+func TestEmptyConversion(t *testing.T) {
+	js, err := ConvertBytes(xdr.SorobanTransactionData{}, []byte{})
+	require.NoError(t, err)
+	require.Equal(t, "", string(js))
+}

lib/ffi/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "ffi"
+version = "22.0.0"
+publish = false
+edition = "2021"
+
+[lib]
+crate-type = ["lib"]
+
+[dependencies]
+libc = { workspace = true }