ui: arb + market maker ui (decred#2615)

* ui: arb + market maker ui

Implement arbmm ui. This required a fair amount refactoring on
both the front and back ends. For the back end, the primary
changes are about when and how cexes are loaded. Since we need
to know what markets are available before the bot it started,
we need to call Markets() on cex initialization, before the CEX
is running. This occurs either during startup (MarketMaker is
now a dex.Connector), or when the cex is added (through the new
UpdateCEXConfig method).

In order to facilitate ui testing, I've made the MarketMaker an
interface for use by WebServer (like clientCore), and then
implemented the interface in live_test.go.

On the front end, most effort is focused around mmsettings, and
there was a lot of refactoring done to facilitate the changes
and clean things up.

.gitignore

@@ -39,5 +39,6 @@ client/asset/btc/electrum/example/wallet/wallet
 client/asset/eth/cmd/getgas/getgas
 client/asset/eth/cmd/deploy/deploy
 client/cmd/dexc-desktop/pkg/installers
+client/cmd/testbinance/testbinance
 server/noderelay/cmd/sourcenode/sourcenode
 tatanka/cmd/demo/demo

client/app/config.go

@@ -211,15 +211,6 @@ func (cfg *Config) Core(log dex.Logger) *core.Config {
 	}
 }
 
-// MarketMakerConfigPath returns the path to the market maker config file.
-func (cfg *Config) MarketMakerConfigPath() string {
-	if cfg.MMConfig.BotConfigPath != "" {
-		return cfg.MMConfig.BotConfigPath
-	}
-	_, _, mmCfgPath := setNet(cfg.AppData, cfg.Net.String())
-	return mmCfgPath
-}
-
 var DefaultConfig = Config{
 	AppData:    defaultApplicationDirectory,
 	ConfigPath: defaultConfigPath,
@@ -303,17 +294,17 @@ func ResolveConfig(appData string, cfg *Config) error {
 
 	cfg.AppData = appData
 
-	var defaultDBPath, defaultLogPath string
+	var defaultDBPath, defaultLogPath, defaultMMConfigPath string
 	switch {
 	case cfg.Testnet:
 		cfg.Net = dex.Testnet
-		defaultDBPath, defaultLogPath, _ = setNet(appData, "testnet")
+		defaultDBPath, defaultLogPath, defaultMMConfigPath = setNet(appData, "testnet")
 	case cfg.Simnet:
 		cfg.Net = dex.Simnet
-		defaultDBPath, defaultLogPath, _ = setNet(appData, "simnet")
+		defaultDBPath, defaultLogPath, defaultMMConfigPath = setNet(appData, "simnet")
 	default:
 		cfg.Net = dex.Mainnet
-		defaultDBPath, defaultLogPath, _ = setNet(appData, "mainnet")
+		defaultDBPath, defaultLogPath, defaultMMConfigPath = setNet(appData, "mainnet")
 	}
 	defaultHost := DefaultHostByNetwork(cfg.Net)
 
@@ -342,6 +333,10 @@ func ResolveConfig(appData string, cfg *Config) error {
 		cfg.LogPath = defaultLogPath
 	}
 
+	if cfg.MMConfig.BotConfigPath == "" {
+		cfg.MMConfig.BotConfigPath = defaultMMConfigPath
+	}
+
 	if cfg.ReloadHTML {
 		fmt.Println("The --reload-html switch is deprecated. Use --no-embed-site instead, which has the same reloading effect.")
 		cfg.NoEmbedSite = cfg.ReloadHTML

client/asset/btc/spv_wrapper.go

@@ -395,8 +395,10 @@ func (w *spvWallet) sendRawTransaction(tx *wire.MsgTx) (*chainhash.Hash, error)
 			res <- err
 			return
 		}
-		defer w.log.Tracef("PublishTransaction(%v) completed in %v", tx.TxHash(),
-			time.Since(tStart)) // after outpoint unlocking and signalling
+		defer func() {
+			w.log.Tracef("PublishTransaction(%v) completed in %v", tx.TxHash(),
+				time.Since(tStart)) // after outpoint unlocking and signalling
+		}()
 		res <- nil
 	}()
 

client/cmd/dexc-desktop/app.go

@@ -184,10 +184,18 @@ func mainCore() error {
 	if cfg.Experimental {
 		// TODO: on shutdown, stop market making and wait for trades to be
 		// canceled.
-		marketMaker, err = mm.NewMarketMaker(clientCore, cfg.MarketMakerConfigPath(), logMaker.Logger("MM"))
+		marketMaker, err = mm.NewMarketMaker(clientCore, cfg.BotConfigPath, logMaker.Logger("MM"))
 		if err != nil {
 			return fmt.Errorf("error creating market maker: %w", err)
 		}
+		cm := dex.NewConnectionMaster(marketMaker)
+		if err := cm.ConnectOnce(appCtx); err != nil {
+			return fmt.Errorf("error connecting market maker")
+		}
+		defer func() {
+			cancel()
+			cm.Wait()
+		}()
 	}
 
 	if cfg.RPCOn {

client/cmd/dexc-desktop/app_darwin.go

@@ -244,13 +244,22 @@ func mainCore() error {
 	<-clientCore.Ready()
 
 	var marketMaker *mm.MarketMaker
+	var mmCM *dex.ConnectionMaster
 	if cfg.Experimental {
 		// TODO: on shutdown, stop market making and wait for trades to be
 		// canceled.
-		marketMaker, err = mm.NewMarketMaker(clientCore, cfg.MarketMakerConfigPath(), logMaker.Logger("MM"))
+		marketMaker, err = mm.NewMarketMaker(clientCore, cfg.BotConfigPath, logMaker.Logger("MM"))
 		if err != nil {
 			return fmt.Errorf("error creating market maker: %w", err)
 		}
+		cm := dex.NewConnectionMaster(marketMaker)
+		if err := cm.ConnectOnce(appCtx); err != nil {
+			return fmt.Errorf("error connecting market maker")
+		}
+		defer func() {
+			cancel()
+			cm.Wait()
+		}()
 	}
 
 	if cfg.RPCOn {

client/cmd/dexc/main.go

@@ -101,7 +101,7 @@ func runCore(cfg *app.Config) error {
 	if cfg.Experimental {
 		// TODO: on shutdown, stop market making and wait for trades to be
 		// canceled.
-		marketMaker, err = mm.NewMarketMaker(clientCore, cfg.MarketMakerConfigPath(), logMaker.Logger("MM"))
+		marketMaker, err = mm.NewMarketMaker(clientCore, cfg.MMConfig.BotConfigPath, logMaker.Logger("MM"))
 		if err != nil {
 			return fmt.Errorf("error creating market maker: %w", err)
 		}
@@ -131,12 +131,23 @@ func runCore(cfg *app.Config) error {
 
 	<-clientCore.Ready()
 
+	var mmCM *dex.ConnectionMaster
 	defer func() {
 		log.Info("Exiting dexc main.")
 		cancel()  // no-op with clean rpc/web server setup
 		wg.Wait() // no-op with clean setup and shutdown
+		if mmCM != nil {
+			mmCM.Wait()
+		}
 	}()
 
+	if marketMaker != nil {
+		mmCM = dex.NewConnectionMaster(marketMaker)
+		if err := mmCM.ConnectOnce(appCtx); err != nil {
+			return fmt.Errorf("Error connecting market maker")
+		}
+	}
+
 	if cfg.RPCOn {
 		rpcSrv, err := rpcserver.New(cfg.RPC(clientCore, marketMaker, logMaker.Logger("RPC")))
 		if err != nil {

client/cmd/testbinance/main.go

@@ -112,6 +112,14 @@ func runServer() error {
 				free:   1000.2358249,
 				locked: 0,
 			},
+			"zec": {
+				free:   1000.2358249,
+				locked: 0,
+			},
+			"polygon": {
+				free:   1000.2358249,
+				locked: 0,
+			},
 		},
 		withdrawalHistory: make([]*transfer, 0),
 		depositHistory:    make([]*transfer, 0),

client/mm/config.go

@@ -17,6 +17,16 @@ type MarketMakingConfig struct {
 	CexConfigs []*CEXConfig `json:"cexConfigs"`
 }
 
+func (cfg *MarketMakingConfig) Copy() *MarketMakingConfig {
+	c := &MarketMakingConfig{
+		BotConfigs: make([]*BotConfig, len(cfg.BotConfigs)),
+		CexConfigs: make([]*CEXConfig, len(cfg.CexConfigs)),
+	}
+	copy(c.BotConfigs, cfg.BotConfigs)
+	copy(c.CexConfigs, cfg.CexConfigs)
+	return c
+}
+
 // CEXConfig is a configuration for connecting to a CEX API.
 type CEXConfig struct {
 	// Name is the name of the cex.
@@ -41,11 +51,13 @@ type BotCEXCfg struct {
 // The balance fields are the initial amounts that will be reserved to use for
 // this bot. As the bot trades, the amounts reserved for it will be updated.
 type BotConfig struct {
-	Host             string      `json:"host"`
-	BaseAsset        uint32      `json:"baseAsset"`
-	QuoteAsset       uint32      `json:"quoteAsset"`
-	BaseBalanceType  BalanceType `json:"baseBalanceType"`
-	BaseBalance      uint64      `json:"baseBalance"`
+	Host    string `json:"host"`
+	BaseID  uint32 `json:"baseID"`
+	QuoteID uint32 `json:"quoteID"`
+
+	BaseBalanceType BalanceType `json:"baseBalanceType"`
+	BaseBalance     uint64      `json:"baseBalance"`
+
 	QuoteBalanceType BalanceType `json:"quoteBalanceType"`
 	QuoteBalance     uint64      `json:"quoteBalance"`
 
@@ -55,7 +67,7 @@ type BotConfig struct {
 	// Only one of the following configs should be set
 	BasicMMConfig        *BasicMarketMakingConfig `json:"basicMarketMakingConfig,omitempty"`
 	SimpleArbConfig      *SimpleArbConfig         `json:"simpleArbConfig,omitempty"`
-	ArbMarketMakerConfig *ArbMarketMakerConfig    `json:"arbMarketMakerConfig,omitempty"`
+	ArbMarketMakerConfig *ArbMarketMakerConfig    `json:"arbMarketMakingConfig,omitempty"`
 
 	Disabled bool `json:"disabled"`
 }

client/mm/libxc/binance.go

@@ -285,6 +285,7 @@ func binanceCoinNetworkToDexSymbol(coin, network string) string {
 }
 
 type bncAssetConfig struct {
+	assetID uint32
 	// symbol is the DEX asset symbol, always lower case
 	symbol string
 	// coin is the asset symbol on binance, always upper case.
@@ -317,6 +318,7 @@ func bncAssetCfg(assetID uint32) (*bncAssetConfig, error) {
 	}
 
 	return &bncAssetConfig{
+		assetID:          assetID,
 		symbol:           symbol,
 		coin:             mapDexToBinanceSymbol(coin),
 		chain:            mapDexToBinanceSymbol(chain),
@@ -369,7 +371,7 @@ type binance struct {
 	tokenIDs atomic.Value // map[string][]uint32
 
 	balanceMtx sync.RWMutex
-	balances   map[string]*bncBalance
+	balances   map[uint32]*bncBalance
 
 	marketStreamMtx sync.RWMutex
 	marketStream    comms.WsConn
@@ -410,7 +412,7 @@ func newBinance(apiKey, secretKey string, log dex.Logger, net dex.Network, binan
 		apiKey:             apiKey,
 		secretKey:          secretKey,
 		knownAssets:        knownAssets,
-		balances:           make(map[string]*bncBalance),
+		balances:           make(map[uint32]*bncBalance),
 		books:              make(map[string]*binanceOrderBook),
 		net:                net,
 		tradeInfo:          make(map[string]*tradeInfo),
@@ -431,10 +433,19 @@ func (bnc *binance) setBalances(coinsData []*binanceCoinInfo) {
 	defer bnc.balanceMtx.Unlock()
 
 	for _, nfo := range coinsData {
-		bnc.balances[nfo.Coin] = &bncBalance{
-			available: nfo.Free,
-			locked:    nfo.Locked,
+		for _, net := range nfo.NetworkList {
+			assetID, found := dex.BipSymbolID(binanceCoinNetworkToDexSymbol(nfo.Coin, net.Coin))
+			if !found {
+				bnc.log.Tracef("no dex asset known for binance coin %q, network %q", nfo.Coin, net.Coin)
+				continue
+			}
+
+			bnc.balances[assetID] = &bncBalance{
+				available: nfo.Free,
+				locked:    nfo.Locked,
+			}
 		}
+
 	}
 }
 
@@ -492,7 +503,7 @@ func (bnc *binance) getCoinInfo(ctx context.Context) error {
 	return nil
 }
 
-func (bnc *binance) getMarkets(ctx context.Context) error {
+func (bnc *binance) getMarkets(ctx context.Context) (map[string]*binanceMarket, error) {
 	var exchangeInfo struct {
 		Timezone   string `json:"timezone"`
 		ServerTime int64  `json:"serverTime"`
@@ -506,7 +517,7 @@ func (bnc *binance) getMarkets(ctx context.Context) error {
 	}
 	err := bnc.getAPI(ctx, "/api/v3/exchangeInfo", nil, false, false, &exchangeInfo)
 	if err != nil {
-		return fmt.Errorf("error getting markets from Binance: %w", err)
+		return nil, fmt.Errorf("error getting markets from Binance: %w", err)
 	}
 
 	marketsMap := make(map[string]*binanceMarket, len(exchangeInfo.Symbols))
@@ -516,7 +527,7 @@ func (bnc *binance) getMarkets(ctx context.Context) error {
 
 	bnc.markets.Store(marketsMap)
 
-	return nil
+	return marketsMap, nil
 }
 
 // Connect connects to the binance API.
@@ -527,7 +538,7 @@ func (bnc *binance) Connect(ctx context.Context) (*sync.WaitGroup, error) {
 		return nil, fmt.Errorf("error getting coin info: %v", err)
 	}
 
-	if err := bnc.getMarkets(ctx); err != nil {
+	if _, err := bnc.getMarkets(ctx); err != nil {
 		return nil, fmt.Errorf("error getting markets: %v", err)
 	}
 
@@ -543,7 +554,7 @@ func (bnc *binance) Connect(ctx context.Context) (*sync.WaitGroup, error) {
 		for {
 			select {
 			case <-nextTick:
-				err := bnc.getMarkets(ctx)
+				_, err := bnc.getMarkets(ctx)
 				if err != nil {
 					bnc.log.Errorf("Error fetching markets: %v", err)
 					nextTick = time.After(time.Minute)
@@ -609,7 +620,7 @@ func (bnc *binance) Balance(assetID uint32) (*ExchangeBalance, error) {
 	bnc.balanceMtx.RLock()
 	defer bnc.balanceMtx.RUnlock()
 
-	bal, found := bnc.balances[assetConfig.coin]
+	bal, found := bnc.balances[assetConfig.assetID]
 	if !found {
 		return nil, fmt.Errorf("no %q balance found", assetConfig.coin)
 	}
@@ -933,11 +944,38 @@ func (bnc *binance) CancelTrade(ctx context.Context, baseID, quoteID uint32, tra
 	return bnc.requestInto(req, &struct{}{})
 }
 
-func (bnc *binance) Markets() ([]*Market, error) {
-	binanceMarkets := bnc.markets.Load().(map[string]*binanceMarket)
-	markets := make([]*Market, 0, 16)
+func (bnc *binance) Balances() (map[uint32]*ExchangeBalance, error) {
+	bnc.balanceMtx.RLock()
+	defer bnc.balanceMtx.RUnlock()
+
+	balances := make(map[uint32]*ExchangeBalance)
+
+	for assetID, bal := range bnc.balances {
+		assetConfig, err := bncAssetCfg(assetID)
+		if err != nil {
+			continue
+		}
+
+		balances[assetConfig.assetID] = &ExchangeBalance{
+			Available: uint64(bal.available * float64(assetConfig.conversionFactor)),
+			Locked:    uint64(bal.locked * float64(assetConfig.conversionFactor)),
+		}
+	}
+
+	return balances, nil
+}
+
+func (bnc *binance) Markets(ctx context.Context) (_ []*Market, err error) {
+	bnMarkets := bnc.markets.Load().(map[string]*binanceMarket)
+	if len(bnMarkets) == 0 {
+		bnMarkets, err = bnc.getMarkets(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("error getting markets: %v", err)
+		}
+	}
+	markets := make([]*Market, 0, len(bnMarkets))
 	tokenIDs := bnc.tokenIDs.Load().(map[string][]uint32)
-	for _, mkt := range binanceMarkets {
+	for _, mkt := range bnMarkets {
 		dexMarkets := binanceMarketToDexMarkets(mkt.BaseAsset, mkt.QuoteAsset, tokenIDs)
 		markets = append(markets, dexMarkets...)
 	}
@@ -1057,12 +1095,35 @@ func (bnc *binance) handleOutboundAccountPosition(update *binanceStreamUpdate) {
 		bnc.log.Tracef("balance: %+v", bal)
 	}
 
+	supportedTokens := bnc.tokenIDs.Load().(map[string][]uint32)
+
+	processSymbol := func(symbol string, bal *binanceWSBalance) {
+		if parentIDs := dex.TokenChains[symbol]; parentIDs != nil {
+			supported := supportedTokens[symbol]
+			for _, tokenID := range supported {
+				bnc.balances[tokenID] = &bncBalance{
+					available: bal.Free,
+					locked:    bal.Locked,
+				}
+			}
+			return
+		}
+		assetID, found := dex.BipSymbolID(symbol)
+		if !found {
+			return
+		}
+		bnc.balances[assetID] = &bncBalance{
+			available: bal.Free,
+			locked:    bal.Locked,
+		}
+	}
+
 	bnc.balanceMtx.Lock()
 	for _, bal := range update.Balances {
 		symbol := strings.ToLower(bal.Asset)
-		bnc.balances[symbol] = &bncBalance{
-			available: bal.Free,
-			locked:    bal.Locked,
+		processSymbol(symbol, bal)
+		if symbol == "eth" {
+			processSymbol("weth", bal)
 		}
 	}
 	bnc.balanceMtx.Unlock()