feat: merkl report

fix: fetch merkl-v2 data fix: usage of public buckets fixes cleaning fix fix: rmv error
AngleProtocol · Feb 16, 2024 · 6dd713a · 6dd713a
1 parent 1599074
commit 6dd713a
Show file tree

Hide file tree

Showing 14 changed files with 170 additions and 495 deletions.
diff --git a/package.json b/package.json
@@ -26,7 +26,6 @@
     "@types/chai": "^4.3.6",
     "@types/mocha": "^10.0.1",
     "axios": "^0.27.2",
-    "chai": "^4.3.8",
     "chalk": "^4.1.0",
     "commander": "^11.0.0",
     "discord.js": "^14.7.1",
@@ -49,7 +48,6 @@
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-prettier": "^4.0.0",
     "eslint-plugin-simple-import-sort": "^7.0.0",
-    "mocha": "^10.2.0",
     "nodemon": "^2.0.22",
     "prettier": "^2.4.1",
     "ts-node": "^10.4.0",

diff --git a/scripts/templates/cloudrun.yaml b/scripts/templates/cloudrun.yaml
@@ -21,6 +21,8 @@ spec:
               value: prod
             - name: BOT_NAME
               value: CHANGE_ME # spec.template.spec.containers.env[2].value
+            - name: MERKL_API_URL
+              value: 'https://api.angle.money/v3'
             - name: DISPUTE_BOT_PRIVATE_KEY
               valueFrom:
                 secretKeyRef:

diff --git a/src/bot/context.ts b/src/bot/context.ts
@@ -4,9 +4,6 @@ import ConsoleLogger from '../helpers/logger/ConsoleLogger';
 import DiscordWebhookLogger from '../helpers/logger/DiscordWebhookLogger';
 import Logger from '../helpers/logger/Logger';
 import Loggers from '../helpers/logger/Loggers';
-import GithubRootsProvider from '../providers/merkl-roots/GithubRootsProvider';
-import GoogleRootsProvider from '../providers/merkl-roots/GoogleRootsProvider';
-import MerkleRootsProvider from '../providers/merkl-roots/MerkleRootsProvider';
 import OnChainProvider from '../providers/on-chain/OnChainProvider';
 import RpcProvider from '../providers/on-chain/RpcProvider';
 

diff --git a/src/bot/runner.ts b/src/bot/runner.ts
@@ -1,19 +1,14 @@
 import { HOUR, MerklChainId } from '@angleprotocol/sdk';
-import { getAddress } from 'ethers/lib/utils';
 import moment from 'moment';
 
-import { ALLOWED_OVER_CLAIM, NULL_ADDRESS } from '../constants';
+import { NULL_ADDRESS } from '../constants';
 import { ALERTING_DELAY } from '../constants/alertingDelay';
-import { buildMerklTree } from '../helpers';
-import createDiffTable from '../helpers/diffTable';
+import { BaseTree } from '../providers/tree';
 import { BotError, MerklReport, Resolver, Result, Step, StepResult } from '../types/bot';
-import { HoldersReport } from '../types/holders';
+import { gtStrings } from '../utils/addString';
+import { fetchCampaigns, fetchLeaves } from '../utils/merklAPI';
 import { DisputeContext } from './context';
 import { approveDisputeStake, createSigner, disputeTree } from './dispute';
-import { validateClaims, validateHolders } from './validity';
-import { BaseTree, ExpandedLeaf } from '../providers/tree';
-import { fetchCampaigns, fetchLeaves } from '../utils/merklAPI';
-import { gtStrings } from '../utils/addString';
 
 export const checkBlockTime: Step = async (context, report) => {
   try {
@@ -36,7 +31,7 @@ export const checkOnChainParams: Step = async ({ onChainProvider, logger }, repo
 
     logger?.onChainParams(params, report.startTime);
 
-    return Result.Success({ ...report, params});
+    return Result.Success({ ...report, params });
   } catch (err) {
     console.error(err);
     return Result.Error({ code: BotError.OnChainFetch, reason: `Unable to get on-chain params: ${err}`, report });
@@ -73,13 +68,17 @@ export const checkTrees: Step = async ({ logger }, report) => {
 
     const startRoot = params.startRoot;
     const endRoot = params.endRoot;
-    
+
     const startLeaves = await fetchLeaves(chainId, startRoot);
     const startTree = new BaseTree(startLeaves, chainId as MerklChainId);
 
+    startTree.buildMerklTree();
+
     const endLeaves = await fetchLeaves(chainId, endRoot);
     const endTree = new BaseTree(endLeaves, chainId as MerklChainId);
 
+    endTree.buildMerklTree();
+
     return Result.Success({ ...report, startTree, endTree, startRoot, endRoot });
   } catch (err) {
     return Result.Error({ code: BotError.TreeFetch, reason: `Unable to get trees: ${err}`, report });
@@ -102,31 +101,41 @@ export const checkRoots: Step = async ({ logger }, report) => {
   }
 };
 
-export const checkOverDistribution: Step = async ({ onChainProvider }, report) => {
+export const checkOverDistribution: Step = async ({}, report) => {
   const { chainId, startTree, endTree } = report;
 
   try {
     const campaigns = await fetchCampaigns(chainId);
-  
+
     const { diffCampaigns, diffRecipients, negativeDiffs } = BaseTree.computeDiff(startTree, endTree, campaigns);
 
     if (negativeDiffs.length > 0) {
-      return Result.Error({ code: BotError.NegativeDiff, reason: negativeDiffs.join('\n'), report: { ...report, diffCampaigns, diffRecipients } });
+      return Result.Error({
+        code: BotError.NegativeDiff,
+        reason: negativeDiffs.join('\n'),
+        report: { ...report, diffCampaigns, diffRecipients },
+      });
     }
 
     const overDistributed = [];
     for (const diffCampaign of diffCampaigns) {
       if (gtStrings(diffCampaign.total, campaigns[diffCampaign.campaignId].amount)) {
         overDistributed.push(
-          `${diffCampaign.campaignId} - Distributed (${diffCampaign.total}) > Total (${campaigns[diffCampaign.campaignId].amount})`);
+          `${diffCampaign.campaignId} - Distributed (${diffCampaign.total}) > Total (${campaigns[diffCampaign.campaignId].amount})`
+        );
       }
     }
     if (overDistributed.length > 0) {
-      return Result.Error({ code: BotError.OverDistributed, reason: overDistributed.join('\n'), report: { ...report, diffCampaigns, diffRecipients } });
+      return Result.Error({
+        code: BotError.OverDistributed,
+        reason: overDistributed.join('\n'),
+        report: { ...report, diffCampaigns, diffRecipients },
+      });
     }
 
     return Result.Success({ ...report, diffCampaigns, diffRecipients });
   } catch (reason) {
+    console.log(reason);
     return Result.Error({ code: BotError.NegativeDiff, reason, report: { ...report } });
   }
 };
@@ -181,14 +190,7 @@ export const checkOverDistribution: Step = async ({ onChainProvider }, report) =
 
 export async function runSteps(
   context: DisputeContext,
-  steps: Step[] = [
-    checkBlockTime,
-    checkOnChainParams,
-    checkDisputeWindow,
-    checkTrees,
-    checkRoots,
-    checkOverDistribution,
-  ],
+  steps: Step[] = [checkBlockTime, checkOnChainParams, checkDisputeWindow, checkTrees, checkRoots, checkOverDistribution],
   report: MerklReport = {}
 ): Promise<StepResult> {
   return new Promise(async function (resolve: Resolver) {
@@ -205,7 +207,12 @@ export async function runSteps(
 
     for (let i = 0; i < steps.length && !resolved; i++) await handleStep(steps[i]);
 
-    resolve(Result.Exit({ reason: 'No problemo', report }));
+    resolve(
+      Result.Exit({
+        reason: `No problem detected. Report at https://storage.cloud.google.com/merkl-production-reports/${context.chainId}/${report.endRoot}.html`,
+        report,
+      })
+    );
   });
 }
 
@@ -215,14 +222,7 @@ export default async function run(context: DisputeContext) {
 
   const checkUpResult = await runSteps(
     context,
-    [
-      checkBlockTime,
-      checkOnChainParams,
-      checkDisputeWindow,
-      checkTrees,
-      checkRoots,
-      checkOverDistribution,
-    ],
+    [checkBlockTime, checkOnChainParams, checkDisputeWindow, checkTrees, checkRoots, checkOverDistribution],
     report
   );
 

diff --git a/src/bot/validity.ts b/src/bot/validity.ts
@@ -150,7 +150,7 @@ export async function validateHolders(
 
 export async function validateClaims(onChainProvider: OnChainProvider, holdersReport: HoldersReport): Promise<HoldersReport> {
   const { details, unclaimed } = holdersReport;
-  const alreadyClaimed: HolderClaims = await onChainProvider.fetchClaimed(details);
+  // const alreadyClaimed: HolderClaims = await onChainProvider.fetchClaimed(details);
 
   const overclaimed: string[] = [];
 
@@ -161,21 +161,21 @@ export async function validateClaims(onChainProvider: OnChainProvider, holdersRe
         a.poolName > b.poolName ? 1 : b.poolName > a.poolName ? -1 : a.percent > b.percent ? -1 : b.percent > a.percent ? 1 : 0
       )
       .map(async (d) => {
-        const alreadyClaimedValue = round(Int256.from(alreadyClaimed[d.holder][d.tokenAddress], d.decimals).toNumber(), 2);
+        // const alreadyClaimedValue = round(Int256.from(alreadyClaimed[d.holder][d.tokenAddress], d.decimals).toNumber(), 2);
         const totalCumulated = round(unclaimed[d.holder][d.symbol].toNumber(), 2);
 
-        if (totalCumulated < alreadyClaimedValue) {
-          overclaimed.push(`${d.holder}: ${alreadyClaimedValue} / ${totalCumulated} ${d.symbol}`);
-        }
+        // if (totalCumulated < alreadyClaimedValue) {
+        //   overclaimed.push(`${d.holder}: ${alreadyClaimedValue} / ${totalCumulated} ${d.symbol}`);
+        // }
         return {
           ...d,
           diff: round(d.diff, 4),
           percent: round(d.percent, 4),
           averageBoost: round(d.diffAverageBoost, 2),
           distribution: d.distribution.slice(0, 5),
           totalCumulated,
-          alreadyClaimed: alreadyClaimedValue,
-          issueSpotted: totalCumulated < alreadyClaimedValue,
+          // alreadyClaimed: alreadyClaimedValue,
+          // issueSpotted: totalCumulated < alreadyClaimedValue,
         };
       })
   );

diff --git a/src/compareFiles.ts b/src/compareFiles.ts
@@ -0,0 +1,26 @@
+import fs from 'fs';
+
+import { DisputeContext } from './bot/context';
+import { validateClaims, validateHolders } from './bot/validity';
+import createDiffTable from './helpers/diffTable';
+import ConsoleLogger from './helpers/logger/ConsoleLogger';
+
+export default async function (context: DisputeContext) {
+  const { onChainProvider } = context;
+  const logger = new ConsoleLogger();
+
+  // const startTree = JSON.parse(fs.readFileSync('old_polygon.json', 'utf8'));
+  // const endTree = JSON.parse(fs.readFileSync('new_polygon.json', 'utf8'));
+
+  // logger.trees(0, startTree, 0, endTree);
+
+  // const endRoot = buildMerklTree(endTree.rewards).getHexRoot();
+  // const startRoot = buildMerklTree(startTree.rewards).getHexRoot();
+
+  // logger.computedRoots(startRoot, endRoot);
+
+  // const holdersReport = await validateClaims(onChainProvider, await validateHolders(onChainProvider, startTree, endTree));
+
+  // const res = await createDiffTable(holdersReport.details, holdersReport.changePerDistrib, !context.uploadDiffTable);
+  // context.uploadDiffTable && console.log('output:', res);
+}
diff --git a/src/constants/index.ts b/src/constants/index.ts
@@ -14,3 +14,5 @@ export const ALLOWED_OVER_CLAIM = [
   '0x3f9763cE4F230368437f45CE81Be598c253Db338',
   '0x2A6Be69cd729288006f831737D5032f15626d52c',
 ];
+
+export const MERKL_API_URL = process.env.MERKL_API_URL ? process.env.MERKL_API_URL : 'https://api.angle.money/v3';
diff --git a/src/diff.ts b/src/diff.ts
@@ -1,11 +1,13 @@
+import { MerklChainId } from '@angleprotocol/sdk';
+import axios from 'axios';
 import moment from 'moment';
 
 import { DisputeContext } from './bot/context';
-import { validateClaims, validateHolders } from './bot/validity';
-import { buildMerklTree } from './helpers';
-import createDiffTable from './helpers/diffTable';
+import { MERKL_API_URL } from './constants';
 import ConsoleLogger from './helpers/logger/ConsoleLogger';
 import blockFromTimestamp from './providers/blockNumberFromTimestamp';
+import { BaseTree } from './providers/tree';
+import { fetchCampaigns, fetchLeaves } from './utils/merklAPI';
 
 export default async function (context: DisputeContext, fromTimeStamp: number, toTimeStamp: number) {
   const { onChainProvider } = context;
@@ -20,22 +22,27 @@ export default async function (context: DisputeContext, fromTimeStamp: number, t
   console.log(`Using block ${endBlock} as onchain reference`);
 
   onChainProvider.setBlock(endBlock);
+  const startRootData = (await axios.get(`${MERKL_API_URL}/rootForTimestamp?chainId=${context.chainId}&timestamp=${fromTimeStamp}`)).data;
+  const endRootData = (await axios.get(`${MERKL_API_URL}/rootForTimestamp?chainId=${context.chainId}&timestamp=${toTimeStamp}`)).data;
 
-  // TODO - update this job to use the new logic (need to get the roots from the API)
-  // const startEpoch = await merkleRootsProvider.epochFromTimestamp(fromTimeStamp);
-  // const endEpoch = await merkleRootsProvider.epochFromTimestamp(toTimeStamp);
-  // const startTree = await merkleRootsProvider.fetchTreeFor(startEpoch);
-  // const endTree = await merkleRootsProvider.fetchTreeFor(endEpoch);
+  const startLeaves = await fetchLeaves(context.chainId, startRootData.root);
+  const startTree = new BaseTree(startLeaves, context.chainId as MerklChainId);
+
+  const endLeaves = await fetchLeaves(context.chainId, endRootData.root);
+  const endTree = new BaseTree(endLeaves, context.chainId as MerklChainId);
 
   // logger.trees(startEpoch, startTree, endEpoch, endTree);
 
-  // const endRoot = buildMerklTree(endTree.rewards).tree.getHexRoot();
-  // const startRoot = buildMerklTree(startTree.rewards).tree.getHexRoot();
+  const endRoot = startTree.merklRoot();
+  const startRoot = endTree.merklRoot();
+
+  logger.computedRoots(startRoot, endRoot);
 
-  // logger.computedRoots(startRoot, endRoot);
+  const campaigns = await fetchCampaigns(context.chainId);
 
-  // const holdersReport = await validateClaims(onChainProvider, await validateHolders(onChainProvider, startTree, endTree));
+  const { diffCampaigns, diffRecipients, negativeDiffs } = BaseTree.computeDiff(startTree, endTree, campaigns);
 
-  // const res = await createDiffTable(holdersReport.details, holdersReport.changePerDistrib, !context.uploadDiffTable);
-  // context.uploadDiffTable && console.log('output:', res);
+  console.log('diffCampaigns:', diffCampaigns);
+  console.log('diffRecipients:', diffRecipients);
+  console.log('negativeDiffs:', negativeDiffs);
 }
diff --git a/src/helpers/index.ts b/src/helpers/index.ts
@@ -1,17 +1,6 @@
-import {
-  AggregatedRewardsType,
-  AMM,
-  AMMAlgorithm,
-  AMMAlgorithmMapping,
-  Erc20__factory,
-  Multicall__factory,
-  UnderlyingTreeType,
-} from '@angleprotocol/sdk';
-import { BigNumber, ethers, utils } from 'ethers';
-import keccak256 from 'keccak256';
-import MerkleTree from 'merkletreejs';
+import { AMM, AMMAlgorithm, AMMAlgorithmMapping, Erc20__factory, Multicall__factory } from '@angleprotocol/sdk';
 
-import { MERKL_TREE_OPTIONS, MULTICALL_ADDRESS } from '../constants';
+import { MULTICALL_ADDRESS } from '../constants';
 import { httpProvider } from '../providers';
 import { PoolInterface } from '../types';
 
@@ -70,66 +59,3 @@ export const fetchPoolName = async (chainId: number, pool: string, amm: AMM) =>
 };
 
 export const round = (n: number, dec: number) => Math.round(n * 10 ** dec) / 10 ** dec;
-
-export const buildMerklTree = (
-  underylingTreeData: UnderlyingTreeType
-): {
-  tree: MerkleTree;
-  tokens: string[];
-} => {
-  /**
-   * 1 - Build the global list of users
-   */
-  const users: string[] = [];
-  for (const id of Object.keys(underylingTreeData)) {
-    const rewardUsers = Object.keys(underylingTreeData[id].holders);
-    for (const r of rewardUsers) {
-      if (!users.includes(r)) {
-        users.push(r);
-      }
-    }
-  }
-
-  /**
-   * 2 - Build the global list of tokens
-   */
-  const tokens: string[] = tokensFromTree(underylingTreeData);
-
-  /**
-   * 3 - Build the tree
-   */
-  const leaves = [];
-  for (const u of users) {
-    for (const t of tokens) {
-      let sum = BigNumber.from(0);
-      for (const id of Object.keys(underylingTreeData)) {
-        const distribution = underylingTreeData[id];
-        if (distribution.token === t) {
-          sum = sum?.add(distribution?.holders[u]?.amount.toString() ?? 0);
-        }
-      }
-      if (!!sum && sum.gt(0)) {
-        const hash = ethers.utils.keccak256(
-          ethers.utils.defaultAbiCoder.encode(['address', 'address', 'uint256'], [utils.getAddress(u), t, sum])
-        );
-        leaves.push(hash);
-      }
-    }
-  }
-  const tree = new MerkleTree(leaves, keccak256, MERKL_TREE_OPTIONS);
-
-  return {
-    tokens,
-    tree,
-  };
-};
-
-export const tokensFromTree = (json: AggregatedRewardsType['rewards']): string[] => {
-  const tokens: string[] = [];
-  for (const id of Object.keys(json)) {
-    if (!tokens.includes(json[id].token)) {
-      tokens.push(json[id].token);
-    }
-  }
-  return tokens;
-};