feat: Add github proxy (#4418)

### Description This PR adds github-proxy CloudFlare Worker project which will attach an api key to github requests to increase read capacity. It is mostly a passthrough proxy that has a simple allowlist. This project is created and deployed using CloudFlare's [cloudflare create CLI](https://developers.cloudflare.com/workers/get-started/guide/) It is deployed using `yarn deploy` (aka wrangler) ### Drive-by changes - `yarn up [email protected]` - `yarn up [email protected]` - `yarn up [email protected]` ### Related issues Fixes: hyperlane-xyz/hyperlane-registry#163 ### Backward compatibility Yes ### Testing Manual/Unit Tests - Use a script to ddos github, then try cli command `hyperlane warp read` - Unit tests for the Worker, and GithubRegistry changes
hyperlane-xyz · Sep 13, 2024 · 0e2f94b · 0e2f94b
1 parent 1ab8751
commit 0e2f94b
Show file tree

Hide file tree

Showing 29 changed files with 1,658 additions and 98 deletions.
diff --git a/.changeset/famous-ants-tan.md b/.changeset/famous-ants-tan.md
@@ -0,0 +1,5 @@
+---
+'github-proxy': major
+---
+
+Add github proxy to reduce github API load
diff --git a/Dockerfile b/Dockerfile
@@ -18,6 +18,7 @@ COPY typescript/cli/package.json ./typescript/cli/
 COPY typescript/infra/package.json ./typescript/infra/
 COPY typescript/ccip-server/package.json ./typescript/ccip-server/
 COPY typescript/widgets/package.json ./typescript/widgets/
+COPY typescript/github-proxy/package.json ./typescript/github-proxy/
 COPY solidity/package.json ./solidity/
 
 RUN yarn install && yarn cache clean

diff --git a/solidity/package.json b/solidity/package.json
@@ -19,7 +19,7 @@
     "@typechain/ethers-v6": "^0.5.1",
     "@typechain/hardhat": "^9.1.0",
     "@types/node": "^18.14.5",
-    "chai": "^4.3.6",
+    "chai": "4.5.0",
     "ethereum-waffle": "^4.0.10",
     "ethers": "^5.7.2",
     "hardhat": "^2.22.2",

diff --git a/typescript/cli/cli.ts b/typescript/cli/cli.ts
@@ -12,6 +12,7 @@ import { deployCommand } from './src/commands/deploy.js';
 import { hookCommand } from './src/commands/hook.js';
 import { ismCommand } from './src/commands/ism.js';
 import {
+  disableProxyCommandOption,
   keyCommandOption,
   logFormatCommandOption,
   logLevelCommandOption,
@@ -46,6 +47,7 @@ try {
     .option('registry', registryUriCommandOption)
     .option('overrides', overrideRegistryUriCommandOption)
     .option('key', keyCommandOption)
+    .option('disableProxy', disableProxyCommandOption)
     .option('yes', skipConfirmationOption)
     .global(['log', 'verbosity', 'registry', 'overrides', 'yes'])
     .middleware([

diff --git a/typescript/cli/package.json b/typescript/cli/package.json
@@ -5,7 +5,7 @@
   "dependencies": {
     "@aws-sdk/client-kms": "^3.577.0",
     "@aws-sdk/client-s3": "^3.577.0",
-    "@hyperlane-xyz/registry": "2.5.0",
+    "@hyperlane-xyz/registry": "4.0.0",
     "@hyperlane-xyz/sdk": "5.1.0",
     "@hyperlane-xyz/utils": "5.1.0",
     "@inquirer/prompts": "^3.0.0",
@@ -16,7 +16,7 @@
     "latest-version": "^8.0.0",
     "terminal-link": "^3.0.0",
     "tsx": "^4.7.1",
-    "yaml": "^2.4.1",
+    "yaml": "2.4.5",
     "yargs": "^17.7.2",
     "zod": "^3.21.2",
     "zod-validation-error": "^3.3.0",
@@ -30,12 +30,12 @@
     "@types/yargs": "^17.0.24",
     "@typescript-eslint/eslint-plugin": "^7.4.0",
     "@typescript-eslint/parser": "^7.4.0",
-    "chai": "^4.3.6",
+    "chai": "4.5.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "mocha": "^10.2.0",
     "prettier": "^2.8.8",
-    "typescript": "^5.1.6"
+    "typescript": "5.3.3"
   },
   "scripts": {
     "hyperlane": "node ./dist/cli.js",

diff --git a/typescript/cli/src/commands/options.ts b/typescript/cli/src/commands/options.ts
@@ -54,6 +54,13 @@ export const keyCommandOption: Options = {
   defaultDescription: 'process.env.HYP_KEY',
 };
 
+export const disableProxyCommandOption: Options = {
+  type: 'boolean',
+  description:
+    'Disable routing of Github API requests through the Hyperlane registry proxy.',
+  default: false,
+};
+
 /* Command-specific options */
 
 export const coreTargetsCommandOption: Options = {

diff --git a/typescript/cli/src/consts.ts b/typescript/cli/src/consts.ts
@@ -2,3 +2,4 @@ export const MINIMUM_CORE_DEPLOY_GAS = (1e8).toString();
 export const MINIMUM_WARP_DEPLOY_GAS = (6e8).toString(); // Rough calculation through deployments to testnets with 2x buffer
 export const MINIMUM_TEST_SEND_GAS = (3e5).toString();
 export const MINIMUM_AVS_GAS = (3e6).toString();
+export const PROXY_DEPLOYED_URL = 'https://proxy.hyperlane.xyz';
diff --git a/typescript/cli/src/context/context.ts b/typescript/cli/src/context/context.ts
@@ -2,6 +2,7 @@ import { confirm } from '@inquirer/prompts';
 import { ethers } from 'ethers';
 
 import {
+  DEFAULT_GITHUB_REGISTRY,
   GithubRegistry,
   IRegistry,
   MergedRegistry,
@@ -16,6 +17,7 @@ import {
 import { isHttpsUrl, isNullish, rootLogger } from '@hyperlane-xyz/utils';
 
 import { isSignCommand } from '../commands/signCommands.js';
+import { PROXY_DEPLOYED_URL } from '../consts.js';
 import { forkNetworkToMultiProvider, verifyAnvil } from '../deploy/dry-run.js';
 import { logBlue } from '../logger.js';
 import { runSingleChainSelectionStep } from '../utils/chains.js';
@@ -37,6 +39,7 @@ export async function contextMiddleware(argv: Record<string, any>) {
     key: argv.key,
     fromAddress: argv.fromAddress,
     requiresKey,
+    disableProxy: argv.disableProxy,
     skipConfirmation: argv.yes,
   };
   if (!isDryRun && settings.fromAddress)
@@ -59,8 +62,9 @@ export async function getContext({
   key,
   requiresKey,
   skipConfirmation,
+  disableProxy = false,
 }: ContextSettings): Promise<CommandContext> {
-  const registry = getRegistry(registryUri, registryOverrideUri);
+  const registry = getRegistry(registryUri, registryOverrideUri, !disableProxy);
 
   let signer: ethers.Wallet | undefined = undefined;
   if (key || requiresKey) {
@@ -89,10 +93,11 @@ export async function getDryRunContext(
     key,
     fromAddress,
     skipConfirmation,
+    disableProxy = false,
   }: ContextSettings,
   chain?: ChainName,
 ): Promise<CommandContext> {
-  const registry = getRegistry(registryUri, registryOverrideUri);
+  const registry = getRegistry(registryUri, registryOverrideUri, !disableProxy);
   const chainMetadata = await registry.getMetadata();
 
   if (!chain) {
@@ -137,6 +142,7 @@ export async function getDryRunContext(
 function getRegistry(
   primaryRegistryUri: string,
   overrideRegistryUri: string,
+  enableProxy: boolean,
 ): IRegistry {
   const logger = rootLogger.child({ module: 'MergedRegistry' });
   const registries = [primaryRegistryUri, overrideRegistryUri]
@@ -145,7 +151,14 @@ function getRegistry(
     .map((uri, index) => {
       const childLogger = logger.child({ uri, index });
       if (isHttpsUrl(uri)) {
-        return new GithubRegistry({ uri, logger: childLogger });
+        return new GithubRegistry({
+          uri,
+          logger: childLogger,
+          proxyUrl:
+            enableProxy && isCanonicalRepoUrl(uri)
+              ? PROXY_DEPLOYED_URL
+              : undefined,
+        });
       } else {
         return new FileSystemRegistry({
           uri,
@@ -159,6 +172,10 @@ function getRegistry(
   });
 }
 
+function isCanonicalRepoUrl(url: string) {
+  return url === DEFAULT_GITHUB_REGISTRY;
+}
+
 /**
  * Retrieves a new MultiProvider based on all known chain metadata & custom user chains
  * @param customChains Custom chains specified by the user

diff --git a/typescript/cli/src/context/types.ts b/typescript/cli/src/context/types.ts
@@ -14,6 +14,7 @@ export interface ContextSettings {
   key?: string;
   fromAddress?: string;
   requiresKey?: boolean;
+  disableProxy?: boolean;
   skipConfirmation?: boolean;
 }
 

diff --git a/typescript/cli/src/tests/commands/core.ts b/typescript/cli/src/tests/commands/core.ts
@@ -11,7 +11,6 @@ export async function hyperlaneCoreDeploy(
 ) {
   return $`yarn workspace @hyperlane-xyz/cli run hyperlane core deploy \
         --registry ${REGISTRY_PATH} \
-        --overrides " " \
         --config ${coreInputPath} \
         --chain ${chain} \
         --key ${ANVIL_KEY} \

diff --git a/typescript/cli/src/tests/commands/warp.ts b/typescript/cli/src/tests/commands/warp.ts
@@ -14,7 +14,6 @@ $.verbose = true;
 export async function hyperlaneWarpDeploy(warpCorePath: string) {
   return $`yarn workspace @hyperlane-xyz/cli run hyperlane warp deploy \
         --registry ${REGISTRY_PATH} \
-        --overrides " " \
         --config ${warpCorePath} \
         --key ${ANVIL_KEY} \
         --verbosity debug \
@@ -30,7 +29,6 @@ export async function hyperlaneWarpApply(
 ) {
   return $`yarn workspace @hyperlane-xyz/cli run hyperlane warp apply \
         --registry ${REGISTRY_PATH} \
-        --overrides " " \
         --config ${warpDeployPath} \
         --warp ${warpCorePath} \
         --key ${ANVIL_KEY} \
@@ -45,7 +43,6 @@ export async function hyperlaneWarpRead(
 ) {
   return $`yarn workspace @hyperlane-xyz/cli run hyperlane warp read \
         --registry ${REGISTRY_PATH} \
-        --overrides " " \
         --address ${warpAddress} \
         --chain ${chain} \
         --key ${ANVIL_KEY} \

diff --git a/typescript/cli/src/tests/warp-read.e2e-test.ts b/typescript/cli/src/tests/warp-read.e2e-test.ts
@@ -0,0 +1,54 @@
+import { expect } from 'chai';
+
+import { WarpRouteDeployConfig } from '@hyperlane-xyz/sdk';
+
+import { readYamlOrJson, writeYamlOrJson } from '../utils/files.js';
+
+import {
+  ANVIL_KEY,
+  REGISTRY_PATH,
+  deployOrUseExistingCore,
+} from './commands/helpers.js';
+import { hyperlaneWarpDeploy, readWarpConfig } from './commands/warp.js';
+
+const CHAIN_NAME_2 = 'anvil2';
+
+const EXAMPLES_PATH = './examples';
+const CORE_CONFIG_PATH = `${EXAMPLES_PATH}/core-config.yaml`;
+const WARP_CONFIG_PATH_EXAMPLE = `${EXAMPLES_PATH}/warp-route-deployment.yaml`;
+
+const TEMP_PATH = '/tmp'; // /temp gets removed at the end of all-test.sh
+const WARP_CONFIG_PATH_2 = `${TEMP_PATH}/anvil2/warp-route-deployment-anvil2.yaml`;
+const WARP_CORE_CONFIG_PATH_2 = `${REGISTRY_PATH}/deployments/warp_routes/ETH/anvil2-config.yaml`;
+
+const TEST_TIMEOUT = 60_000; // Long timeout since these tests can take a while
+describe('WarpRead e2e tests', async function () {
+  let anvil2Config: WarpRouteDeployConfig;
+  this.timeout(TEST_TIMEOUT);
+  before(async function () {
+    await deployOrUseExistingCore(CHAIN_NAME_2, CORE_CONFIG_PATH, ANVIL_KEY);
+
+    // Create a new warp config using the example
+    const exampleWarpConfig: WarpRouteDeployConfig = readYamlOrJson(
+      WARP_CONFIG_PATH_EXAMPLE,
+    );
+    anvil2Config = { anvil2: { ...exampleWarpConfig.anvil1 } };
+    writeYamlOrJson(WARP_CONFIG_PATH_2, anvil2Config);
+  });
+
+  beforeEach(async function () {
+    await hyperlaneWarpDeploy(WARP_CONFIG_PATH_2);
+  });
+
+  it('should be able to read a warp route', async function () {
+    const warpConfigPath = `${TEMP_PATH}/warp-route-deployment-2.yaml`;
+    const warpConfig = await readWarpConfig(
+      CHAIN_NAME_2,
+      WARP_CORE_CONFIG_PATH_2,
+      warpConfigPath,
+    );
+    expect(warpConfig[CHAIN_NAME_2].type).to.be.equal(
+      anvil2Config[CHAIN_NAME_2].type,
+    );
+  });
+});
diff --git a/typescript/github-proxy/.dev.vars.example b/typescript/github-proxy/.dev.vars.example
@@ -0,0 +1 @@
+GITHUB_API_KEY=