Automated build process generation exploration

.gitignore

@@ -1,4 +1,5 @@
 .pulumi
+.task
 bin
 obj
 node_modules

Makefile

@@ -1,3 +1,4 @@
+MAKEFLAGS 		:= --jobs=$(shell nproc) --warn-undefined-variables
 PROJECT_NAME 	:= Pulumi Infrastructure Components for AWS
 
 VERSION         := $(shell pulumictl get version)
@@ -30,7 +31,7 @@ awsx/schema.json: bin/${CODEGEN}
 	cd schemagen/cmd/$(CODEGEN) && go run . schema $(WORKING_DIR)/$(PACK)
 
 awsx/node_modules: awsx/package.json awsx/yarn.lock
-	yarn install --cwd awsx
+	yarn install --cwd awsx --no-progress
 	@touch awsx/node_modules
 
 awsx/schema-types.ts: awsx/node_modules awsx/schema.json
@@ -71,7 +72,7 @@ sdk/nodejs/bin:: bin/${CODEGEN} awsx/schema.json ${AWSX_CLASSIC_SRC}
 	rm -rf sdk/nodejs
 	bin/${CODEGEN} nodejs sdk/nodejs awsx/schema.json $(VERSION)
 	cd sdk/nodejs && \
-		yarn install && \
+		yarn install --no-progress && \
 		yarn run tsc --version && \
 		yarn run tsc && \
 		sed -e 's/\$${VERSION}/$(VERSION)/g' < package.json > bin/package.json && \
@@ -136,7 +137,7 @@ install_dotnet_sdk:: sdk/dotnet/bin
 
 lint_classic:
 	cd awsx-classic && \
-		yarn install && \
+		yarn install --no-progress && \
 		yarn lint
 
 lint:: awsx/node_modules

Taskfile.yml

@@ -0,0 +1,163 @@
+version: 3
+
+vars:
+  CWD:
+    sh: pwd
+  VERSION:
+    sh: pulumictl get version
+
+tasks:
+  clean:
+    run: once
+    cmds:
+      - rm -rf bin dist obj awsx/bin awsx/node_modules
+
+  schemagen:
+    run: once
+    cmds:
+      - go build -o ../bin/pulumi-gen-awsx ./cmd/pulumi-gen-awsx
+    dir: schemagen
+    generates:
+      - ../bin/pulumi-gen-awsx
+    sources:
+      - "**/*"
+
+  schema:
+    run: once
+    cmds:
+      - bin/pulumi-gen-awsx schema awsx
+    deps:
+      - schemagen
+    generates:
+      - awsx/schema.json
+    sources:
+      - bin/pulumi-gen-awsx
+
+  provider:install:
+    run: once
+    cmds:
+      - yarn install --no-progress
+    dir: awsx
+    generates:
+      - node_modules/**/*
+    sources:
+      - package.json
+      - yarn.lock
+
+  provider:gen:
+    run: once
+    cmds:
+      - yarn gen-types
+    deps:
+      - provider:install
+      - schema
+    dir: awsx
+    generates:
+      - schema-types.ts
+    sources:
+      - schema.json
+      - scripts/generate-provider-types.ts
+
+  provider:transpile:
+    run: once
+    cmds:
+      - yarn tsc
+      - cp package.json schema.json ./bin/
+      - sed -i.bak -e "s/\${VERSION}/{{ .VERSION }}/g" ./bin/package.json
+    deps:
+      - provider:install
+      - provider:gen
+    dir: awsx
+    generates:
+      - bin/**/*
+    sources:
+      - "**/*.ts"
+      - "*.*"
+
+  provider:build:
+    run: when_changed
+    vars:
+      TARGET: '{{ default "node16" .TARGET }}'
+      OUTPUT: '{{ default "../bin/pulumi-resource-awsx" .OUTPUT }}'
+    cmds:
+      - yarn run pkg . --no-bytecode --public-packages "*" --public --target {{ .TARGET }} --output {{ .OUTPUT }}
+    deps:
+      - provider:transpile
+    dir: awsx
+    generates:
+      - "{{ .OUTPUT }}"
+    sources:
+      - bin/**/*
+      - "*"
+
+  provider:build-linux-amd64:
+    deps:
+      - task: provider:build
+        vars:
+          TARGET: node16-linux-x64
+          OUTPUT: ../obj/provider/linux-amd64/pulumi-resource-awsx
+  provider:build-linux-arm64:
+    deps:
+      - task: provider:build
+        vars:
+          TARGET: node16-linux-arm64
+          OUTPUT: ../obj/provider/linux-arm64/pulumi-resource-awsx
+  provider:build-darwin-amd64:
+    deps:
+      - task: provider:build
+        vars:
+          TARGET: node16-macos-x64
+          OUTPUT: ../obj/provider/darwin-amd64/pulumi-resource-awsx
+  provider:build-darwin-arm64:
+    deps:
+      - task: provider:build
+        vars:
+          TARGET: node16-macos-arm64
+          OUTPUT: ../obj/provider/darwin-arm64/pulumi-resource-awsx
+  provider:build-windows-amd64:
+    deps:
+      - task: provider:build
+        vars:
+          TARGET: node16-win-x64
+          OUTPUT: ../obj/provider/windows-amd64/pulumi-resource-awsx.exe
+
+  provider:build-all:
+    deps:
+      - task: provider:build-linux-amd64
+      - task: provider:build-linux-arm64
+      - task: provider:build-darwin-amd64
+      - task: provider:build-darwin-arm64
+      - task: provider:build-windows-amd64
+
+  provider:dist-dir:
+    dir: dist
+
+  provider:pack:
+    vars:
+      TAR: pulumi-resource-awsx-v{{ .VERSION }}-{{ .PLAT }}.tar.gz
+      BIN_DIR: obj/provider/{{ .PLAT }}
+    cmds:
+      - tar --gzip -cf dist/{{ .TAR }} README.md LICENSE -C {{ .BIN_DIR }} .
+    deps:
+      - provider:build-{{ .PLAT }}
+      - provider:dist-dir
+    generates:
+      - dist/{{ .TAR }}
+
+  dist:
+    deps:
+      - task: provider:pack
+        vars:
+          PLAT: linux-amd64
+      - task: provider:pack
+        vars:
+          PLAT: linux-arm64
+      - task: provider:pack
+        vars:
+          PLAT: darwin-amd64
+      - task: provider:pack
+        vars:
+          PLAT: darwin-arm64
+      - task: provider:pack
+        vars:
+          PLAT: windows-amd64

awsx/package.json

@@ -14,6 +14,7 @@
   "scripts": {
     "gen-scripts": "curl https://raw.githubusercontent.com/pulumi/pulumi/master/pkg/codegen/schema/pulumi.json | json2ts -o scripts/pulumi-schema.d.ts --unreachableDefinitions --ignoreMinAndMaxItems",
     "gen-types": "ts-node scripts/generate-provider-types.ts",
+    "gen-make": "ts-node scripts/generate-makefile.ts",
     "format": "prettier --list-different --write .",
     "lint": "tslint -c tslint.json -p tsconfig.json",
     "test": "jest",

awsx/scripts/generate-makefile.ts

@@ -0,0 +1,14 @@
+import { writeFileSync } from "fs";
+import { render } from "./makefile";
+import { typescriptProvider } from "./makefile-builders";
+
+function main() {
+  const makefile = typescriptProvider("awsx", {
+    goDir: "schemagen",
+    providerDir: "awsx",
+    schemaDir: "awsx",
+  });
+  writeFileSync("Makefile", render(makefile), { encoding: "utf-8" });
+}
+
+main();

awsx/scripts/makefile-builders.ts

@@ -0,0 +1,164 @@
+import { Target, Makefile } from "./makefile";
+
+/** Add `&&` between each */
+function shellChain(...commands: string[]): string {
+  return commands.join(" && ");
+}
+
+function cwd(dir: string, ...commands: string[]): string {
+  return shellChain(`cd ${dir}`, ...commands);
+}
+
+function goBuild({ binaryName, goDir }: { binaryName: string; goDir: string }): Target {
+  return {
+    name: `bin/${binaryName}`,
+    dependencies: ["$(GO_SRC)"],
+    commands: [
+      shellChain(
+        `cd ${goDir}`,
+        `go build -o $(CWD)/bin/${binaryName} $(VERSION_FLAGS) $(CWD)/${goDir}/cmd/${binaryName}`,
+      ),
+    ],
+  };
+}
+
+interface GenSchemaArgs {
+  schemaDir: string;
+  genBinaryName: string;
+}
+
+function genSchema({ schemaDir, genBinaryName }: GenSchemaArgs): Target {
+  return {
+    name: `${schemaDir}/schema.json`,
+    dependencies: [`bin/${genBinaryName}`],
+    commands: [`bin/${genBinaryName} schema $(CWD)/${schemaDir}`],
+  };
+}
+
+function yarnInstall({ dir }: { dir: string }): Target {
+  return {
+    name: `${dir}/node_modules`,
+    dependencies: [`${dir}/package.json`, `${dir}/yarn.lock`],
+    commands: [`yarn install --cwd ${dir} --no-progress`, `@touch ${dir}/node_modules`],
+  };
+}
+
+type PkgMapping = {
+  plat: string;
+  target: string;
+  ext?: string;
+};
+
+const pkgMappings: PkgMapping[] = [
+  { plat: "linux-amd64", target: "node16-linux-x64" },
+  { plat: "linux-arm64", target: "node16-linux-arm64" },
+  { plat: "darwin-amd64", target: "node16-macos-x64" },
+  { plat: "darwin-arm64", target: "node16-macos-arm64" },
+  { plat: "windows-amd64", target: "node16-win-x64", ext: ".exe" },
+];
+
+export function typescriptProvider(
+  providerName: string,
+  config?: {
+    goDir?: string;
+    providerDir?: string;
+    schemaDir?: string;
+  },
+): Makefile {
+  const genBinaryName = `pulumi-gen-${providerName}`;
+  const goDir = config?.goDir ?? "provider";
+  const providerDir = config?.providerDir ?? "provider";
+  const providerBinaryName = `pulumi-resource-${providerName}`;
+  const schemaDir = config?.schemaDir ?? `${providerDir}/cmd/${providerBinaryName}`;
+
+  const variables = {
+    MAKEFLAGS: "--jobs=$(shell nproc) --warn-undefined-variables",
+    GOPATH: {
+      value: "$(HOME)/go",
+      type: "conditional",
+    },
+    GOBIN: {
+      value: "$(GOPATH)/bin",
+      type: "conditional",
+    },
+    CWD: "$(shell pwd)",
+    VERSION: "$(shell pulumictl get version)",
+    GO_SRC: `$(wildcard ${goDir}/go.*) $(wildcard ${goDir}/pkg/*/*.go) $(wildcard ${goDir}/cmd/${genBinaryName}/*.go)`,
+    PROVIDER_SRC: `$(wildcard ${providerDir}/*.*) $(wildcard ${providerDir}/*/*.ts)`,
+  } as const;
+
+  const genBin = goBuild({ binaryName: genBinaryName, goDir });
+  const schema = genSchema({ schemaDir, genBinaryName });
+  const providerNodeModules = yarnInstall({ dir: providerDir });
+  const schemaTypes = {
+    name: `${providerDir}/schema-types.ts`,
+    dependencies: [providerNodeModules, schema],
+  };
+  const providerJs = {
+    name: `${providerDir}/bin`,
+    dependencies: [providerNodeModules, "$(PROVIDER_SRC)"],
+    commands: [
+      cwd(providerDir, `yarn tsc`),
+      `cp ${providerDir}/package.json ${providerDir}/schema.json ${providerDir}/bin/`,
+      `sed -i.bak -e "s/\\$\${VERSION}/$(VERSION)/g" ${providerDir}/bin/package.json`,
+    ],
+  };
+  const providerBin = {
+    name: `bin/${providerBinaryName}`,
+    dependencies: [providerJs, providerNodeModules],
+  };
+  const pkgOutputDir = (mapping: PkgMapping) => `obj/provider/${mapping.plat}`;
+  const pkgOutputBinary = (mapping: PkgMapping) => `${providerBinaryName}${mapping.ext ?? ""}`;
+  const pkgOutput = (mapping: PkgMapping) => `${pkgOutputDir(mapping)}/${pkgOutputBinary(mapping)}`;
+  const pkgCmd = (mapping: PkgMapping) =>
+    `yarn run pkg . --no-bytecode --public-packages "*" --public --target ${
+      mapping.target
+    } --output $(CWD)/${pkgOutput(mapping)}`;
+  const providerGzipOutput = (mapping: PkgMapping) =>
+    `dist/${providerBinaryName}-v$(VERSION)-${mapping.plat}.tar.gz`;
+  const providerGzipCommand = (mapping: PkgMapping) =>
+    `tar --gzip -cf ${providerGzipOutput(mapping)} README.md LICENSE -C ${pkgOutputDir(mapping)} .`;
+  const providerDists = pkgMappings.map((mapping) => ({
+    name: providerGzipOutput(mapping),
+    dependencies: [pkgOutput(mapping)],
+    commands: ["@mkdir -p dist", providerGzipCommand(mapping)],
+  }));
+  return {
+    variables,
+    targets: [
+      genBin,
+      schema,
+      providerNodeModules,
+      schemaTypes,
+      providerJs,
+      providerBin,
+      ...pkgMappings.map((t) => ({
+        name: pkgOutput(t),
+        dependencies: [providerJs, providerNodeModules],
+        commands: [cwd(providerDir, pkgCmd(t))],
+      })),
+      ...providerDists,
+      // Phonies
+      {
+        name: "schemagen",
+        dependencies: [genBin],
+        phony: true,
+      },
+      {
+        name: "schema",
+        dependencies: [schema],
+        phony: true,
+      },
+      {
+        name: "dist",
+        dependencies: providerDists,
+        phony: true,
+      },
+      {
+        name: "clean",
+        commands: [`rm -rf bin obj ${providerDir}/bin ${providerDir}/node_modules`],
+        phony: true,
+      },
+    ],
+  };
+}