Postgres-pglite as submodule (#461)

* moved postgres-pglite to own repo; building as before * added postgres-pglite as submodule * build postgres-pglite as submodule and copy output to the expected folder * remove cibuild folder * move instead of copy wasm output * switch back to copying the output * changed postgres-pglite branch * updated readme * github actions fixes * readd sudo to install * update dist * change linkweb.sh path in cibuild.sh * removed cibuild.sh; updated submodule * submodule deps * set remote branch of postgres-pglite to pglite/16.4 * update postgres-pglite submodule branch * remove patches and extra folders --------- Co-authored-by: tudor <[email protected]>
electric-sql · Jan 16, 2025 · ac7622c · ac7622c
1 parent d6b981b
commit ac7622c
Show file tree

Hide file tree

Showing 114 changed files with 45 additions and 19,889 deletions.
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -180,7 +180,10 @@ jobs:
     runs-on: ubuntu-22.04
     needs: [build-and-test-pglite, build-and-test-pglite-dependents]
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
       - uses: pnpm/action-setup@v4
       - uses: actions/setup-node@v4
         with:
@@ -206,6 +209,10 @@ jobs:
           path: /tmp/web
 
       - name: Build demo site
+        env: 
+          PGSRC: ${{ github.workspace }}/postgres-pglite
+          POSTGRES_PGLITE_OUT: ${{ github.workspace }}/packages/pglite/release
+        working-directory: ${{ github.workspace }}/postgres-pglite/pglite
         run: |
           NO_SDK_CHECK=true SDKROOT=/opt/python-wasm-sdk bash ./cibuild.sh demo-site
 

diff --git a/.github/workflows/build_wasm_postgres.yml b/.github/workflows/build_wasm_postgres.yml
@@ -31,6 +31,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 1
+          submodules: true
 
       - name: Set build parameters
         run: |
@@ -47,46 +48,70 @@ jobs:
             /tmp/sdk/postgres-*.tar.gz
             postgres
             postgresql-*
-          key: build-cache-${{ hashFiles('.buildconfig', 'cibuild.sh', '.github/workflows/build_wasm_postgres.yml', 'package.json', 'cibuild/**', 'extra/**', 'patches/*.c', 'patches/*.h', 'patches/**/*.diff') }}
+          key: build-cache-${{ hashFiles('postgres-pglite/pglite/.buildconfig', 'postgres-pglite/pglite/cibuild.sh', '.github/workflows/build_wasm_postgres.yml', 'package.json', 'postgres-pglite/pglite/cibuild/**', 'extra/**') }}
 
       - name: Install python-wasm-sdk for emsdk/wasi+prebuilts
         if: steps.cache-restore.outputs.cache-hit != 'true'
         working-directory: /
         run: |
-          sudo apt-get install -y lz4 wget pv bash
+          sudo apt-get update && sudo apt-get install -y lz4 wget pv bash build-essential libreadline-dev zlib1g-dev bison flex git
           echo https://github.com/pygame-web/python-wasm-sdk/releases/download/$SDK_VERSION/$SDK_ARCHIVE
           curl -sL --retry 5 https://github.com/pygame-web/python-wasm-sdk/releases/download/$SDK_VERSION/$SDK_ARCHIVE | tar xvP --use-compress-program=lz4 | pv -p -l -s 34500 >/dev/null
           echo https://github.com/pygame-web/python-wasi-sdk/releases/download/$WASI_SDK_VERSION/$WASI_SDK_ARCHIVE
           curl -sL --retry 5 https://github.com/pygame-web/python-wasi-sdk/releases/download/$WASI_SDK_VERSION/$WASI_SDK_ARCHIVE | tar xvP --use-compress-program=lz4 | pv -p -l -s 23000 >/dev/null
 
       - name: Build postgres WASM via emsdk
+        working-directory: ${{ github.workspace }}/postgres-pglite/pglite
+        env: 
+          PGSRC: ${{ github.workspace }}/postgres-pglite
+          POSTGRES_PGLITE_OUT: ${{ github.workspace }}/packages/pglite/release
         if: steps.cache-restore.outputs.cache-hit != 'true'
         run: |
-          bash ./cibuild.sh
+          bash cibuild.sh
 
       - name: Build contributed extensions
+        working-directory: ${{ github.workspace }}/postgres-pglite/pglite
+        env: 
+          PGSRC: ${{ github.workspace }}/postgres-pglite
+          POSTGRES_PGLITE_OUT: ${{ github.workspace }}/packages/pglite/release
         if: steps.cache-restore.outputs.cache-hit != 'true'
         run: |
           bash ./cibuild.sh contrib
 
       - name: Build extra extensions
+        working-directory: ${{ github.workspace }}/postgres-pglite/pglite
+        env: 
+          PGSRC: ${{ github.workspace }}/postgres-pglite
+          POSTGRES_PGLITE_OUT: ${{ github.workspace }}/packages/pglite/release
         if: steps.cache-restore.outputs.cache-hit != 'true'
         run: |
           bash ./cibuild.sh extra
 
       - name: Pack node distribution
+        working-directory: ${{ github.workspace }}/postgres-pglite/pglite
+        env: 
+          PGSRC: ${{ github.workspace }}/postgres-pglite
+          POSTGRES_PGLITE_OUT: ${{ github.workspace }}/packages/pglite/release
         if: steps.cache-restore.outputs.cache-hit != 'true'
         run: |
           bash ./cibuild.sh node
 
       - name: Link postgres WASM for web
+        working-directory: ${{ github.workspace }}/postgres-pglite/pglite
+        env: 
+          PGSRC: ${{ github.workspace }}/postgres-pglite
+          POSTGRES_PGLITE_OUT: ${{ github.workspace }}/packages/pglite/release
         if: steps.cache-restore.outputs.cache-hit != 'true'
         run: |
           bash ./cibuild.sh linkweb
 
       - name: Copy release for PGlite build
+        working-directory: ${{ github.workspace }}/postgres-pglite/pglite
+        env: 
+          PGSRC: ${{ github.workspace }}/postgres-pglite
+          POSTGRES_PGLITE_OUT: ${{ github.workspace }}/packages/pglite/release
         run: |
-          bash ./cibuild.sh pglite-prep
+          bash ./cibuild.sh postgres-pglite-dist
 
       - name: Upload Postgres to Github artifacts
         id: upload-postgres-wasm

diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,4 @@
+[submodule "postgres-pglite"]
+	path = postgres-pglite
+	url = https://github.com/electric-sql/postgres-pglite.git
+	branch = pglite/REL_16_STABLE
diff --git a/README.md b/README.md
@@ -136,14 +136,14 @@ Docker is required to build the WASM module, along with Node (v20 or above) and
 To start checkout the repository and install dependencies:
 
 ```bash
-git clone https://github.com/electric-sql/pglite
+git clone --recurse-submodules https://github.com/electric-sql/pglite
 cd pglite
 pnpm install
 ```
 
 To build everything, we have the convenient `pnpm build:all` command in the root of the repository. This command will:
 
-1. Use Docker to build the Postgres WASM module. The artifacts from this are then copied to `/packages/pglite/release`.
+1. Use Docker to build the Postgres WASM module. The artifacts produced by this step are then copied to `/packages/pglite/release`.
 2. Build the PGlite client library and other TypeScript packages.
 
 To _only_ build the Postgres WASM module (i.e. point 1 above), run