-
Notifications
You must be signed in to change notification settings - Fork 770
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Update Github actions to build new GRR Docker image
The new docker image and docker-compose setup replaces the GRR debian package. The current e2e testing is temporarily removed as it depends on the debian package and will be re-introduced after the docker-compose setup is available.
- Loading branch information
1 parent
727545a
commit 2b3e0d6
Showing
2 changed files
with
84 additions
and
156 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,7 +3,6 @@ on: [push, pull_request] | |
env: | ||
GCS_BUCKET: autobuilds.grr-response.com | ||
GCS_BUCKET_OPENAPI: autobuilds-grr-openapi | ||
GCS_LATEST_PATH: _latest_server_deb | ||
DOCKER_REPOSITORY: ghcr.io/google/grr | ||
jobs: | ||
test-devenv: | ||
|
@@ -71,7 +70,7 @@ jobs: | |
travis/build_api_documentation.sh "_openapi_artifacts/openapi_description/openapi_description.json" "_openapi_artifacts/documentation/openapi_documentation.html" | ||
ls -la _openapi_artifacts/* | ||
- name: Upload OpenAPI to GitHub artifacts | ||
uses: actions/upload-artifact@v3 | ||
uses: actions/upload-artifact@v4 | ||
with: | ||
name: openapi | ||
path: _openapi_artifacts/ | ||
|
@@ -97,7 +96,7 @@ jobs: | |
travis/build_templates.sh | ||
ls -la gcs_upload_dir | ||
- name: Upload installers to GitHub artifacts | ||
uses: actions/upload-artifact@v3 | ||
uses: actions/upload-artifact@v4 | ||
with: | ||
name: ubuntu-installers | ||
path: gcs_upload_dir/ | ||
|
@@ -120,7 +119,7 @@ jobs: | |
travis/build_templates.sh | ||
ls -la gcs_upload_dir | ||
- name: Upload installers to GitHub artifacts | ||
uses: actions/upload-artifact@v3 | ||
uses: actions/upload-artifact@v4 | ||
with: | ||
name: osx-installers | ||
path: gcs_upload_dir/ | ||
|
@@ -154,7 +153,7 @@ jobs: | |
docker exec "${DOCKER_CONTAINER}" rpm -vih gcs_upload_dir/*.rpm | ||
ls -la gcs_upload_dir | ||
- name: Upload installers to GitHub artifacts | ||
uses: actions/upload-artifact@v3 | ||
uses: actions/upload-artifact@v4 | ||
with: | ||
name: centos-installers | ||
path: gcs_upload_dir/ | ||
|
@@ -177,133 +176,71 @@ jobs: | |
mv -v output*/* gcs_upload_dir | ||
ls -la gcs_upload_dir | ||
- name: Upload installers to GitHub artifacts | ||
uses: actions/upload-artifact@v3 | ||
uses: actions/upload-artifact@v4 | ||
with: | ||
name: windows-installers | ||
path: gcs_upload_dir/ | ||
retention-days: 1 | ||
|
||
build-server-deb: | ||
runs-on: ubuntu-22.04 | ||
build-push-docker-image: | ||
env: | ||
GCS_TAG: server_deb | ||
REGISTRY: ghcr.io | ||
IMAGE_NAME: ${{ github.repository }} | ||
runs-on: ubuntu-22.04 | ||
needs: | ||
- build-centos | ||
- build-ubuntu | ||
- build-osx | ||
- build-windows | ||
steps: | ||
- uses: actions/checkout@v3 | ||
- name: Checkout repository | ||
uses: actions/checkout@v3 | ||
- name: Download installers from GitHub artifacts | ||
id: download | ||
uses: actions/download-artifact@v3 | ||
uses: actions/download-artifact@v4 | ||
with: | ||
path: ~/_artifacts | ||
- name: Set up | ||
run: | | ||
sudo apt-get update | ||
sudo apt-get install -y fakeroot debhelper libffi-dev libssl-dev python3-dev python3-pip python3-venv python3-mysqldb wget openjdk-8-jdk zip git devscripts libmysqlclient-dev dh-virtualenv dh-make libc6-i386 lib32z1 | ||
python3 -m venv --system-site-packages "${HOME}/INSTALL" | ||
"${HOME}/INSTALL/bin/python3" -m pip install --upgrade pip 'setuptools<58.3.1' wheel | ||
- name: Build | ||
run: | | ||
travis/install.sh | ||
mkdir -p grr/config/grr_response_templates/templates | ||
mv -v ~/_artifacts/windows-installers/GRR_*_amd64.msi.zip grr/config/grr_response_templates/templates | ||
mv -v ~/_artifacts/ubuntu-installers/grr_*_amd64.deb.zip grr/config/grr_response_templates/templates | ||
mv -v ~/_artifacts/centos-installers/grr_*_amd64.rpm.zip grr/config/grr_response_templates/templates | ||
mv -v ~/_artifacts/osx-installers/grr_*_amd64.xar.zip grr/config/grr_response_templates/templates | ||
travis/build_local_pyindex.sh | ||
travis/build_server_deb.sh | ||
ls -la gcs_upload_dir | ||
- name: Upload installers to GitHub artifacts | ||
uses: actions/upload-artifact@v3 | ||
path: ./_artifacts | ||
pattern: '*installer*' | ||
- name: Login to GitHub Container registry | ||
if: ${{ github.event_name == 'push' }} | ||
uses: docker/login-action@v3 | ||
with: | ||
name: server-deb | ||
path: gcs_upload_dir/ | ||
retention-days: 1 | ||
registry: ${{ env.REGISTRY }} | ||
username: ${{ github.actor }} | ||
password: ${{ secrets.GITHUB_TOKEN }} | ||
|
||
test-ubuntu-e2e: | ||
continue-on-error: true # Debug follow up step. | ||
runs-on: ubuntu-22.04 | ||
env: | ||
GRR_ADMIN_PASS: 'e2e_tests' | ||
APPVEYOR_MYSQL_PASS: 'root' | ||
needs: | ||
- build-server-deb | ||
steps: | ||
- uses: actions/checkout@v3 | ||
- name: Set up MySQL | ||
run: | | ||
printf "\n[mysqld]\nmax_allowed_packet=42M\nlog_bin_trust_function_creators=1\n" | sudo tee -a /etc/mysql/my.cnf | ||
sudo /etc/init.d/mysql start | ||
- name: Download installers from GitHub artifacts | ||
id: download | ||
uses: actions/download-artifact@v3 | ||
with: | ||
name: server-deb | ||
path: _artifacts | ||
- name: Install | ||
run: | | ||
free -hmw | ||
lscpu | ||
sudo -EH ./appveyor/e2e_tests/install_mem_usage_cron.sh | ||
sudo -EH ./appveyor/e2e_tests/install_latest_server_deb.sh | ||
- name: Test | ||
run: | | ||
sudo -EH ./appveyor/e2e_tests/run_e2e_tests.sh | ||
sudo -EH ./appveyor/e2e_tests/test_repack.sh | ||
- name: Upload logs and configs to GitHub artifacts | ||
uses: actions/upload-artifact@v3 | ||
if: always() | ||
- name: Extract metadata (tags, labels) for Docker | ||
if: ${{ github.event_name == 'push' }} | ||
id: meta | ||
uses: docker/metadata-action@v5 | ||
with: | ||
name: e2e-test | ||
path: /var/log/grr | ||
retention-days: 1 | ||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | ||
|
||
build-push-docker: | ||
runs-on: ubuntu-22.04 | ||
needs: | ||
- build-server-deb | ||
# - test-ubuntu-e2e # TODO: Comment back in after debugging is finished. | ||
- test-ubuntu | ||
- build-openapi | ||
steps: | ||
- uses: actions/checkout@v3 | ||
- name: Download installers from GitHub artifacts | ||
id: download | ||
uses: actions/download-artifact@v3 | ||
- name: Build and push Docker image | ||
if: ${{ github.event_name == 'push' }} | ||
uses: docker/build-push-action@v5 | ||
with: | ||
name: server-deb | ||
path: _artifacts | ||
- name: Build Docker image | ||
run: | | ||
export BRANCH=$(echo $GITHUB_REF | cut -d'/' -f 3) | ||
./appveyor/docker_build/build_docker_image.sh | ||
- if: ${{ github.event_name == 'push' }} | ||
name: Login to GitHub Container registry | ||
uses: docker/login-action@v2 | ||
with: | ||
registry: ghcr.io | ||
username: ${{ github.actor }} | ||
password: ${{ secrets.GITHUB_TOKEN }} | ||
- if: ${{ github.event_name == 'push' }} | ||
name: Push to GitHub Container registry | ||
run: | | ||
docker push -a ${{ env.DOCKER_REPOSITORY }} | ||
upload: | ||
context: . | ||
push: true | ||
tags: ${{ steps.meta.outputs.tags }} | ||
labels: ${{ steps.meta.outputs.labels }} | ||
|
||
upload-artifacts: | ||
if: ${{ github.event_name == 'push' }} | ||
permissions: | ||
contents: 'read' | ||
id-token: 'write' | ||
runs-on: ubuntu-22.04 | ||
needs: | ||
- build-push-docker | ||
- build-centos | ||
- build-ubuntu | ||
- build-osx | ||
- build-windows | ||
steps: | ||
- uses: actions/checkout@v3 | ||
- name: Download installers from GitHub artifacts | ||
id: download | ||
uses: actions/download-artifact@v3 | ||
uses: actions/download-artifact@v4 | ||
with: | ||
path: _artifacts | ||
- run: | | ||
|
@@ -319,8 +256,6 @@ jobs: | |
mv -v _artifacts/osx-installers/* $OUTPUT_DIR/osx | ||
mkdir -p $OUTPUT_DIR/windows/ | ||
mv -v _artifacts/windows-installers/* $OUTPUT_DIR/windows | ||
mkdir -p $OUTPUT_DIR/server_deb/ | ||
mv -v _artifacts/server-deb/* $OUTPUT_DIR/server_deb | ||
- name: Authenticate | ||
uses: 'google-github-actions/auth@v1' | ||
with: | ||
|
@@ -335,10 +270,6 @@ jobs: | |
destination: ${{ env.GCS_BUCKET }} | ||
# Omit `path` (e.g. /home/runner/deploy/) in final GCS path. | ||
parent: false | ||
- name: Replace ${{ env.GCS_LATEST_PATH }} folder in GCS | ||
run: | | ||
gsutil rm gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/** || true | ||
gsutil cp -r $OUTPUT_DIR/server_deb/* gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/ | ||
- name: Upload OpenAPI to GCS | ||
uses: google-github-actions/[email protected] | ||
with: | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,78 +2,75 @@ | |
# | ||
# See https://hub.docker.com/r/grrdocker/grr/ | ||
# | ||
# We have configured Travis to trigger an image build every time a new server | ||
# deb is been uploaded to GCS. | ||
# We have configured Github Actions to trigger an image build every time a new | ||
# a PUSH happens in the GRR github repository. | ||
# | ||
# Run the container with: | ||
# Example: Run the grr admin_ui component: | ||
# | ||
# docker run \ | ||
# -e EXTERNAL_HOSTNAME="localhost" \ | ||
# -e ADMIN_PASSWORD="demo" \ | ||
# -p 0.0.0.0:8000:8000 \ | ||
# -p 0.0.0.0:8080:8080 \ | ||
# grrdocker/grr | ||
# docker run -it \ | ||
# -v $(pwd)/docker_config_files:/configs | ||
# ghcr.io/google/grr:grr-docker-compose | ||
# "-component" "admin_ui" | ||
# "-config" "/configs/server/grr.server.yaml" | ||
|
||
FROM mariadb:jammy | ||
FROM ubuntu:22.04 AS builder | ||
|
||
LABEL maintainer="[email protected]" | ||
|
||
ARG GCS_BUCKET | ||
ARG GRR_COMMIT | ||
|
||
ENV GRR_VENV /usr/share/grr-server | ||
ENV DEBIAN_FRONTEND noninteractive | ||
# Buffering output (sometimes indefinitely if a thread is stuck in | ||
# a loop) makes for a non-optimal user experience when containers | ||
# are run in the foreground, so we disable that. | ||
ENV PYTHONUNBUFFERED=0 | ||
|
||
SHELL ["/bin/bash", "-c"] | ||
ENV PYTHONUNBUFFERED 0 | ||
|
||
RUN apt-get update && \ | ||
apt-get install -y \ | ||
debhelper \ | ||
default-jre \ | ||
dpkg-dev \ | ||
git \ | ||
libffi-dev \ | ||
libssl-dev \ | ||
python-is-python3 \ | ||
python3-dev \ | ||
python3-pip \ | ||
python3-venv \ | ||
python3-mysqldb \ | ||
rpm \ | ||
wget \ | ||
zip \ | ||
python3-mysqldb | ||
build-essential \ | ||
linux-headers-generic \ | ||
dh-make \ | ||
rpm | ||
|
||
RUN pwd | ||
RUN ls -lha | ||
RUN ls -lha / | ||
|
||
# Only available when building as part of Github Actions. | ||
COPY ./_artifacts* /client_templates | ||
|
||
ENV VIRTUAL_ENV /usr/share/grr-server | ||
ENV GRR_SOURCE /usr/src/grr | ||
|
||
RUN python -m venv --system-site-packages $VIRTUAL_ENV | ||
ENV PATH="$VIRTUAL_ENV/bin:$PATH" | ||
|
||
# Limiting setuptools version due to | ||
# https://github.com/pypa/setuptools/issues/3278 | ||
# (it behaves incorrectly on Ubuntu 22 on virtualenvs with access to | ||
# globally installed packages). | ||
RUN pip3 install --upgrade 'setuptools<58.3.1' && \ | ||
python3 -m venv --system-site-packages $GRR_VENV | ||
RUN pip install wheel nodeenv grpcio-tools==1.60 | ||
|
||
RUN $GRR_VENV/bin/pip install --upgrade --no-cache-dir pip wheel six setuptools nodeenv && \ | ||
$GRR_VENV/bin/nodeenv -p --prebuilt --node=16.13.0 && \ | ||
echo '{ "allow_root": true }' > /root/.bowerrc | ||
RUN nodeenv -p --prebuilt --node=16.13.0 | ||
|
||
# Copy the GRR code over. | ||
ADD . /usr/src/grr | ||
RUN mkdir ${GRR_SOURCE} | ||
ADD . ${GRR_SOURCE} | ||
|
||
RUN cd /usr/src/grr && bash -x /usr/src/grr/docker/install_grr_from_gcs.sh | ||
WORKDIR ${GRR_SOURCE} | ||
|
||
ENTRYPOINT ["/usr/src/grr/docker/docker-entrypoint.sh"] | ||
RUN cd grr/server/grr_response_server/gui/static && \ | ||
npm ci && npm run gulp compile | ||
|
||
# Port for the admin UI GUI | ||
EXPOSE 8000 | ||
RUN python grr/proto/makefile.py && \ | ||
python grr/core/grr_response_core/artifacts/makefile.py | ||
|
||
# Port for clients to talk to | ||
EXPOSE 8080 | ||
RUN pip install -e grr/proto \ | ||
pip install -e grr/core \ | ||
pip install -e grr/client \ | ||
pip install -e grr/server \ | ||
pip install -e grr/client_builder \ | ||
pip install -e api_client/python | ||
|
||
# Directories used by GRR at runtime, which can be mounted from the host's | ||
# filesystem. Note that volumes can be mounted even if they do not appear in | ||
# this list. | ||
VOLUME ["/usr/share/grr-server/install_data/etc"] | ||
WORKDIR / | ||
|
||
CMD ["grr"] | ||
ENTRYPOINT [ "grr_server" ] |