Merge pull request #829 from jiridanek/jd_try_oc_command_cleanup

RHOAIENG-17695: chore(ci): create a test for calling `oc version` in the test, which can be run with ci testing

.github/workflows/build-notebooks-TEMPLATE.yaml

@@ -18,7 +18,10 @@ name: Build & Publish Notebook Servers (TEMPLATE)
 
 jobs:
   build:
-    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        os: [ubuntu-22.04]
+    runs-on: ${{matrix.os}}
     env:
       # Some pieces of code (image pulls for example) in podman consult TMPDIR or default to /var/tmp
       TMPDIR: /home/runner/.local/share/containers/tmpdir
@@ -34,6 +37,8 @@ jobs:
       TRIVY_VULNDB: "/home/runner/.local/share/containers/trivy_db"
       # Targets (and their folder) that should be scanned using FS instead of IMAGE scan due to resource constraints
       TRIVY_SCAN_FS_JSON: '{}'
+      # Poetry version for use in running tests
+      POETRY_VERSION: '2.0.0'
 
     steps:
 
@@ -258,6 +263,47 @@ jobs:
 
       # endregion
 
+      # region Pytest image tests
+
+      - name: Install poetry
+        if: steps.cache-poetry-restore.outputs.cache-hit != 'true'
+        run: pipx install poetry==${{ env.POETRY_VERSION }}
+        env:
+          PIPX_HOME: /home/runner/.local/pipx
+          PIPX_BIN_DIR: /home/runner/.local/bin
+
+      - name: Check poetry is installed correctly
+        run: poetry env info
+
+      - name: Set up Python
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'poetry'
+
+      - name: Configure poetry
+        run: poetry env use "${{ steps.setup-python.outputs.python-path }}"
+
+      - name: Install deps
+        run: poetry install --sync
+
+      - name: Run container tests (in PyTest)
+        run: |
+          set -Eeuxo pipefail
+          # retry to increase CI reliability
+          for i in {1..5}; do
+            if podman pull --retry 10 "${RYUK_CONTAINER_IMAGE}"; then break; fi
+          done
+          # now run the tests
+          poetry run pytest tests/containers --image="${{ steps.calculated_vars.outputs.OUTPUT_IMAGE }}"
+        env:
+          DOCKER_HOST: "unix:///var/run/podman/podman.sock"
+          TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE: "/var/run/podman/podman.sock"
+          RYUK_CONTAINER_IMAGE: "testcontainers/ryuk:0.8.1"
+
+      # endregion Pytest image tests
+
       # region Makefile image tests
 
       - name: "Check if we have tests or not"

pyproject.toml

@@ -14,6 +14,8 @@ python = "~3.12"
 pytest = "^8.2.2"
 pytest-subtests = "^0.12.1"
 pyfakefs = "^5.7.2"
+testcontainers = "^4.9.0"
+docker = "^7.1.0"
 
 [build-system]
 requires = ["poetry-core"]

tests/__init__.py

@@ -0,0 +1,7 @@
+import pathlib
+
+PROJECT_ROOT = pathlib.Path(__file__).parent.parent
+
+__all__ = [
+    PROJECT_ROOT,
+]

tests/containers/base_image_test.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import logging
+import pathlib
+import tempfile
+from typing import TYPE_CHECKING
+
+import testcontainers.core.container
+import testcontainers.core.waiting_utils
+
+from tests.containers import docker_utils
+
+logging.basicConfig(level=logging.DEBUG)
+LOGGER = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    import pytest_subtests
+
+
+class TestBaseImage:
+    """Tests that are applicable for all images we have in this repository."""
+
+    def test_oc_command_runs(self, image: str):
+        container = testcontainers.core.container.DockerContainer(image=image, user=123456, group_add=[0])
+        container.with_command("/bin/sh -c 'sleep infinity'")
+        try:
+            container.start()
+            ecode, output = container.exec(["/bin/sh", "-c", "oc version"])
+        finally:
+            docker_utils.NotebookContainer(container).stop(timeout=0)
+
+        logging.debug(output.decode())
+        assert ecode == 0
+
+    def test_oc_command_runs_fake_fips(self, image: str, subtests: pytest_subtests.SubTests):
+        """Establishes a best-effort fake FIPS environment and attempts to execute `oc` binary in it.
+
+        Related issue: RHOAIENG-4350 In workbench the oc CLI tool cannot be used on FIPS enabled cluster"""
+        with tempfile.TemporaryDirectory() as tmp_crypto:
+            # Ubuntu does not even have /proc/sys/crypto directory, unless FIPS is activated and machine
+            #  is rebooted, see https://ubuntu.com/security/certifications/docs/fips-enablement
+            # NOTE: mounting a temp file as `/proc/sys/crypto/fips_enabled` is further discussed in
+            #  * https://issues.redhat.com/browse/RHOAIENG-4350
+            #  * https://github.com/junaruga/fips-mode-user-space/blob/main/fips-mode-user-space-setup
+            tmp_crypto = pathlib.Path(tmp_crypto)
+            (tmp_crypto / 'crypto').mkdir()
+            (tmp_crypto / 'crypto' / 'fips_enabled').write_text("1\n")
+            (tmp_crypto / 'crypto' / 'fips_name').write_text("Linux Kernel Cryptographic API\n")
+            (tmp_crypto / 'crypto' / 'fips_version').write_text("6.10.10-200.fc40.aarch64\n")
+            # tmpdir is by-default created with perms restricting access to user only
+            tmp_crypto.chmod(0o777)
+
+            container = testcontainers.core.container.DockerContainer(image=image, user=654321, group_add=[0])
+            container.with_volume_mapping(str(tmp_crypto), "/proc/sys")
+            container.with_command("/bin/sh -c 'sleep infinity'")
+
+            try:
+                container.start()
+
+                with subtests.test("/proc/sys/crypto/fips_enabled is 1"):
+                    ecode, output = container.exec(["/bin/sh", "-c", "sysctl crypto.fips_enabled"])
+                    assert ecode == 0, output.decode()
+                    assert "crypto.fips_enabled = 1\n" == output.decode(), output.decode()
+
+                # 0: enabled, 1: partial success, 2: not enabled
+                with subtests.test("/fips-mode-setup --is-enabled reports 1"):
+                    ecode, output = container.exec(["/bin/sh", "-c", "fips-mode-setup --is-enabled"])
+                    assert ecode == 1, output.decode()
+
+                with subtests.test("/fips-mode-setup --check reports partial success"):
+                    ecode, output = container.exec(["/bin/sh", "-c", "fips-mode-setup --check"])
+                    assert ecode == 1, output.decode()
+                    assert "FIPS mode is enabled.\n" in output.decode(), output.decode()
+                    assert "Inconsistent state detected.\n" in output.decode(), output.decode()
+
+                with subtests.test("oc version command runs"):
+                    ecode, output = container.exec(["/bin/sh", "-c", "oc version"])
+                    assert ecode == 0, output.decode()
+            finally:
+                docker_utils.NotebookContainer(container).stop(timeout=0)

tests/containers/conftest.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import testcontainers.core.config
+import testcontainers.core.container
+import testcontainers.core.docker_client
+
+import pytest
+
+if TYPE_CHECKING:
+    from pytest import ExitCode, Session, Parser, Metafunc
+
+SHUTDOWN_RYUK = False
+
+# NOTE: Configure Testcontainers through `testcontainers.core.config` and not through env variables.
+# Importing `testcontainers` above has already read out env variables, and so at this point, setting
+#  * DOCKER_HOST
+#  * TESTCONTAINERS_RYUK_DISABLED
+#  * TESTCONTAINERS_RYUK_PRIVILEGED
+#  * TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE
+# would have no effect.
+
+# We'd get selinux violations with podman otherwise, so either ryuk must be privileged, or we need to disable selinux.
+# https://github.com/testcontainers/testcontainers-java/issues/2088#issuecomment-1169830358
+testcontainers.core.config.testcontainers_config.ryuk_privileged = True
+
+
+def pytest_addoption(parser: Parser) -> None:
+    parser.addoption("--image", action="append", default=[],
+                     help="Image to use, can be specified multiple times")
+
+
+def pytest_generate_tests(metafunc: Metafunc) -> None:
+    if image.__name__ in metafunc.fixturenames:
+        metafunc.parametrize(image.__name__, metafunc.config.getoption("--image"))
+
+
+# https://docs.pytest.org/en/stable/how-to/fixtures.html#parametrizing-fixtures
+# indirect parametrization https://stackoverflow.com/questions/18011902/how-to-pass-a-parameter-to-a-fixture-function-in-pytest
+@pytest.fixture(scope="session")
+def image(request):
+    yield request.param
+
+
+def pytest_sessionstart(session: Session) -> None:
+    # first preflight check: ping the Docker API
+    client = testcontainers.core.docker_client.DockerClient()
+    assert client.client.ping(), "Failed to connect to Docker"
+
+    # second preflight check: start the Reaper container
+    assert testcontainers.core.container.Reaper.get_instance() is not None, "Failed to start Reaper container"
+
+
+# https://docs.pytest.org/en/latest/reference/reference.html#pytest.hookspec.pytest_sessionfinish
+def pytest_sessionfinish(session: Session, exitstatus: int | ExitCode) -> None:
+    # resolves a shutdown resource leak warning that would be otherwise reported
+    if SHUTDOWN_RYUK:
+        testcontainers.core.container.Reaper.delete_instance()

tests/containers/docker_utils.py

@@ -0,0 +1,143 @@
+from __future__ import annotations
+
+import io
+import logging
+import os.path
+import sys
+import tarfile
+import time
+from typing import TYPE_CHECKING
+
+import testcontainers.core.container
+
+if TYPE_CHECKING:
+    from docker.models.containers import Container
+
+
+class NotebookContainer:
+    @classmethod
+    def wrap(cls, container: testcontainers.core.container.DockerContainer):
+        return NotebookContainer(container)
+
+    def __init__(self, container: testcontainers.core.container.DockerContainer) -> None:
+        self.testcontainer = container
+
+    def stop(self, timeout: int = 10):
+        """Stop container with customizable timeout.
+
+        DockerContainer.stop() has unchangeable 10s timeout between SIGSTOP and SIGKILL."""
+        self.testcontainer.get_wrapped_container().stop(timeout=timeout)
+        self.testcontainer.stop()
+
+    def wait_for_exit(self) -> int:
+        container = self.testcontainer.get_wrapped_container()
+        container.reload()
+        while container.status != "exited":
+            time.sleep(0.2)
+            container.reload()
+        return container.attrs["State"]["ExitCode"]
+
+
+def container_cp(container: Container, src: str, dst: str,
+                 user: int | None = None, group: int | None = None) -> None:
+    """
+    Copies a directory into a container
+    From https://stackoverflow.com/questions/46390309/how-to-copy-a-file-from-host-to-container-using-docker-py-docker-sdk
+    """
+    fh = io.BytesIO()
+    tar = tarfile.open(fileobj=fh, mode="w:gz")
+
+    tar_filter = None
+    if user or group:
+        def tar_filter(f: tarfile.TarInfo) -> tarfile.TarInfo:
+            if user:
+                f.uid = user
+            if group:
+                f.gid = group
+            return f
+
+    logging.debug(f"Adding {src=} to archive {dst=}")
+    try:
+        tar.add(src, arcname=os.path.basename(src), filter=tar_filter)
+    finally:
+        tar.close()
+
+    fh.seek(0)
+    container.put_archive(dst, fh)
+
+
+def container_exec(
+        container: Container,
+        cmd: str | list[str],
+        stdout: bool = True,
+        stderr: bool = True,
+        stdin: bool = False,
+        tty: bool = False,
+        privileged: bool = False,
+        user: str = "",
+        detach: bool = False,
+        stream: bool = False,
+        socket: bool = False,
+        environment: dict[str, str] | None = None,
+        workdir: str | None = None,
+) -> ContainerExec:
+    """
+    An enhanced version of #docker.Container.exec_run() which returns an object
+    that can be properly inspected for the status of the executed commands.
+    Usage example:
+    result = tools.container_exec(container, cmd, stream=True, **kwargs)
+    res = result.communicate(line_prefix=b'--> ')
+    if res != 0:
+        error('exit code {!r}'.format(res))
+    From https://github.com/docker/docker-py/issues/1989
+    """
+
+    exec_id = container.client.api.exec_create(
+        container.id,
+        cmd,
+        stdout=stdout,
+        stderr=stderr,
+        stdin=stdin,
+        tty=tty,
+        privileged=privileged,
+        user=user,
+        environment=environment,
+        workdir=workdir,
+    )["Id"]
+
+    output = container.client.api.exec_start(exec_id, detach=detach, tty=tty, stream=stream, socket=socket)
+
+    return ContainerExec(container.client, exec_id, output)
+
+
+class ContainerExec:
+    def __init__(self, client, id, output: list[int] | list[str]):
+        self.client = client
+        self.id = id
+        self.output = output
+
+    def inspect(self):
+        return self.client.api.exec_inspect(self.id)
+
+    def poll(self):
+        return self.inspect()["ExitCode"]
+
+    def communicate(self, line_prefix=b""):
+        for data in self.output:
+            if not data:
+                continue
+            offset = 0
+            while offset < len(data):
+                sys.stdout.buffer.write(line_prefix)
+                nl = data.find(b"\n", offset)
+                if nl >= 0:
+                    slice = data[offset: nl + 1]
+                    offset = nl + 1
+                else:
+                    slice = data[offset:]
+                    offset += len(slice)
+                sys.stdout.buffer.write(slice)
+            sys.stdout.flush()
+        while self.poll() is None:
+            raise RuntimeError("Hm could that really happen?")
+        return self.poll()

tests/test_main.py

@@ -2,18 +2,19 @@
 
 import os
 import logging
-import pathlib
 import shutil
 import subprocess
 import tomllib
 from typing import TYPE_CHECKING
 
+from tests import PROJECT_ROOT
+
 if TYPE_CHECKING:
     import pytest_subtests
 
-PROJECT_ROOT = pathlib.Path(__file__).parent.parent
 MAKE = shutil.which("gmake") or shutil.which("make")
 
+
 def test_image_pipfiles(subtests: pytest_subtests.plugin.SubTests):
     for file in PROJECT_ROOT.glob("**/Pipfile"):
         with subtests.test(msg="checking Pipfile", pipfile=file):