From ee8a86f222ff1326ccef8117406c4d6e7a847c19 Mon Sep 17 00:00:00 2001 From: Teresa Noviello <85874226+teresanovielloatinstana@users.noreply.github.com> Date: Fri, 17 Jan 2025 17:18:51 +0100 Subject: [PATCH] Build on ubuntu+clang17, fix cross-build, QA (#10) Migrate the build from DockerHub debian:testing to AWS ubuntu:22.04. Attention: Clang version on this image is 17, not 16. Isolate docker files in directory docker-image. Introduce an entrypoint for the docker image, to better deal with the GO environment. Dockerfile: install GolangCi deps in the docker image, copy the binaries in the container's workdir in /agent/go/bin. Makefile: fix the clean target. Makefiles: substitute LLVM commands versionsxi, example clang-16 with clang-17. This is to avoid introducing aliases in the building container. Makefile: remove the installation of GolangCi deps during linting. Makefile: add targets for GO deps management Makefile: pin GO deps versions Makefile: pin porto version to "v0.6.0" (instead of latest). The latest version of porto requires GO 1.23, while the current toolchain is based on Go 1.22. Makefile: improve target linter-version, print all GolangCi deps versions. GO: touch go/bin/.gitkeep, the directory bin contains the GO deps print_instruction_count: fix OBJDUMP_CMD from llvm-objdump-16 to llvm-objdump. Makefile: Fix cross-compilation Always use NATIVE_ARCH as GOARCH for go:generate. The previous approach may fail on systems that can not execute cross-compiled binaries on the build host (e.g. if an appropriate qemu binary is not installed). --- Makefile | 45 ++++++++++++++++++----- Dockerfile => docker-image/Dockerfile | 47 ++++++++++++++++++------- docker-image/entrypoint.sh | 29 +++++++++++++++ go/bin/.gitkeep | 0 support/ebpf/Makefile | 6 ++-- support/ebpf/print_instruction_count.sh | 4 +-- 6 files changed, 104 insertions(+), 27 deletions(-) rename Dockerfile => docker-image/Dockerfile (57%) create mode 100755 docker-image/entrypoint.sh create mode 100644 go/bin/.gitkeep diff --git a/Makefile b/Makefile index 402949bd..1ad89053 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ endif COMPILER_TARGET ?= $(COMPILER_TARGET_ARCH)-redhat-linux SYSROOT_PATH ?= / -export CC = clang-16 +export CC = clang-17 export COMPILER_TARGET export CGO_CFLAGS = --sysroot=/usr/sysroot --target=$(COMPILER_TARGET) export CGO_ENABLED = 1 @@ -54,6 +54,10 @@ GO_FLAGS := -buildvcs=false -ldflags="$(LDFLAGS)" MAKEFLAGS += -j$(shell nproc) +ifneq ($(strip $(BASE_IMAGE)),) +$(info BASE_IMAGE is $(BASE_IMAGE)) +endif + all: ebpf-profiler debug: GO_TAGS := $(GO_TAGS),debugtracer @@ -66,10 +70,10 @@ clean: @$(MAKE) -s -C support/ebpf clean @rm -f support/*.test @chmod -Rf u+w go/ || true - @rm -rf go .cache + @rm -rf go/pkg go/.cache .cache generate: - go generate ./... + GOARCH=$(NATIVE_ARCH) go generate ./... ebpf: $(MAKE) $(EBPF_FLAGS) -C support/ebpf @@ -77,22 +81,40 @@ ebpf: ebpf-profiler: generate ebpf go build $(GO_FLAGS) -tags $(GO_TAGS) +PROTOC_GEN_VERSION = "v1.31.0" +PROTOC_GRPC_VERSION = "v1.3.0" GOLANGCI_LINT_VERSION = "v1.60.1" +PORTO_VERSION = "v0.6.0" + +install-grpc-deps: + go install google.golang.org/protobuf/cmd/protoc-gen-go@$(PROTOC_GEN_VERSION) + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@$(PROTOC_GRPC_VERSION) + +install-ci-deps: + go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) + go install github.com/jcchavezs/porto/cmd/porto@$(PORTO_VERSION) + +install-go-deps: install-grpc-deps install-ci-deps + +clean-go-deps: clean + @rm go/bin/protoc* + @rm go/bin/porto* + @rm go/bin/golang* + lint: generate vanity-import-check - go run github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) version - go run github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) run + golangci-lint version + golangci-lint run linter-version: - @echo $(GOLANGCI_LINT_VERSION) + @echo golangci-lint version: $(GOLANGCI_LINT_VERSION) + @echo porto version: $(PORTO_VERSION) .PHONY: vanity-import-check vanity-import-check: - @go install github.com/jcchavezs/porto/cmd/porto@latest @porto --include-internal -l . || ( echo "(run: make vanity-import-fix)"; exit 1 ) .PHONY: vanity-import-fix vanity-import-fix: $(PORTO) - @go install github.com/jcchavezs/porto/cmd/porto@latest @porto --include-internal -w . test: generate ebpf test-deps @@ -118,8 +140,13 @@ integration-test-binaries: generate ebpf ./$(test_name)) || exit ; \ ) +ifeq ($(strip $(BASE_IMAGE)),) docker-image: - docker build -t profiling-agent -f Dockerfile . + docker build -t profiling-agent -f docker-image/Dockerfile . +else +docker-image: + docker build --build-arg image=$(BASE_IMAGE) -t profiling-agent -f docker-image/Dockerfile . +endif agent: docker run -v "$$PWD":/agent -v $(SYSROOT_PATH):/usr/sysroot -it --rm --user $(shell id -u):$(shell id -g) profiling-agent \ diff --git a/Dockerfile b/docker-image/Dockerfile similarity index 57% rename from Dockerfile rename to docker-image/Dockerfile index 6455aa90..1e40db9f 100644 --- a/Dockerfile +++ b/docker-image/Dockerfile @@ -1,31 +1,48 @@ -FROM debian:testing +ARG image=public.ecr.aws/ubuntu/ubuntu:22.04 -WORKDIR /agent +FROM ${image} + +WORKDIR /agent-go + +RUN apt-get update -y \ + && apt-get install -y --no-install-recommends --no-install-suggests \ + ca-certificates \ + gnupg2 \ + software-properties-common \ + wget \ + git \ + findutils \ + unzip \ + make -RUN apt-get update -y && \ - apt-get dist-upgrade -y && \ - apt-get install -y clang-16 git lld-16 make pkgconf unzip wget && \ - apt-get clean autoclean && \ - apt-get autoremove --yes +RUN wget https://apt.llvm.org/llvm.sh \ + && chmod +x llvm.sh \ + && ./llvm.sh 17 COPY go.mod /tmp/go.mod # Extract Go version from go.mod RUN GO_VERSION=$(grep -oPm1 '^go \K([[:digit:].]+)' /tmp/go.mod) && \ GOARCH=$(uname -m) && if [ "$GOARCH" = "x86_64" ]; then GOARCH=amd64; elif [ "$GOARCH" = "aarch64" ]; then GOARCH=arm64; fi && \ - wget -qO- https://golang.org/dl/go${GO_VERSION}.linux-${GOARCH}.tar.gz | tar -C /usr/local -xz + wget https://golang.org/dl/go${GO_VERSION}.linux-${GOARCH}.tar.gz && \ + tar -C /usr/local -xvzf ./go${GO_VERSION}.linux-${GOARCH}.tar.gz && \ + rm -rf ./go${GO_VERSION}.linux-${GOARCH}.tar.gz # Set Go environment variables -ENV GOPATH="/agent/go" -ENV GOCACHE="/agent/.cache" +ENV GOPATH="/agent-go" +ENV GOCACHE="$GOPATH/.cache" +ENV GOBIN="$GOPATH/bin" +# not addin GOBIN on purpose, see entrypoint ENV PATH="/usr/local/go/bin:$PATH" # gRPC dependencies RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.31.0 RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0 +RUN go install github.com/jcchavezs/porto/cmd/porto@v0.6.0 +RUN go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.1 RUN \ PB_URL="https://github.com/protocolbuffers/protobuf/releases/download/v24.4/"; \ - PB_FILE="protoc-24.4-linux-x86_64.zip"; \ + PB_FILE="protoc-24.4-linux-x86_64.zip"; \ INSTALL_DIR="/usr/local"; \ \ wget -q "$PB_URL/$PB_FILE" \ @@ -36,6 +53,10 @@ RUN && rm "$PB_FILE" # Append to /etc/profile for login shells -RUN echo 'export PATH="/usr/local/go/bin:$PATH"' >> /etc/profile +RUN echo 'export PATH="$PATH"' >> /etc/profile + +WORKDIR /agent -ENTRYPOINT ["/bin/bash", "-l", "-c"] +COPY docker-image/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker-image/entrypoint.sh b/docker-image/entrypoint.sh new file mode 100755 index 00000000..12a5bb66 --- /dev/null +++ b/docker-image/entrypoint.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +ORIG_GOPATH="/agent-go" + +NEW_GOPATH="/agent/go" +GOPATH="$NEW_GOPATH" +GOCACHE="$GOPATH/.cache" +GOBIN="$GOPATH/bin" +PATH="$PATH:$GOBIN" +GOLANGCI_LINT_CACHE=$GOCACHE + +# Check if /agent/go exists, and create it if not +if [ ! -d "${GOPATH}" ]; then + mkdir -p ${GOPATH} + mkdir -p ${GOBIN} +fi + +cp --recursive $ORIG_GOPATH/bin/* $GOBIN + +export GOPATH +export GOCACHE +export GOBIN +export PATH +export GOLANGCI_LINT_CACHE + +git config --global --add safe.directory /agent + +# Run the actual command (e.g., bash or other processes) +exec $@ diff --git a/go/bin/.gitkeep b/go/bin/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/support/ebpf/Makefile b/support/ebpf/Makefile index a271e745..528b6c73 100644 --- a/support/ebpf/Makefile +++ b/support/ebpf/Makefile @@ -1,7 +1,7 @@ SHELL ?= bash -BPF_CLANG ?= clang-16 -BPF_LINK ?= llvm-link-16 -LLC ?= llc-16 +BPF_CLANG ?= clang-17 +BPF_LINK ?= llvm-link-17 +LLC ?= llc-17 DEBUG_FLAGS = -DOPTI_DEBUG -g diff --git a/support/ebpf/print_instruction_count.sh b/support/ebpf/print_instruction_count.sh index c1155690..3105191c 100755 --- a/support/ebpf/print_instruction_count.sh +++ b/support/ebpf/print_instruction_count.sh @@ -7,9 +7,9 @@ export SHELLOPTS file="$1" -OBJDUMP_CMD=llvm-objdump +OBJDUMP_CMD=llvm-objdump-17 if ! type -p "${OBJDUMP_CMD}"; then - OBJDUMP_CMD=llvm-objdump-16 + OBJDUMP_CMD=llvm-objdump-17 fi echo -e "\nInstruction counts for ${file}:\n"