From ee00f078f4e5f003759ee245ebd77546aeee35ac Mon Sep 17 00:00:00 2001 From: lengrongfu <1275177125@qq.com> Date: Mon, 13 Nov 2023 10:57:37 +0800 Subject: [PATCH] add support container runtime use systemd cgroups Signed-off-by: lengrongfu --- pkg/benchmark/pod.go | 9 ++++-- pkg/benchmark/pod_container.go | 7 +++-- pkg/common/pod_config.go | 39 ++++++++++++++++++++++++++ pkg/framework/util.go | 16 +++++++---- pkg/validate/multi_container_linux.go | 5 ++++ pkg/validate/networking.go | 17 +++++++++-- pkg/validate/pod.go | 6 +++- pkg/validate/pod_linux.go | 6 ++-- pkg/validate/security_context_linux.go | 7 +++++ 9 files changed, 96 insertions(+), 16 deletions(-) create mode 100644 pkg/common/pod_config.go diff --git a/pkg/benchmark/pod.go b/pkg/benchmark/pod.go index a781152518..32029b127b 100644 --- a/pkg/benchmark/pod.go +++ b/pkg/benchmark/pod.go @@ -22,6 +22,7 @@ import ( "path" "time" + "github.com/kubernetes-sigs/cri-tools/pkg/common" "github.com/kubernetes-sigs/cri-tools/pkg/framework" "github.com/sirupsen/logrus" internalapi "k8s.io/cri-api/pkg/apis" @@ -85,11 +86,13 @@ var _ = framework.KubeDescribe("PodSandbox", func() { podSandboxName := "PodSandbox-for-creating-performance-test-" + framework.NewUUID() uid := framework.DefaultUIDPrefix + framework.NewUUID() namespace := framework.DefaultNamespacePrefix + framework.NewUUID() - + cgroupParent := common.GetCgroupParent(context.TODO(), c) config := &runtimeapi.PodSandboxConfig{ Metadata: framework.BuildPodSandboxMetadata(podSandboxName, uid, namespace, framework.DefaultAttempt), - Linux: &runtimeapi.LinuxPodSandboxConfig{}, - Labels: framework.DefaultPodLabels, + Linux: &runtimeapi.LinuxPodSandboxConfig{ + CgroupParent: cgroupParent, + }, + Labels: framework.DefaultPodLabels, } By(fmt.Sprintf("Creating a pod %d", idx)) diff --git a/pkg/benchmark/pod_container.go b/pkg/benchmark/pod_container.go index e1d2c398c6..014c0e1016 100644 --- a/pkg/benchmark/pod_container.go +++ b/pkg/benchmark/pod_container.go @@ -19,6 +19,7 @@ package benchmark import ( "context" + "github.com/kubernetes-sigs/cri-tools/pkg/common" "github.com/kubernetes-sigs/cri-tools/pkg/framework" internalapi "k8s.io/cri-api/pkg/apis" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" @@ -62,10 +63,12 @@ var _ = framework.KubeDescribe("PodSandbox", func() { podSandboxName := "PodSandbox-for-creating-pod-and-container-performance-test-" + framework.NewUUID() uid := framework.DefaultUIDPrefix + framework.NewUUID() namespace := framework.DefaultNamespacePrefix + framework.NewUUID() - + cgroupParent := common.GetCgroupParent(context.TODO(), rc) config := &runtimeapi.PodSandboxConfig{ Metadata: framework.BuildPodSandboxMetadata(podSandboxName, uid, namespace, framework.DefaultAttempt), - Linux: &runtimeapi.LinuxPodSandboxConfig{}, + Linux: &runtimeapi.LinuxPodSandboxConfig{ + CgroupParent: cgroupParent, + }, } benchmark := func() { diff --git a/pkg/common/pod_config.go b/pkg/common/pod_config.go new file mode 100644 index 0000000000..5d2538c998 --- /dev/null +++ b/pkg/common/pod_config.go @@ -0,0 +1,39 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "context" + + internalapi "k8s.io/cri-api/pkg/apis" + runtimev1 "k8s.io/cri-api/pkg/apis/runtime/v1" +) + +func GetCgroupParent(ctx context.Context, c internalapi.RuntimeService) string { + runtimeConfig, err := c.RuntimeConfig(ctx) + if err != nil { + return "" + } + if runtimeConfig == nil || runtimeConfig.Linux == nil { + return "" + } + cgroupDriver := runtimeConfig.Linux.GetCgroupDriver() + if cgroupDriver == runtimev1.CgroupDriver_CGROUPFS { + return "" + } + return "/test.slice" +} diff --git a/pkg/framework/util.go b/pkg/framework/util.go index 0990e6364e..0764ed3fe0 100644 --- a/pkg/framework/util.go +++ b/pkg/framework/util.go @@ -26,6 +26,7 @@ import ( "github.com/distribution/reference" "github.com/google/uuid" + "github.com/kubernetes-sigs/cri-tools/pkg/common" "gopkg.in/yaml.v3" internalapi "k8s.io/cri-api/pkg/apis" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" @@ -192,11 +193,13 @@ func RunDefaultPodSandbox(c internalapi.RuntimeService, prefix string) string { podSandboxName := prefix + NewUUID() uid := DefaultUIDPrefix + NewUUID() namespace := DefaultNamespacePrefix + NewUUID() - + cgroupParent := common.GetCgroupParent(context.TODO(), c) config := &runtimeapi.PodSandboxConfig{ Metadata: BuildPodSandboxMetadata(podSandboxName, uid, namespace, DefaultAttempt), - Linux: &runtimeapi.LinuxPodSandboxConfig{}, - Labels: DefaultPodLabels, + Linux: &runtimeapi.LinuxPodSandboxConfig{ + CgroupParent: cgroupParent, + }, + Labels: DefaultPodLabels, } return RunPodSandbox(c, config) } @@ -223,10 +226,13 @@ func CreatePodSandboxForContainer(c internalapi.RuntimeService) (string, *runtim podSandboxName := "create-PodSandbox-for-container-" + NewUUID() uid := DefaultUIDPrefix + NewUUID() namespace := DefaultNamespacePrefix + NewUUID() + cgroupParent := common.GetCgroupParent(context.TODO(), c) config := &runtimeapi.PodSandboxConfig{ Metadata: BuildPodSandboxMetadata(podSandboxName, uid, namespace, DefaultAttempt), - Linux: &runtimeapi.LinuxPodSandboxConfig{}, - Labels: DefaultPodLabels, + Linux: &runtimeapi.LinuxPodSandboxConfig{ + CgroupParent: cgroupParent, + }, + Labels: DefaultPodLabels, } podID := RunPodSandbox(c, config) diff --git a/pkg/validate/multi_container_linux.go b/pkg/validate/multi_container_linux.go index ef13db8fd2..cbab9f7fc8 100644 --- a/pkg/validate/multi_container_linux.go +++ b/pkg/validate/multi_container_linux.go @@ -23,6 +23,7 @@ import ( "strings" "time" + "github.com/kubernetes-sigs/cri-tools/pkg/common" "github.com/kubernetes-sigs/cri-tools/pkg/framework" internalapi "k8s.io/cri-api/pkg/apis" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" @@ -113,6 +114,7 @@ func createMultiContainerTestPodSandbox(c internalapi.RuntimeService) (string, * uid := framework.DefaultUIDPrefix + framework.NewUUID() namespace := framework.DefaultNamespacePrefix + framework.NewUUID() logDir, podLogPath := createLogTempDir(podSandboxName) + cgroupParent := common.GetCgroupParent(context.TODO(), c) podConfig := &runtimeapi.PodSandboxConfig{ Metadata: framework.BuildPodSandboxMetadata(podSandboxName, uid, namespace, framework.DefaultAttempt), LogDirectory: podLogPath, @@ -122,6 +124,9 @@ func createMultiContainerTestPodSandbox(c internalapi.RuntimeService) (string, * }, }, Labels: framework.DefaultPodLabels, + Linux: &runtimeapi.LinuxPodSandboxConfig{ + CgroupParent: cgroupParent, + }, } return framework.RunPodSandbox(c, podConfig), podConfig, logDir } diff --git a/pkg/validate/networking.go b/pkg/validate/networking.go index 7a9aab2c0c..d2371d96c4 100644 --- a/pkg/validate/networking.go +++ b/pkg/validate/networking.go @@ -23,6 +23,7 @@ import ( "strings" "time" + "github.com/kubernetes-sigs/cri-tools/pkg/common" "github.com/kubernetes-sigs/cri-tools/pkg/framework" internalapi "k8s.io/cri-api/pkg/apis" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" @@ -132,10 +133,14 @@ func createPodSandWithHostname(c internalapi.RuntimeService, hostname string) (s podSandboxName := "create-PodSandbox-with-hostname" + framework.NewUUID() uid := framework.DefaultUIDPrefix + framework.NewUUID() namespace := framework.DefaultNamespacePrefix + framework.NewUUID() + cgroupParent := common.GetCgroupParent(context.TODO(), c) config := &runtimeapi.PodSandboxConfig{ Metadata: framework.BuildPodSandboxMetadata(podSandboxName, uid, namespace, framework.DefaultAttempt), Hostname: hostname, Labels: framework.DefaultPodLabels, + Linux: &runtimeapi.LinuxPodSandboxConfig{ + CgroupParent: cgroupParent, + }, } podID := framework.RunPodSandbox(c, config) @@ -147,6 +152,7 @@ func createPodSandWithDNSConfig(c internalapi.RuntimeService) (string, *runtimea podSandboxName := "create-PodSandbox-with-DNS-config" + framework.NewUUID() uid := framework.DefaultUIDPrefix + framework.NewUUID() namespace := framework.DefaultNamespacePrefix + framework.NewUUID() + cgroupParent := common.GetCgroupParent(context.TODO(), c) config := &runtimeapi.PodSandboxConfig{ Metadata: framework.BuildPodSandboxMetadata(podSandboxName, uid, namespace, framework.DefaultAttempt), DnsConfig: &runtimeapi.DNSConfig{ @@ -154,7 +160,9 @@ func createPodSandWithDNSConfig(c internalapi.RuntimeService) (string, *runtimea Searches: []string{defaultDNSSearch}, Options: []string{defaultDNSOption}, }, - Linux: &runtimeapi.LinuxPodSandboxConfig{}, + Linux: &runtimeapi.LinuxPodSandboxConfig{ + CgroupParent: cgroupParent, + }, Labels: framework.DefaultPodLabels, } @@ -167,11 +175,14 @@ func createPodSandboxWithPortMapping(c internalapi.RuntimeService, portMappings podSandboxName := "create-PodSandbox-with-port-mapping" + framework.NewUUID() uid := framework.DefaultUIDPrefix + framework.NewUUID() namespace := framework.DefaultNamespacePrefix + framework.NewUUID() + cgroupParent := common.GetCgroupParent(context.TODO(), c) config := &runtimeapi.PodSandboxConfig{ Metadata: framework.BuildPodSandboxMetadata(podSandboxName, uid, namespace, framework.DefaultAttempt), PortMappings: portMappings, - Linux: &runtimeapi.LinuxPodSandboxConfig{}, - Labels: framework.DefaultPodLabels, + Linux: &runtimeapi.LinuxPodSandboxConfig{ + CgroupParent: cgroupParent, + }, + Labels: framework.DefaultPodLabels, } if hostNet { config.Linux.SecurityContext = &runtimeapi.LinuxSandboxSecurityContext{ diff --git a/pkg/validate/pod.go b/pkg/validate/pod.go index 82e7a4d4f4..65b1029d50 100644 --- a/pkg/validate/pod.go +++ b/pkg/validate/pod.go @@ -21,6 +21,7 @@ import ( "os" "path/filepath" + "github.com/kubernetes-sigs/cri-tools/pkg/common" "github.com/kubernetes-sigs/cri-tools/pkg/framework" internalapi "k8s.io/cri-api/pkg/apis" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" @@ -175,10 +176,13 @@ func createPodSandboxWithLogDirectory(c internalapi.RuntimeService) (string, *ru namespace := framework.DefaultNamespacePrefix + framework.NewUUID() hostPath, podLogPath := createLogTempDir(podSandboxName) - + cgroupParent := common.GetCgroupParent(context.TODO(), c) podConfig := &runtimeapi.PodSandboxConfig{ Metadata: framework.BuildPodSandboxMetadata(podSandboxName, uid, namespace, framework.DefaultAttempt), LogDirectory: podLogPath, + Linux: &runtimeapi.LinuxPodSandboxConfig{ + CgroupParent: cgroupParent, + }, } return framework.RunPodSandbox(c, podConfig), podConfig, hostPath } diff --git a/pkg/validate/pod_linux.go b/pkg/validate/pod_linux.go index 5a0c249f28..f197bea31e 100644 --- a/pkg/validate/pod_linux.go +++ b/pkg/validate/pod_linux.go @@ -21,6 +21,7 @@ import ( "strings" "time" + "github.com/kubernetes-sigs/cri-tools/pkg/common" "github.com/kubernetes-sigs/cri-tools/pkg/framework" internalapi "k8s.io/cri-api/pkg/apis" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" @@ -89,11 +90,12 @@ func createSandboxWithSysctls(rc internalapi.RuntimeService, sysctls map[string] podSandboxName := "pod-sandbox-with-sysctls-" + framework.NewUUID() uid := framework.DefaultUIDPrefix + framework.NewUUID() namespace := framework.DefaultNamespacePrefix + framework.NewUUID() - + cgroupParent := common.GetCgroupParent(context.TODO(), rc) podConfig := &runtimeapi.PodSandboxConfig{ Metadata: framework.BuildPodSandboxMetadata(podSandboxName, uid, namespace, framework.DefaultAttempt), Linux: &runtimeapi.LinuxPodSandboxConfig{ - Sysctls: sysctls, + CgroupParent: cgroupParent, + Sysctls: sysctls, }, } return framework.RunPodSandbox(rc, podConfig), podConfig diff --git a/pkg/validate/security_context_linux.go b/pkg/validate/security_context_linux.go index a36600f323..daa82c0599 100644 --- a/pkg/validate/security_context_linux.go +++ b/pkg/validate/security_context_linux.go @@ -26,6 +26,7 @@ import ( "strings" "time" + "github.com/kubernetes-sigs/cri-tools/pkg/common" "github.com/kubernetes-sigs/cri-tools/pkg/framework" internalapi "k8s.io/cri-api/pkg/apis" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" @@ -944,12 +945,14 @@ func createNamespacePodSandbox(rc internalapi.RuntimeService, podSandboxNamespac By("create NamespaceOption podSandbox") uid := framework.DefaultUIDPrefix + framework.NewUUID() namespace := framework.DefaultNamespacePrefix + framework.NewUUID() + cgroupParent := common.GetCgroupParent(context.TODO(), rc) config := &runtimeapi.PodSandboxConfig{ Metadata: framework.BuildPodSandboxMetadata(podSandboxName, uid, namespace, framework.DefaultAttempt), Linux: &runtimeapi.LinuxPodSandboxConfig{ SecurityContext: &runtimeapi.LinuxSandboxSecurityContext{ NamespaceOptions: podSandboxNamespace, }, + CgroupParent: cgroupParent, }, LogDirectory: podLogPath, Labels: framework.DefaultPodLabels, @@ -1016,12 +1019,14 @@ func createPrivilegedPodSandbox(rc internalapi.RuntimeService, privileged bool) podSandboxName := "create-Privileged-PodSandbox-for-container-" + framework.NewUUID() uid := framework.DefaultUIDPrefix + framework.NewUUID() namespace := framework.DefaultNamespacePrefix + framework.NewUUID() + cgroupParent := common.GetCgroupParent(context.TODO(), rc) config := &runtimeapi.PodSandboxConfig{ Metadata: framework.BuildPodSandboxMetadata(podSandboxName, uid, namespace, framework.DefaultAttempt), Linux: &runtimeapi.LinuxPodSandboxConfig{ SecurityContext: &runtimeapi.LinuxSandboxSecurityContext{ Privileged: privileged, }, + CgroupParent: cgroupParent, }, Labels: framework.DefaultPodLabels, } @@ -1152,12 +1157,14 @@ func seccompTestContainer(rc internalapi.RuntimeService, ic internalapi.ImageMan podSandboxName := "seccomp-sandbox-" + framework.NewUUID() uid := framework.DefaultUIDPrefix + framework.NewUUID() namespace := framework.DefaultNamespacePrefix + framework.NewUUID() + cgroupParent := common.GetCgroupParent(context.TODO(), rc) podConfig := &runtimeapi.PodSandboxConfig{ Metadata: framework.BuildPodSandboxMetadata(podSandboxName, uid, namespace, framework.DefaultAttempt), Linux: &runtimeapi.LinuxPodSandboxConfig{ SecurityContext: &runtimeapi.LinuxSandboxSecurityContext{ Seccomp: profile, }, + CgroupParent: cgroupParent, }, Labels: framework.DefaultPodLabels, }