Skip to content

Commit

Permalink
runtime: Allocate default workload vcpus
Browse files Browse the repository at this point in the history 
- similar to the static_sandbox_default_workload_mem option,
  assign a default number of vcpus to the VM when no limits
  are given, 1 vcpu in this case
- similar to commit c7b8ee9, do not allocate additional vcpus
  when limits are provided

Signed-off-by: Manuel Huber <[email protected]>
  • Loading branch information
@ms-mahuber
ms-mahuber committed Jan 4, 2025
1 parent 3627201 commit 35d4c62
Show file tree
Hide file tree
Showing 9 changed files with 51 additions and 27 deletions.
6 changes: 4 additions & 2 deletions src/runtime/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ STRATOVIRTPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTCMD)
STRATOVIRTVALIDHYPERVISORPATHS := [\"$(STRATOVIRTPATH)\"]

# Default number of vCPUs
DEFVCPUS := 1
DEFVCPUS ?= 1
# Default maximum number of vCPUs
DEFMAXVCPUS := 0
# Default memory size in MiB
Expand Down Expand Up @@ -263,8 +263,9 @@ DEFSANDBOXCGROUPONLY ?= false
DEFSTATICRESOURCEMGMT ?= false
DEFSTATICRESOURCEMGMT_TEE = true

# Default memory for use for workloads within the sandbox if no specific workload memory value is requested
# Default memory and vcpus for use for workloads within the sandbox if no specific workload values are requested
DEFSTATICSANDBOXWORKLOADMEM ?= 2048
DEFSTATICSANDBOXWORKLOADVCPUS ?= 1

DEFBINDMOUNTS := []

Expand Down Expand Up @@ -694,6 +695,7 @@ USER_VARS += DEFSTATICRESOURCEMGMT_FC
USER_VARS += DEFSTATICRESOURCEMGMT_STRATOVIRT
USER_VARS += DEFSTATICRESOURCEMGMT_TEE
USER_VARS += DEFSTATICSANDBOXWORKLOADMEM
USER_VARS += DEFSTATICSANDBOXWORKLOADVCPUS
USER_VARS += DEFBINDMOUNTS
USER_VARS += DEFSERVICEOFFLOAD
USER_VARS += DEFVFIOMODE
Expand Down
7 changes: 6 additions & 1 deletion src/runtime/config/configuration-clh-snp.toml.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ kernel_params = "@KERNELPARAMS@"
# < 0 --> will be set to the actual number of physical cores
# > 0 <= number of physical cores --> will be set to the specified number
# > number of physical cores --> will be set to the actual number of physical cores
default_vcpus = 1
default_vcpus = @DEFVCPUS@

# Default maximum number of vCPUs per SB/VM:
# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number
Expand Down Expand Up @@ -398,6 +398,11 @@ static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_TEE@
# default amount of memory available within the sandbox.
static_sandbox_default_workload_mem=@DEFSTATICSANDBOXWORKLOADMEM@

# If set, the runtime will use the value as the default number of vcpus for the sandbox when no workload vcpu request is passed
# down to the shim via the OCI when static sandbox resource management is enabled. With this, we ensure that workloads have a proper
# default amount of vcpus available within the sandbox.
static_sandbox_default_workload_vcpus=@DEFSTATICSANDBOXWORKLOADVCPUS@

# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.
# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts`
Expand Down
7 changes: 6 additions & 1 deletion src/runtime/config/configuration-clh.toml.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ kernel_params = "@KERNELPARAMS@"
# < 0 --> will be set to the actual number of physical cores
# > 0 <= number of physical cores --> will be set to the specified number
# > number of physical cores --> will be set to the actual number of physical cores
default_vcpus = 1
default_vcpus = @DEFVCPUS@

# Default maximum number of vCPUs per SB/VM:
# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number
Expand Down Expand Up @@ -418,6 +418,11 @@ static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_CLH@
# default amount of memory available within the sandbox.
static_sandbox_default_workload_mem=@DEFSTATICSANDBOXWORKLOADMEM@

# If set, the runtime will use the value as the default number of vcpus for the sandbox when no workload vcpu request is passed
# down to the shim via the OCI when static sandbox resource management is enabled. With this, we ensure that workloads have a proper
# default amount of vcpus available within the sandbox.
static_sandbox_default_workload_vcpus=@DEFSTATICSANDBOXWORKLOADVCPUS@

# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.
# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts`
Expand Down
2 changes: 1 addition & 1 deletion src/runtime/pkg/katautils/config-settings.go.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ var systemdUnitName = "kata-containers.target"
const defaultKernelParams = ""
const defaultMachineType = "q35"

const defaultVCPUCount uint32 = 1
const defaultVCPUCount uint32 = 0
const defaultMaxVCPUCount uint32 = 0
const defaultMemSize uint32 = 2048 // MiB
const defaultMemSlots uint32 = 10
Expand Down
38 changes: 20 additions & 18 deletions src/runtime/pkg/katautils/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,24 +170,25 @@ type hypervisor struct {
}

type runtime struct {
InterNetworkModel string `toml:"internetworking_model"`
JaegerEndpoint string `toml:"jaeger_endpoint"`
JaegerUser string `toml:"jaeger_user"`
JaegerPassword string `toml:"jaeger_password"`
VfioMode string `toml:"vfio_mode"`
GuestSeLinuxLabel string `toml:"guest_selinux_label"`
SandboxBindMounts []string `toml:"sandbox_bind_mounts"`
Experimental []string `toml:"experimental"`
Tracing bool `toml:"enable_tracing"`
DisableNewNetNs bool `toml:"disable_new_netns"`
DisableGuestSeccomp bool `toml:"disable_guest_seccomp"`
EnableVCPUsPinning bool `toml:"enable_vcpus_pinning"`
Debug bool `toml:"enable_debug"`
SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"`
StaticSandboxResourceMgmt bool `toml:"static_sandbox_resource_mgmt"`
EnablePprof bool `toml:"enable_pprof"`
DisableGuestEmptyDir bool `toml:"disable_guest_empty_dir"`
StaticSandboxWorkloadDefaultMem uint32 `toml:"static_sandbox_default_workload_mem"`
InterNetworkModel string `toml:"internetworking_model"`
JaegerEndpoint string `toml:"jaeger_endpoint"`
JaegerUser string `toml:"jaeger_user"`
JaegerPassword string `toml:"jaeger_password"`
VfioMode string `toml:"vfio_mode"`
GuestSeLinuxLabel string `toml:"guest_selinux_label"`
SandboxBindMounts []string `toml:"sandbox_bind_mounts"`
Experimental []string `toml:"experimental"`
Tracing bool `toml:"enable_tracing"`
DisableNewNetNs bool `toml:"disable_new_netns"`
DisableGuestSeccomp bool `toml:"disable_guest_seccomp"`
EnableVCPUsPinning bool `toml:"enable_vcpus_pinning"`
Debug bool `toml:"enable_debug"`
SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"`
StaticSandboxResourceMgmt bool `toml:"static_sandbox_resource_mgmt"`
EnablePprof bool `toml:"enable_pprof"`
DisableGuestEmptyDir bool `toml:"disable_guest_empty_dir"`
StaticSandboxWorkloadDefaultMem uint32 `toml:"static_sandbox_default_workload_mem"`
StaticSandboxWorkloadDefaultVcpus float32 `toml:"static_sandbox_default_workload_vcpus"`
}

type agent struct {
Expand Down Expand Up @@ -1588,6 +1589,7 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
config.GuestSeLinuxLabel = tomlConf.Runtime.GuestSeLinuxLabel
config.StaticSandboxResourceMgmt = tomlConf.Runtime.StaticSandboxResourceMgmt
config.StaticSandboxWorkloadDefaultMem = tomlConf.Runtime.StaticSandboxWorkloadDefaultMem
config.StaticSandboxWorkloadDefaultVcpus = tomlConf.Runtime.StaticSandboxWorkloadDefaultVcpus
config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly
config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs
config.EnablePprof = tomlConf.Runtime.EnablePprof
Expand Down
12 changes: 10 additions & 2 deletions src/runtime/pkg/oci/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,9 @@ type RuntimeConfig struct {
// Memory to allocate for workloads within the sandbox when workload memory is unspecified
StaticSandboxWorkloadDefaultMem uint32

// vcpus to allocate for workloads within the sandbox when workload vcpus is unspecified
StaticSandboxWorkloadDefaultVcpus float32

// Determines if create a netns for hypervisor process
DisableNewNetNs bool

Expand Down Expand Up @@ -996,6 +999,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st

StaticWorkloadDefaultMem: runtime.StaticSandboxWorkloadDefaultMem,

StaticWorkloadDefaultVcpus: runtime.StaticSandboxWorkloadDefaultVcpus,

ShmSize: shmSize,

VfioMode: runtime.VfioMode,
Expand All @@ -1022,11 +1027,14 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st
// with the base number of CPU/memory (which is equal to the default CPU/memory specified for the runtime
// configuration or annotations) as well as any specified workload resources.
if sandboxConfig.StaticResourceMgmt {
// If no Limits are set in pod config, use StaticWorkloadDefaultMem to ensure the containers generally
// have a reasonable amount of memory available
// If no Limits are set in pod config, use StaticWorkloadDefaultMem/Vcpus to ensure the containers generally
// have a reasonable amount of resources available
if sandboxConfig.SandboxResources.WorkloadMemMB == 0 {
sandboxConfig.SandboxResources.WorkloadMemMB = sandboxConfig.StaticWorkloadDefaultMem
}
if sandboxConfig.SandboxResources.WorkloadCPUs == 0 {
sandboxConfig.SandboxResources.WorkloadCPUs = sandboxConfig.StaticWorkloadDefaultVcpus
}

sandboxConfig.SandboxResources.BaseCPUs = sandboxConfig.HypervisorConfig.NumVCPUsF
sandboxConfig.SandboxResources.BaseMemMB = sandboxConfig.HypervisorConfig.MemorySize
Expand Down
2 changes: 1 addition & 1 deletion src/runtime/virtcontainers/hypervisor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ const (

procCPUInfo = "/proc/cpuinfo"

defaultVCPUs = float32(1)
defaultVCPUs = float32(0)
// 2 GiB
defaultMemSzMiB = 2048

Expand Down
2 changes: 2 additions & 0 deletions src/runtime/virtcontainers/sandbox.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,8 @@ type SandboxConfig struct {

StaticWorkloadDefaultMem uint32

StaticWorkloadDefaultVcpus float32

// Memory to allocate for workloads within the sandbox when workload memory is unspecified
ShmSize uint64

Expand Down
2 changes: 1 addition & 1 deletion tools/osbuilder/node-builder/azure-linux/package_build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ source "${common_file}"

# these options ensure we produce the proper CLH config file
runtime_make_flags="SKIP_GO_VERSION_CHECK=1 QEMUCMD= FCCMD= ACRNCMD= STRATOVIRTCMD= DEFAULT_HYPERVISOR=cloud-hypervisor
DEFMEMSZ=0 DEFSTATICSANDBOXWORKLOADMEM=512 DEFVIRTIOFSDAEMON=${VIRTIOFSD_BINARY_LOCATION} PREFIX=${INSTALL_PATH_PREFIX}"
DEFMEMSZ=0 DEFSTATICSANDBOXWORKLOADMEM=512 DEFVCPUS=0 DEFSTATICSANDBOXWORKLOADVCPUS=1 DEFVIRTIOFSDAEMON=${VIRTIOFSD_BINARY_LOCATION} PREFIX=${INSTALL_PATH_PREFIX}"

# - for vanilla Kata we use the kernel binary. For ConfPods we use IGVM, so no need to provide kernel path.
# - for vanilla Kata we explicitly set DEFSTATICRESOURCEMGMT_CLH. For ConfPods,
Expand Down

0 comments on commit 35d4c62

Please sign in to comment.