diff --git a/.clang-format b/.clang-format index c58fbe7..f7e7406 100644 --- a/.clang-format +++ b/.clang-format @@ -13,10 +13,30 @@ Language: Cpp AccessModifierOffset: -8 AlignAfterOpenBracket: Align AlignArrayOfStructures: None -AlignConsecutiveMacros: Consecutive -AlignConsecutiveAssignments: Consecutive -AlignConsecutiveBitFields: None -AlignConsecutiveDeclarations: Consecutive +AlignConsecutiveAssignments: + Enabled: true + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + PadOperators: true +AlignConsecutiveBitFields: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + PadOperators: true +AlignConsecutiveDeclarations: + Enabled: true + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + PadOperators: true +AlignConsecutiveMacros: + Enabled: true + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + PadOperators: true AlignEscapedNewlines: Right AlignOperands: Align AlignTrailingComments: true @@ -49,6 +69,11 @@ AttributeMacros: - RELEASE_READ - REQUIRE_READ - EXCLUDE_READ + - ACQUIRE_RCU_READ + - TRY_ACQUIRE_RCU_READ + - RELEASE_RCU_READ + - REQUIRE_RCU_READ + - EXCLUDE_RCU_READ - LOCK_IMPL - ACQUIRE_SPINLOCK - ACQUIRE_SPINLOCK_NP @@ -84,7 +109,7 @@ BraceWrapping: SplitEmptyRecord: true SplitEmptyNamespace: true BreakBeforeBinaryOperators: None -BreakBeforeConceptDeclarations: true +BreakBeforeConceptDeclarations: Always BreakBeforeBraces: Custom BreakBeforeInheritanceComma: false BreakInheritanceList: BeforeComma @@ -95,6 +120,7 @@ BreakAfterJavaFieldAnnotations: false BreakStringLiterals: false ColumnLimit: 80 CommentPragmas: '^ FIXME:' +QualifierAlignment: Left CompactNamespaces: false ConstructorInitializerIndentWidth: 8 ContinuationIndentWidth: 8 @@ -105,6 +131,10 @@ DisableFormat: false EmptyLineAfterAccessModifier: Never EmptyLineBeforeAccessModifier: LogicalBlock ExperimentalAutoDetectBinPacking: false +PackConstructorInitializers: BinPack +BasedOnStyle: '' +ConstructorInitializerAllOnOneLineOrOnePerLine: false +AllowAllConstructorInitializersOnNextLine: true FixNamespaceComments: true ForEachMacros: - list_foreach @@ -155,9 +185,10 @@ IndentCaseBlocks: false IndentGotoLabels: true IndentPPDirectives: None IndentExternBlock: AfterExternBlock -IndentRequires: false +IndentRequiresClause: false IndentWidth: 8 IndentWrappedFunctionNames: false +InsertBraces: true InsertTrailingCommas: None JavaScriptQuotes: Leave JavaScriptWrapImports: true @@ -176,6 +207,7 @@ PenaltyBreakAssignment: 10 PenaltyBreakBeforeFirstCallParameter: 30 PenaltyBreakComment: 10 PenaltyBreakFirstLessLess: 0 +PenaltyBreakOpenParenthesis: 0 PenaltyBreakString: 10 PenaltyBreakTemplateDeclaration: 10 PenaltyExcessCharacter: 100 @@ -185,6 +217,9 @@ PointerAlignment: Right PPIndentWidth: -1 ReferenceAlignment: Pointer ReflowComments: true +RemoveBracesLLVM: false +RequiresClausePosition: OwnLine +SeparateDefinitionBlocks: Always ShortNamespaceLines: 1 SortIncludes: CaseSensitive SortJavaStaticImport: Before @@ -198,6 +233,16 @@ SpaceBeforeCpp11BracedList: false SpaceBeforeCtorInitializerColon: true SpaceBeforeInheritanceColon: true SpaceBeforeParens: ControlStatements +SpaceBeforeParensOptions: + AfterControlStatements: true + AfterForeachMacros: true + AfterFunctionDefinitionName: false + AfterFunctionDeclarationName: false + AfterIfMacros: true + AfterOverloadedOperator: false + AfterRequiresInClause: false + AfterRequiresInExpression: false + BeforeNonEmptyParentheses: false SpaceAroundPointerQualifiers: Default SpaceBeforeRangeBasedForLoopColon: false SpaceInEmptyBlock: true diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..b5cb7d6 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,16 @@ +root = True + +[*] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +charset = utf-8 +tab_width = 8 + +[*.{c,h,tc,ev,S,hvc}] +indent_style = tab +indent_size = 8 + +[*.{py,md}] +indent_style = space +indent_size = 4 diff --git a/.gitignore b/.gitignore index d0fec74..8bd4bf9 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ /.ninja_log /.sconsign.dblite /build*/ +hyp/**/*.o # Ignore Python precompiled bitcode *.pyc @@ -20,3 +21,10 @@ # Ignore Vim temporary files .*.sw[a-p] .*.un~ + +# Ignore host test temporary files +*.o + +# Ignore documentation output +*.pdf +*.html diff --git a/CHANGELOG.md b/CHANGELOG.md index 53cbb98..55a361d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ -# Changelog +# Status and Changelog -All notable changes to this project will be documented in this file. +This page documents current status, known issues and work in progress. Some of +these may impact your development or hypervisor usage. ## Open Issues @@ -9,15 +10,29 @@ All notable changes to this project will be documented in this file. The Resource Manager being the root-VM, manages creation of the Primary VM (HLOS) and controls the rights to create additional VMs. In the Gunyah Resource Manager design, VM management services are provided by the Resource Manager -(although it is technically possible for these rights to be delegated to other -VMs). -The current Resource Manager does not support Secondary VM loading. Support -will be added in a subsequent contribution. +Gunyah patches are required in Linux and the CrosVM VMM to support SVM loading. -### 2. Virtio support +### Known issues: -Virtio support is under development and should be contributed along with secondary VM support. +- Only QEMU serial communication is tested. Using host Linux networking (qemu + virtio) with adb (network) connection will permit greater flexibility in + connecting to the device. +- SVM booting with Crosvm uses uart emulation, its very slow. +- Crosvm opens the UART console in the current terminal, so it is via the host + uart terminal. We have not configured a way to open multiple terminals yet. +- Debugging a system running QEMU with a remote gdb connection is unstable. + +### Untested scenarios: + +- Launching of multiple SVM's simultaneously from PVM, because of the known + issue of having only one console available. + +### TODO list: + +- Hardcoded platform parameters + + Memory address ranges are hardcoded (get from dtb nodes) + + Dtb address is hardcoded (get from register) ## Unreleased @@ -27,7 +42,7 @@ Unreleased changes in the `develop` branch may be added here. Individual releases are tagged, and the latest release will be available in the `main` branch. -* No releases have been made at this time. +* No tagged releases have been made at this time. ## Contributions @@ -38,4 +53,5 @@ Significant contributions are listed here. This is the initial contribution of source code to the Gunyah Hypervisor. * Support for QEMU AArch64 Simulator -* Support unmodified Linux Primary VM kernel +* Support unmodified Linux Primary VM kernel or with Gunyah patches for VM loading +* Support unmodified Linux Secondary VM kernel diff --git a/README.md b/README.md index 825bbe1..bf25cad 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,27 @@ -![Qualcomm Innovation Center.](docs/images/logo-quic-on%40h68.png) +[Qualcomm Innovation Center](https://github.com/quic) # Gunyah Hypervisor +Gunyah is a high performance, scalable and flexible hypervisor built for +demanding battery powered, real-time, safety and security use cases. + +The Gunyah Hypervisor open source project provides a reference Type-1 +hypervisor configuration suitable for general purpose hosting of multiple +trusted and dependent VMs. + +## Gunyah Origins + *Gunyah* is an Australian Aboriginal word. See: https://en.wiktionary.org/wiki/gunyah The Gunyah Hypervisor was developed by Qualcomm in Sydney Australia. ## Type-1 Hypervisor Concept -Gunyah is a Type-1 hypervisor, meaning that it is independent of any high-level -OS kernel, and runs in a higher CPU privilege level. It does not depend on any -lower-privileged OS kernel/code for its core functionality. This increases its -security and can support a much smaller trusted computing base than a Type-2 -hypervisor. +Gunyah is a Type-1 hypervisor, meaning that it runs independently of any +high-level OS kernel - such as Linux, and runs in a higher CPU privilege level +than VMs. It does not depend on any lower-privileged OS kernel/code for its +core functionality. This increases its security and can support a much smaller +trusted computing base than a Type-2 like hosted-hypervisors. Gunyah's design principle is not dissimilar to a traditional microkernel in that it provides only a minimal set of critical services to its clients, and @@ -21,15 +30,16 @@ less-privileged) processes, wherever this is possible without an adverse impact on performance or security. The hypervisor uses the CPU's virtualization mode and features to isolate -itself from OS kernels in VMs. On ARM, this includes trapping privileged -registers, using GIC virtualization support, and the Stage-2 MMU to provide -isolated VMs in EL1/0. +itself from OS kernels in VMs and isolate VMs from each other. On ArM, this +includes trapping and emulating registers as required, virtualizing core +platform devices, Arm's GIC virtualization support, and the CPU's Stage-2 MMU +to provide isolated VMs in EL1/0. ## Why Gunyah -- **strong security**: Mobile payments, secure user-interface, and many more security sensitive use-cases all require strong security. Gunyah's design is suited to providing strong isolation guarantees and its small size is conducive to audit. -- **performance**: Mobile devices are particularly demanding. Battery powered devices demand low software overheads to get the most performance per-watt. Gunyah is designed to have high performance with minimal impact to high-level operating systems. -- **modularity**: The hypervisor is designed to be modular, allowing customization and enhancement by swapping out module implementations and adding new feature via new modules. +- **Strong security**: Mobile payments, secure user-interface, and many more security sensitive use-cases all require strong security. Gunyah's design is suited to providing strong isolation guarantees and its small size is conducive to audit. +- **Performance**: Mobile devices are particularly demanding. Battery powered devices demand low software overheads to get the most performance per-watt. Gunyah is designed to have high performance with minimal impact to high-level operating systems. +- **Modularity**: The hypervisor is designed to be modular, allowing customization and enhancement by swapping out module implementations and adding new feature via new modules. ## Features @@ -41,17 +51,23 @@ isolated VMs in EL1/0. ## Platform Support -Gunyah is architected to support other CPU architectures, so its core design ensures architecture independence and portability in non-architecture specific areas. +Gunyah is architected to support multiple CPU architectures, so its core design +ensures architecture independence and portability in non-architecture specific +areas. -Gunyah currently supports ARMv8.2+ platforms as it uses AArch64 EL2 in VHE mode. Some porting is required to support ARMv8.0. +Gunyah currently supports the ARM64 (ARMv8+) architecure, it uses AArch64 EL2 +in VHE mode by default. -We have developed an initial port of Gunyah to the QEMU ARMv8 simulator. *Note QEMU v5+ is required*. Additional platforms are expected to be supported in future contributions. +We have developed an initial port of Gunyah to the QEMU Arm System emulator. +*Note QEMU v7+ is recommended*. Additional platforms are expected to be +supported in future contributions. ## Getting Started +- [Terminology](docs/terminology.md) - [Setup Instructions](docs/setup.md) + + [Quick Start Instructions](https://github.com/quic/gunyah-support-scripts/blob/develop/quickstart.md) - [Build Instructions](docs/build.md) -- [Testing Instructions](docs/test.md) -- [Changelog](CHANGELOG.md) +- [Status and Changelog](CHANGELOG.md) ## Resources - [Gunyah Hypercall API](docs/api/gunyah_api.md) diff --git a/SConstruct b/SConstruct index 8b8f160..d7c9052 100644 --- a/SConstruct +++ b/SConstruct @@ -10,8 +10,10 @@ import os env_vars = { 'PATH': os.environ['PATH'], - 'LLVM': os.environ['LLVM'], } +if 'LLVM' in os.environ: + env_vars['LLVM'] = os.environ['LLVM'] + env = Environment(tools={}, SCANNERS=[], BUILDERS={}, ENV=env_vars) configure.SConsBuild(env, Builder, Action, arguments=SCons.Script.ARGUMENTS)() diff --git a/config/arch/aarch64.conf b/config/arch/aarch64.conf index d718fd4..a5c6221 100644 --- a/config/arch/aarch64.conf +++ b/config/arch/aarch64.conf @@ -6,6 +6,5 @@ is_abi # Use the Linux target because it knows how to link with LLD target_triple aarch64-linux-gnu defines_link -defines_registers flags -mgeneral-regs-only -mtp=el2 configs ARCH_IS_64BIT=1 ARCH_ENDIAN_LITTLE=1 diff --git a/config/arch/cortex-a-v8_0.conf b/config/arch/cortex-a-v8_0.conf new file mode 100644 index 0000000..7357280 --- /dev/null +++ b/config/arch/cortex-a-v8_0.conf @@ -0,0 +1,28 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +base_arch armv8-64 + +configs ARCH_AARCH64_32BIT_EL0=1 +configs ARCH_AARCH64_32BIT_EL0_ALL_CORES=1 +# FIXME +configs ARCH_AARCH64_32BIT_EL1=0 + +# Mandatory architecture extensions in v8.0 +configs ARCH_ARM_PMU_VER=3 + +# The number of implemented ICH_LR_EL2 registers. +configs CPU_GICH_LR_COUNT=4U + +# The number of implemented ICH_APR[01]R_EL2 registers. +configs CPU_GICH_APR_COUNT=1U + +# The number of implemented DBGB[CV]R_EL1 (HW breakpoint) registers. +configs CPU_DEBUG_BP_COUNT=6U + +# The number of implemented DBGW[CV]R_EL1 (HW watchpoint) registers. +configs CPU_DEBUG_WP_COUNT=4U + +# These CPUs always have an ETM. +configs PLATFORM_HAS_NO_ETM_BASE=0 diff --git a/config/arch/cortex-a-v8_2.conf b/config/arch/cortex-a-v8_2.conf index 0091510..b3b8c14 100644 --- a/config/arch/cortex-a-v8_2.conf +++ b/config/arch/cortex-a-v8_2.conf @@ -5,28 +5,36 @@ base_arch armv8-64 configs ARCH_AARCH64_32BIT_EL0=1 +configs ARCH_AARCH64_32BIT_EL0_ALL_CORES=1 configs ARCH_AARCH64_32BIT_EL1=0 +# Checked for Cortex-(A55,A65,A75,A76,A77,A78,X1) +configs CPU_HAS_NO_ACTLR_EL1=1 +configs CPU_HAS_NO_AMAIR_EL1=1 +configs CPU_HAS_NO_AFSR0_EL1=1 +configs CPU_HAS_NO_AFSR1_EL1=1 + # Mandatory architecture extensions in v8.2 -configs ARCH_ARM_8_0_CSV2=1 -configs ARCH_ARM_8_0_CSV3=1 - -configs ARCH_ARM_8_1_HPD=1 -configs ARCH_ARM_8_1_LSE=1 -configs ARCH_ARM_8_1_LOR=1 -configs ARCH_ARM_8_1_PAN=1 -configs ARCH_ARM_8_1_RDMA=1 -configs ARCH_ARM_8_1_VHE=1 - -configs ARCH_ARM_8_2_A64ISA=1 -configs ARCH_ARM_8_2_ATS1E1=1 -configs ARCH_ARM_8_2_DCPOP=1 -configs ARCH_ARM_8_2_DEBUG=1 -configs ARCH_ARM_8_2_DOTPROD=1 -configs ARCH_ARM_8_2_RAS=1 -configs ARCH_ARM_8_2_TTCNP=1 -configs ARCH_ARM_8_2_TTS2UXN=1 -configs ARCH_ARM_8_2_UAO=1 +configs ARCH_ARM_FEAT_CSV2=1 +configs ARCH_ARM_FEAT_CSV3=1 + +configs ARCH_ARM_FEAT_HPDS=1 +configs ARCH_ARM_FEAT_LSE=1 +configs ARCH_ARM_FEAT_LOR=1 +configs ARCH_ARM_FEAT_PAN=1 +configs ARCH_ARM_FEAT_RDM=1 +configs ARCH_ARM_FEAT_VHE=1 +configs ARCH_ARM_FEAT_CRC32=1 + +configs ARCH_ARM_FEAT_ASMv8p2=1 +configs ARCH_ARM_FEAT_PAN2=1 +configs ARCH_ARM_FEAT_DPB=1 +configs ARCH_ARM_FEAT_DEBUGv8p2=1 +configs ARCH_ARM_FEAT_DotProd=1 +configs ARCH_ARM_FEAT_RAS=1 +configs ARCH_ARM_FEAT_TTCNP=1 +configs ARCH_ARM_FEAT_XNX=1 +configs ARCH_ARM_FEAT_UAO=1 configs ARCH_AARCH64_ASID16=1 ARCH_ARM_PMU_VER=3 @@ -44,9 +52,7 @@ configs CPU_DEBUG_WP_COUNT=4U # The level of support for ARMv8.4-TTRem on this CPU (encoded the same way # as ID_AA64MMFR2_EL1.BBM). -configs CPU_PGTABLE_BLOCK_SPLIT_LEVEL=2U -# The FORCE config is set to ignore the ID register, since these CPUs do not -# implement the extension but for most purposes behave as if they do so at -# level 2; the only difference apart from the ID register is that TLB conflict -# aborts may be reported to EL1 when stage 2 is enabled. -configs CPU_PGTABLE_BLOCK_SPLIT_LEVEL_FORCE=1 +configs CPU_PGTABLE_BBM_LEVEL=0U + +# These CPUs always have an ETM. +configs PLATFORM_HAS_NO_ETM_BASE=0 diff --git a/config/arch/cortex-a53.conf b/config/arch/cortex-a53.conf new file mode 100644 index 0000000..1b85801 --- /dev/null +++ b/config/arch/cortex-a53.conf @@ -0,0 +1,19 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +base_arch cortex-a-v8_0 +flags -mcpu=cortex-a53 + +configs ARCH_CORE_IDS=CORTEX_A53 + +configs CPU_PGTABLE_BBM_LEVEL=0U + +configs CPU_HAS_NO_ACTLR_EL1=1 +configs CPU_HAS_NO_AMAIR_EL1=1 +configs CPU_HAS_NO_AFSR0_EL1=1 +configs CPU_HAS_NO_AFSR1_EL1=1 + +# Fixed fields in cortex-a53 +configs PLATFORM_MPIDR_AFF3_MASK=0 +configs PLATFORM_MPIDR_AFF3_SHIFT=0 diff --git a/config/arch/cortex-a55-a76.conf b/config/arch/cortex-a55-a76.conf index a42412b..6585b53 100644 --- a/config/arch/cortex-a55-a76.conf +++ b/config/arch/cortex-a55-a76.conf @@ -4,5 +4,8 @@ base_arch cortex-a-v8_2 flags -mcpu=cortex-a76 -configs ARCH_AARCH64_USE_PAN=1 +configs ARCH_CORE_IDS=CORTEX_A55,CORTEX_A76 + +configs ARCH_AARCH64_BIG_END_ALL_CORES=1 + configs ARCH_ARM_PMU_HPMN_UNPREDICTABLE=1 diff --git a/config/arch/cortex-a55-a77.conf b/config/arch/cortex-a55-a77.conf index 967024c..3300340 100644 --- a/config/arch/cortex-a55-a77.conf +++ b/config/arch/cortex-a55-a77.conf @@ -4,10 +4,13 @@ base_arch cortex-a-v8_2 flags -mcpu=cortex-a77 -configs ARCH_AARCH64_USE_PAN=1 -configs ARCH_ARM_8_1_VMID16=1 +configs ARCH_CORE_IDS=CORTEX_A55,CORTEX_A77 + +configs ARCH_AARCH64_BIG_END_ALL_CORES=1 + +configs ARCH_ARM_FEAT_VMID16=1 configs ARCH_ARM_PMU_HPMN_UNPREDICTABLE=1 -configs ARCH_ARM_8_1_PMU=1 -configs ARCH_ARM_8_2_IESB=1 -configs ARCH_ARM_8_2_TTPBHA=1 +configs ARCH_ARM_FEAT_PMUv3p1=1 +configs ARCH_ARM_FEAT_IESB=1 +configs ARCH_ARM_FEAT_HPDS2=1 diff --git a/config/arch/cortex-a55-a78-x1.conf b/config/arch/cortex-a55-a78-x1.conf index eb63bb3..68bb977 100644 --- a/config/arch/cortex-a55-a78-x1.conf +++ b/config/arch/cortex-a55-a78-x1.conf @@ -4,10 +4,13 @@ base_arch cortex-a-v8_2 flags -mcpu=cortex-x1 -configs ARCH_AARCH64_USE_PAN=1 -configs ARCH_ARM_8_1_VMID16=1 + +configs ARCH_AARCH64_BIG_END_ALL_CORES=1 + +configs ARCH_CORE_IDS=CORTEX_A55,CORTEX_A78,CORTEX_X1 +configs ARCH_ARM_FEAT_VMID16=1 configs ARCH_ARM_PMU_HPMN_UNPREDICTABLE=1 -configs ARCH_ARM_8_1_PMU=1 -configs ARCH_ARM_8_2_IESB=1 -configs ARCH_ARM_8_2_TTPBHA=1 +configs ARCH_ARM_FEAT_PMUv3p1=1 +configs ARCH_ARM_FEAT_IESB=1 +configs ARCH_ARM_FEAT_HPDS2=1 diff --git a/config/arch/cortex-a55.conf b/config/arch/cortex-a55.conf index 102472b..e12504b 100644 --- a/config/arch/cortex-a55.conf +++ b/config/arch/cortex-a55.conf @@ -4,10 +4,10 @@ base_arch cortex-a-v8_2 flags -mcpu=cortex-a55 -configs ARCH_AARCH64_USE_PAN=1 -configs ARCH_ARM_8_1_VMID16=1 +configs ARCH_CORE_IDS=CORTEX_A55 +configs ARCH_ARM_FEAT_VMID16=1 configs ARCH_ARM_PMU_HPMN_UNPREDICTABLE=1 -configs ARCH_ARM_8_1_PMU=1 -configs ARCH_ARM_8_2_IESB=1 -configs ARCH_ARM_8_2_TTPBHA=1 +configs ARCH_ARM_FEAT_PMUv3p1=1 +configs ARCH_ARM_FEAT_IESB=1 +configs ARCH_ARM_FEAT_HPDS2=1 diff --git a/config/arch/cortex-a78.conf b/config/arch/cortex-a78.conf index 54f9223..07c2af1 100644 --- a/config/arch/cortex-a78.conf +++ b/config/arch/cortex-a78.conf @@ -4,10 +4,10 @@ base_arch cortex-a-v8_2 flags -mcpu=cortex-a78 -configs ARCH_AARCH64_USE_PAN=1 -configs ARCH_ARM_8_1_VMID16=1 +configs ARCH_CORE_IDS=CORTEX_A78 +configs ARCH_ARM_FEAT_VMID16=1 configs ARCH_ARM_PMU_HPMN_UNPREDICTABLE=1 -configs ARCH_ARM_8_1_PMU=1 -configs ARCH_ARM_8_2_IESB=1 -configs ARCH_ARM_8_2_TTPBHA=1 +configs ARCH_ARM_FEAT_PMUv3p1=1 +configs ARCH_ARM_FEAT_IESB=1 +configs ARCH_ARM_FEAT_HPDS2=1 diff --git a/config/arch/gic-500.conf b/config/arch/gic-500.conf new file mode 100644 index 0000000..338c577 --- /dev/null +++ b/config/arch/gic-500.conf @@ -0,0 +1,15 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +configs GICV3_EXT_IRQS=0 +# There may be an ITS, but it is not useful to the hypervisor without vLPIs +configs GICV3_HAS_ITS=0 +configs GICV3_HAS_LPI=0 +configs GICV3_HAS_VLPI=0 +configs GICV3_HAS_VLPI_V4_1=0 +configs GICV3_HAS_1N=1 +configs GICV3_HAS_GICD_ICLAR=0 +configs GICV3_HAS_SECURITY_DISABLED=0 +configs PLATFORM_GICD_BASE=PLATFORM_GIC_BASE +configs PLATFORM_GICR_SIZE=(0x20000*PLATFORM_MAX_CORES) diff --git a/config/arch/gic-600.conf b/config/arch/gic-600.conf new file mode 100644 index 0000000..7a9a45d --- /dev/null +++ b/config/arch/gic-600.conf @@ -0,0 +1,20 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +configs GICV3_EXT_IRQS=0 +# There may be an ITS, but it is not useful to the hypervisor without vLPIs +configs GICV3_HAS_ITS=0 +configs GICV3_HAS_LPI=0 +configs GICV3_HAS_VLPI=0 +configs GICV3_HAS_VLPI_V4_1=0 +configs GICV3_HAS_1N=1 +configs GICV3_HAS_GICD_ICLAR=1 +configs GICV3_HAS_SECURITY_DISABLED=0 +configs PLATFORM_GICD_BASE=PLATFORM_GIC_BASE +configs PLATFORM_GICA_BASE=(PLATFORM_GIC_BASE+0x10000U) +configs PLATFORM_GITS_BASE=(PLATFORM_GIC_BASE+0x40000U) +configs PLATFORM_GITS_SIZE=(0x20000U*PLATFORM_GITS_COUNT) +configs PLATFORM_GICR_BASE=(PLATFORM_GITS_BASE+PLATFORM_GITS_SIZE) +configs PLATFORM_GICR_SIZE=(0x20000U*PLATFORM_GICR_COUNT) +configs PLATFORM_GIC_SIZE=(0x50000U+PLATFORM_GITS_SIZE+PLATFORM_GICR_SIZE) diff --git a/config/arch/gic-700-vlpi.conf b/config/arch/gic-700-vlpi.conf new file mode 100644 index 0000000..24e41d8 --- /dev/null +++ b/config/arch/gic-700-vlpi.conf @@ -0,0 +1,19 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +configs GICV3_EXT_IRQS=0 +configs GICV3_HAS_ITS=1 +configs GICV3_HAS_LPI=1 +configs GICV3_HAS_VLPI=1 +configs GICV3_HAS_VLPI_V4_1=1 +configs GICV3_HAS_1N=1 +configs GICV3_HAS_GICD_ICLAR=1 +configs GICV3_HAS_SECURITY_DISABLED=0 +configs PLATFORM_GICD_BASE=PLATFORM_GIC_BASE +configs PLATFORM_GICA_BASE=(PLATFORM_GIC_BASE+0x10000U) +configs PLATFORM_GITS_BASE=(PLATFORM_GIC_BASE+0x40000U) +configs PLATFORM_GITS_SIZE=(0x40000U*PLATFORM_GITS_COUNT) +configs PLATFORM_GICR_BASE=(PLATFORM_GITS_BASE+PLATFORM_GITS_SIZE) +configs PLATFORM_GICR_SIZE=(0x40000U*PLATFORM_GICR_COUNT) +configs PLATFORM_GIC_SIZE=(0x50000U+PLATFORM_GITS_SIZE+PLATFORM_GICR_SIZE) diff --git a/config/arch/gic-qemu.conf b/config/arch/gic-qemu.conf index 6fb1f46..d0105bf 100644 --- a/config/arch/gic-qemu.conf +++ b/config/arch/gic-qemu.conf @@ -5,12 +5,15 @@ configs GICV3_EXT_IRQS=0 # There may be an ITS, but it is not useful to the hypervisor without vLPIs configs GICV3_HAS_ITS=0 +configs GICV3_HAS_LPI=0 +configs GICV3_HAS_VLPI=0 +configs GICV3_HAS_VLPI_V4_1=0 configs GICV3_HAS_1N=0 configs GICV3_HAS_GICD_ICLAR=0 configs GICV3_HAS_SECURITY_DISABLED=1 configs PLATFORM_GICD_BASE=PLATFORM_GIC_BASE configs PLATFORM_GITS_BASE=(PLATFORM_GIC_BASE+0x80000U) -configs PLATFORM_GITS_SIZE=(0x20000*PLATFORM_GITS_COUNT) +configs PLATFORM_GITS_SIZE=(0x20000U*PLATFORM_GITS_COUNT) configs PLATFORM_GICR_BASE=(PLATFORM_GITS_BASE+PLATFORM_GITS_SIZE) -configs PLATFORM_GICR_SIZE=(0x20000*PLATFORM_MAX_CORES) -configs PLATFORM_GIC_SIZE=(0x50000+PLATFORM_GITS_SIZE+PLATFORM_GICR_SIZE) +configs PLATFORM_GICR_SIZE=(0x20000U*PLATFORM_MAX_CORES) +configs PLATFORM_GIC_SIZE=(0x50000U+PLATFORM_GITS_SIZE+PLATFORM_GICR_SIZE) diff --git a/config/arch/qemu-armv8-5a-rng.conf b/config/arch/qemu-armv8-5a-rng.conf index b6e2ee7..3661e22 100644 --- a/config/arch/qemu-armv8-5a-rng.conf +++ b/config/arch/qemu-armv8-5a-rng.conf @@ -1,8 +1,79 @@ -# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# © 2022 Qualcomm Innovation Center, Inc. All rights reserved. # # SPDX-License-Identifier: BSD-3-Clause -base_arch cortex-a55-a77 +base_arch armv8-64 flags -march=armv8.5-a+rng -configs ARCH_ARM_8_5_RNG=1 +configs PLATFORM_MAX_CORES=8U +configs PLATFORM_USABLE_CORES=0xFFU + +configs PLATFORM_MPIDR_AFF0_MASK=0x7U +configs PLATFORM_MPIDR_AFF0_SHIFT=0 +configs PLATFORM_MPIDR_AFF1_MASK=0U +configs PLATFORM_MPIDR_AFF1_SHIFT=0 +configs PLATFORM_MPIDR_AFF2_MASK=0U +configs PLATFORM_MPIDR_AFF2_SHIFT=0 +configs PLATFORM_MPIDR_AFF3_MASK=0U +configs PLATFORM_MPIDR_AFF3_SHIFT=0 + +configs ARCH_CORE_IDS=QEMU + +configs ARCH_ARM_FEAT_AES=1 +configs ARCH_ARM_FEAT_PMULL=1 +configs ARCH_ARM_FEAT_SHA1=1 +configs ARCH_ARM_FEAT_RNG=1 + +configs ARCH_AARCH64_BIG_END_ALL_CORES=1 +configs ARCH_AARCH64_32BIT_EL0=1 +configs ARCH_AARCH64_32BIT_EL0_ALL_CORES=1 +configs ARCH_AARCH64_32BIT_EL1=0 + +configs ARCH_ARM_FEAT_VMID16=1 +configs ARCH_ARM_PMU_HPMN_UNPREDICTABLE=1 + +configs ARCH_ARM_FEAT_PMUv3p1=1 +configs ARCH_ARM_FEAT_IESB=1 +configs ARCH_ARM_FEAT_HPDS2=1 +# Assume sve128=on +configs ARCH_ARM_FEAT_SVE=1 +configs PLATFORM_SVE_REG_SIZE=16U + +configs ARCH_ARM_FEAT_CSV2=1 +configs ARCH_ARM_FEAT_CSV3=1 + +configs ARCH_ARM_FEAT_HPDS=1 +configs ARCH_ARM_FEAT_LSE=1 +configs ARCH_ARM_FEAT_LOR=1 +configs ARCH_ARM_FEAT_PAN=1 +configs ARCH_ARM_FEAT_RDM=1 +configs ARCH_ARM_FEAT_VHE=1 +configs ARCH_ARM_FEAT_CRC32=1 + +configs ARCH_ARM_FEAT_ASMv8p2=1 +configs ARCH_ARM_FEAT_PAN2=1 +configs ARCH_ARM_FEAT_DPB=1 +configs ARCH_ARM_FEAT_DEBUGv8p2=1 +configs ARCH_ARM_FEAT_DotProd=1 +configs ARCH_ARM_FEAT_RAS=1 +configs ARCH_ARM_FEAT_TTCNP=1 +configs ARCH_ARM_FEAT_XNX=1 +configs ARCH_ARM_FEAT_UAO=1 + +configs ARCH_AARCH64_ASID16=1 ARCH_ARM_PMU_VER=3 + +# The number of implemented ICH_LR_EL2 registers. +configs CPU_GICH_LR_COUNT=4U + +# The number of implemented ICH_APR[01]R_EL2 registers. +configs CPU_GICH_APR_COUNT=1U + +# The number of implemented DBGB[CV]R_EL1 (HW breakpoint) registers. +configs CPU_DEBUG_BP_COUNT=6U + +# The number of implemented DBGW[CV]R_EL1 (HW watchpoint) registers. +configs CPU_DEBUG_WP_COUNT=4U + +# The level of support for ARMv8.4-TTRem on this CPU (encoded the same way +# as ID_AA64MMFR2_EL1.BBM). +configs CPU_PGTABLE_BBM_LEVEL=0U diff --git a/config/featureset/gunyah-rm-qemu.conf b/config/featureset/gunyah-rm-qemu.conf index 59c5eb6..0f545ca 100644 --- a/config/featureset/gunyah-rm-qemu.conf +++ b/config/featureset/gunyah-rm-qemu.conf @@ -27,6 +27,8 @@ module core/irq module core/task_queue module core/timer module core/power +module core/wait_queue_broadcast +module core/globals module debug/object_lists module debug/symbol_version module ipc/doorbell @@ -37,22 +39,28 @@ module mem/memdb module mem/hyp_aspace module mem/pgtable module mem/addrspace -module mem/memextent +module mem/memextent_sparse module misc/elf module misc/prng_hw module misc/prng_simple module misc/trace_standard module misc/smc_trace module misc/log_standard +module platform/arm_generic module platform/arm_smccc module platform/arm_trng_fi +arch_module aarch64 misc/spectre_arm +module misc/qcbor module vm/rootvm module vm/rootvm_package -module vm/vcpu module vm/slat +module vm/vcpu +module vm/vcpu_power +module vm/vcpu_run arch_module armv8 vm/smccc arch_module armv8 vm/psci_pc arch_module armv8 vm/vdebug arch_module armv8 vm/vgic arch_module armv8 vm/arm_vm_timer arch_module armv8 vm/arm_vm_pmu +arch_module armv8 vm/arm_vm_sve_simple diff --git a/config/featureset/unittests-qemu.conf b/config/featureset/unittests-qemu.conf new file mode 100644 index 0000000..4072ff8 --- /dev/null +++ b/config/featureset/unittests-qemu.conf @@ -0,0 +1,54 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +configs HYP_CONF_STR=unittest UNITTESTS=1 +configs UNIT_TESTS=1 +platforms qemu + +module core/api +module core/base +module core/boot +module core/util +module misc/abort +module core/object_standard +module core/thread_standard +module core/idle +module core/scheduler_fprr +module core/partition_standard +module core/preempt +module core/cpulocal +module core/spinlock_ticket +module core/mutex_trivial +module core/rcu_bitmap +module core/cspace_twolevel +module core/tests +module core/vectors +module core/debug +module core/ipi +module core/irq +module core/virq_null +module core/timer +module core/power +module debug/object_lists +module debug/symbol_version +module mem/allocator_list +configs ALLOCATOR_DEBUG=1 +module mem/allocator_boot +module mem/memdb +module mem/hyp_aspace +module mem/pgtable +module mem/addrspace +module mem/memextent_sparse +module misc/elf +module misc/gpt +module misc/prng_simple +module misc/trace_standard +module misc/log_standard +module misc/smc_trace +module misc/qcbor +arch_module aarch64 misc/spectre_arm +module platform/arm_generic +module platform/arm_smccc +module vm/slat +configs POWER_START_ALL_CORES=1 diff --git a/config/include/debug_no_cspace_rand.conf b/config/include/debug_no_cspace_rand.conf new file mode 100644 index 0000000..689ec86 --- /dev/null +++ b/config/include/debug_no_cspace_rand.conf @@ -0,0 +1,5 @@ +# © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +configs DISABLE_CSPACE_RAND=1 diff --git a/config/include/debug_no_kaslr.conf b/config/include/debug_no_kaslr.conf new file mode 100644 index 0000000..c9f4e87 --- /dev/null +++ b/config/include/debug_no_kaslr.conf @@ -0,0 +1,5 @@ +# © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +configs DISABLE_KASLR=1 diff --git a/config/include/debug_no_rootvm_aslr.conf b/config/include/debug_no_rootvm_aslr.conf new file mode 100644 index 0000000..1a2915c --- /dev/null +++ b/config/include/debug_no_rootvm_aslr.conf @@ -0,0 +1,5 @@ +# © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +configs DISABLE_ROOTVM_ASLR=1 diff --git a/config/platform/qemu.conf b/config/platform/qemu.conf index 1a6794d..bc612f2 100644 --- a/config/platform/qemu.conf +++ b/config/platform/qemu.conf @@ -11,28 +11,47 @@ module platform/soc_qemu module platform/arm_arch_timer module platform/arm_pmu module platform/arm_rng + +# PLATFORM_LMA_BASE is used in linker script +# this is used to link Hyp image to this address configs PLATFORM_LMA_BASE=0x80000000 -configs PLATFORM_MAX_CORES=8 -configs PLATFORM_USABLE_CORES=0xFF + +# These define what is the installed memory range of +# the platform hardware (memsize while invoking qemu) +configs PLATFORM_DDR_BASE=0x40000000U +configs PLATFORM_DDR_SIZE=0x80000000U + +# These define the amount of memory given to the HLOS +# which is defined in the DT loaded for linux +configs HLOS_VM_DDR_BASE=0x40000000U +configs HLOS_VM_DDR_SIZE=0x40000000U + +# Address locations where Linux kernel, DT and RAMFS are +# loaded when Qemu is started +configs HLOS_ENTRY_POINT=0x41000000 +configs HLOS_DT_BASE=0x40F00000 +configs HLOS_RAM_FS_BASE=0x40800000 + configs PLATFORM_HEAP_PRIVATE_SIZE=0x200000 -configs PLATFORM_ROOTVM_LMA_BASE=0x80480000 -configs PLATFORM_ROOTVM_LMA_SIZE=0xa0000 -configs HYP_ASPACE_MAP_DIRECT_BITS=36 +configs PLATFORM_RW_DATA_SIZE=0x200000 +configs PLATFORM_ROOTVM_LMA_BASE=0x80480000U +configs PLATFORM_ROOTVM_LMA_SIZE=0xa0000U +configs PLATFORM_PHYS_ADDRESS_BITS=36 configs PLATFORM_VM_ADDRESS_SPACE_BITS=36 configs PLATFORM_PGTABLE_4K_GRANULE=1 configs PLATFORM_ARCH_TIMER_FREQ=62500000 configs PLATFORM_HYP_ARCH_TIMER_IRQ=26 -configs PLATFORM_VM_ARCH_VIRTUAL_TIMER_IRQ=27 +configs PLATFORM_VM_ARCH_VIRTUAL_TIMER_IRQ=27U configs PLATFORM_VM_ARCH_PHYSICAL_TIMER_IRQ=30 configs PLATFORM_GIC_BASE=0x08000000U +configs PLATFORM_GICR_COUNT=PLATFORM_MAX_CORES +configs PLATFORM_MAX_CLUSTERS=1U +configs PLATFORM_MAX_HIERARCHY=1U configs PLATFORM_GITS_COUNT=1 configs PLATFORM_GICH_IRQ=25 configs PLATFORM_PMU_IRQ=23 configs PLATFORM_VM_PMU_IRQ=23 configs PLATFORM_PMU_CNT_NUM=4 -configs ARCH_ARM_8_0_AES=1 -configs ARCH_ARM_8_0_AES_PMULL=1 -configs ARCH_ARM_8_0_SHA=1 # We currently do not have a wdog QEMU platform implementation configs WATCHDOG_DISABLE=1 # QEMU does not use affinity levels and uses original powerstate format diff --git a/config/quality/debug.conf b/config/quality/debug.conf index 0645c26..8ffdf8f 100644 --- a/config/quality/debug.conf +++ b/config/quality/debug.conf @@ -5,13 +5,16 @@ configs VERBOSE=1 configs VERBOSE_TRACE=1 configs RESET_ON_ABORT=0 -configs DISABLE_CSPACE_RAND=1 -configs DISABLE_ROOTVM_ASLR=1 configs QUALITY=debug flags -O1 -g -mstrict-align -arch_configs aarch64 ARCH_SANITIZE_STACK_SIZE=1536 +#include include/debug_no_kaslr +include include/debug_no_cspace_rand +include include/debug_no_rootvm_aslr + +arch_configs aarch64 ARCH_SANITIZE_STACK_SIZE=1536U +arch_configs aarch64 VCPU_MIN_STACK_SIZE=8192U # The trace entry numbers include the header arch_configs qemu TRACE_BOOT_ENTRIES=128 PER_CPU_TRACE_ENTRIES=4096 TRACE_FORMAT=1 -arch_configs qemu TRACE_AREA_SIZE=0x2000000U EXTRA_PRIVATE_HEAP_SIZE=0x100000U EXTRA_ROOT_HEAP_SIZE=0x300000U +arch_configs qemu TRACE_AREA_SIZE=0x2000000 EXTRA_PRIVATE_HEAP_SIZE=0x100000 EXTRA_ROOT_HEAP_SIZE=0x300000 diff --git a/config/quality/production-unchecked.conf b/config/quality/production-unchecked.conf index a41a046..7e8b57f 100644 --- a/config/quality/production-unchecked.conf +++ b/config/quality/production-unchecked.conf @@ -7,8 +7,9 @@ configs RESET_ON_ABORT=1 configs QUALITY=perf flags -flto -O3 -g +# This is slightly overkill for LTO builds, but safe. arch_configs aarch64 ARCH_SANITIZE_STACK_SIZE=1024 # The trace entry numbers include the header arch_configs qemu TRACE_BOOT_ENTRIES=128 PER_CPU_TRACE_ENTRIES=4096 TRACE_FORMAT=1 -arch_configs qemu TRACE_AREA_SIZE=0x200000U EXTRA_PRIVATE_HEAP_SIZE=0x100000U EXTRA_ROOT_HEAP_SIZE=0x300000U +arch_configs qemu TRACE_AREA_SIZE=0x2000000U EXTRA_PRIVATE_HEAP_SIZE=0x100000U EXTRA_ROOT_HEAP_SIZE=0x300000U diff --git a/configure.py b/configure.py index adde557..91ed1cc 100755 --- a/configure.py +++ b/configure.py @@ -501,6 +501,12 @@ def __call__(self, gen_cmd=None, **kwargs): # Add a phony rule for always-built targets self.add_alias(self._phony_always, []) + # Add phony rules for all of the generator sources, so Ninja + # does not fail if one of them disappears (e.g. if a module + # is renamed, or an older branch is checked out) + for f in sorted(self._gen_sources): + self.add_alias(f, []) + # Add a rule and targets for all of the automatically created parent # directories. We do this in deepest-first order at the end of the # build file because ninja -t clean always processes targets in the diff --git a/docs/api/Makefile b/docs/api/Makefile new file mode 100644 index 0000000..15efcd1 --- /dev/null +++ b/docs/api/Makefile @@ -0,0 +1,9 @@ +%.pdf: %.md Makefile + pandoc -s --toc --pdf-engine=xelatex -N --top-level-division=part \ + --metadata=title:'Gunyah Hypercall API' \ + --variable=class:book \ + --variable=mainfont:LiberationSans \ + --variable=monofont:LiberationMono \ + $< -o $@ + +all: gunyah_api.pdf diff --git a/docs/api/gunyah_api.md b/docs/api/gunyah_api.md index 19a1495..b4df733 100644 --- a/docs/api/gunyah_api.md +++ b/docs/api/gunyah_api.md @@ -1,26 +1,6 @@ -# Gunyah Hypercall API - -1. [AARCH64 HVC ABI](#aarch64-hvc-abi)
-1. [Common Types](#common-types)
-1. [Object Rights](#object-rights)
-1. [Hypervisor Identification](#hypervisor-identification)
-1. [Partitions](#partitions) -1. [Object Management](#object-management) -1. [Communication APIs](#communication-apis)
- 7.1 [Doorbell Management](#doorbell-management)
- 7.2 [Message Queue Management](#message-queue-management)
-1. [Capability Management](#capability-management) -1. [Interrupt Management](#interrupt-management) -1. [Address Space Management](#address-space-management) -1. [Memory Extent Management](#memory-extent-management) -1. [VCPU Management](#vcpu-management) -1. [Scheduler Management](#scheduler-management) -1. [Virtual PM Group Management](#virtual-pm-group-management) -1. [Trace Buffer Management](#trace-buffer-management) -1. [Watchdog Management](#watchdog-management) -1. [Error Results](#error-results) - -## AARCH64 HVC ABI +# Gunyah API + +## AArch64 HVC ABI The Gunyah AArch64 hypercall interface generally follows the ARM AAPCS64 conventions for general purpose register argument and result passing, and preservation of registers, unless explicitly documented otherwise. The hypervisor does not use SIMD, Floating-Point or SVE registers in the hypercall interface. Gunyah hypercalls use a range of HVC opcode immediate numbers, and reserves the following HVC immediate range: @@ -35,8 +15,8 @@ Note, Gunyah hypercalls encode the Call-ID in the HVC immediate, encoded within ### General-purpose Register -| Register | Role in AAPCS64 | Role in Gunyah HVC | -|------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------| +| Register | Role in AAPCS64 | Role in Gunyah HVC | +|--|---|-----| | SP_EL0 / SP_EL1 | The Stack Pointers | Preserved. (Callee-saved) | | r30 / LR | The Link Register | Preserved. (Callee-saved) | | r29 / FR | The Frame Register | Preserved. (Callee-saved) | @@ -92,14 +72,18 @@ A pointer in the VM’s current virtual address space, in the context of the cal A pointer in the VM’s physical address space. In ARMv8 terminology, this is an IPA. +### Access Rights + +An enumeration describing the rights given to a memory mapping. Follows the standard RWX format. + ### Virtual IRQ Info A bitfield type that identifies a virtual IRQ within a virtual interrupt controller. *Virtual IRQ Info:* -| Bit Numbers | Mask | Description | -|----------------------|----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Bits | Mask | Description | +|-|---|-----| | 23:0 | `0x00FFFFFF` | Virtual IRQ Number. The valid range of this field is defined by the platform-specific virtual interrupt controller implementation. The range may be discontiguous, and some sub-ranges may have special meanings (e.g. there may be a range reserved for VCPU-specific VIRQs). | | 31..24 | `0xFF000000` | Target VCPU index. This is the attachment index of a VCPU as defined by the hypercall API that configures the virtual interrupt controller. Valid only if the virtual IRQ number is in a range reserved by the virtual interrupt controller for VCPU-specific IRQs, and the operation being performed is implemented for VCPU-specific IRQs. | | 63:32 | `0xFFFFFFFF.00000000` | Reserved, Must be Zero | @@ -121,6 +105,7 @@ Generic rights are valid for all object types. | Right | Value | |-------------------|-------------------| | Partition Object Create | `0x00000001` | +| Partition Donate | `0x00000002` | ### Capability Space Rights @@ -137,6 +122,7 @@ Generic rights are valid for all object types. |-------------------|-------------------| | Address Space Attach | `0x00000001` | | Address Space Map | `0x00000002` | +| Address Space Lookup | `0x00000004` | ### Memory Extent Rights @@ -145,6 +131,8 @@ Generic rights are valid for all object types. | Memory Extent Map | `0x00000001` | | Memory Extent Derive | `0x00000002` | | Memory Extent Attach | `0x00000004` | +| Memory Extent Lookup | `0x00000008` | +| Memory Extent Donate | `0x00000010` | ### Thread Rights @@ -155,7 +143,11 @@ Generic rights are valid for all object types. | Thread Set Priority | `0x00000004` | | Thread Set Timeslice | `0x00000008` | | Thread Yield To | `0x00000010` | +| Thread Bind VIRQ | `0x00000020` | +| Thread Access State | `0x00000040` | | Thread Lifecycle | `0x00000080` | +| Thread Write Context | `0x00000100` | +| Thread Disable | `0x00000200` | ### Doorbell Rights @@ -238,8 +230,8 @@ Identifies the hypervisor version and feature set. *Hyp API Info:* -| Bit Numbers | Mask | Description | -|-----------------|------------------------|---------------------------------------------------------------| +| Bits | Mask | Description | +|-|---|-----| | 13:0 | `0x00001FFF` | API Version = “1” | | 14 | `0x00004000` | 0 = API is Little Endian.
1 = API is Big Endian. | | 15 | `0x00008000` | If set to 1, the API is 64-bit, otherwise 32-bit. | @@ -248,8 +240,8 @@ Identifies the hypervisor version and feature set. *API Flags 0:* -| Bit Numbers | Mask | Description | -|----------------------|----------------------------|--------------------------------------------------------------------------| +| Bits | Mask | Description | +|-|---|-----| | 0 | `0x1` | 1 = Partition and CSpace APIs supported | | 1 | `0x2` | 1 = Doorbell APIs supported | | 2 | `0x4` | 1 = Message Queue APIs supported | @@ -265,15 +257,15 @@ Identifies the hypervisor version and feature set. *API Flags 1:* -| Bit Numbers | Mask | Description | -|----------------------|----------------------------|---------------------------------| +| Bits | Mask | Description | +|-|---|-----| | 0 | `0x1` | 1 = ARM v8.2 SVE support | | 63:1 | `0xFFFFFFFF.FFFFFFFF ` | Reserved = 0 | *API Flags 2:* -| Bit Numbers | Mask | Description | -|----------------------|----------------------------|----------------------| +| Bits | Mask | Description | +|-|---|-----| | 63:0 | `0xFFFFFFFF.FFFFFFFF` | Reserved = 0 | @@ -288,7 +280,7 @@ Allocates a new Partition object from the Partition and allocates a Capability I | Call number: | `hvc 0x6001` | | Inputs: | X0: Partition CapID | | | X1: CSpace CapID | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | | | X1: Partition CapID | @@ -311,7 +303,7 @@ Allocates a new CSpace object from the Partition and allocates a Capability ID f | Call number: | `hvc 0x6002` | | Inputs: | X0: Partition CapID | | | X1: CSpace CapID | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | | | X1: Cspace CapID | @@ -335,7 +327,7 @@ Allocates a new Address Space object from the Partition and allocates a Capabili | Call number: | `hvc 0x6003` | | Inputs: | X0: Partition CapID | | | X1: CSpace CapID | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | On successful creation, the new Address Space object is created and its state is OBJECT_STATE_INIT. @@ -357,7 +349,7 @@ Allocates a new Memory Extent object from the Partition and allocates a Capabili | Call number: | `hvc 0x6004` | | Inputs: | X0: Partition CapID | | | X1: CSpace CapID | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | | | X1: MemExtent CapID | @@ -380,7 +372,7 @@ Allocates a new Thread object from the Partition and allocates a Capability ID f | Call number: | `hvc 0x6005` | | Inputs: | X0: Partition CapID | | | X1: CSpace CapID | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | | | X1: Thread CapID | @@ -403,7 +395,7 @@ Allocates a new Doorbell object from the Partition and allocates a Capability ID | Call number: | `hvc 0x6006` | | Inputs: | X0: Partition CapID | | | X1: CSpace CapID | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | | | X1: Doorbell CapID | @@ -426,7 +418,7 @@ Allocates a new Message Queue object from the Partition and allocates a Capabili | Call number: | `hvc 0x6007` | | Inputs: | X0: Partition CapID | | | X1: CSpace CapID | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | | | X1: MessageQueue CapID | @@ -449,7 +441,7 @@ Allocates a new Watchdog object from the Partition and allocates a Capability ID | Call number: | `hvc 0x6009` | | Inputs: | X0: Partition CapID | | | X1: CSpace CapID | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | | | X1: Watchdog CapID | @@ -472,7 +464,7 @@ Allocates a new Virtual Interrupt Controller object from the Partition and alloc | Call number: | `hvc 0x600A` | | Inputs: | X0: Partition CapID | | | X1: CSpace CapID | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | | | X1: Virtual IC CapID | @@ -495,7 +487,7 @@ Allocates a new Virtual PM Group object from the Partition and allocates a Capab | Call number: | `hvc 0x600B` | | Inputs: | X0: Partition CapID | | | X1: CSpace CapID | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | | | X1: VPMGroup CapID | @@ -509,6 +501,29 @@ ERROR_NOMEM – the creation failed due to memory allocation error. Also see: [Capability Errors](#capability-errors) +### Virtual IO MMIO object creation + +Allocates a new Virtual IO MMIO object from the Partition and allocates a Capability ID from the CSpace. + +| **Hypercall**: | `partition_create_virtio_mmio` | +|-------------------------|---------------------------------------| +| Call number: | `hvc 0x6048` | +| Inputs: | X0: Partition CapID | +| | X1: CSpace CapID | +| | X2: Reserved — Must be Zero | +| Outputs: | X0: Error Result | +| | X1: VirtioMMIO CapID | + +On successful creation, the new Virtual IO MMIO object is created and its state is OBJECT_STATE_INIT. + +**Errors:** + +OK – the operation was successful, and the result is valid. + +ERROR_NOMEM – the creation failed due to memory allocation error. + +Also see: [Capability Errors](#capability-errors) + ## Object Management ### Activate an Object @@ -519,14 +534,14 @@ Activate an object. |-------------------------|--------------------------------------| | Call number: | `hvc 0x600C` | | Inputs: | X0: Cap CapID | -| | X1: Reserved – Must be Zero | +| | X1: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** -OK – the operation was successful, and the result is valid. +OK – the operation was successful, and the object has moved into `OBJECT_STATE_ACTIVE` state. -ERROR_OBJECT_STATE - if the object is not in OBJECT_STATE_INIT state. +ERROR_OBJECT_STATE – if the object is not in OBJECT_STATE_INIT state. Additional [error codes](#error-code-enumeration) can be returned depending on the type of object to be activated. @@ -541,14 +556,14 @@ Activate an object from a Cspace. | Call number: | `hvc 0x600D` | | Inputs: | X0: CSpace CapID | | | X1: Cap CapID | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** OK – the operation was successful, and the result is valid. -ERROR_OBJECT_STATE - if the object is not in OBJECT_STATE_INIT state. +ERROR_OBJECT_STATE – if the object is not in OBJECT_STATE_INIT state. Additional [error codes](#error-code-enumeration) can be returned depending on the type of object to be activated. @@ -562,14 +577,14 @@ Reset an object to its initial state. |-------------------------|--------------------------------------| | Call number: | `hvc 0x600E` | | Inputs: | X0: Cap CapID | -| | X1: Reserved – Must be Zero | +| | X1: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** OK – the operation was successful, and the result is valid. -ERROR_UNIMPLEMENTED - if functionality not implemented. +ERROR_UNIMPLEMENTED – if functionality not implemented. Additional [error codes](#error-code-enumeration) can be returned depending on the type of object to be reset. @@ -584,14 +599,14 @@ Reset an object from a Cspace to its initial state. | Call number: | `hvc 0x600F` | | Inputs: | X0: CSpace CapID | | | X1: Cap CapID | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** OK – the operation was successful, and the result is valid. -ERROR_UNIMPLEMENTED - if functionality not implemented. +ERROR_UNIMPLEMENTED – if functionality not implemented. Additional [error codes](#error-code-enumeration) can be returned depending on the type of object to be reset. @@ -601,7 +616,7 @@ Also see: [Capability Errors](#capability-errors) -0x6008 - `Reserved` +0x6008 – `Reserved` ## Communication APIs @@ -617,7 +632,7 @@ Binds a Doorbell to a virtual interrupt. | Inputs: | X0: Doorbell CapID | | | X1: Virtual IC CapID | | | X2: Virtual IRQ Info | -| | X3: Reserved – Must be Zero | +| | X3: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -642,7 +657,7 @@ Unbinds a Doorbell from a virtual IRQ number. |-------------------------|--------------------------------------| | Call number: | `hvc 0x6011` | | Inputs: | X0: Doorbell CapID | -| | X1: Reserved – Must be Zero | +| | X1: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -660,7 +675,7 @@ Sets flags in the Doorbell. If following the send, the set of enabled flags as d | Call number: | `hvc 0x6012` | | Inputs: | X0: Doorbell CapID | | | X1: NewFlags FlagsBitmap – Must be non-zero. | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | | | X1: OldFlags FlagsBitmap | @@ -686,8 +701,8 @@ Reads and clears the flags of the Doorbell. If there is a pending bound virtual |-------------------------|-------------------------------------------------------| | Call number: | `hvc 0x6013` | | Inputs: | X0: Doorbell CapID | -| | X1: ClearFlags FlagsBitmap - Must be non-zero. | -| | X2: Reserved – Must be Zero | +| | X1: ClearFlags FlagsBitmap – Must be non-zero. | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | | | X1: OldFlags FlagsBitmap | @@ -713,7 +728,7 @@ Clears all the flags of the Doorbell and sets all bits in the Doorbell’s mask. |-------------------------|--------------------------------------| | Call number: | `hvc 0x6014` | | Inputs: | X0: Doorbell CapID | -| | X1: Reserved – Must be Zero | +| | X1: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -732,7 +747,7 @@ Sets the Doorbell object’s masks. A Doorbell object has two masks which are co | Inputs: | X0: Doorbell CapID | | | X1: EnableMask FlagsBitmap | | | X2: AckMask FlagsBitmap | -| | X3: Reserved – Must be Zero | +| | X3: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Types:** @@ -749,7 +764,7 @@ Also see: [Capability Errors](#capability-errors) -0x6016 - `Reserved` +0x6016 – `Reserved` ### Message Queue Management @@ -763,7 +778,7 @@ Binds a Message Queue send interface to a virtual IRQ number. | Inputs: | X0: Message Queue CapID | | | X1: Virtual IC CapID | | | X2: Virtual IRQ Info | -| | X3: Reserved – Must be Zero | +| | X3: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -790,7 +805,7 @@ Binds a Message Queue receive interface to a virtual IRQ number. | Inputs: | X0: Message Queue CapID | | | X1: Virtual IC CapID | | | X2: Virtual IRQ Info | -| | X3: Reserved – Must be Zero | +| | X3: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -815,7 +830,7 @@ Unbinds a Message Queue send interface virtual IRQ number. |-------------------------|--------------------------------------| | Call number: | `hvc 0x6019` | | Inputs: | X0: Message Queue CapID | -| | X1: Reserved – Must be Zero | +| | X1: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -832,7 +847,7 @@ Unbinds a Message Queue receive interface virtual IRQ number. |-------------------------|--------------------------------------| | Call number: | `hvc 0x601A` | | Inputs: | X0: Message Queue CapID | -| | X1: Reserved – Must be Zero | +| | X1: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -849,10 +864,10 @@ Append a message to the tail of a Message Queue, if it is not full. The message |-------------------------|--------------------------------------------| | Call number: | `hvc 0x601B` | | Inputs: | X0: Message Queue CapID | -| | X1: Size Size Must be non-zero. | +| | X1: Size Size — Must be non-zero. | | | X2: Data VMAddr | | | X3: MsgQSendFlags | -| | X4: Reserved – Must be Zero | +| | X4: Reserved — Must be Zero | | Outputs: | X0: Error Result | | | X1: NotFull Boolean | @@ -860,10 +875,10 @@ Append a message to the tail of a Message Queue, if it is not full. The message *MsgQSendFlags:* -| Bit Numbers | Mask | Description | -|----------------------|----------------------------|---------------------------------| +| Bits | Mask | Description | +|-|---|-----| | 0 | `0x1` | Message Push | -| 63:1 | `0xFFFFFFFF.FFFFFFFE` | Reserved, Must be Zero | +| 63:1 | `0xFFFFFFFF.FFFFFFFE` | Reserved — Must be Zero | Message Push: If set to 0x1, this flag indicates that the hypervisor should push the message immediately to the receiver. This may cause a receive interrupt to be raised immediately, regardless of any interrupt threshold or interrupt delay configuration. @@ -889,7 +904,7 @@ Fetch a message from the head of a Message Queue, if it is not empty, into a spe | Inputs: | X0: Message Queue CapID | | | X1: Buffer VMAddr | | | X2: MaximumSize Size | -| | X3: Reserved – Must be Zero | +| | X3: Reserved — Must be Zero | | Outputs: | X0: Error Result | | | X1: Size Size | | | X2: NotEmpty Boolean | @@ -914,7 +929,7 @@ Rmoves all messages from a Message Queue. If the Message Queue was previously no |-------------------------|--------------------------------------| | Call number: | `hvc 0x601D` | | Inputs: | X0: Message Queue CapID | -| | X1: Reserved – Must be Zero | +| | X1: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -927,7 +942,7 @@ Also see: [Capability Errors](#capability-errors) -0x601E - `Reserved` +0x601E – `Reserved` #### Message Queue Configure Send @@ -939,7 +954,7 @@ Modify configuration of a Message Queue send interface. The interface allows for | Inputs: | X0: Message Queue CapID | | | X1: NotFull interrupt threshold | | | X2: NotFull threshold delay (microseconds) | -| | X3: Reserved – Must be -1 | +| | X3: Reserved — Must be -1 | | Outputs: | X0: Error Result | Any parameter passed in as -1 indicates no change to the corresponding is requested. @@ -966,7 +981,7 @@ Modify configuration of a Message Queue receive interface. The interface allows | Inputs: | X0: Message Queue CapID | | | X1: NotEmpty interrupt threshold | | | X2: NotEmpty threshold delay (microseconds) | -| | X3: Reserved – Must be -1 | +| | X3: Reserved — Must be -1 | | Outputs: | X0: Error Result | Any parameter passed in as -1 indicates no change to the corresponding is requested. @@ -992,15 +1007,15 @@ Configure a Message Queue whose state is OBJECT_STATE_INIT. | Call number: | `hvc 0x6021` | | Inputs: | X0: Message Queue CapID | | | X1: MessageQueueCreateInfo | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Types:** *MessageQueueCreateInfo:* -| Bit Numbers | Mask | Description | -|----------------------|----------------------------|---------------------------------| +| Bits | Mask | Description | +|-|---|-----| | 15:0 | `0x0000FFFF` | Queue Depth | | 31:15 | `0xFFFF0000` | Max Message Size | | 63:32 | `0xFFFFFFFF.00000000` | Reserved, Must be Zero | @@ -1028,7 +1043,7 @@ Delete a Capability in a CSpace. | Call number: | `hvc 0x6022` | | Inputs: | X0: CSpace CapID | | | X1: Cap CapID | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | @@ -1049,7 +1064,7 @@ Copy a Capability from one CSpace to another. | | X1: SourceCap CapID | | | X2: DestCSpace CapID | | | X3: RightsMask | -| | X4: Reserved – Must be Zero | +| | X4: Reserved — Must be Zero | | Outputs: | X0: Error Result | | | X1: New CapID | @@ -1070,14 +1085,14 @@ Revoke a Capability from another CSpace. | Call number: | `hvc 0x6024` | | Inputs: | X0: CSpace CapID | | | X1: Cap CapID | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** OK – the operation was successful, and the result is valid. -ERROR_UNIMPLEMENTED - if functionality not implemented. +ERROR_UNIMPLEMENTED – if functionality not implemented. `TODO: TBD. Currently unimplemented` @@ -1092,7 +1107,7 @@ Configure a CSpace whose state is OBJECT_STATE_INIT. | Call number: | `hvc 0x6025` | | Inputs: | X0: CSpace CapID | | | X1: MaxCaps | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -1101,7 +1116,7 @@ OK – the operation was successful, and the result is valid. ERROR_OBJECT_STATE – if the Cspace is not in OBJECT_STATE_INIT state. -ERROR_ARGUMENT_INVALID - a value passed in an argument was invalid. This could be due to an invalid Max Caps value. +ERROR_ARGUMENT_INVALID – a value passed in an argument was invalid. This could be due to an invalid Max Caps value. Also see: [Capability Errors](#capability-errors) @@ -1116,7 +1131,7 @@ Attaches a thread to a CSpace. The Cspace object must have been activated before | Call number: | `hvc 0x603e` | | Inputs: | X0: CSpace CapID | | | X1: Thread CapID | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -1136,7 +1151,7 @@ Revoke children Capabilities from a CSpace. | Call number: | `hvc 0x6059` | | Inputs: | X0: CSpace CapID | | | X1: MasterCap CapID | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -1157,7 +1172,7 @@ Binds a hardware IRQ number to a virtual IRQ number. | Inputs: | X0: HW IRQ CapID | | | X1: Virtual IC CapID | | | X2: Virtual IRQ Info | -| | X3: Reserved – Must be Zero | +| | X3: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -1182,7 +1197,7 @@ Unbinds a hardware IRQ number from a virtual IRQ number. |-------------------------|--------------------------------------| | Call number: | `hvc 0x6027` | | Inputs: | X0: HW IRQ CapID | -| | X1: Reserved – Must be Zero | +| | X1: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -1205,7 +1220,7 @@ Note that both of these numbers may have implementation-defined upper bounds. Al | Inputs: | X0: VIC CapID | | | X1: MaxVCPUs | | | X2: MaxSharedVIRQs | -| | X3: Reserved – Must be Zero | +| | X3: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -1228,7 +1243,7 @@ Attaches a VCPU to a Virtual Interrupt Controller. The Virtual Interrupt Control | Inputs: | X0: Virtual IC CapID | | | X1: VCPU CapID | | | X2: Index | -| | X3: Reserved – Must be Zero | +| | X3: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -1257,7 +1272,7 @@ In the current implementation, the only type of MSI source supported is a GICv4 | Inputs: | X0: Virtual IC CapID | | | X1: MSI Source (platform-specific object type) CapID | | | X2: Index | -| | X3: Reserved – Must be Zero | +| | X3: Reserved — Must be Zero | | Outputs: | X0: Error Result | @@ -1284,7 +1299,7 @@ Attaches an address space to a thread. The address space object must have been a | Call number: | `hvc 0x602a` | | Inputs: | X0: Address Space CapID | | | X1: Thread CapID | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -1299,7 +1314,11 @@ Also see: [capability errors](#capability-errors) ### Address Space Map -Maps a memory extent into a specified address space. The entire memory extent range is mapped, except for any carveouts contained within the extent. +Map a memory extent into a specified address space. By default, the entire memory extent is mapped, except for any carveouts contained within the extent. + +If the Partial flag is set in Map Flags, only the range of the memory extent specified by Offset and Size will be mapped. If not set, these arguments are ignored. Partial mappings are only supported by sparse memory extents. + +If successful, the hypervisor will automatically synchronise with other cores to ensure they have observed the map operation. This behaviour is skipped if the NoSync flag is set. | **Hypercall**: | `addrspace_map` | |-------------------------|--------------------------------------| @@ -1308,39 +1327,53 @@ Maps a memory extent into a specified address space. The entire memory extent ra | | X1: Memory Extent CapID | | | X2: Base VMAddr | | | X3: Map Attributes | -| | X4: Reserved – Must be Zero | +| | X4: Map Flags | +| | X5: Offset | +| | X6: Size | | Outputs: | X0: Error Result | **Types:** -*Map Atrributes:* +*Map Attributes:* -| Bit Numbers | Mask | Description | -|----------------------|----------------------------|-----------------------------------| +| Bits | Mask | Description | +|-|---|-----| | 2..0 | `0x7` | User Access (if Supported) | | 6..4 | `0x70` | Kernel Access | | 23:16 | `0xFF0000` | Memory Type | | 63:24,15:7,3 | `0xFFFFFFFF.0000FF88` | Reserved, Must be Zero | +*Map Flags:* + +| Bits | Mask | Description | +|-|---|-----| +| 0 | `0x1` | Partial | +| 31 | `0x80000000` | NoSync | +| 30:1 | `0x7FFFFFFE` | Reserved, Must be Zero | + **Errors:** OK – the operation was successful, and the result is valid. ERROR_ARGUMENT_INVALID – a value passed in an argument was invalid. This could be due to an invalid Address Space. -ERROR_MEMEXTENT_MAPPINGS_FULL– the memory extent has exceeded its mappings capacity. Currently it can have up to 4 mappings. +ERROR_MEMEXTENT_MAPPINGS_FULL – the memory extent has exceeded its mappings capacity. Currently it can have up to 4 mappings. -ERROR_DENIED - the specified Address Space is not allowed to execute map operations. +ERROR_DENIED – the specified Address Space is not allowed to execute map operations. -ERROR_ARGUMENT_ALIGNMENT - the specificied base address is not page size aligned. +ERROR_ARGUMENT_ALIGNMENT – the specified base address is not page size aligned. -ERROR_ADDR_OVERFLOW - the specified base address may cause an overflow. +ERROR_ADDR_OVERFLOW – the specified base address may cause an overflow. Also see: [capability errors](#capability-errors) ### Address Space Unmap -Unmaps a memory extent from a specified address space. The entire memory extent range is unmapped, except for any carveouts contained within the extent. +Unmaps a memory extent from a specified address space. By default, the entire memory extent range is unmapped, except for any carveouts contained within the extent. + +If the Partial flag is set in Map Flags, only the range of the Memory Extent specified by Offset and Size will be unmapped. If not set, these arguments are ignored. Partial unmappings are only supported by sparse memory memextents. + +If successful, the hypervisor will automatically synchronise with other cores to ensure they have observed the unmap operation. This behaviour is skipped if the NoSync flag is set. | **Hypercall**: | `addrspace_unmap` | |-------------------------|--------------------------------------| @@ -1348,7 +1381,9 @@ Unmaps a memory extent from a specified address space. The entire memory extent | Inputs: | X0: Address Space CapID | | | X1: Memory Extent CapID | | | X2: Base VMAddr | -| | X3: Reserved – Must be Zero | +| | X3: Map Flags | +| | X4: Offset | +| | X5: Size | | Outputs: | X0: Error Result | **Errors:** @@ -1357,9 +1392,9 @@ OK – the operation was successful, and the result is valid. ERROR_ARGUMENT_INVALID – a value passed in an argument was invalid. This could be due to an invalid Address Space or a non-existing mapping. -ERROR_DENIED - the specified Address Space is not allowed to execute map operations. +ERROR_DENIED – the specified Address Space is not allowed to execute map operations. -ERROR_ARGUMENT_ALIGNMENT - the specificied base address is not page size aligned. +ERROR_ARGUMENT_ALIGNMENT – the specified base address is not page size aligned. Also see: [capability errors](#capability-errors) @@ -1367,22 +1402,28 @@ Also see: [capability errors](#capability-errors) Update access rights on an existing mapping. +If the Partial flag is set in Map Flags, only the range of the Memory Extent specified by Offset and Size will be updated. If not set, these arguments are ignored. Partial access updates are only supported by sparse memory extents. + +If successful, the hypervisor will automatically synchronise with other cores to ensure they have observed the mapping update. This behaviour is skipped if the NoSync flag is set. + | **Hypercall**: | `addrspace_update_access` | |-------------------------|--------------------------------------| | Call number: | `hvc 0x602d` | | Inputs: | X0: Address Space CapID | | | X1: Memory Extent CapID | | | X2: Base VMAddr | -| | X3: Update Attributes | -| | X5: Reserved – Must be Zero | +| | X3: Update Attributes | +| | X4: Map Flags | +| | X5: Offset | +| | X6: Size | | Outputs: | X0: Error Result | **Types:** *Update Attributes:* -| Bit Numbers | Mask | Description | -|----------------------|----------------------------|-----------------------------------| +| Bits | Mask | Description | +|-|---|-----| | 2..0 | `0x7` | User Access (if Supported) | | 6..4 | `0x70` | Kernel Access | | 63:7,3 | `0xFFFFFFFF.FFFFFF88` | Reserved, Must be Zero | @@ -1393,22 +1434,22 @@ OK – the operation was successful, and the result is valid. ERROR_ARGUMENT_INVALID – a value passed in an argument was invalid. This could be due to an invalid Address Space or a non-existing mapping. -ERROR_ARGUMENT_ALIGNMENT - the specificied base address is not page size aligned. +ERROR_ARGUMENT_ALIGNMENT – the specified base address is not page size aligned. -ERROR_DENIED - the specified Address Space is not allowed to update access of mappings. +ERROR_DENIED – the specified Address Space is not allowed to update access of mappings. Also see: [capability errors](#capability-errors) ### Configure an Address Space -Configure a address space whose state is OBJECT_STATE_INIT. +Configure an address space whose state is OBJECT_STATE_INIT. -| **Hypercall**: | `addrspace_configure` | +| **Hypercall**: | `addrspace_configure` | |-------------------------|--------------------------------------| | Call number: | `hvc 0x602e` | | Inputs: | X0: Address Space CapID | | | X1: VMID | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Types:** @@ -1425,6 +1466,31 @@ ERROR_ARGUMENT_INVALID – a value passed in an argument was invalid. This could Also see: [capability errors](#capability-errors) +### Configure the information area of an Address Space + +Configure the information area of an address space whose state is OBJECT_STATE_INIT. + +| **Hypercall**: | `addrspace_configure_info_area` | +|-------------------------|--------------------------------------| +| Call number: | `hvc 0x605b` | +| Inputs: | X0: Address Space CapID | +| | X1: Info area memextent CapID | +| | X2: Info area IPA | +| | X3: Reserved — Must be Zero | +| Outputs: | X0: Error Result | + +**Errors:** + +OK – The operation was successful, and the result is valid. + +ERROR_OBJECT_STATE – The Address Space object has already been activated. + +ERROR_ADDR_INVALID – The provided IPA is invalid. + +ERROR_ARGUMENT_INVALID – A value passed in an argument was invalid. + +Also see: [capability errors](#capability-errors) + ### Address Space to DMA-capable Object Attachment Attaches an address space to any type of object that has a virtual DMA port which it can use to independently access memory in a VM address space. For types of object that have more than one virtual DMA port (e.g. a DMA-based IPC object), an index may be specified to indicate which port should be attached. Note that VCPUs do not access the VM address spaces through a virtual DMA port when executing VM code; they use a separate attachment call, described in [section](#address-space-to-thread-attachment) above. @@ -1439,40 +1505,208 @@ In the current implementation, the only object type with a virtual DMA port is t | Inputs: | X0: Address Space CapID | | | X1: Virtual DMA-capable Object CapID | | | X2: Virtual DMA Port Index | -| | X3: Reserved – Must be Zero | +| | X3: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** -OK – the operation was successful, and the result is valid. +OK – the operation was successful. ERROR_NOMEM – the operation failed due to memory allocation error. ERROR_ARGUMENT_INVALID – the specified object is virtual DMA capable, but the port index is outside the valid range for the object. +ERROR_CSPACE_WRONG_OBJECT_TYPE – the specified virtual device object does not have any virtual DMA ports; or the specified address space object is not an address space. + ERROR_BUSY – the specified port already has an address space attached, and the object does not support changing an existing attachment. ERROR_OBJECT_STATE – the Address Space object has not yet been activated. Also see: [capability errors](#capability-errors) +### Address Space to Virtual Device Attachment + +Attaches an address space to any type of object that presents a virtual memory-mapped device register interfaces. For types of object that have more than one virtual device interface, an index may be specified to indicate which interface should be attached. The meaning of this index depends on the object type. + +After this call succeeds, accesses by any VCPU attached to the address space that lie within the specified IPA range and fault in the IPA translation will be forwarded to the specified virtual device for emulation. The addresses, access sizes, access types, and semantics of the emulated registers depend entirely on the device implementation. Also, the behaviour of any access that does not match an emulated register depends on the device implementation, and may include either faulting as if the virtual device was not attached, or returning a constant value (typically 0 or 0xff) for reads and ignoring writes. + +Note that the register interface will not function correctly if any memory extent is mapped in the specified IPA range. The hypervisor will not check for such overlapping mappings. + +The specified IPA range must be large enough to contain the selected register interface, and must not be attached to any other virtual device. If the specified range is undersized, some registers may not be accessible. If the specified range is oversized, any extra space will become unavailable to other virtual devices; the behaviour of an access to this extra space is unspecified. + +| **Hypercall**: | `addrspace_attach_vdevice` | +|-------------------------|---------------------------------------------| +| Call number: | `hvc 0x6062` | +| Inputs: | X0: Address Space CapID | +| | X1: Virtual Device Object CapID | +| | X2: Virtual Device Interface Index | +| | X3: Base IPA | +| | X4: Size | +| | X5: Reserved — Must be Zero | +| Outputs: | X0: Error Result | + +**Index values:** + +| **Type** | **Index** | **Size** | **Description** | +|--|-|--|-----| +| vGIC | 0 | 64KiB | GIC Distributor registers | +| vGIC | 1..N | 64KiB | GIC Redistributor registers for VCPUs 0..(N-1) | +| vITS | 0 | 64KiB | GIC ITS registers | + +**Errors:** + +OK – the operation was successful. + +ERROR_NOMEM – the operation failed due to memory allocation error. + +ERROR_ARGUMENT_INVALID – the specified object is a virtual device, but the interface index is outside the valid range for the object. + +ERROR_CSPACE_WRONG_OBJECT_TYPE – the specified virtual device object does not support memory-mapped interfaces or is not a virtual device; or the specified address space object is not an address space. + +ERROR_BUSY – the specified address range already contains a virtual device. + +ERROR_OBJECT_STATE – the Address Space object has not yet been activated. + +Also see: [capability errors](#capability-errors) + +### Address Space Lookup + +Lookup a memextent mapping in an address space. If successful, returns the offset and size within the memextent, as well as the attributes of the mapping. + +| **Hypercall**: | `addrspace_lookup` | +|-------------------------|--------------------------------------| +| Call number: | `hvc 0x605a` | +| Inputs: | X0: Address Space CapID | +| | X1: Memory Extent CapID | +| | X2: Base VMAddr | +| | X3: Size | +| | X4: Reserved — Must be Zero | +| Outputs: | X0: Error Result | +| | X1: Offset | +| | X2: Size | +| | X3: Map Attributes | + +**Types:** + +*Map Attributes:* + +See: [Address Space Map](#address-space-map) + +**Errors:** + +OK – the operation was successful, and the result is valid. + +ERROR_ARGUMENT_INVALID – one of the given arguments is invalid. This could be due to an invalid Address Space. + +ERROR_ARGUMENT_SIZE – the specified size is invalid. + +ERROR_ARGUMENT_ALIGNMENT – the specified base address or size is not page size aligned. + +ERROR_ADDR_OVERFLOW – the specified base address may cause an overflow. + +ERROR_ADDR_INVALID – the specified base address is not mapped in the Address Space. + +ERROR_MEMDB_NOT_OWNER – the memory mapped in the Address Space is not owned by the specified Memory Extent. + +Also see: [capability errors](#capability-errors) + +### Address Space Virtual MMIO Area Configuration + +Configure the virtual MMIO device regions for the address space. + +A virtual MMIO device region is a region of the address space in which translation faults may be handled by an unprivileged VMM residing in another VM. +This allows the unprivileged VMM to emulate memory-mapped I/O devices. +Note that other types of fault, such as permission or alignment faults, cannot be handled by this mechanism. +Also, depending on the architecture, this mechanism may only support translation faults generated by specific types of instruction. +On AArch64, it is limited to single-register load & store instructions without base register writeback, which are decoded by the CPU into the `ESR_EL2` syndrome bits. + +This call may be made before or after activation of the address space object. +This is to permit delegation of the right to call this API to the VM that runs in the address space, so it can explicitly acknowledge that the specified region should not be used for sensitive data. + +An address range that is added must not overlap any existing range, and must not wrap around the end of the address space. +There are no other restrictions on the size or alignment of ranges added to the address space. +However, a limit may be imposed on the total number of ranges added to an address space. + +A removed address range must exactly match a single previously added address range. Note that removal of a range will prevent the VMM receiving any new faults that occur in that range after the removal operation completes, but does not guarantee that the VMM has finished handling all faults in the removed range. + +| **Hypercall**: | `addrspace_configure_vmmio` | +|-------------------------|------------------------------------| +| Call number: | `hvc 0x6060` | +| Inputs: | X0: Address Space CapID | +| | X1: Base VMAddr | +| | X2: Size | +| | X3: VMMIOConfigureOperation | +| | X4: Reserved – Must be Zero | +| Outputs: | X0: Error Result | + +**Types:** + +*VMMIOConfigureOperation:* + +| Operation Enumerator | Integer Value | +|-----------------------------------------|------------------------| +| VMMIO_CONFIGURE_OP_ADD_RANGE | 0 | +| VMMIO_CONFIGURE_OP_REMOVE_RANGE | 1 | + +**Errors:** + +OK – the operation was successful. + +ERROR_ADDR_OVERFLOW – the specified range wraps around the end of the address space. + +ERROR_ADDR_INVALID – the specified range is not completely within the input address range of the address space. + +ERROR_ARGUMENT_INVALID – the specified range to be added overlaps a previously added range, or the specified range to be removed does not match a previously added range. + +ERROR_NORESOURCES – the number of nominated ranges has reached an implementation-defined limit, or the hypervisor was unable to allocate memory for bookkeeping. + +ERROR_UNIMPLEMENTED — unprivileged VMMs are unable to handle faults in this configuration, or an unknown operation was requested. + +Also see: [capability errors](#capability-errors) + ## Memory Extent Management -### Memory Extent Unmap All +### Memory Extent Modify + +Perform a modification on a memory extent. -Unmaps a memory extent from all address spaces it was mapped into. The entire range is unmapped, except for any carveouts contained within the extent. +For range operations, only the range of the memory extent specified by Offset and Size will be modified. For all other operations these arguments are ignored. -| **Hypercall**: | `memextent_unmap_all` | +For operations that affect address space mappings, the hypervisor will automatically synchronise with other cores to ensure they have observed any successful changes in mappings. This behaviour is skipped if the NoSync flag is set. For other operations the NoSync flag must be set as specified below. + +| **Hypercall**: | `memextent_modify` | |-------------------------|--------------------------------------| | Call number: | `hvc 0x6030` | | Inputs: | X0: Memory Extent CapID | -| | X1: Reserved – Must be Zero | +| | X1: Memextent Modify Flags | +| | X2: Offset | +| | X3: Size | | Outputs: | X0: Error Result | +**Types:** + +*MemExtent Modify Flags:* + +| Bit Numbers | Mask | Description | +|---------------------|-------------------|---------------------------------| +| 7:0 | `0xFF` | Memextent Modify Operation | +| 31 | `0x80000000` | NoSync | +| 30:8 | `0x7FFFFF00` | Reserved, Must be Zero | + +*MemExtent Modify Operation:* + +| Modify Operation | Integer Value | Description | +|------------------------------------|-------------------|-------------------------------------------------------------------------------------------------| +| MEMEXTENT_MODIFY_OP_UNMAP_ALL | 0 | Unmap the memory extent from all address spaces it was mapped into. | +| MEMEXTENT_MODIFY_OP_ZERO_RANGE | 1 | Zero the owned memory of an extent within the specified range. The NoSync flag must be set. | +| MEMEXTENT_MODIFY_OP_SYNC_ALL | 255 | Synchronise all previous memory extent operations. The NoSync flag must not be set. | + **Errors:** OK – the operation was successful, and the result is valid. +ERROR_ARGUMENT_INVALID – the specified modify flags are invalid. + Also see: [Capability Errors](#capability-errors) ### Configure a Memory Extent @@ -1486,19 +1720,36 @@ Configure a memory extent whose state is OBJECT_STATE_INIT. | | X1: Phys Base | | | X2: Size | | | X3: MemExtent Attributes | -| | X4: Reserved – Must be Zero | +| | X4: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Types:** *MemExtent Attributes:* -| Bit Numbers | Mask | Description | -|----------------------|-------------------|---------------------------------| +| Bits | Mask | Description | +|-|---|-----| | 2..0 | `0x7` | Access Rights | | 9:8 | `0x300` | MemExtent MemType | +| 17:16 | `0x30000` | MemExtent Type | | 31 | `0x80000000` | List Append | -| 30:19,7:3 | `0x7FFFFCF8` | Reserved, Must be Zero | +| 30:18,15:10,7:3 | `0x7FFCFCF8` | Reserved, Must be Zero | + +*Memextent Type* + +| Memextent Type | Integer Value | Description | +|-----------------------|---------------------|------------------------------------------------------| +| BASIC | 0 | Extent with basic functionality. | +| SPARSE | 1 | Extent supporting donation and partial mappings. | + +*Memextent MemType* + +| Memextent MemType | Integer Value | Description | +|-------------------------|---------------------|----------------------------------------------------| +| ANY | 0 | Allow mappings of any memory type. | +| DEVICE | 1 | Restrict mappings to device memory types only. | +| UNCACHED | 2 | Force mappings to be uncached. | +| CACHED | 3 | Force mappings to be writeback cacheable. | **Errors:** @@ -1520,7 +1771,7 @@ Configure a derived memory extent whose state is OBJECT_STATE_INIT. The extent w | | X2: Offset | | | X3: Size | | | X4: MemExtent Attributes | -| | X5: Reserved – Must be Zero | +| | X5: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -1531,11 +1782,54 @@ ERROR_ARGUMENT_INVALID – a value passed in an argument was invalid. This could Also see: [Capability Errors](#capability-errors) -### Memory Extent Reserved +### Memory Extent Donate - +Donate memory from one extent to another. This includes donations from parent to child, child to parent and between siblings. + +For non-derived memory extents, the parent is considered to be the partition that was used to create the extent. Donation is only supported for sparse memory extents. + +If successful, the hypervisor will automatically synchronise with other cores to ensure they have observed the donation and any mapping changes that may have occurred. This behaviour is skipped if the NoSync flag is set. + +| **Hypercall**: | `memextent_donate` | +|-------------------------|---------------------------------------| +| Call number: | `hvc 0x6033` | +| Inputs: | X0: Memextent Donate Options | +| | X1: From CapID | +| | X2: To CapID | +| | X3: Offset | +| | X4: Size | +| | X5: Reserved — Must be Zero | +| Outputs: | X0: Error Result | -0x6033 - `Reserved` +**Types:** + +*Memextent Donate Options* + +| Bits | Mask | Description | +|-|---|-----| +| 7:0 | `0xFF` | Memextent Donate Type | +| 31 | `0x80000000` | NoSync | +| 30:8 | `0x7FFFFF00` | Reserved — Must be Zero | + +*Memextent Donate Type* + +| Memextent Donate Type | Integer Value | Description | +|------------------------------|---------------------|-------------------------------------------------| +| TO_CHILD | 0 | Donate to a child extent from its parent. | +| TO_PARENT | 1 | Donate from a child extent to its parent. | +| TO_SIBLING | 2 | Donate from one sibling extent to another. | + +**Errors:** + +OK – the operation was successful, and the result is valid. + +ERROR_ARGUMENT_INVALID – a value passed in an argument was invalid. This could be due to an invalid donate option, offset or size. + +ERROR_ARGUMENT_SIZE – the Size provided is zero, or leads to an overflow. + +ERROR_MEMDB_NOT_OWNER – the donating memory extent did not have ownership of the specified memory range. + +Also see: [Capability Errors](#capability-errors) ## VCPU Management @@ -1548,15 +1842,15 @@ Configure a VCPU Thread whose state is OBJECT_STATE_INIT. | Call number: | `hvc 0x6034` | | Inputs: | X0: vCPU CapID | | | X1: vCPUOptionFlags | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Types:** *vCPUOptionFlags:* -| Bit Numbers | Mask | Description | -|----------------------|----------------------------|-----------------------------------------| +| Bits | Mask | Description | +|-|---|-----| | 0 | `0x1` | AArch64 Self-hosted Debug Enable | | 1 | `0x2` | VCPU containing HLOS VM | | 63:2 | `0xFFFFFFFF.FFFFFFFE` | Reserved, Must be Zero | @@ -1569,7 +1863,7 @@ OK – the operation was successful, and the result is valid. ERROR_OBJECT_STATE – if the VCPU object is not in OBJECT_STATE_INIT state. -ERROR_ARGUMENT_INVALID – a value passed in an argument was invalid. This could be due to an invalid VCPU or option flags. +ERROR_ARGUMENT_INVALID – a value passed in an argument was invalid. This could be due to an invalid VCPU or option flag. Also see: [Capability Errors](#capability-errors) @@ -1588,12 +1882,16 @@ If the call targets a VCPU that is currently running on a different physical CPU | Call number: | `hvc 0x603d` | | Inputs: | X0: vCPU CapID | | | X1: Affinity CPUIndex | -| | X2: Reserved – Must be -1 | +| | X2: Reserved — Must be -1 | | Outputs: | X0: Error Result | **Types:** -CPUIndex — a number identifying the target physical CPU. For hardware platforms with physical CPUs that are linearly numbered from 0, this is equal to the physical CPU number; for AArch64 platforms, this is the case if three of the four affinity fields in MPIDR_EL1 have a zero value on every physical PE, and the CPUIndex corresponds to the value of the remaining MPIDR_EL1 affinity field. Otherwise, the hypervisor’s platform driver defines the mapping between CPUIndex values and physical CPUs, and VMs may be informed of this mapping at boot time via the boot environment data. If the scheduler supports directed yields and/or automatic migration of threads, the value -1 may be used to indicate that the VCPU should not have affinity to any physical CPU. +CPUIndex — a number identifying the target physical CPU. + +For hardware platforms with physical CPUs that are linearly numbered from 0, this is equal to the physical CPU number; for AArch64 platforms, this is the case if three of the four affinity fields in `MPIDR_EL1` have a zero value on every physical PE, and the CPUIndex corresponds to the value of the remaining `MPIDR_EL1` affinity field. Otherwise, the hypervisor’s platform driver defines the mapping between CPUIndex values and physical CPUs, and VMs may be informed of this mapping at boot time via the boot environment data. + +The value -1 (`CPU_INDEX_INVALID`) may be used to indicate that the VCPU should not have affinity to any physical CPU. If the scheduler does not support automatic migration of threads, this will effectively disable the VCPU, so an additional object right (Thread Disable) is required in this case. **Errors:** @@ -1607,43 +1905,109 @@ ERROR_DENIED – the specified VCPU is not permitted to change affinity because Also see: [Capability Errors](#capability-errors) +### Write to the Register Context of a VCPU Thread + +Write a specified value to one of a VCPU's registers. + +This may be called for any VCPU thread object that is currently in a virtual power-off state. +This includes VCPU objects that have not yet been activated. +Note that powering on a VCPU using a platform-specific power control API, such as `PSCI_CPU_ON`, might overwrite values set by this call. + +The register to write is identified by an architecture-specific enumeration identifying the set or group of registers, and an index into that set or group. +The primary purpose of this hypercall is to set the initial state of a VCPU before it is powered on. +Therefore, the architecture will typically only define access to the general-purpose registers, excluding extended register sets such as system control registers and floating-point or vector registers. + +| **Hypercall**: | `vcpu_register_write` | +|-------------------------|--------------------------------------| +| Call number: | `hvc 0x6064` | +| Inputs: | X0: vCPU CapID | +| | X1: RegisterSet | +| | X2: Index | +| | X3: Value | +| | X4: Reserved — Must be Zero | +| Outputs: | X0: Error Result | + +**Types:** + +*RegisterSet (AArch64)* + +| **RegisterSet** | **Name** | **Indices** | **Description** | +|-|---|-|-----| +| 0 | `VCPU_REGISTER_SET_X` | 0–31 | 64-bit general purpose registers X0-X30 | +| 1 | `VCPU_REGISTER_SET_PC` | 0 | Program counter (4-byte aligned) | +| 2 | `VCPU_REGISTER_SET_SP_EL` | 0–1 | Stack pointers for EL0 and EL1 | + ### Power on a VCPU Thread -Set a VCPU Thread’s initial execution state, including its entry point and context. +Bring a VCPU Thread out of its initial virtual power-off state. + +This call can also set the minimal initial execution state of the VCPU, including its entry point and a context pointer, avoiding the need to call `vcpu_register_write`. +The hypervisor does not dereference, check, or otherwise define any particular meaning for the context pointer. +It will be written to the first argument register in the VCPU's standard calling convention; for an AArch64 VCPU, this is X0. + +The entry point and context pointer each have a corresponding flag in the flags argument which will cause this call to discard the provided value and preserve the current state of the respective VCPU register. | **Hypercall**: | `vcpu_poweron` | |-------------------------|--------------------------------------| | Call number: | `hvc 0x6038` | | Inputs: | X0: vCPU CapID | -| | X1: vCPU EntryPointAddr | -| | X2: vCPU Context | -| | X3: Reserved – Must be Zero | +| | X1: EntryPointAddr VMPhysAddr | +| | X2: ContextPtr Register | +| | X3: vCPUPowerOnFlags | | Outputs: | X0: Error Result | +**Types:** + +*vCPUPowerOnFlags:* + +| Bits | Mask | Description | +|-|---|-----| +| 0 | `0x1` | Preserve entry point | +| 1 | `0x2` | Preserve context | +| 63:2 | `0xFFFFFFFF.FFFFFFFC` | Reserved — Must be Zero | + **Errors:** OK – the operation was successful, and the result is valid. ERROR_ARGUMENT_INVALID – a value passed in an argument was invalid. This could be due to an invalid VCPU. -ERROR_BUSY - the specified VCPU is currently busy and cannot be powered on at the moment. +ERROR_BUSY – the specified VCPU is currently busy and cannot be powered on at the moment. Also see: [Capability Errors](#capability-errors) ### Power off a VCPU Thread -Tear down the current thread’s VCPU execution state. This call will not return when successful. +Halt execution of the calling VCPU, and apply architecture-defined reset values to its register context. +The effect of the reset is architecture-specific, but will typically disable the first stage of address translation, and may also disable caches, mask interrupts, etc. + +This call will not return when successful. + +The specified VCPU capability must refer to the calling VCPU. Specifying any other VCPU is invalid. + +The last-VCPU bit in the flags argument must be set if, and only if, the caller is either the sole powered-on VCPU attached to a Virtual PM Group, or not attached to a Virtual PM Group at all. If this flag is not set correctly, the call may return ERROR_DENIED. This requirement prevents a VM inadvertently powering off all of its VCPUs, which is a state it cannot recover from without outside assistance. | **Hypercall**: | `vcpu_poweroff` | |-------------------------|--------------------------------------| | Call number: | `hvc 0x6039` | | Inputs: | X0: vCPU CapID | -| | X1: Reserved – Must be Zero | +| | X1: vCPUPowerOffFlags | | Outputs: | X0: Error Result | +**Types:** + +*vCPUPowerOffFlags:* + +| Bits | Mask | Description | +|-|---|-----| +| 0 | `0x1` | Last VCPU to power off in VM | +| 63:1 | `0xFFFFFFFF.FFFFFFFE` | Reserved — Must be Zero | + **Errors:** -ERROR_ARGUMENT_INVALID – a value passed in an argument was invalid. This could be due to an invalid VCPU. +ERROR_ARGUMENT_INVALID – a value passed in an argument was invalid. This could be due to an unrecognised flag value, or specifying a VCPU that is not the caller. + +ERROR_DENIED — the caller is the sole powered-on VCPU in a Virtual PM Group, and the last-VCPU flag was not set; or the caller is not the sole powered-on VCPU in a Virtual PM Group, and the last-VCPU flag was set. Also see: [Capability Errors](#capability-errors) @@ -1699,7 +2063,9 @@ Also see: [Capability Errors](#capability-errors) ### VCPU vIRQ Bind -Binds a VCPU interface to a virtual interrupt. +Each VCPU may have one or more associated virtual interrupt sources, depending on its configuration. This API binds one of those sources to a virtual IRQ number. + +If the IRQ type is set to `VCPU_RUN_WAKEUP`, binding the IRQ will automatically place the VCPU into a state in which it can only be scheduled by calling `vcpu_run`. Refer to the [documentation](#run-a-proxy-scheduled-vcpu-thread) for that hypercall for further details. | **Hypercall**: | `vcpu_bind_virq` | |-------------------------|--------------------------------------| @@ -1708,14 +2074,14 @@ Binds a VCPU interface to a virtual interrupt. | | X1: Virtual IC CapID | | | X2: Virtual IRQ Info | | | X3: VCPU Virtual IRQ Type | -| | X4: Reserved – Must be Zero | +| | X4: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Types:** | VCPU Virtual IRQ Type | Integer Value | |--------------------------------|------------------------| -| HALT | 0 | +| VCPU_RUN_WAKEUP | 1 | **Errors:** @@ -1733,98 +2099,172 @@ Also see: [Capability Errors](#capability-errors) ### VCPU vIRQ Unbind -Unbinds a VCPU interface from a virtual IRQ number. +Unbinds a VCPU interrupt source from a virtual IRQ number. + +If the IRQ type is set to `VCPU_RUN_WAKEUP`, unbinding the IRQ will allow the VCPU to run without a `vcpu_run` call, subject to its normal scheduling parameters and state. Note that in some cases this can cause incorrect execution in the VCPU. Refer to the [documentation](#run-a-proxy-scheduled-vcpu-thread) for that hypercall for further details. | **Hypercall**: | `vcpu_unbind_virq` | |-------------------------|--------------------------------------| | Call number: | `hvc 0x605d` | | Inputs: | X0: VCPU CapID | | | X1: VCPU Virtual IRQ Type | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | +**Types:** + +| VCPU Virtual IRQ Type | Integer Value | +|--------------------------------|------------------------| +| VCPU_RUN_WAKEUP | 1 | + **Errors:** OK – the operation was successful, or the VCPU interrupt was already unbound. Also see: [Capability Errors](#capability-errors) -### VCPU Write State +### Kill a VCPU thread -It updates the VCPU information related to the VCPUState specified. +Places the VCPU thread in a killed state, forcing it to exit and end execution. The VCPU can no longer be scheduled once it has exited. If the calling VCPU is targeting itself, this call will not return if successful. -| **Hypercall**: | `vcpu_write_state` | -|-------------------------|--------------------------------------------| -| Call number: | `hvc 0x605e` | -| Inputs: | X0: VCPU CapID | -| | X1: VCPUState | -| | X2: Data VMAddr | -| | X3: Size Size Must be non-zero. | -| | X4: Reserved – Must be Zero | -| Outputs: | X0: Error Result | +| **Hypercall**: | `vcpu_kill` | +|-------------------------|--------------------------------------| +| Call number: | `hvc 0x603a` | +| Inputs: | X0: VCPU CapID | +| | X1: Reserved — Must be Zero | +| Outputs: | X0: Error Result | +**Errors:** -**Types:** +OK – the operation was successful. -*VCPUState:* +ERROR_ARGUMENT_INVALID – a value passed in an argument was invalid. This could be due to an invalid VCPU. -| VCPUState | Integer Value | -|--------------------|------------------------| -| HALT | 0 | +ERROR_OBJECT_STATE – the VCPU thread was not active, or has already been killed. -**Errors:** +Also see: [Capability Errors](#capability-errors) -OK – the operation was successful, and the result is valid. +### Run a Proxy-Scheduled VCPU thread -ERROR_UNIMPLEMENTED - if functionality not implemented. +Donates CPU time to a VCPU that is configured for proxy scheduling. This is an optional mechanism that gives a privileged VM's scheduler limited control over the scheduling of another VM's VCPUs. -`TODO: TBD. Currently unimplemented` +This call may only be used on a VCPU that has a VIRQ bound to its `VCPU_RUN_WAKEUP` interrupt source. A VCPU that is in that state cannot be scheduled normally by the hypervisor scheduler; it will only execute when this hypercall is used to give it CPU time. -Also see: [Capability Errors](#capability-errors) +If all arguments are valid, this hypercall will attempt to context-switch to the specified VCPU. It returns when the caller is preempted or when the specified VCPU is unable to continue running. The VCPU state result indicates the reason that it was unable to continue. -### VCPU Read State +Some states may return additional state-specific data to allow the caller to take appropriate actions, and/or require additional data to resume execution which must be passed to the next `vcpu_run` call for the same VCPU. Also, some states may persist for some length of time that can't be directly predicted by the caller; when the VCPU leaves one of these states, it will assert the VIRQ bound to its `VCPU_RUN_WAKEUP` interrupt source. -It fetches the VCPU information related to the VCPUState specified. +For this call to behave as intended, the specified VCPU should have lower scheduling priority than the caller. Otherwise, the return from this call may be delayed until execution of the specified VCPU is blocked or its own timeslice expires. This rule is not enforced by the implementation. -| **Hypercall**: | `vcpu_read_state` | +| **Hypercall**: | `vcpu_run` | |-------------------------|--------------------------------------| -| Call number: | `hvc 0x605f` | +| Call number: | `hvc 0x6065` | | Inputs: | X0: VCPU CapID | -| | X1: VCPUState | -| | X2: Buffer VMAddr | -| | X3: MaximumSize Size | -| | X4: Reserved – Must be Zero | +| | X1: State-specific Resume Data 1 | +| | X2: State-specific Resume Data 2 | +| | X3: State-specific Resume Data 3 | +| | X4: Reserved —Must be Zero | | Outputs: | X0: Error Result | -| | X1: Size Size | +| | X1: VCPU Run State | +| | X2: State-specific Data 1 | +| | X3: State-specific Data 2 | +| | X4: State-specific Data 3 | -Size: is the number of bytes received. +**Types**: -**Errors:** +*VCPU Run State*: -OK – the operation was successful. +The following table shows the expected types of the state-specific data and resume data for each state. A 0 indicates that the argument or result is currently reserved and must be zero. -ERROR_ARGUMENT_INVALID – a value passed in an argument was invalid. This could be due to an invalid VCPU State value. +| State | Name | State Data 1 | State Data 2 | State Data 3 | Resume Data 1 | +|-|--|--|--|--|--| +| 0x0 | `READY` | 0 | 0 | 0 | 0 | +| 0x1 | `EXPECTS_WAKEUP` | VCPU Sleep Type | 0 | 0 | 0 | +| 0x2 | `POWERED_OFF` | VCPU Poweroff Type | 0 | 0 | 0 | +| 0x3 | `BLOCKED` | 0 | 0 | 0 | 0 | +| 0x4 | `ADDRSPACE_VMMIO_READ` | VMPhysAddr | Size | 0 | Register | +| 0x5 | `ADDRSPACE_VMMIO_WRITE` | VMPhysAddr | Size | Register | 0 | +| 0x100 | `PSCI_SYSTEM_RESET` | PSCI Reset Type | 0 | 0 | 0 | -ERROR_DENIED – VCPU State passed does not comply with current VCPU state. +The Resume Data 2 and 3 arguments are currently unused and must be zero for all states. -ERROR_ARGUMENT_SIZE – the MaximumSize provided is smaller than the information to be fetched. +0x0 `READY` +:The caller's hypervisor timeslice ended, or the caller received an interrupt. The caller should retry after handling any pending interrupts. -ERROR_ADDR_OVERFLOW – the information to be fetched is larger than the provided buffer, and could not be received. +0x1 `EXPECTS_WAKEUP` +:The VCPU is waiting to receive an interrupt; for example, it may have executed a WFI instruction, or made a firmware call requesting entry into a low-power state. In the latter case, the state-specific data in X2 will be a platform-specific nonzero value indicating the requested power state. For a platform that implements Arm's PSCI standard, it is in the same format as the state argument to a `PSCI_CPU_SUSPEND` call. The `VCPU_RUN_WAKEUP` VIRQ will be asserted when the VCPU leaves this state. -ERROR_ADDR_INVALID – some, or the whole of the provided buffer is not mapped. +0x2 `POWERED_OFF` +:The VCPU has not yet been started by calling `vcpu_poweron`, or has stopped itself by calling `vcpu_poweroff`, or has been terminated due to a reset request from another VM. If PSCI is implemented, this state is also reachable via PSCI calls. The `VCPU_RUN_WAKEUP` VIRQ will be asserted when the VCPU leaves this state. The first state data word contains a VCPU Poweroff Type value (defined below). + +0x3 `BLOCKED` +:The VCPU is temporarily unable to run due to a hypervisor operation. This may include a hypercall made by the VCPU that transiently blocks it, or by an incomplete migration from another physical CPU. The caller should retry after yielding to the calling VM's scheduler. + +0x4 `ADDRSPACE_VMMIO_READ` +:The VCPU has performed a read access to an unmapped stage 2 address inside a range previously nominated by a call to `addrspace_configure_vmmio`. The first two state data words contain the base IPA and the access size, respectively. The VCPU will be automatically resumed by the next `vcpu_run` call. The first resume data word for that call should be set to the value that will be returned by the read access. + +0x5 `ADDRSPACE_VMMIO_WRITE` +:The VCPU has performed a write access to an unmapped stage 2 address inside a range previously nominated by a call to `addrspace_configure_vmmio`. The three state data words contain the base IPA, access size, and the value written by the access, respectively. The VCPU will be automatically resumed by the next `vcpu_run` call. + +0x6 `FAULT` +: The VCPU has an unrecoverable fault. + +0x100 `PSCI_SYSTEM_RESET` +:On a platform that implements PSCI, the VCPU has made a call to `PSCI_SYSTEM_RESET` or `PSCI_SYSTEM_RESET2`. The first state data word contains a PSCI Reset Type value (defined below). For a `PSCI_SYSTEM_RESET2` call, the second state data word contains the cookie value. + +*VCPU Sleep Type:* + +This is a platform-specific unsigned word indicating a low-power suspend state. The value 0 is reserved for a trapped wait-for-interrupt or halt instruction, such as the AArch64 `WFI` instruction. + +If the platform implements PSCI, nonzero values are power state values as passed to `PSCI_CPU_SUSPEND`. + +*VCPU Poweroff Type*: + +| Value | Description | +|-|-----| +| 0 | Recoverable power-off state, e.g. `vcpu_poweroff` called. | +| 1 | Terminated; cannot run until the VM resets. | +| >1 | Reserved. | + +*PSCI Reset Type:* + +| Bits | Mask | Description | +|-|---|-----| +| 31:0 | `0xffffffff` | Reset type for `PSCI_SYSTEM_RESET2`; 0 for `PSCI_SYSTEM_RESET` | +| 61:32 | `0x3FFFFFFF.00000000` | Reserved — Must be Zero | +| 62 | `0x40000000.00000000` | 1: `PSCI_SYSTEM_RESET2` SMC64 call, 0: SMC32 call | +| 63 | `0x80000000.00000000` | 1: `PSCI_SYSTEM_RESET` call, 0: `PSCI_SYSTEM_RESET2` | + +**Errors:** + +OK – the operation was successful. + +ERROR_ARGUMENT_INVALID – a value passed in an argument was invalid. This could be due to an invalid VCPU. + +ERROR_BUSY – the specified VCPU does not have a bound `VCPU_RUN_WAKEUP` VIRQ. + +ERROR_OBJECT_STATE – the VCPU thread was not active, or has already been killed. Also see: [Capability Errors](#capability-errors) -### Kill a VCPU thread +### Check the State of a Halted VCPU -Places the VCPU thread in a killed state, forcing it to exit and end execution. The VCPU can no longer be scheduled once it has exited. If the calling VCPU is targeting itself, this call will not return if successful. +Query the state of a VCPU that has generated a halt VIRQ to determine why it halted. The state is described the same way as for `vcpu_run`, but the VCPU is not required to be proxy-scheduled. -| **Hypercall**: | `vcpu_kill` | +| **Hypercall**: | `vcpu_run_check` | |-------------------------|--------------------------------------| -| Call number: | `hvc 0x603a` | +| Call number: | `hvc 0x6068` | | Inputs: | X0: VCPU CapID | -| | X1: Reserved – Must be Zero | +| | X4: Reserved – Must be Zero | | Outputs: | X0: Error Result | +| | X1: VCPU Run State | +| | X2: State-specific Data | +| | X3: State-specific Data | +| | X4: State-specific Data | + +**Types**: + +Refer to the [documentation for `vcpu_run_thread`](#run-a-proxy-scheduled-vcpu-thread). **Errors:** @@ -1832,6 +2272,8 @@ OK – the operation was successful. ERROR_ARGUMENT_INVALID – a value passed in an argument was invalid. This could be due to an invalid VCPU. +ERROR_BUSY — the specified VCPU is not halted. + ERROR_OBJECT_STATE – the VCPU thread was not active, or has already been killed. Also see: [Capability Errors](#capability-errors) @@ -1840,20 +2282,20 @@ Also see: [Capability Errors](#capability-errors) ### Scheduler Yield -Attaches a VCPU to a Virtual PM Group. The Virtual PM Group object must have been activated before this function is called. The VCPU object must not have been activated. An attachment index must be specified which must be a non-negative integer less than the maximum number of attachments supported by this Virtual PM Group object. +Informs the hypervisor scheduler that the caller is executing a low priority task or waiting for a non-wakeup event to occur, and wants to give other VCPUs a chance to run. A hint argument may be provided to suggest to the scheduler that a particular VCPU or class of VCPUs should be run instead. | **Hypercall**: | `scheduler_yield` | |-------------------------|--------------------------------------| | Call number: | `hvc 0x603b` | | Inputs: | X0: control | | | X1: arg1 | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | *Control:* -| Bit Numbers | Mask | Description | -|----------------------|-------------------|----------------------------------------------------------------------------------------------------------------------------------------| +| Bits | Mask | Description | +|-|---|-----| | 15:0 | `0xffff` | hint: Yield type hint. | | 31 | `0x80000000` | imp_def: Implementation defined flag. If set, the hint value specifies a scheduler implementation specific yield operation. | @@ -1872,7 +2314,42 @@ Also see: [Capability Errors](#capability-errors) ## Virtual PM Group Management -A Virtual PM Group is a collection of VCPUs which share a virtual power management state. This state may be accessible via a virtualised platform-specific interface; on AArch64 this is the ARM PSCI (Platform State Configuration Interface) API. Attachment to this object type is optional for VCPUs in single-processor VMs that do not participate in power management decisions. +A Virtual PM Group is a collection of VCPUs which share a virtual power management state. This state may be accessible via a virtualised platform-specific interface; on AArch64 this is the Arm PSCI (Platform State Configuration Interface) API. Attachment to this object type is optional for VCPUs in single-processor VMs that do not participate in power management decisions. + +### Configure a Virtual PM Group + +Set configuration options for a Virtual PM Group whose state is `OBJECT_STATE_INIT`. Making this call is optional. + +| **Hypercall**: | `vpm_group_configure` | +|-------------------------|--------------------------------------| +| Call number: | `hvc 0x6066` | +| Inputs: | X0: VPMGroup CapID | +| | X1: VPMGroupOptionFlags | +| | X2: Reserved — Must be Zero | +| Outputs: | X0: Error Result | + +**Types:** + +*VPMGroupOptionFlags:* + +| Bit Numbers | Mask | Description | +|----------------------|----------------------------|---------------------------------| +| 0 | `0x1` | Exclude from aggregation | +| 63:1 | `0xFFFFFFFF.FFFFFFFE` | Reserved — Must be Zero | + +**Errors:** + +OK – the operation was successful. + +ERROR_ARGUMENT_INVALID – an unsupported or invalid configuration option was specified. + +Also see: [Capability Errors](#capability-errors) + +#### Power State Aggregation + +If the flags argument's "exclude from aggregation" bit is clear, which is the default configuration, the Virtual PM Group will collect power state votes from its attached VCPUs. These votes will be used when determining what power state the physical device should enter when one or more physical CPUs becomes idle. In general, a physical CPU will enter the shallowest available idle state permitted by the votes of its VCPUs, i.e. a state with wakeup latency no higher than the acceptable limit for each of the VCPUs. + +If the "exclude from aggregation" bit is set, the platform-specific power management API calls will still be available, but their effect on the physical power state may be limited. Also, validation of the power management API calls may be relaxed; e.g. for Arm PSCI implementations, the power state argument to `PSCI_CPU_SUSPEND` will not be validated against the states supported by the physical device. ### Virtual PM Group to VCPU Attachment @@ -1884,7 +2361,7 @@ Attaches a VCPU to a Virtual PM Group. The Virtual PM Group object must have bee | Inputs: | X0: VPMGroup CapID | | | X1: VCPU CapID | | | X2: Index | -| | X3: Reserved – Must be Zero | +| | X3: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -1909,7 +2386,7 @@ Binds a Virtual PM Group to a virtual interrupt. | Inputs: | X0: VPMGroup CapID | | | X1: Virtual IC CapID | | | X2: Virtual IRQ Info | -| | X3: Reserved – Must be Zero | +| | X3: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -1934,7 +2411,7 @@ Unbinds a Virtual PM Group from a virtual IRQ number. |-------------------------|--------------------------------------| | Call number: | `hvc 0x6044` | | Inputs: | X0: VPMGroup CapID | -| | X1: Reserved – Must be Zero | +| | X1: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -1951,7 +2428,7 @@ Gets the state of the Virtual PM Group. |-------------------------|--------------------------------------| | Call number: | `hvc 0x6045` | | Inputs: | X0: VPMGroup CapID | -| | X1: Reserved – Must be Zero | +| | X1: Reserved — Must be Zero | | Outputs: | X0: Error Result | | | X1: VPMState | @@ -1983,7 +2460,7 @@ Update the trace class flags values by specifying which flags to set and clear. | Call number: | `hvc 0x603f` | | Inputs: | X0: SetFlags | | | X1: ClearFlags | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | | | X1: SetFlags | @@ -2002,15 +2479,15 @@ Configure a Watchdog whose state is OBJECT_STATE_INIT. | Call number: | `hvc 0x6058` | | Inputs: | X0: Watchdog CapID | | | X1: WatchdogOptionFlags | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Types:** *WatchdogOptionFlags:* -| Bit Numbers | Mask | Description | -|----------------------|----------------------------|---------------------------------| +| Bits | Mask | Description | +|-|---|-----| | 0 | `0x1` | Critical bite | | 63:1 | `0xFFFFFFFF.FFFFFFFE` | Reserved, Must be Zero | @@ -2033,7 +2510,7 @@ Attaches a Watchdog object to a vCPU. The Watchdog object must have been activat | Call number: | `hvc 0x6040` | | Inputs: | X0: Watchdog CapID | | | X1: vCPU CapID | -| | X2: Reserved – Must be Zero | +| | X2: Reserved — Must be Zero | | Outputs: | X0: Error Result | **Errors:** @@ -2063,8 +2540,8 @@ Binds a Watchgdog (bark or bite) interface to a virtual IRQ number. *WatchdogBindOptionFlags:* -| Bit Numbers | Mask | Description | -|----------------------|----------------------------|----------------------------------------| +| Bits | Mask | Description | +|-|---|-----| | 0 | `0x1` | Bite virq (If unset, bark virq) | | 63:1 | `0xFFFFFFFF.FFFFFFFE` | Reserved, Must be Zero | @@ -2099,8 +2576,8 @@ Unbinds a Watchdog (bark or bite) interface virtual IRQ number. *WatchdogBindOptionFlags:* -| Bit Numbers | Mask | Description | -|----------------------|----------------------------|----------------------------------------| +| Bits | Mask | Description | +|-|---|-----| | 0 | `0x1` | Bite virq (If unset, bark virq) | | 63:1 | `0xFFFFFFFF.FFFFFFFE` | Reserved, Must be Zero | @@ -2112,6 +2589,342 @@ OK – the operation was successful, or the watchdog interrupt was already unbou Also see: [Capability Errors](#capability-errors) +### Watchdog Runtime Management + +Performs miscellaneous management operations on an arbitrary watchdog object (not necessarily the calling VM's watchdog). Currently, three operations are defined: + +1. Freeze a watchdog's counter, preventing a bark or bite occurring (if no such event has already occurred). +1. Freeze a watchdog's counter as above, and also reset the counter to 0. +1. Unfreeze a watchdog's counter. + +This is intended primarily for use by the manager of a proxy-scheduled VM, to prevent watchdog events occurring in the VM if the proxy threads cannot be scheduled. + +Note that freeze and unfreeze operations are counted, and the watchdog counter will only progress while the the freeze count is zero (i.e. freeze and unfreeze operations are balanced). Also, freeze and unfreeze operations may be performed automatically by the hypervisor in some cases. + +| **Hypercall**: | `watchdog_manage` | +|-------------------------|------------------------------------| +| Call number: | `hvc 0x6063` | +| Inputs: | X0: Watchdog CapID | +| | X1: WatchdogManageOperation | +| Outputs: | X0: Error Result | + +**Types:** + +*WatchdogManageOperation*: + +| Operation Enumerator | Integer Value | +|-----------------------------------------|------------------------| +| WATCHDOG_MANAGE_OP_FREEZE | 0 | +| WATCHDOG_MANAGE_OP_FREEZE_AND_RESET | 1 | +| WATCHDOG_MANAGE_OP_UNFREEZE | 2 | + +**Errors:** + +OK – the operation was successful, or the watchdog interrupt was already unbound. + +ERROR_BUSY – the operation failed because it would otherwise have overflowed or underflowed the watchdog's freeze count. + +Also see: [Capability Errors](#capability-errors) + +## Virtual IO MMIO Management + +### Configure a Virtual IO MMIO + +Configure a Virtual IO MMIO whose state is OBJECT_STATE_INIT. The Virtual IO MMIO device needs to get a reference to memextent that covers its range. + +| **Hypercall**: | `virtio_mmio_configure` | +|-------------------------|--------------------------------------| +| Call number: | `hvc 0x6049` | +| Inputs: | X0: VirtioMMIO CapID | +| | X1: Memextent CapID | +| | X3: VQsNum | +| | X3: Reserved — Must be Zero | +| Outputs: | X0: Error Result | + + +**Errors:** + +OK – the operation was successful, and the result is valid. + +ERROR_OBJECT_STATE – if the Virtual IO MMIO object is not in OBJECT_STATE_INIT state. + +ERROR_ARGUMENT_INVALID – a value passed in an argument was invalid. This could be due to a VQsNum out of range or if the specified memextent is not contiguous. + +Also see: [Capability Errors](#capability-errors) + +### Virtual IO MMIO Backend vIRQ Bind + +Binds a Virtual IO MMIO backend interface to a virtual interrupt. + +| **Hypercall**: | `virtio_mmio_bind_backend_virq` | +|-------------------------|----------------------------------------| +| Call number: | `hvc 0x604a` | +| Inputs: | X0: VirtioMMIO CapID | +| | X1: Virtual IC CapID | +| | X2: Virtual IRQ Info | +| | X3: Reserved — Must be Zero | +| Outputs: | X0: Error Result | + +**Errors:** + +OK – the operation was successful, and the result is valid. + +ERROR_NOMEM – the operation failed due to memory allocation error. + +ERROR_VIRQ_BOUND – the specified virtual IO MMIO is already bound to a VIRQ number. + +ERROR_BUSY – the specified VIRQ number is already bound to a source. + +ERROR_ARGUMENT_INVALID – a value passed in an argument was invalid. This could be due to an invalid Virtual IRQ Info value. + +Also see: [Capability Errors](#capability-errors) + +### Virtual IO MMIO Backend vIRQ Unbind + +Unbinds a Virtual IO MMIO backend interface from a virtual IRQ number. + +| **Hypercall**: | `virtio_mmio_unbind_backend_virq` | +|-------------------------|------------------------------------------| +| Call number: | `hvc 0x604b` | +| Inputs: | X0: VirtioMMIO CapID | +| | X1: Reserved — Must be Zero | +| Outputs: | X0: Error Result | + +**Errors:** + +OK – the operation was successful, or the virtual IO MMIO interrupt was already unbound. + +Also see: [Capability Errors](#capability-errors) + +### Virtual IO MMIO Frontend vIRQ Bind + +Binds a Virtual IO MMIO frontend interface to a virtual interrupt. + +| **Hypercall**: | `virtio_mmio_bind_frontend_virq` | +|-------------------------|-----------------------------------------| +| Call number: | `hvc 0x604c` | +| Inputs: | X0: VirtioMMIO CapID | +| | X1: Virtual IC CapID | +| | X2: Virtual IRQ Info | +| | X3: Reserved — Must be Zero | +| Outputs: | X0: Error Result | + +**Errors:** + +OK – the operation was successful, and the result is valid. + +ERROR_NOMEM – the operation failed due to memory allocation error. + +ERROR_VIRQ_BOUND – the specified virtual IO MMIO is already bound to a VIRQ number. + +ERROR_BUSY – the specified VIRQ number is already bound to a source. + +ERROR_ARGUMENT_INVALID – a value passed in an argument was invalid. This could be due to an invalid Virtual IRQ Info value. + +Also see: [Capability Errors](#capability-errors) + +### Virtual IO MMIO Frontend vIRQ Unbind + +Unbinds a Virtual IO MMIO backend interface from a virtual IRQ number. + +| **Hypercall**: | `virtio_mmio_unbind_frontend_virq` | +|-------------------------|-------------------------------------------| +| Call number: | `hvc 0x604d` | +| Inputs: | X0: VirtioMMIO CapID | +| | X1: Reserved — Must be Zero | +| Outputs: | X0: Error Result | + +**Errors:** + +OK – the operation was successful, or the virtual IO MMIO interrupt was already unbound. + +Also see: [Capability Errors](#capability-errors) + +### Virtual IO MMIO Backend Assert vIRQ + +The backend makes this call to assert the virtual IRQ directed to the frontend and writes a bit mask of events that caused the assertion. + +| **Hypercall**: | `virtio_mmio_backend_assert_virq` | +|-------------------------|------------------------------------------| +| Call number: | `hvc 0x604e` | +| Inputs: | X0: VirtioMMIO CapID | +| | X1: InterruptStatus | +| | X2: Reserved — Must be Zero | +| Outputs: | X0: Error Result | + +**Errors:** + +OK – the operation was successful, or the virtual IO MMIO interrupt was already unbound. + +ERROR_DENIED – Cannot assert irq since there is a reset currently pending. + +ERROR_ARGUMENT_INVALID – A value passed in an argument was invalid. + +Also see: [Capability Errors](#capability-errors) + +### Virtual IO MMIO Backend Set DeviceFeatures + +Set the device features flags based on the specified device features selector. The device features specified must comply with the features enforced by the hypervisor (VIRTIO_F_VERSION_1, VIRTIO_F_ACCESS_PLATFORM, !VIRTIO_F_NOTIFICATION_DATA). + +| **Hypercall**: | `virtio_mmio_backend_set_dev_features` | +|-------------------------|-----------------------------------------------| +| Call number: | `hvc 0x604f` | +| Inputs: | X0: VirtioMMIO CapID | +| | X1: DeviceFeaturesSel | +| | X2: DeviceFeatures | +| | X3: Reserved — Must be Zero | +| Outputs: | X0: Error Result | + +**Errors:** + +OK – The operation was successful, and the result is valid. + +ERROR_ARGUMENT_INVALID – A value passed in an argument was invalid. + +ERROR_DENIED – Device features passed do not comply with the features enforced by the hypervisor. + +Also see: [Capability Errors](#capability-errors) + +### Virtual IO MMIO Backend Set QueueNumMax + +Set maximum virtual queue size of the queue specified by the queue selector. + +| **Hypercall**: | `virtio_mmio_backend_set_queue_num_max` | +|-------------------------|------------------------------------------------| +| Call number: | `hvc 0x6050` | +| Inputs: | X0: VirtioMMIO CapID | +| | X1: QueueSel | +| | X2: QueueNumMax | +| | X3: Reserved — Must be Zero | +| Outputs: | X0: Error Result | + +**Errors:** + +OK – The operation was successful, and the result is valid. + +ERROR_ARGUMENT_INVALID – A value passed in an argument was invalid. + +Also see: [Capability Errors](#capability-errors) + +### Virtual IO MMIO Backend Get DriverFeatures + +Get the driver features flags based on the specified driver features selector. + +| **Hypercall**: | `virtio_mmio_backend_get_drv_features` | +|-------------------------|-----------------------------------------------| +| Call number: | `hvc 0x6051` | +| Inputs: | X0: VirtioMMIO CapID | +| | X1: DriverFeaturesSel | +| | X2: Reserved — Must be Zero | +| Outputs: | X0: Error Result | +| | X1: DriverFeatures | + +**Errors:** + +OK – The operation was successful, and the result is valid. + +ERROR_ARGUMENT_INVALID – A value passed in an argument was invalid. + +Also see: [Capability Errors](#capability-errors) + +### Virtual IO MMIO Backend Get Queue Info + +Get information from the queue specified by the queue selector. + +| **Hypercall**: | `virtio_mmio_backend_get_queue_info` | +|-------------------------|---------------------------------------------| +| Call number: | `hvc 0x6052` | +| Inputs: | X0: VirtioMMIO CapID | +| | X1: QueueSel | +| | X2: Reserved — Must be Zero | +| Outputs: | X0: Error Result | +| | X1: QueueNum | +| | X2: QueueReady | +| | X3: QueueDesc (low and high) | +| | X4: QueueDriver (low and high) | +| | X5: QueueDevice (low and high) | + +**Errors:** + +OK – The operation was successful, and the result is valid. + +ERROR_ARGUMENT_INVALID – A value passed in an argument was invalid. + +Also see: [Capability Errors](#capability-errors) + +### Virtual IO MMIO Backend Get Notification + +The backend should make this call, when its VIRQ is asserted, to get a bitmap of the virtual queues that need to be notified and a bitmap of the reasons why the VIRQ was asserted. This calls also deasserts the backend’s VIRQ. + +| **Hypercall**: | `virtio_mmio_backend_get_notification` | +|-------------------------|-----------------------------------------------| +| Call number: | `hvc 0x6053` | +| Inputs: | X0: VirtioMMIO CapID | +| | X1: Reserved — Must be Zero | +| Outputs: | X0: Error Result | +| | X1: VQs Bitmap | +| | X2: NotifyReason Bitmap | + +**Types:** + +*NotifyReason:* + +| Bits | Mask | Description | +|-|---|-----| +| 0 | `0x1` | 1 = NEW_BUFFER: notifies the device that there are new buffers to process in a queue. | +| 1 | `0x2` | 1 = RESET_RQST: notifies the device that a device reset has been requested. | +| 2 | `0x4` | 1 = INTERRUPT_ACK: notifies the device that the frontend wrote to the InterruptACK register. | +| 3 | `0x8` | 1 = DRIVER_OK: notifies the device that the frontend has set the DRIVER_OK bit of the Status register. | +| 4 | `0x10` | 1 = FAILED: notifies the device that the frontend has set the FAILED bit of the Status register. | +| 63:5 | `0xFFFFFFFF.FFFFFFE0` | Reserved = 0 [TBD notify reasons] | + +**Errors:** + +OK – The operation was successful, and the result is valid. + +ERROR_ARGUMENT_INVALID – A value passed in an argument was invalid. + +Also see: [Capability Errors](#capability-errors) + +### Virtual IO MMIO Backend Acknowledge Reset + +The backend should make this call after a device reset is completed. This call will clear all bits in QueueReady for all queues in the device. + +| **Hypercall**: | `virtio_mmio_backend_acknowledge_reset` | +|-------------------------|------------------------------------------------| +| Call number: | `hvc 0x6054` | +| Inputs: | X0: VirtioMMIO CapID | +| | X1: Reserved — Must be Zero | +| Outputs: | X0: Error Result | + +**Errors:** + +OK – The operation was successful, and the result is valid. + +ERROR_ARGUMENT_INVALID – A value passed in an argument was invalid. + +Also see: [Capability Errors](#capability-errors) + +### Virtual IO MMIO Backend Set Status + +This calls sets status register. + +| **Hypercall**: | `virtio_mmio_backend_set_status` | +|-------------------------|-----------------------------------------| +| Call number: | `hvc 0x6055` | +| Inputs: | X0: VirtioMMIO CapID | +| | X1: Status | +| | X2: Reserved — Must be Zero | +| Outputs: | X0: Error Result | + +**Errors:** + +OK – The operation was successful, and the result is valid. + +ERROR_ARGUMENT_INVALID – A value passed in an argument was invalid. + +Also see: [Capability Errors](#capability-errors) + ## PRNG Management ### PRNG Get Entropy @@ -2122,7 +2935,7 @@ Gets random numbers from a DRBG that is seeded by a TRNG. Typically this API wil |---------------------|--------------------------------| | Call number: | `hvc 0x6057` | | Inputs: | X0: NumBytes | -| | X1: Reserved – Must be Zero | +| | X1: Reserved — Must be Zero | | Outputs: | X0: Error Result | | | X1: Data0 | | | X2: Data1 | @@ -2135,13 +2948,14 @@ OK – the operation was successful, and the result is valid. ERROR_ARGUMENT_SIZE – the NumBytes provided is zero, or exceeds the possible bytes to be returned in the Data output registers. -ERROR_BUSY - Called within the read rate-limit window. +ERROR_BUSY – Called within the read rate-limit window. -ERROR_UNIMPLEMENTED - if functionality not implemented. +ERROR_UNIMPLEMENTED – if functionality not implemented. Also see: [Capability Errors](#capability-errors) ## Error Results + ### Error Code Enumeration | Error Enumerator | Integer Value | @@ -2179,17 +2993,19 @@ Also see: [Capability Errors](#capability-errors) | ERROR_MSGQUEUE_EMPTY | 60 | | ERROR_MSGQUEUE_FULL | 61 | | | | +| ERROR_MEMDB_NOT_OWNER | 111 | | ERROR_MEMEXTENT_MAPPINGS_FULL | 120 | +| ERROR_MEMEXTENT_TYPE | 121 | | ERROR_EXISTING_MAPPING | 200 | ### Capability Errors -ERROR_CSPACE_CAP_NULL - invalid CapID. +ERROR_CSPACE_CAP_NULL – invalid CapID. -ERROR_CSPACE_CAP_REVOKED - CapID no longer valid since it has already been revoked. +ERROR_CSPACE_CAP_REVOKED – CapID no longer valid since it has already been revoked. -ERROR_CSPACE_WRONG_OBJECT_TYPE - CapID does not correspond with the specified object type. +ERROR_CSPACE_WRONG_OBJECT_TYPE – CapID does not correspond with the specified object type. -ERROR_CSPACE_INSUFFICIENT_RIGHTS - CapID has not enough rights to execute operation. +ERROR_CSPACE_INSUFFICIENT_RIGHTS – CapID has not enough rights to execute operation. -ERROR_CSPACE_FULL - CSpace has reached maximum number of capabilities allowed. +ERROR_CSPACE_FULL – CSpace has reached maximum number of capabilities allowed. diff --git a/docs/build.md b/docs/build.md index b7bbea4..f3e803d 100644 --- a/docs/build.md +++ b/docs/build.md @@ -15,54 +15,73 @@ Always ensure you have activated the `gunyah-venv` *before* running `configure` The following repositories are needed to build a Gunyah Hypervisor image: -- [Gunyah Hypervisor](https://github.com/quic/gunyah-hypervisor) - The Gunyah Hypervisor. +These should all be cloned into the same top-level directory (this assumed in the Docker setup). + +- [Gunyah Hypervisor](https://github.com/quic/gunyah-hypervisor) — The Gunyah Hypervisor. ```bash - git clone https://github.com/quic/gunyah-hypervisor.git + git clone https://github.com/quic/gunyah-hypervisor.git hyp ``` -- [Resource Manager](https://github.com/quic/gunyah-resource-manager) - The privileged root VM and VM manager supporting the Gunyah Hypervisor. +- [Resource Manager](https://github.com/quic/gunyah-resource-manager) — The privileged root VM and VM manager supporting the Gunyah Hypervisor. ```bash - git clone https://github.com/quic/gunyah-resource-manager.git + git clone https://github.com/quic/gunyah-resource-manager.git resource-manager ``` -- [Gunyah C Runtime](https://github.com/quic/gunyah-c-runtime) - A runtime for light-weight OS-less application VMs. +- [Gunyah C Runtime](https://github.com/quic/gunyah-c-runtime) — A runtime for light-weight OS-less application VMs. ```bash -git clone https://github.com/quic/gunyah-c-runtime.git +git clone https://github.com/quic/gunyah-c-runtime.git musl-c-runtime ``` ## Build Configuration The build system has several configuration parameters that must be set: -* `platform`: selects the target hardware platform; -* `featureset`: selects a named set of enabled features; and -* `quality`: specifies the build quality, e.g. `debug`, `production` etc, which affect compilation of runtime assertions, compiler optimisations etc. +* `platform`: selects the target hardware platform +* `featureset`: selects a named hypervisor architecture configuration +* `quality`: specifies the build quality, e.g. `debug`, `production` etc., which modify the build - such as including runtime assertions, compiler optimisations etc. -These parameters must be set on the build system's command line; if one or more of them is left unset, the build system will print the known values for the missing parameter and abort. You may specify a comma-separated list to select multiple values for a parameter, or `all` to select every known value. You may also specify `all=true`, which is equivalent to specifying `all` for every parameter that is not otherwise specified. The when multiple options are selected, each combination (variant) will be built in separate output directories under the `build` directory. +These parameters must be set on the build system's command line; if one or more +of them is left unset, the build system will print the known values for the +missing parameter and abort. You may specify a comma-separated list to select +multiple values for a parameter, or `all` to select every valid combination. +You may also specify simply `all=true`, which is equivalent to specifying `all` +for every parameter that is not otherwise specified. The when multiple options +are selected, each combination (variant) will be built in separate output +directories under the `build` directory. -Each project may be build using `ninja` or `scons` and the process for build configuration depends on the selected build tool used. See the sections below. +Each project may be built using `ninja` or `scons` and the process for build configuration depends on the selected build tool used. See the sections below. ## Building -The Gunyah Hypervisor, Resource Manager and C runtime are built separately, each following the same build instructions below. These will be packages together into a final boot image. +The Gunyah Hypervisor, Resource Manager and C runtime are built separately, +each following the similar build instructions below. These separate images need +to be packaged together into a final boot image. -> IMPORTANT! If making hypervisor public API changes, these changes need to be added to the Resource Manager and libc Runtime. +> IMPORTANT! If making hypervisor public API changes, these changes will need to be updated in the Resource Manager and Runtime sources. ### Building with Ninja -To configure the build for using *Ninja*, run `./configure.py `, specifying the configuration parameters. +To configure the build for use with *Ninja*, run `./configure.py ` +in the top-level source repository of the component, specifying the desired +configuration parameters. -For example, in each of the Gunyah Hypervisor, Resource Manager and Gunyah C Runtime checkouts, run: +For example, in each of the Gunyah Hypervisor, Resource Manager and Gunyah C Runtime source directories, run: ```sh ./configure.py platform=qemu featureset=gunyah-rm-qemu quality=production ``` -or to build all available configurations for QEMU: +or to build all available configurations for the QEMU platform: ```sh ./configure.py platform=qemu all=true ``` This will create a `build` directory and Ninja build rules file for each enabled build variant. Generally, the `configure` step only needs to be run once. -Run `ninja` to build. There is usually no need to specify `-j` or similar, as Ninja will select this automatically. Ninja also will incrementally re-build if run again after making code changes. +```sh +ninja +``` + +Run `ninja` to build. There is usually no need to specify `-j` or similar, as +Ninja will select this automatically. Ninja also will incrementally re-build if +run again after making code changes. > Note, if configuration files are modified, Ninja will rerun the configuration tool with the previous parameters. However, you must manually rerun the configuration step if you rename or delete an existing module or configuration parameter, as Ninja will refuse to run if a build configuration file is missing. To build a specific file (for example, a single variant when multiple variants have been configured), specify its full name as the target on the `ninja` command line. @@ -71,19 +90,21 @@ To clean the build, run `ninja -t clean`. It should not be necessary to do this ### Building with SCons -To perform a standalone SCons build, run `scons`, specifying the configuration parameters. For example, to build debug builds of all available feature sets for the QEMU: +To perform a standalone SCons build, run `scons`, specifying the configuration +parameters. For example, to build debug builds of all available feature sets +for the QEMU platform: ```sh scons platform=qemu featureset=all quality=debug ``` -Note, configuration parameters *must* be specified on every SCons build; they will not be cached. +Note, configuration parameters *must* be specified on every time you perform a SCons build; configuration is not cached. To clean the build, run `scons -c all=true`, or use configuration parameters to select a specific variant to clean. It should not be necessary to do this routinely. ## Producing a Boot Image -Once you have built the Gunyah Hypervisor, Resource Manager and C Runtime, a boot image can be prepared. +Once you have built the Gunyah Hypervisor, Resource Manager and C Runtime, a boot image needs be generated. To reduce the size of the boot image, the generated binaries of Resource Manager and C Runtime need to be stripped with the following commands: ```bash @@ -93,7 +114,9 @@ $LLVM/bin/llvm-strip -o /build/runtime.strip git clone https://github.com/eliben/pyelftools.git @@ -102,10 +125,10 @@ git clone https://github.com/eliben/pyelftools.git To generate `hypvm.elf` boot image run these steps (substituting ``s for each tool / executable): ```bash cd -PYTHONPATH=/pyelftools tools/elf/package_apps.py \ +tools/elf/package_apps.py \ -a /build/resource-manager.strip \ -r /build/runtime.strip \ /build/qemu/gunyah-rm-qemu/production/hyp.elf \ -o /hypvm.elf ``` -> Note, you may wish to pick a different hypervisor `hyp.elf` from a different build variant (i.e. `build/qemu/gunyah-rm-qemu/production/`). +> Note, you may wish to pick a different hypervisor `hyp.elf` from a different build variant (i.e. `build/qemu/gunyah-rm-qemu/debug/`). diff --git a/docs/docker.md b/docs/docker.md deleted file mode 100644 index 1682834..0000000 --- a/docs/docker.md +++ /dev/null @@ -1,31 +0,0 @@ -# Docker - -A Docker container can be used to host the build tools and QEMU simulator. This is an alternative to installing them directly on a host Linux workstation. - -## Installation - -Install Docker in your machine following the instructions: https://docs.docker.com/engine/install/ - -## Build the Docker image from a Dockerfile - -A Dockerfile may be found here: [Gunyah support scripts](https://github.com/quic/gunyah-support-scripts) - -To build the Docker image, first go to the directory that contains the [Dockerfile](https://github.com/quic/gunyah-support-scripts/tree/develop/gunyah-qemu-aarch64/Dockerfile): -```bash -cd -``` - -Build the Dockerfile and give it a name and optionally a tag using the following command: -```bash -docker build -f Dockerfile -t : . -``` - -> Note, for building this image you may need to increase the available disk space for Docker. - -Finally, run the generated Docker image with this command: -```bash -docker run -it : -``` -## Development - -Once your Docker container is setup, proceed to building the hypervisor: [Build instructions](build.md) diff --git a/docs/setup.md b/docs/setup.md index 200c99d..1bde4c6 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -1,48 +1,37 @@ # Setup Instructions -The following instructions describe setting up a Linux machine for Gunyah development and testing. You may alternatively wish to use Docker: +## Development Environment -[Docker Instructions](docker.md) +We recommend using a Docker container with the required compilers and tools for +convenience. -## Toolchain -The Gunyah Hypervisor projects use the LLVM 10+ toolchain, cross-compiled for AArch64. Standalone applications (Resource Manager) are built with the musl libc library. +A separate _Gunyah Support Scripts_ repository is maintained with reference +Docker based environment instructions and scripts. -A [script](https://github.com/quic/gunyah-support-scripts/tree/develop/gunyah-qemu-aarch64/llvm_musl_build.sh) to help build a suitable LLVM compiler (with musl libc) can be found in [Gunyah support scripts](https://github.com/quic/gunyah-support-scripts). +See: +[Gunyah Support Scripts](https://github.com/quic/gunyah-support-scripts/tree/develop) ```bash git clone https://github.com/quic/gunyah-support-scripts.git ``` -To build the LLVM toolchain, first execute the provided [script](https://github.com/quic/gunyah-support-scripts/tree/develop/gunyah-qemu-aarch64/llvm_musl_build.sh) in the directory where you want it to be installed. +## Custom Dev Environment -```bash -cd -.//llvm_musl_build.sh -``` - -> Note, this script requires cmake to be installed. - -This script will generate a `llvm-musl-install` folder in the current directory. +If you intend to setup your own development environment, you can follow the +reference Docker setup on your development host. This process is not +documented. -## C Application Sysroot +### Toolchain -The Resource Manager needs to be compiled using the libfdt library, cross-compiled for AArch64. - -The following instructions indicate how to get the libfdt source code and cross compile it for AArch64: - -```bash -git clone https://github.com/dgibson/dtc.git -cd dtc -CC=aarch64-linux-gnu-gcc make install libfdt PREFIX= -``` +The Gunyah Hypervisor projects use the LLVM v15 toolchain, cross-compiled for +AArch64 and musl libc. This is due to standalone application VMs (Resource +Manager) are built with a runtime supporting the musl libc library. -> Note, \ refers to the desired path where the built will be installed. +### Set up environment variables -## Set up environment variables You must set the following environment variables: -- To point to the toolchain (assumed to be Clang 10.0 or later): ```bash -export LLVM=/path/to/llvm10/ +export LLVM=/path/to/llvm15/ ``` > Note, when using the toolchain built with the provided script, point to the "llvm-musl-install" generated folder. > `export LLVM=/path/to/llvm-musl-install` @@ -52,7 +41,7 @@ export LLVM=/path/to/llvm10/ export LOCAL_SYSROOT=/path/to/c-application-sysroot ``` -## Install the Python dependencies +### Install the Python dependencies Create a virtual environment, activate it, and install the modules used by the auto-generation code: @@ -62,4 +51,6 @@ python3 -m venv gunyah-venv pip install -r /tools/requirements.txt ``` -We recommend installing the Python environment _outside_ the Gunyah source directory. This is so the automatic dependency detection in the Python scripts ignores modules imported from the virtual environment. +We recommend installing the Python environment _outside_ the Gunyah source +directory. This is so the automatic dependency detection in the Python scripts +ignores modules imported from the virtual environment. diff --git a/docs/terminology.md b/docs/terminology.md new file mode 100644 index 0000000..a267e70 --- /dev/null +++ b/docs/terminology.md @@ -0,0 +1,12 @@ +## Terminology: + +### PVM : **P**rimary **V**irtual **M**achine +- This is the VM that's created for the primary HLOS to execute +- Generally all the resources are assigned to this VM + +### HLOS : **H**igh **L**evel **OS** +- Mean the main OS owning the majority of the HW device access. +- Typically used as a synonym for PVM. + +### SVM : **S**econdary **V**irtual **M**achine +- These VM's are created and launched from the PVM diff --git a/docs/test.md b/docs/test.md index f36abe5..68613c6 100644 --- a/docs/test.md +++ b/docs/test.md @@ -1,54 +1,18 @@ # Testing Gunyah Hypervisor We provide two ways of testing the hypervisor: -1. [Using a Docker container](#1-using-a-docker-container) -2. [Using local machine](#2-using-a-local-linux-machine) +1. [Using a Docker container](#using-a-docker-container) +2. [Using local machine](#using-a-local-linux-machine) -## 1. Using a Docker container +## Using a Docker container -A Docker image can been built (with this [Dockerfile](https://github.com/quic/gunyah-support-scripts/tree/develop/gunyah-qemu-aarch64/Dockerfile)) to make the testing easier, without needing to install the tools directly on your machine. +A Docker image can been built with the required compiler and QEMU Arm System emulator. +See [Setup Instructions](setup.md). -The Dockerfile provided contains: -- Latest QEMU (version required: >= 5.0.0) -- GDB 9.2 (version required: >= 9.2) -- Latest Linux image (to be used as primary VM) -- RAM disk generated with Busybox-1.33.0 (to be used by primary VM) -- Device tree (to be used by primary VM) -- LLVM 10.0.1 toolchain with musl libc for AArch64 -- Virtual environment (gunyah-venv) -- Script with the QEMU start command (start_cmd.sh) -- Required environment variables set -- Required dependencies installed +## Using a local Linux machine -Once a Docker image is built, the required files can be found in `$OUTPUT_DIR` (`/usr/local/src/out`). - -### Instructions - -1. Build the Docker image: - [Docker instructions](docker.md) - -> Note, make sure that at this point you are running on the Docker container's shell. (e.g. `docker run -it :`) - -2. Download and build the hypervisor source code: - [Build instructions](build.md) -3. Boot the Gunyah Hypervisor with the Linux VM on the QEMU simulator. - - Use the following QEMU start command: -```bash -qemu-system-aarch64 -machine virt,virtualization=on,gic-version=3,highmem=off \ --cpu max -m size=2G -smp cpus=8 -nographic \ --kernel /hypvm.elf \ --device loader,file=$OUTPUT_DIR/Image,addr=$LINUX_BASE \ --device loader,file=$OUTPUT_DIR/virt.dtb,addr=$DT_BASE \ --device loader,file=$OUTPUT_DIR/initrd.img,addr=$INITRD_BASE -``` - -> Note, see [Hardcoded Parameters](#hardcoded-parameters) section for an explanation of why these `_BASE` values have been set as environment variables. - -## 2. Using a local Linux machine - -1. Install latest QEMU: - [QEMU instructions](qemu.md) +1. Build and install a recent QEMU (v7.2 is tested): + - See Docker support scripts for reference 2. Download and build the hypervisor source code: - [Setup Instructions](setup.md) - [Build instructions](build.md) @@ -56,69 +20,7 @@ qemu-system-aarch64 -machine virt,virtualization=on,gic-version=3,highmem=off \ [Linux instructions](linux.md) 4. Create a RAM disk for Linux: [RAM disk instructions](ramdisk.md) -5. Generate a device tree with 512M of RAM: - -```bash -qemu-system-aarch64 \ --machine virt,virtualization=on,gic-version=3,highmem=off \ --cpu max -m size=512M -smp cpus=8 -nographic \ --kernel /hypvm.elf \ --append "rw root=/dev/ram rdinit=/sbin/init earlyprintk=serial,ttyAMA0 console=ttyAMA0" \ --machine dumpdtb=virt_qemu.dtb -``` - -6. Modify the "chosen" node in the device tree based on the created RAM disk: - - Create a device tree overlay with the new "chosen" node values and apply it to the dtb generated in the previous step. - -```bash -export INITRD_BASE=0x44400000 -export INITRD_SIZE=$(stat -Lc %s /initrd.img) -export INITRD_END=$(printf "0x%x" $((${INITRD_BASE} + ${INITRD_SIZE}))) -cat < overlay.dts -/dts-v1/; -/{ -fragment@0{ -target-path = "/chosen"; -__overlay__{ -linux,initrd-start = <${INITRD_BASE}>; -linux,initrd-end = <${INITRD_END}>; -}; -}; -}; -EOF -dtc -@ -I dts -O dtb overlay.dts -o overlay.dtbo -fdtoverlay -v -i virt_qemu.dtb -o virt.dtb overlay.dtbo -rm overlay.dts overlay.dtbo virt_qemu.dtb -dtc -I dtb -O dts virt.dtb > virt.dts -cp virt.dts /. -cp virt.dtb /. -``` - +5. Generate a device tree for the QEMU platform: + - See Docker support scripts for reference 7. Boot the Gunyah Hypervisor with the Linux VM on the QEMU simulator: - -```bash -qemu-system-aarch64 \ --machine virt,virtualization=on,gic-version=3,highmem=off \ --cpu max -m size=2G -smp cpus=8 -nographic \ --kernel /hypvm.elf \ --device loader,file=/Image,addr=$LINUX_BASE \ --device loader,file=/virt.dtb,addr=$DT_BASE \ --device loader,file=/initrd.img,addr=$INITRD_BASE -``` - -> Note, see [Hardcoded Parameters](#hardcoded-parameters) section for an explanation of why these `_BASE` values have been set as environment variables. - -## Hardcoded parameters - -In the [Gunyah Hypervisor](https://github.com/quic/gunyah-hypervisor) repository, specifically in `hyp/platform/soc_qemu/src/boot.c`, we currently have hardcoded the base addresses to be used in the QEMU start command. We hope to address this in future contributions such that these values only need to be set in the QEMU command. For now, in case you want to change these values in the start command, please make sure you also modify these values in the code, without overlapping with the range assigned to the hypervisor RAM. - -The current hardcoded addresses that need to be used in the QEMU start command are: -(set in [boot.c](https://github.com/quic/gunyah-hypervisor/tree/develop/hyp/platform/soc_qemu/src/boot.c) in function soc_qemu_handle_rootvm_init()) -- LINUX_BASE=0x41080000 -- DT_BASE=0x44200000 -- INITRD_BASE=0x44400000 - -Current hypervisor RAM range: -(set in [boot.c](https://github.com/quic/gunyah-hypervisor/tree/develop/hyp/platform/soc_qemu/src/boot.c) in function platform_ram_probe()) -- 0x80000000..0xBFFFFFFF + - See Docker support scripts for reference diff --git a/hyp/arch/aarch64/include/asm/asm_defs.inc b/hyp/arch/aarch64/include/asm/asm_defs.inc index a0a3869..6f789b9 100644 --- a/hyp/arch/aarch64/include/asm/asm_defs.inc +++ b/hyp/arch/aarch64/include/asm/asm_defs.inc @@ -49,7 +49,7 @@ .endm // Branch target identification helpers -#if defined(ARCH_ARM_8_5_BTI) +#if defined(ARCH_ARM_FEAT_BTI) #define BRANCH_TARGET(type, ...) \ bti type; \ __VA_ARGS__ diff --git a/hyp/arch/aarch64/include/asm/barrier.h b/hyp/arch/aarch64/include/asm/barrier.h index e64e4c9..dab8db1 100644 --- a/hyp/arch/aarch64/include/asm/barrier.h +++ b/hyp/arch/aarch64/include/asm/barrier.h @@ -21,4 +21,4 @@ // The asm_ordering variable is used as an artificial dependency to order // different individual asm statements with respect to each other in a way that // is lighter weight than a full "memory" clobber. -extern struct asm_ordering_dummy asm_ordering; +extern asm_ordering_dummy_t asm_ordering; diff --git a/hyp/arch/aarch64/include/asm/panic.inc b/hyp/arch/aarch64/include/asm/panic.inc index f26f27e..ac26422 100644 --- a/hyp/arch/aarch64/include/asm/panic.inc +++ b/hyp/arch/aarch64/include/asm/panic.inc @@ -7,6 +7,6 @@ local panic_str\@: .asciz "\panic_str" .popsection - adr x0, LOCAL(panic_str\@) + adrl x0, LOCAL(panic_str\@) bl panic .endm diff --git a/hyp/arch/aarch64/include/asm/sysregs.h b/hyp/arch/aarch64/include/asm/sysregs.h index ba6649a..6b89691 100644 --- a/hyp/arch/aarch64/include/asm/sysregs.h +++ b/hyp/arch/aarch64/include/asm/sysregs.h @@ -4,17 +4,17 @@ #define sysreg64_read(reg, val) \ do { \ - register_t _val; \ - __asm__ volatile("mrs %0, " #reg ";" : "=r"(_val)); \ - val = (__typeof__(val))_val; \ + register_t val_; \ + __asm__ volatile("mrs %0, " #reg ";" : "=r"(val_)); \ + val = (__typeof__(val))val_; \ } while (0) #define sysreg64_read_ordered(reg, val, ordering_var) \ do { \ - register_t _val; \ + register_t val_; \ __asm__ volatile("mrs %0, " #reg ";" \ - : "=r"(_val), "+m"(ordering_var)); \ - val = (__typeof__(val))_val; \ + : "=r"(val_), "+m"(ordering_var)); \ + val = (__typeof__(val))val_; \ } while (0) #define sysreg64_write(reg, val) \ diff --git a/hyp/arch/aarch64/include/asm/system_registers.h b/hyp/arch/aarch64/include/asm/system_registers.h index 4e84023..a6a2c00 100644 --- a/hyp/arch/aarch64/include/asm/system_registers.h +++ b/hyp/arch/aarch64/include/asm/system_registers.h @@ -40,12 +40,14 @@ #define ISS_MRS_MSR_ID_AA64PFR0_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 0, 4, 0) #define ISS_MRS_MSR_ID_AA64PFR1_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 0, 4, 1) #define ISS_MRS_MSR_ID_AA64ZFR0_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 0, 4, 4) +#define ISS_MRS_MSR_ID_AA64SMFR0_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 0, 4, 5) #define ISS_MRS_MSR_ID_AA64DFR0_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 0, 5, 0) #define ISS_MRS_MSR_ID_AA64DFR1_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 0, 5, 1) #define ISS_MRS_MSR_ID_AA64AFR0_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 0, 5, 4) #define ISS_MRS_MSR_ID_AA64AFR1_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 0, 5, 5) #define ISS_MRS_MSR_ID_AA64ISAR0_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 0, 6, 0) #define ISS_MRS_MSR_ID_AA64ISAR1_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 0, 6, 1) +#define ISS_MRS_MSR_ID_AA64ISAR2_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 0, 6, 2) #define ISS_MRS_MSR_ID_AA64MMFR0_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 0, 7, 0) #define ISS_MRS_MSR_ID_AA64MMFR1_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 0, 7, 1) #define ISS_MRS_MSR_ID_AA64MMFR2_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 0, 7, 2) @@ -68,6 +70,7 @@ #define ISS_MRS_MSR_PMCCFILTR_EL0 ISS_OP0_OP1_CRN_CRM_OP2(3, 3, 14, 15, 7) #define ISS_MRS_MSR_SCTLR_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 1, 0, 0) +#define ISS_MRS_MSR_ACTLR_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 1, 0, 1) #define ISS_MRS_MSR_TTBR0_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 2, 0, 0) #define ISS_MRS_MSR_TTBR1_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 2, 0, 1) #define ISS_MRS_MSR_TCR_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 2, 0, 2) @@ -109,7 +112,7 @@ #define ISS_MRS_MSR_DC_CISW ISS_OP0_OP1_CRN_CRM_OP2(1, 0, 7, 14, 2) #define ISS_MRS_MSR_DC_ISW ISS_OP0_OP1_CRN_CRM_OP2(1, 0, 7, 6, 2) -#if defined(ARCH_ARM_8_4_AMU) || defined(ARCH_ARM_8_6_AMU) +#if defined(ARCH_ARM_FEAT_AMUv1) || defined(ARCH_ARM_FEAT_AMUv1p1) #define ISS_MRS_MSR_AMCR_EL0 ISS_OP0_OP1_CRN_CRM_OP2(3, 3, 13, 2, 0) #define ISS_MRS_MSR_AMCFGR_EL0 ISS_OP0_OP1_CRN_CRM_OP2(3, 3, 13, 2, 1) #define ISS_MRS_MSR_AMCGCR_EL0 ISS_OP0_OP1_CRN_CRM_OP2(3, 3, 13, 2, 2) @@ -118,7 +121,7 @@ #define ISS_MRS_MSR_AMCNTENSET0_EL0 ISS_OP0_OP1_CRN_CRM_OP2(3, 3, 13, 2, 5) #define ISS_MRS_MSR_AMCNTENCLR1_EL0 ISS_OP0_OP1_CRN_CRM_OP2(3, 3, 13, 3, 0) #define ISS_MRS_MSR_AMCNTENSET1_EL0 ISS_OP0_OP1_CRN_CRM_OP2(3, 3, 13, 3, 1) -#if defined(ARCH_ARM_8_6_AMU) +#if defined(ARCH_ARM_FEAT_AMUv1p1) #define ISS_MRS_MSR_AMCG1IDR_EL0 ISS_OP0_OP1_CRN_CRM_OP2(3, 3, 13, 2, 6) #endif #endif diff --git a/hyp/arch/aarch64/include/reg/registers_arm.inc b/hyp/arch/aarch64/include/reg/registers_arm.inc new file mode 100644 index 0000000..6233a3d --- /dev/null +++ b/hyp/arch/aarch64/include/reg/registers_arm.inc @@ -0,0 +1,13 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier; BSD-3-Clause + +#if defined(ARCH_ARM_FEAT_VHE) +#define VHE(X) X##2 [X!] +#define VHE_V(X, V) X##2 [V] +#define VHE_T(X, T) X##2 [X!] +#else +#define VHE(X) X +#define VHE_V(X, V) X [V] +#define VHE_T(X, T) X +#endif diff --git a/hyp/arch/aarch64/link.lds b/hyp/arch/aarch64/link.lds index da09d39..42d0f5a 100644 --- a/hyp/arch/aarch64/link.lds +++ b/hyp/arch/aarch64/link.lds @@ -32,6 +32,7 @@ SECTIONS image_phys_start = LOADADDR(.text); .text : AT (PLATFORM_LMA_BASE) { *(.text.boot) + *(.text.boot.*) *(SORT_BY_ALIGNMENT(.text .text.*)) KEEP(*(.text.debug)); } : text @@ -61,7 +62,11 @@ SECTIONS /* Package data for RootVM */ . = ALIGN(4096); +#ifdef PLATFORM_ROOTVM_PKG_START_BASE + image_pkg_start = PLATFORM_ROOTVM_PKG_START_BASE; +#else image_pkg_start = . - image_virt_start + image_phys_start; +#endif /* align RW sections to the next 2MB page */ . = ALIGN(0x200000); @@ -86,9 +91,12 @@ SECTIONS #endif . = ALIGN(64); +#if PLATFORM_RW_DATA_SIZE < 0x200000 +#error PLATFORM_RW_DATA_SIZE too small +#endif .heap.root (NOLOAD) : { heap_private_start = .; - . = ALIGN(PLATFORM_HEAP_PRIVATE_SIZE); + . = data_base + PLATFORM_RW_DATA_SIZE; heap_private_end = .; } : NONE image_virt_last = . - 1; diff --git a/hyp/arch/aarch64/registers.reg b/hyp/arch/aarch64/registers.reg index b353dfa..1425360 100644 --- a/hyp/arch/aarch64/registers.reg +++ b/hyp/arch/aarch64/registers.reg @@ -2,15 +2,7 @@ // // SPDX-License-Identifier; BSD-3-Clause -#if defined(ARCH_ARM_8_1_VHE) -#define VHE(X) X##2 [X!] -#define VHE_V(X, V) X##2 [V] -#define VHE_T(X, T) X##2 [X!] -#else -#define VHE(X) X -#define VHE_V(X, V) X [V] -#define VHE_T(X, T) X -#endif +#include ACTLR_EL1 ACTLR_EL2 @@ -25,7 +17,7 @@ CCSIDR_EL1 R CCSIDR2_EL1 R CLIDR_EL1 r CNTFRQ_EL0 -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) CNTHCTL_EL2 [E2H0 E2H1] #else CNTHCTL_EL2 [E2H0] @@ -56,13 +48,16 @@ VHE_T(CNTV_CVAL_EL0, CNT_CVAL!) oOrwR VHE_T(CNTV_TVAL_EL0, CNT_TVAL!) CNTVCT_EL0 R +#if defined(ARCH_ARM_FEAT_ECV) +CNTPOFF_EL2 +#endif CNTVOFF_EL2 VHE(CONTEXTIDR_EL1) -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) CONTEXTIDR_EL2 #endif VHE(CPACR_EL1) -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) CPTR_EL2 [E2H0 E2H1] oOrw #else CPTR_EL2 [E2H0] oOrw @@ -81,7 +76,7 @@ VHE(ELR_EL1) //ELR_EL2 //SPSR_EL2 [A32 A64] -#if (defined(ARCH_ARM_8_2_RAS) || defined(ARCH_ARM_8_4_RAS)) +#if (defined(ARCH_ARM_FEAT_RAS) || defined(ARCH_ARM_FEAT_RASv1p1)) //ERRIDR_EL1 r ERRSELR_EL1 Orw //ERXADDR_EL1 @@ -112,8 +107,9 @@ HSTR_EL2 //ID_AA64AFR1_EL1 r ID_AA64DFR0_EL1 r //ID_AA64DFR1_EL1 r -//ID_AA64ISAR0_EL1 r +ID_AA64ISAR0_EL1 r ID_AA64ISAR1_EL1 r +ID_AA64ISAR2_EL1 r ID_AA64MMFR0_EL1 r ID_AA64MMFR1_EL1 r ID_AA64MMFR2_EL1 r @@ -246,7 +242,7 @@ PMCR_EL0 oOrw //PMXEVTYPER_EL0 //REVIDR_EL1 r VHE(SCTLR_EL1) -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) SCTLR_EL2 [VM E2H_TGE] #else SCTLR_EL2 [VM] @@ -258,7 +254,7 @@ SP_EL2 SPSel VHE_V(SPSR_EL1, SPSR_EL1_A64!) VHE(TCR_EL1) -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) TCR_EL2 [E2H0 E2H1] rwW #else TCR_EL2 [E2H0] rwW @@ -267,7 +263,7 @@ TPIDR_EL0 TPIDR_EL1 TPIDR_EL2 TPIDRRO_EL0 -#if defined(ARCH_ARM_8_4_TRACE) +#if defined(ARCH_ARM_FEAT_TRF) VHE(TRFCR_EL1) Orw #endif VHE(TTBR0_EL1) @@ -285,7 +281,7 @@ VSESR_EL2 w //VSTCR_EL2 //VSTTBR_EL2 VTCR_EL2 -VTTBR_EL2 +VTTBR_EL2 oOrw //VHE(ZCR_EL1) //ZCR_EL2 @@ -335,7 +331,7 @@ ICH_VMCR_EL2 oOrw ICH_VTR_EL2 r #endif -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) APDAKeyHi_EL1 rw APDAKeyLo_EL1 rw APDBKeyHi_EL1 rw @@ -348,7 +344,7 @@ APGAKeyHi_EL1 rw APGAKeyLo_EL1 rw #endif -#if defined(ARCH_ARM_8_5_MEMTAG) && defined(INTERFACE_VCPU) +#if defined(ARCH_ARM_FEAT_MTE) && defined(INTERFACE_VCPU) GCR_EL1 rw RGSR_EL1 rw VHE(TFSR_EL1) rw @@ -395,7 +391,9 @@ DBGCLAIMSET_EL1 Ow //DBGDTRRX_EL0 Or //DBGDTRTX_EL0 Ow //DBGPRCR_EL1 -//DBGVCR32_EL2 +#if ARCH_AARCH64_32BIT_EL1 +DBGVCR32_EL2 Orw +#endif DBGWCR0_EL1 Orw DBGWCR1_EL1 Orw DBGWCR2_EL1 Orw @@ -442,12 +440,189 @@ OSLSR_EL1 Or //SDER32_EL2 #endif -#if defined(ARCH_ARM_8_5_RNG) +#if defined(ARCH_ARM_FEAT_TRF) +TRFCR_EL2 Orw +#endif + +#if defined(MODULE_PLATFORM_ETE) +TRCLAR Orw +TRCIDR0 r +TRCIDR2 r +TRCIDR3 r +TRCIDR4 r +TRCIDR5 r +TRCSTATR Orw +TRCPRGCTLR Orw +TRCCONFIGR Orw +TRCAUXCTLR Orw +TRCEVENTCTL0R Orw +TRCEVENTCTL1R Orw +TRCRSR Orw +TRCSTALLCTLR Orw +TRCTSCTLR Orw +TRCSYNCPR Orw +TRCCCCTLR Orw +TRCBBCTLR Orw +TRCTRACEIDR Orw +TRCQCTLR Orw +TRCVICTLR Orw +TRCVIIECTLR Orw +TRCVISSCTLR Orw +TRCVIPCSSCTLR Orw +TRCSEQEVR0 Orw +TRCSEQEVR1 Orw +TRCSEQEVR2 Orw +TRCSEQRSTEVR Orw +TRCSEQSTR Orw +TRCEXTINSELR0 Orw +TRCEXTINSELR1 Orw +TRCEXTINSELR2 Orw +TRCEXTINSELR3 Orw +TRCCNTRLDVR0 Orw +TRCCNTRLDVR1 Orw +TRCCNTRLDVR2 Orw +TRCCNTRLDVR3 Orw +TRCCNTCTLR0 Orw +TRCCNTCTLR1 Orw +TRCCNTCTLR2 Orw +TRCCNTCTLR3 Orw +TRCCNTVR0 Orw +TRCCNTVR1 Orw +TRCCNTVR2 Orw +TRCCNTVR3 Orw +TRCIMSPEC1 Orw +TRCIMSPEC2 Orw +TRCIMSPEC3 Orw +TRCIMSPEC4 Orw +TRCIMSPEC5 Orw +TRCIMSPEC6 Orw +TRCIMSPEC7 Orw +TRCRSCTLR2 Orw +TRCRSCTLR3 Orw +TRCRSCTLR4 Orw +TRCRSCTLR5 Orw +TRCRSCTLR6 Orw +TRCRSCTLR7 Orw +TRCRSCTLR8 Orw +TRCRSCTLR9 Orw +TRCRSCTLR10 Orw +TRCRSCTLR11 Orw +TRCRSCTLR12 Orw +TRCRSCTLR13 Orw +TRCRSCTLR14 Orw +TRCRSCTLR15 Orw +TRCRSCTLR16 Orw +TRCRSCTLR17 Orw +TRCRSCTLR18 Orw +TRCRSCTLR19 Orw +TRCRSCTLR20 Orw +TRCRSCTLR21 Orw +TRCRSCTLR22 Orw +TRCRSCTLR23 Orw +TRCRSCTLR24 Orw +TRCRSCTLR25 Orw +TRCRSCTLR26 Orw +TRCRSCTLR27 Orw +TRCRSCTLR28 Orw +TRCRSCTLR29 Orw +TRCRSCTLR30 Orw +TRCRSCTLR31 Orw +TRCSSCCR0 Orw +TRCSSCCR1 Orw +TRCSSCCR2 Orw +TRCSSCCR3 Orw +TRCSSCCR4 Orw +TRCSSCCR5 Orw +TRCSSCCR6 Orw +TRCSSCCR7 Orw +TRCSSCSR0 Orw +TRCSSCSR1 Orw +TRCSSCSR2 Orw +TRCSSCSR3 Orw +TRCSSCSR4 Orw +TRCSSCSR5 Orw +TRCSSCSR6 Orw +TRCSSCSR7 Orw +TRCSSPCICR0 Orw +TRCSSPCICR1 Orw +TRCSSPCICR2 Orw +TRCSSPCICR3 Orw +TRCSSPCICR4 Orw +TRCSSPCICR5 Orw +TRCSSPCICR6 Orw +TRCSSPCICR7 Orw +TRCACVR0 Orw +TRCACVR1 Orw +TRCACVR2 Orw +TRCACVR3 Orw +TRCACVR4 Orw +TRCACVR5 Orw +TRCACVR6 Orw +TRCACVR7 Orw +TRCACVR8 Orw +TRCACVR9 Orw +TRCACVR10 Orw +TRCACVR11 Orw +TRCACVR12 Orw +TRCACVR13 Orw +TRCACVR14 Orw +TRCACVR15 Orw +TRCACATR0 Orw +TRCACATR1 Orw +TRCACATR2 Orw +TRCACATR3 Orw +TRCACATR4 Orw +TRCACATR5 Orw +TRCACATR6 Orw +TRCACATR7 Orw +TRCACATR8 Orw +TRCACATR9 Orw +TRCACATR10 Orw +TRCACATR11 Orw +TRCACATR12 Orw +TRCACATR13 Orw +TRCACATR14 Orw +TRCACATR15 Orw +TRCCIDCVR0 Orw +TRCCIDCVR1 Orw +TRCCIDCVR2 Orw +TRCCIDCVR3 Orw +TRCCIDCVR4 Orw +TRCCIDCVR5 Orw +TRCCIDCVR6 Orw +TRCCIDCVR7 Orw +TRCVMIDCVR0 Orw +TRCVMIDCVR1 Orw +TRCVMIDCVR2 Orw +TRCVMIDCVR3 Orw +TRCVMIDCVR4 Orw +TRCVMIDCVR5 Orw +TRCVMIDCVR6 Orw +TRCVMIDCVR7 Orw +TRCCIDCCTLR0 Orw +TRCCIDCCTLR1 Orw +TRCVMIDCCTLR0 Orw +TRCVMIDCCTLR1 Orw + +TRCCLAIMCLR Orw +TRCCLAIMSET Orw +#endif + +#if defined(MODULE_PLATFORM_TBRE) +TRBLIMITR_EL1 Orw +TRBPTR_EL1 Orw +TRBBASER_EL1 Orw +TRBSR_EL1 Orw +TRBMAR_EL1 Orw +TRBTRG_EL1 Orw +#endif + +#if defined(ARCH_ARM_FEAT_RNG) RNDR O RNDRRS O #endif -#if defined(ARCH_ARM_8_4_AMU) || defined(ARCH_ARM_8_6_AMU) +#if defined(ARCH_ARM_FEAT_AMUv1) || defined(ARCH_ARM_FEAT_AMUv1p1) AMCR_EL0 r AMCFGR_EL0 r AMCGCR_EL0 r @@ -456,7 +631,7 @@ AMCNTENCLR1_EL0 r AMCNTENSET0_EL0 r AMCNTENSET1_EL0 r AMUSERENR_EL0 -#if defined(ARCH_ARM_8_6_AMU) +#if defined(ARCH_ARM_FEAT_AMUv1p1) AMCG1IDR_EL0 r #endif #endif @@ -464,3 +639,10 @@ AMCG1IDR_EL0 r #if defined(MODULE_PLATFORM_ARM_DSU) && defined(INTERFACE_VCPU) S3_0_C15_C3_1 [IMP_CLUSTERIDR_EL1!] Or #endif + +#if defined(ARCH_ARM_FEAT_CSV2_2) || defined(ARCH_ARM_FEAT_CSV2_1p2) || \ + defined(ARCH_ARM_FEAT_CSV2_3) +SCXTNUM_EL0 +SCXTNUM_EL1 +SCXTNUM_EL2 +#endif diff --git a/hyp/arch/aarch64/src/asm_ordering.c b/hyp/arch/aarch64/src/asm_ordering.c index c8b5198..03f2149 100644 --- a/hyp/arch/aarch64/src/asm_ordering.c +++ b/hyp/arch/aarch64/src/asm_ordering.c @@ -6,4 +6,4 @@ #include -struct asm_ordering_dummy asm_ordering; +asm_ordering_dummy_t asm_ordering; diff --git a/hyp/arch/aarch64/src/nospec_checks.c b/hyp/arch/aarch64/src/nospec_checks.c index 4a4aa6b..176fab3 100644 --- a/hyp/arch/aarch64/src/nospec_checks.c +++ b/hyp/arch/aarch64/src/nospec_checks.c @@ -16,8 +16,9 @@ nospec_range_check(index_t val, index_t limit) "cset %w[valid], lo;" "csel %w[result_r], %w[val], wzr, lo;" "csdb;" - : [valid] "=r"(valid), [result_r] "=r"(result.r) - : [val] "r"(val), [limit] "r"(limit)); + : [valid] "=&r"(valid), [result_r] "=r"(result.r) + : [val] "r"(val), [limit] "r"(limit) + : "cc"); if (valid) { result.e = OK; diff --git a/hyp/arch/aarch64/registers.tmpl b/hyp/arch/aarch64/templates/hypregisters.h.tmpl similarity index 96% rename from hyp/arch/aarch64/registers.tmpl rename to hyp/arch/aarch64/templates/hypregisters.h.tmpl index 450d940..90a664e 100644 --- a/hyp/arch/aarch64/registers.tmpl +++ b/hyp/arch/aarch64/templates/hypregisters.h.tmpl @@ -34,7 +34,7 @@ ${t}_raw(${v})#slurp static inline ${vt}_t register_${vn}_read${suffix}( #if ordered -struct asm_ordering_dummy *ordering_var +asm_ordering_dummy_t *ordering_var #else void #end if @@ -67,7 +67,7 @@ void static inline void register_${vn}_write${suffix}(const ${vt}_t val #if ordered -, struct asm_ordering_dummy *ordering_var +, asm_ordering_dummy_t *ordering_var #end if ) { diff --git a/hyp/arch/aarch64/types.tc b/hyp/arch/aarch64/types.tc index 4e0952f..82af192 100644 --- a/hyp/arch/aarch64/types.tc +++ b/hyp/arch/aarch64/types.tc @@ -6,16 +6,16 @@ define asm_ordering_dummy structure { dummy char; }; -define AARCH64_INST_EXCEPTION_VAL constant = 0xd4000000; -define AARCH64_INST_EXCEPTION_MASK constant = 0xff000000; +define AARCH64_INST_EXCEPTION_VAL constant uint32 = 0xd4000000; +define AARCH64_INST_EXCEPTION_MASK constant uint32 = 0xff000000; -define AARCH64_INST_EXCEPTION_IMM16_MASK constant = 0x001fffe0; +define AARCH64_INST_EXCEPTION_IMM16_MASK constant uint32 = 0x001fffe0; define AARCH64_INST_EXCEPTION_IMM16_SHIFT constant = 5; -define AARCH64_INST_EXCEPTION_SUBTYPE_HLT_VAL constant = 0x00400000; -define AARCH64_INST_EXCEPTION_SUBTYPE_MASK constant = 0x00e0001f; +define AARCH64_INST_EXCEPTION_SUBTYPE_HLT_VAL constant uint32 = 0x00400000; +define AARCH64_INST_EXCEPTION_SUBTYPE_MASK constant uint32 = 0x00e0001f; -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) define aarch64_pauth_key structure(aligned(16)) { lo uint64; hi uint64; @@ -28,4 +28,4 @@ define aarch64_pauth_keys structure(aligned(16)) { ib structure aarch64_pauth_key; ga structure aarch64_pauth_key; }; -#endif // defined(ARCH_ARM_8_3_PAUTH) +#endif // defined(ARCH_ARM_FEAT_PAuth) diff --git a/hyp/arch/armv8/include/asm/cache.h b/hyp/arch/armv8/include/asm/cache.h index 25a7fe2..0744532 100644 --- a/hyp/arch/armv8/include/asm/cache.h +++ b/hyp/arch/armv8/include/asm/cache.h @@ -2,40 +2,43 @@ // // SPDX-License-Identifier: BSD-3-Clause -#define CACHE_DO_OP(x, size, op, is_object) \ +#define CACHE_BARRIER_OBJECT_LOAD(x, order) \ + __asm__ volatile("" : "=r"(order) : "m"(*(x))) +#define CACHE_BARRIER_OBJECT_STORE(x, order) \ + __asm__ volatile("" : "+r"(order), "+m"(*(x))) +#define CACHE_BARRIER_MEMORY_LOAD(x, order) \ + __asm__ volatile("" : "=r"(order) : : "memory") +#define CACHE_BARRIER_MEMORY_STORE(x, order) \ + __asm__ volatile("" : "+r"(order) : : "memory") + +#define CACHE_DO_OP(x, size, op, CACHE_BARRIER) \ do { \ - size_t _line_size = 1U << CPU_L1D_LINE_BITS; \ + const size_t line_size_ = 1U << CPU_L1D_LINE_BITS; \ \ - uintptr_t _base = \ - util_balign_down((uintptr_t)(x), _line_size); \ - uintptr_t _end = (uintptr_t)(x) + (size); \ + uintptr_t line_base_ = \ + util_balign_down((uintptr_t)(x), line_size_); \ + uintptr_t end_ = (uintptr_t)(x) + (size); \ register_t ordering; \ \ - assert(!util_add_overflows((uintptr_t)x, (size)-1)); \ + assert(!util_add_overflows((uintptr_t)x, (size)-1U)); \ \ - if (is_object) { \ - __asm__ volatile("" : "=r"(ordering) : "m"(*(x))); \ - } else { \ - __asm__ volatile("" : "=r"(ordering) : : "memory"); \ - } \ + CACHE_BARRIER##_LOAD(x, ordering); \ \ do { \ __asm__ volatile("DC " #op ", %1" \ : "+r"(ordering) \ - : "r"(_base)); \ - _base = _base + _line_size; \ - } while (_base < _end); \ + : "r"(line_base_)); \ + line_base_ = line_base_ + line_size_; \ + } while (line_base_ < end_); \ \ __asm__ volatile("dsb ish" : "+r"(ordering)); \ - if (is_object) { \ - __asm__ volatile("" : "+r"(ordering), "+m"(*(x))); \ - } else { \ - __asm__ volatile("" : "+r"(ordering) : : "memory"); \ - } \ + CACHE_BARRIER##_STORE(x, ordering); \ } while (0) -#define CACHE_OP_RANGE(x, size, op) CACHE_DO_OP(x, size, op, false) -#define CACHE_OP_OBJECT(x, op) CACHE_DO_OP(&(x), sizeof(x), op, true) +#define CACHE_OP_RANGE(x, size, op) \ + CACHE_DO_OP(x, size, op, CACHE_BARRIER_MEMORY) +#define CACHE_OP_OBJECT(x, op) \ + CACHE_DO_OP(&(x), sizeof(x), op, CACHE_BARRIER_OBJECT) #define CACHE_CLEAN_RANGE(x, size) CACHE_OP_RANGE(x, size, CVAC) #define CACHE_INVALIDATE_RANGE(x, size) CACHE_OP_RANGE(x, size, IVAC) @@ -49,8 +52,8 @@ do { \ struct { \ char p[size]; \ - } *_x = (void *)x; \ - CACHE_OP_OBJECT(*_x, op); \ + } *x_ = (void *)x; \ + CACHE_OP_OBJECT(*x_, op); \ } while (0) #define CACHE_CLEAN_FIXED_RANGE(x, size) CACHE_OP_FIXED_RANGE(x, size, CVAC) diff --git a/hyp/arch/build.conf b/hyp/arch/build.conf index bc25161..3461f5b 100644 --- a/hyp/arch/build.conf +++ b/hyp/arch/build.conf @@ -2,7 +2,12 @@ # # SPDX-License-Identifier: BSD-3-Clause -arch_source aarch64 timestamp.c asm_ordering.c nospec_checks.c arch_types aarch64 types.tc +arch_template registers aarch64 hypregisters.h +arch_registers aarch64 registers.reg + +arch_source aarch64 timestamp.c asm_ordering.c nospec_checks.c arch_source cortex-a-v8_2 sysreg_init.c arch_events cortex-a-v8_2 sysreg_init.ev +arch_source cortex-a-v9 sysreg_init.c +arch_events cortex-a-v9 sysreg_init.ev diff --git a/hyp/arch/cortex-a-v8_0/include/asm/cpu.h b/hyp/arch/cortex-a-v8_0/include/asm/cpu.h new file mode 100644 index 0000000..7bde800 --- /dev/null +++ b/hyp/arch/cortex-a-v8_0/include/asm/cpu.h @@ -0,0 +1,17 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +// Miscellaneous definitions describing the CPU implementation. + +// The size in address bits of a line in the innermost visible data cache. +#define CPU_L1D_LINE_BITS 6U + +// The size in address bits of the CPU's DC ZVA block. This is nearly always +// the same as CPU_L1D_LINE_BITS. +#define CPU_DCZVA_BITS 6U + +// The largest difference between the source and destination pointers during +// the optimised memcpy() for this CPU. This is here because it might depend +// on CPU_L1D_LINE_BITS in some implementations. +#define CPU_MEMCPY_STRIDE 256U diff --git a/hyp/arch/cortex-a-v8_0/include/asm/system_registers_cpu.h b/hyp/arch/cortex-a-v8_0/include/asm/system_registers_cpu.h new file mode 100644 index 0000000..c12bd69 --- /dev/null +++ b/hyp/arch/cortex-a-v8_0/include/asm/system_registers_cpu.h @@ -0,0 +1,11 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +// AArch64 System Register Encoding (CPU Implementation Defined Registers) +// +// This list is not exhaustive, it contains mostly registers likely to be +// trapped and accessed indirectly. + +#define ISS_MRS_MSR_CPUACTLR_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 1, 15, 2, 0) +#define ISS_MRS_MSR_CPUECTLR_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 1, 15, 2, 1) diff --git a/hyp/arch/cortex-a-v8_0/src/sysreg_init.c b/hyp/arch/cortex-a-v8_0/src/sysreg_init.c new file mode 100644 index 0000000..2bd9d78 --- /dev/null +++ b/hyp/arch/cortex-a-v8_0/src/sysreg_init.c @@ -0,0 +1,20 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include + +#include + +#include "event_handlers.h" + +void +arch_cortex_a_v80_handle_boot_cpu_warm_init(void) +{ + // ACTLR_EL2 controls EL1 access to some important features such as + // CLUSTERPMU and power control registers, which can be exploited to + // perform security/DoS attacks. Therefore, we deny all these accesses + // by writing 0 to this register. + ACTLR_EL2_t val = ACTLR_EL2_default(); + register_ACTLR_EL2_write(val); +} diff --git a/hyp/arch/cortex-a-v8_0/sysreg_init.ev b/hyp/arch/cortex-a-v8_0/sysreg_init.ev new file mode 100644 index 0000000..5f3db9b --- /dev/null +++ b/hyp/arch/cortex-a-v8_0/sysreg_init.ev @@ -0,0 +1,8 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module arch + +subscribe boot_cpu_warm_init + handler arch_cortex_a_v80_handle_boot_cpu_warm_init diff --git a/hyp/arch/cortex-a-v9/include/asm/cpu.h b/hyp/arch/cortex-a-v9/include/asm/cpu.h new file mode 100644 index 0000000..7bde800 --- /dev/null +++ b/hyp/arch/cortex-a-v9/include/asm/cpu.h @@ -0,0 +1,17 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +// Miscellaneous definitions describing the CPU implementation. + +// The size in address bits of a line in the innermost visible data cache. +#define CPU_L1D_LINE_BITS 6U + +// The size in address bits of the CPU's DC ZVA block. This is nearly always +// the same as CPU_L1D_LINE_BITS. +#define CPU_DCZVA_BITS 6U + +// The largest difference between the source and destination pointers during +// the optimised memcpy() for this CPU. This is here because it might depend +// on CPU_L1D_LINE_BITS in some implementations. +#define CPU_MEMCPY_STRIDE 256U diff --git a/hyp/arch/cortex-a-v9/include/asm/system_registers_cpu.h b/hyp/arch/cortex-a-v9/include/asm/system_registers_cpu.h new file mode 100644 index 0000000..332aa23 --- /dev/null +++ b/hyp/arch/cortex-a-v9/include/asm/system_registers_cpu.h @@ -0,0 +1,13 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +// AArch64 System Register Encoding (CPU Implementation Defined Registers) +// +// This list is not exhaustive, it contains mostly registers likely to be +// trapped and accessed indirectly. + +#define ISS_MRS_MSR_CPUACTLR_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 15, 1, 0) +#define ISS_MRS_MSR_A7X_CPUACTLR2_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 15, 1, 1) +#define ISS_MRS_MSR_CPUECTLR_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 15, 1, 4) +#define ISS_MRS_MSR_CPUPWRCTLR_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 15, 2, 7) diff --git a/hyp/arch/cortex-a-v9/src/sysreg_init.c b/hyp/arch/cortex-a-v9/src/sysreg_init.c new file mode 100644 index 0000000..d7364d2 --- /dev/null +++ b/hyp/arch/cortex-a-v9/src/sysreg_init.c @@ -0,0 +1,20 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include + +#include + +#include "event_handlers.h" + +void +arch_cortex_a_v9_handle_boot_cpu_warm_init(void) +{ + // ACTLR_EL2 controls EL1 access to some important features such as + // CLUSTERPMU and power control registers, which can be exploited to + // perform security/DoS attacks. Therefore, we deny all these accesses + // by writing 0 to this register. + ACTLR_EL2_t val = ACTLR_EL2_default(); + register_ACTLR_EL2_write(val); +} diff --git a/hyp/arch/cortex-a-v9/sysreg_init.ev b/hyp/arch/cortex-a-v9/sysreg_init.ev new file mode 100644 index 0000000..009544b --- /dev/null +++ b/hyp/arch/cortex-a-v9/sysreg_init.ev @@ -0,0 +1,8 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module arch + +subscribe boot_cpu_warm_init + handler arch_cortex_a_v9_handle_boot_cpu_warm_init diff --git a/hyp/arch/qemu-armv8-5a-rng/include/asm/cpu.h b/hyp/arch/qemu-armv8-5a-rng/include/asm/cpu.h new file mode 100644 index 0000000..f17f918 --- /dev/null +++ b/hyp/arch/qemu-armv8-5a-rng/include/asm/cpu.h @@ -0,0 +1,17 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +// Miscellaneous definitions describing the CPU implementation. + +// The size in address bits of a line in the innermost visible data cache. +#define CPU_L1D_LINE_BITS 6U + +// The size in address bits of the CPU's DC ZVA block. This is nearly always +// the same as CPU_L1D_LINE_BITS. +#define CPU_DCZVA_BITS 6U + +// The largest difference between the source and destination pointers during +// the optimised memcpy() for this CPU. This is here because it might depend +// on CPU_L1D_LINE_BITS in some implementations. +#define CPU_MEMCPY_STRIDE 256U diff --git a/hyp/core/api/aarch64/templates/c_wrapper.c.tmpl b/hyp/core/api/aarch64/templates/c_wrapper.c.tmpl index fef2f20..1b420f4 100644 --- a/hyp/core/api/aarch64/templates/c_wrapper.c.tmpl +++ b/hyp/core/api/aarch64/templates/c_wrapper.c.tmpl @@ -35,17 +35,17 @@ #def trace_out(hypcall_num, hypcall) #set trace_fmt = $hypcall.name + " ret:" - #for i, input in $hypcall.outputs[:4] + #for i, input in $hypcall.outputs[:5] #if not $input.ignore #set trace_fmt = trace_fmt + " {:#x}" #end if #end for TRACE(USER, HYPERCALL, "${trace_fmt}" #if len($hypcall.outputs) == 1 - , (register_t)_ret + , (register_t)ret_ #else - #for i, output in $hypcall.outputs[:4] - , (register_t)_ret.$register_expr($output) + #for i, output in $hypcall.outputs[:5] + , (register_t)ret_.$register_expr($output) #end for #end if ); @@ -71,7 +71,7 @@ ${type_signature($hypcall, suffix=$wrapper_suffix, ignored_inputs=True)} ${type_signature($hypcall, suffix=$wrapper_suffix, ignored_inputs=True)} { #if $hypcall.outputs - $return_type($hypcall) _ret; + $return_type($hypcall) ret_; #end if trigger_thread_entry_from_user_event(THREAD_ENTRY_REASON_HYPERCALL); @@ -97,9 +97,9 @@ ${type_signature($hypcall, suffix=$wrapper_suffix, ignored_inputs=True)} { #set has_ignores = True if (compiler_unexpected(${input.name} != (${input.ctype})${input.default}U)) { #if len($hypcall.outputs) == 1 - _ret = ERROR_ARGUMENT_INVALID; + ret_ = ERROR_ARGUMENT_INVALID; #else - _ret = ($return_type($hypcall)){ .${error_ret.name} = ERROR_ARGUMENT_INVALID }; + ret_ = ($return_type($hypcall)){ .${error_ret.name} = ERROR_ARGUMENT_INVALID }; #end if goto out; } @@ -116,7 +116,7 @@ ${type_signature($hypcall, suffix=$wrapper_suffix, ignored_inputs=True)} { ## call the implementation #if $hypcall.outputs - _ret = + ret_ = #end if $prefix${hypcall.name}(#slurp #set sep='' @@ -140,7 +140,7 @@ out: ## return the result, if any #if $hypcall.outputs - return _ret; + return ret_; #else #if has_ignores: return; diff --git a/hyp/core/api/aarch64/templates/hypcall_table.S.tmpl b/hyp/core/api/aarch64/templates/hypcall_table.S.tmpl index 44bcd16..245b627 100644 --- a/hyp/core/api/aarch64/templates/hypcall_table.S.tmpl +++ b/hyp/core/api/aarch64/templates/hypcall_table.S.tmpl @@ -10,7 +10,7 @@ \#include \#include -\#if defined(ARCH_ARM_8_5_BTI) +\#if defined(ARCH_ARM_FEAT_BTI) #define BTI_J bti j #define BTI_NOP nop #define HYPERCALL_ALIGN 0x10 diff --git a/hyp/core/api/api.tc b/hyp/core/api/api.tc index 9f12e66..04b59ef 100644 --- a/hyp/core/api/api.tc +++ b/hyp/core/api/api.tc @@ -3,7 +3,7 @@ // SPDX-License-Identifier: BSD-3-Clause extend trace_class enumeration { - USER; + USER = 2; }; extend trace_id enumeration { diff --git a/hyp/core/base/aarch64/enums.tc b/hyp/core/base/aarch64/enums.tc index 760b5b8..b04c96c 100644 --- a/hyp/core/base/aarch64/enums.tc +++ b/hyp/core/base/aarch64/enums.tc @@ -22,30 +22,59 @@ extend esr_ec enumeration { BRK = 0x3c; }; -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) extend esr_ec enumeration { PAUTH = 0x09; -#if defined(ARCH_ARM_8_3_NV) +#if defined(ARCH_ARM_FEAT_NV) ERET = 0x1a; #endif -#if defined(ARCH_ARM_8_3_FPAC) +#if defined(ARCH_ARM_FEAT_FPAC) FPAC = 0x1c; #endif }; #endif -#if defined(ARCH_ARM_8_5_BTI) +#if defined(ARCH_ARM_FEAT_BTI) extend esr_ec enumeration { BTI = 0x0d; }; #endif -#if defined(ARCH_ARM_8_2_SVE) +#if defined(ARCH_ARM_FEAT_SVE) extend esr_ec enumeration { SVE = 0x19; }; #endif +#if defined(ARCH_ARM_FEAT_LS64) +extend esr_ec enumeration { + LD64B_ST64B = 0x0a; +}; +#endif + +#if defined(ARCH_ARM_FEAT_TME) +extend esr_ec enumeration { + TSTART = 0x1b; +}; +#endif + +#if defined(ARCH_ARM_FEAT_SME) +extend esr_ec enumeration { + SME = 0x1d; +}; +#endif + +#if defined(ARCH_ARM_FEAT_RME) +extend esr_ec enumeration { + RME = 0x1e; +}; +#endif + +#if defined(ARCH_ARM_FEAT_MOPS) +extend esr_ec enumeration { + MOPS = 0x27; +}; +#endif extend iss_da_ia_fsc enumeration { TRANSLATION_0 = 0x04; @@ -59,7 +88,7 @@ extend iss_da_ia_fsc enumeration { PAGE_DOMAIN = 0x3e; }; -#if defined(ARCH_ARM_8_1_TTHM) +#if defined(ARCH_ARM_FEAT_HAFDBS) extend iss_da_ia_fsc enumeration { ATOMIC_HW_UPDATE = 0x31; }; @@ -134,7 +163,7 @@ define tcr_ps enumeration { SIZE_52BITS = 0b110; }; -#ifdef ARCH_ARM_8_4_TLBI +#ifdef ARCH_ARM_FEAT_TLBIRANGE define tlbi_range_tg enumeration { GRANULE_SIZE_4KB = 0b01; GRANULE_SIZE_16KB = 0b10; @@ -147,3 +176,10 @@ define cptr_zen enumeration { trap_all = 0x02; trap_none = 0x03; }; + +#if defined(ARCH_ARM_FEAT_WFxT) +extend iss_wfx_ti enumeration { + WFIT = 0b10; + WFET = 0b11; +}; +#endif diff --git a/hyp/core/base/aarch64/src/core_id.c b/hyp/core/base/aarch64/src/core_id.c new file mode 100644 index 0000000..5ab57b9 --- /dev/null +++ b/hyp/core/base/aarch64/src/core_id.c @@ -0,0 +1,119 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +// FIXME: +#if !defined(MODULE_PLATFORM_SOC_QCOM) +// Platforms may override this with their own implementation +core_id_t WEAK +platform_cpu_get_coreid(MIDR_EL1_t midr) +{ + (void)midr; + return CORE_ID_UNKNOWN; +} +#endif + +static core_id_t +get_core_id(uint16_t partnum, uint8_t variant, uint8_t revision) +{ + static const coreid_info_t core_id_data[] = { + { .part_num = 0xD03, .core_ID = CORE_ID_CORTEX_A53 }, + { .part_num = 0xD05, .core_ID = CORE_ID_CORTEX_A55 }, + { .part_num = 0xD07, .core_ID = CORE_ID_CORTEX_A57 }, + { .part_num = 0xD08, .core_ID = CORE_ID_CORTEX_A72 }, + { .part_num = 0xD09, .core_ID = CORE_ID_CORTEX_A73 }, + { .part_num = 0xD0A, .core_ID = CORE_ID_CORTEX_A75 }, + { .part_num = 0xD0B, .core_ID = CORE_ID_CORTEX_A76 }, + { .part_num = 0xD0C, .core_ID = CORE_ID_NEOVERSE_N1 }, + { .part_num = 0xD0D, .core_ID = CORE_ID_CORTEX_A77 }, + { .part_num = 0xD0E, .core_ID = CORE_ID_CORTEX_A76AE }, + { .part_num = 0xD40, .core_ID = CORE_ID_NEOVERSE_V1 }, + { .part_num = 0xD41, .core_ID = CORE_ID_CORTEX_A78 }, + { .part_num = 0xD42, .core_ID = CORE_ID_CORTEX_A78AE }, + { .part_num = 0xD44, .core_ID = CORE_ID_CORTEX_X1 }, + { .part_num = 0xD46, .core_ID = CORE_ID_CORTEX_A510 }, + { .part_num = 0xD47, .core_ID = CORE_ID_CORTEX_A710 }, + { .part_num = 0xD48, .core_ID = CORE_ID_CORTEX_X2 }, + { .part_num = 0xD49, .core_ID = CORE_ID_NEOVERSE_N2 }, + { .part_num = 0xD4B, .core_ID = CORE_ID_CORTEX_A78C }, + { .part_num = 0xD4D, .core_ID = CORE_ID_CORTEX_A715 }, + { .part_num = 0xD4E, .core_ID = CORE_ID_CORTEX_X3 }, + }; + + static const count_t NUM_CORE_ID = + (count_t)util_array_size(core_id_data); + core_id_t coreid = CORE_ID_UNKNOWN; + + uint32_t start; + + bool core_identified = false; + + for (start = 0; ((start < NUM_CORE_ID) && (!core_identified)); + start++) { + if (partnum == core_id_data[start].part_num) { + if ((partnum == 0xD81U) || (partnum == 0xD82U)) { + if ((variant == 0U) && (revision == 0U)) { + coreid = core_id_data[start].core_ID; + } else { + coreid = CORE_ID_UNKNOWN; + } + } else { + coreid = core_id_data[start].core_ID; + } + core_identified = true; + } + } + + return coreid; +} + +core_id_t +get_current_core_id(void) REQUIRE_PREEMPT_DISABLED +{ + core_id_t coreid; + + assert_cpulocal_safe(); + + MIDR_EL1_t midr = register_MIDR_EL1_read(); + + uint8_t implementer = MIDR_EL1_get_Implementer(&midr); + uint16_t partnum = MIDR_EL1_get_PartNum(&midr); + uint8_t variant = MIDR_EL1_get_Variant(&midr); + uint8_t revision = MIDR_EL1_get_Revision(&midr); + + if ((char)implementer == 'A') { + coreid = get_core_id(partnum, variant, revision); + } else { + coreid = CORE_ID_UNKNOWN; + } + + if (coreid == CORE_ID_UNKNOWN) { + coreid = platform_cpu_get_coreid(midr); + } + +#if defined(VERBOSE) && VERBOSE + if (coreid == CORE_ID_UNKNOWN) { + cpu_index_t cpu = cpulocal_get_index(); + + LOG(DEBUG, WARN, + "detected unknown core ID, cpu: {:d}, MIDR: {:#08x}", cpu, + MIDR_EL1_raw(midr)); + } +#endif + + return coreid; +} diff --git a/hyp/core/base/aarch64/sysregs.tc b/hyp/core/base/aarch64/sysregs.tc index b98e9be..a140996 100644 --- a/hyp/core/base/aarch64/sysregs.tc +++ b/hyp/core/base/aarch64/sysregs.tc @@ -49,7 +49,7 @@ define CCSIDR_EL1 bitfield<64> { 63:32 unknown=0; }; -#if defined(ARCH_ARM_8_3_CCIDX) +#if defined(ARCH_ARM_FEAT_CCIDX) extend CCSIDR_EL1 bitfield { delete Associativity; 23:3 Associativity uint32; @@ -80,7 +80,7 @@ define CLIDR_EL1 bitfield<64> { 63:33 unknown=0; }; -#if defined(ARCH_ARM_8_5_MEMTAG) +#if defined(ARCH_ARM_FEAT_MTE) extend CLIDR_EL1 bitfield { 34:33 Ttype1 uint8; 36:35 Ttype2 uint8; @@ -106,7 +106,7 @@ define CNTHCTL_EL2_E2H0 bitfield<64> { 63:8 unknown=0; }; -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) define CNTHCTL_EL2_E2H1 bitfield<64> { 0 EL0PCTEN bool; 1 EL0VCTEN bool; @@ -121,6 +121,28 @@ define CNTHCTL_EL2_E2H1 bitfield<64> { }; #endif +#if defined(ARCH_ARM_FEAT_ECV) +extend CNTHCTL_EL2_E2H0 bitfield { + 12 ECV bool; + 13 EL1TVT bool; + 14 EL1TVCT bool; + 15 EL1NVPCT bool; + 16 EL1NVVCT bool; + 17 EVNTIS bool; +}; + +#if defined(ARCH_ARM_FEAT_VHE) +extend CNTHCTL_EL2_E2H1 bitfield { + 12 ECV bool; + 13 EL1TVT bool; + 14 EL1TVCT bool; + 15 EL1NVPCT bool; + 16 EL1NVVCT bool; + 17 EVNTIS bool; +}; +#endif +#endif + define CNT_CTL bitfield<64> { 0 ENABLE bool; 1 IMASK bool; @@ -156,6 +178,12 @@ define CNTVCT_EL0 bitfield<64> { 63:0 CountValue uint64; }; +#if defined(ARCH_ARM_FEAT_ECV) +define CNTPOFF_EL2 bitfield<64> { + 63:0 PhysicalOffset uint64; +}; +#endif + define CNTVOFF_EL2 bitfield<64> { 63:0 VirtualOffset uint64; }; @@ -177,7 +205,7 @@ define CPACR_EL1 bitfield<64> { 63:29 unknown=0; }; -#if defined(ARCH_ARM_8_2_SVE) +#if defined(ARCH_ARM_FEAT_SVE) extend CPACR_EL1 bitfield { 17:16 ZEN uint8; }; @@ -198,7 +226,7 @@ define CPTR_EL2_E2H0 bitfield<64> { 63:32 unknown=0; }; -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) define CPTR_EL2_E2H1 bitfield<64> { 15:0 unknown=0; 17:16 unknown=0; @@ -213,19 +241,19 @@ define CPTR_EL2_E2H1 bitfield<64> { }; #endif -#if defined(ARCH_ARM_8_2_SVE) +#if defined(ARCH_ARM_FEAT_SVE) extend CPTR_EL2_E2H0 bitfield { 8 TZ bool; }; #endif -#if defined(ARCH_ARM_8_2_SVE) +#if defined(ARCH_ARM_FEAT_SVE) extend CPTR_EL2_E2H1 bitfield { 17:16 ZEN enumeration cptr_zen; }; #endif -#if defined(ARCH_ARM_8_4_AMU) || defined(ARCH_ARM_8_6_AMU) +#if defined(ARCH_ARM_FEAT_AMUv1) || defined(ARCH_ARM_FEAT_AMUv1p1) extend CPTR_EL2_E2H1 bitfield { 30 TAM bool; }; @@ -240,7 +268,7 @@ define CSSELR_EL1 bitfield<64> { 63:4 unknown=0; }; -#if defined(ARCH_ARM_8_5_MEMTAG) +#if defined(ARCH_ARM_FEAT_MTE) extend CSSELR_EL1 bitfield { 4 TnD bool; }; @@ -260,7 +288,7 @@ define CTR_EL0 bitfield<64> { 63:32 unknown=0; }; -#if defined(ARCH_ARM_8_5_MEMTAG) +#if defined(ARCH_ARM_FEAT_MTE) extend CTR_EL0 bitfield { 37:32 TminLine uint8; }; @@ -320,7 +348,7 @@ define ELR_##el bitfield<64> { \ ELR(EL1) ELR(EL2) -#if defined(ARCH_ARM_8_2_RAS) || defined(ARCH_ARM_8_4_RAS) +#if defined(ARCH_ARM_FEAT_RAS) || defined(ARCH_ARM_FEAT_RASv1p1) define ERRSELR_EL1 bitfield<64> { 15:0 SEL uint16; others unknown=0; @@ -469,54 +497,66 @@ define HCR_EL2 bitfield<64> { 63:48 unknown=0; }; -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) extend HCR_EL2 bitfield { 34 E2H bool; 35 TLOR bool; }; #endif -#if defined(ARCH_ARM_8_2_RAS) || defined(ARCH_ARM_8_4_RAS) +#if defined(ARCH_ARM_FEAT_RAS) || defined(ARCH_ARM_FEAT_RASv1p1) extend HCR_EL2 bitfield { 36 TERR bool; 37 TEA bool; }; #endif -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) extend HCR_EL2 bitfield { 40 APK bool; 41 API bool; }; #endif -#if defined(ARCH_ARM_8_3_NV) +#if defined(ARCH_ARM_FEAT_NV) extend HCR_EL2 bitfield { + 42 NV bool; + 43 NV1 bool; 44 AT bool; }; #endif -#if defined(ARCH_ARM_8_4_NV) +#if defined(ARCH_ARM_FEAT_NV2) extend HCR_EL2 bitfield { - 42 NV bool; - 43 NV1 bool; 45 NV2 bool; }; #endif -#if defined(ARCH_ARM_8_4_S2FWB) +#if defined(ARCH_ARM_FEAT_S2FWB) extend HCR_EL2 bitfield { 46 FWB bool; }; #endif -#if defined(ARCH_ARM_8_4_RAS) +#if defined(ARCH_ARM_FEAT_RASv1p1) extend HCR_EL2 bitfield { 47 FIEN bool; }; #endif -#if defined(ARCH_ARM_8_5_MEMTAG) +#if defined(ARCH_ARM_FEAT_CSV2_2) || defined(ARCH_ARM_FEAT_CSV2_1p2) || \ + defined(ARCH_ARM_FEAT_CSV2_3) +#if !defined(ARCH_ARM_FEAT_CSV2) +#error ARCH_ARM_FEAT_CSV2 not defined +#endif +define ARCH_ARM_HAVE_SCXT constant bool = 1; + +extend HCR_EL2 bitfield { + 53 EnSCXT bool; +}; +#endif + +#if defined(ARCH_ARM_FEAT_MTE) extend HCR_EL2 bitfield { 56 ATA bool; 57 DCT bool; @@ -532,14 +572,14 @@ define HPFAR_EL2 bitfield<64> { 63 unknown=0; }; -#if defined(ARCH_ARM_8_2_LPA) +#if defined(ARCH_ARM_FEAT_LPA) extend HPFAR_EL2 bitfield { delete FIPA; 43:4 FIPA uint64 lsl(12); }; #endif -#if defined(ARCH_ARM_8_4_SECEL2) +#if defined(ARCH_ARM_FEAT_SEL2) extend HPFAR_EL2 bitfield { 63 NS bool; }; @@ -594,6 +634,8 @@ define ID_AA64MMFR0_EL1 bitfield<64> { 43:40 TGran4_2 uint8; 47:44 ExS uint8; 63:48 unknown=0; + 59:56 FGT uint8; + 63:60 ECV uint8; }; define ID_AA64MMFR1_EL1 bitfield<64> { @@ -605,7 +647,14 @@ define ID_AA64MMFR1_EL1 bitfield<64> { 23:20 PAN uint8; 27:24 SpecSEI uint8; 31:28 XNX uint8; - 63:32 unknown=0; + 35:32 TWED uint8; + 39:36 ETS uint8; + 43:40 HCX uint8; + 47:44 AFP uint8; + 51:48 nTLBPA uint8; + 55:52 TIDCP1 uint8; + 59:56 CMOW uint8; + 63:60 ECBHB uint8; }; define ID_AA64MMFR2_EL1 bitfield<64> { @@ -641,7 +690,7 @@ define ID_AA64PFR0_EL1 bitfield<64> { 43:40 MPAM uint8; 47:44 AMU uint8; 51:48 DIT uint8; - 55:52 unknown=0; + 55:52 RME uint8; 59:56 CSV2 uint8; 63:60 CSV3 uint8; }; @@ -651,7 +700,12 @@ define ID_AA64PFR1_EL1 bitfield<64> { 7:4 SSBS uint8; 11:8 MTE uint8; 15:12 RAS_frac uint8; - 63:16 unknown=0; + 19:16 MPAM_frac uint8; + 27:24 SME uint8; + 31:28 RNDR_trap uint8; + 35:32 CSV2_frac uint8; + 39:36 NMI uint8; + others unknown=0; }; define ID_AA64ISAR0_EL1 bitfield<64> { @@ -691,6 +745,18 @@ define ID_AA64ISAR1_EL1 bitfield<64> { others unknown=0; }; +define ID_AA64ISAR2_EL1 bitfield<64> { + 3:0 WFxT uint8; + 7:4 RPRES uint8; + 11:8 GPA3 uint8; + 15:12 APA3 uint8; + 19:16 MOPS uint8; + 23:20 BC uint8; + 27:24 PAC_frac uint8; + 31:28 CLRBHB uint8; + 63:32 unknown=0; +}; + define ID_DFR0_EL1 bitfield<64> { 3:0 CopDbg uint8; 7:4 CopSDbg uint8; @@ -888,18 +954,20 @@ define MAIR_ATTR enumeration { DEVICE_nGRE = 0x8; DEVICE_GRE = 0xc; + DEVICE_nGnRnE_XS= 0x1; + DEVICE_nGnRE_XS = 0x5; + DEVICE_nGRE_XS = 0x9; + DEVICE_GRE_XS = 0xd; + NORMAL_NC = 0x44; // Inner+outer non-cacheable NORMAL_WB_OUTER_NC = 0x4f; // Inner writeback RW alloc -#if defined(ARCH_ARM_8_5_MEMTAG) +#if defined(ARCH_ARM_FEAT_MTE) TAGGED_NORMAL_WB = 0xf0; // Inner+outer writeback RW alloc, MT enabled #endif NORMAL_WB = 0xff; // Inner/outer writeback RW alloc // Other combinations of types not enumerated }; -// Bits valid for device types -define MAIR_ATTR_DEVICE_MASK constant enumeration MAIR_ATTR = 0xc; - // Bits that indicate RW alloc hints for normal memory types (other than // TAGGED_NORMAL_WB which is special) define MAIR_ATTR_ALLOC_HINT_MASK constant enumeration MAIR_ATTR = 0x33; @@ -939,20 +1007,20 @@ extend MDCR_EL2 bitfield { }; #endif -#if defined(ARCH_ARM_SPE) +#if defined(ARCH_ARM_FEAT_SPEv1p1) extend MDCR_EL2 bitfield { 13:12 E2PB uint8; 14 TPMS bool; }; #endif -#if defined(ARCH_ARM_8_1_PMU) || defined(ARCH_ARM_8_4_PMU) +#if defined(ARCH_ARM_FEAT_PMUv3p1) || defined(ARCH_ARM_FEAT_PMUv3p4) extend MDCR_EL2 bitfield { 17 HPMD bool; }; #endif -#if defined(ARCH_ARM_8_4_TRACE) +#if defined(ARCH_ARM_FEAT_TRF) extend MDCR_EL2 bitfield { 19 TTRF bool; }; @@ -1018,7 +1086,7 @@ define PAR_EL1_F0 bitfield<64> { 63:56 ATTR enumeration MAIR_ATTR; }; -#if defined(ARCH_ARM_8_2_LPA) +#if defined(ARCH_ARM_FEAT_LPA) extend PAR_EL1_F0 bitfield { delete PA; 51:12 PA uint64 lsl(12); @@ -1060,7 +1128,7 @@ define PMCR_EL0 bitfield<64> { 63:32 unknown=0; }; -#if defined(ARCH_ARM_8_5_PMU) +#if defined(ARCH_ARM_FEAT_PMUv3p5) extend PMCR_EL0 bitfield { 7 LP bool; }; @@ -1100,26 +1168,26 @@ define SCTLR_EL1 bitfield<64> { 63:32 unknown=0; }; -#if defined(ARCH_ARM_8_1_PAN) +#if defined(ARCH_ARM_FEAT_PAN) extend SCTLR_EL1 bitfield { 23 SPAN bool; }; #endif -#if defined(ARCH_ARM_8_2_IESB) +#if defined(ARCH_ARM_FEAT_IESB) extend SCTLR_EL1 bitfield { 21 IESB bool; }; #endif -#if defined(ARCH_ARM_8_2_LSMAOC) +#if defined(ARCH_ARM_FEAT_LSMAOC) extend SCTLR_EL1 bitfield { 28 nTLSMD bool; 29 LSMAOE bool; }; #endif -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) extend SCTLR_EL1 bitfield { 13 EnDB bool; 27 EnDA bool; @@ -1128,13 +1196,13 @@ extend SCTLR_EL1 bitfield { }; #endif -#if defined(ARCH_ARM_8_4_LSE) +#if defined(ARCH_ARM_FEAT_LSE2) extend SCTLR_EL1 bitfield { 6 nAA bool; }; #endif -#if defined(ARCH_ARM_8_5_MEMTAG) +#if defined(ARCH_ARM_FEAT_MTE) extend SCTLR_EL1 bitfield { 43 ATA bool; 42 ATA0 bool; @@ -1144,7 +1212,7 @@ extend SCTLR_EL1 bitfield { }; #endif -#if defined(ARCH_ARM_8_0_SSBS) +#if defined(ARCH_ARM_FEAT_SSBS) extend SCTLR_EL1 bitfield { 44 DSSBS bool; }; @@ -1179,7 +1247,7 @@ define SCTLR_EL2_VM bitfield<64> { 63:32 unknown=0; }; -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) define SCTLR_EL2_E2H_TGE bitfield<64> { 0 M bool; 1 A bool; @@ -1215,7 +1283,7 @@ define SCTLR_EL2_E2H_TGE bitfield<64> { }; #endif -#if defined(ARCH_ARM_8_2_IESB) +#if defined(ARCH_ARM_FEAT_IESB) extend SCTLR_EL2_VM bitfield { 21 IESB bool; }; @@ -1224,14 +1292,14 @@ extend SCTLR_EL2_E2H_TGE bitfield { }; #endif -#if defined(ARCH_ARM_8_2_LSMAOC) +#if defined(ARCH_ARM_FEAT_LSMAOC) extend SCTLR_EL2_E2H_TGE bitfield { 28 nTLSMD bool; 29 LSMAOE bool; }; #endif -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) extend SCTLR_EL2_VM bitfield { 13 EnDB bool; 27 EnDA bool; @@ -1246,7 +1314,7 @@ extend SCTLR_EL2_E2H_TGE bitfield { }; #endif -#if defined(ARCH_ARM_8_4_LSE) +#if defined(ARCH_ARM_FEAT_LSE2) extend SCTLR_EL2_VM bitfield { 6 nAA bool; }; @@ -1255,7 +1323,7 @@ extend SCTLR_EL2_E2H_TGE bitfield { }; #endif -#if defined(ARCH_ARM_8_5_MEMTAG) +#if defined(ARCH_ARM_FEAT_MTE) extend SCTLR_EL2_VM bitfield { 43 ATA bool; 42 ATA0 bool; @@ -1270,7 +1338,7 @@ extend SCTLR_EL2_E2H_TGE bitfield { }; #endif -#if defined(ARCH_ARM_8_0_SSBS) +#if defined(ARCH_ARM_FEAT_SSBS) extend SCTLR_EL2_VM bitfield { 44 DSSBS bool; }; @@ -1314,13 +1382,13 @@ define SPSR_EL2_A32 bitfield<64> { 63:32 unknown=0; }; -#if defined(ARCH_ARM_8_1_PAN) +#if defined(ARCH_ARM_FEAT_PAN) extend SPSR_EL2_A32 bitfield { 22 PAN bool; }; #endif -#if defined(ARCH_ARM_8_4_DIT) +#if defined(ARCH_ARM_FEAT_DIT) extend SPSR_EL2_A32 bitfield { 23 DIT bool; }; @@ -1350,7 +1418,7 @@ define SPSR_##el##_A64 bitfield<64> { \ SPSR_A64(EL1) SPSR_A64(EL2) -#if defined(ARCH_ARM_8_5_BTI) +#if defined(ARCH_ARM_FEAT_BTI) extend SPSR_EL1_A64 bitfield { 11:10 BTYPE uint8; }; @@ -1359,7 +1427,7 @@ extend SPSR_EL2_A64 bitfield { }; #endif -#if defined(ARCH_ARM_8_0_SSBS) +#if defined(ARCH_ARM_FEAT_SSBS) extend SPSR_EL1_A64 bitfield { 12 SSBS bool; }; @@ -1368,7 +1436,7 @@ extend SPSR_EL2_A64 bitfield { }; #endif -#if defined(ARCH_ARM_8_1_PAN) +#if defined(ARCH_ARM_FEAT_PAN) extend SPSR_EL1_A64 bitfield { 22 PAN bool; }; @@ -1377,7 +1445,7 @@ extend SPSR_EL2_A64 bitfield { }; #endif -#if defined(ARCH_ARM_8_2_UAO) +#if defined(ARCH_ARM_FEAT_UAO) extend SPSR_EL1_A64 bitfield { 23 UAO bool; }; @@ -1386,7 +1454,7 @@ extend SPSR_EL2_A64 bitfield { }; #endif -#if defined(ARCH_ARM_8_4_DIT) +#if defined(ARCH_ARM_FEAT_DIT) extend SPSR_EL1_A64 bitfield { 24 DIT bool; }; @@ -1395,7 +1463,7 @@ extend SPSR_EL2_A64 bitfield { }; #endif -#if defined(ARCH_ARM_8_5_MEMTAG) +#if defined(ARCH_ARM_FEAT_MTE) extend SPSR_EL1_A64 bitfield { 25 TCO bool; }; @@ -1404,7 +1472,19 @@ extend SPSR_EL2_A64 bitfield { }; #endif -#if defined(ARCH_ARM_8_5_MEMTAG) +define SPSR_EL2_base bitfield<64> { + 3:0 unknown=0; + 4 M4 bool; + 63:5 unknown=0; +}; + +define SPSR_EL2 union { + a32 bitfield SPSR_EL2_A32; + a64 bitfield SPSR_EL2_A64; + base bitfield SPSR_EL2_base; +}; + +#if defined(ARCH_ARM_FEAT_MTE) define TCO bitfield<64> { 24:0 unknown=0; 25 TCO bool; @@ -1440,21 +1520,21 @@ define TCR_EL1 bitfield<64> { 63:55 unknown=0; }; -#if defined(ARCH_ARM_8_1_TTHM) +#if defined(ARCH_ARM_FEAT_HAFDBS) extend TCR_EL1 bitfield { 39 HA bool; 40 HD bool; }; #endif -#if defined(ARCH_ARM_8_1_HPD) +#if defined(ARCH_ARM_FEAT_HPDS) extend TCR_EL1 bitfield { 41 HPD0 bool; 42 HPD1 bool; }; #endif -#if defined(ARCH_ARM_8_2_TTPBHA) +#if defined(ARCH_ARM_FEAT_HPDS2) extend TCR_EL1 bitfield { 43 HWU059 bool; 44 HWU060 bool; @@ -1467,21 +1547,21 @@ extend TCR_EL1 bitfield { }; #endif -#if defined(ARCH_ARM_8_2_SVE) +#if defined(ARCH_ARM_FEAT_SVE) extend TCR_EL1 bitfield { 53 NFD0 bool; 54 NFD1 bool; }; #endif -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) extend TCR_EL1 bitfield { 51 TBID0 bool; 52 TBID1 bool; }; #endif -#if defined(ARCH_ARM_8_5_MEMTAG) +#if defined(ARCH_ARM_FEAT_MTE) extend TCR_EL1 bitfield { 57 TCMA0 bool; 58 TCMA1 bool; @@ -1508,7 +1588,7 @@ define TCR_EL2_E2H0 bitfield<64> { 63:32 unknown=0; }; -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) define TCR_EL2_E2H1 bitfield<64> { 5:0 T0SZ uint8; 6 unknown=0; @@ -1533,7 +1613,7 @@ define TCR_EL2_E2H1 bitfield<64> { }; #endif -#if defined(ARCH_ARM_8_1_TTHM) +#if defined(ARCH_ARM_FEAT_HAFDBS) extend TCR_EL2_E2H0 bitfield { 21 HA bool; 22 HD bool; @@ -1544,7 +1624,7 @@ extend TCR_EL2_E2H1 bitfield { }; #endif -#if defined(ARCH_ARM_8_1_HPD) +#if defined(ARCH_ARM_FEAT_HPDS) extend TCR_EL2_E2H0 bitfield { 24 HPD bool; }; @@ -1554,13 +1634,13 @@ extend TCR_EL2_E2H1 bitfield { }; #endif -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) extend TCR_EL2_E2H1 bitfield { 37 TBI0 bool; }; #endif -#if defined(ARCH_ARM_8_2_TTPBHA) +#if defined(ARCH_ARM_FEAT_HPDS2) extend TCR_EL2_E2H0 bitfield { 25 HWU059 bool; 26 HWU060 bool; @@ -1579,14 +1659,14 @@ extend TCR_EL2_E2H1 bitfield { }; #endif -#if defined(ARCH_ARM_8_2_SVE) +#if defined(ARCH_ARM_FEAT_SVE) extend TCR_EL2_E2H1 bitfield { 53 NFD0 bool; 54 NFD1 bool; }; #endif -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) extend TCR_EL2_E2H0 bitfield { 29 TBID bool; }; @@ -1596,7 +1676,7 @@ extend TCR_EL2_E2H1 bitfield { }; #endif -#if defined(ARCH_ARM_8_5_MEMTAG) +#if defined(ARCH_ARM_FEAT_MTE) extend TCR_EL2_E2H0 bitfield { 30 TCMA bool; }; @@ -1626,7 +1706,7 @@ TTBR(1,EL1) TTBR(0,EL2) TTBR(1,EL2) -#if defined(ARCH_ARM_8_4_TRACE) +#if defined(ARCH_ARM_FEAT_TRF) define TRFCR_EL2 bitfield<64> { 0 E0HTRE bool = 0; 1 E2TRE bool = 0; @@ -1680,20 +1760,20 @@ define VTCR_EL2 bitfield<64> { 63:32 unknown=0; }; -#if defined(ARCH_ARM_8_1_VMID16) +#if defined(ARCH_ARM_FEAT_VMID16) extend VTCR_EL2 bitfield { 19 VS bool; }; #endif -#if defined(ARCH_ARM_8_1_TTHM) +#if defined(ARCH_ARM_FEAT_HAFDBS) extend VTCR_EL2 bitfield { 21 HA bool; 22 HD bool; }; #endif -#if defined(ARCH_ARM_8_2_TTPBHA) +#if defined(ARCH_ARM_FEAT_HPDS2) extend VTCR_EL2 bitfield { 25 HWU059 bool; 26 HWU060 bool; @@ -1702,7 +1782,7 @@ extend VTCR_EL2 bitfield { }; #endif -#if defined(ARCH_ARM_8_4_SECEL2) +#if defined(ARCH_ARM_FEAT_SEL2) extend VTCR_EL2 bitfield { 29 NSW bool; 30 NSA bool; @@ -1717,7 +1797,7 @@ define VTTBR_EL2 bitfield<64> { }; -#if defined(ARCH_ARM_8_1_VMID16) +#if defined(ARCH_ARM_FEAT_VMID16) extend VTTBR_EL2 bitfield { delete VMID; 63:48 VMID uint32; @@ -1728,12 +1808,21 @@ extend VTTBR_EL2 bitfield { // ESR_EL2 ISS encodings (only some ISS encodings have been added) define ESR_EL2_ISS_WFI_WFE bitfield<25> { - 0 TI bool; + 0 TI enumeration iss_wfx_ti; 19:1 unknown=0; 23:20 COND uint8; 24 CV bool; }; +#if defined(ARCH_ARM_FEAT_WFxT) +extend ESR_EL2_ISS_WFI_WFE bitfield { + delete TI; + 1:0 TI enumeration iss_wfx_ti; + 2 RV bool; + 9:5 Rn uint8; +}; +#endif + define ESR_EL2_ISS_HVC bitfield<25> { 15:0 imm16 uint16; 24:16 unknown=0; @@ -1835,10 +1924,10 @@ define ESR_EL2_ISS_LDC_STC bitfield<25> { }; #endif -#if defined(ARCH_ARM_8_4_AMU) || defined(ARCH_ARM_8_6_AMU) +#if defined(ARCH_ARM_FEAT_AMUv1) || defined(ARCH_ARM_FEAT_AMUv1p1) define AMCR_EL0 bitfield<64> { 10 HDBG bool; -#if defined(ARCH_ARM_8_6_AMU) +#if defined(ARCH_ARM_FEAT_AMUv1p1) 17 CG1RZ bool; #endif others unknown=0; @@ -1858,3 +1947,49 @@ define AMCGCR_EL0 bitfield<64> { others unknown=0; }; #endif + +#if defined(ARCH_ARM_FEAT_FGT) +define HFGWTR_EL2 bitfield<64> { + 0 AFSR0_EL1 bool; + 1 AFSR1_EL1 bool; + 3 AMAIR_EL1 bool; + 4 APDAKey bool; + 5 APDBKey bool; + 6 APGAKey bool; + 7 APIAKey bool; + 8 APIBKey bool; + 11 CONTEXTIDR_EL1 bool; + 12 CPACR_EL1 bool; + 13 CSSELR_EL1 bool; + 16 ESR_EL1 bool; + 17 FAR_EL1 bool; + 19 LORC_EL1 bool; + 20 LOREA_EL1 bool; + 22 LORN_EL1 bool; + 23 LORSA_EL1 bool; + 24 MAIR_EL1 bool; + 27 PAR_EL1 bool; + 29 SCTLR_EL1 bool; + 30 SCTXNUM_EL1 bool; + 31 SCTXNUM_EL0 bool; + 32 TCR_EL1 bool; + 33 TPIDR_EL1 bool; + 34 TPIDRRO_EL0 bool; + 35 TPIDR_EL0 bool; + 36 TTBR0_EL1 bool; + 37 TTBR1_EL1 bool; + 38 VBAR_EL1 bool; + 39 ICC_IGRPENn_EL1 bool; + 41 ERRSELR_EL1 bool; + 43 ERXCTLR_EL1 bool; + 44 ERXSTATUS_EL1 bool; + 45 ERXMISCn_EL1 bool; + 47 ERXPFGCTL_EL1 bool; + 48 ERXPFGCDN_EL1 bool; + 49 ERXADDR_EL1 bool; + 50 nACCDATA_EL1 bool; + 54 nSMPRI_EL1 bool; + 55 nTPIDR2_EL0 bool; + others unknown=0; +}; +#endif diff --git a/hyp/core/base/aarch64/types.tc b/hyp/core/base/aarch64/types.tc index 136198c..be7f31f 100644 --- a/hyp/core/base/aarch64/types.tc +++ b/hyp/core/base/aarch64/types.tc @@ -3,3 +3,11 @@ // SPDX-License-Identifier: BSD-3-Clause define paddr_t public newtype uregister; + +define PADDR_INVALID constant type paddr_t = -1; +define VADDR_INVALID constant uintptr = -1; + +define coreid_info structure { + part_num uint16; + core_ID enumeration core_id; +}; diff --git a/hyp/core/base/armv8/enums.tc b/hyp/core/base/armv8/enums.tc index afa0b90..0bc72b3 100644 --- a/hyp/core/base/armv8/enums.tc +++ b/hyp/core/base/armv8/enums.tc @@ -65,3 +65,8 @@ define spsr_32bit_mode enumeration { Undefined = 0b11011; System = 0b11111; }; + +define iss_wfx_ti enumeration { + WFI = 0b00; + WFE = 0b01; +}; diff --git a/hyp/core/base/build.conf b/hyp/core/base/build.conf index d6d3420..2189a03 100644 --- a/hyp/core/base/build.conf +++ b/hyp/core/base/build.conf @@ -4,9 +4,12 @@ interface base types types.tc +source base.c +arch_source aarch64 core_id.c arch_types armv8 enums.tc arch_types aarch64 types.tc enums.tc sysregs.tc arch_types cortex-a-v8_2 sysregs_cpu.tc +arch_types cortex-a-v9 sysregs_cpu.tc template typed hypconstants.h template typed hypcontainers.h template typed hypresult.h diff --git a/hyp/core/base/cortex-a-v9/sysregs_cpu.tc b/hyp/core/base/cortex-a-v9/sysregs_cpu.tc new file mode 100644 index 0000000..9d7ce12 --- /dev/null +++ b/hyp/core/base/cortex-a-v9/sysregs_cpu.tc @@ -0,0 +1,15 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +// This needs to be double-checked. +// FIXME: +extend ACTLR_EL2 bitfield { + 0 ACTLREN bool; + 1 ECTLREN bool; + 4 AMEN bool; + 5 ERXPFGEN bool; + 7 PWREN bool; + 11 SMEN bool; + 12 CLUSTERPMUEN bool; +}; diff --git a/hyp/core/base/src/base.c b/hyp/core/base/src/base.c new file mode 100644 index 0000000..e7fb378 --- /dev/null +++ b/hyp/core/base/src/base.c @@ -0,0 +1,13 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +// Gunyah compiler assumption sanity checks. +#if ARCH_IS_64BIT +static_assert(SIZE_MAX == UINT64_MAX, "SIZE_MAX smaller than machine width"); +#else +#error unsupported +#endif diff --git a/hyp/core/base/templates/accessors.c.tmpl b/hyp/core/base/templates/accessors.c.tmpl index 2282568..7da3d48 100644 --- a/hyp/core/base/templates/accessors.c.tmpl +++ b/hyp/core/base/templates/accessors.c.tmpl @@ -15,8 +15,10 @@ #for $definition in $definitions #if $definition.category == "bitfield" #set type_name=$definition.type_name +#set unit_type=$definition.unit_type #set unit_cnt=$definition.unit_count #set compare_masks=$definition.compare_masks +#set init_values=$definition.init_values #set all_fields_boolean=$definition.all_fields_boolean #set boolean_masks=$definition.boolean_masks #set trivial = True @@ -32,29 +34,37 @@ ${type_name}_init(${type_name}_t *bit_field) { } #if $unit_cnt == 1 -${definition.unit_type} +$unit_type ${type_name}_raw(${type_name}_t bit_field) { return bit_field.bf[0]; } -_Atomic ${definition.unit_type} * +_Atomic $unit_type * ${type_name}_atomic_ptr_raw(_Atomic ${type_name}_t *ptr) { - return (_Atomic ${definition.unit_type} *)&((${type_name}_t *)ptr)->bf[0]; + return (_Atomic $unit_type *)&((${type_name}_t *)ptr)->bf[0]; } #end if +#set $unit_size_mask = (1 << definition.unit_size) - 1 ${type_name}_t -${type_name}_clean(${type_name}_t val) { +${type_name}_clean(${type_name}_t bit_field) { #if trivial - (void)val; + (void)bit_field; #end if return (${type_name}_t) { .bf = { #slurp #for i in range($unit_cnt) #if $compare_masks[$i] != 0 #set mask = hex($compare_masks[$i]) - val.bf[$i] & ${mask}U, +#if $init_values[$i] != 0 +#set init = hex($init_values[$i]) +## Keep any default values, only for `unknown = XX` fields + // (${init}U & ~${mask}U) | + ($unit_type)(${hex($init_values[$i] & ($compare_masks[$i] ^ unit_size_mask))}U) | +#end if + (bit_field.bf[$i] & ${mask}U), #else - 0, +#set init = hex($init_values[$i]) + ${init}, #end if #end for } }; @@ -90,6 +100,41 @@ ${type_name}_is_equal(${type_name}_t b1, ${type_name}_t b2) { #set bool_trivial = False #end if #end for + +#if $all_fields_boolean +bool +${type_name}_is_empty(${type_name}_t bit_field) +{ +return#slurp +#set $first = True +#for $unit in range($unit_cnt) +#if not $first + &&#slurp +#end if +#set $first = False + ((bit_field.bf[$unit] & ${hex($boolean_masks[$unit])}U) == 0U)#slurp +#end for +; +} +#end if + +#if $all_fields_boolean +bool +${type_name}_is_clean(${type_name}_t bit_field) +{ +return#slurp +#set $first = True +#for $unit in range($unit_cnt) +#if not $first + &&#slurp +#end if +#set $first = False + ((bit_field.bf[$unit] & ${hex($boolean_masks[$unit] ^ $unit_size_mask)}U) == ${hex($init_values[$i] & ~$compare_masks[$i])}U)#slurp +#end for +; +} +#end if + ${type_name}_t ${type_name}_union(${type_name}_t b1, ${type_name}_t b2) { @@ -122,7 +167,7 @@ ${type_name}_intersection(${type_name}_t b1, ${type_name}_t b2) b1.bf[$i] & b2.bf[$i], #elif $boolean_masks[$i] != 0 #set mask = hex($boolean_masks[$i]) - b1.bf[$i] & (b2.bf[$i] | ~($definition.unit_type)${mask}U), + b1.bf[$i] & (b2.bf[$i] | ~($unit_type)${mask}U), #else b1.bf[$i], #end if @@ -157,7 +202,7 @@ ${type_name}_difference(${type_name}_t b1, ${type_name}_t b2) ${type_name}_t ${type_name}_atomic_union(_Atomic ${type_name}_t *b1, ${type_name}_t b2, memory_order order) { - _Atomic ${definition.unit_type} *bf = (_Atomic ${definition.unit_type} *) & ((${type_name}_t *) b1)->bf[0]; + _Atomic $unit_type *bf = (_Atomic $unit_type *) & ((${type_name}_t *) b1)->bf[0]; return (${type_name}_t){ .bf = { #slurp #if $all_fields_boolean atomic_fetch_or_explicit(bf, b2.bf[0], order) @@ -170,12 +215,12 @@ ${type_name}_atomic_union(_Atomic ${type_name}_t *b1, ${type_name}_t b2, memory_ ${type_name}_t ${type_name}_atomic_intersection(_Atomic ${type_name}_t *b1, ${type_name}_t b2, memory_order order) { - _Atomic ${definition.unit_type} *bf = (_Atomic ${definition.unit_type} *) & ((${type_name}_t *) b1)->bf[0]; + _Atomic $unit_type *bf = (_Atomic $unit_type *) & ((${type_name}_t *) b1)->bf[0]; return (${type_name}_t){ .bf = { #slurp #if $all_fields_boolean atomic_fetch_and_explicit(bf, b2.bf[0], order) #else - atomic_fetch_and_explicit(bf, b2.bf[0] | ~($definition.unit_type)${mask}U, order) + atomic_fetch_and_explicit(bf, b2.bf[0] | ~($unit_type)${mask}U, order) #end if } }; } @@ -190,32 +235,46 @@ ${type_name}_atomic_difference(_Atomic ${type_name}_t *b1, ${type_name}_t b2, me #end if #for $dec in $definition._all_declarations: #if not $dec.is_ignore -#set field_type = $dec.compound_type.gen_type_name(unqualified=True) +#set field_type = $dec.compound_type +#set field_type_type_name = $dec.compound_type.gen_type_name(unqualified=True) #if $dec.is_nested_bitfield -#set field_type_name = $dec.compound_type.type_name -#set field_unit_type = $dec.compound_type.definition.unit_type -#set val_expr = $field_type_name + '_raw(val)' +#set field_type_name = $field_type.type_name +#set field_unit_type = $field_type.definition.unit_type +#set val_expr = '(' + $dec.unit_type + ')' + $field_type_name + '_raw(val)' #else -#set val_expr = 'val' +#set val_expr = '(' + $dec.unit_type + ')val' #end if #if not $dec.is_const: -void ${dec.bf_type_name}_set_${dec.field_name}(${dec.bf_type_name}_t *bit_field, ${dec.compound_type.gen_declaration('val')}) { - ${dec.unit_type} *bf = (${dec.unit_type} *)bit_field; +void ${dec.bf_type_name}_set_${dec.field_name}(${dec.bf_type_name}_t *bit_field, ${field_type.gen_declaration('val')}) { +## Handle MISRA Boolean type casting +#if not $dec.is_nested_bitfield +#if ($field_type.basic_type.bitsize == 1) and ($field_type.basic_type.category == 'primitive') + ${dec.unit_type} bool_val = val ? (${dec.unit_type})1 : (${dec.unit_type})0; +#set val_expr = 'bool_val' +#end if +#end if + ${dec.unit_type} *bf = &bit_field->bf[0]; #for $map in $dec.field_maps: #set $m = (1 << $map.length) - 1 #set unit = $map.mapped_bit // $dec.unit_size #set mapped_bit = $map.mapped_bit % $dec.unit_size bf[$unit] &= (${dec.unit_type})${hex(((2 ** dec.unit_size) - 1) ^ (m << $mapped_bit))}U; - bf[$unit] |= ((((${dec.unit_type})${val_expr}) >> ${map.field_bit}U) & (${dec.unit_type})${hex(m)}U) << ${mapped_bit}U; + bf[$unit] |= ((${val_expr} >> ${map.field_bit}U) & (${dec.unit_type})${hex(m)}U) << ${mapped_bit}U; #end for } #end if #if not $dec.is_writeonly: -${field_type} +${field_type_type_name} ${dec.bf_type_name}_get_${dec.field_name}(const ${dec.bf_type_name}_t *bit_field) { - ${dec.unit_type} val = 0U; + ${dec.unit_type} val = 0; const ${dec.unit_type} *bf = (const ${dec.unit_type} *)&bit_field->bf[0]; +#set $bool_type = False +#if not $dec.is_nested_bitfield +#if ($field_type.basic_type.bitsize == 1) and ($field_type.basic_type.category == 'primitive') +#set $bool_type = True +#end if +#end if #for $map in $dec.field_maps: #set $m = (1 << $map.length) - 1 @@ -225,10 +284,16 @@ ${dec.bf_type_name}_get_${dec.field_name}(const ${dec.bf_type_name}_t *bit_field #end for #if $dec.field_signed #set l=hex(1 << ($dec.field_length - 1)) + 'U' - return (${field_type})((val ^ $l) - $l); + return (${field_type_type_name})((val ^ $l) - $l); #else #if not $dec.is_nested_bitfield - return (${field_type})val; +#if $bool_type + return val != (${dec.unit_type})0; +#else if $field_type.is_pointer + return (${field_type_type_name})(uintptr_t)val; +#else + return (${field_type_type_name})val; +#end if #else return ${field_type_name}_cast((${field_unit_type})val); #end if diff --git a/hyp/core/base/templates/hypresult.c.tmpl b/hyp/core/base/templates/hypresult.c.tmpl index 00bee88..97b91ef 100644 --- a/hyp/core/base/templates/hypresult.c.tmpl +++ b/hyp/core/base/templates/hypresult.c.tmpl @@ -74,7 +74,7 @@ ${name}_ptr_result_ok(${type_name} * ret) #continue #end if #end if -#if $category in ['enumeration', 'primitive', 'bitfield', 'structure'] +#if $category in ['enumeration', 'primitive', 'bitfield', 'structure'] and $d.size != 0 $declare_result(d.indicator, type_name) $declare_result_ptr(d.indicator, type_name) #else if $category in ['union', 'object'] diff --git a/hyp/core/base/templates/hypresult.h.tmpl b/hyp/core/base/templates/hypresult.h.tmpl index 8e6c9b7..48b965d 100644 --- a/hyp/core/base/templates/hypresult.h.tmpl +++ b/hyp/core/base/templates/hypresult.h.tmpl @@ -60,7 +60,7 @@ ${name}_ptr_result_ok(${type_name} * ret); #set category = $basic_type.category #end if #end if -#if $category in ['enumeration', 'primitive', 'bitfield', 'structure'] +#if $category in ['enumeration', 'primitive', 'bitfield', 'structure'] and $d.size != 0 $declare_result(d.indicator, type_name) $declare_result_ptr(d.indicator, type_name) #else if $category in ['union', 'object'] diff --git a/hyp/core/base/types.tc b/hyp/core/base/types.tc index 2a391c7..b648672 100644 --- a/hyp/core/base/types.tc +++ b/hyp/core/base/types.tc @@ -52,6 +52,9 @@ define sregister_t public newtype sregister; define count_t public newtype uint32; define index_t public newtype uint32; +define COUNT_INVALID constant type count_t = -1; +define INDEX_INVALID constant type index_t = -1; + // Dummy lockable structure used for thread safety analysis of public lock // APIs that lock static variables, without having to expose the static // variable. These are always declared extern; they don't need to be defined. diff --git a/hyp/core/boot/aarch64/include/arch/reloc.h b/hyp/core/boot/aarch64/include/arch/reloc.h index 142b63d..c152919 100644 --- a/hyp/core/boot/aarch64/include/arch/reloc.h +++ b/hyp/core/boot/aarch64/include/arch/reloc.h @@ -6,4 +6,4 @@ (((R_TYPE(r_info) == R_AARCH64_NONE) || \ (R_TYPE(r_info) == R_AARCH64_NULL) || \ (R_TYPE(r_info) == R_AARCH64_RELATIVE)) && \ - (R_SYM(r_info) == 0)) + (R_SYM(r_info) == 0U)) diff --git a/hyp/core/boot/aarch64/src/aarch64_boot.c b/hyp/core/boot/aarch64/src/aarch64_boot.c index 6060cd1..3dfd689 100644 --- a/hyp/core/boot/aarch64/src/aarch64_boot.c +++ b/hyp/core/boot/aarch64/src/aarch64_boot.c @@ -13,7 +13,7 @@ void aarch64_handle_boot_runtime_init(void) { -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) CPTR_EL2_E2H1_t cptr = CPTR_EL2_E2H1_default(); register_CPTR_EL2_E2H1_write_ordered(cptr, &asm_ordering); #else diff --git a/hyp/core/boot/aarch64/src/init_el2.S b/hyp/core/boot/aarch64/src/init_el2.S index 83f608a..a2d8028 100644 --- a/hyp/core/boot/aarch64/src/init_el2.S +++ b/hyp/core/boot/aarch64/src/init_el2.S @@ -11,10 +11,10 @@ // This stack is used by boot_cold_init. It must be large enough to run // all the first-boot event triggers. This memory is then reclaimed by // the hypervisor_partition. - _bss aarch64_boot_stack, BOOT_STACK_SIZE, THREAD_STACK_MAP_ALIGN + _bss aarch64_boot_stack, BOOT_STACK_SIZE, 16 .space BOOT_STACK_SIZE -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) // Pointer authentication keys shared by all EL2 threads. _bss aarch64_pauth_keys, AARCH64_PAUTH_KEYS_SIZE, \ AARCH64_PAUTH_KEYS_ALIGN @@ -45,7 +45,8 @@ // x0-x3: 256-bit RNG seed from the platform code // w4: Logical CPU number from the platform code -function aarch64_init + .section .text.boot.init +function aarch64_init, section=nosection // Disable debug, aborts and interrupts. msr DAIFSet, 0xf @@ -109,7 +110,7 @@ function aarch64_init cmp x8, x22 beq aarch64_boot_error -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) // Generate the PAC keys adrl x21, aarch64_pauth_keys mov x23, AARCH64_PAUTH_KEYS_SIZE @@ -145,7 +146,7 @@ function_end aarch64_init // x0: Virtual pointer to the CPU's idle thread // w1: Logical CPU number from the platform code -function aarch64_secondary_init +function aarch64_secondary_init, section=nosection // Disable debug, aborts and interrupts. msr DAIFSet, 0xf @@ -195,7 +196,7 @@ function aarch64_secondary_init cmp x8, x22 beq aarch64_boot_error -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) // Load the PAC keys into registers kernel_pauth_entry x0, x1, x2 @@ -222,7 +223,7 @@ function_end aarch64_secondary_init // Input arguments: // x0: Virtual pointer to the CPU's idle thread. -function aarch64_warm_init +function aarch64_warm_init, section=nosection // Disable debug, aborts and interrupts. msr DAIFSet, 0xf @@ -266,7 +267,7 @@ function aarch64_warm_init cmp x8, x22 beq aarch64_boot_error -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) // Load the PAC keys into registers kernel_pauth_entry x0, x1, x2 @@ -284,7 +285,7 @@ function aarch64_warm_init b boot_warm_init function_end aarch64_warm_init -function aarch64_boot_error +function aarch64_boot_error, section=nosection 9: wfi b 9b diff --git a/hyp/core/boot/aarch64/src/init_el2_mmu.S b/hyp/core/boot/aarch64/src/init_el2_mmu.S index 1b85452..0925c63 100644 --- a/hyp/core/boot/aarch64/src/init_el2_mmu.S +++ b/hyp/core/boot/aarch64/src/init_el2_mmu.S @@ -9,9 +9,9 @@ bss64 aarch64_kaslr_base, local bss64 hypervisor_prng_nonce - .section .text + .section .text.boot.init -function aarch64_init_generate_seed, local +function aarch64_init_generate_seed, local, section=nosection // KASLR - Seed is passed in x0-x3, however, for portability it may not // be random. Mix in some values from registers that are UNKNOWN after // reset: TPIDR_EL2, TTBR0_EL2, MAIR_EL2, VBAR_EL2 and FAR_EL2. @@ -93,10 +93,14 @@ function_end aarch64_init_generate_seed #define VSMAv8_AF_SHIFT (LOWER_ATTRS + VMSA_STG1_LOWER_ATTRS_AF_SHIFT) #define VSMAv8_NG_SHIFT (LOWER_ATTRS + VMSA_STG1_LOWER_ATTRS_NG_SHIFT) +#if defined(ARCH_ARM_FEAT_VHE) #define VSMAv8_UXN_SHIFT (UPPER_ATTRS + VMSA_STG1_UPPER_ATTRS_UXN_SHIFT) #define VSMAv8_PXN_SHIFT (UPPER_ATTRS + VMSA_STG1_UPPER_ATTRS_PXN_SHIFT) +#else +#define VSMAv8_XN_SHIFT (UPPER_ATTRS + VMSA_STG1_UPPER_ATTRS_XN_SHIFT) +#endif -#if defined(ARCH_ARM_8_5_BTI) +#if defined(ARCH_ARM_FEAT_BTI) #define VSMAv8_GP_SHIFT (UPPER_ATTRS + VMSA_STG1_UPPER_ATTRS_GP_SHIFT) #endif @@ -128,41 +132,45 @@ function_end aarch64_init_generate_seed (1 << VSMAv8_PXN_SHIFT) // (DBM << 51) | (CONTIG << 52) | (PXN << 53) | (UXN << 54) -#if defined(ARCH_ARM_8_5_BTI) +#if defined(ARCH_ARM_FEAT_BTI) #define GUARDED_PAGE (1 << VSMAv8_GP_SHIFT) #endif -// FIXME: move these to type system -#define TCR_RGN_NORMAL_NC ENUM_TCR_RGN_NORMAL_NONCACHEABLE -#define TCR_RGN_NORMAL_WB_RA_WA ENUM_TCR_RGN_NORMAL_WRITEBACK_RA_WA -#define TCR_RGN_NORMAL_WT_RA_NWA ENUM_TCR_RGN_NORMAL_WRITETHROUGH_RA -#define TCR_RGN_NORMAL_WB_RA_NWA ENUM_TCR_RGN_NORMAL_WRITEBACK_RA -#define TCR_SH_NONE ENUM_TCR_SH_NON_SHAREABLE -#define TCR_SH_OUTER ENUM_TCR_SH_OUTER_SHAREABLE -#define TCR_SH_INNER ENUM_TCR_SH_INNER_SHAREABLE -#define TCR_TG0_4KB ENUM_TCR_TG0_GRANULE_SIZE_4KB -#define TCR_TG1_4KB ENUM_TCR_TG1_GRANULE_SIZE_4KB -#define TCR_IPS_32BIT ENUM_TCR_PS_SIZE_32BITS -#define TCR_IPS_36BIT ENUM_TCR_PS_SIZE_36BITS -#define TCR_IPS_40BIT ENUM_TCR_PS_SIZE_40BITS -#define TCR_IPS_42BIT ENUM_TCR_PS_SIZE_42BITS -#define TCR_IPS_44BIT ENUM_TCR_PS_SIZE_44BITS -#define TCR_IPS_48BIT ENUM_TCR_PS_SIZE_48BITS -#define TCR_IPS_52BIT ENUM_TCR_PS_SIZE_52BITS +#define TCR_RGN ENUM_TCR_RGN_NORMAL_WRITEBACK_RA_WA +#define TCR_SH ENUM_TCR_SH_INNER_SHAREABLE +#define TCR_TG0 ENUM_TCR_TG0_GRANULE_SIZE_4KB +#define TCR_TG1 ENUM_TCR_TG1_GRANULE_SIZE_4KB +// Note: the nested macros are are needed to expand the config to a number +// before it is pasted into the enum macro name. +#define TCR_PS TCR_PS_FOR_CONFIG(PLATFORM_PHYS_ADDRESS_BITS) +#define TCR_PS_FOR_CONFIG(x) TCR_PS_FOR_SIZE(x) +#define TCR_PS_FOR_SIZE(x) ENUM_TCR_PS_SIZE_##x##BITS + +#if defined(ARCH_ARM_FEAT_VHE) #define TCR_EL2_HYP ( \ TCR_EL2_E2H1_EPD0_MASK | \ (HYP_T1SZ << TCR_EL2_E2H1_T1SZ_SHIFT) | \ (0 << TCR_EL2_E2H1_A1_SHIFT) | \ - (TCR_RGN_NORMAL_WB_RA_WA << TCR_EL2_E2H1_IRGN1_SHIFT) | \ - (TCR_RGN_NORMAL_WB_RA_WA << TCR_EL2_E2H1_ORGN1_SHIFT) | \ - (TCR_SH_INNER << TCR_EL2_E2H1_SH1_SHIFT) | \ - (TCR_TG1_4KB << TCR_EL2_E2H1_TG1_SHIFT) | \ - (TCR_IPS_40BIT << TCR_EL2_E2H1_IPS_SHIFT)) + (TCR_RGN << TCR_EL2_E2H1_IRGN1_SHIFT) | \ + (TCR_RGN << TCR_EL2_E2H1_ORGN1_SHIFT) | \ + (TCR_SH << TCR_EL2_E2H1_SH1_SHIFT) | \ + (TCR_TG1 << TCR_EL2_E2H1_TG1_SHIFT) | \ + (TCR_PS << TCR_EL2_E2H1_IPS_SHIFT)) +#else +#define TCR_EL2_HYP ( \ + (HYP_T0SZ << TCR_EL2_E2H0_T0SZ_SHIFT) | \ + (TCR_RGN << TCR_EL2_E2H0_IRGN0_SHIFT) | \ + (TCR_RGN << TCR_EL2_E2H0_ORGN0_SHIFT) | \ + (TCR_SH << TCR_EL2_E2H0_SH0_SHIFT) | \ + (TCR_TG0 << TCR_EL2_E2H0_TG0_SHIFT) | \ + (TCR_PS << TCR_EL2_E2H0_PS_SHIFT)) +#endif #if defined(__ARM_FEATURE_UNALIGNED) #define SCTLR_EL2_VM_HYP_DEBUG 0 #else +// FIXME: //#define SCTLR_EL2_VM_HYP_DEBUG SCTLR_EL2_VM_A_MASK #define SCTLR_EL2_VM_HYP_DEBUG 0 #endif @@ -174,6 +182,8 @@ function_end aarch64_init_generate_seed SCTLR_EL2_VM_I_MASK | \ SCTLR_EL2_VM_WXN_MASK) +#if defined(ARCH_ARM_FEAT_VHE) +// Enable EL2 E2H (hosted hypervisor) mode #define HCR_EL2_HYP ( \ HCR_EL2_FMO_MASK | HCR_EL2_IMO_MASK | HCR_EL2_AMO_MASK | \ HCR_EL2_TGE_MASK | HCR_EL2_E2H_MASK) @@ -185,6 +195,19 @@ function_end aarch64_init_generate_seed #define HYP_LEVEL1_ALIGN (37 - HYP_T1SZ) #define HYP_LEVEL1_ENTRIES (1U << (HYP_ASPACE_HIGH_BITS - 30)) +#else +#define HCR_EL2_HYP ( \ + HCR_EL2_FMO_MASK | HCR_EL2_IMO_MASK | HCR_EL2_AMO_MASK | \ + HCR_EL2_TGE_MASK) + +// Kernel main pagetable in TTBR0_EL2 +// 39 bit address space (3-level) for KASLR +// HYP_T0SZ = 64 - 39 = 25 +#define HYP_T0SZ (64 - HYP_ASPACE_LOW_BITS) +#define HYP_LEVEL1_ALIGN (40 - HYP_T0SZ) +#define HYP_LEVEL1_ENTRIES (1U << (HYP_ASPACE_LOW_BITS - 30)) +#endif + #define BITS_2MiB 21 #define BITS_1GiB 30 #define SIZE_2MiB (1 << BITS_2MiB) @@ -203,17 +226,20 @@ function_end aarch64_init_generate_seed // x19-x28: preserved as per AAPCS64 // Returns: // x0: KASLR base -function aarch64_init_kaslr +function aarch64_init_kaslr, section=nosection stp x29, x30, [sp, -16]! mov x29, sp bl aarch64_init_generate_seed - // To test pathological seed, uncomment below - //mov x0, 0 + // To test pathological seed, define this below +#if defined(DISABLE_KASLR) && DISABLE_KASLR + mov x0, 0 +#endif - // - calculate KASLR virtual base address + // Calculate KASLR virtual base address local mm_calc_virt_base: +#if defined(ARCH_ARM_FEAT_VHE) mov x10, 0xffffffffffe00000 ubfiz x9, x0, BITS_2MiB, (HYP_ASPACE_HIGH_BITS - BITS_2MiB) eor x14, x10, x9 @@ -226,10 +252,33 @@ local mm_calc_virt_base: adds x10, x10, x14 bcc 1f - // -- virt address range either wraps, or ends at -1; try again + // -- virtual address wraps or ends at -1, try again lsr x0, x0, 18 movk x0, 0x5555, lsl 48 b LOCAL(mm_calc_virt_base) +#else + // - KASLR base needs to be in the lower half of the the address space, + // because the upper half is reserved for physaccess + ubfiz x9, x0, BITS_2MiB, (HYP_ASPACE_LOWER_HALF_BITS - BITS_2MiB) + // Ensure the KASLR base is outside of the 1:1 area + mov x10, (1 << HYP_ASPACE_MAP_DIRECT_BITS) + orr x14, x10, x9 + + // - ensure that the hypervisor image doesn't go out of the lower half of + // the address space + adr x10, image_virt_start + adrl x11, image_virt_last + sub x10, x11, x10 + add x10, x10, x14 + mov x9, (1 << HYP_ASPACE_LOWER_HALF_BITS) + cmp x9, x10 + bhi 1f + + // -- virtual address overlaps the upper half, try again + lsr x0, x0, (HYP_ASPACE_LOWER_HALF_BITS - BITS_2MiB) + movk x0, 0x5555, lsl 32 + b LOCAL(mm_calc_virt_base) +#endif 1: adrp x9, aarch64_kaslr_base @@ -246,14 +295,13 @@ function_end aarch64_init_kaslr // pointer may be invalid and should not be accessed. // // Input / preserve registers: -// w2: bool cold_boot +// w2: bool first_boot // x30: Physical address of return (to be translated to virtual address) // x19-x28: preserved as per AAPCS64 -function aarch64_init_address_space +function aarch64_init_address_space, section=nosection // Ensure TLB-EL2 is clean, dsb and isb deferred until before mmu enable tlbi alle2 - // Enable EL2 E2H (hosted hypervisor) mode abs64 x9, HCR_EL2_HYP msr HCR_EL2, x9 isb @@ -268,11 +316,17 @@ function aarch64_init_address_space abs64 x9, MAIR_DEFAULTS msr MAIR_EL2, x9 - // - setup TTBR1_EL2 (hypervisor code / data) - adrl x12, aarch64_pt_ttbr1_level1 // get physical addresses for el2 pt + // - setup TTBRx_EL2 (hypervisor code / data) + adrl x12, aarch64_pt_ttbr_level1 // physical addresses of EL2 PT +#if defined(ARCH_ARM_FEAT_VHE) orr x9, x12, TTBR1_EL2_CNP_MASK // -- preserve x12 for use below msr TTBR1_EL2, x9 +#else + orr x9, x12, TTBR0_EL2_CNP_MASK + // -- preserve x12 for use below + msr TTBR0_EL2, x9 +#endif isb // Load the KASLR base address @@ -289,7 +343,7 @@ function aarch64_init_address_space add x9, x9, x14 // - set the MMU-init vectors to: vectors_boot_aarch64 - // -- on enabling the mmu, the resulting Data Abort will jump there + // -- on enabling the MMU, the resulting prefetch abort will jump there msr VBAR_EL2, x9 // Skip the PT setup if this is not the first boot @@ -297,9 +351,13 @@ function aarch64_init_address_space // === Setup the TTBR1_EL2 mappings for the hypervisor // - create hypervisor text/ro 2MB mapping - adrl x13, aarch64_pt_ttbr1_level2 + adrl x13, aarch64_pt_ttbr_level2 // -- calculate 2MB table entries - ubfx x10, x9, VSMAv8_ADDRESS_BITS_LEVEL1, HYP_ASPACE_HIGH_BITS-VSMAv8_ADDRESS_BITS_LEVEL1 +#if defined(ARCH_ARM_FEAT_VHE) + ubfx x10, x9, VSMAv8_ADDRESS_BITS_LEVEL1, HYP_ASPACE_HIGH_BITS - VSMAv8_ADDRESS_BITS_LEVEL1 +#else + ubfx x10, x9, VSMAv8_ADDRESS_BITS_LEVEL1, HYP_ASPACE_LOW_BITS - VSMAv8_ADDRESS_BITS_LEVEL1 +#endif add x10, x12, x10, lsl VSMAv8_ENTRY_BITS // x10 points to Level-1 table entry ubfx x11, x9, VSMAv8_ADDRESS_BITS_LEVEL2, VSMAv8_LEVEL_BITS add x11, x13, x11, lsl VSMAv8_ENTRY_BITS // x11 points to Level-2 table entry @@ -314,7 +372,7 @@ function aarch64_init_address_space bic x12, x12, MASK_2MiB // orr x9, x12, VSMAv8_BLOCK_TYPE movz x10, LOWER_ATTRIBUTES_HYP_R -#if defined(ARCH_ARM_8_5_BTI) +#if defined(ARCH_ARM_FEAT_BTI) movk x10, ((UPPER_ATTRIBUTES_HYP_X | GUARDED_PAGE) >> 48), LSL 48 #else movk x10, (UPPER_ATTRIBUTES_HYP_X >> 48), LSL 48 @@ -336,17 +394,25 @@ function aarch64_init_address_space // -- RW section falls on the next 1GiB page, so we need a new table // entry and use the second level2 page-table - adrl x12, aarch64_pt_ttbr1_level1 + adrl x12, aarch64_pt_ttbr_level1 add x13, x13, (1 << (VSMAv8_LEVEL_BITS + VSMAv8_ENTRY_BITS)) // -- Update entry count for first level 1 entry - ubfx x10, x14, VSMAv8_ADDRESS_BITS_LEVEL1, HYP_ASPACE_HIGH_BITS-VSMAv8_ADDRESS_BITS_LEVEL1 +#if defined(ARCH_ARM_FEAT_VHE) + ubfx x10, x14, VSMAv8_ADDRESS_BITS_LEVEL1, HYP_ASPACE_HIGH_BITS - VSMAv8_ADDRESS_BITS_LEVEL1 +#else + ubfx x10, x14, VSMAv8_ADDRESS_BITS_LEVEL1, HYP_ASPACE_LOW_BITS - VSMAv8_ADDRESS_BITS_LEVEL1 +#endif add x10, x12, x10, lsl VSMAv8_ENTRY_BITS // x10 points to Level-1 table entry ldr x11, [x10] sub x11, x11, (1 << VSMAv8_TABLE_REFCOUNT_SHIFT) str x11, [x10] // -- calculate 2MB hyp table entries - ubfx x10, x15, VSMAv8_ADDRESS_BITS_LEVEL1, HYP_ASPACE_HIGH_BITS-VSMAv8_ADDRESS_BITS_LEVEL1 +#if defined(ARCH_ARM_FEAT_VHE) + ubfx x10, x15, VSMAv8_ADDRESS_BITS_LEVEL1, HYP_ASPACE_HIGH_BITS - VSMAv8_ADDRESS_BITS_LEVEL1 +#else + ubfx x10, x15, VSMAv8_ADDRESS_BITS_LEVEL1, HYP_ASPACE_LOW_BITS - VSMAv8_ADDRESS_BITS_LEVEL1 +#endif add x10, x12, x10, lsl VSMAv8_ENTRY_BITS // x10 points to Level-1 table entry ubfx x11, x15, VSMAv8_ADDRESS_BITS_LEVEL2, VSMAv8_LEVEL_BITS add x11, x13, x11, lsl VSMAv8_ENTRY_BITS // x11 points to Level-2 table entry @@ -360,7 +426,9 @@ local mm_init_rw_level2: add x11, x13, x11, lsl VSMAv8_ENTRY_BITS // x11 points to Level-2 table entry orr x9, x9, VSMAv8_BLOCK_TYPE movz x10, LOWER_ATTRIBUTES_HYP_RW +#if defined(ARCH_ARM_FEAT_VHE) movk x10, (UPPER_ATTRIBUTES_HYP_NX >> 48), LSL 48 +#endif orr x9, x9, x10 str x9, [x11] @@ -396,17 +464,22 @@ vector_aarch64_\name\(): .endm - .section .text.boot.vectors + .section .text.boot_vectors // Alignment for Boot Vectors .balign 2048 // Vectors for booting EL2 + // w2 = bool first_boot // x30 = virtual return address vectors_boot_aarch64: boot_vector self_sync_mmu_init 0x0 - adr x9, vectors_aarch64 + // For the first boot we must use the emergency vectors + adr x9, kernel_vectors_aarch64 + adr x10, emergency_vectors_aarch64 + cmp w2, 0 + csel x9, x9, x10, eq msr VBAR_EL2, x9 isb msr DAIFclr, 4 @@ -417,8 +490,15 @@ boot_vector self_sync_mmu_init 0x0 .section .bss.hyp_pt.level1, "aw", @nobits .p2align HYP_LEVEL1_ALIGN + .global aarch64_pt_ttbr_level1 +aarch64_pt_ttbr_level1: +#if defined(ARCH_ARM_FEAT_VHE) .global aarch64_pt_ttbr1_level1 aarch64_pt_ttbr1_level1: +#else + .global aarch64_pt_ttbr0_level1 +aarch64_pt_ttbr0_level1: +#endif .space HYP_LEVEL1_ENTRIES * 8 // We have two 4KB level2 tables here, to handle the case where the @@ -426,5 +506,5 @@ aarch64_pt_ttbr1_level1: // level 1 entries). .section .bss.hyp.pt.level2, "aw", @nobits .balign 4096 -aarch64_pt_ttbr1_level2: +aarch64_pt_ttbr_level2: .space (1 << (VSMAv8_LEVEL_BITS + VSMAv8_ENTRY_BITS)) * 2 diff --git a/hyp/core/boot/boot.tc b/hyp/core/boot/boot.tc index 64857ea..569cd57 100644 --- a/hyp/core/boot/boot.tc +++ b/hyp/core/boot/boot.tc @@ -9,7 +9,5 @@ define boot_env_phys_range public structure { size size; }; -define boot_env_data public structure { - free_ranges array(BOOT_ENV_RANGES_NUM) structure boot_env_phys_range; - free_ranges_count type count_t; +define hyp_env_data structure { }; diff --git a/hyp/core/boot/include/boot_init.h b/hyp/core/boot/include/boot_init.h index 8dba29a..a09d593 100644 --- a/hyp/core/boot/include/boot_init.h +++ b/hyp/core/boot/include/boot_init.h @@ -21,6 +21,7 @@ boot_secondary_init(cpu_index_t cpu); noreturn void boot_warm_init(void); -// Add address range to free ranges in env_data +// Add address range to free ranges in env data stream error_t -boot_add_free_range(paddr_t base, size_t size, void *arg); +boot_add_free_range(uintptr_t object, memdb_type_t type, + qcbor_enc_ctxt_t *qcbor_enc_ctxt); diff --git a/hyp/core/boot/src/boot.c b/hyp/core/boot/src/boot.c index d657cad..3128e4c 100644 --- a/hyp/core/boot/src/boot.c +++ b/hyp/core/boot/src/boot.c @@ -10,7 +10,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -47,15 +49,17 @@ boot_cold_init(cpu_index_t cpu) LOCK_IMPL assert(guard_r.e == OK); __stack_chk_guard = (uintptr_t)guard_r.r; - // We can't trace yet because the CPU index in the thread is possibly - // wrong (if cpu is nonzero), but the cpulocal handler for - // boot_cpu_cold_init will do it for us. - LOG(ERROR, WARN, "Hypervisor cold boot, version: {:s} ({:s})", - (register_t)hypervisor_version, (register_t)hypervisor_build_date); + // We can't trace/log early because the CPU index and preemption count + // in the thread are still uninitialized. trigger_boot_cpu_early_init_event(); trigger_boot_cold_init_event(cpu); trigger_boot_cpu_cold_init_event(cpu); + + // It's safe to log now. + LOG(ERROR, WARN, "Hypervisor cold boot, version: {:s} ({:s})", + (register_t)hypervisor_version, (register_t)hypervisor_build_date); + TRACE(DEBUG, INFO, "boot_cpu_warm_init"); trigger_boot_cpu_warm_init_event(); TRACE(DEBUG, INFO, "boot_hypervisor_start"); @@ -68,7 +72,7 @@ boot_cold_init(cpu_index_t cpu) LOCK_IMPL #if defined(VERBOSE) && VERBOSE #define STACK_GUARD_BYTE 0xb8 -#define STACK_GUARD_SIZE 256 +#define STACK_GUARD_SIZE 256U #include #include @@ -81,7 +85,11 @@ boot_handle_boot_cold_init(void) { #if defined(VERBOSE) && VERBOSE // Add a red-zone to the boot stack - memset(&aarch64_boot_stack, STACK_GUARD_BYTE, STACK_GUARD_SIZE); + errno_t err_mem = memset_s(aarch64_boot_stack, STACK_GUARD_SIZE, + STACK_GUARD_BYTE, STACK_GUARD_SIZE); + if (err_mem != 0) { + panic("Error in memset_s operation!"); + } #endif } @@ -89,10 +97,10 @@ void boot_handle_idle_start(void) { #if defined(VERBOSE) && VERBOSE - char *stack_bottom = (char *)&aarch64_boot_stack; + char *stack_bottom = (char *)aarch64_boot_stack; // Check red-zone in the boot stack for (index_t i = 0; i < STACK_GUARD_SIZE; i++) { - if (stack_bottom[i] != STACK_GUARD_BYTE) { + if (stack_bottom[i] != (char)STACK_GUARD_BYTE) { panic("boot stack overflow!"); } } @@ -102,14 +110,18 @@ boot_handle_idle_start(void) noreturn void boot_secondary_init(cpu_index_t cpu) LOCK_IMPL { - // We can't trace yet because the CPU index in the thread is invalid, - // but the cpulocal handler for boot_cpu_cold_init setup it up for us. - LOG(ERROR, INFO, "secondary cpu ({:d}) cold boot", (register_t)cpu); + // We can't trace/log early because the CPU index and preemption count + // in the thread are still uninitialized trigger_boot_cpu_early_init_event(); trigger_boot_cpu_cold_init_event(cpu); + + // It's safe to log now. + LOG(ERROR, INFO, "secondary cpu ({:d}) cold boot", (register_t)cpu); + trigger_boot_cpu_warm_init_event(); trigger_boot_cpu_start_event(); + TRACE_LOCAL(DEBUG, INFO, "cpu cold boot complete"); thread_boot_set_idle(); } @@ -126,53 +138,43 @@ boot_warm_init(void) LOCK_IMPL thread_boot_set_idle(); } -error_t -boot_add_free_range(paddr_t base, size_t size, void *arg) +static error_t +boot_do_memdb_walk(paddr_t base, size_t size, void *arg) { - error_t ret = OK; - boot_env_data_t *env_data = (boot_env_data_t *)arg; - bool first_entry = true; + qcbor_enc_ctxt_t *qcbor_enc_ctxt = (qcbor_enc_ctxt_t *)arg; if ((size == 0U) && (util_add_overflows(base, size - 1))) { - ret = ERROR_ARGUMENT_SIZE; - goto error; + return ERROR_ARGUMENT_SIZE; } - index_t index = env_data->free_ranges_count; + QCBOREncode_OpenArray(qcbor_enc_ctxt); - if (index != 0U) { - index--; - first_entry = false; - } + QCBOREncode_AddUInt64(qcbor_enc_ctxt, base); + QCBOREncode_AddUInt64(qcbor_enc_ctxt, size); - assert((env_data->free_ranges[index].base + - env_data->free_ranges[index].size) <= base); - - // Check if the address range to be added is contiguous with the last - // range added. If so, append range. If not, add in next index. - if ((env_data->free_ranges[index].base + - env_data->free_ranges[index].size) == base) { - if (util_add_overflows(env_data->free_ranges[index].base, - env_data->free_ranges[index].size + - size - 1)) { - ret = ERROR_ARGUMENT_SIZE; - goto error; - } else { - env_data->free_ranges[index].size += size; - } - } else { - if (first_entry == false) { - index++; - } - if (index >= BOOT_ENV_RANGES_NUM) { - LOG(ERROR, WARN, "env_data: no more free ranges"); - } else { - env_data->free_ranges[index].base = base; - env_data->free_ranges[index].size = size; - env_data->free_ranges_count++; - } - } + QCBOREncode_CloseArray(qcbor_enc_ctxt); + + return OK; +} + +error_t +boot_add_free_range(uintptr_t object, memdb_type_t type, + qcbor_enc_ctxt_t *qcbor_enc_ctxt) +{ + error_t ret; + + QCBOREncode_OpenArrayInMap(qcbor_enc_ctxt, "free_ranges"); + + ret = memdb_walk(object, type, boot_do_memdb_walk, + (void *)qcbor_enc_ctxt); + + QCBOREncode_CloseArray(qcbor_enc_ctxt); -error: return ret; } + +void +boot_start_hypervisor_handover(void) +{ + trigger_boot_hypervisor_handover_event(); +} diff --git a/hyp/core/boot/src/rel_init.c b/hyp/core/boot/src/rel_init.c index 59dd4db..ce7fe9d 100644 --- a/hyp/core/boot/src/rel_init.c +++ b/hyp/core/boot/src/rel_init.c @@ -17,7 +17,7 @@ __attribute__((no_stack_protector)) void boot_rel_fixup(Elf_Dyn *dyni, Elf_Addr addr_offset, Elf_Addr rel_offset) { Elf_Xword dyn[DT_CNT]; - Elf_Rel *rel = NULL; + Elf_Rel *rel = NULL; Elf_Rela *rela = NULL; Elf_Xword sz = 0; @@ -32,7 +32,7 @@ boot_rel_fixup(Elf_Dyn *dyni, Elf_Addr addr_offset, Elf_Addr rel_offset) rel = (Elf_Rel *)(dyn[DT_REL] + addr_offset); sz = dyn[DT_RELSZ]; - for (; sz > 0; sz -= sizeof(*rel), ++rel) { + for (; sz > 0u; sz -= sizeof(*rel), ++rel) { if (!ARCH_CAN_PATCH(rel->r_info)) { continue; } @@ -42,7 +42,7 @@ boot_rel_fixup(Elf_Dyn *dyni, Elf_Addr addr_offset, Elf_Addr rel_offset) rela = (Elf_Rela *)(dyn[DT_RELA] + addr_offset); sz = dyn[DT_RELASZ]; - for (; sz > 0; sz -= sizeof(*rela), ++rela) { + for (; sz > 0u; sz -= sizeof(*rela), ++rela) { if (!ARCH_CAN_PATCH(rela->r_info)) { continue; } diff --git a/hyp/core/cpulocal/build.conf b/hyp/core/cpulocal/build.conf index 2892335..e99c64f 100644 --- a/hyp/core/cpulocal/build.conf +++ b/hyp/core/cpulocal/build.conf @@ -3,6 +3,10 @@ # SPDX-License-Identifier: BSD-3-Clause interface cpulocal + +assert_config PLATFORM_USABLE_CORES > 0 +assert_config PLATFORM_MAX_CORES >= bin(PLATFORM_USABLE_CORES).count("1") + types cpulocal.tc events cpulocal.ev source cpulocal.c diff --git a/hyp/core/cpulocal/cpulocal.ev b/hyp/core/cpulocal/cpulocal.ev index b18f639..2cfc64a 100644 --- a/hyp/core/cpulocal/cpulocal.ev +++ b/hyp/core/cpulocal/cpulocal.ev @@ -15,3 +15,4 @@ subscribe object_create_thread // the handlers run. subscribe thread_context_switch_post(prev) priority first + require_preempt_disabled diff --git a/hyp/core/cpulocal/cpulocal.tc b/hyp/core/cpulocal/cpulocal.tc index 98d311f..77f87aa 100644 --- a/hyp/core/cpulocal/cpulocal.tc +++ b/hyp/core/cpulocal/cpulocal.tc @@ -4,7 +4,7 @@ define cpu_index_t public newtype uint16; -define CPU_INDEX_INVALID constant type cpu_index_t = -1; +define CPU_INDEX_INVALID public constant type cpu_index_t = -1; extend thread object module cpulocal { current_cpu type cpu_index_t; diff --git a/hyp/core/cpulocal/src/cpulocal.c b/hyp/core/cpulocal/src/cpulocal.c index a19265a..9c16790 100644 --- a/hyp/core/cpulocal/src/cpulocal.c +++ b/hyp/core/cpulocal/src/cpulocal.c @@ -15,7 +15,7 @@ bool cpulocal_index_valid(cpu_index_t index) { - return index < PLATFORM_MAX_CORES; + return index < (cpu_index_t)PLATFORM_MAX_CORES; } cpu_index_t @@ -33,10 +33,10 @@ cpulocal_get_index_for_thread(const thread_t *thread) } cpu_index_t -cpulocal_get_index(void) +cpulocal_get_index_unsafe(void) { const thread_t *self = thread_get_self(); - return cpulocal_check_index(cpulocal_get_index_for_thread(self)); + return cpulocal_get_index_for_thread(self); } void diff --git a/hyp/core/cspace_twolevel/cspace.tc b/hyp/core/cspace_twolevel/cspace.tc index 6581719..6dcad85 100644 --- a/hyp/core/cspace_twolevel/cspace.tc +++ b/hyp/core/cspace_twolevel/cspace.tc @@ -49,6 +49,7 @@ define cap_data structure { info bitfield cap_info; }; +// FIXME: define cap structure(aligned(16)) { data structure cap_data(atomic); cap_list_node structure list_node(contained); @@ -84,6 +85,7 @@ define CAP_TABLE_NUM_CAP_SLOTS constant type index_t = sizeof(structure cap)) * sizeof(type register_t))) / sizeof(structure cap); +// FIXME: define CSPACE__CAP_TABLE_MEM_AVAIL constant size = CSPACE_ALLOC_SIZE - sizeof(object cspace_inner) - sizeof(structure object_header) - sizeof(structure list_node); diff --git a/hyp/core/cspace_twolevel/cspace_tests.ev b/hyp/core/cspace_twolevel/cspace_tests.ev index 4a13336..bbebdef 100644 --- a/hyp/core/cspace_twolevel/cspace_tests.ev +++ b/hyp/core/cspace_twolevel/cspace_tests.ev @@ -11,5 +11,6 @@ subscribe tests_init subscribe tests_start handler tests_cspace_start() + require_preempt_disabled #endif diff --git a/hyp/core/cspace_twolevel/hypercalls.hvc b/hyp/core/cspace_twolevel/hypercalls.hvc index 9279750..4ae1a74 100644 --- a/hyp/core/cspace_twolevel/hypercalls.hvc +++ b/hyp/core/cspace_twolevel/hypercalls.hvc @@ -6,6 +6,6 @@ define cspace_configure hypercall { call_num 0x25; cspace input type cap_id_t; max_caps input type count_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; diff --git a/hyp/core/cspace_twolevel/src/cspace_tests.c b/hyp/core/cspace_twolevel/src/cspace_tests.c index d5e4e91..592e292 100644 --- a/hyp/core/cspace_twolevel/src/cspace_tests.c +++ b/hyp/core/cspace_twolevel/src/cspace_tests.c @@ -21,7 +21,7 @@ #include "event_handlers.h" -static cspace_t *test_cspace; +static cspace_t *test_cspace; static cap_id_t test_cspace_master_cap; static _Atomic count_t test_cspace_wait_count; static _Atomic count_t test_cspace_finish_count; @@ -81,7 +81,7 @@ bool tests_cspace_start(void) { cap_id_result_t cap_ret; - cap_id_t *cap = CPULOCAL(test_caps); + cap_id_t *cap = CPULOCAL(test_caps); cap_rights_cspace_t cspace_rights = cap_rights_cspace_default(); cap_rights_t rights; error_t err; diff --git a/hyp/core/cspace_twolevel/src/cspace_twolevel.c b/hyp/core/cspace_twolevel/src/cspace_twolevel.c index 8262d90..006b133 100644 --- a/hyp/core/cspace_twolevel/src/cspace_twolevel.c +++ b/hyp/core/cspace_twolevel/src/cspace_twolevel.c @@ -106,14 +106,14 @@ cspace_init_id_encoder(cspace_t *cspace) } // Calculate a 16-bit random multiplier and its inverse cspace->id_mult = rand_mult.r & 0xffffU; - cspace->id_inv = (((uint64_t)1U << 32U) / cspace->id_mult) + 1U; + cspace->id_inv = (util_bit(32) / cspace->id_mult) + 1U; } out: #else cspace->id_rand_base = 0U; cspace->id_mult = 1U; - cspace->id_inv = ((uint64_t)1U << 32U) + 1U; + cspace->id_inv = util_bit(32U) + 1U; err = OK; #endif return err; @@ -254,8 +254,8 @@ cspace_allocate_cap_table(cspace_t *cspace, cap_table_t **table, error_t err; void_ptr_result_t ret; index_t index; - cap_table_t *new_table; - partition_t *partition = cspace->header.partition; + cap_table_t *new_table; + partition_t *partition = cspace->header.partition; do { if (!bitmap_atomic_ffc(cspace->allocated_tables, @@ -276,8 +276,8 @@ cspace_allocate_cap_table(cspace_t *cspace, cap_table_t **table, goto allocate_cap_table_error; } - memset(ret.r, 0, sizeof(cap_table_t)); new_table = (cap_table_t *)ret.r; + (void)memset_s(new_table, sizeof(*new_table), 0, sizeof(*new_table)); new_table->partition = object_get_partition_additional(partition); new_table->cspace = cspace; @@ -295,14 +295,14 @@ rcu_update_status_t cspace_destroy_cap_table(rcu_entry_t *entry) { index_t index; - cap_table_t *table = cap_table_container_of_rcu_entry(entry); - partition_t *partition = table->partition; + cap_table_t *table = cap_table_container_of_rcu_entry(entry); + partition_t *partition = table->partition; rcu_update_status_t ret = rcu_update_status_default(); // If called via cspace destroy, there may still // be valid caps which also require destruction. for (; table->cap_count > 0U; table->cap_count--) { - cap_t *cap; + cap_t *cap; cap_data_t data; object_type_t type; object_header_t *header; @@ -327,7 +327,7 @@ cspace_destroy_cap_table(rcu_entry_t *entry) type = cap_info_get_type(&data.info); header = object_get_header(type, data.object); spinlock_acquire(&header->cap_list_lock); - list_delete_node(&header->cap_list, &cap->cap_list_node); + (void)list_delete_node(&header->cap_list, &cap->cap_list_node); cap_list_empty = list_is_empty(&header->cap_list); spinlock_release(&header->cap_list_lock); @@ -336,7 +336,7 @@ cspace_destroy_cap_table(rcu_entry_t *entry) } } - partition_free(partition, table, sizeof(cap_table_t)); + (void)partition_free(partition, table, sizeof(cap_table_t)); object_put_partition(partition); return ret; @@ -344,6 +344,7 @@ cspace_destroy_cap_table(rcu_entry_t *entry) static error_t cspace_allocate_cap_slot(cspace_t *cspace, cap_t **cap, cap_id_t *cap_id) + REQUIRE_RCU_READ { error_t err; cap_table_t *table; @@ -445,7 +446,7 @@ cspace_lookup_object(cspace_t *cspace, cap_id_t cap_id, object_type_t type, cap_rights_t rights, bool active_only) { error_t err; - cap_t *cap; + cap_t *cap; cap_data_t cap_data; object_ptr_result_t ret; @@ -490,7 +491,7 @@ cspace_lookup_object_any(cspace_t *cspace, cap_id_t cap_id, cap_rights_generic_t rights, object_type_t *type) { error_t err; - cap_t *cap; + cap_t *cap; cap_data_t cap_data; object_ptr_result_t ret; object_type_t obj_type = OBJECT_TYPE_ANY; @@ -565,7 +566,7 @@ cspace_configure(cspace_t *cspace, count_t max_caps) error_t cspace_twolevel_handle_object_activate_cspace(cspace_t *cspace) { - if (cspace->max_caps != 0) { + if (cspace->max_caps != 0U) { return OK; } else { return ERROR_OBJECT_CONFIG; @@ -589,7 +590,7 @@ cspace_create_master_cap(cspace_t *cspace, object_ptr_t object, object_type_t type) { error_t err; - cap_t *new_cap; + cap_t *new_cap; cap_data_t cap_data; cap_id_t new_cap_id; cap_id_result_t ret; @@ -631,7 +632,7 @@ cspace_copy_cap(cspace_t *target_cspace, cspace_t *parent_cspace, cap_id_t parent_id, cap_rights_t rights_mask) { error_t err; - cap_t *new_cap, *parent_cap; + cap_t *new_cap, *parent_cap; cap_data_t cap_data; cap_id_t new_cap_id; object_header_t *header; @@ -705,7 +706,7 @@ error_t cspace_delete_cap(cspace_t *cspace, cap_id_t cap_id) { error_t err; - cap_t *cap; + cap_t *cap; cap_data_t cap_data, null_cap_data = { 0 }; cap_state_t state; object_type_t type; @@ -730,8 +731,8 @@ cspace_delete_cap(cspace_t *cspace, cap_id_t cap_id) err = cspace_update_cap_slot(cap, &cap_data, null_cap_data); if (err == OK) { - list_delete_node(&header->cap_list, - &cap->cap_list_node); + (void)list_delete_node(&header->cap_list, + &cap->cap_list_node); cap_list_empty = list_is_empty(&header->cap_list); } @@ -741,8 +742,8 @@ cspace_delete_cap(cspace_t *cspace, cap_id_t cap_id) err = cspace_update_cap_slot(cap, &cap_data, null_cap_data); if (err == OK) { - list_delete_node(&cspace->revoked_cap_list, - &cap->cap_list_node); + (void)list_delete_node(&cspace->revoked_cap_list, + &cap->cap_list_node); } spinlock_release(&cspace->revoked_cap_list_lock); @@ -766,7 +767,7 @@ error_t cspace_revoke_caps(cspace_t *cspace, cap_id_t master_cap_id) { error_t err; - cap_t *master_cap; + cap_t *master_cap; cap_data_t master_cap_data; object_header_t *header; @@ -805,6 +806,7 @@ cspace_revoke_caps(cspace_t *cspace, cap_id_t master_cap_id) list_t *list = &header->cap_list; assert(list_get_head(list) == &master_cap->cap_list_node); + // FIXME: cap_t *curr_cap = NULL; list_foreach_container_maydelete (curr_cap, list, cap, cap_list_node) { @@ -819,7 +821,7 @@ cspace_revoke_caps(cspace_t *cspace, cap_id_t master_cap_id) // Clear the object this cap points to, since the object could // be deleted by deleting the last valid cap, revoked caps // pointing to freed memory would make debugging confusing. - memset(&curr_cap_data.object, 0, sizeof(curr_cap_data.object)); + curr_cap_data.object = (object_ptr_t){ 0 }; // It is safe to get the child cap's cspace, as the child // cap must be destroyed before the cspace can be, and @@ -830,7 +832,8 @@ cspace_revoke_caps(cspace_t *cspace, cap_id_t master_cap_id) // Child cap data won't change while we hold the locks, // so just atomically store the invalid data. atomic_store_relaxed(&curr_cap->data, curr_cap_data); - list_delete_node(&header->cap_list, &curr_cap->cap_list_node); + (void)list_delete_node(&header->cap_list, + &curr_cap->cap_list_node); list_insert_at_head(&curr_cspace->revoked_cap_list, &curr_cap->cap_list_node); spinlock_release_nopreempt(&curr_cspace->revoked_cap_list_lock); diff --git a/hyp/core/cspace_twolevel/src/hypercalls.c b/hyp/core/cspace_twolevel/src/hypercalls.c index f2ce64c..10f506c 100644 --- a/hyp/core/cspace_twolevel/src/hypercalls.c +++ b/hyp/core/cspace_twolevel/src/hypercalls.c @@ -107,7 +107,7 @@ error_t hypercall_cspace_configure(cap_id_t cspace_cap, count_t max_caps) { error_t err; - cspace_t *cspace = cspace_get_self(); + cspace_t *cspace = cspace_get_self(); object_type_t type; object_ptr_result_t o = cspace_lookup_object_any( @@ -143,7 +143,7 @@ error_t hypercall_cspace_attach_thread(cap_id_t cspace_cap, cap_id_t thread_cap) { error_t ret; - cspace_t *cspace = cspace_get_self(); + cspace_t *cspace = cspace_get_self(); object_type_t type; object_ptr_result_t o = cspace_lookup_object_any( diff --git a/hyp/core/debug/aarch64/debug.ev b/hyp/core/debug/aarch64/debug.ev index 19c92d0..b5108f5 100644 --- a/hyp/core/debug/aarch64/debug.ev +++ b/hyp/core/debug/aarch64/debug.ev @@ -14,4 +14,7 @@ subscribe power_cpu_suspend(may_poweroff) subscribe power_cpu_resume(was_poweroff) require_preempt_disabled +subscribe power_cpu_offline + require_preempt_disabled + #endif diff --git a/hyp/core/debug/aarch64/debug.tc b/hyp/core/debug/aarch64/debug.tc index 27ca11a..543ee25 100644 --- a/hyp/core/debug/aarch64/debug.tc +++ b/hyp/core/debug/aarch64/debug.tc @@ -66,6 +66,9 @@ define debug_common_registers structure { bcrs array(CPU_DEBUG_BP_COUNT) bitfield DBGBCR_EL1; wcrs array(CPU_DEBUG_WP_COUNT) bitfield DBGWCR_EL1; mdscr bitfield MDSCR_EL1; +#if ARCH_AARCH64_32BIT_EL1 + dbgvcr uint32; +#endif }; #if PLATFORM_DEBUG_SAVE_STATE diff --git a/hyp/core/debug/aarch64/src/debug.c b/hyp/core/debug/aarch64/src/debug.c index 1082580..5e68cb5 100644 --- a/hyp/core/debug/aarch64/src/debug.c +++ b/hyp/core/debug/aarch64/src/debug.c @@ -17,7 +17,7 @@ #include "event_handlers.h" #if PLATFORM_DEBUG_SAVE_STATE -static struct asm_ordering_dummy debug_asm_order; +static asm_ordering_dummy_t debug_asm_order; CPULOCAL_DECLARE_STATIC(debug_ext_state_t, debug_ext_state); @@ -51,6 +51,12 @@ debug_handle_power_cpu_online(void) debug_os_unlock(); } +void +debug_handle_power_cpu_offline(void) +{ + (void)debug_handle_power_cpu_suspend(true); +} + error_t debug_handle_power_cpu_suspend(bool may_poweroff) { diff --git a/hyp/core/debug/aarch64/templates/debug_bps.h.tmpl b/hyp/core/debug/aarch64/templates/debug_bps.h.tmpl index 950a886..06a57a9 100644 --- a/hyp/core/debug/aarch64/templates/debug_bps.h.tmpl +++ b/hyp/core/debug/aarch64/templates/debug_bps.h.tmpl @@ -5,7 +5,7 @@ \#if defined(MODULE_CORE_DEBUG) static inline bool debug_save_common(debug_common_registers_t *regs, - struct asm_ordering_dummy *order) + asm_ordering_dummy_t *order) { bool in_use = false; @@ -28,12 +28,16 @@ debug_save_common(debug_common_registers_t *regs, in_use = in_use || MDSCR_EL1_get_TDCC(®s->mdscr); in_use = in_use || MDSCR_EL1_get_SS(®s->mdscr); +\#if ARCH_AARCH64_32BIT_EL1 + regs->dbgvcr = register_DBGVCR32_EL2_read_ordered(order); +\#endif + return in_use; } static inline void debug_load_common(const debug_common_registers_t *regs, - struct asm_ordering_dummy *order) + asm_ordering_dummy_t *order) { #for bp in range(0, $CPU_DEBUG_BP_COUNT) register_DBGBVR${bp}_EL1_write_ordered(regs->bvrs[${bp}], order); @@ -49,5 +53,9 @@ debug_load_common(const debug_common_registers_t *regs, #end for register_MDSCR_EL1_write_ordered(regs->mdscr, order); + +\#if ARCH_AARCH64_32BIT_EL1 + register_DBGVCR32_EL2_write_ordered(regs->dbgvcr, order); +\#endif } \#endif diff --git a/hyp/core/globals/build.conf b/hyp/core/globals/build.conf new file mode 100644 index 0000000..a50c3bb --- /dev/null +++ b/hyp/core/globals/build.conf @@ -0,0 +1,8 @@ +# © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +interface globals + +events globals.ev +source globals.c diff --git a/hyp/core/globals/globals.ev b/hyp/core/globals/globals.ev new file mode 100644 index 0000000..08bf09e --- /dev/null +++ b/hyp/core/globals/globals.ev @@ -0,0 +1,7 @@ +// © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module globals + +subscribe boot_cold_init() diff --git a/hyp/core/globals/src/globals.c b/hyp/core/globals/src/globals.c new file mode 100644 index 0000000..4e0edba --- /dev/null +++ b/hyp/core/globals/src/globals.c @@ -0,0 +1,41 @@ +// © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include + +#include +#include + +#include "event_handlers.h" + +static global_options_t global_options; +static spinlock_t global_options_lock; + +void +globals_handle_boot_cold_init(void) +{ + spinlock_init(&global_options_lock); +} + +const global_options_t * +globals_get_options(void) +{ + return &global_options; +} + +void +globals_set_options(global_options_t set) +{ + spinlock_acquire(&global_options_lock); + global_options = global_options_union(global_options, set); + spinlock_release(&global_options_lock); +} + +void +globals_clear_options(global_options_t clear) +{ + spinlock_acquire(&global_options_lock); + global_options = global_options_difference(global_options, clear); + spinlock_release(&global_options_lock); +} diff --git a/hyp/core/idle/idle.ev b/hyp/core/idle/idle.ev index 67a6d42..61b424e 100644 --- a/hyp/core/idle/idle.ev +++ b/hyp/core/idle/idle.ev @@ -8,6 +8,7 @@ subscribe object_create_thread subscribe boot_hypervisor_start handler idle_thread_init() + require_preempt_disabled subscribe thread_get_entry_fn[THREAD_KIND_IDLE] diff --git a/hyp/core/idle/idle.tc b/hyp/core/idle/idle.tc index dd3e5e5..8a37114 100644 --- a/hyp/core/idle/idle.tc +++ b/hyp/core/idle/idle.tc @@ -3,7 +3,7 @@ // SPDX-License-Identifier: BSD-3-Clause extend thread_kind enumeration { - idle; + idle = 1; }; extend scheduler_block enumeration { diff --git a/hyp/core/idle/src/idle.c b/hyp/core/idle/src/idle.c index e9a2149..00b8312 100644 --- a/hyp/core/idle/src/idle.c +++ b/hyp/core/idle/src/idle.c @@ -37,6 +37,8 @@ idle_thread_create(cpu_index_t i) thread_create_t params = { .scheduler_affinity = i, .scheduler_affinity_valid = true, + .scheduler_priority = SCHEDULER_MIN_PRIORITY, + .scheduler_priority_valid = true, .kind = THREAD_KIND_IDLE, }; @@ -69,6 +71,8 @@ idle_thread_init_boot(thread_t *thread, cpu_index_t i) thread_create_t params = { .scheduler_affinity = i, .scheduler_affinity_valid = true, + .scheduler_priority = SCHEDULER_MIN_PRIORITY, + .scheduler_priority_valid = true, .kind = THREAD_KIND_IDLE, }; @@ -91,7 +95,8 @@ void idle_thread_init(void) { // Allocate some address space for the idle stacks. - size_t aspace_size = THREAD_STACK_MAP_ALIGN * (PLATFORM_MAX_CORES + 1U); + size_t aspace_size = + THREAD_STACK_MAP_ALIGN * ((size_t)PLATFORM_MAX_CORES + 1U); virt_range_result_t stack_range = hyp_aspace_allocate(aspace_size); if (stack_range.e != OK) { @@ -106,14 +111,14 @@ idle_thread_init(void) const cpu_index_t cpu = cpulocal_get_index(); for (cpu_index_t i = 0; cpulocal_index_valid(i); i++) { - thread_t *idle_thread; + thread_t *thread_idle; if (cpu == i) { thread_t *self = thread_get_self(); idle_thread_init_boot(self, i); - idle_thread = self; + thread_idle = self; } else { - idle_thread = idle_thread_create(i); + thread_idle = idle_thread_create(i); } // Each idle thread needs a single extra reference to prevent it @@ -121,10 +126,10 @@ idle_thread_init(void) // be switching from itself in thread_boot_set_idle(), so when // it releases the reference to the previous thread in // thread_arch_main(), it will in fact be releasing itself. - object_get_thread_additional(idle_thread); + (void)object_get_thread_additional(thread_idle); - CPULOCAL_BY_INDEX(idle_thread, i) = idle_thread; - if (object_activate_thread(idle_thread) != OK) { + CPULOCAL_BY_INDEX(idle_thread, i) = thread_idle; + if (object_activate_thread(thread_idle) != OK) { panic("Error activating idle thread"); } @@ -152,6 +157,7 @@ idle_handle_idle_start(void) // Free the boot stack // Find a better place to free the boot stack + // FIXME: error_t err = partition_add_heap( private, partition_virt_to_phys(private, (uintptr_t)&aarch64_boot_stack), @@ -164,6 +170,11 @@ idle_handle_idle_start(void) static noreturn void idle_loop(uintptr_t unused_params) { + // We generally run the idle thread with preemption disabled. Handlers + // for the idle_yield event may re-enable preemption, as long as they + // are guaranteed to stop waiting and return true if preemption occurs. + preempt_disable(); + const cpu_index_t this_cpu = cpulocal_get_index(); if (compiler_unexpected(this_cpu == boot_cpu)) { @@ -176,11 +187,6 @@ idle_loop(uintptr_t unused_params) (void)unused_params; - // We generally run the idle thread with preemption disabled. Handlers - // for the idle_yield event may re-enable preemption, as long as they - // are guaranteed to stop waiting and return true if preemption occurs. - preempt_disable(); - assert(scheduler_is_blocked(thread_get_self(), SCHEDULER_BLOCK_IDLE)); do { @@ -210,7 +216,7 @@ idle_handle_thread_get_stack_base(thread_kind_t kind, thread_t *thread) cpu_index_t cpu = thread->scheduler_affinity; - return idle_stack_base + (cpu * THREAD_STACK_MAP_ALIGN); + return idle_stack_base + ((uintptr_t)cpu * THREAD_STACK_MAP_ALIGN); } thread_t * diff --git a/hyp/core/ipi/ipi.ev b/hyp/core/ipi/ipi.ev index 065cf82..5a7b857 100644 --- a/hyp/core/ipi/ipi.ev +++ b/hyp/core/ipi/ipi.ev @@ -19,7 +19,9 @@ subscribe thread_exit_to_user require_preempt_disabled subscribe optional preempt_interrupt + require_preempt_disabled // This handler IPIs all cores to stop, it has high priority subscribe scheduler_stop() priority 1000 + require_preempt_disabled diff --git a/hyp/core/ipi/src/ipi.c b/hyp/core/ipi/src/ipi.c index b024971..63d5e1a 100644 --- a/hyp/core/ipi/src/ipi.c +++ b/hyp/core/ipi/src/ipi.c @@ -31,7 +31,7 @@ // Set this to 1 to disable the fast idle optimisation for debugging #define IPI_DEBUG_NO_FAST_IDLE 0 -#define REGISTER_BITS (sizeof(register_t) * CHAR_BIT) +#define REGISTER_BITS (sizeof(register_t) * (size_t)CHAR_BIT) // We enable the fast wakeup support by default if asm_event_wait() can sleep // (as it will busy-wait otherwise) and preemption is enabled. We can possibly @@ -41,6 +41,7 @@ // If interrupts are handled by a VM, we need to be able to ask the VM to send // an IPI for us. This is not currently implemented, so we force fast wakeups in // such configurations even though they will block pending interrupts. +// FIXME: #if (!ASM_EVENT_WAIT_IS_NOOP && !defined(PREEMPT_NULL) && \ !IPI_DEBUG_NO_FAST_IDLE) || \ defined(IPI_FORCE_FAST_WAKEUP_HACK) @@ -233,8 +234,9 @@ ipi_handle_relaxed(void) prefetch_store_keep(local_pending); register_t pending = atomic_load_relaxed(local_pending); while (compiler_unexpected(pending != 0U)) { - ipi_reason_t ipi = (ipi_reason_t)(REGISTER_BITS - 1U - - compiler_clz(pending)); + ipi_reason_t ipi = + (ipi_reason_t)((register_t)(REGISTER_BITS - 1U - + compiler_clz(pending))); if (ipi_clear_relaxed(ipi) && trigger_ipi_received_event(ipi)) { reschedule = true; } @@ -250,7 +252,7 @@ ipi_handle_thread_exit_to_user(thread_entry_reason_t reason) // Relaxed IPIs are handled directly by the IRQ module for interrupts. if (reason != THREAD_ENTRY_REASON_INTERRUPT) { if (ipi_handle_relaxed()) { - scheduler_schedule(); + (void)scheduler_schedule(); } } } @@ -266,8 +268,9 @@ ipi_handle_idle_yield(bool in_idle_thread) register_t pending; do { // Mark ourselves as waiting in idle. - atomic_fetch_or_explicit(local_pending, IPI_WAITING_IN_IDLE, - memory_order_relaxed); + (void)atomic_fetch_or_explicit(local_pending, + IPI_WAITING_IN_IDLE, + memory_order_relaxed); // Sleep until there is at least one event to handle or a // preemption clears IPI_WAITING_IN_IDLE. @@ -348,7 +351,7 @@ ipi_handle_scheduler_stop(void) uint32_t freq = platform_timer_get_frequency(); uint64_t now = platform_timer_get_current_ticks(); - uint64_t end = now + (freq / 1024); + uint64_t end = now + ((uint64_t)freq / 1024U); while (now < end) { asm_yield(); diff --git a/hyp/core/irq/src/irq.c b/hyp/core/irq/src/irq.c index 7a72e9e..c85fdf0 100644 --- a/hyp/core/irq/src/irq.c +++ b/hyp/core/irq/src/irq.c @@ -30,8 +30,9 @@ // compare-exchange, at both levels. Empty levels are never freed, on the // assumption that IRQ numbers are set by hardware and therefore are likely to // be reused. -#define IRQ_TABLE_L2_SIZE PGTABLE_HYP_PAGE_SIZE -#define IRQ_TABLE_L2_ENTRIES (IRQ_TABLE_L2_SIZE / sizeof(hwirq_t *_Atomic)) +#define IRQ_TABLE_L2_SIZE PGTABLE_HYP_PAGE_SIZE +#define IRQ_TABLE_L2_ENTRIES \ + (count_t)((IRQ_TABLE_L2_SIZE / sizeof(hwirq_t *_Atomic))) static hwirq_t *_Atomic *_Atomic *irq_table_l1; #else // Dynamically allocated array of RCU-protected pointers to hwirq objects. @@ -67,29 +68,29 @@ irq_handle_boot_cold_init(void) if (ptr_r.e != OK) { panic("Unable to allocate IRQ table"); } - assert(ptr_r.r != NULL); #if IRQ_SPARSE_IDS - irq_table_l1 = memset(ptr_r.r, 0, alloc_size); + irq_table_l1 = ptr_r.r; + (void)memset_s(irq_table_l1, alloc_size, 0, alloc_size); #else - irq_table = memset(ptr_r.r, 0, alloc_size); + irq_table = ptr_r.r; + (void)memset_s(irq_table, alloc_size, 0, alloc_size); #endif #if IRQ_HAS_MSI irq_msi_bitmap_size = platform_irq_msi_max() - platform_irq_msi_base + 1U; count_t msi_bitmap_words = BITMAP_NUM_WORDS(irq_msi_bitmap_size); + alloc_size = msi_bitmap_words * sizeof(register_t); - ptr_r = partition_alloc(partition_get_private(), - msi_bitmap_words * sizeof(register_t), + ptr_r = partition_alloc(partition_get_private(), alloc_size, alignof(register_t)); if (ptr_r.e != OK) { panic("Unable to allocate MSI allocator bitmap"); } - assert(ptr_r.r != NULL); - irq_msi_bitmap = - memset(ptr_r.r, 0, msi_bitmap_words * sizeof(register_t)); + irq_msi_bitmap = ptr_r.r; + (void)memset_s(irq_msi_bitmap, alloc_size, 0, alloc_size); #endif } @@ -111,8 +112,7 @@ irq_find_entry(irq_t irq, bool allocate) partition_get_private(), alloc_size, alloc_align); if (ptr_r.e == OK) { - assert(ptr_r.r != NULL); - memset(ptr_r.r, 0, alloc_size); + (void)memset_s(ptr_r.r, alloc_size, 0, alloc_size); if (atomic_compare_exchange_strong_explicit( &irq_table_l1[irq_l1_index], &irq_table_l2, @@ -122,8 +122,8 @@ irq_find_entry(irq_t irq, bool allocate) irq_table_l2 = (hwirq_t *_Atomic *)ptr_r.r; } else { assert(irq_table_l2 != NULL); - partition_free(partition_get_private(), ptr_r.r, - alloc_size); + (void)partition_free(partition_get_private(), + ptr_r.r, alloc_size); } } } @@ -186,7 +186,7 @@ irq_handle_object_activate_hwirq(hwirq_t *hwirq) // The IRQ is fully registered; give the handler an opportunity to // enable it if desired. - trigger_irq_registered_event(hwirq->action, hwirq->irq, hwirq); + (void)trigger_irq_registered_event(hwirq->action, hwirq->irq, hwirq); out: return err; @@ -277,63 +277,76 @@ irq_handle_object_deactivate_hwirq(hwirq_t *hwirq) atomic_store_relaxed(entry, NULL); } -bool -irq_interrupt_dispatch(void) +static void +disable_unhandled_irq(irq_result_t irq_r) REQUIRE_PREEMPT_DISABLED { - bool spurious = true; + TRACE(DEBUG, WARN, "disabling unhandled HW IRQ {:d}", irq_r.r); + if (platform_irq_is_percpu(irq_r.r)) { + platform_irq_disable_local(irq_r.r); + } else { + platform_irq_disable(irq_r.r); + } + platform_irq_priority_drop(irq_r.r); + platform_irq_deactivate(irq_r.r); +} - while (true) { - irq_result_t irq_r = platform_irq_acknowledge(); - - if (irq_r.e == ERROR_RETRY) { - // IRQ handled by the platform, probably an IPI - spurious = false; - continue; - } else if (compiler_unexpected(irq_r.e == ERROR_IDLE)) { - // No IRQs are pending; exit - break; - } else { - assert(irq_r.e == OK); - - spurious = false; - TRACE(DEBUG, INFO, "acknowledged HW IRQ {:d}", irq_r.r); - - // The entire IRQ delivery is an RCU critical section. - // - // Note that this naturally true anyway if we don't - // allow interrupt nesting. - // - // Also, the alternative is to take a reference to the - // hwirq, which might force us to tear down the hwirq - // (and potentially the whole partition) in the - // interrupt handler. - rcu_read_start(); - hwirq_t *hwirq = irq_lookup_hwirq(irq_r.r); - - if (compiler_unexpected(hwirq == NULL)) { - TRACE(DEBUG, WARN, - "disabling unhandled HW IRQ {:d}", - irq_r.r); - if (platform_irq_is_percpu(irq_r.r)) { - platform_irq_disable_local(irq_r.r); - } else { - platform_irq_disable(irq_r.r); - } - platform_irq_priority_drop(irq_r.r); - platform_irq_deactivate(irq_r.r); - rcu_read_finish(); - continue; - } - assert(hwirq->irq == irq_r.r); +static bool +irq_interrupt_dispatch_one(void) REQUIRE_PREEMPT_DISABLED +{ + irq_result_t irq_r = platform_irq_acknowledge(); + bool ret = true; - bool handled = trigger_irq_received_event( - hwirq->action, irq_r.r, hwirq); - platform_irq_priority_drop(irq_r.r); - if (handled) { - platform_irq_deactivate(irq_r.r); - } + if (irq_r.e == ERROR_RETRY) { + // IRQ handled by the platform, probably an IPI + goto out; + } else if (compiler_unexpected(irq_r.e == ERROR_IDLE)) { + // No IRQs are pending; exit + ret = false; + goto out; + } else { + assert(irq_r.e == OK); + TRACE(DEBUG, INFO, "acknowledged HW IRQ {:d}", irq_r.r); + + // The entire IRQ delivery is an RCU critical section. + // + // Note that this naturally true anyway if we don't + // allow interrupt nesting. + // + // Also, the alternative is to take a reference to the + // hwirq, which might force us to tear down the hwirq + // (and potentially the whole partition) in the + // interrupt handler. + rcu_read_start(); + hwirq_t *hwirq = irq_lookup_hwirq(irq_r.r); + + if (compiler_unexpected(hwirq == NULL)) { + disable_unhandled_irq(irq_r); rcu_read_finish(); + goto out; } + + assert(hwirq->irq == irq_r.r); + + bool handled = trigger_irq_received_event(hwirq->action, + irq_r.r, hwirq); + platform_irq_priority_drop(irq_r.r); + if (handled) { + platform_irq_deactivate(irq_r.r); + } + rcu_read_finish(); + } + +out: + return ret; +} + +bool +irq_interrupt_dispatch(void) +{ + bool spurious = true; + + while (irq_interrupt_dispatch_one()) { + spurious = false; } if (spurious) { diff --git a/hyp/core/object_standard/hypercalls.hvc b/hyp/core/object_standard/hypercalls.hvc index 54ea20e..2116719 100644 --- a/hyp/core/object_standard/hypercalls.hvc +++ b/hyp/core/object_standard/hypercalls.hvc @@ -5,7 +5,7 @@ define object_activate hypercall { call_num 0xc; cap input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; @@ -13,14 +13,14 @@ define object_activate_from hypercall { call_num 0xd; cspace input type cap_id_t; cap input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; define object_reset hypercall { call_num 0xe; cap input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; @@ -28,6 +28,6 @@ define object_reset_from hypercall { call_num 0xf; cspace input type cap_id_t; cap input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; diff --git a/hyp/core/object_standard/src/hypercalls.c b/hyp/core/object_standard/src/hypercalls.c index 0fae4db..e6a2157 100644 --- a/hyp/core/object_standard/src/hypercalls.c +++ b/hyp/core/object_standard/src/hypercalls.c @@ -18,7 +18,7 @@ error_t hypercall_object_activate(cap_id_t cap) { error_t err; - cspace_t *cspace = cspace_get_self(); + cspace_t *cspace = cspace_get_self(); object_type_t type; object_ptr_result_t o = cspace_lookup_object_any( @@ -38,7 +38,7 @@ error_t hypercall_object_activate_from(cap_id_t cspace_cap, cap_id_t cap) { error_t err; - cspace_t *cspace = cspace_get_self(); + cspace_t *cspace = cspace_get_self(); object_type_t type; cspace_ptr_result_t c; diff --git a/hyp/core/object_standard/templates/object.tc.tmpl b/hyp/core/object_standard/templates/object.tc.tmpl index b9ef74b..a26a20d 100644 --- a/hyp/core/object_standard/templates/object.tc.tmpl +++ b/hyp/core/object_standard/templates/object.tc.tmpl @@ -6,8 +6,8 @@ define object_header structure { type enumeration object_type; state enumeration object_state(atomic); refcount structure refcount; - rcu_entry structure rcu_entry(contained); lock structure spinlock; + rcu_entry structure rcu_entry(contained); }; #for obj in $object_list @@ -17,7 +17,7 @@ define object_header structure { // at the start of the object, needed for partition_revoke // " header structure object_header priority=first;" extend $o object { - header structure object_header(contained); + header structure object_header(contained, group(__object_header)); }; #end for diff --git a/hyp/core/partition_standard/armv8/phys_access.ev b/hyp/core/partition_standard/armv8/phys_access.ev index b4ea5b2..4d6175c 100644 --- a/hyp/core/partition_standard/armv8/phys_access.ev +++ b/hyp/core/partition_standard/armv8/phys_access.ev @@ -6,3 +6,4 @@ module partition_standard subscribe boot_cpu_warm_init handler partition_phys_access_cpu_warm_init() + priority 10 diff --git a/hyp/core/partition_standard/armv8/src/phys_access.c b/hyp/core/partition_standard/armv8/src/phys_access.c index 46a804a..32dadaf 100644 --- a/hyp/core/partition_standard/armv8/src/phys_access.c +++ b/hyp/core/partition_standard/armv8/src/phys_access.c @@ -27,6 +27,41 @@ partition_phys_access_cpu_warm_init(void) #endif } +static bool +memory_attr_type_check(MAIR_ATTR_t memattr, paddr_t check_pa) +{ + bool ret = true; + + switch (memattr) { + case MAIR_ATTR_DEVICE_NGNRNE: + case MAIR_ATTR_DEVICE_NGNRE: + case MAIR_ATTR_DEVICE_NGRE: + case MAIR_ATTR_DEVICE_GRE: + ret = false; + break; + case MAIR_ATTR_NORMAL_NC: + case MAIR_ATTR_NORMAL_WB_OUTER_NC: +#if defined(ARCH_ARM_FEAT_MTE) + case MAIR_ATTR_TAGGED_NORMAL_WB: +#endif + case MAIR_ATTR_NORMAL_WB: + break; + case MAIR_ATTR_DEVICE_NGNRNE_XS: + case MAIR_ATTR_DEVICE_NGNRE_XS: + case MAIR_ATTR_DEVICE_NGRE_XS: + case MAIR_ATTR_DEVICE_GRE_XS: + default: + LOG(ERROR, WARN, + "Unexpected look-up result in partition_phys_valid." + " PA:{:#x}, attr : {:#x}", + check_pa, (register_t)memattr); + ret = false; + break; + } + + return ret; +} + bool partition_phys_valid(paddr_t paddr, size_t size) { @@ -41,13 +76,12 @@ partition_phys_valid(paddr_t paddr, size_t size) goto out; } -#if ARCH_AARCH64_USE_PAN for (paddr_t check_pa = paddr; check_pa < (paddr + size); check_pa += PGTABLE_HYP_PAGE_SIZE) { paddr_t pa_lookup; MAIR_ATTR_t memattr; - void *check_va = (void *)((uintptr_t)check_pa + - HYP_ASPACE_PHYSACCESS_OFFSET); + void *check_va = (void *)((uintptr_t)check_pa + + hyp_aspace_get_physaccess_offset()); error_t err = hyp_aspace_va_to_pa_el2_read(check_va, &pa_lookup, &memattr, NULL); @@ -67,14 +101,14 @@ partition_phys_valid(paddr_t paddr, size_t size) panic("partition_phys_valid: Bad look-up result"); } - // We map the HYP_ASPACE_PHYSACCESS_OFFSET as device type when + // We map the hyp_aspace_physaccess_offset as device type when // invalid. - if ((memattr & ~MAIR_ATTR_DEVICE_MASK) == 0U) { - ret = false; + ret = memory_attr_type_check(memattr, check_pa); + + if (!ret) { break; } } -#endif out: return ret; @@ -83,24 +117,17 @@ partition_phys_valid(paddr_t paddr, size_t size) void * partition_phys_map(paddr_t paddr, size_t size) { -#if ARCH_AARCH64_USE_PAN assert(!util_add_overflows(paddr, size)); + assert_debug(partition_phys_valid(paddr, size)); -#if defined(VERBOSE) && VERBOSE - assert(partition_phys_valid(paddr, size)); -#endif - - return (void *)((uintptr_t)paddr + HYP_ASPACE_PHYSACCESS_OFFSET); -#else -#error not implemented: locate allocator and convert paddr to virtual, or map -#endif + return (void *)((uintptr_t)paddr + hyp_aspace_get_physaccess_offset()); } void partition_phys_access_enable(const void *ptr) { -#if ARCH_AARCH64_USE_PAN (void)ptr; +#if ARCH_AARCH64_USE_PAN __asm__ volatile("msr PAN, 0" ::: "memory"); #else // Nothing to do here. @@ -110,8 +137,8 @@ partition_phys_access_enable(const void *ptr) void partition_phys_access_disable(const void *ptr) { -#if ARCH_AARCH64_USE_PAN (void)ptr; +#if ARCH_AARCH64_USE_PAN __asm__ volatile("msr PAN, 1" ::: "memory"); #else // Nothing to do here. @@ -128,6 +155,10 @@ partition_phys_unmap(const void *vaddr, paddr_t paddr, size_t size) // Nothing to do here. #else -#error not implemented: unmap if mapped above + (void)vaddr; + (void)paddr; + (void)size; + + // Nothing to do here. #endif } diff --git a/hyp/core/partition_standard/hypercalls.hvc b/hyp/core/partition_standard/hypercalls.hvc index 7b250cd..c722a98 100644 --- a/hyp/core/partition_standard/hypercalls.hvc +++ b/hyp/core/partition_standard/hypercalls.hvc @@ -6,7 +6,7 @@ define partition_create_partition hypercall { call_num 0x1; src_partition input type cap_id_t; cspace input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; new_cap output type cap_id_t; }; @@ -18,7 +18,7 @@ define partition_create_cspace hypercall { src_partition input type cap_id_t; cspace input type cap_id_t; // cspace size - _res0 input uregister; + res0 input uregister; error output enumeration error; new_cap output type cap_id_t; }; @@ -27,7 +27,7 @@ define partition_create_addrspace hypercall { call_num 0x3; src_partition input type cap_id_t; cspace input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; new_cap output type cap_id_t; }; @@ -37,7 +37,7 @@ define partition_create_memextent hypercall { src_partition input type cap_id_t; cspace input type cap_id_t; // base, size - _res0 input uregister; + res0 input uregister; error output enumeration error; new_cap output type cap_id_t; }; @@ -46,7 +46,7 @@ define partition_create_thread hypercall { call_num 0x5; src_partition input type cap_id_t; cspace input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; new_cap output type cap_id_t; }; @@ -55,7 +55,7 @@ define partition_create_doorbell hypercall { call_num 0x6; src_partition input type cap_id_t; cspace input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; new_cap output type cap_id_t; }; @@ -65,7 +65,7 @@ define partition_create_msgqueue hypercall { src_partition input type cap_id_t; cspace input type cap_id_t; // create info - _res0 input uregister; + res0 input uregister; error output enumeration error; new_cap output type cap_id_t; }; @@ -75,7 +75,7 @@ define partition_create_watchdog hypercall { call_num 0x9; src_partition input type cap_id_t; cspace input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; new_cap output type cap_id_t; }; @@ -85,7 +85,7 @@ define partition_create_vic hypercall { call_num 0xa; src_partition input type cap_id_t; cspace input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; new_cap output type cap_id_t; }; @@ -94,7 +94,7 @@ define partition_create_vpm_group hypercall { call_num 0xb; src_partition input type cap_id_t; cspace input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; new_cap output type cap_id_t; }; @@ -104,7 +104,7 @@ define partition_create_virtio_mmio hypercall { call_num 0x48; src_partition input type cap_id_t; cspace input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; new_cap output type cap_id_t; }; diff --git a/hyp/core/partition_standard/partition.ev b/hyp/core/partition_standard/partition.ev index 1c6cab1..f56571b 100644 --- a/hyp/core/partition_standard/partition.ev +++ b/hyp/core/partition_standard/partition.ev @@ -7,6 +7,12 @@ module partition_standard subscribe boot_cold_init() priority first +subscribe boot_cold_init + handler partition_standard_boot_add_private_heap() + // This must run after hyp_aspace_handle_boot_cold_init (priority 20) + // and memdb_handle_boot_cold_init (priority 10) + priority 9 + subscribe boot_hypervisor_start() // Run this early since it can add more heap memory priority 100 diff --git a/hyp/core/partition_standard/partition.tc b/hyp/core/partition_standard/partition.tc index dd20fa6..4e58c01 100644 --- a/hyp/core/partition_standard/partition.tc +++ b/hyp/core/partition_standard/partition.tc @@ -23,6 +23,7 @@ extend partition object { extend cap_rights_partition bitfield { 0 object_create bool; + 1 donate bool; }; define partition_option_flags bitfield<64> { diff --git a/hyp/core/partition_standard/src/init.c b/hyp/core/partition_standard/src/init.c index 937445f..bfe1a51 100644 --- a/hyp/core/partition_standard/src/init.c +++ b/hyp/core/partition_standard/src/init.c @@ -24,6 +24,7 @@ #include #include +#include #include #include @@ -34,6 +35,7 @@ static partition_t hyp_partition; static partition_t *root_partition; extern const char image_virt_start; +extern const char image_virt_last; extern const char image_phys_start; extern const char image_phys_last; @@ -41,6 +43,14 @@ static const uintptr_t virt_start = (uintptr_t)&image_virt_start; static const paddr_t phys_start = (paddr_t)&image_phys_start; static const paddr_t phys_last = (paddr_t)&image_phys_last; +#if defined(ARCH_ARM) && ARCH_IS_64BIT +// Ensure hypervisor is 2MiB page size aligned to use AArch64 2M block mappings +static_assert((PLATFORM_RW_DATA_SIZE & 0x1fffffU) == 0U, + "PLATFORM_RW_DATA_SIZE must be 2MB aligned"); +static_assert((PLATFORM_HEAP_PRIVATE_SIZE & 0x1fffffU) == 0U, + "PLATFORM_HEAP_PRIVATE_SIZE must be 2MB aligned"); +#endif + void NOINLINE partition_standard_handle_boot_cold_init(void) { @@ -49,16 +59,21 @@ partition_standard_handle_boot_cold_init(void) hyp_partition.header.type = OBJECT_TYPE_PARTITION; atomic_store_release(&hyp_partition.header.state, OBJECT_STATE_ACTIVE); + paddr_t hyp_heap_end = + (phys_last + 1U) - ((size_t)PLATFORM_RW_DATA_SIZE - + (size_t)PLATFORM_HEAP_PRIVATE_SIZE); // Add hypervisor memory as a mapped range. hyp_partition.mapped_ranges[0].virt = virt_start; hyp_partition.mapped_ranges[0].phys = phys_start; hyp_partition.mapped_ranges[0].size = - (size_t)(phys_last - phys_start + 1U); + (size_t)(hyp_heap_end - phys_start); // Allocate management structures for the hypervisor allocator. - allocator_init(&hyp_partition.allocator); + if (allocator_init(&hyp_partition.allocator) != OK) { + panic("allocator_init() failed for hyp partition"); + } - // Configure partition to be priviledged + // Configure partition to be privileged partition_option_flags_set_privileged(&hyp_partition.options, true); // Get remaining boot memory and assign it to hypervisor allocator. @@ -68,10 +83,35 @@ partition_standard_handle_boot_cold_init(void) panic("no boot mem"); } - error_t err = allocator_heap_add_memory(&hyp_partition.allocator, ret.r, - hyp_alloc_size); + paddr_t phys = partition_virt_to_phys(&hyp_partition, (uintptr_t)ret.r); + assert(phys != PADDR_INVALID); + + error_t err = trigger_allocator_add_ram_range_event( + &hyp_partition, phys, (uintptr_t)ret.r, hyp_alloc_size); if (err != OK) { - panic("Error passing on bootmem to hypervisor allocator"); + panic("Error moving bootmem to hyp_partition allocator"); + } +} + +void NOINLINE +partition_standard_boot_add_private_heap(void) +{ + // Only the first 2MiB of RW data was mapped in the assembly mmu_init. + // The remainder is mapped by hyp_aspace_handle_boot_cold_init. Because + // of this, the additional memory if any needs to be added to the + // hyp_partition allocator here. + if ((size_t)PLATFORM_HEAP_PRIVATE_SIZE > 0x200000U) { + size_t remaining_size = + (size_t)PLATFORM_HEAP_PRIVATE_SIZE - 0x200000U; + paddr_t remaining_phys = + (phys_last + 1U) - + ((size_t)PLATFORM_RW_DATA_SIZE - 0x200000U); + + error_t err = partition_add_heap(&hyp_partition, remaining_phys, + remaining_size); + if (err != OK) { + panic("Error expanding hyp_partition allocator"); + } } } diff --git a/hyp/core/partition_standard/src/partition.c b/hyp/core/partition_standard/src/partition.c index 2d2be84..f438232 100644 --- a/hyp/core/partition_standard/src/partition.c +++ b/hyp/core/partition_standard/src/partition.c @@ -14,12 +14,15 @@ #include #include #include +#include #include #include +#include + #include "event_handlers.h" -void_ptr_result_t +void_ptr_result_t NOINLINE partition_alloc(partition_t *partition, size_t bytes, size_t min_alignment) { void_ptr_result_t ret; @@ -29,6 +32,9 @@ partition_alloc(partition_t *partition, size_t bytes, size_t min_alignment) ret = allocator_allocate_object(&partition->allocator, bytes, min_alignment); + if (compiler_expected(ret.e == OK)) { + assert(ret.r != NULL); + } return ret; } @@ -37,21 +43,28 @@ partition_free(partition_t *partition, void *mem, size_t bytes) { error_t ret; assert((bytes > 0U) && !util_add_overflows((uintptr_t)mem, bytes - 1U)); - assert(partition_virt_to_phys(partition, (uintptr_t)mem) != ~0UL); + assert(partition_virt_to_phys(partition, (uintptr_t)mem) != + PADDR_INVALID); ret = allocator_deallocate_object(&partition->allocator, mem, bytes); return ret; } +// FIXME: partition->mapped_ranges is not updated atomically. Its not an issue +// yet since its only done during single-threaded init. Once we support dynamic +// heap adjustment, it will become a problem. +// FIXME: + static uintptr_t phys_to_virt(partition_t *partition, paddr_t phys, size_t size) { - uintptr_t virt = ~0UL; + uintptr_t virt = VADDR_INVALID; assert(!util_add_overflows(phys, size - 1U)); - for (count_t i = 0U; i < PARTITION_NUM_MAPPED_RANGE; i++) { + for (count_t i = 0U; i < util_array_size(partition->mapped_ranges); + i++) { partition_mapped_range_t *mr = &partition->mapped_ranges[i]; if (mr->size == 0U) { continue; @@ -71,7 +84,7 @@ partition_free_phys(partition_t *partition, paddr_t phys, size_t bytes) { uintptr_t virt = phys_to_virt(partition, phys, bytes); - if (virt == ~0UL) { + if (virt == VADDR_INVALID) { panic("Attempt to free memory not in partition"); } @@ -81,9 +94,10 @@ partition_free_phys(partition_t *partition, paddr_t phys, size_t bytes) paddr_t partition_virt_to_phys(partition_t *partition, uintptr_t addr) { - paddr_t phys = ~0UL; + paddr_t phys = PADDR_INVALID; - for (count_t i = 0U; i < PARTITION_NUM_MAPPED_RANGE; i++) { + for (count_t i = 0U; i < util_array_size(partition->mapped_ranges); + i++) { partition_mapped_range_t *mr = &partition->mapped_ranges[i]; if (mr->size == 0U) { continue; @@ -104,9 +118,7 @@ partition_standard_handle_object_create_partition(partition_create_t create) partition_t *partition = create.partition; assert(partition != NULL); - allocator_init(&partition->allocator); - - return OK; + return allocator_init(&partition->allocator); } error_t @@ -132,7 +144,7 @@ partition_standard_handle_object_activate_partition(partition_t *partition) // still be executing on other CPUs; this self-reference will be deleted // after that is done. This destruction operation is not yet // implemented. - object_get_partition_additional(partition); + (void)object_get_partition_additional(partition); err = OK; out: @@ -156,8 +168,8 @@ partition_mem_donate(partition_t *src_partition, paddr_t base, size_t size, partition_t *hyp_partition = partition_get_private(); - if ((size != 0U) && (!util_add_overflows(base, size - 1))) { - ret = memdb_update(hyp_partition, base, base + (size - 1), + if ((size != 0U) && (!util_add_overflows(base, size - 1U))) { + ret = memdb_update(hyp_partition, base, base + (size - 1U), (uintptr_t)dst_partition, MEMDB_TYPE_PARTITION, (uintptr_t)src_partition, @@ -179,8 +191,8 @@ partition_add_heap(partition_t *partition, paddr_t base, size_t size) partition_t *hyp_partition = partition_get_private(); - if ((size != 0U) && (!util_add_overflows(base, size - 1))) { - ret = memdb_update(hyp_partition, base, base + (size - 1), + if ((size != 0U) && (!util_add_overflows(base, size - 1U))) { + ret = memdb_update(hyp_partition, base, base + (size - 1U), (uintptr_t)&partition->allocator, MEMDB_TYPE_ALLOCATOR, (uintptr_t)partition, MEMDB_TYPE_PARTITION); @@ -190,29 +202,99 @@ partition_add_heap(partition_t *partition, paddr_t base, size_t size) if (ret == OK) { uintptr_t virt = phys_to_virt(partition, base, size); - assert(virt != ~0UL); - ret = allocator_heap_add_memory(&partition->allocator, - (void *)virt, size); + assert(virt != VADDR_INVALID); + ret = trigger_allocator_add_ram_range_event(partition, base, + virt, size); } return ret; } +static error_t +new_memory_add(partition_t *partition, partition_t *hyp_partition, paddr_t phys, + size_t size) +{ + error_t ret = OK; + uintptr_t virt; + + partition_mapped_range_t *mr = NULL; + for (count_t i = 0U; i < util_array_size(partition->mapped_ranges); + i++) { + if (partition->mapped_ranges[i].size == 0U) { + mr = &partition->mapped_ranges[i]; + break; + } + } + + if (mr == NULL) { + ret = ERROR_NORESOURCES; + goto out; + } + + // Use large page size for virt-phys alignment. + paddr_t phys_align_base = + util_balign_down(phys, PGTABLE_HYP_LARGE_PAGE_SIZE); + size_t phys_align_offset = phys - phys_align_base; + size_t phys_align_size = phys_align_offset + size; + + virt_range_result_t vr = hyp_aspace_allocate(phys_align_size); + if (vr.e != OK) { + ret = vr.e; + goto out; + } + + virt = vr.r.base + phys_align_offset; + + pgtable_hyp_start(); + // FIXME: + ret = pgtable_hyp_map(hyp_partition, virt, size, phys, + PGTABLE_HYP_MEMTYPE_WRITEBACK, PGTABLE_ACCESS_RW, + VMSA_SHAREABILITY_INNER_SHAREABLE); + pgtable_hyp_commit(); + if (ret == OK) { + ret = trigger_allocator_add_ram_range_event(partition, phys, + virt, size); + } + if (ret != OK) { + // FIXME: + // This should unmap the failed range, freeing to the target + // partition and preserve the levels that were preallocated, + // followed by unmapping the preserved tables (if they are + // empty), freeing to the hyp_partition. + pgtable_hyp_start(); + pgtable_hyp_unmap(hyp_partition, virt, size, + PGTABLE_HYP_UNMAP_PRESERVE_NONE); + pgtable_hyp_commit(); + hyp_aspace_deallocate(partition, vr.r); + } else { + mr->virt = virt; + mr->phys = phys; + mr->size = size; + + LOG(DEBUG, INFO, + "added heap: partition {:#x}, virt {:#x}, phys {:#x}, size {:#x}", + (uintptr_t)partition, virt, phys, size); + } + +out: + return ret; +} + error_t partition_map_and_add_heap(partition_t *partition, paddr_t phys, size_t size) { - error_t ret; - error_t err = OK; - uintptr_t virt; + error_t ret; + error_t err = OK; assert(partition != NULL); assert(size != 0U); // This should not be called for memory already mapped. - if (phys_to_virt(partition, phys, size) != ~0UL) { + if (phys_to_virt(partition, phys, size) != VADDR_INVALID) { panic("Attempt to add memory already in partition"); } + // FIXME: // Mapping the partition should preallocate top page-table levels from // the hyp partition and then map with the target partition, but we // have a chicken-and-egg problem to solve: if the target partition has @@ -240,18 +322,46 @@ partition_map_and_add_heap(partition_t *partition, paddr_t phys, size_t size) goto out; } + spinlock_acquire(&partition->header.lock); + // Add a new mapped range for the memory. - partition_mapped_range_t *mr = NULL; - for (count_t i = 0U; i < PARTITION_NUM_MAPPED_RANGE; i++) { - if (partition->mapped_ranges[i].virt == 0U) { - mr = &partition->mapped_ranges[i]; + ret = new_memory_add(partition, hyp_partition, phys, size); + + spinlock_release(&partition->header.lock); + + if (ret != OK) { + err = memdb_update(hyp_partition, phys, phys + (size - 1U), + (uintptr_t)partition, MEMDB_TYPE_PARTITION, + (uintptr_t)&partition->allocator, + MEMDB_TYPE_ALLOCATOR); + if (err != OK) { + panic("Error updating memdb."); + } + } +out: + return ret; +} + +#if defined(PLATFORM_TRACE_STANDALONE_REGION) + +static error_t +new_memory_add_trace(partition_t *partition, paddr_t phys, size_t size, + partition_mapped_range_t **mr, uintptr_result_t *virt_ret) +{ + error_t ret = OK; + uintptr_t virt; + + for (count_t i = 0U; i < util_array_size(partition->mapped_ranges); + i++) { + if (partition->mapped_ranges[i].size == 0U) { + *mr = &partition->mapped_ranges[i]; break; } } - if (mr == NULL) { + if (*mr == NULL) { ret = ERROR_NORESOURCES; - goto memdb_revert; + goto out; } // Use large page size for virt-phys alignment. @@ -263,44 +373,75 @@ partition_map_and_add_heap(partition_t *partition, paddr_t phys, size_t size) virt_range_result_t vr = hyp_aspace_allocate(phys_align_size); if (vr.e != OK) { ret = vr.e; - goto memdb_revert; + goto out; } - virt = vr.r.base + phys_align_offset; - mr->virt = virt; - mr->phys = phys; - mr->size = size; + virt = vr.r.base + phys_align_offset; + (*mr)->virt = virt; + (*mr)->phys = phys; + (*mr)->size = size; pgtable_hyp_start(); - ret = pgtable_hyp_map(hyp_partition, virt, size, phys, + ret = pgtable_hyp_map(partition, virt, size, phys, PGTABLE_HYP_MEMTYPE_WRITEBACK, PGTABLE_ACCESS_RW, VMSA_SHAREABILITY_INNER_SHAREABLE); if (ret == OK) { pgtable_hyp_commit(); - ret = allocator_heap_add_memory(&partition->allocator, - (void *)virt, size); } else { - // This should unmap the failed range, freeing to the target - // partition and preserve the levels that were preallocated, - // followed by unmapping the preserved tables (if they are - // empty), freeing to the hyp_partition. - pgtable_hyp_unmap(hyp_partition, virt, size, + pgtable_hyp_unmap(partition, virt, size, PGTABLE_HYP_UNMAP_PRESERVE_NONE); pgtable_hyp_commit(); hyp_aspace_deallocate(partition, vr.r); } if (ret == OK) { + (*virt_ret).r = virt; LOG(DEBUG, INFO, - "added heap: partition {:#x}, virt {:#x}, phys {:#x}, size {:#x}", - (uintptr_t)partition, virt, phys, size); + "added trace: partition {:#x}, virt {:#x}, phys {:#x}, size {:#x}", + (uintptr_t)partition, (*virt_ret).r, phys, size); + } + +out: + return ret; +} + +uintptr_result_t +partition_map_and_add_trace(partition_t *partition, paddr_t phys, size_t size) +{ + error_t ret; + error_t err = OK; + uintptr_result_t virt_ret = { 0 }; + + assert(partition != NULL); + assert(size != 0U); + + if ((size == 0U) || (util_add_overflows(phys, size - 1U))) { + ret = ERROR_ARGUMENT_SIZE; + goto out; } -memdb_revert: + if (!util_is_baligned(phys, PGTABLE_HYP_PAGE_SIZE) || + !util_is_baligned(size, PGTABLE_HYP_PAGE_SIZE)) { + ret = ERROR_ARGUMENT_ALIGNMENT; + goto out; + } + + partition_t *hyp_partition = partition_get_private(); + ret = memdb_update(hyp_partition, phys, phys + (size - 1U), + (uintptr_t)NULL, MEMDB_TYPE_TRACE, + (uintptr_t)partition, MEMDB_TYPE_PARTITION); + if (ret != OK) { + goto out; + } + + // Add a new mapped range for the memory. + partition_mapped_range_t *mr = NULL; + + ret = new_memory_add_trace(partition, phys, size, &mr, &virt_ret); + if (ret != OK) { err = memdb_update(hyp_partition, phys, phys + (size - 1U), (uintptr_t)partition, MEMDB_TYPE_PARTITION, - (uintptr_t)&partition->allocator, - MEMDB_TYPE_ALLOCATOR); + (uintptr_t)NULL, MEMDB_TYPE_TRACE); if (err != OK) { panic("Error updating memdb."); } @@ -312,5 +453,7 @@ partition_map_and_add_heap(partition_t *partition, paddr_t phys, size_t size) } } out: - return ret; + virt_ret.e = ret; + return virt_ret; } +#endif diff --git a/hyp/core/partition_standard/templates/object.c.tmpl b/hyp/core/partition_standard/templates/object.c.tmpl index 1024f59..413752f 100644 --- a/hyp/core/partition_standard/templates/object.c.tmpl +++ b/hyp/core/partition_standard/templates/object.c.tmpl @@ -8,6 +8,7 @@ \#include \#include +\#include \#include \#include \#include @@ -36,6 +37,8 @@ partition_allocate_${o}(partition_t *parent, ${o}_create_t create) #if o == "thread" const size_t size = thread_size; const size_t align = thread_align; + + assert(size >= sizeof(${o}_t)); #else const size_t size = sizeof(${o}_t); const size_t align = alignof(${o}_t); @@ -46,9 +49,9 @@ partition_allocate_${o}(partition_t *parent, ${o}_create_t create) obj_ret = ${o}_ptr_result_error(alloc_ret.e); goto allocate_${o}_error; } - memset(alloc_ret.r, 0, size); ${o} = (${o}_t *)alloc_ret.r; + (void)memset_s(${o}, size, 0, size); refcount_init(&${o}->header.refcount); spinlock_init(&${o}->header.lock); @@ -84,9 +87,9 @@ partition_destroy_${o}(rcu_entry_t *entry) partition_t *parent = ${o}->header.partition; #if o == "thread" - partition_free(parent, ${o}, thread_size); + (void)partition_free(parent, ${o}, thread_size); #else - partition_free(parent, ${o}, sizeof(${o}_t)); + (void)partition_free(parent, ${o}, sizeof(${o}_t)); #end if object_put_partition(parent); diff --git a/hyp/core/power/power.ev b/hyp/core/power/power.ev index 5a0be3d..707e1ce 100644 --- a/hyp/core/power/power.ev +++ b/hyp/core/power/power.ev @@ -6,18 +6,39 @@ module power subscribe boot_cold_init priority 100 - -subscribe boot_cpu_cold_init + require_preempt_disabled subscribe boot_cpu_warm_init - priority 100 + priority first require_preempt_disabled -subscribe power_cpu_resume() +subscribe power_cpu_resume(was_poweroff) priority first + require_preempt_disabled subscribe power_cpu_suspend(state) priority last + require_preempt_disabled -subscribe power_cpu_offline - priority last +// This has higher priority than PSCI to avoid PSCI put the core into CPU_SUSPEND first +subscribe idle_yield + priority 100 + require_preempt_disabled + +subscribe timer_action[TIMER_ACTION_POWER_CPU_ON_RETRY] + handler power_handle_timer_action(timer) + require_preempt_disabled + +#if defined(MODULE_VM_ROOTVM) +subscribe rootvm_started() + require_preempt_disabled +#endif + +subscribe boot_hypervisor_handover + priority first + +#if defined(POWER_START_ALL_CORES) +subscribe boot_hypervisor_start + priority -1000 + require_preempt_disabled +#endif diff --git a/hyp/core/power/power.tc b/hyp/core/power/power.tc index c967936..0d231c8 100644 --- a/hyp/core/power/power.tc +++ b/hyp/core/power/power.tc @@ -2,10 +2,23 @@ // // SPDX-License-Identifier: BSD-3-Clause -define cpu_power_state enumeration(explicit) { - off = 0; // never booted - boot = 1; // boot CPU in cold boot - suspend = 2; - offline = 3; // hotplugged - online = 4; +define power_voting structure { + // Spinlock protecting the rest of this structure, and also the + // CPU's entry in the power state array. + lock structure spinlock; + // Number of votes to keep this core powered on. + vote_count type count_t; + // Timer that is enqueued when an ERROR_RETRY error is returned + // from platform_cpu_on(). + retry_timer structure timer(contained); + // Number of consecutive retries. + retry_count type count_t; }; + +// Action for the timer above. +extend timer_action enumeration { + power_cpu_on_retry; +}; + +define POWER_CPU_ON_RETRY_DELAY_NS constant type nanoseconds_t = 100000; +define MAX_CPU_ON_RETRIES constant type count_t = 2; diff --git a/hyp/core/power/src/power.c b/hyp/core/power/src/power.c index 260b3b4..f0b44aa 100644 --- a/hyp/core/power/src/power.c +++ b/hyp/core/power/src/power.c @@ -5,94 +5,391 @@ #include #include +#include + +#include +#include #include +#include +#include +#include #include +#include +#include +#include #include +#include #include #include #include "event_handlers.h" -CPULOCAL_DECLARE(cpu_power_state_t, cpu_power_state); +static ticks_t power_cpu_on_retry_delay_ticks; static spinlock_t power_system_lock; -static register_t power_system_online_cpus PROTECTED_BY(power_system_lock); +static BITMAP_DECLARE(PLATFORM_MAX_CORES, power_system_online_cpus) + PROTECTED_BY(power_system_lock); static platform_power_state_t power_system_suspend_state PROTECTED_BY(power_system_lock); -void -power_handle_boot_cold_init(cpu_index_t boot_cpu) +CPULOCAL_DECLARE_STATIC(power_voting_t, power_voting); + +// This is protected by the lock in the corresponding power_voting_t structure, +// but must remain a separate array because it is exposed in crash minidumps. +CPULOCAL_DECLARE_STATIC(cpu_power_state_t, power_state); + +const cpu_power_state_array_t * +power_get_cpu_states_for_debug(void) { - CPULOCAL_BY_INDEX(cpu_power_state, boot_cpu) = CPU_POWER_STATE_BOOT; - spinlock_init(&power_system_lock); + return &cpulocal_power_state; } void -power_handle_boot_cpu_cold_init(cpu_index_t cpu) +power_handle_boot_cold_init(cpu_index_t boot_cpu) { - spinlock_acquire(&power_system_lock); - power_system_online_cpus |= util_bit(cpu); - spinlock_release(&power_system_lock); + power_cpu_on_retry_delay_ticks = + timer_convert_ns_to_ticks(POWER_CPU_ON_RETRY_DELAY_NS); + assert(power_cpu_on_retry_delay_ticks != 0U); + + for (cpu_index_t cpu = 0U; cpu < PLATFORM_MAX_CORES; cpu++) { + spinlock_init(&CPULOCAL_BY_INDEX(power_voting, cpu).lock); + spinlock_acquire_nopreempt( + &CPULOCAL_BY_INDEX(power_voting, cpu).lock); + + timer_init_object( + &CPULOCAL_BY_INDEX(power_voting, cpu).retry_timer, + TIMER_ACTION_POWER_CPU_ON_RETRY); + CPULOCAL_BY_INDEX(power_voting, cpu).retry_count = 0U; + + // Initialize the boot CPU's vote count to 1 while booting to + // prevent the cpu going to suspend. This will be decremented + // once the rootvm setup is completed and the rootvm VCPU has + // voted to keep the boot core powered on. + CPULOCAL_BY_INDEX(power_voting, cpu).vote_count = + (cpu == boot_cpu) ? 1U : 0U; + + CPULOCAL_BY_INDEX(power_state, cpu) = + (cpu == boot_cpu) ? CPU_POWER_STATE_COLD_BOOT + : CPU_POWER_STATE_OFF; + + spinlock_release_nopreempt( + &CPULOCAL_BY_INDEX(power_voting, cpu).lock); + } + + spinlock_init(&power_system_lock); + + // FIXME: + spinlock_acquire_nopreempt(&power_system_lock); + bitmap_set(power_system_online_cpus, (index_t)boot_cpu); + spinlock_release_nopreempt(&power_system_lock); } void power_handle_boot_cpu_warm_init(void) { -#if PLATFORM_MAX_CORES > 1 - cpu_power_state_t state = CPULOCAL(cpu_power_state); - - assert(state != CPU_POWER_STATE_ONLINE); + spinlock_acquire_nopreempt(&CPULOCAL(power_voting).lock); + cpu_power_state_t state = CPULOCAL(power_state); - CPULOCAL(cpu_power_state) = CPU_POWER_STATE_ONLINE; + assert((state == CPU_POWER_STATE_COLD_BOOT) || + (state == CPU_POWER_STATE_STARTED) || + (state == CPU_POWER_STATE_SUSPEND)); + CPULOCAL(power_state) = CPU_POWER_STATE_ONLINE; - if (state == CPU_POWER_STATE_OFF) { + if (state == CPU_POWER_STATE_STARTED) { trigger_power_cpu_online_event(); } -#endif -} + spinlock_release_nopreempt(&CPULOCAL(power_voting).lock); -void -power_handle_power_cpu_offline(void) -{ - CPULOCAL(cpu_power_state) = CPU_POWER_STATE_OFFLINE; + // FIXME: + spinlock_acquire_nopreempt(&power_system_lock); + if (bitmap_empty(power_system_online_cpus, PLATFORM_MAX_CORES)) { + // STARTED could be seen due to a last-cpu-suspend/cpu_on race. + assert((state == CPU_POWER_STATE_STARTED) || + (state == CPU_POWER_STATE_SUSPEND)); + trigger_power_system_resume_event(power_system_suspend_state); + } + bitmap_set(power_system_online_cpus, (index_t)cpulocal_get_index()); + spinlock_release_nopreempt(&power_system_lock); } error_t power_handle_power_cpu_suspend(platform_power_state_t state) { - error_t err = OK; - register_t cpu_bit = util_bit(cpulocal_get_index()); + error_t err = OK; + cpu_index_t cpu_id = cpulocal_get_index(); - spinlock_acquire(&power_system_lock); - power_system_online_cpus &= ~cpu_bit; - if (power_system_online_cpus == 0U) { + // FIXME: + spinlock_acquire_nopreempt(&power_system_lock); + bitmap_clear(power_system_online_cpus, (index_t)cpu_id); + if (bitmap_empty(power_system_online_cpus, PLATFORM_MAX_CORES)) { power_system_suspend_state = state; err = trigger_power_system_suspend_event(state); if (err != OK) { - power_system_online_cpus |= cpu_bit; + bitmap_set(power_system_online_cpus, (index_t)cpu_id); } } - spinlock_release(&power_system_lock); + spinlock_release_nopreempt(&power_system_lock); if (err == OK) { - CPULOCAL(cpu_power_state) = CPU_POWER_STATE_SUSPEND; + spinlock_acquire_nopreempt(&CPULOCAL(power_voting).lock); + assert(CPULOCAL(power_state) == CPU_POWER_STATE_ONLINE); + CPULOCAL(power_state) = CPU_POWER_STATE_SUSPEND; + spinlock_release_nopreempt(&CPULOCAL(power_voting).lock); } return err; } void -power_handle_power_cpu_resume(void) +power_handle_power_cpu_resume(bool was_poweroff) { - register_t cpu_bit = util_bit(cpulocal_get_index()); + // A cpu that was warm booted updates its state in the cpu warm-boot + // event. + if (!was_poweroff) { + spinlock_acquire_nopreempt(&CPULOCAL(power_voting).lock); + assert(CPULOCAL(power_state) == CPU_POWER_STATE_SUSPEND); + CPULOCAL(power_state) = CPU_POWER_STATE_ONLINE; + spinlock_release_nopreempt(&CPULOCAL(power_voting).lock); - CPULOCAL(cpu_power_state) = CPU_POWER_STATE_ONLINE; + // FIXME: + spinlock_acquire_nopreempt(&power_system_lock); + if (bitmap_empty(power_system_online_cpus, + PLATFORM_MAX_CORES)) { + trigger_power_system_resume_event( + power_system_suspend_state); + } + bitmap_set(power_system_online_cpus, + (index_t)cpulocal_get_index()); + spinlock_release_nopreempt(&power_system_lock); + } else { + spinlock_acquire_nopreempt(&power_system_lock); + // power_system_online_cpus should be updated in the warm init + // event. + assert(!bitmap_empty(power_system_online_cpus, + PLATFORM_MAX_CORES)); + spinlock_release_nopreempt(&power_system_lock); + } +} - spinlock_acquire(&power_system_lock); - if (power_system_online_cpus == 0U) { - trigger_power_system_resume_event(power_system_suspend_state); +static error_t +power_try_cpu_on(power_voting_t *voting, cpu_index_t cpu) + REQUIRE_LOCK(voting->lock) +{ + error_t ret; + + if (!platform_cpu_exists(cpu)) { + ret = ERROR_ARGUMENT_INVALID; + goto out; + } + + cpu_power_state_t *state = &CPULOCAL_BY_INDEX(power_state, cpu); + if ((*state != CPU_POWER_STATE_OFF) && + (*state != CPU_POWER_STATE_OFFLINE)) { + // CPU has already been started, or didn't get to power off. + ret = OK; + goto out; + } + + ret = platform_cpu_on(cpu); + + if (ret == OK) { + // Mark the CPU as started so we don't call cpu_on twice. + *state = CPU_POWER_STATE_STARTED; + voting->retry_count = 0U; + goto out; + } else if ((ret == ERROR_RETRY) && + (voting->retry_count < MAX_CPU_ON_RETRIES)) { + // We are racing with a power-off, and it is too late to prevent + // the power-off completing. We need to wait until power-off is + // complete and then retry. Enqueue the retry timer, if it is + // not already queued. + if (!timer_is_queued(&voting->retry_timer)) { + timer_enqueue(&voting->retry_timer, + timer_get_current_timer_ticks() + + power_cpu_on_retry_delay_ticks); + } + + // If we're racing with power-off, that means the CPU is + // functional and the power-on should not fail, so report + // success to the caller. If the retry does fail, we panic. + ret = OK; + } else if (ret == ERROR_RETRY) { + // We ran out of retry attempts. + ret = ERROR_FAILURE; + } else { + // platform_cpu_on() failed and cannot be retried; just return + // the error status. + } + +out: + return ret; +} + +error_t +power_vote_cpu_on(cpu_index_t cpu) +{ + error_t ret; + + assert(cpulocal_index_valid(cpu)); + power_voting_t *voting = &CPULOCAL_BY_INDEX(power_voting, cpu); + + spinlock_acquire(&voting->lock); + if (voting->vote_count == 0U) { + ret = power_try_cpu_on(voting, cpu); + if (ret != OK) { + goto out; + } + } + + voting->vote_count++; + ret = OK; + +out: + spinlock_release(&voting->lock); + return ret; +} + +void +power_vote_cpu_off(cpu_index_t cpu) +{ + assert(cpulocal_index_valid(cpu)); + power_voting_t *voting = &CPULOCAL_BY_INDEX(power_voting, cpu); + + spinlock_acquire(&voting->lock); + assert(voting->vote_count > 0U); + voting->vote_count--; + + if (voting->vote_count == 0U) { + // Any outstanding retries can be cancelled. + voting->retry_count = 0U; + timer_dequeue(&voting->retry_timer); + + // Send an IPI to rerun the idle handlers in case the CPU + // is already idle in WFI or suspend. + ipi_one(IPI_REASON_IDLE, cpu); + } + spinlock_release(&voting->lock); +} + +idle_state_t +power_handle_idle_yield(bool in_idle_thread) +{ + idle_state_t idle_state = IDLE_STATE_IDLE; + + if (!in_idle_thread) { + goto out; + } + + if (rcu_has_pending_updates()) { + goto out; + } + + power_voting_t *voting = &CPULOCAL(power_voting); + spinlock_acquire_nopreempt(&voting->lock); + if (voting->vote_count == 0U) { + error_t err = OK; + + spinlock_acquire_nopreempt(&power_system_lock); + cpu_index_t cpu_id = cpulocal_get_index(); + bitmap_clear(power_system_online_cpus, (index_t)cpu_id); + if (bitmap_empty(power_system_online_cpus, + PLATFORM_MAX_CORES)) { + power_system_suspend_state = + (platform_power_state_t){ 0 }; + err = trigger_power_system_suspend_event( + power_system_suspend_state); + if (err != OK) { + bitmap_set(power_system_online_cpus, + (index_t)cpu_id); + } + } + spinlock_release_nopreempt(&power_system_lock); + + if (err == OK) { + assert(CPULOCAL(power_state) == CPU_POWER_STATE_ONLINE); + trigger_power_cpu_offline_event(); + CPULOCAL(power_state) = CPU_POWER_STATE_OFFLINE; + spinlock_release_nopreempt(&voting->lock); + + platform_cpu_off(); + + idle_state = IDLE_STATE_WAKEUP; + } else { + spinlock_release_nopreempt(&voting->lock); + } + } else { + spinlock_release_nopreempt(&voting->lock); } - power_system_online_cpus |= cpu_bit; - spinlock_release(&power_system_lock); + +out: + return idle_state; } + +bool +power_handle_timer_action(timer_t *timer) +{ + assert(timer != NULL); + + power_voting_t *voting = power_voting_container_of_retry_timer(timer); + cpu_index_t cpu = CPULOCAL_PTR_INDEX(power_voting, voting); + + spinlock_acquire_nopreempt(&voting->lock); + error_t ret = OK; + if (voting->vote_count > 0U) { + voting->retry_count++; + ret = power_try_cpu_on(voting, cpu); + } + spinlock_release_nopreempt(&voting->lock); + + if (ret != OK) { + panic("Failed to power on a CPU that was previously on"); + } + + return true; +} + +#if defined(MODULE_VM_ROOTVM) +// The Boot CPU power count is initialised to 1. Decrement the count after the +// root VM initialization. +void +power_handle_rootvm_started(void) +{ + power_vote_cpu_off(cpulocal_get_index()); +} +#endif + +void +power_handle_boot_hypervisor_handover(void) +{ + // Ensure the running core is the only core online. There is no easy way + // to do this race-free, but it doesn't really matter for our purpose. + count_t on_count = 0; + for (cpu_index_t cpu = 0U; cpu < PLATFORM_MAX_CORES; cpu++) { + cpu_power_state_t state = CPULOCAL_BY_INDEX(power_state, cpu); + if ((state != CPU_POWER_STATE_OFF) && + (state != CPU_POWER_STATE_OFFLINE)) { + on_count++; + } + } + + if (on_count != 1U) { + panic("Hypervisor hand-over requested with multiple CPUs on"); + } +} + +#if defined(POWER_START_ALL_CORES) +void +power_handle_boot_hypervisor_start(void) +{ + cpu_index_t boot_cpu = cpulocal_get_index(); + + for (cpu_index_t cpu = 0U; cpulocal_index_valid(cpu); cpu++) { + if (cpu == boot_cpu) { + continue; + } + + power_vote_cpu_on(cpu); + } +} +#endif diff --git a/hyp/core/preempt/preempt.ev b/hyp/core/preempt/preempt.ev index 86743e1..9cc1ff0 100644 --- a/hyp/core/preempt/preempt.ev +++ b/hyp/core/preempt/preempt.ev @@ -32,3 +32,4 @@ subscribe thread_exit_to_user subscribe scheduler_stop() priority first + acquire_preempt_disabled diff --git a/hyp/core/preempt/src/preempt.c b/hyp/core/preempt/src/preempt.c index 1722678..2608646 100644 --- a/hyp/core/preempt/src/preempt.c +++ b/hyp/core/preempt/src/preempt.c @@ -23,11 +23,22 @@ static _Thread_local count_t preempt_disable_count; static const count_t preempt_count_mask = - util_bit(PREEMPT_BITS_COUNT_MAX + 1) - 1; -static const count_t preempt_count_max = preempt_count_mask; -static const count_t preempt_cpu_init = util_bit(PREEMPT_BITS_CPU_INIT); -static const count_t preempt_in_interrupt = util_bit(PREEMPT_BITS_IN_INTERRUPT); -static const count_t preempt_abort_kernel = util_bit(PREEMPT_BITS_ABORT_KERNEL); + (count_t)util_bit((index_t)PREEMPT_BITS_COUNT_MAX + 1U) - 1U; +static const count_t preempt_count_max = preempt_count_mask; + +static_assert(PREEMPT_BITS_COUNT_MAX < PREEMPT_BITS_CPU_INIT, + "PREEMPT_BITS_CPU_INIT invalid"); +static_assert(PREEMPT_BITS_COUNT_MAX < PREEMPT_BITS_IN_INTERRUPT, + "PREEMPT_BITS_IN_INTERRUPT invalid"); +static_assert(PREEMPT_BITS_COUNT_MAX < PREEMPT_BITS_ABORT_KERNEL, + "PREEMPT_BITS_ABORT_KERNEL invalid"); + +static const count_t preempt_cpu_init = + (count_t)util_bit((index_t)PREEMPT_BITS_CPU_INIT); +static const count_t preempt_in_interrupt = + (count_t)util_bit((index_t)PREEMPT_BITS_IN_INTERRUPT); +static const count_t preempt_abort_kernel = + (count_t)util_bit((index_t)PREEMPT_BITS_ABORT_KERNEL); void preempt_handle_boot_cpu_early_init(void) LOCK_IMPL @@ -45,7 +56,7 @@ preempt_handle_boot_cpu_start(void) LOCK_IMPL } void -preempt_handle_thread_start() LOCK_IMPL +preempt_handle_thread_start(void) LOCK_IMPL { // Arrange for preemption to be enabled by the first // preempt_enable() call. @@ -68,7 +79,9 @@ preempt_handle_thread_entry_from_user(thread_entry_reason_t reason) } preempt_enable(); +#if defined(VERBOSE) && VERBOSE assert_preempt_enabled(); +#endif } void @@ -117,7 +130,7 @@ preempt_interrupt_dispatch(void) LOCK_IMPL // reschedule. scheduler_trigger(); } else { - scheduler_schedule(); + (void)scheduler_schedule(); } } @@ -139,7 +152,7 @@ preempt_enable_in_irq(void) LOCK_IMPL } bool -preempt_abort_dispatch(void) +preempt_abort_dispatch(void) LOCK_IMPL { preempt_disable_count |= preempt_in_interrupt; @@ -151,7 +164,7 @@ preempt_abort_dispatch(void) } void -preempt_handle_scheduler_stop(void) +preempt_handle_scheduler_stop(void) LOCK_IMPL { count_t old_count = preempt_disable_count; asm_interrupt_disable_acquire(&preempt_disable_count); diff --git a/hyp/core/rcu_bitmap/rcu.tc b/hyp/core/rcu_bitmap/rcu.tc index 4321174..4624998 100644 --- a/hyp/core/rcu_bitmap/rcu.tc +++ b/hyp/core/rcu_bitmap/rcu.tc @@ -36,6 +36,7 @@ extend rcu_entry structure { // We should pack the class in here so we can use a unified queue. + // FIXME: next pointer structure rcu_entry; }; @@ -61,6 +62,7 @@ extend ipi_reason enumeration { // A batch of RCU updates, from one grace period on one CPU. define rcu_batch structure { // We have a list head for each update class. + // FIXME: heads array(maxof(enumeration rcu_update_class) + 1) pointer structure rcu_entry; }; @@ -69,6 +71,7 @@ define rcu_batch structure { // Note that we use a uint32 bitmap here rather than the generic bitmap type // because we want to be able to pack this into 64 bits, so that 64-bit // machines can access it with atomic load and store accesses. +// FIXME: define rcu_grace_period structure(aligned(8)) { generation type count_t; cpu_bitmap uint32; diff --git a/hyp/core/rcu_bitmap/src/rcu_bitmap.c b/hyp/core/rcu_bitmap/src/rcu_bitmap.c index 02c2b9e..6a1e52c 100644 --- a/hyp/core/rcu_bitmap/src/rcu_bitmap.c +++ b/hyp/core/rcu_bitmap/src/rcu_bitmap.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -19,6 +20,8 @@ #include "event_handlers.h" +static_assert(PLATFORM_MAX_CORES <= 32U, "PLATFORM_MAX_CORES > 32"); + static rcu_state_t rcu_state; CPULOCAL_DECLARE_STATIC(rcu_cpu_state_t, rcu_state); @@ -67,7 +70,7 @@ rcu_bitmap_refresh_active(void) // loop, so the idle_yield event won't be rerun and the CPU // won't be deactivated. ipi_one(IPI_REASON_RESCHEDULE, cpu); - active_cpus &= ~util_bit(cpu); + active_cpus &= (uint32_t)(~util_bit(cpu)); } } @@ -89,7 +92,7 @@ rcu_enqueue(rcu_entry_t *rcu_entry, rcu_update_class_t rcu_update_class) cpu_index_t cpu = cpulocal_get_index(); rcu_cpu_state_t *my_state = &CPULOCAL_BY_INDEX(rcu_state, cpu); - rcu_batch_t *batch = &my_state->next_batch; + rcu_batch_t *batch = &my_state->next_batch; if (atomic_fetch_add_explicit(&my_state->update_count, 1U, memory_order_relaxed) == 0U) { @@ -126,7 +129,7 @@ rcu_bitmap_activate_cpu(void) REQUIRE_PREEMPT_DISABLED uint32_t cpu_bit = (uint32_t)util_bit(cpu); rcu_cpu_state_t *my_state = &CPULOCAL_BY_INDEX(rcu_state, cpu); - if (!my_state->is_active) { + if (compiler_unexpected(!my_state->is_active)) { // We're not in the active CPU set. Add ourselves. my_state->is_active = true; @@ -198,8 +201,8 @@ rcu_bitmap_deactivate_cpu(void) REQUIRE_PREEMPT_DISABLED // ensure that it is done after the end of any critical sections. // However, it does not need ordering relative to the quiesce below; // if it happens late then at worst we might get a redundant IPI. - atomic_fetch_and_explicit(&rcu_state.active_cpus, ~cpu_bit, - memory_order_relaxed); + (void)atomic_fetch_and_explicit(&rcu_state.active_cpus, ~cpu_bit, + memory_order_relaxed); // This sequential consistency fence matches the one in // rcu_bitmap_quiesce when a new grace period starts, to ensure that @@ -420,15 +423,15 @@ rcu_bitmap_notify(void) // Advance the batches bool waiting_updates = false; - for (rcu_update_class_t update_class = RCU_UPDATE_CLASS__MIN; - update_class <= RCU_UPDATE_CLASS__MAX; update_class++) { + ENUM_FOREACH(RCU_UPDATE_CLASS, update_class) + { // Ready batch should have been emptied by rcu_bitmap_update() assert(my_state->ready_batch.heads[update_class] == NULL); // Collect the heads to be shifted for this class - struct rcu_entry *waiting_head = + rcu_entry_t *waiting_head = my_state->waiting_batch.heads[update_class]; - struct rcu_entry *next_head = + rcu_entry_t *next_head = my_state->next_batch.heads[update_class]; // Trigger further batch processing if necessary @@ -478,26 +481,27 @@ rcu_bitmap_update(void) goto out; } - for (rcu_update_class_t update_class = RCU_UPDATE_CLASS__MIN; - update_class <= RCU_UPDATE_CLASS__MAX; update_class++) { - struct rcu_entry *entry = - my_state->ready_batch.heads[update_class]; + ENUM_FOREACH(RCU_UPDATE_CLASS, update_class) + { + rcu_entry_t *entry = my_state->ready_batch.heads[update_class]; my_state->ready_batch.heads[update_class] = NULL; while (entry != NULL) { // We must read the next pointer _before_ triggering // the update, in case the update handler frees the // object. - struct rcu_entry *next = entry->next; - status = rcu_update_status_union( - trigger_rcu_update_event(update_class, entry), - status); + rcu_entry_t *next = entry->next; + status = rcu_update_status_union( + trigger_rcu_update_event( + (rcu_update_class_t)update_class, + entry), + status); entry = next; update_count++; } } - if ((update_count != 0) && + if ((update_count != 0U) && (atomic_fetch_sub_explicit(&my_state->update_count, update_count, memory_order_relaxed) == update_count)) { (void)atomic_fetch_sub_explicit(&rcu_state.waiter_count, 1U, @@ -509,3 +513,25 @@ rcu_bitmap_update(void) out: return rcu_update_status_get_need_schedule(&status); } + +void +rcu_bitmap_handle_power_cpu_offline(void) +{ + // We shouldn't get here if there are any pending updates on this CPU. + // The power aggregation code should have checked this by calling + // rcu_has_pending_updates() before deciding to offline the core. + assert(atomic_load_relaxed(&CPULOCAL(rcu_state).update_count) == 0U); + + // Always deactivate & quiesce the CPU, even if RCU doesn't need to run + // at the moment. This is because the CPU might have been left active + // when the last update was run, and it won't be able to deactivate + // once it goes offline. + rcu_bitmap_deactivate_cpu(); +} + +bool +rcu_has_pending_updates(void) +{ + return compiler_unexpected(rcu_bitmap_should_run()) && + (atomic_load_relaxed(&CPULOCAL(rcu_state).update_count) != 0U); +} diff --git a/hyp/core/rcu_sync/src/rcu_sync.c b/hyp/core/rcu_sync/src/rcu_sync.c index 961d0c5..476e1e3 100644 --- a/hyp/core/rcu_sync/src/rcu_sync.c +++ b/hyp/core/rcu_sync/src/rcu_sync.c @@ -124,6 +124,8 @@ rcu_sync_handle_update(rcu_entry_t *entry) } #if defined(UNITTESTS) && UNITTESTS +#include + static _Atomic count_t rcu_sync_test_ready_count; static _Atomic bool rcu_sync_test_start_flag; static _Atomic bool rcu_sync_test_success_flag; @@ -136,9 +138,6 @@ rcu_sync_handle_tests_init(void) atomic_init(&rcu_sync_test_success_flag, false); } -static_assert((1U << ((PLATFORM_MAX_CORES - 1U) * 3U)) != 0U, - "Spin count will overflow"); - bool rcu_sync_handle_tests_start(void) { @@ -172,7 +171,9 @@ rcu_sync_handle_tests_start(void) } // Spin for a while to give rcu_sync() time to return early - for (count_t i = 0; i < (1U << (my_order * 3U)); i++) { + for (count_t i = 0; + i < util_bit((24 * (my_order + 1)) / PLATFORM_MAX_CORES); + i++) { asm_yield(); } diff --git a/hyp/core/scheduler_fprr/scheduler_fprr.ev b/hyp/core/scheduler_fprr/scheduler_fprr.ev index 9da5bbd..8d0a6ef 100644 --- a/hyp/core/scheduler_fprr/scheduler_fprr.ev +++ b/hyp/core/scheduler_fprr/scheduler_fprr.ev @@ -25,12 +25,12 @@ subscribe vcpu_expects_wakeup subscribe object_deactivate_thread -subscribe thread_context_switch_pre +subscribe thread_context_switch_pre(next) unwinder() priority first require_preempt_disabled -subscribe thread_context_switch_post +subscribe thread_context_switch_post(prev) priority last require_preempt_disabled @@ -43,6 +43,7 @@ subscribe timer_action[TIMER_ACTION_RESCHEDULE] subscribe rcu_update[RCU_UPDATE_CLASS_AFFINITY_CHANGED] handler scheduler_fprr_handle_affinity_change_update(entry) + require_preempt_disabled subscribe thread_killed // Run last so other handlers can prepare for the thread @@ -60,6 +61,7 @@ subscribe tests_init subscribe tests_start handler tests_scheduler_start() + require_preempt_disabled subscribe thread_get_entry_fn[THREAD_KIND_SCHED_TEST] handler sched_test_get_entry_fn diff --git a/hyp/core/scheduler_fprr/scheduler_fprr.tc b/hyp/core/scheduler_fprr/scheduler_fprr.tc index f640e00..9fcc2c7 100644 --- a/hyp/core/scheduler_fprr/scheduler_fprr.tc +++ b/hyp/core/scheduler_fprr/scheduler_fprr.tc @@ -8,12 +8,17 @@ define SCHEDULER_VARIANT constant enumeration scheduler_variant = 0x1; define SCHEDULER_NUM_PRIORITIES constant type priority_t = 64; define SCHEDULER_MIN_PRIORITY public constant type priority_t = 0; -define SCHEDULER_MAX_PRIORITY public constant type priority_t = SCHEDULER_NUM_PRIORITIES - 1; -define SCHEDULER_DEFAULT_PRIORITY public constant type priority_t = SCHEDULER_NUM_PRIORITIES / 2; - -define SCHEDULER_MAX_TIMESLICE public constant type nanoseconds_t = 100000000; // 100ms -define SCHEDULER_MIN_TIMESLICE public constant type nanoseconds_t = 100000; // 100us -define SCHEDULER_DEFAULT_TIMESLICE public constant type nanoseconds_t = 5000000; // 5ms +define SCHEDULER_MAX_PRIORITY public constant type priority_t = + SCHEDULER_NUM_PRIORITIES - 1; +define SCHEDULER_DEFAULT_PRIORITY public constant type priority_t = + SCHEDULER_NUM_PRIORITIES / 2; + +define SCHEDULER_MAX_TIMESLICE public constant type nanoseconds_t = + 100000000; // 100ms +define SCHEDULER_MIN_TIMESLICE public constant type nanoseconds_t = + 100000; // 100µs +define SCHEDULER_DEFAULT_TIMESLICE public constant type nanoseconds_t = + 5000000; // 5ms define SCHEDULER_NUM_BLOCK_BITS constant type index_t = maxof(enumeration scheduler_block) + 1; @@ -38,8 +43,7 @@ define scheduler structure { runqueue array(SCHEDULER_NUM_PRIORITIES) structure list; active_thread pointer object thread; timer structure timer; - timeout type ticks_t; - timeout_set bool; + schedtime type ticks_t; lock structure spinlock; }; @@ -85,7 +89,7 @@ extend rcu_update_class enumeration { #if defined(UNIT_TESTS) extend thread_kind enumeration { - sched_test; + sched_test = 2; }; define sched_test_op enumeration { diff --git a/hyp/core/scheduler_fprr/src/scheduler_fprr.c b/hyp/core/scheduler_fprr/src/scheduler_fprr.c index e2f48e9..7a4545a 100644 --- a/hyp/core/scheduler_fprr/src/scheduler_fprr.c +++ b/hyp/core/scheduler_fprr/src/scheduler_fprr.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -51,11 +52,13 @@ static_assert(SCHEDULER_BLOCK__MAX < BITMAP_WORD_BITS, "Scheduler block flags must fit in a register"); static ticks_t -get_target_timeout(thread_t *target) +get_target_timeout(scheduler_t *scheduler, thread_t *target) + REQUIRE_PREEMPT_DISABLED { + assert(scheduler != NULL); assert(target != NULL); - return target->scheduler_schedtime + target->scheduler_active_timeslice; + return scheduler->schedtime + target->scheduler_active_timeslice; } static void @@ -96,11 +99,13 @@ end_directed_yield(thread_t *target) } static bool -update_timeslice(thread_t *target, ticks_t curticks) +update_timeslice(scheduler_t *scheduler, thread_t *target, ticks_t curticks) + REQUIRE_PREEMPT_DISABLED { + assert(scheduler != NULL); assert(target != NULL); - ticks_t timeout = get_target_timeout(target); + ticks_t timeout = get_target_timeout(scheduler, target); bool expired = timeout <= curticks; if (expired) { @@ -145,7 +150,7 @@ remove_from_runqueue(scheduler_t *scheduler, thread_t *target) assert_preempt_disabled(); index_t i = SCHEDULER_MAX_PRIORITY - target->scheduler_priority; - list_t *list = &scheduler->runqueue[i]; + list_t *list = &scheduler->runqueue[i]; list_node_t *node = &target->scheduler_list_node; bool was_head = node == list_get_head(list); @@ -178,7 +183,7 @@ pop_runqueue_head(scheduler_t *scheduler, index_t i) } static bool -can_be_scheduled(thread_t *thread) REQUIRE_SCHEDULER_LOCK(thread) +can_be_scheduled(const thread_t *thread) REQUIRE_SCHEDULER_LOCK(thread) { assert_spinlock_held(&thread->scheduler_lock); @@ -204,10 +209,11 @@ scheduler_fprr_handle_boot_cold_init(void) } } - for (scheduler_block_t block = SCHEDULER_BLOCK__MIN; - block <= SCHEDULER_BLOCK__MAX; block++) { + ENUM_FOREACH(SCHEDULER_BLOCK, block) + { scheduler_block_properties_t props = - trigger_scheduler_get_block_properties_event(block); + trigger_scheduler_get_block_properties_event( + (scheduler_block_t)block); if (scheduler_block_properties_get_non_killable(&props)) { bitmap_set(non_killable_block_mask, block); } @@ -222,7 +228,8 @@ scheduler_fprr_handle_object_create_thread(thread_create_t thread_create) assert(thread != NULL); assert(atomic_load_relaxed(&thread->state) == THREAD_STATE_INIT); assert(!sched_state_get_init(&thread->scheduler_state)); - assert(!scheduler_is_runnable(thread)); + assert(!bitmap_empty(thread->scheduler_block_bits, + SCHEDULER_NUM_BLOCK_BITS)); spinlock_init(&thread->scheduler_lock); atomic_init(&thread->scheduler_active_affinity, CPU_INDEX_INVALID); @@ -273,7 +280,7 @@ scheduler_fprr_handle_object_activate_thread(thread_t *thread) #if defined(INTERFACE_VCPU) bool -scheduler_fprr_handle_vcpu_activate_thread(thread_t *thread, +scheduler_fprr_handle_vcpu_activate_thread(thread_t *thread, vcpu_option_flags_t options) { bool ret = false, pin = false; @@ -393,8 +400,8 @@ scheduler_fprr_handle_timer_reschedule(void) { assert_preempt_disabled(); - CPULOCAL(scheduler).timeout_set = false; scheduler_trigger(); + return true; } @@ -415,23 +422,23 @@ scheduler_fprr_handle_affinity_change_update(rcu_entry_t *entry) { rcu_update_status_t ret = rcu_update_status_default(); - thread_t *thread = thread_container_of_scheduler_rcu_entry(entry); + thread_t *thread = thread_container_of_scheduler_rcu_entry(entry); cpu_index_t prev_cpu, next_cpu; - scheduler_lock(thread); + scheduler_lock_nopreempt(thread); assert(scheduler_is_blocked(thread, SCHEDULER_BLOCK_AFFINITY_CHANGED)); prev_cpu = thread->scheduler_prev_affinity; next_cpu = thread->scheduler_affinity; - scheduler_unlock(thread); + scheduler_unlock_nopreempt(thread); trigger_scheduler_affinity_changed_sync_event(thread, prev_cpu, next_cpu); - scheduler_lock(thread); + scheduler_lock_nopreempt(thread); if (scheduler_unblock(thread, SCHEDULER_BLOCK_AFFINITY_CHANGED)) { rcu_update_status_set_need_schedule(&ret, true); } - scheduler_unlock(thread); + scheduler_unlock_nopreempt(thread); object_put_thread(thread); @@ -456,41 +463,28 @@ set_next_timeout(scheduler_t *scheduler, thread_t *target) } if (need_timeout) { - ticks_t timeout = get_target_timeout(target); - if (scheduler->timeout_set) { - // The timer only needs to be updated - // if the timeout actually changed. - if (timeout != scheduler->timeout) { - timer_update(&scheduler->timer, timeout); - scheduler->timeout = timeout; - } - } else { - timer_enqueue(&scheduler->timer, timeout); - scheduler->timeout = timeout; - scheduler->timeout_set = true; - } - } else if (scheduler->timeout_set) { - timer_dequeue(&scheduler->timer); - scheduler->timeout_set = false; + ticks_t timeout = get_target_timeout(scheduler, target); + timer_update(&scheduler->timer, timeout); } else { - // No timeout set, nothing to do. + timer_dequeue(&scheduler->timer); } } static thread_t * -get_next_target(scheduler_t *scheduler) REQUIRE_SPINLOCK(scheduler->lock) +get_next_target(scheduler_t *scheduler, ticks_t curticks) + REQUIRE_SPINLOCK(scheduler->lock) { assert(scheduler != NULL); assert_spinlock_held(&scheduler->lock); thread_t *prev = scheduler->active_thread; thread_t *target = prev; - ticks_t curticks = timer_get_current_timer_ticks(); bool timeslice_expired = false; index_t i; if (target != NULL) { - timeslice_expired = update_timeslice(target, curticks); + timeslice_expired = + update_timeslice(scheduler, target, curticks); } if (bitmap_ffs(scheduler->prio_bitmap, SCHEDULER_NUM_PRIORITIES, &i)) { @@ -508,13 +502,14 @@ get_next_target(scheduler_t *scheduler) REQUIRE_SPINLOCK(scheduler->lock) } if (target != NULL) { - target->scheduler_schedtime = curticks; - scheduler->active_thread = target; + scheduler->active_thread = target; } else { target = idle_thread(); scheduler->active_thread = NULL; } + scheduler->schedtime = curticks; + if ((prev != NULL) && (target != prev)) { add_to_runqueue(scheduler, prev, timeslice_expired); } @@ -527,7 +522,7 @@ can_yield_to(thread_t *yield_to) REQUIRE_SCHEDULER_LOCK(yield_to) { assert_preempt_disabled(); - thread_t *current = thread_get_self(); + thread_t *current = thread_get_self(); cpu_index_t cpu = cpulocal_get_index(); cpu_index_t affinity = yield_to->scheduler_affinity; bool yield = true; @@ -592,12 +587,19 @@ scheduler_schedule(void) while (must_schedule) { scheduler_t *scheduler = &CPULOCAL(scheduler); - thread_t *target; + ticks_t curticks = timer_get_current_timer_ticks(); + thread_t *current = thread_get_self(); + thread_t *target; rcu_read_start(); + trigger_scheduler_schedule_event(current, + CPULOCAL(yielded_from), + scheduler->schedtime, + curticks); + spinlock_acquire_nopreempt(&scheduler->lock); - target = get_next_target(scheduler); + target = get_next_target(scheduler, curticks); bool can_idle = bitmap_empty(scheduler->prio_bitmap, SCHEDULER_NUM_PRIORITIES); set_next_timeout(scheduler, target); @@ -607,7 +609,7 @@ scheduler_schedule(void) trigger_scheduler_selected_thread_event(target, &can_idle); - if (target == thread_get_self()) { + if (target == current) { rcu_read_finish(); trigger_scheduler_quiescent_event(); must_schedule = false; @@ -616,7 +618,8 @@ scheduler_schedule(void) // thread stops running. rcu_read_finish(); - if (compiler_expected(thread_switch_to(target) == OK)) { + if (compiler_expected( + thread_switch_to(target, curticks) == OK)) { switched = true; must_schedule = ipi_clear(IPI_REASON_RESCHEDULE); @@ -904,7 +907,7 @@ scheduler_fprr_handle_thread_context_switch_pre(thread_t *next) scheduler_lock_nopreempt(next); cpu_index_t affinity = next->scheduler_affinity; - thread_t *yielded_from = CPULOCAL(yielded_from); + thread_t *yielded_from = CPULOCAL(yielded_from); // The next thread's affinity could have changed between target // selection and now; it may have been blocked by or is already running @@ -978,11 +981,18 @@ scheduler_block(thread_t *thread, scheduler_block_t block) { TRACE(DEBUG, INFO, "scheduler: block {:#x}, reason: {:d}, others: {:#x}", - (uintptr_t)thread, block, thread->scheduler_block_bits[0]); + (uintptr_t)thread, (register_t)block, + thread->scheduler_block_bits[0]); assert_spinlock_held(&thread->scheduler_lock); assert(block <= SCHEDULER_BLOCK__MAX); - bitmap_set(thread->scheduler_block_bits, block); + + if (!bitmap_isset(thread->scheduler_block_bits, (index_t)block)) { + trigger_scheduler_blocked_event(thread, block, + can_be_scheduled(thread)); + } + + bitmap_set(thread->scheduler_block_bits, (index_t)block); if (sched_state_get_queued(&thread->scheduler_state) && !can_be_scheduled(thread)) { remove_thread_from_scheduler(thread); @@ -994,7 +1004,7 @@ scheduler_block_init(thread_t *thread, scheduler_block_t block) { assert(!sched_state_get_init(&thread->scheduler_state)); assert(block <= SCHEDULER_BLOCK__MAX); - bitmap_set(thread->scheduler_block_bits, block); + bitmap_set(thread->scheduler_block_bits, (index_t)block); } bool @@ -1004,8 +1014,11 @@ scheduler_unblock(thread_t *thread, scheduler_block_t block) assert_spinlock_held(&thread->scheduler_lock); assert(block <= SCHEDULER_BLOCK__MAX); bool was_blocked = !can_be_scheduled(thread); - bitmap_clear(thread->scheduler_block_bits, block); - bool need_schedule = was_blocked && can_be_scheduled(thread); + bool block_was_set = + bitmap_isset(thread->scheduler_block_bits, (index_t)block); + bitmap_clear(thread->scheduler_block_bits, (index_t)block); + bool now_runnable = can_be_scheduled(thread); + bool need_schedule = was_blocked && now_runnable; if (need_schedule) { assert(!sched_state_get_queued(&thread->scheduler_state)); @@ -1024,8 +1037,12 @@ scheduler_unblock(thread_t *thread, scheduler_block_t block) TRACE(DEBUG, INFO, "scheduler: unblock {:#x}, reason: {:d}, others: {:#x}, local run: {:d}", - (uintptr_t)thread, block, thread->scheduler_block_bits[0], - need_schedule); + (uintptr_t)thread, (register_t)block, + thread->scheduler_block_bits[0], (register_t)need_schedule); + + if (block_was_set) { + trigger_scheduler_unblocked_event(thread, block, now_runnable); + } return need_schedule; } @@ -1034,14 +1051,13 @@ bool scheduler_is_blocked(const thread_t *thread, scheduler_block_t block) { assert(block <= SCHEDULER_BLOCK__MAX); - return bitmap_isset(thread->scheduler_block_bits, block); + return bitmap_isset(thread->scheduler_block_bits, (index_t)block); } bool scheduler_is_runnable(const thread_t *thread) { - return bitmap_empty(thread->scheduler_block_bits, - SCHEDULER_NUM_BLOCK_BITS); + return can_be_scheduled(thread); } thread_t * @@ -1121,6 +1137,12 @@ scheduler_set_affinity(thread_t *thread, cpu_index_t target_cpu) goto out; } + err = trigger_scheduler_set_affinity_prepare_event(thread, prev_cpu, + target_cpu); + if (err != OK) { + goto out; + } + // Block the thread so affinity changes are serialised. We need to get // an additional reference to the thread, otherwise it may be deleted // prior to the completion of the affinity change. @@ -1189,8 +1211,12 @@ scheduler_set_priority(thread_t *thread, priority_t priority) assert_spinlock_held(&thread->scheduler_lock); - if ((priority < SCHEDULER_MIN_PRIORITY) || - (priority > SCHEDULER_MAX_PRIORITY)) { + // Verify the SCHEDULER_MIN_PRIORITY is configured other than 0 to add + // another check for the 'priority' variable. + static_assert(SCHEDULER_MIN_PRIORITY == 0U, + "zero minimum priority expected"); + + if ((priority > SCHEDULER_MAX_PRIORITY)) { err = ERROR_ARGUMENT_INVALID; goto out; } @@ -1272,11 +1298,13 @@ scheduler_fprr_handle_thread_killed(thread_t *thread) // scheduled to run, so there is nothing to do. } - scheduler_unlock(thread); + scheduler_unlock_nopreempt(thread); if (need_schedule) { scheduler_trigger(); } + + preempt_enable(); } void diff --git a/hyp/core/scheduler_fprr/src/scheduler_tests.c b/hyp/core/scheduler_fprr/src/scheduler_tests.c index 21e6562..f0109bf 100644 --- a/hyp/core/scheduler_fprr/src/scheduler_tests.c +++ b/hyp/core/scheduler_fprr/src/scheduler_tests.c @@ -37,6 +37,8 @@ static uintptr_t sched_test_stack_base; static uintptr_t sched_test_stack_end; static _Atomic uintptr_t sched_test_stack_alloc; +static _Atomic count_t sync_flag; + CPULOCAL_DECLARE_STATIC(_Atomic uint8_t, wait_flag); CPULOCAL_DECLARE_STATIC(thread_t *, test_thread); CPULOCAL_DECLARE_STATIC(count_t, test_passed_count); @@ -44,6 +46,7 @@ CPULOCAL_DECLARE_STATIC(_Atomic count_t, affinity_count); static thread_ptr_result_t create_thread(priority_t prio, cpu_index_t cpu, sched_test_op_t op) + REQUIRE_PREEMPT_DISABLED { thread_ptr_result_t ret; @@ -75,6 +78,17 @@ create_thread(priority_t prio, cpu_index_t cpu, sched_test_op_t op) return ret; } +static void +destroy_thread(thread_t *thread) +{ + // Wait for the thread to exit so subsequent tests do not race with it. + while (atomic_load_relaxed(&thread->state) != THREAD_STATE_EXITED) { + scheduler_yield_to(thread); + } + + object_put_thread(thread); +} + static void schedule_check_switched(thread_t *thread, bool switch_expected) { @@ -127,7 +141,7 @@ tests_scheduler_start(void) old = atomic_load_relaxed(&CPULOCAL(wait_flag)); assert(old == 1U); atomic_store_relaxed(&CPULOCAL(wait_flag), 0U); - object_put_thread(ret.r); + destroy_thread(ret.r); CPULOCAL(test_passed_count)++; // priority == default: switch on yield @@ -139,7 +153,7 @@ tests_scheduler_start(void) scheduler_yield(); } atomic_store_relaxed(&CPULOCAL(wait_flag), 0U); - object_put_thread(ret.r); + destroy_thread(ret.r); CPULOCAL(test_passed_count)++; // priority < default: switch on directed yield @@ -153,31 +167,27 @@ tests_scheduler_start(void) scheduler_yield_to(ret.r); } atomic_store_relaxed(&CPULOCAL(wait_flag), 0U); - object_put_thread(ret.r); + destroy_thread(ret.r); CPULOCAL(test_passed_count)++; // Test 2: wait for timeslice expiry - ret = create_thread(SCHEDULER_MIN_PRIORITY, cpulocal_get_index(), + ret = create_thread(SCHEDULER_DEFAULT_PRIORITY, cpulocal_get_index(), SCHED_TEST_OP_WAKE); assert(ret.e == OK); - // Yield to reset the current thread's timeslice. + // Yield to reset the current thread's timeslice, then wait for the + // other thread to run and update the wait flag. scheduler_yield(); - - scheduler_lock(ret.r); - scheduler_set_priority(ret.r, SCHEDULER_DEFAULT_PRIORITY); - scheduler_unlock(ret.r); - - // To ensure a timeout is set, we need to reschedule. The timeslice - // shouldn't have expired yet due to the earlier yield. - schedule_check_switched(ret.r, false); - - while (asm_event_load_before_wait(&CPULOCAL(wait_flag)) == 0U) { - asm_event_wait(&CPULOCAL(wait_flag)); + _Atomic uint8_t *wait_flag = &CPULOCAL(wait_flag); + atomic_store_relaxed(wait_flag, 1U); + preempt_enable(); + while (asm_event_load_before_wait(wait_flag) == 1U) { + asm_event_wait(wait_flag); } + preempt_disable(); - asm_event_store_and_wake(&CPULOCAL(wait_flag), 0U); - object_put_thread(ret.r); + assert(atomic_load_relaxed(&CPULOCAL(wait_flag)) == 0U); + destroy_thread(ret.r); CPULOCAL(test_passed_count)++; // Test 3: double directed yield @@ -198,8 +208,8 @@ tests_scheduler_start(void) } atomic_store_relaxed(&CPULOCAL(wait_flag), 0U); - object_put_thread(ret.r); - object_put_thread(CPULOCAL(test_thread)); + destroy_thread(ret.r); + destroy_thread(CPULOCAL(test_thread)); CPULOCAL(test_passed_count)++; #if SCHEDULER_CAN_MIGRATE @@ -213,17 +223,22 @@ tests_scheduler_start(void) schedule_check_switched(ret.r, false); CPULOCAL(test_thread) = thread_get_self(); - scheduler_lock(ret.r); + scheduler_lock_nopreempt(ret.r); err = scheduler_set_affinity(ret.r, cpulocal_get_index()); - scheduler_unlock(ret.r); + scheduler_unlock_nopreempt(ret.r); assert(err == OK); schedule_check_switched(ret.r, true); scheduler_yield_to(ret.r); - object_put_thread(ret.r); + destroy_thread(ret.r); CPULOCAL(test_passed_count)++; + (void)atomic_fetch_add_explicit(&sync_flag, 1U, memory_order_relaxed); + while (asm_event_load_before_wait(&sync_flag) < PLATFORM_MAX_CORES) { + asm_event_wait(&sync_flag); + } + // Test 5: migrate running thread ret = create_thread(SCHEDULER_DEFAULT_PRIORITY, cpulocal_get_index(), SCHED_TEST_OP_AFFINITY); @@ -232,15 +247,24 @@ tests_scheduler_start(void) while (atomic_load_relaxed(&CPULOCAL(affinity_count)) < NUM_AFFINITY_SWITCH) { scheduler_yield(); - scheduler_lock(ret.r); + scheduler_lock_nopreempt(ret.r); cpu_index_t affinity = (scheduler_get_affinity(ret.r) + 1U) % PLATFORM_MAX_CORES; err = scheduler_set_affinity(ret.r, affinity); - scheduler_unlock(ret.r); + scheduler_unlock_nopreempt(ret.r); assert((err == OK) || (err == ERROR_RETRY)); } - object_put_thread(ret.r); + // Ensure the thread is running on the current CPU so we can yield to it + // and ensure it exits. + do { + scheduler_lock_nopreempt(ret.r); + err = scheduler_set_affinity(ret.r, cpulocal_get_index()); + scheduler_unlock_nopreempt(ret.r); + assert((err == OK) || (err == ERROR_RETRY)); + } while (err == ERROR_RETRY); + + destroy_thread(ret.r); CPULOCAL(test_passed_count)++; #endif @@ -248,9 +272,9 @@ tests_scheduler_start(void) } static void -sched_test_thread_entry(uintptr_t param) EXCLUDE_PREEMPT_DISABLED +sched_test_thread_entry(uintptr_t param) { - assert_preempt_enabled(); + cpulocal_begin(); sched_test_param_t test_param = sched_test_param_cast((uint32_t)param); sched_test_op_t op = sched_test_param_get_op(&test_param); @@ -260,12 +284,16 @@ sched_test_thread_entry(uintptr_t param) EXCLUDE_PREEMPT_DISABLED (void)atomic_fetch_add_explicit(&CPULOCAL(wait_flag), 1U, memory_order_relaxed); break; - case SCHED_TEST_OP_WAKE: - asm_event_store_and_wake(&CPULOCAL(wait_flag), 1U); - while (asm_event_load_before_wait(&CPULOCAL(wait_flag)) == 1U) { - asm_event_wait(&CPULOCAL(wait_flag)); + case SCHED_TEST_OP_WAKE: { + _Atomic uint8_t *wait_flag = &CPULOCAL(wait_flag); + cpulocal_end(); + while (asm_event_load_before_wait(wait_flag) == 0U) { + asm_event_wait(wait_flag); } + asm_event_store_and_wake(wait_flag, 0U); + cpulocal_begin(); break; + } case SCHED_TEST_OP_YIELDTO: while (atomic_load_relaxed(&CPULOCAL(wait_flag)) == 1U) { scheduler_yield_to(CPULOCAL(test_thread)); @@ -285,6 +313,8 @@ sched_test_thread_entry(uintptr_t param) EXCLUDE_PREEMPT_DISABLED default: panic("Invalid param for sched test thread!"); } + + cpulocal_end(); } thread_func_t diff --git a/hyp/core/scheduler_trivial/src/scheduler_trivial.c b/hyp/core/scheduler_trivial/src/scheduler_trivial.c index f05bc5f..557275f 100644 --- a/hyp/core/scheduler_trivial/src/scheduler_trivial.c +++ b/hyp/core/scheduler_trivial/src/scheduler_trivial.c @@ -205,6 +205,13 @@ scheduler_block(thread_t *thread, scheduler_block_t block) assert_preempt_disabled(); assert(block <= SCHEDULER_BLOCK__MAX); + bool block_was_set = bitmap_isset(thread->scheduler_block_bits, block); + + if (!bitmap_isset(thread->scheduler_block_bits, block)) { + trigger_scheduler_blocked_event(thread, block, + scheduler_is_runnable(thread)); + } + bitmap_set(thread->scheduler_block_bits, block); } @@ -226,9 +233,10 @@ scheduler_unblock(thread_t *thread, scheduler_block_t block) { assert_preempt_disabled(); assert(block <= SCHEDULER_BLOCK__MAX); - bool was_blocked = bitmap_isset(thread->scheduler_block_bits, block); + bool block_was_set = bitmap_isset(thread->scheduler_block_bits, block); bitmap_clear(thread->scheduler_block_bits, block); - bool need_schedule = was_blocked && scheduler_is_runnable(thread); + bool now_runnable = scheduler_is_runnable(thread); + bool need_schedule = block_was_set && now_runnable; if (need_schedule) { cpu_index_t cpu = cpulocal_get_index(); @@ -244,6 +252,10 @@ scheduler_unblock(thread_t *thread, scheduler_block_t block) (uintptr_t)thread, block, thread->scheduler_block_bits[0], need_schedule); + if (block_was_set) { + trigger_scheduler_unblocked_event(thread, block, now_runnable); + } + return need_schedule; } @@ -294,8 +306,9 @@ scheduler_set_affinity(thread_t *thread, cpu_index_t target_cpu) { assert_preempt_disabled(); - error_t err = OK; - cpu_index_t prev_cpu = thread->scheduler_affinity; + error_t err = OK; + bool need_sync = false; + cpu_index_t prev_cpu = thread->scheduler_affinity; if (prev_cpu == target_cpu) { goto out; @@ -307,7 +320,13 @@ scheduler_set_affinity(thread_t *thread, cpu_index_t target_cpu) } thread->scheduler_affinity = target_cpu; - trigger_scheduler_affinity_changed_event(thread, prev_cpu, target_cpu); + err = trigger_scheduler_set_affinity_prepare_event(thread, prev_cpu, + target_cpu); + if (err != OK) { + goto out; + } + trigger_scheduler_affinity_changed_event(thread, prev_cpu, target_cpu, + &need_sync); out: return err; diff --git a/hyp/core/spinlock_ticket/include/spinlock_attrs.h b/hyp/core/spinlock_ticket/include/spinlock_attrs.h index 3fa2878..40da670 100644 --- a/hyp/core/spinlock_ticket/include/spinlock_attrs.h +++ b/hyp/core/spinlock_ticket/include/spinlock_attrs.h @@ -10,8 +10,7 @@ #define ACQUIRE_SPINLOCK(lock) ACQUIRE_LOCK(lock) ACQUIRE_PREEMPT_DISABLED #define ACQUIRE_SPINLOCK_NP(lock) ACQUIRE_LOCK(lock) REQUIRE_PREEMPT_DISABLED #define TRY_ACQUIRE_SPINLOCK(success, lock) \ - TRY_ACQUIRE_LOCK(success, lock) \ - TRY_ACQUIRE_PREEMPT_DISABLED(success) + TRY_ACQUIRE_LOCK(success, lock) TRY_ACQUIRE_PREEMPT_DISABLED(success) #define TRY_ACQUIRE_SPINLOCK_NP(success, lock) \ TRY_ACQUIRE_LOCK(success, lock) REQUIRE_PREEMPT_DISABLED #define RELEASE_SPINLOCK(lock) RELEASE_LOCK(lock) RELEASE_PREEMPT_DISABLED diff --git a/hyp/core/spinlock_ticket/spinlock.tc b/hyp/core/spinlock_ticket/spinlock.tc index 63baf9a..70d6092 100644 --- a/hyp/core/spinlock_ticket/spinlock.tc +++ b/hyp/core/spinlock_ticket/spinlock.tc @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: BSD-3-Clause -define spinlock structure(lockable) { +define spinlock structure(lockable, aligned(4)) { now_serving uint16(atomic); next_ticket uint16(atomic); }; diff --git a/hyp/core/spinlock_ticket/src/spinlock_ticket.c b/hyp/core/spinlock_ticket/src/spinlock_ticket.c index bce0f0f..b3cbf8d 100644 --- a/hyp/core/spinlock_ticket/src/spinlock_ticket.c +++ b/hyp/core/spinlock_ticket/src/spinlock_ticket.c @@ -105,7 +105,7 @@ spinlock_release_nopreempt(spinlock_t *lock) LOCK_IMPL } void -assert_spinlock_held(spinlock_t *lock) +assert_spinlock_held(const spinlock_t *lock) { assert_preempt_disabled(); trigger_spinlock_assert_held_event(lock); diff --git a/hyp/core/tests/build.conf b/hyp/core/tests/build.conf new file mode 100644 index 0000000..c563236 --- /dev/null +++ b/hyp/core/tests/build.conf @@ -0,0 +1,10 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +interface tests +types tests.tc +events tests.ev +source tests.c +source spinlock_tests.c +source print_version.c diff --git a/hyp/core/tests/src/print_version.c b/hyp/core/tests/src/print_version.c new file mode 100644 index 0000000..40787df --- /dev/null +++ b/hyp/core/tests/src/print_version.c @@ -0,0 +1,38 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include + +#include + +#include +#include +#include +#include + +#include + +#include "event_handlers.h" + +extern const char hypervisor_version[]; +static const char *msg_ptr; + +void +test_print_hyp_version_init(void) +{ + msg_ptr = hypervisor_version; + return; +} + +error_t +test_print_hyp_version(tests_run_id_t test_id) + +{ + if (test_id == TESTS_RUN_ID_SMC_0) { + LOG(USER, TEST, "{:s}", (register_t)msg_ptr); + return OK; + } else { + return ERROR_UNIMPLEMENTED; + } +} diff --git a/hyp/core/tests/src/spinlock_tests.c b/hyp/core/tests/src/spinlock_tests.c new file mode 100644 index 0000000..7b57525 --- /dev/null +++ b/hyp/core/tests/src/spinlock_tests.c @@ -0,0 +1,170 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "event_handlers.h" + +#define TEST_ITERATIONS 100 + +extern test_info_t test_info; +test_info_t test_info; +extern test_info_t test_spinlock_multi_info; +test_info_t test_spinlock_multi_info; +extern test_info_t test_spinlock_multi_lock[PLATFORM_MAX_CORES]; +test_info_t test_spinlock_multi_lock[PLATFORM_MAX_CORES]; + +#if defined(UNIT_TESTS) +void +tests_spinlock_single_lock_init(void) +{ + // Initialize spinlock with boot cpu + spinlock_init(&test_info.lock); + test_info.count = 0; +} +#endif + +// Only one core can increment the count at the time. +bool +tests_spinlock_single_lock(void) +{ + bool ret = false; + bool wait_all_cores_start = true; + bool wait_all_cores_end = true; + + spinlock_acquire_nopreempt(&test_info.lock); + test_info.count++; + spinlock_release_nopreempt(&test_info.lock); + + // Wait until all cores have reached this point to start. + while (wait_all_cores_start) { + spinlock_acquire_nopreempt(&test_info.lock); + + if (!cpulocal_index_valid((cpu_index_t)test_info.count)) { + wait_all_cores_start = false; + } + + spinlock_release_nopreempt(&test_info.lock); + } + + for (count_t i = 0; i < TEST_ITERATIONS; i++) { + spinlock_acquire_nopreempt(&test_info.lock); + test_info.count++; + spinlock_release_nopreempt(&test_info.lock); + } + + // If test succeeds, count should be (TEST_ITERATIONS * + // PLATFORM_MAX_CORES) + PLATFORM_MAX_CORES + while (wait_all_cores_end) { + spinlock_acquire_nopreempt(&test_info.lock); + + if (test_info.count == ((TEST_ITERATIONS * PLATFORM_MAX_CORES) + + PLATFORM_MAX_CORES)) { + wait_all_cores_end = false; + } + + spinlock_release_nopreempt(&test_info.lock); + } + + return ret; +} + +#if defined(UNIT_TESTS) +void +tests_spinlock_multiple_locks_init(void) +{ + // Initialize spinlocks with boot cpu + spinlock_init(&test_spinlock_multi_info.lock); + test_spinlock_multi_info.count = 0; + + for (int i = 0; cpulocal_index_valid((cpu_index_t)i); i++) { + spinlock_init(&test_spinlock_multi_lock[i].lock); + test_spinlock_multi_lock[i].count = 0; + } +} +#endif + +// Dinning philosophers example +// Only philosophers that hold two forks can be eating at the same time. +// To avoid deadlock the odd and even numbers start picking differently. +bool +tests_spinlock_multiple_locks(void) +{ + bool ret = false; + bool wait_all_cores_start = true; + bool wait_all_cores_end = true; + + const cpu_index_t cpu = cpulocal_get_index(); + + spinlock_acquire_nopreempt(&test_spinlock_multi_info.lock); + test_spinlock_multi_info.count++; + spinlock_release_nopreempt(&test_spinlock_multi_info.lock); + + index_t left = cpu; + index_t right = cpu + 1; + + if (cpu == (PLATFORM_MAX_CORES - 1)) { + right = 0; + } + + // Wait until all cores have reached this point to start. + while (wait_all_cores_start) { + spinlock_acquire_nopreempt(&test_spinlock_multi_info.lock); + + if (test_spinlock_multi_info.count == PLATFORM_MAX_CORES) { + wait_all_cores_start = false; + } + + spinlock_release_nopreempt(&test_spinlock_multi_info.lock); + } + + for (count_t i = 0; i < TEST_ITERATIONS; i++) { + if ((cpu % 2) == 0U) { + spinlock_acquire_nopreempt( + &test_spinlock_multi_lock[left].lock); + spinlock_acquire_nopreempt( + &test_spinlock_multi_lock[right].lock); + } else { + spinlock_acquire_nopreempt( + &test_spinlock_multi_lock[right].lock); + spinlock_acquire_nopreempt( + &test_spinlock_multi_lock[left].lock); + } + + test_spinlock_multi_lock[left].count++; + test_spinlock_multi_lock[right].count++; + + spinlock_release_nopreempt( + &test_spinlock_multi_lock[left].lock); + spinlock_release_nopreempt( + &test_spinlock_multi_lock[right].lock); + } + + // If test succeeds, each fork count should be (2 * TEST_ITERATIONS) + while (wait_all_cores_end) { + spinlock_acquire_nopreempt( + &test_spinlock_multi_lock[left].lock); + + if (test_spinlock_multi_lock[left].count == + (2 * TEST_ITERATIONS)) { + wait_all_cores_end = false; + } + + spinlock_release_nopreempt( + &test_spinlock_multi_lock[left].lock); + } + + return ret; +} diff --git a/hyp/core/tests/src/string_test.c b/hyp/core/tests/src/string_test.c new file mode 100644 index 0000000..14ac3be --- /dev/null +++ b/hyp/core/tests/src/string_test.c @@ -0,0 +1,50 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include + +#include + +#include +#include +#include +#include +#include + +#include + +static uint8_t test_data_buff[512], test_buff[2048]; + +#define CPU_MEMCPY_STRIDE 256 + +void +memmove_test(void) +{ + uint16_t i; + uint8_t b = 0; + + for (i = 0; i < sizeof(test_data_buff); ++i) { + test_data_buff[i] = b; + test_buff[i] = b; + + ++b; + if (b == 251) { + b = 0; + } + } + + uint8_t *test_src, *test_dst; + + test_src = test_buff; + test_dst = test_src + CPU_MEMCPY_STRIDE + 1; + + memmove(test_dst, test_src, CPU_MEMCPY_STRIDE + 13); + + for (i = 0; i < (CPU_MEMCPY_STRIDE + 13); ++i) { + if (test_dst[i] != test_data_buff[i]) { + LOG(ERROR, WARN, "Err: pos {:d}, exp {:d}, act,{:d}\n", + i, test_dst[i], test_data_buff[i]); + } + } +} diff --git a/hyp/core/tests/src/tests.c b/hyp/core/tests/src/tests.c new file mode 100644 index 0000000..906787f --- /dev/null +++ b/hyp/core/tests/src/tests.c @@ -0,0 +1,177 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "event_handlers.h" + +static uintptr_t test_thread_stack_base; + +#if defined(UNIT_TESTS) +static thread_t * +tests_thread_create(cpu_index_t i) +{ + thread_create_t params = { + .scheduler_affinity = i, + .scheduler_affinity_valid = true, + .kind = THREAD_KIND_TEST, + .params = i, + }; + + thread_ptr_result_t ret = + partition_allocate_thread(partition_get_private(), params); + + if (ret.e != OK) { + panic("Unable to create test thread"); + } + + if (object_activate_thread(ret.r) != OK) { + panic("Error activating test thread"); + } + + return ret.r; +} +#endif + +error_t +tests_handle_object_create_thread(thread_create_t create) +{ + thread_t *thread = create.thread; + assert(thread != NULL); + + if (thread->kind == THREAD_KIND_TEST) { + scheduler_block_init(thread, SCHEDULER_BLOCK_TEST); + } + + return OK; +} + +#if defined(UNIT_TESTS) +static void +tests_add_root_partition_heap(void) +{ + // Grab some kernel heap from the hypervisor_partition and give it to + // the root partition allocator. + void_ptr_result_t ret; + size_t root_alloc_size = 0x20000; + + partition_t *hyp_partition = partition_get_private(); + partition_t *root_partition = partition_get_root(); + + ret = partition_alloc(hyp_partition, root_alloc_size, 4096U); + if (ret.e != OK) { + panic("Error allocating root partition heap"); + } + + paddr_t root_alloc_base = + partition_virt_to_phys(hyp_partition, (uintptr_t)ret.r); + + error_t err = memdb_update(hyp_partition, root_alloc_base, + root_alloc_base + (root_alloc_size - 1U), + (uintptr_t)root_partition, + MEMDB_TYPE_PARTITION, + (uintptr_t)&hyp_partition->allocator, + MEMDB_TYPE_ALLOCATOR); + if (err != OK) { + panic("Error adding root partition heap memory"); + } + + err = partition_map_and_add_heap(root_partition, root_alloc_base, + root_alloc_size); + if (err != OK) { + panic("Error mapping root partition heap memory"); + } +} +#endif + +static void +tests_alloc_stack_space(void) +{ + size_t aspace_size = THREAD_STACK_MAP_ALIGN * (PLATFORM_MAX_CORES + 1); + + virt_range_result_t stack_range = hyp_aspace_allocate(aspace_size); + if (stack_range.e != OK) { + panic("Unable to allocate address space for test thread stacks"); + } + + // Start the idle stack range at the next alignment boundary. + test_thread_stack_base = + util_balign_up(stack_range.r.base + 1U, THREAD_STACK_MAP_ALIGN); +} + +void +tests_thread_init(void) +{ +#if defined(UNIT_TESTS) + tests_add_root_partition_heap(); +#endif + tests_alloc_stack_space(); + + trigger_tests_init_event(); + +#if defined(UNIT_TESTS) + for (cpu_index_t i = 0; cpulocal_index_valid(i); i++) { + thread_t *thread = tests_thread_create(i); + scheduler_lock(thread); + scheduler_unblock(thread, SCHEDULER_BLOCK_TEST); + scheduler_unlock(thread); + + // The thread has a reference to itself now (until it exits); we + // don't need to hold onto it. + object_put_thread(thread); + } +#endif +} + +static void +tests_main(uintptr_t cpu_index) +{ + preempt_disable(); + if (trigger_tests_start_event()) { + panic("Tests are failing."); + } else { + LOG(DEBUG, INFO, "Tests completed successfully on CPU {:d}", + cpu_index); + } + preempt_enable(); +} + +thread_func_t +tests_handle_thread_get_entry_fn(thread_kind_t kind) +{ + assert(kind == THREAD_KIND_TEST); + + return tests_main; +} + +uintptr_t +tests_handle_thread_get_stack_base(thread_kind_t kind, thread_t *thread) +{ + assert(kind == THREAD_KIND_TEST); + assert(thread != NULL); + + cpu_index_t cpu = thread->scheduler_affinity; + + return test_thread_stack_base + (cpu * THREAD_STACK_MAP_ALIGN); +} diff --git a/hyp/core/tests/tests.ev b/hyp/core/tests/tests.ev new file mode 100644 index 0000000..a93c72f --- /dev/null +++ b/hyp/core/tests/tests.ev @@ -0,0 +1,39 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module tests + +subscribe boot_hypervisor_start + handler tests_thread_init() + priority -100 + +subscribe thread_get_stack_base[THREAD_KIND_TEST] + +#if defined (UNIT_TESTS) +subscribe tests_init + handler tests_spinlock_single_lock_init() +#endif + +subscribe tests_start + handler tests_spinlock_single_lock() + require_preempt_disabled + +#if defined (UNIT_TESTS) +subscribe tests_init + handler tests_spinlock_multiple_locks_init() +#endif + +subscribe tests_start + handler tests_spinlock_multiple_locks() + require_preempt_disabled + +subscribe thread_get_entry_fn[THREAD_KIND_TEST] + +subscribe object_create_thread + +subscribe tests_init + handler test_print_hyp_version_init() + +subscribe tests_run[TESTS_RUN_ID_SMC_0] + handler test_print_hyp_version(test_id) diff --git a/hyp/core/tests/tests.tc b/hyp/core/tests/tests.tc new file mode 100644 index 0000000..186105d --- /dev/null +++ b/hyp/core/tests/tests.tc @@ -0,0 +1,24 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +define test_info structure { + count type count_t; + lock structure spinlock; +}; + +extend thread_kind enumeration { + test; +}; + +extend scheduler_block enumeration { + test; +}; + +extend tests_run_id enumeration { + SMC_0 = 0x0; +}; + +extend trace_id enumeration { + TEST = 4; +}; diff --git a/hyp/core/thread_standard/aarch64/src/thread_arch.c b/hyp/core/thread_standard/aarch64/src/thread_arch.c index 3fc4711..940d0e7 100644 --- a/hyp/core/thread_standard/aarch64/src/thread_arch.c +++ b/hyp/core/thread_standard/aarch64/src/thread_arch.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -22,28 +23,37 @@ #include "event_handlers.h" #include "thread_arch.h" +typedef register_t (*fptr_t)(register_t arg); +typedef void (*fptr_noreturn_t)(register_t arg); + const size_t thread_stack_min_align = 16; const size_t thread_stack_alloc_align = PGTABLE_HYP_PAGE_SIZE; const size_t thread_stack_size_default = PGTABLE_HYP_PAGE_SIZE; -static uintptr_t -thread_get_tls_base(thread_t *thread) +static size_t +thread_get_tls_offset(void) { size_t offset = 0; __asm__("add %0, %0, :tprel_hi12:current_thread ;" "add %0, %0, :tprel_lo12_nc:current_thread ;" : "+r"(offset)); - return (uintptr_t)thread - offset; + return offset; +} + +static uintptr_t +thread_get_tls_base(thread_t *thread) +{ + return (uintptr_t)thread - thread_get_tls_offset(); } static noreturn void -thread_arch_main(thread_t *prev) LOCK_IMPL +thread_arch_main(thread_t *prev, ticks_t schedtime) LOCK_IMPL { thread_t *thread = thread_get_self(); trigger_thread_start_event(); - trigger_thread_context_switch_post_event(prev); + trigger_thread_context_switch_post_event(prev, schedtime, (ticks_t)0UL); object_put_thread(prev); thread_func_t thread_func = @@ -59,21 +69,22 @@ thread_arch_main(thread_t *prev) LOCK_IMPL } thread_t * -thread_arch_switch_thread(thread_t *next_thread) +thread_arch_switch_thread(thread_t *next_thread, ticks_t *schedtime) { - // Note: the old thread must be in X0 so that a switch to a new thread - // with its PC set to thread_arch_main() will get the old thread as - // its argument. - register thread_t *old __asm__("x0") = thread_get_self(); + // The previous thread and the scheduling time must be kept in X0 and X1 + // to ensure that thread_arch_main() receives them as arguments on the + // first context switch. + register thread_t *old __asm__("x0") = thread_get_self(); + register ticks_t ticks __asm__("x1") = *schedtime; // The remaining hard-coded registers here are only needed to ensure a // correct clobber list below. The union of the clobber list, hard-coded // registers and explicitly saved registers (x29, sp and pc) must be the // entire integer register state. - register register_t old_pc __asm__("x1"); - register register_t old_sp __asm__("x2"); - register register_t old_fp __asm__("x3"); - register uintptr_t old_context __asm__("x4") = + register register_t old_pc __asm__("x2"); + register register_t old_sp __asm__("x3"); + register register_t old_fp __asm__("x4"); + register uintptr_t old_context __asm__("x5") = (uintptr_t)&old->context.pc; static_assert(offsetof(thread_t, context.sp) == offsetof(thread_t, context.pc) + @@ -104,17 +115,21 @@ thread_arch_switch_thread(thread_t *next_thread) "str %[old_fp], [%[old_context], 16] ;" "br %[new_pc] ;" ".Lthread_continue.%=: ;" -#if defined(ARCH_ARM_8_5_BTI) +#if defined(ARCH_ARM_FEAT_BTI) "bti j ;" #endif - : [old] "+r"(old), [old_pc] "=&r"(old_pc), [old_sp] "=&r"(old_sp), - [old_fp] "=&r"(old_fp), [old_context] "+r"(old_context), - [new_pc] "+r"(new_pc), [new_sp] "+r"(new_sp), - [new_fp] "+r"(new_fp), [new_tls_base] "+r"(new_tls_base) + : [old] "+r"(old), [old_pc] "=&r"(old_pc), + [old_sp] "=&r"(old_sp), [old_fp] "=&r"(old_fp), + [old_context] "+r"(old_context), [new_pc] "+r"(new_pc), + [new_sp] "+r"(new_sp), [new_fp] "+r"(new_fp), + [new_tls_base] "+r"(new_tls_base), [ticks] "+r"(ticks) : /* This must not have any inputs */ - : "x5", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x17", - "x18", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", - "x27", "x28", "x30", "cc", "memory"); + : "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x17", "x18", + "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", + "x28", "x30", "cc", "memory"); + + // Update schedtime from the tick count passed by the previous thread + *schedtime = ticks; return old; } @@ -128,10 +143,14 @@ thread_arch_set_thread(thread_t *thread) assert(thread == thread_get_self()); assert(thread == idle_thread()); - // Note: the old thread must be in X0 so that a switch to a new thread - // with its PC set to thread_arch_main() will get the old thread as - // its argument. - register thread_t *old __asm__("x0") = thread; + // The previous thread and the scheduling time must be kept in X0 and X1 + // to ensure that thread_arch_main() receives them as arguments on the + // first context switch during CPU cold boot. The scheduling time is set + // to 0 because we consider the idle thread to have been scheduled at + // the epoch. These are unused on warm boot, which is always resuming a + // thread_freeze() call. + register thread_t *old __asm__("x0") = thread; + register ticks_t ticks __asm__("x1") = (ticks_t)0U; // The new PC must be in x16 or x17 so ARMv8.5-BTI will treat the BR // below as a call trampoline, and thus allow it to jump to the BTI C @@ -145,15 +164,15 @@ thread_arch_set_thread(thread_t *thread) "mov sp, %[new_sp] ;" "mov x29, %[new_fp] ;" "br %[new_pc] ;" - : [old] "+r"(old) - : [new_pc] "r"(new_pc), [new_sp] "r"(new_sp), [new_fp] "r"(new_fp) + : + : [old] "r"(old), [ticks] "r"(ticks), [new_pc] "r"(new_pc), + [new_sp] "r"(new_sp), [new_fp] "r"(new_fp) : "memory"); __builtin_unreachable(); } register_t -thread_freeze(register_t (*fn)(register_t), register_t param, - register_t resumed_result) +thread_freeze(fptr_t fn, register_t param, register_t resumed_result) { TRACE(DEBUG, INFO, "thread_freeze start fn: {:#x} param: {:#x}", (uintptr_t)fn, (uintptr_t)param); @@ -163,6 +182,8 @@ thread_freeze(register_t (*fn)(register_t), register_t param, thread_t *thread = thread_get_self(); assert(thread != NULL); + // The parameter must be kept in X0 so the freeze function gets it as an + // argument. register register_t x0 __asm__("x0") = param; // The remaining hard-coded registers here are only needed to @@ -173,8 +194,8 @@ thread_freeze(register_t (*fn)(register_t), register_t param, register register_t saved_sp __asm__("x2"); register uintptr_t context __asm__("x3") = (uintptr_t)&thread->context.pc; - register register_t (*fn_reg)(register_t) __asm__("x4") = fn; - register bool is_resuming __asm__("x5"); + register fptr_t fn_reg __asm__("x4") = fn; + register bool is_resuming __asm__("x5"); static_assert(offsetof(thread_t, context.sp) == offsetof(thread_t, context.pc) + @@ -194,7 +215,7 @@ thread_freeze(register_t (*fn)(register_t), register_t param, "mov %[is_resuming], 0 ;" "b .Lthread_freeze.done.%= ;" ".Lthread_freeze.resumed.%=: ;" -#if defined(ARCH_ARM_8_5_BTI) +#if defined(ARCH_ARM_FEAT_BTI) "bti j ;" #endif "mov %[is_resuming], 1 ;" @@ -220,9 +241,9 @@ thread_freeze(register_t (*fn)(register_t), register_t param, } noreturn void -thread_reset_stack(void (*fn)(register_t), register_t param) +thread_reset_stack(fptr_noreturn_t fn, register_t param) { - thread_t *thread = thread_get_self(); + thread_t *thread = thread_get_self(); register register_t x0 __asm__("x0") = param; uintptr_t new_sp = (uintptr_t)thread->stack_base + thread->stack_size; @@ -245,15 +266,6 @@ thread_arch_init_context(thread_t *thread) thread->context.fp = (uintptr_t)0; } -void -thread_standard_handle_boot_hypervisor_start(void) -{ - thread_t *thread = idle_thread(); - assert(thread != NULL); - - thread->context.pc = (uintptr_t)thread_arch_main; -} - error_t thread_arch_map_stack(thread_t *thread) { diff --git a/hyp/core/thread_standard/include/thread_arch.h b/hyp/core/thread_standard/include/thread_arch.h index b517dbc..1cc7e90 100644 --- a/hyp/core/thread_standard/include/thread_arch.h +++ b/hyp/core/thread_standard/include/thread_arch.h @@ -35,11 +35,17 @@ thread_arch_init_context(thread_t *thread); // specifying this thread as an argument. At that point, it returns a pointer to // the thread it switched from, which need not be the same as the thread that // the original call switched to. +// +// The tick counter is the time at which the specified thread was scheduled; +// it is passed through to the new thread's context switch handlers to avoid a +// double read of the timer which can cause gaps in the time accounting and +// might be expensive on some platforms. On return, it is updated to be the +// value passed through by the previous thread. thread_t * -thread_arch_switch_thread(thread_t *next_thread); +thread_arch_switch_thread(thread_t *next_thread, ticks_t *schedtime); // Set the current thread, assuming there was no previous current thread. // // This function is called at the end of the CPU power-on sequence. noreturn void -thread_arch_set_thread(thread_t *next_thread); +thread_arch_set_thread(thread_t *next_thread) REQUIRE_PREEMPT_DISABLED; diff --git a/hyp/core/thread_standard/src/init.c b/hyp/core/thread_standard/src/init.c index 2231d8b..d4eee88 100644 --- a/hyp/core/thread_standard/src/init.c +++ b/hyp/core/thread_standard/src/init.c @@ -41,7 +41,12 @@ thread_standard_handle_boot_runtime_first_init(void) // reference count. The real setup will be done in the idle module after // partitions and allocators are working. idle_thread = (thread_t *)ret.r; - memset(idle_thread, 0, thread_size); + + assert(thread_size >= sizeof(*idle_thread)); + errno_t err_mem = memset_s(idle_thread, thread_size, 0, thread_size); + if (err_mem != 0) { + panic("Error in memset_s operation!"); + } refcount_init(&idle_thread->header.refcount); // This must be the last operation in boot_runtime_first_init. diff --git a/hyp/core/thread_standard/src/thread.c b/hyp/core/thread_standard/src/thread.c index 3b93cbb..2d280fc 100644 --- a/hyp/core/thread_standard/src/thread.c +++ b/hyp/core/thread_standard/src/thread.c @@ -67,7 +67,7 @@ thread_standard_handle_object_create_thread(thread_create_t thread_create) #if !defined(NDEBUG) // Fill the stack with a pattern so we can detect maximum stack // depth - memset(stack.r, 0x57, stack_size); + (void)memset_s(stack.r, stack_size, 0x57, stack_size); #endif thread->stack_mem = (uintptr_t)stack.r; @@ -89,8 +89,9 @@ thread_standard_unwind_object_create_thread(error_t result, assert(atomic_load_relaxed(&thread->state) == THREAD_STATE_INIT); if (thread->stack_mem != 0U) { - partition_free(thread->header.partition, - (void *)thread->stack_mem, thread->stack_size); + (void)partition_free(thread->header.partition, + (void *)thread->stack_mem, + thread->stack_size); thread->stack_mem = 0U; } } @@ -151,8 +152,9 @@ thread_standard_handle_object_deactivate_thread(thread_t *thread) } if (thread->stack_mem != 0U) { - partition_free(thread->header.partition, - (void *)thread->stack_mem, thread->stack_size); + (void)partition_free(thread->header.partition, + (void *)thread->stack_mem, + thread->stack_size); thread->stack_mem = 0U; } } @@ -170,7 +172,7 @@ thread_get_self(void) } error_t -thread_switch_to(thread_t *thread) +thread_switch_to(thread_t *thread, ticks_t schedtime) { assert_preempt_disabled(); @@ -181,16 +183,18 @@ thread_switch_to(thread_t *thread) (uintptr_t)current, (uintptr_t)thread); trigger_thread_save_state_event(); - error_t err = trigger_thread_context_switch_pre_event(thread); + error_t err = + trigger_thread_context_switch_pre_event(thread, schedtime); if (compiler_unexpected(err != OK)) { object_put_thread(thread); goto out; } - thread_t *prev = thread_arch_switch_thread(thread); + ticks_t prevticks = schedtime; + thread_t *prev = thread_arch_switch_thread(thread, &schedtime); assert(prev != NULL); - trigger_thread_context_switch_post_event(prev); + trigger_thread_context_switch_post_event(prev, schedtime, prevticks); object_put_thread(prev); trigger_thread_load_state_event(false); @@ -205,13 +209,19 @@ thread_kill(thread_t *thread) { assert(thread != NULL); - error_t err = OK; + error_t err; thread_state_t expected_state = THREAD_STATE_READY; if (atomic_compare_exchange_strong_explicit( &thread->state, &expected_state, THREAD_STATE_KILLED, memory_order_relaxed, memory_order_relaxed)) { trigger_thread_killed_event(thread); + err = OK; + } else if ((expected_state == THREAD_STATE_KILLED) || + (expected_state == THREAD_STATE_EXITED)) { + // Thread was already killed, or has exited + err = OK; } else { + // Thread had not started yet err = ERROR_OBJECT_STATE; } @@ -225,6 +235,13 @@ thread_is_dying(const thread_t *thread) return atomic_load_relaxed(&thread->state) == THREAD_STATE_KILLED; } +bool +thread_has_exited(const thread_t *thread) +{ + assert(thread != NULL); + return atomic_load_relaxed(&thread->state) == THREAD_STATE_EXITED; +} + noreturn void thread_exit(void) { diff --git a/hyp/core/thread_standard/thread.ev b/hyp/core/thread_standard/thread.ev index 8f9a5b4..d849cd2 100644 --- a/hyp/core/thread_standard/thread.ev +++ b/hyp/core/thread_standard/thread.ev @@ -25,8 +25,5 @@ subscribe object_deactivate_thread // Add a dummy handler so the scheduler can always register an unwinder. subscribe thread_context_switch_pre() -subscribe boot_hypervisor_start - priority -20 - subscribe thread_exit_to_user() priority last diff --git a/hyp/core/timer/src/timer_queue.c b/hyp/core/timer/src/timer_queue.c index 1523fbe..feac058 100644 --- a/hyp/core/timer/src/timer_queue.c +++ b/hyp/core/timer/src/timer_queue.c @@ -7,12 +7,15 @@ #include +#include #include #include +#include #include #include #include #include +#include #include #include #include @@ -26,30 +29,37 @@ CPULOCAL_DECLARE_STATIC(timer_queue_t, timer_queue); void -timer_handle_boot_cpu_cold_init(cpu_index_t cpu_index) +timer_handle_boot_cold_init(cpu_index_t boot_cpu_index) { - timer_queue_t *tq = &CPULOCAL_BY_INDEX(timer_queue, cpu_index); - tq->timeout = TIMER_INVALID_TIMEOUT; - list_init(&tq->list); - spinlock_init(&tq->lock); + // Initialise all timer queues here as online CPUs may try to move + // timers to CPUs that have not booted yet + // Secondary CPUs will be set online by `power_cpu_online()` handler + for (cpu_index_t cpu_index = 0U; cpu_index < PLATFORM_MAX_CORES; + cpu_index++) { + timer_queue_t *tq = &CPULOCAL_BY_INDEX(timer_queue, cpu_index); + spinlock_init(&tq->lock); + list_init(&tq->list); + tq->timeout = TIMER_INVALID_TIMEOUT; + tq->online = (cpu_index == boot_cpu_index); + } } #if !defined(UNITTESTS) || !UNITTESTS void -timer_handle_rootvm_init(boot_env_data_t *env_data) +timer_handle_rootvm_init(hyp_env_data_t *hyp_env) { - env_data->timer_freq = timer_get_timer_frequency(); + hyp_env->timer_freq = timer_get_timer_frequency(); } #endif uint32_t -timer_get_timer_frequency() +timer_get_timer_frequency(void) { return platform_timer_get_frequency(); } ticks_t -timer_get_current_timer_ticks() +timer_get_current_timer_ticks(void) { return platform_timer_get_current_ticks(); } @@ -69,18 +79,10 @@ timer_convert_ticks_to_ns(ticks_t ticks) static bool is_timeout_a_smaller_than_b(list_node_t *node_a, list_node_t *node_b) { - bool smaller = false; - - ticks_t timeout_a = - timer_container_of_timer_queue_list_node(node_a)->timeout; - ticks_t timeout_b = - timer_container_of_timer_queue_list_node(node_b)->timeout; - - if (timeout_a < timeout_b) { - smaller = true; - } + ticks_t timeout_a = timer_container_of_list_node(node_a)->timeout; + ticks_t timeout_b = timer_container_of_list_node(node_b)->timeout; - return smaller; + return timeout_a < timeout_b; } void @@ -88,11 +90,9 @@ timer_init_object(timer_t *timer, timer_action_t action) { assert(timer != NULL); - timer->timer_queue = NULL; - timer->timeout = TIMER_INVALID_TIMEOUT; - timer->action = action; - timer->queued = false; - spinlock_init(&timer->lock); + timer->timeout = TIMER_INVALID_TIMEOUT; + timer->action = action; + atomic_init(&timer->queue, NULL); } bool @@ -100,23 +100,7 @@ timer_is_queued(timer_t *timer) { assert(timer != NULL); - bool queued = false; - spinlock_acquire(&timer->lock); - timer_queue_t *tq = timer->timer_queue; - if (tq == NULL) { - goto out; - } - - spinlock_acquire_nopreempt(&tq->lock); - if (timer->queued) { - queued = true; - } else { - timer->timer_queue = NULL; - } - spinlock_release_nopreempt(&tq->lock); -out: - spinlock_release(&timer->lock); - return queued; + return atomic_load_relaxed(&timer->queue) != NULL; } ticks_t @@ -125,99 +109,102 @@ timer_queue_get_next_timeout(void) timer_queue_t *tq = &CPULOCAL(timer_queue); ticks_t timeout; - spinlock_acquire(&tq->lock); + spinlock_acquire_nopreempt(&tq->lock); timeout = tq->timeout; - spinlock_release(&tq->lock); + spinlock_release_nopreempt(&tq->lock); return timeout; } static void -timer_update_timeout(timer_queue_t *tq) REQUIRE_PREEMPT_DISABLED +timer_update_timeout(timer_queue_t *tq) REQUIRE_SPINLOCK(tq->lock) { assert_preempt_disabled(); + assert(tq == &CPULOCAL(timer_queue)); - if (tq == &CPULOCAL(timer_queue)) { - if (tq->timeout != TIMER_INVALID_TIMEOUT) { - platform_timer_set_timeout(tq->timeout); - } else { - platform_timer_cancel_timeout(); - } + if (tq->timeout != TIMER_INVALID_TIMEOUT) { + platform_timer_set_timeout(tq->timeout); + } else { + platform_timer_cancel_timeout(); } } static void -timer_enqueue_internal(timer_t *timer, ticks_t timeout) REQUIRE_PREEMPT_DISABLED +timer_enqueue_internal(timer_queue_t *tq, timer_t *timer, ticks_t timeout) + REQUIRE_SPINLOCK(tq->lock) { assert_preempt_disabled(); - - if (compiler_unexpected(timer->timer_queue != NULL)) { - // This timer object already belongs to a queue + assert(tq == &CPULOCAL(timer_queue)); + assert(tq->online); + + // Set the timer's queue pointer. We need acquire ordering to ensure we + // observe any previous dequeues on other CPUs. + timer_queue_t *old_tq = NULL; + if (!atomic_compare_exchange_strong_explicit(&timer->queue, &old_tq, tq, + memory_order_acquire, + memory_order_relaxed)) { + // This timer is already queued; it is the caller's + // responsibility to avoid this. panic("Request to enqueue a timer that is already queued"); } - timer_queue_t *tq = &CPULOCAL(timer_queue); - // There is no need to check if the timeout is already in the past, as // the timer module generates a level-triggered interrupt if the timer // condition is already met. + timer->timeout = timeout; - timer->timeout = timeout; - timer->timer_queue = tq; - timer->queued = true; - - bool new_head = list_insert_in_order(&tq->list, - &timer->timer_queue_list_node, + bool new_head = list_insert_in_order(&tq->list, &timer->list_node, is_timeout_a_smaller_than_b); - if (new_head) { tq->timeout = timeout; + timer_update_timeout(tq); } } -static void -timer_dequeue_internal(timer_t *timer, bool timer_locked) - REQUIRE_PREEMPT_DISABLED +static bool +timer_dequeue_internal(timer_queue_t *tq, timer_t *timer) + REQUIRE_SPINLOCK(tq->lock) { assert_preempt_disabled(); - timer_queue_t *tq = timer->timer_queue; - - if (compiler_unexpected(tq == NULL)) { - // The timer object is not in a queue - panic("Request to dequeue a timer that is not in a queue"); - } - - bool new_head = - list_delete_node(&tq->list, &timer->timer_queue_list_node); - - if (new_head) { - list_node_t *head = list_get_head(&tq->list); - ticks_t timeout = - timer_container_of_timer_queue_list_node(head)->timeout; + bool new_timeout = false; - tq->timeout = timeout; - - } else if (list_is_empty(&tq->list)) { - tq->timeout = TIMER_INVALID_TIMEOUT; - } + // The timer may have expired between loading the timer's queue and + // acquiring its lock. Ensure the timer's queue has not changed before + // dequeuing. + if (compiler_expected(atomic_load_relaxed(&timer->queue) == tq)) { + bool new_head = list_delete_node(&tq->list, &timer->list_node); + if (new_head) { + list_node_t *head = list_get_head(&tq->list); + tq->timeout = + timer_container_of_list_node(head)->timeout; + new_timeout = true; + } else if (list_is_empty(&tq->list)) { + tq->timeout = TIMER_INVALID_TIMEOUT; + new_timeout = true; + } else { + // The queue's timeout has not changed. + } - if (timer_locked) { - timer->timer_queue = NULL; + // Clear the timer's queue pointer. We need release ordering to + // ensure this dequeue is observed by the next enqueue. + atomic_store_release(&timer->queue, NULL); } - timer->queued = false; + return new_timeout; } static void -timer_update_internal(timer_t *timer, ticks_t timeout) REQUIRE_PREEMPT_DISABLED +timer_update_internal(timer_queue_t *tq, timer_t *timer, ticks_t timeout) + REQUIRE_SPINLOCK(tq->lock) { assert_preempt_disabled(); + assert(tq == &CPULOCAL(timer_queue)); + assert(tq->online); - timer_queue_t *tq = &CPULOCAL(timer_queue); - - if (compiler_unexpected(tq != timer->timer_queue)) { - // The timer object is not queued on this CPU + if (compiler_unexpected(tq != atomic_load_relaxed(&timer->queue))) { + // There is a race with timer updates; it is the caller's + // responsibility to prevent this. panic("Request to update a timer that is not queued on this CPU"); } @@ -228,22 +215,20 @@ timer_update_internal(timer_t *timer, ticks_t timeout) REQUIRE_PREEMPT_DISABLED // Delete timer from queue, update it, and add it again to queue - bool new_head_delete = list_delete_node( - &tq->list, &timer->timer_queue_list_node); + bool new_head_delete = + list_delete_node(&tq->list, &timer->list_node); timer->timeout = timeout; - bool new_head_insert = list_insert_in_order( - &tq->list, &timer->timer_queue_list_node, - is_timeout_a_smaller_than_b); + bool new_head_insert = + list_insert_in_order(&tq->list, &timer->list_node, + is_timeout_a_smaller_than_b); if (new_head_delete || new_head_insert) { list_node_t *head = list_get_head(&tq->list); - ticks_t head_timeout = - timer_container_of_timer_queue_list_node(head) - ->timeout; - - tq->timeout = head_timeout; + tq->timeout = + timer_container_of_list_node(head)->timeout; + timer_update_timeout(tq); } } } @@ -254,24 +239,13 @@ timer_enqueue(timer_t *timer, ticks_t timeout) assert(timer != NULL); preempt_disable(); - timer_queue_t *tq = &CPULOCAL(timer_queue); - spinlock_acquire_nopreempt(&timer->lock); - if (timer->timer_queue != NULL) { - spinlock_acquire_nopreempt(&timer->timer_queue->lock); - if (compiler_unexpected(timer->queued)) { - panic("Request to enqueue a queued timer"); - } - spinlock_release_nopreempt(&timer->timer_queue->lock); - timer->timer_queue = NULL; - } + timer_queue_t *tq = &CPULOCAL(timer_queue); spinlock_acquire_nopreempt(&tq->lock); - timer_enqueue_internal(timer, timeout); - timer_update_timeout(tq); - + timer_enqueue_internal(tq, timer, timeout); spinlock_release_nopreempt(&tq->lock); - spinlock_release_nopreempt(&timer->lock); + preempt_enable(); } @@ -280,23 +254,16 @@ timer_dequeue(timer_t *timer) { assert(timer != NULL); - spinlock_acquire(&timer->lock); + timer_queue_t *tq = atomic_load_relaxed(&timer->queue); - timer_queue_t *tq = timer->timer_queue; - if (tq == NULL) { - goto out; - } - - spinlock_acquire_nopreempt(&tq->lock); - if (timer->queued) { - timer_dequeue_internal(timer, true); - timer_update_timeout(tq); - } else { - timer->timer_queue = NULL; + if (tq != NULL) { + spinlock_acquire(&tq->lock); + if (timer_dequeue_internal(tq, timer) && + (tq == &CPULOCAL(timer_queue))) { + timer_update_timeout(tq); + } + spinlock_release(&tq->lock); } - spinlock_release_nopreempt(&tq->lock); -out: - spinlock_release(&timer->lock); } void @@ -304,39 +271,27 @@ timer_update(timer_t *timer, ticks_t timeout) { assert(timer != NULL); - spinlock_acquire(&timer->lock); + preempt_disable(); - timer_queue_t *tq = timer->timer_queue; + timer_queue_t *old_tq = atomic_load_relaxed(&timer->queue); + timer_queue_t *new_tq = &CPULOCAL(timer_queue); // If timer is queued on another CPU, it needs to be dequeued. - if ((tq != NULL) && (tq != &CPULOCAL(timer_queue))) { - spinlock_acquire_nopreempt(&tq->lock); - if (timer->queued) { - timer_dequeue_internal(timer, true); - } else { - timer->timer_queue = NULL; - } - spinlock_release_nopreempt(&tq->lock); + if ((old_tq != NULL) && (old_tq != new_tq)) { + spinlock_acquire_nopreempt(&old_tq->lock); + (void)timer_dequeue_internal(old_tq, timer); + spinlock_release_nopreempt(&old_tq->lock); } - tq = &CPULOCAL(timer_queue); - - spinlock_acquire_nopreempt(&tq->lock); - if (timer->timer_queue != NULL) { - if (timer->queued) { - timer_update_internal(timer, timeout); - } else { - timer->timer_queue = NULL; - timer_enqueue_internal(timer, timeout); - } + spinlock_acquire_nopreempt(&new_tq->lock); + if (old_tq == new_tq) { + timer_update_internal(new_tq, timer, timeout); } else { - timer_enqueue_internal(timer, timeout); + timer_enqueue_internal(new_tq, timer, timeout); } + spinlock_release_nopreempt(&new_tq->lock); - timer_update_timeout(tq); - - spinlock_release_nopreempt(&tq->lock); - spinlock_release(&timer->lock); + preempt_enable(); } static void @@ -350,9 +305,9 @@ timer_dequeue_expired(void) REQUIRE_PREEMPT_DISABLED spinlock_acquire_nopreempt(&tq->lock); while (tq->timeout <= current_ticks) { - list_node_t *head = list_get_head(&tq->list); - timer_t *timer = timer_container_of_timer_queue_list_node(head); - timer_dequeue_internal(timer, false); + list_node_t *head = list_get_head(&tq->list); + timer_t *timer = timer_container_of_list_node(head); + (void)timer_dequeue_internal(tq, timer); spinlock_release_nopreempt(&tq->lock); (void)trigger_timer_action_event(timer->action, timer); spinlock_acquire_nopreempt(&tq->lock); @@ -390,4 +345,132 @@ void timer_handle_power_cpu_online(void) { timer_dequeue_expired(); + + // Mark this CPU timer queue as online + timer_queue_t *tq = &CPULOCAL(timer_queue); + assert_preempt_disabled(); + spinlock_acquire_nopreempt(&tq->lock); + tq->online = true; + spinlock_release_nopreempt(&tq->lock); +} + +// A timer_queue operation has occurred that requires synchronisation, process +// our timer queue. Handle any expired timers as the timer might have expired +// since it was queued on this CPU and reprogram the platform timer if required. +bool NOINLINE +timer_handle_ipi_received(void) +{ + timer_dequeue_expired(); + + return true; +} + +static bool +timer_try_move_to_cpu(timer_t *timer, cpu_index_t target) + REQUIRE_PREEMPT_DISABLED +{ + bool moved = false; + timer_queue_t *ttq = &CPULOCAL_BY_INDEX(timer_queue, target); + + assert_preempt_disabled(); + + spinlock_acquire_nopreempt(&ttq->lock); + + // We can only use active CPU timer queues + if (ttq->online) { + // Update the timer queue to be on the new CPU + timer_queue_t *old_ttq = NULL; + if (!atomic_compare_exchange_strong_explicit( + &timer->queue, &old_ttq, ttq, memory_order_acquire, + memory_order_relaxed)) { + panic("Request to move timer that is already queued"); + } + + // Call IPI if the queue HEAD changed so the target CPU can + // update its local timer + bool new_head = + list_insert_in_order(&ttq->list, &timer->list_node, + is_timeout_a_smaller_than_b); + if (new_head) { + ttq->timeout = timer->timeout; + spinlock_release_nopreempt(&ttq->lock); + ipi_one(IPI_REASON_TIMER_QUEUE_SYNC, target); + } else { + spinlock_release_nopreempt(&ttq->lock); + } + moved = true; + } else { + spinlock_release_nopreempt(&ttq->lock); + } + + return moved; +} + +void +timer_handle_power_cpu_offline(void) +{ + // Try to move any timers to the next one up from this one. + // If this is the last core, wrap around + cpu_index_t our_index = cpulocal_get_index(); + cpu_index_t start = + (cpu_index_t)((our_index + 1U) % PLATFORM_MAX_CORES); + timer_queue_t *tq = &CPULOCAL(timer_queue); + + assert_preempt_disabled(); + spinlock_acquire_nopreempt(&tq->lock); + + // Mark this CPU timer queue as going down and cancel any pending timers + tq->online = false; + platform_timer_cancel_timeout(); + + // Move all active timers in this CPU timer queue to an active CPU + while (tq->timeout != TIMER_INVALID_TIMEOUT) { + list_node_t *head = list_get_head(&tq->list); + timer_t *timer = timer_container_of_list_node(head); + + // Remove timer from this core. + (void)timer_dequeue_internal(tq, timer); + spinlock_release_nopreempt(&tq->lock); + + // The target core might go down while we are searching, so + // always check if the target is active. Hopefully the target + // Queue stays online so we check the last-used CPU first. If + // we cannot find any active timer queues we panic. In reality + // at least one CPU timer queue should always be online. + bool found_target = false; + cpu_index_t target = start; + while (!found_target) { + if (platform_cpu_exists(target)) { + if (timer_try_move_to_cpu(timer, target)) { + found_target = true; + start = target; + break; + } + } + + // Skip our CPU as we know we are going down + // This could happen if the previously saved core is + // down now and the initial search wrapped. + target = (cpu_index_t)((target + 1U) % + PLATFORM_MAX_CORES); + if (target == our_index) { + target = (cpu_index_t)((target + 1U) % + PLATFORM_MAX_CORES); + } + if (target == start) { + // we looped around without finding a target, + // this should never happen. + break; + } + } + + if (!found_target) { + panic("Could not find target CPU for timer migration"); + } + + // Get the lock back to check the next timer + spinlock_acquire_nopreempt(&tq->lock); + } + + spinlock_release_nopreempt(&tq->lock); } diff --git a/hyp/core/timer/src/timer_tests.c b/hyp/core/timer/src/timer_tests.c index 9af6070..7f0c478 100644 --- a/hyp/core/timer/src/timer_tests.c +++ b/hyp/core/timer/src/timer_tests.c @@ -12,12 +12,17 @@ #include #include #include +#include #include #include #include "event_handlers.h" +#if defined(PLATFORM_QEMU) +#define MAX_TICKS_DIFFERENCE 0x500000 +#else #define MAX_TICKS_DIFFERENCE 0x100 +#endif CPULOCAL_DECLARE_STATIC(timer_t, timer1); CPULOCAL_DECLARE_STATIC(timer_t, timer2); @@ -26,13 +31,15 @@ CPULOCAL_DECLARE_STATIC(_Atomic bool, in_progress); CPULOCAL_DECLARE_STATIC(ticks_t, expected_timeout); bool -tests_timer() +tests_timer(void) { ticks_t current_ticks; timer_t *timer1 = &CPULOCAL(timer1); timer_t *timer2 = &CPULOCAL(timer2); ticks_t *expected_timeout = &CPULOCAL(expected_timeout); + _Atomic bool *in_progress = &CPULOCAL(in_progress); + // Test 1 // Enqueue a timer and make sure its expiry is received CPULOCAL(test_num) = 1; @@ -44,8 +51,10 @@ tests_timer() atomic_store_relaxed(&CPULOCAL(in_progress), true); timer_enqueue(timer1, *expected_timeout); - while (atomic_load_relaxed(&CPULOCAL(in_progress))) - ; + preempt_enable(); + while (atomic_load_relaxed(in_progress)) { + } + preempt_disable(); // Test 2 // Enqueue two timers, dequeue the first one and make sure only the @@ -64,8 +73,10 @@ tests_timer() timer_dequeue(timer1); - while (atomic_load_relaxed(&CPULOCAL(in_progress))) - ; + preempt_enable(); + while (atomic_load_relaxed(in_progress)) { + } + preempt_disable(); // TODO: Add more tests diff --git a/hyp/core/timer/timer.ev b/hyp/core/timer/timer.ev index 378f0d0..78b6860 100644 --- a/hyp/core/timer/timer.ev +++ b/hyp/core/timer/timer.ev @@ -4,16 +4,20 @@ module timer -subscribe boot_cpu_cold_init +subscribe boot_cold_init + +subscribe ipi_received[IPI_REASON_TIMER_QUEUE_SYNC]() + require_preempt_disabled #if !defined(UNITTESTS) || !UNITTESTS -subscribe rootvm_init(env_data) +subscribe rootvm_init(hyp_env) #endif subscribe platform_timer_expiry require_preempt_disabled subscribe power_cpu_suspend() + unwinder timer_handle_power_cpu_online() // Run early since it may reject suspends priority 100 require_preempt_disabled @@ -23,3 +27,6 @@ subscribe power_cpu_online() subscribe power_cpu_resume handler timer_handle_power_cpu_online() + +subscribe power_cpu_offline() + require_preempt_disabled diff --git a/hyp/core/timer/timer.tc b/hyp/core/timer/timer.tc index 38dad6d..5e1da49 100644 --- a/hyp/core/timer/timer.tc +++ b/hyp/core/timer/timer.tc @@ -4,21 +4,29 @@ define TIMER_INVALID_TIMEOUT constant type ticks_t = -1; +// An IPI used to manage timer queue synchronization. If a timer is moved from +// from one CPU timer queue to another the target CPU may need to update it's +// local timer with the new timeout. +extend ipi_reason enumeration { + timer_queue_sync; +}; + extend timer structure { - timeout type ticks_t; - action enumeration timer_action; - timer_queue pointer structure timer_queue; - timer_queue_list_node structure list_node(contained); - lock structure spinlock; - queued bool; + timeout type ticks_t; + action enumeration timer_action; + queue pointer(atomic) structure timer_queue; + list_node structure list_node(contained); }; define timer_queue structure { timeout type ticks_t; list structure list; lock structure spinlock; + + // False if the pCPU for this queue is powering off + online bool; }; -extend boot_env_data structure { +extend hyp_env_data structure { timer_freq uint64; }; diff --git a/hyp/core/timer/timer_tests.ev b/hyp/core/timer/timer_tests.ev index 03359a7..ac45100 100644 --- a/hyp/core/timer/timer_tests.ev +++ b/hyp/core/timer/timer_tests.ev @@ -8,8 +8,10 @@ module timer subscribe tests_start handler tests_timer() + require_preempt_disabled subscribe timer_action[TIMER_ACTION_TEST] handler tests_timer_action(timer) + require_preempt_disabled #endif diff --git a/hyp/core/timer_lp/build.conf b/hyp/core/timer_lp/build.conf new file mode 100644 index 0000000..8374f68 --- /dev/null +++ b/hyp/core/timer_lp/build.conf @@ -0,0 +1,8 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +interface timer_lp +types timer_lp.tc +events timer_lp.ev +source timer_lp_queue.c diff --git a/hyp/core/timer_lp/src/timer_lp_queue.c b/hyp/core/timer_lp/src/timer_lp_queue.c new file mode 100644 index 0000000..1234a53 --- /dev/null +++ b/hyp/core/timer_lp/src/timer_lp_queue.c @@ -0,0 +1,206 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "event_handlers.h" + +static spinlock_t timer_lp_queue_lock; +static timer_lp_queue_t timer_lp_queue PROTECTED_BY(timer_lp_queue_lock); + +CPULOCAL_DECLARE_STATIC(timer_lp_t, timer_lp); + +void +timer_lp_queue_handle_boot_cold_init(void) +{ + spinlock_init(&timer_lp_queue_lock); + spinlock_acquire(&timer_lp_queue_lock); + timer_lp_queue.timeout = TIMER_INVALID_TIMEOUT; + list_init(&timer_lp_queue.list); + spinlock_release(&timer_lp_queue_lock); +} + +void +timer_lp_queue_handle_boot_cpu_cold_init(cpu_index_t cpu_index) +{ + timer_lp_t *timer = &CPULOCAL_BY_INDEX(timer_lp, cpu_index); + timer->timeout = TIMER_INVALID_TIMEOUT; + timer->cpu_index = cpu_index; +} + +static bool +is_timeout_a_smaller_than_b(list_node_t *node_a, list_node_t *node_b) +{ + bool smaller = false; + + ticks_t timeout_a = timer_lp_container_of_node(node_a)->timeout; + ticks_t timeout_b = timer_lp_container_of_node(node_b)->timeout; + + if (timeout_a < timeout_b) { + smaller = true; + } + + return smaller; +} + +static void +timer_lp_enqueue(timer_lp_t *timer, ticks_t timeout) + REQUIRE_SPINLOCK(timer_lp_queue_lock) +{ + timer->timeout = timeout; + + bool new_head = list_insert_in_order(&timer_lp_queue.list, &timer->node, + is_timeout_a_smaller_than_b); + + if (new_head) { + timer_lp_queue.timeout = timer->timeout; + platform_timer_lp_set_timeout_and_route(timer->timeout, + timer->cpu_index); + } +} + +static bool +timer_lp_dequeue(timer_lp_t *timer) REQUIRE_SPINLOCK(timer_lp_queue_lock) +{ + bool new_head = list_delete_node(&timer_lp_queue.list, &timer->node); + bool need_update; + + if (new_head) { + list_node_t *head = list_get_head(&timer_lp_queue.list); + timer_lp_t *head_timer = timer_lp_container_of_node(head); + + timer_lp_queue.timeout = head_timer->timeout; + need_update = true; + } else if (list_is_empty(&timer_lp_queue.list)) { + timer_lp_queue.timeout = TIMER_INVALID_TIMEOUT; + need_update = true; + } else { + need_update = false; + } + + timer->timeout = TIMER_INVALID_TIMEOUT; + + return need_update; +} + +static void +timer_lp_queue_save_arch_timer(void) REQUIRE_SPINLOCK(timer_lp_queue_lock) +{ + // Get the next timeout of the local arch timer queue + + ticks_t timeout = timer_queue_get_next_timeout(); + if (timeout == TIMER_INVALID_TIMEOUT) { + goto out; + } + + timer_lp_t *timer = &CPULOCAL(timer_lp); + assert(timer->timeout == TIMER_INVALID_TIMEOUT); + + timer_lp_enqueue(timer, timeout); + +out: + return; +} + +error_t +timer_lp_handle_power_cpu_suspend(void) +{ + assert_preempt_disabled(); + + // TODO: Delay or reject attempted suspend if timeout is due to expire + // sooner than the CPU can reach the requested power state. + + spinlock_acquire_nopreempt(&timer_lp_queue_lock); + + timer_lp_queue_save_arch_timer(); + + spinlock_release_nopreempt(&timer_lp_queue_lock); + + return OK; +} + +static void +timer_lp_sync(bool force_update) REQUIRE_SPINLOCK(timer_lp_queue_lock) +{ + cpu_index_t cpu_index = cpulocal_get_index(); + ticks_t current_ticks = platform_timer_lp_get_current_ticks(); + bool do_update = force_update; + + assert_preempt_disabled(); + + while (timer_lp_queue.timeout <= current_ticks) { + list_node_t *head = list_get_head(&timer_lp_queue.list); + timer_lp_t *timer = timer_lp_container_of_node(head); + + (void)timer_lp_dequeue(timer); + // We just dequeued the head, so always update the timer + do_update = true; + + if (timer->cpu_index != cpu_index) { + ipi_one(IPI_REASON_RESCHEDULE, timer->cpu_index); + } + } + + if (!do_update) { + // Queue head didn't change, nothing to do + } else if (timer_lp_queue.timeout == TIMER_INVALID_TIMEOUT) { + // Queue is now empty + platform_timer_lp_cancel_timeout(); + } else { + // Schedule the next timeout + list_node_t *head = list_get_head(&timer_lp_queue.list); + timer_lp_t *head_timer = timer_lp_container_of_node(head); + platform_timer_lp_set_timeout_and_route(head_timer->timeout, + head_timer->cpu_index); + } +} + +static void +timer_lp_queue_restore_arch_timer(void) REQUIRE_SPINLOCK(timer_lp_queue_lock) +{ + timer_lp_t *timer = &CPULOCAL(timer_lp); + if (timer->timeout == TIMER_INVALID_TIMEOUT) { + goto out; + } + + if (timer_lp_dequeue(timer)) { + timer_lp_sync(true); + } + +out: + return; +} + +void +timer_lp_handle_power_cpu_resume(void) +{ + assert_preempt_disabled(); + + spinlock_acquire_nopreempt(&timer_lp_queue_lock); + + timer_lp_queue_restore_arch_timer(); + + spinlock_release_nopreempt(&timer_lp_queue_lock); +} + +void +timer_lp_handle_platform_timer_lp_expiry(void) +{ + spinlock_acquire_nopreempt(&timer_lp_queue_lock); + + timer_lp_sync(false); + + spinlock_release_nopreempt(&timer_lp_queue_lock); +} diff --git a/hyp/core/timer_lp/timer_lp.ev b/hyp/core/timer_lp/timer_lp.ev new file mode 100644 index 0000000..48c20a0 --- /dev/null +++ b/hyp/core/timer_lp/timer_lp.ev @@ -0,0 +1,21 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module timer_lp + +subscribe boot_cold_init + handler timer_lp_queue_handle_boot_cold_init() + +subscribe boot_cpu_cold_init + handler timer_lp_queue_handle_boot_cpu_cold_init + +subscribe power_cpu_suspend() + unwinder timer_lp_handle_power_cpu_resume() + require_preempt_disabled + +subscribe power_cpu_resume() + require_preempt_disabled + +subscribe platform_timer_lp_expiry + require_preempt_disabled diff --git a/hyp/core/timer_lp/timer_lp.tc b/hyp/core/timer_lp/timer_lp.tc new file mode 100644 index 0000000..ea04417 --- /dev/null +++ b/hyp/core/timer_lp/timer_lp.tc @@ -0,0 +1,14 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +define timer_lp structure { + timeout type ticks_t; + cpu_index type cpu_index_t; + node structure list_node(contained); +}; + +define timer_lp_queue structure { + timeout type ticks_t; + list structure list; +}; diff --git a/hyp/core/util/aarch64/src/memcpy.S b/hyp/core/util/aarch64/src/memcpy.S index 3511bac..3bb45be 100644 --- a/hyp/core/util/aarch64/src/memcpy.S +++ b/hyp/core/util/aarch64/src/memcpy.S @@ -6,7 +6,7 @@ #include #include -#if defined(ARCH_ARM_8_5_BTI) +#if defined(ARCH_ARM_FEAT_BTI) // Two instructions per jump #define JUMP_SHIFT 3 #else diff --git a/hyp/core/util/aarch64/src/memset.S b/hyp/core/util/aarch64/src/memset.S index bd27e99..f7facde 100644 --- a/hyp/core/util/aarch64/src/memset.S +++ b/hyp/core/util/aarch64/src/memset.S @@ -7,7 +7,7 @@ #include -#if defined(ARCH_ARM_8_5_BTI) +#if defined(ARCH_ARM_FEAT_BTI) // Two instructions per jump #define JUMP_SHIFT 3 #else @@ -236,50 +236,3 @@ function_chain memset_alignable, memset_align16 cbnz x2, LOCAL(memset_below16) ret function_end memset_align16 - -// Optimize memset_s to perform size, alignment and store zero checks and call -// optimized functions above. -function memset_s - cbz x0, LOCAL(memset_s_null) - uxtb w2, w2 - cbz x3, LOCAL(out) - cmp x3, x1 - bhi LOCAL(memset_s_size) - // Note, 15.99MiB is smaller than RSIZE_MAX, its the largest immediate - cmp x1, 4095, lsl 12 - b.hi LOCAL(memset_s_size) - - cmp x3, 8 - blo 3f - // Copy is at least 8 bytes, assume pointers are aligned - - // Vector duplicate the byte value in x2 - orr x2, x2, x2, lsl 8 - orr x2, x2, x2, lsl 16 - orr x2, x2, x2, lsl 32 - - bic x1, x3, 7 - // Copy 8 bytes per loop -2: - str x2, [x0], #8 - subs x1, x1, 8 - b.gt 2b - - and x3, x3, 7 - cbnz x3, 3f -local out: - mov x0, xzr - ret - -3: // Slow copy 1 byte per loop - strb w2, [x0], #1 - subs x3, x3, 1 - b.ne 3b - - b LOCAL(out) - -local memset_s_null: - panic "memset_s: null pointer" -local memset_s_size: - panic "memset_s: bad size" -function_end memset_s diff --git a/hyp/core/util/aarch64/src/string.c b/hyp/core/util/aarch64/src/string.c index cb7fa4b..b249dac 100644 --- a/hyp/core/util/aarch64/src/string.c +++ b/hyp/core/util/aarch64/src/string.c @@ -4,9 +4,11 @@ #include #include +#include #include #include +#include #include #include @@ -78,9 +80,9 @@ memcpy(void *restrict s1, const void *restrict s2, size_t n) { assert(compiler_sizeof_object(s1) >= n); assert(compiler_sizeof_object(s2) >= n); - if (n == 0) { + if (n == 0U) { // Nothing to do. - } else if (n < 32) { + } else if (n < 32U) { prefetch_store_keep(s1); prefetch_load_stream(s2); memcpy_below32(s1, s2, n); @@ -88,7 +90,7 @@ memcpy(void *restrict s1, const void *restrict s2, size_t n) prefetch_store_keep(s1); prefetch_load_stream(s2); uintptr_t a16 = (uintptr_t)s1 & (uintptr_t)15; - if (a16 == 0) { + if (a16 == 0U) { memcpy_align16(s1, s2, n); } else { memcpy_alignable(s1, s2, n); @@ -98,61 +100,78 @@ memcpy(void *restrict s1, const void *restrict s2, size_t n) return s1; } -size_t -memscpy(void *restrict s1, size_t s1_size, const void *restrict s2, - size_t s2_size) +static void +memmove_bytes_reverse(uint8_t *dst, const uint8_t *src, size_t n) { - size_t copy_size = util_min(s1_size, s2_size); - - memcpy(s1, s2, copy_size); - - return copy_size; + assert((uintptr_t)src < (uintptr_t)dst); + + // move to a higher address, copy backwards + const uint8_t *srcr; + uint8_t *dstr; + srcr = src + (n - 1U); + dstr = dst + (n - 1U); + + for (; n != 0; n--) { + *dstr = *srcr; + dstr--; + srcr--; + } } void * -memmove(void *s1, const void *s2, size_t n) +memmove(void *dst, const void *src, size_t n) { - // The hypervisor should never need memmove(), but unfortunately the - // test program won't link if we don't define it. This is because - // static glibc defines it in the same object as memcpy(), so if we - // don't define it the calls in the glibc startup will pull in the - // glibc version of memcpy() and cause duplicate definition errors. - // - // In cases where we know our fast memcpy will work, just call that. - // Otherwise call a slow memcpy which is only defined in the test - // program. - if ((uintptr_t)s1 == (uintptr_t)s2) { + if (n == 0) { + goto out; + } + + if (util_add_overflows((uintptr_t)dst, n - 1U) || + util_add_overflows((uintptr_t)src, n - 1U)) { + panic("memmove_bytes addr overflow"); + } + + if ((uintptr_t)dst == (uintptr_t)src) { // Nothing to do. - } else if ((uintptr_t)s1 < (uintptr_t)s2) { - (void)memcpy(s1, s2, n); - } else if ((uintptr_t)s2 + CPU_MEMCPY_STRIDE < (uintptr_t)s1) { - (void)memcpy(s1, s2, n); - } else if (((uintptr_t)s1 + n <= (uintptr_t)s2) && - ((uintptr_t)s2 + n <= (uintptr_t)s1)) { - (void)memcpy(s1, s2, n); + } else if ((uintptr_t)dst < (uintptr_t)src) { + (void)memcpy(dst, src, n); + } else if ((uintptr_t)src + (n - 1) < (uintptr_t)dst) { + (void)memcpy(dst, src, n); } else { - (void)memcpy_bytes(s1, s2, n); + (void)memmove_bytes_reverse(dst, src, n); } - return s1; + +out: + return dst; } -void * -memset(void *s, int c, size_t n) +errno_t +memset_s(void *s, rsize_t smax, int c, rsize_t n) { - assert(compiler_sizeof_object(s) >= n); + assert(compiler_sizeof_object(s) >= smax); uintptr_t a16 = (uintptr_t)s & (uintptr_t)15; - if (n == 0) { + errno_t err = 0; + + if (s == NULL) { + err = 1; + goto out_null; + } + if (n > smax) { + err = 1; + n = smax; + } + + if (n == 0U) { // Nothing to do. } else if (c == 0) { uintptr_t a_zva = (uintptr_t)s & - (uintptr_t)((1 << CPU_DCZVA_BITS) - 1); - if (n < 32) { + (uintptr_t)util_mask(CPU_DCZVA_BITS); + if (n < 32U) { prefetch_store_keep(s); memset_zeros_below32(s, n); - } else if ((a_zva == 0) && ((n >> CPU_DCZVA_BITS) > 0U)) { + } else if ((a_zva == 0U) && ((n >> CPU_DCZVA_BITS) > 0U)) { memset_zeros_dczva(s, n); - } else if (a16 == 0) { + } else if (a16 == 0U) { prefetch_store_keep(s); memset_zeros_align16(s, n); } else { @@ -164,10 +183,10 @@ memset(void *s, int c, size_t n) cs |= cs << 8; cs |= cs << 16; cs |= cs << 32; - if (n < 32) { + if (n < 32U) { prefetch_store_keep(s); memset_below32(s, cs, n); - } else if (a16 == 0) { + } else if (a16 == 0U) { prefetch_store_keep(s); memset_align16(s, cs, n); } else { @@ -176,6 +195,14 @@ memset(void *s, int c, size_t n) } } +out_null: + return err; +} + +void * +memset(void *s, int c, size_t n) +{ + (void)memset_s(s, n, c, n); return s; } @@ -199,7 +226,7 @@ strchr(const char *str, int c) const char *end = str; for (; *end != '\0'; end++) { - if (*end == c) { + if (*end == (char)c) { ret = (uintptr_t)end; break; } diff --git a/hyp/core/util/src/assert.c b/hyp/core/util/src/assert.c index ca1a216..dd7eae7 100644 --- a/hyp/core/util/src/assert.c +++ b/hyp/core/util/src/assert.c @@ -19,8 +19,9 @@ #include #include -noreturn void NOINLINE +noreturn void NOINLINE COLD assert_failed(const char *file, int line, const char *func, const char *err) + LOCK_IMPL { const char *file_short; @@ -28,12 +29,12 @@ assert_failed(const char *file, int line, const char *func, const char *err) trigger_scheduler_stop_event(); size_t len = strlen(file); - if (len < 64) { + if (len < 64U) { file_short = file; } else { file_short = file + len - 64; - char *file_strchr = strchr(file_short, '/'); + char *file_strchr = strchr(file_short, (int)'/'); if (file_strchr != NULL) { file_short = file_strchr + 1; } diff --git a/hyp/core/util/src/bitmap.c b/hyp/core/util/src/bitmap.c index 267c591..0ee87f1 100644 --- a/hyp/core/util/src/bitmap.c +++ b/hyp/core/util/src/bitmap.c @@ -10,11 +10,11 @@ #include #include -#define BITMAP_SET_BIT(x) ((register_t)1U << ((x % BITMAP_WORD_BITS))) +#define BITMAP_SET_BIT(x) ((register_t)1U << (((x) % BITMAP_WORD_BITS))) #define BITMAP_WORD(x) ((x) / BITMAP_WORD_BITS) #define BITMAP_SIZE_ASSERT(bitmap, bit) \ - assert((compiler_sizeof_object(bitmap) / sizeof(register_t)) > \ - BITMAP_WORD(bit)) + assert((index_t)(compiler_sizeof_object(bitmap) / \ + sizeof(register_t)) > BITMAP_WORD(bit)) bool bitmap_isset(const register_t *bitmap, index_t bit) diff --git a/hyp/core/util/src/list.c b/hyp/core/util/src/list.c index f29e687..428b58e 100644 --- a/hyp/core/util/src/list.c +++ b/hyp/core/util/src/list.c @@ -31,13 +31,9 @@ list_get_head(list_t *list) { assert(list != NULL); - list_node_t *node = NULL; + list_node_t *node = atomic_load_relaxed(&list->head.next); - if (!list_is_empty(list)) { - node = atomic_load_relaxed(&list->head.next); - } - - return node; + return (node != &list->head) ? node : NULL; } static inline void @@ -92,24 +88,6 @@ list_insert_at_tail_release(list_t *list, list_node_t *node) list_insert_at_tail_explicit(list, node, memory_order_release); } -static list_node_t * -find_prev_node_based_on_order(list_node_t *head, list_node_t *new_node, - bool (*compare_fn)(list_node_t *a, - list_node_t *b)) -{ - list_node_t *node = head; - assert(node != NULL); - - while (atomic_load_relaxed(&node->next) != head) { - if (compare_fn(new_node, atomic_load_relaxed(&node->next))) { - break; - } - node = atomic_load_relaxed(&node->next); - } - - return node; -} - static inline bool list_insert_in_order_explicit(list_t *list, list_node_t *node, bool (*compare_fn)(list_node_t *a, @@ -122,9 +100,17 @@ list_insert_in_order_explicit(list_t *list, list_node_t *node, bool new_head = false; list_node_t *head = &list->head; - list_node_t *prev = - find_prev_node_based_on_order(head, node, compare_fn); - list_node_t *next = atomic_load_relaxed(&prev->next); + list_node_t *prev = head; + list_node_t *next = atomic_load_relaxed(&head->next); + + while (next != head) { + if (compare_fn(node, next)) { + break; + } + + prev = next; + next = atomic_load_relaxed(&prev->next); + } if (prev == head) { new_head = true; diff --git a/hyp/core/util/src/panic.c b/hyp/core/util/src/panic.c index b37163c..47d827a 100644 --- a/hyp/core/util/src/panic.c +++ b/hyp/core/util/src/panic.c @@ -16,8 +16,8 @@ #include -noreturn void NOINLINE -panic(const char *str) +noreturn void NOINLINE COLD +panic(const char *str) LOCK_IMPL { void *from = __builtin_return_address(0); void *frame = __builtin_frame_address(0); @@ -25,7 +25,7 @@ panic(const char *str) // Stop all cores and disable preemption trigger_scheduler_stop_event(); -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) __asm__("xpaci %0;" : "+r"(from)); #endif diff --git a/hyp/core/util/tests/Makefile b/hyp/core/util/tests/Makefile new file mode 100644 index 0000000..dd66f9b --- /dev/null +++ b/hyp/core/util/tests/Makefile @@ -0,0 +1,20 @@ +CC=clang -target aarch64-linux-gnu + +CFLAGS=-std=c11 --sysroot=/usr/aarch64-linux-gnu -O3 -flto -Wno-gcc-compat +CPPFLAGS=-D_DEFAULT_SOURCE -D__STDC_WANT_LIB_EXT1__=1 -DHYP_STANDALONE_TEST \ + -I../../../interfaces/util/include \ + -I../../../arch/aarch64/include \ + -I../../../arch/generic/include \ + -Iqemu/include + +SRC=../tests/string.c \ + ../aarch64/src/string.c \ + ../aarch64/src/memset.S \ + ../aarch64/src/memcpy.S + +LDFLAGS=-static + +OBJ=string-test + +$(OBJ): $(SRC) + $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ -o $@ diff --git a/hyp/core/util/tests/qemu/include/asm/cpu.h b/hyp/core/util/tests/qemu/include/asm/cpu.h new file mode 100644 index 0000000..8bf70b4 --- /dev/null +++ b/hyp/core/util/tests/qemu/include/asm/cpu.h @@ -0,0 +1,17 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +// Miscellaneous definitions describing the CPU implementation. + +// The size in address bits of a line in the innermost visible data cache. +#define CPU_L1D_LINE_BITS 6U + +// The size in address bits of the CPU's DC ZVA block. This is nearly always +// the same as CPU_L1D_LINE_BITS. +#define CPU_DCZVA_BITS 9U + +// The largest difference between the source and destination pointers during +// the optimised memcpy() for this CPU. This is here because it might depend +// on CPU_L1D_LINE_BITS in some implementations. +#define CPU_MEMCPY_STRIDE 256U diff --git a/hyp/core/util/tests/string.c b/hyp/core/util/tests/string.c index e57ec35..bfdfc16 100644 --- a/hyp/core/util/tests/string.c +++ b/hyp/core/util/tests/string.c @@ -2,15 +2,20 @@ // // SPDX-License-Identifier: BSD-3-Clause +// FIXME: // Integrate these tests into unittest configuration +#include #include #include +#include #include #include #include +#include #include +#include #include #include @@ -34,6 +39,21 @@ typedef size_t rsize_t; extern errno_t memset_s(void *s, rsize_t smax, int c, size_t n); +noreturn void NOINLINE +assert_failed(const char *file, int line, const char *func, const char *err) +{ + fprintf(stderr, "Assert failed in %s at %s:%d: %s\n", func, file, line, + err); + abort(); +} + +noreturn void NOINLINE +panic(const char *msg) +{ + fprintf(stderr, "panic: %s\n", msg); + abort(); +} + static size_t memchk(const volatile uint8_t *p, int c, size_t n) { @@ -45,6 +65,17 @@ memchk(const volatile uint8_t *p, int c, size_t n) return n; } +static size_t +memcmpchk(const volatile uint8_t *p, const volatile uint8_t *q, size_t n) +{ + for (size_t i = 0; i < n; i++) { + if (p[i] != q[i]) { + return i; + } + } + return n; +} + static void memset_test(size_t size, size_t misalign, int c) { @@ -157,8 +188,8 @@ memcpy_test(size_t size, size_t src_misalign, size_t dst_misalign) size_t dst_end = dst_start + size; size_t pos; - // We assume that we can memset the whole buffer safely... hopefully - // any bugs in it won't crash the test before we find them! + // We tested memset first, so it should be safe to use it to clear + // the destination buffer. memset(dst_buffer, INIT_BYTE, BUFFER_SIZE); memcpy(&dst_buffer[dst_start], &src_buffer[src_start], size); @@ -172,11 +203,8 @@ memcpy_test(size_t size, size_t src_misalign, size_t dst_misalign) exit(2); } - for (pos = 0; pos < size; pos++) { - if (dst_buffer[dst_start + pos] == - src_buffer[src_start + pos]) { - continue; - } + pos = memcmpchk(&dst_buffer[dst_start], &src_buffer[src_start], size); + if (pos < size) { fprintf(stderr, "FAILED: memcpy(dst + %#zx, src + %#zx, %#zx) set byte at offset %#zx to %#x (should be %#x)\n", dst_start - BUFFER_PAD, src_start - BUFFER_PAD, size, @@ -217,6 +245,108 @@ memcpy_tests(void) printf("\nPASS\n"); } +static void +memmove_test(size_t size, ptrdiff_t overlap) +{ + // We assume here that memmove() is based on memcpy(), so we don't need + // to re-test with different alignments; just different amounts of + // overlap is enough. + size_t src_start = BUFFER_PAD + overlap; + size_t src_end = src_start + size; + size_t dst_start = BUFFER_PAD; + size_t dst_end = dst_start + size; + size_t pos; + + // We tested memset first, so it should be safe to use it to clear + // the destination buffer. + memset(dst_buffer, INIT_BYTE, BUFFER_SIZE); + + // We also tested memcpy already, so it should be safe to use it to copy + // some random bytes from the source buffer into the destination buffer + // at the source location. + memcpy(&dst_buffer[src_start], src_buffer, size); + + // Now move from the source location to the destination location, both + // within the destination buffer. + memmove(&dst_buffer[dst_start], &dst_buffer[src_start], size); + + size_t start = (overlap > 0) ? dst_start : src_start; + pos = memchk(dst_buffer, INIT_BYTE, start); + if (pos < start) { + fprintf(stderr, + "FAILED: memmove(dst, dst + %td, %#zx) set byte at dst + %td to %#x (1)\n", + overlap, size, dst_start - pos, dst_buffer[pos]); + exit(2); + } + + pos = memcmpchk(&dst_buffer[dst_start], src_buffer, size); + if (pos < size) { + fprintf(stderr, + "FAILED: memmove(dst, dst + %td, %#zx) set byte at dst + %#zx to %#x (should be %#x) (2)\n", + overlap, size, pos, dst_buffer[dst_start + pos], + src_buffer[pos]); + exit(2); + } + + if ((overlap > 0) && (size > overlap)) { + pos = memcmpchk(&dst_buffer[dst_end], + &src_buffer[size - overlap], overlap); + if (pos < overlap) { + fprintf(stderr, + "FAILED: memmove(dst, dst + %td, %#zx) set byte at dst + %td to %#x (3a, should be %#x, %#zx, %#zx)\n", + overlap, size, size + pos, + dst_buffer[dst_end + pos], + src_buffer[size - overlap + pos], pos, overlap); + exit(2); + } + } else if ((overlap < 0) && (size > -overlap)) { + pos = memcmpchk(&dst_buffer[src_start], src_buffer, -overlap); + if (pos < -overlap) { + fprintf(stderr, + "FAILED: memmove(dst, dst + %td, %#zx) set byte at dst + %td to %#x (3b, should be %#x, %#zx, %#zx)\n", + overlap, size, overlap + pos, + dst_buffer[src_start + pos], src_buffer[pos], + pos, overlap); + exit(2); + } + } + + size_t end = (overlap > 0) ? src_end : dst_end; + size_t cnt = BUFFER_SIZE - end; + pos = memchk(&dst_buffer[end], INIT_BYTE, cnt); + if (pos < cnt) { + fprintf(stderr, + "FAILED: memmove(dst, dst + %td, %#zx) set byte at dst + %#zx to %#x (4)\n", + overlap, size, end - dst_start, dst_buffer[end + pos]); + exit(2); + } +} + +void +memmove_tests(void) +{ + size_t size; + ptrdiff_t overlap; + printf("Testing memmove..."); + + for (size = 0; size <= MAX_SIZE; size++) { + if ((size % 64) == 0) { + printf("\n%#5zx: .", size); + } else { + printf("."); + } + static_assert(BUFFER_PAD <= (2 * LARGE_ALIGN), + "Buffer padding too small"); + for (overlap = (-2 * LARGE_ALIGN); overlap <= (2 * LARGE_ALIGN); + overlap++) { + if (overlap == 0) { + continue; + } + memmove_test(size, overlap); + } + } +} + int main(void) { @@ -224,7 +354,8 @@ main(void) __asm__("mrs %0, dczid_el0" : "=r"(dczid)); if (dczid != CPU_DCZVA_BITS - 2) { - fprintf(stderr, "Unexpected DC ZVA ID: %#x (expected %#x)\n", + fprintf(stderr, + "ERROR: Unexpected DC ZVA ID: %#x (expected %#x)\n", (unsigned int)dczid, CPU_DCZVA_BITS - 2); return 1; } @@ -237,6 +368,8 @@ main(void) memcpy_tests(); + memmove_tests(); + return 0; } diff --git a/hyp/core/vdevice/aarch64/src/vdevice.c b/hyp/core/vdevice/aarch64/src/vdevice.c index 1d71449..6a48bdf 100644 --- a/hyp/core/vdevice/aarch64/src/vdevice.c +++ b/hyp/core/vdevice/aarch64/src/vdevice.c @@ -1,151 +1,109 @@ -// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. // // SPDX-License-Identifier: BSD-3-Clause #include #include -#include #include #include #include -#include #include #include #include #include -#include - #include "event_handlers.h" +#include "internal.h" vcpu_trap_result_t -vdevice_handle_vcpu_trap_data_abort_guest(ESR_EL2_t esr, vmaddr_t ipa, +vdevice_handle_vcpu_trap_data_abort_guest(ESR_EL2_t esr, vmaddr_result_t ipa, FAR_EL2_t far) { vcpu_trap_result_t ret = VCPU_TRAP_RESULT_UNHANDLED; register_t val = 0U; - thread_t *thread = thread_get_self(); + thread_t *thread = thread_get_self(); ESR_EL2_ISS_DATA_ABORT_t iss = ESR_EL2_ISS_DATA_ABORT_cast(ESR_EL2_get_ISS(&esr)); if (ESR_EL2_ISS_DATA_ABORT_get_ISV(&iss)) { - bool is_write = ESR_EL2_ISS_DATA_ABORT_get_WnR(&iss); - bool is_acquire_release = ESR_EL2_ISS_DATA_ABORT_get_AR(&iss); - size_t size = 1 << ESR_EL2_ISS_DATA_ABORT_get_SAS(&iss); + bool is_write = ESR_EL2_ISS_DATA_ABORT_get_WnR(&iss); + bool is_acquire_release = ESR_EL2_ISS_DATA_ABORT_get_AR(&iss); + iss_da_sas_t sas = ESR_EL2_ISS_DATA_ABORT_get_SAS(&iss); + size_t size = (size_t)util_bit((count_t)sas); uint8_t reg_num = ESR_EL2_ISS_DATA_ABORT_get_SRT(&iss); iss_da_ia_fsc_t fsc = ESR_EL2_ISS_DATA_ABORT_get_DFSC(&iss); - // Only translation and permission faults are considered for - // vdevice access. + // ISV is not meaningful for a S1 page table walk fault + assert(!ESR_EL2_ISS_DATA_ABORT_get_S1PTW(&iss)); - vdevice_typed_ptr_t vdevice = { .type = VDEVICE_TYPE_NONE }; - paddr_t pa = 0U; + if (is_write) { + val = vcpu_gpr_read(thread, reg_num); + + if (is_acquire_release) { + atomic_thread_fence(memory_order_release); + } + } + // Only translation and permission faults are considered for + // vdevice access. if ((fsc == ISS_DA_IA_FSC_PERMISSION_1) || (fsc == ISS_DA_IA_FSC_PERMISSION_2) || (fsc == ISS_DA_IA_FSC_PERMISSION_3)) { + // A permission fault may be a vdevice associated with + // a physical address with a read-only mapping. Since + // the IPA is not valid for permission faults, we must + // look up the physical address from the faulting VA. rcu_read_start(); - gvaddr_t va = FAR_EL2_get_VirtualAddress(&far); paddr_result_t paddr_res = addrspace_va_to_pa_read(va); - if (paddr_res.e != OK) { - rcu_read_finish(); - goto out; - } - - memdb_obj_type_result_t res = memdb_lookup(paddr_res.r); - if ((res.e != OK) || - (res.r.type != MEMDB_TYPE_EXTENT)) { - rcu_read_finish(); - goto out; - } - memextent_t *me = (memextent_t *)res.r.object; - assert(me != NULL); - vdevice = atomic_load_consume(&me->vdevice); - if (vdevice.type == VDEVICE_TYPE_NONE) { - rcu_read_finish(); - goto out; + // The lookup can fail if the guest unmapped or remapped + // the faulting VA in stage 1 on another CPU after the + // stage 2 fault was triggered. In that case, we must + // retry the faulting instruction. + if (paddr_res.e != OK) { + ret = VCPU_TRAP_RESULT_RETRY; + } else { + ret = vdevice_access_phys(paddr_res.r, size, + &val, is_write); } - - pa = paddr_res.r; - - } else if ((fsc != ISS_DA_IA_FSC_TRANSLATION_1) && - (fsc != ISS_DA_IA_FSC_TRANSLATION_2) && - (fsc != ISS_DA_IA_FSC_TRANSLATION_3)) { - goto out; + rcu_read_finish(); + } else if ((fsc == ISS_DA_IA_FSC_TRANSLATION_0) || + (fsc == ISS_DA_IA_FSC_TRANSLATION_1) || + (fsc == ISS_DA_IA_FSC_TRANSLATION_2) || + (fsc == ISS_DA_IA_FSC_TRANSLATION_3)) { + // A translation fault may be a vdevice associated with + // an IPA, with no underlying physical memory. Note + // that the IPA is always valid for a translation + // fault. + assert(ipa.e == OK); + ret = vdevice_access_ipa(ipa.r, size, &val, is_write); } else { - // Nothing to do + // Wrong fault type; not handled by this module } - // TODO: for now the vdevice handlers with hard-coded address - // will not be part of the vdevice types. If the IPA is not - // mapped in the VM, we will try calling those handlers directly - // since IPA=PA - - if (is_write) { - bool handled = false; - - val = vcpu_gpr_read(thread, reg_num); - - if (is_acquire_release) { - atomic_thread_fence(memory_order_release); + if (!is_write && (ret == VCPU_TRAP_RESULT_EMULATED)) { + // Do we need to sign-extend the result? + if (ESR_EL2_ISS_DATA_ABORT_get_SSE(&iss) && + (size != sizeof(uint64_t))) { + uint64_t mask = util_bit((size * 8U) - 1U); + val = (val ^ mask) - mask; } - if (vdevice.type != VDEVICE_TYPE_NONE) { - handled = trigger_vdevice_access_event( - vdevice.type, vdevice.ptr, pa, ipa, - size, &val, true); - rcu_read_finish(); - } else { - handled = - trigger_vdevice_access_fixed_addr_event( - ipa, size, &val, true); + // Adjust the width if necessary + if (!ESR_EL2_ISS_DATA_ABORT_get_SF(&iss)) { + val = (uint32_t)val; } - if (handled) { - ret = VCPU_TRAP_RESULT_EMULATED; - } - } else { - bool handled = false; - if (vdevice.type != VDEVICE_TYPE_NONE) { - handled = trigger_vdevice_access_event( - vdevice.type, vdevice.ptr, pa, ipa, - size, &val, false); - rcu_read_finish(); - } else { - handled = - trigger_vdevice_access_fixed_addr_event( - ipa, size, &val, false); - } - if (handled) { - // Do we need to sign-extend the result? - if (ESR_EL2_ISS_DATA_ABORT_get_SSE(&iss) && - (size != sizeof(uint64_t))) { - uint64_t mask = 1ULL - << ((size * 8) - 1); - val = (val ^ mask) - mask; - } - - // Adjust the width if necessary - if (!ESR_EL2_ISS_DATA_ABORT_get_SF(&iss)) { - val = (uint32_t)val; - } - - vcpu_gpr_write(thread, reg_num, val); - - if (is_acquire_release) { - atomic_thread_fence( - memory_order_acquire); - } - - ret = VCPU_TRAP_RESULT_EMULATED; + vcpu_gpr_write(thread, reg_num, val); + + if (is_acquire_release) { + atomic_thread_fence(memory_order_acquire); } } } -out: return ret; } diff --git a/hyp/core/vdevice/build.conf b/hyp/core/vdevice/build.conf index a5fd814..2fd1e96 100644 --- a/hyp/core/vdevice/build.conf +++ b/hyp/core/vdevice/build.conf @@ -2,7 +2,11 @@ # # SPDX-License-Identifier: BSD-3-Clause +base_module hyp/misc/gpt interface vdevice types vdevice.tc +events vdevice.ev +local_include +source vdevice.c access.c arch_events aarch64 vdevice_aarch64.ev arch_source aarch64 vdevice.c diff --git a/hyp/core/vdevice/include/internal.h b/hyp/core/vdevice/include/internal.h new file mode 100644 index 0000000..f790f82 --- /dev/null +++ b/hyp/core/vdevice/include/internal.h @@ -0,0 +1,28 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +// Emulate an access to a virtual device backed by physical memory. +// +// This function looks up the given physical address in the memdb, finds the +// corresponding memextent object, and checks whether it is associated with a +// virtual device. If so, it triggers the access event. +// +// This function is intended to be called from a permission fault handler after +// obtaining the physical address from the guest's address space. Since the +// address space might be concurrently modified to unmap the physical address, +// this must be called from an RCU critical section to ensure that the physical +// address is not reused before it has finished. +vcpu_trap_result_t +vdevice_access_phys(paddr_t pa, size_t size, register_t *val, bool is_write) + REQUIRE_RCU_READ; + +// Emulate an access to a virtual device that is not backed by physical memory. +// +// This function looks up the IPA in the current guest address space's virtual +// device mapping. If a virtual device is found, the access event will be +// triggered. +// +// This function is intended to be called from a translation fault handler. +vcpu_trap_result_t +vdevice_access_ipa(vmaddr_t ipa, size_t size, register_t *val, bool is_write); diff --git a/hyp/core/vdevice/src/access.c b/hyp/core/vdevice/src/access.c new file mode 100644 index 0000000..739b78c --- /dev/null +++ b/hyp/core/vdevice/src/access.c @@ -0,0 +1,87 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include "internal.h" + +vcpu_trap_result_t +vdevice_access_phys(paddr_t pa, size_t size, register_t *val, bool is_write) +{ + vcpu_trap_result_t ret; + + memdb_obj_type_result_t res = memdb_lookup(pa); + if ((res.e != OK) || (res.r.type != MEMDB_TYPE_EXTENT)) { + ret = VCPU_TRAP_RESULT_UNHANDLED; + goto out; + } + + memextent_t *me = (memextent_t *)res.r.object; + assert(me != NULL); + vdevice_t *vdevice = atomic_load_consume(&me->vdevice); + if (vdevice == NULL) { + ret = VCPU_TRAP_RESULT_UNHANDLED; + goto out; + } + + size_result_t offset_r = memextent_get_offset_for_pa(me, pa, size); + if (offset_r.e != OK) { + ret = VCPU_TRAP_RESULT_UNHANDLED; + goto out; + } + + ret = trigger_vdevice_access_event(vdevice->type, vdevice, offset_r.r, + size, val, is_write); + +out: + return ret; +} + +vcpu_trap_result_t +vdevice_access_ipa(vmaddr_t ipa, size_t size, register_t *val, bool is_write) +{ + vcpu_trap_result_t ret; + + addrspace_t *addrspace = addrspace_get_self(); + assert(addrspace != NULL); + + rcu_read_start(); + + gpt_lookup_result_t lookup_ret = + gpt_lookup(&addrspace->vdevice_gpt, ipa, size); + + if (lookup_ret.size != size) { + ret = VCPU_TRAP_RESULT_UNHANDLED; + } else if (lookup_ret.entry.type == GPT_TYPE_VDEVICE) { + vdevice_t *vdevice = lookup_ret.entry.value.vdevice; + assert(vdevice != NULL); + assert((ipa >= vdevice->ipa) && + ((ipa + size - 1U) <= + (vdevice->ipa + vdevice->size - 1U))); + + ret = trigger_vdevice_access_event(vdevice->type, vdevice, + ipa - vdevice->ipa, size, + val, is_write); + } else { + assert(lookup_ret.entry.type == GPT_TYPE_EMPTY); + + // FIXME: + ret = trigger_vdevice_access_fixed_addr_event(ipa, size, val, + is_write); + } + + rcu_read_finish(); + + return ret; +} diff --git a/hyp/core/vdevice/src/vdevice.c b/hyp/core/vdevice/src/vdevice.c new file mode 100644 index 0000000..b6b3daf --- /dev/null +++ b/hyp/core/vdevice/src/vdevice.c @@ -0,0 +1,134 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "event_handlers.h" + +error_t +vdevice_attach_phys(vdevice_t *vdevice, memextent_t *memextent) +{ + assert(vdevice != NULL); + assert(memextent != NULL); + assert(vdevice->type != VDEVICE_TYPE_NONE); + + vdevice_t *null_vdevice = NULL; + return atomic_compare_exchange_strong_explicit(&memextent->vdevice, + &null_vdevice, vdevice, + memory_order_release, + memory_order_release) + ? OK + : ERROR_BUSY; +} + +void +vdevice_detach_phys(vdevice_t *vdevice, memextent_t *memextent) +{ + vdevice_t *old_vdevice = atomic_exchange_explicit( + &memextent->vdevice, NULL, memory_order_relaxed); + assert(old_vdevice == vdevice); +} + +bool +vdevice_handle_gpt_values_equal(gpt_type_t type, gpt_value_t x, gpt_value_t y) +{ + assert(type == GPT_TYPE_VDEVICE); + + return x.vdevice == y.vdevice; +} + +error_t +vdevice_handle_object_create_addrspace(addrspace_create_t params) +{ + addrspace_t *addrspace = params.addrspace; + assert(addrspace != NULL); + + spinlock_init(&addrspace->vdevice_lock); + + gpt_config_t config = gpt_config_default(); + gpt_config_set_max_bits(&config, VDEVICE_MAX_GPT_BITS); + gpt_config_set_rcu_read(&config, true); + + return gpt_init(&addrspace->vdevice_gpt, addrspace->header.partition, + config, util_bit((index_t)GPT_TYPE_VDEVICE)); +} + +void +vdevice_handle_object_cleanup_addrspace(addrspace_t *addrspace) +{ + assert(addrspace != NULL); + + gpt_destroy(&addrspace->vdevice_gpt); +} + +error_t +vdevice_attach_vmaddr(vdevice_t *vdevice, addrspace_t *addrspace, vmaddr_t ipa, + size_t size) +{ + error_t err; + + assert(vdevice != NULL); + assert(addrspace != NULL); + assert(vdevice->type != VDEVICE_TYPE_NONE); + + if (vdevice->addrspace != NULL) { + err = ERROR_BUSY; + goto out; + } + + gpt_entry_t entry = { + .type = GPT_TYPE_VDEVICE, + .value = { .vdevice = vdevice }, + }; + + spinlock_acquire(&addrspace->vdevice_lock); + + err = gpt_insert(&addrspace->vdevice_gpt, ipa, size, entry, true); + + spinlock_release(&addrspace->vdevice_lock); + + if (err == OK) { + vdevice->addrspace = object_get_addrspace_additional(addrspace); + vdevice->ipa = ipa; + vdevice->size = size; + } + +out: + return err; +} + +void +vdevice_detach_vmaddr(vdevice_t *vdevice) +{ + assert(vdevice != NULL); + assert(vdevice->type != VDEVICE_TYPE_NONE); + + addrspace_t *addrspace = vdevice->addrspace; + assert(addrspace != NULL); + + gpt_entry_t entry = { + .type = GPT_TYPE_VDEVICE, + .value = { .vdevice = vdevice }, + }; + + spinlock_acquire(&addrspace->vdevice_lock); + + error_t err = gpt_remove(&addrspace->vdevice_gpt, vdevice->ipa, + vdevice->size, entry); + assert(err == OK); + + spinlock_release(&addrspace->vdevice_lock); + + vdevice->addrspace = NULL; + + object_put_addrspace(addrspace); +} diff --git a/hyp/core/vdevice/vdevice.ev b/hyp/core/vdevice/vdevice.ev new file mode 100644 index 0000000..0621bd0 --- /dev/null +++ b/hyp/core/vdevice/vdevice.ev @@ -0,0 +1,12 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module vdevice + +subscribe gpt_values_equal[GPT_TYPE_VDEVICE] + +subscribe object_create_addrspace + unwinder + +subscribe object_cleanup_addrspace(addrspace) diff --git a/hyp/core/vdevice/vdevice.tc b/hyp/core/vdevice/vdevice.tc index 6aa068d..2d95a65 100644 --- a/hyp/core/vdevice/vdevice.tc +++ b/hyp/core/vdevice/vdevice.tc @@ -2,11 +2,28 @@ // // SPDX-License-Identifier: BSD-3-Clause -define vdevice_typed_ptr structure(aligned(16)) { - ptr union vdevice_ptr; - type enumeration vdevice_type; -}; +// FIXME: Get this from the addrspace instead? +define VDEVICE_MAX_GPT_BITS constant type count_t = 44; extend memextent object { - vdevice structure vdevice_typed_ptr(atomic); + vdevice pointer(atomic) structure vdevice; +}; + +extend vdevice structure { + addrspace pointer object addrspace; + ipa type vmaddr_t; + size size; +}; + +extend addrspace object { + vdevice_gpt structure gpt; + vdevice_lock structure spinlock; +}; + +extend gpt_type enumeration { + vdevice; +}; + +extend gpt_value union { + vdevice pointer structure vdevice; }; diff --git a/hyp/core/vectors/aarch64/include/trap_dispatch.h b/hyp/core/vectors/aarch64/include/trap_dispatch.h index 3ac02d9..eedc118 100644 --- a/hyp/core/vectors/aarch64/include/trap_dispatch.h +++ b/hyp/core/vectors/aarch64/include/trap_dispatch.h @@ -3,10 +3,11 @@ // SPDX-License-Identifier: BSD-3-Clause void -vectors_exception_dispatch(kernel_trap_frame_full_t *frame); +vectors_exception_dispatch(kernel_trap_frame_full_t *frame) + REQUIRE_PREEMPT_DISABLED; SPSR_EL2_A64_t -vectors_interrupt_dispatch(void); +vectors_interrupt_dispatch(void) REQUIRE_PREEMPT_DISABLED; void vectors_dump_regs(kernel_trap_frame_full_t *frame); diff --git a/hyp/core/vectors/aarch64/include/vectors_el2.inc b/hyp/core/vectors/aarch64/include/vectors_el2.inc index f0dc3cd..db5c4eb 100644 --- a/hyp/core/vectors/aarch64/include/vectors_el2.inc +++ b/hyp/core/vectors/aarch64/include/vectors_el2.inc @@ -22,7 +22,7 @@ #endif .endm -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) // Define symbols for pointer auth offsets so we can access them from macros .equ pauth_DA_ofs, OFS_AARCH64_PAUTH_KEYS_DA .equ pauth_DB_ofs, OFS_AARCH64_PAUTH_KEYS_DB @@ -40,7 +40,7 @@ // Macro for restoring the EL2 pointer auth keys (discarding any EL1 keys) .macro kernel_pauth_entry kp:req, kl:req, kh:req -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) adrl \kp, aarch64_pauth_keys kernel_pauth_entry_key DA, \kp, \kl, \kh kernel_pauth_entry_key DB, \kp, \kl, \kh @@ -65,7 +65,7 @@ b.ne handle_stack_fault .endm -.macro save_kernel_context full:req +.macro save_kernel_context full:req overflow:req disable_phys_access .if \full @@ -78,7 +78,7 @@ sub sp, sp, KERNEL_TRAP_FRAME_SIZE .endif -.if \full +.if \overflow // The exception may have been triggered by a stack overflow, // so check for this before storing anything on the stack. check_stack_overflow @@ -98,7 +98,7 @@ .endif stp x4, x5, [sp, OFS_KERNEL_TRAP_FRAME_X(4)] -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) // Insert PAC bits in ELR_EL2 before we save it (matching the ERETAA // in vectors_kernel_return) pacia x0, x2 @@ -166,12 +166,16 @@ // For the nested faults the stack may be corrupted, so we switch to a special // stack set aside for this purpose. .macro save_kernel_context_stack_fault - // At this point we don't care about preserving the value of TPIDR_EL0 // and TPIDR_EL1 msr TPIDR_EL0, x0 msr TPIDR_EL1, x1 + // The emergency stacks must use the emergency vectors. + adr x0, emergency_vectors_aarch64 + msr VBAR_EL2, x0 + isb + // Get the CPU number and use it to calculate the address of the nested // fault context buffer for this CPU adr_threadlocal x1, current_thread + OFS_THREAD_CPULOCAL_CURRENT_CPU @@ -238,7 +242,7 @@ .endm // Hypervisor self vectors -.macro el2_vectors name +.macro el2_vectors name:req, overflow:req vector vector_el2t_sync_\name\() save_kernel_context_full mov x0, sp @@ -270,7 +274,7 @@ // Hypervisor nested vectors vector vector_el2h_sync_\name\() - save_kernel_context 1 + save_kernel_context 1 \overflow mov x0, sp bl vectors_exception_dispatch_full @@ -279,7 +283,7 @@ vector_end vector_el2h_sync_\name\() vector vector_el2h_irq_\name\() - save_kernel_context 0 + save_kernel_context 0 0 bl vectors_interrupt_dispatch b vectors_kernel_return @@ -294,7 +298,7 @@ vector_end vector_el2h_fiq_\name\() vector vector_el2h_serror_\name\() - save_kernel_context 1 + save_kernel_context 1 0 // The dispatcher will inject a virtual SError to the RAS VM. mov x0, sp @@ -303,3 +307,50 @@ b vectors_kernel_return vector_end vector_el2h_serror_\name\() .endm + +.macro default_guest_vectors name:req + vector vector_guest64_sync_\name\() + save_kernel_context_full + panic "64-bit guest vectors" + vector_end vector_guest64_sync_\name\() + + vector vector_guest64_irq_\name\() + save_kernel_context_full + panic "64-bit guest vectors" + vector_end vector_guest64_irq_\name\() + + vector vector_guest64_fiq_\name\() + save_kernel_context_full + panic "64-bit guest vectors" + vector_end vector_guest64_fiq_\name\() + + vector vector_guest64_serror_\name\() + save_kernel_context_full + panic "64-bit guest vectors" + vector_end vector_guest64_serror_\name\() + + vector vector_guest32_sync_\name\() + save_kernel_context_full + panic "32-bit guest vectors" + vector_end vector_guest32_sync_\name\() + + vector vector_guest32_irq_\name\() + save_kernel_context_full + panic "32-bit guest vectors" + vector_end vector_guest32_irq_\name\() + + vector vector_guest32_fiq_\name\() + save_kernel_context_full + panic "32-bit guest vectors" + vector_end vector_guest32_fiq_\name\() + + vector vector_guest32_serror_\name\() + save_kernel_context_full + panic "32-bit guest vectors" + vector_end vector_guest32_serror_\name\() +.endm + +.macro default_vectors name:req, overflow:req + el2_vectors \name \overflow + default_guest_vectors \name +.endm diff --git a/hyp/core/vectors/aarch64/src/exception_debug.c b/hyp/core/vectors/aarch64/src/exception_debug.c index 44364a4..4e5c5ac 100644 --- a/hyp/core/vectors/aarch64/src/exception_debug.c +++ b/hyp/core/vectors/aarch64/src/exception_debug.c @@ -67,6 +67,8 @@ dump_self_sync_fault(kernel_trap_frame_full_t *frame) { TRACE_AND_LOG(ERROR, WARN, "EL2t synchronous fault"); vectors_dump_regs(frame); + + // FIXME: } void @@ -74,6 +76,8 @@ dump_self_irq_fault(kernel_trap_frame_full_t *frame) { TRACE_AND_LOG(ERROR, WARN, "EL2t IRQ"); vectors_dump_regs(frame); + + // FIXME: } void @@ -81,6 +85,8 @@ dump_self_fiq_fault(kernel_trap_frame_full_t *frame) { TRACE_AND_LOG(ERROR, WARN, "EL2t FIQ fault"); vectors_dump_regs(frame); + + // FIXME: } void @@ -88,6 +94,8 @@ dump_self_serror(kernel_trap_frame_full_t *frame) { TRACE_AND_LOG(ERROR, WARN, "EL2t SError fault"); vectors_dump_regs(frame); + + // FIXME: } void @@ -95,4 +103,6 @@ dump_nested_fault(kernel_trap_frame_full_t *frame) { TRACE_AND_LOG(ERROR, WARN, "EL2 stack fault"); vectors_dump_regs(frame); + + // FIXME: } diff --git a/hyp/core/vectors/aarch64/src/trap_dispatch.c b/hyp/core/vectors/aarch64/src/trap_dispatch.c index 5363e8f..12f348e 100644 --- a/hyp/core/vectors/aarch64/src/trap_dispatch.c +++ b/hyp/core/vectors/aarch64/src/trap_dispatch.c @@ -25,32 +25,47 @@ #include "event_handlers.h" #include "trap_dispatch.h" +#if defined(ARCH_ARM_FEAT_PAuth) +static inline uintptr_t +remove_pointer_auth(uintptr_t addr) +{ + __asm__("xpaci %0" : "+r"(addr)); + return addr; +} +#endif + static inline uintptr_t vectors_get_return_address(kernel_trap_frame_t *frame) { -#if defined(ARCH_ARM_8_3_PAUTH) - uintptr_t pc = ELR_EL2_get_ReturnAddress(&frame->pc); - __asm__("xpaci %0" : "+r"(pc)); - return pc; +#if defined(ARCH_ARM_FEAT_PAuth) + return remove_pointer_auth(ELR_EL2_get_ReturnAddress(&frame->pc)); #else return ELR_EL2_get_ReturnAddress(&frame->pc); #endif } -static inline ALWAYS_INLINE void -vectors_set_return_address(kernel_trap_frame_t *frame, uintptr_t pc) +#if defined(ARCH_ARM_FEAT_PAuth) +static inline ALWAYS_INLINE uintptr_t +sign_pc_using_framepointer(uintptr_t pc, uintptr_t fp) { -#if defined(ARCH_ARM_8_3_PAUTH) - uintptr_t signed_pc = pc; // The new PC needs to be signed with a modifier equal to the value the // SP will have after restoring the frame, i.e. the address immediately // after the end of the frame. Note that this must be inlined and BTI // must be enabled to avoid providing a gadget for signing an arbitrary // return address. - __asm__("pacia %0, %1" - : "+r"(signed_pc) - : "r"((uintptr_t)(SP_EL2_raw(frame->sp_el2)))); - ELR_EL2_set_ReturnAddress(&frame->pc, signed_pc); + __asm__("pacia %0, %1" : "+r"(pc) : "r"(fp)); + return pc; +} +#endif + +static inline ALWAYS_INLINE void +vectors_set_return_address(kernel_trap_frame_t *frame, uintptr_t pc) +{ +#if defined(ARCH_ARM_FEAT_PAuth) + ELR_EL2_set_ReturnAddress( + &frame->pc, + sign_pc_using_framepointer( + pc, (uintptr_t)(SP_EL2_raw(frame->sp_el2)))); #else ELR_EL2_set_ReturnAddress(&frame->pc, pc); #endif @@ -67,8 +82,8 @@ vectors_exception_dispatch(kernel_trap_frame_full_t *frame) ESR_EL2_t esr = register_ESR_EL2_read_ordered(&asm_ordering); esr_ec_t ec = ESR_EL2_get_EC(&esr); uintptr_t pc = ELR_EL2_get_ReturnAddress(&frame->base.pc); -#if defined(ARCH_ARM_8_3_PAUTH) - __asm__("xpaci %0" : "+r"(pc)); +#if defined(ARCH_ARM_FEAT_PAuth) + pc = remove_pointer_auth(pc); #endif TRACE(ERROR, WARN, "EL2 exception at PC = {:x} ESR_EL2 = {:#x}, LR = {:#x}, " @@ -81,7 +96,7 @@ vectors_exception_dispatch(kernel_trap_frame_full_t *frame) handled = trigger_vectors_trap_unknown_el2_event(&frame->base); break; -#if defined(ARCH_ARM_8_5_BTI) +#if defined(ARCH_ARM_FEAT_BTI) case ESR_EC_BTI: TRACE_AND_LOG(ERROR, WARN, "BTI abort in EL2 on CPU {:d}, from {:#x}, " @@ -121,7 +136,7 @@ vectors_exception_dispatch(kernel_trap_frame_full_t *frame) handled = trigger_vectors_trap_brk_el2_event(esr); break; -#if defined(ARCH_ARM_8_3_PAUTH) && defined(ARCH_ARM_8_3_FPAC) +#if defined(ARCH_ARM_FEAT_PAuth) && defined(ARCH_ARM_FEAT_FPAC) case ESR_EC_FPAC: handled = trigger_vectors_trap_pauth_failed_el2_event(esr); break; @@ -151,20 +166,36 @@ vectors_exception_dispatch(kernel_trap_frame_full_t *frame) case ESR_EC_SVC64: case ESR_EC_HVC64_EL2: case ESR_EC_SMC64_EL2: -#if defined(ARCH_ARM_8_3_PAUTH) + case ESR_EC_INST_ABT_LO: + case ESR_EC_DATA_ABT_LO: + case ESR_EC_FP64: +#if defined(ARCH_ARM_FEAT_PAuth) case ESR_EC_PAUTH: #endif -#if defined(ARCH_ARM_8_3_PAUTH) && defined(ARCH_ARM_8_3_NV) +#if defined(ARCH_ARM_FEAT_PAuth) && defined(ARCH_ARM_FEAT_NV) case ESR_EC_ERET: #endif case ESR_EC_SYSREG: -#if defined(ARCH_ARM_8_2_SVE) +#if defined(ARCH_ARM_FEAT_SVE) case ESR_EC_SVE: #endif - case ESR_EC_INST_ABT_LO: - case ESR_EC_DATA_ABT_LO: - case ESR_EC_FP64: +#if defined(ARCH_ARM_FEAT_LS64) + case ESR_EC_LD64B_ST64B: +#endif +#if defined(ARCH_ARM_FEAT_TME) + case ESR_EC_TSTART: +#endif +#if defined(ARCH_ARM_FEAT_SME) + case ESR_EC_SME: +#endif +#if defined(ARCH_ARM_FEAT_RME) + case ESR_EC_RME: +#endif +#if defined(ARCH_ARM_FEAT_MOPS) + case ESR_EC_MOPS: +#endif default: + // Unexpected trap, fall through to panic break; } @@ -209,8 +240,12 @@ vectors_handle_abort_kernel(void) // HLT instruction will stop if an external debugger is attached, // otherwise it generates an exception and the trap handler below will // skip the instruction. +#if !defined(QQVP_SIMULATION_PLATFORM) || !QQVP_SIMULATION_PLATFORM + // The QQVP model exits with invalid instruction here. + // This should be resolved in model. __asm__ volatile("hlt 1" ::: "memory"); #endif +#endif } bool @@ -234,8 +269,9 @@ vectors_handle_vectors_trap_unknown_el2(kernel_trap_frame_t *frame) if ((inst & AARCH64_INST_EXCEPTION_MASK) == AARCH64_INST_EXCEPTION_VAL) { - uint16_t imm16 = (inst & AARCH64_INST_EXCEPTION_IMM16_MASK) >> - AARCH64_INST_EXCEPTION_IMM16_SHIFT; + uint16_t imm16 = + (uint16_t)((inst & AARCH64_INST_EXCEPTION_IMM16_MASK) >> + AARCH64_INST_EXCEPTION_IMM16_SHIFT); if ((inst & AARCH64_INST_EXCEPTION_SUBTYPE_MASK) == AARCH64_INST_EXCEPTION_SUBTYPE_HLT_VAL) { diff --git a/hyp/core/vectors/armv8-64/src/return.S b/hyp/core/vectors/armv8-64/src/return.S index d121aa3..bc6d096 100644 --- a/hyp/core/vectors/armv8-64/src/return.S +++ b/hyp/core/vectors/armv8-64/src/return.S @@ -39,7 +39,7 @@ function vectors_kernel_return mov sp, x1 ldp x0, x1, [x0, OFS_KERNEL_TRAP_FRAME_X(0)] -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) // Matching PACIA is in save_kernel_context eretaa #else diff --git a/hyp/core/vectors/armv8-64/src/vectors.S b/hyp/core/vectors/armv8-64/src/vectors.S index ed6c2bc..32fee13 100644 --- a/hyp/core/vectors/armv8-64/src/vectors.S +++ b/hyp/core/vectors/armv8-64/src/vectors.S @@ -14,51 +14,16 @@ // The vector table base is 2KB aligned // (16 vectors, 32 instructions each) .balign 2048 -.global vectors_aarch64 -vectors_aarch64: +.global kernel_vectors_aarch64 +kernel_vectors_aarch64: -el2_vectors kernel +default_vectors kernel 1 -// Guest vectors -vector vector_guest64_sync_panic - save_kernel_context_full - panic "64-bit guest vectors" -vector_end vector_guest64_sync_panic - -vector vector_guest64_irq_panic - save_kernel_context_full - panic "64-bit guest vectors" -vector_end vector_guest64_irq_panic - -vector vector_guest64_fiq_panic - save_kernel_context_full - panic "64-bit guest vectors" -vector_end vector_guest64_fiq_panic - -vector vector_guest64_serror_panic - save_kernel_context_full - panic "64-bit guest vectors" -vector_end vector_guest64_serror_panic - -vector vector_guest32_sync_panic - save_kernel_context_full - panic "32-bit guest vectors" -vector_end vector_guest32_sync_panic - -vector vector_guest32_irq_panic - save_kernel_context_full - panic "32-bit guest vectors" -vector_end vector_guest32_irq_panic - -vector vector_guest32_fiq_panic - save_kernel_context_full - panic "32-bit guest vectors" -vector_end vector_guest32_fiq_panic + .balign 2048 +.global emergency_vectors_aarch64 +emergency_vectors_aarch64: -vector vector_guest32_serror_panic - save_kernel_context_full - panic "32-bit guest vectors" -vector_end vector_guest32_serror_panic +default_vectors emergency 0 function vectors_exception_dispatch_full // In save_kernel_context, space was reserved for a full frame. diff --git a/hyp/core/wait_queue_broadcast/src/wait_queue.c b/hyp/core/wait_queue_broadcast/src/wait_queue.c index 5f1382a..abd7d88 100644 --- a/hyp/core/wait_queue_broadcast/src/wait_queue.c +++ b/hyp/core/wait_queue_broadcast/src/wait_queue.c @@ -16,6 +16,7 @@ #include "event_handlers.h" +// FIXME: // Unify list code in util module scheduler_block_properties_t @@ -43,7 +44,7 @@ wait_queue_init(wait_queue_t *wait_queue) void wait_queue_prepare(wait_queue_t *wait_queue) LOCK_IMPL { - thread_t *self = thread_get_self(); + thread_t *self = thread_get_self(); list_node_t *node = &self->wait_queue_list_node; assert(wait_queue != NULL); @@ -71,7 +72,7 @@ wait_queue_finish(wait_queue_t *wait_queue) LOCK_IMPL // Dequeue the thread list_node_t *node = &self->wait_queue_list_node; - list_delete_node(&wait_queue->list, node); + (void)list_delete_node(&wait_queue->list, node); spinlock_release(&wait_queue->lock); @@ -99,7 +100,7 @@ wait_queue_put(void) LOCK_IMPL assert_preempt_disabled(); scheduler_lock(self); - scheduler_unblock(self, SCHEDULER_BLOCK_WAIT_QUEUE); + (void)scheduler_unblock(self, SCHEDULER_BLOCK_WAIT_QUEUE); scheduler_unlock(self); } @@ -123,7 +124,7 @@ wait_queue_wakeup(wait_queue_t *wait_queue) // Wakeup all waiters thread_t *thread; - list_t *list = &wait_queue->list; + list_t *list = &wait_queue->list; list_foreach_container (thread, list, thread, wait_queue_list_node) { scheduler_lock_nopreempt(thread); @@ -133,9 +134,11 @@ wait_queue_wakeup(wait_queue_t *wait_queue) scheduler_unlock_nopreempt(thread); } - spinlock_release(&wait_queue->lock); + spinlock_release_nopreempt(&wait_queue->lock); if (wakeup_any) { scheduler_trigger(); } + + preempt_enable(); } diff --git a/hyp/debug/object_lists/templates/object_lists.tc.tmpl b/hyp/debug/object_lists/templates/object_lists.tc.tmpl index 0fd8cd1..d4b2bc8 100644 --- a/hyp/debug/object_lists/templates/object_lists.tc.tmpl +++ b/hyp/debug/object_lists/templates/object_lists.tc.tmpl @@ -11,10 +11,21 @@ extend partition object { ${o}_list structure list; - ${o}_list_lock structure spinlock; }; extend $o object { ${o}_list_node structure list_node(contained); }; #end for + +#for obj in $object_list +#set o = str(obj) + +#if o in ('partition','hwirq',) +#continue +#end if + +extend partition object { + ${o}_list_lock structure spinlock; +}; +#end for diff --git a/hyp/debug/symbol_version/aarch64/src/sym_ver.S b/hyp/debug/symbol_version/aarch64/src/sym_ver.S index 5e9142f..92a4edb 100644 --- a/hyp/debug/symbol_version/aarch64/src/sym_ver.S +++ b/hyp/debug/symbol_version/aarch64/src/sym_ver.S @@ -10,6 +10,11 @@ // externally visible and not optimized by LTO - usually for debuggers .section .text.debug, "ax", @progbits adrp x0, hyp_sym_version_pointer + adrp x1, platform_gicd_base + adrp x2, platform_gicrs_bases +#if GICV3_HAS_ITS + adrp x3, platform_gits_base +#endif // We use a random value as the symbols version, put it directly in a symbol and // in memory. The external debugger reads the values of this symbol and the diff --git a/hyp/interfaces/addrspace/addrspace.ev b/hyp/interfaces/addrspace/addrspace.ev index 3744aac..8d3a04c 100644 --- a/hyp/interfaces/addrspace/addrspace.ev +++ b/hyp/interfaces/addrspace/addrspace.ev @@ -9,3 +9,28 @@ handled_event addrspace_attach_vdma param vdma_device_cap: cap_id_t param index: index_t return: error_t = ERROR_CSPACE_WRONG_OBJECT_TYPE + +handled_event addrspace_map + param addrspace: addrspace_t * + param vbase: vmaddr_t + param size: size_t + param phys: paddr_t + param memtype: pgtable_vm_memtype_t + param kernel_access: pgtable_access_t + param user_access: pgtable_access_t + return: error_t = ERROR_UNIMPLEMENTED + +handled_event addrspace_unmap + param addrspace: addrspace_t * + param vbase: vmaddr_t + param size: size_t + param phys: paddr_t + return: error_t = ERROR_UNIMPLEMENTED + +handled_event addrspace_attach_vdevice + param addrspace: addrspace_t * + param vdevice_cap: cap_id_t + param index: index_t + param vbase: vmaddr_t + param size: size_t + return: error_t = ERROR_CSPACE_WRONG_OBJECT_TYPE diff --git a/hyp/interfaces/addrspace/addrspace.hvc b/hyp/interfaces/addrspace/addrspace.hvc index cdabddd..55b446c 100644 --- a/hyp/interfaces/addrspace/addrspace.hvc +++ b/hyp/interfaces/addrspace/addrspace.hvc @@ -6,7 +6,7 @@ define addrspace_attach_thread hypercall { call_num 0x2A; addrspace input type cap_id_t; thread input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; @@ -16,7 +16,11 @@ define addrspace_map hypercall { memextent input type cap_id_t; vbase input type vmaddr_t; map_attrs input bitfield memextent_mapping_attrs; - _res0 input uregister; + map_flags input bitfield addrspace_map_flags; + // Below arguments are used only if partial is set in map_flags; + // otherwise they are ignored for backwards compatibility. + offset input size; + size input size; error output enumeration error; }; @@ -25,7 +29,11 @@ define addrspace_unmap hypercall { addrspace input type cap_id_t; memextent input type cap_id_t; vbase input type vmaddr_t; - _res0 input uregister; + map_flags input bitfield addrspace_map_flags; + // Below arguments are used only if partial is set in map_flags; + // otherwise they are ignored for backwards compatibility. + offset input size; + size input size; error output enumeration error; }; @@ -35,7 +43,11 @@ define addrspace_update_access hypercall { memextent input type cap_id_t; vbase input type vmaddr_t; access_attrs input bitfield memextent_access_attrs; - _res0 input uregister; + map_flags input bitfield addrspace_map_flags; + // Below arguments are used only if partial is set in map_flags; + // otherwise they are ignored for backwards compatibility. + offset input size; + size input size; error output enumeration error; }; @@ -43,7 +55,7 @@ define addrspace_configure hypercall { call_num 0x2E; addrspace input type cap_id_t; vmid input type vmid_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; @@ -52,6 +64,49 @@ define addrspace_attach_vdma hypercall { addrspace input type cap_id_t; dma_device input type cap_id_t; index input type index_t; - _res0 input uregister; + res0 input uregister; + error output enumeration error; +}; + +define addrspace_lookup hypercall { + call_num 0x5A; + addrspace input type cap_id_t; + memextent input type cap_id_t; + vbase input type vmaddr_t; + size input size; + res0 input uregister; + error output enumeration error; + offset output size; + size output size; + map_attrs output bitfield memextent_mapping_attrs; +}; + +define addrspace_configure_info_area hypercall { + call_num 0x5B; + addrspace input type cap_id_t; + info_area_me input type cap_id_t; + ipa input type vmaddr_t; + res0 input uregister; + error output enumeration error; +}; + +define addrspace_configure_vmmio hypercall { + call_num 0x60; + addrspace input type cap_id_t; + vbase input type vmaddr_t; + size input size; + op input enumeration addrspace_vmmio_configure_op; + res0 input uregister; + error output enumeration error; +}; + +define addrspace_attach_vdevice hypercall { + call_num 0x62; + addrspace input type cap_id_t; + vdevice input type cap_id_t; + index input type index_t; + vbase input type vmaddr_t; + size input size; + res0 input uregister; error output enumeration error; }; diff --git a/hyp/interfaces/addrspace/addrspace.tc b/hyp/interfaces/addrspace/addrspace.tc new file mode 100644 index 0000000..dfc1fb8 --- /dev/null +++ b/hyp/interfaces/addrspace/addrspace.tc @@ -0,0 +1,31 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +define addrspace_map_flags public bitfield<32> { + 0 partial bool; + 31 no_sync bool; + 30:1 res0_0 uregister(const) = 0; +}; + +define addrspace_lookup structure { + phys type paddr_t; + size size; + memtype enumeration pgtable_vm_memtype; + kernel_access enumeration pgtable_access; + user_access enumeration pgtable_access; +}; + +// Allocate one page for the info area for now +define MAX_VM_INFO_AREA_SIZE constant = PGTABLE_HYP_PAGE_SIZE; + +// The modules that need to use the VM Info Area will extend this structure +// and add their own entries, then use the generated offsets. This structure +// should never be directly used or referenced. +define addrspace_info_area_layout structure { +}; + +define addrspace_vmmio_configure_op public enumeration(explicit) { + ADD = 0; + REMOVE = 1; +}; diff --git a/hyp/interfaces/addrspace/build.conf b/hyp/interfaces/addrspace/build.conf index 48e0d26..6c45429 100644 --- a/hyp/interfaces/addrspace/build.conf +++ b/hyp/interfaces/addrspace/build.conf @@ -5,3 +5,4 @@ first_class_object addrspace hypercalls addrspace.hvc events addrspace.ev +types addrspace.tc diff --git a/hyp/interfaces/addrspace/include/addrspace.h b/hyp/interfaces/addrspace/include/addrspace.h index 101a1b4..2a271e9 100644 --- a/hyp/interfaces/addrspace/include/addrspace.h +++ b/hyp/interfaces/addrspace/include/addrspace.h @@ -12,6 +12,10 @@ error_t addrspace_attach_thread(addrspace_t *addrspace, thread_t *thread); +// Get a pointer to the thread's addrspace. +addrspace_t * +addrspace_get_self(void); + // Configure the address space. // // The object's header lock must be held and object state must be @@ -19,10 +23,65 @@ addrspace_attach_thread(addrspace_t *addrspace, thread_t *thread); error_t addrspace_configure(addrspace_t *addrspace, vmid_t vmid); +// Configure the address space information area. +// +// The object's header lock must be held and object state must be +// OBJECT_STATE_INIT. +error_t +addrspace_configure_info_area(addrspace_t *addrspace, memextent_t *info_area_me, + vmaddr_t ipa); + +// Nominate an address range as being handled by an unprivileged VMM. +// +// This can return ERROR_NORESOURCES if the implementation has reached a limit +// of nominated address ranges, ERROR_ARGUMENT_INVALID if the specified range +// overlaps an existing VMMIO range, or ERROR_UNIMPLEMENTED if there is no way +// to forward faults to an unprivileged VMM. +error_t +addrspace_add_vmmio_range(addrspace_t *addrspace, vmaddr_t base, size_t size); + +// Remove an address range from the ranges handled by an unprivileged VMM. +// +// The range must match one that was previously added to the address space by +// calling addrspace_add_vmmio_range(). +error_t +addrspace_remove_vmmio_range(addrspace_t *addrspace, vmaddr_t base, + size_t size); + // Translate a VA to PA in the current guest address space. +// +// Returns ERROR_DENIED if the lookup faults in stage 2 (including during the +// stage 1 page table walk), or ERROR_ADDR_INVALID if it faults in stage 1, +// regardless of the cause of the fault. +// +// Requires the RCU read lock, because the returned physical address might be +// unmapped concurrently and then reused after an RCU grace period. paddr_result_t -addrspace_va_to_pa_read(gvaddr_t addr); +addrspace_va_to_pa_read(gvaddr_t addr) REQUIRE_RCU_READ; // Translate a VA to IPA in the current guest address space. +// +// Returns ERROR_DENIED if the lookup faults in stage 2 (during the stage 1 page +// table walk), or ERROR_ADDR_INVALID if it faults in stage 1, regardless of the +// cause of the fault. vmaddr_result_t addrspace_va_to_ipa_read(gvaddr_t addr); + +// Check whether an address range is within the address space. +error_t +addrspace_check_range(addrspace_t *addrspace, vmaddr_t base, size_t size); + +// Map into an addrspace. +error_t +addrspace_map(addrspace_t *addrspace, vmaddr_t vbase, size_t size, paddr_t phys, + pgtable_vm_memtype_t memtype, pgtable_access_t kernel_access, + pgtable_access_t user_access); + +// Unmap from an addrspace. +error_t +addrspace_unmap(addrspace_t *addrspace, vmaddr_t vbase, size_t size, + paddr_t phys); + +// Lookup a mapping in the addrspace. +addrspace_lookup_result_t +addrspace_lookup(addrspace_t *addrspace, vmaddr_t vbase, size_t size); diff --git a/hyp/interfaces/allocator/allocator.ev b/hyp/interfaces/allocator/allocator.ev new file mode 100644 index 0000000..1954afd --- /dev/null +++ b/hyp/interfaces/allocator/allocator.ev @@ -0,0 +1,16 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +interface allocator + +// Add normal memory from the partition to its allocator. +// +// The specified partition is the partition that we expect owns the memory. +setup_event allocator_add_ram_range + param owner: partition_t * + param phys_base: paddr_t + param virt_base: uintptr_t + param size: size_t + return: error_t = ERROR_FAILURE + success: OK diff --git a/hyp/interfaces/allocator/build.conf b/hyp/interfaces/allocator/build.conf index e0c03fc..87c34f6 100644 --- a/hyp/interfaces/allocator/build.conf +++ b/hyp/interfaces/allocator/build.conf @@ -3,3 +3,4 @@ # SPDX-License-Identifier: BSD-3-Clause types allocator.tc +events allocator.ev diff --git a/hyp/interfaces/allocator/include/allocator.h b/hyp/interfaces/allocator/include/allocator.h index bffd5a1..5e32363 100644 --- a/hyp/interfaces/allocator/include/allocator.h +++ b/hyp/interfaces/allocator/include/allocator.h @@ -12,12 +12,6 @@ error_t allocator_init(allocator_t *allocator); -error_t -allocator_heap_add_memory(allocator_t *allocator, void *addr, size_t size); - -error_t -allocator_heap_remove_memory(allocator_t *allocator, void *obj, size_t size); - void_ptr_result_t allocator_allocate_object(allocator_t *allocator, size_t size, size_t alignment); diff --git a/hyp/interfaces/api/api.tc b/hyp/interfaces/api/api.tc index 5a3acd2..71d6b9b 100644 --- a/hyp/interfaces/api/api.tc +++ b/hyp/interfaces/api/api.tc @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: BSD-3-Clause +// FIXME: #define HYP_VARIANT_GUNYAH 0x48 #define HYP_VARIANT_QUALCOMM 0x51 @@ -31,8 +32,9 @@ define hyp_api_flags0 public bitfield<64>(const) { 8 watchdog bool = 0; 9 virtio_mmio bool = 0; 10 prng bool = 0; + 16 reserved bool = 0; 31:28 scheduler enumeration scheduler_variant = SCHEDULER_VARIANT; - 63:32,27:16,15:11 res0_0 uint64 = 0; + 63:32,27:17,15:11 res0_0 uint64 = 0; }; define hyp_api_flags1 public bitfield<64>(const) { diff --git a/hyp/interfaces/arm_fgt/arm_fgt.reg b/hyp/interfaces/arm_fgt/arm_fgt.reg new file mode 100644 index 0000000..133f1d0 --- /dev/null +++ b/hyp/interfaces/arm_fgt/arm_fgt.reg @@ -0,0 +1,7 @@ +// © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier; BSD-3-Clause + +#if defined(ARCH_ARM_FEAT_FGT) +HFGWTR_EL2 +#endif diff --git a/hyp/interfaces/arm_fgt/arm_fgt.tc b/hyp/interfaces/arm_fgt/arm_fgt.tc new file mode 100644 index 0000000..cfea7d5 --- /dev/null +++ b/hyp/interfaces/arm_fgt/arm_fgt.tc @@ -0,0 +1,11 @@ +// © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +extend platform_cpu_features bitfield { + auto fgt_disable bool = 1; +}; + +extend global_options bitfield { + auto fgt bool = 0; +}; diff --git a/hyp/interfaces/arm_fgt/build.conf b/hyp/interfaces/arm_fgt/build.conf new file mode 100644 index 0000000..2b1aa31 --- /dev/null +++ b/hyp/interfaces/arm_fgt/build.conf @@ -0,0 +1,6 @@ +# © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +types arm_fgt.tc +registers arm_fgt.reg diff --git a/hyp/interfaces/arm_fgt/include/arm_fgt.h b/hyp/interfaces/arm_fgt/include/arm_fgt.h new file mode 100644 index 0000000..5d5df8a --- /dev/null +++ b/hyp/interfaces/arm_fgt/include/arm_fgt.h @@ -0,0 +1,6 @@ +// © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +bool +arm_fgt_is_allowed(void); diff --git a/hyp/interfaces/arm_sve/arm_sve.tc b/hyp/interfaces/arm_sve/arm_sve.tc new file mode 100644 index 0000000..b015423 --- /dev/null +++ b/hyp/interfaces/arm_sve/arm_sve.tc @@ -0,0 +1,7 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +extend platform_cpu_features bitfield { + auto sve_disable bool = 1; +}; diff --git a/hyp/mem/addrspace_null/build.conf b/hyp/interfaces/arm_sve/build.conf similarity index 73% rename from hyp/mem/addrspace_null/build.conf rename to hyp/interfaces/arm_sve/build.conf index 64e35da..f393970 100644 --- a/hyp/mem/addrspace_null/build.conf +++ b/hyp/interfaces/arm_sve/build.conf @@ -2,5 +2,4 @@ # # SPDX-License-Identifier: BSD-3-Clause -interface addrspace -types addrspace.tc +types arm_sve.tc diff --git a/hyp/interfaces/base/armv8/base.tc b/hyp/interfaces/base/armv8/base.tc new file mode 100644 index 0000000..a106573 --- /dev/null +++ b/hyp/interfaces/base/armv8/base.tc @@ -0,0 +1,28 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +extend core_id enumeration { + CORTEX_A53; + CORTEX_A55; + CORTEX_A57; + CORTEX_A72; + CORTEX_A73; + CORTEX_A75; + CORTEX_A76; + CORTEX_A76AE; + CORTEX_A77; + CORTEX_A78; + CORTEX_A78AE; + CORTEX_A78C; + CORTEX_X1; + CORTEX_X2; + CORTEX_A710; + CORTEX_A510; + NEOVERSE_N1; + NEOVERSE_N2; + NEOVERSE_V1; + CORTEX_A715; + CORTEX_X3; + QEMU; +}; diff --git a/hyp/interfaces/base/base.tc b/hyp/interfaces/base/base.tc new file mode 100644 index 0000000..383a16c --- /dev/null +++ b/hyp/interfaces/base/base.tc @@ -0,0 +1,7 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +define core_id enumeration { + UNKNOWN; +}; diff --git a/hyp/interfaces/base/build.conf b/hyp/interfaces/base/build.conf index e69de29..8402982 100644 --- a/hyp/interfaces/base/build.conf +++ b/hyp/interfaces/base/build.conf @@ -0,0 +1,6 @@ +# © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +types base.tc +arch_types armv8 base.tc diff --git a/hyp/interfaces/base/include/base.h b/hyp/interfaces/base/include/base.h new file mode 100644 index 0000000..ff9abc3 --- /dev/null +++ b/hyp/interfaces/base/include/base.h @@ -0,0 +1,6 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +core_id_t +get_current_core_id(void); diff --git a/hyp/interfaces/boot/boot.ev b/hyp/interfaces/boot/boot.ev index 429857e..ba7a7e6 100644 --- a/hyp/interfaces/boot/boot.ev +++ b/hyp/interfaces/boot/boot.ev @@ -86,3 +86,6 @@ event boot_hypervisor_start // This runs at the end of every power-on sequence, immediately before // switching to the idle thread. event boot_cpu_start + +// Triggered when handover to another hypervisor is requested. +event boot_hypervisor_handover diff --git a/hyp/interfaces/boot/include/boot.h b/hyp/interfaces/boot/include/boot.h index 18870b0..90709bb 100644 --- a/hyp/interfaces/boot/include/boot.h +++ b/hyp/interfaces/boot/include/boot.h @@ -4,3 +4,7 @@ extern const char hypervisor_version[]; extern const char hypervisor_build_date[]; + +// Triggers the hypervisor hand-over event +void +boot_start_hypervisor_handover(void) REQUIRE_PREEMPT_DISABLED; diff --git a/hyp/interfaces/cpulocal/include/cpulocal.h b/hyp/interfaces/cpulocal/include/cpulocal.h index 7fc939c..b575467 100644 --- a/hyp/interfaces/cpulocal/include/cpulocal.h +++ b/hyp/interfaces/cpulocal/include/cpulocal.h @@ -37,6 +37,14 @@ #define CPULOCAL_BY_INDEX(name, index) \ cpulocal_##name[cpulocal_check_index(index)] +// Given a pointer to an object that is known to be an element of a given +// CPU-local data array, obtain its index. The behaviour is undefined (and +// violates MISRA rule 18.2) if the object is not a member of the specified +// array. +#define CPULOCAL_PTR_INDEX(name, ptr) \ + (assert(ptr != NULL), \ + cpulocal_check_index((cpu_index_t)(ptr - cpulocal_##name))) + // Return true if a CPU index is valid. bool cpulocal_index_valid(cpu_index_t index); @@ -48,14 +56,29 @@ cpulocal_index_valid(cpu_index_t index); cpu_index_t cpulocal_check_index(cpu_index_t index); +// Get the CPU index of the caller at the instant of the call. +// +// This is the same as cpulocal_get_index(), except that it does not require +// preemption to be disabled. The result may therefore be stale by the time +// the caller gets it. The result should only be used for purposes where this +// does not matter, e.g. for debug traces. +// +// The result of this function is not checked and may be invalid if called +// very early in the boot sequence. +cpu_index_t +cpulocal_get_index_unsafe(void); + // Get the CPU index of the caller. // // All calls to this function should be inside a critical section, which may // be either an explicit preemption disable, a spinlock, or a cpulocal // critical section. All uses of its result should occur before the // corresponding critical section ends. -cpu_index_t -cpulocal_get_index(void); +static inline cpu_index_t +cpulocal_get_index(void) REQUIRE_PREEMPT_DISABLED +{ + return cpulocal_check_index(cpulocal_get_index_unsafe()); +} // Get the CPU index of a specified thread. // diff --git a/hyp/interfaces/cspace/hypercalls.hvc b/hyp/interfaces/cspace/hypercalls.hvc index fcf8390..a307397 100644 --- a/hyp/interfaces/cspace/hypercalls.hvc +++ b/hyp/interfaces/cspace/hypercalls.hvc @@ -6,7 +6,7 @@ define cspace_delete_cap_from hypercall { call_num 0x22; cspace input type cap_id_t; cap input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; @@ -16,7 +16,7 @@ define cspace_copy_cap_from hypercall { src_cap input type cap_id_t; dest_cspace input type cap_id_t; rights_mask input type cap_rights_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; new_cap output type cap_id_t; }; @@ -25,7 +25,7 @@ define cspace_revoke_cap_from hypercall { call_num 0x24; src_cspace input type cap_id_t; src_cap input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; @@ -33,7 +33,7 @@ define cspace_attach_thread hypercall { call_num 0x3e; cspace input type cap_id_t; thread input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; @@ -41,6 +41,6 @@ define cspace_revoke_caps_from hypercall { call_num 0x59; src_cspace input type cap_id_t; master_cap input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; diff --git a/hyp/interfaces/doorbell/doorbell.hvc b/hyp/interfaces/doorbell/doorbell.hvc index 32d987b..2864ec0 100644 --- a/hyp/interfaces/doorbell/doorbell.hvc +++ b/hyp/interfaces/doorbell/doorbell.hvc @@ -7,14 +7,14 @@ define doorbell_bind_virq hypercall { doorbell input type cap_id_t; vic input type cap_id_t; virq input type virq_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; define doorbell_unbind_virq hypercall { call_num 0x11; doorbell input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; @@ -22,7 +22,7 @@ define doorbell_send hypercall { call_num 0x12; doorbell input type cap_id_t; new_flags input uint64; - _res0 input uregister; + res0 input uregister; error output enumeration error; old_flags output uint64; }; @@ -31,7 +31,7 @@ define doorbell_receive hypercall { call_num 0x13; doorbell input type cap_id_t; clear_flags input uint64; - _res0 input uregister; + res0 input uregister; error output enumeration error; old_flags output uint64; }; @@ -39,7 +39,7 @@ define doorbell_receive hypercall { define doorbell_reset hypercall { call_num 0x14; doorbell input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; @@ -48,6 +48,6 @@ define doorbell_mask hypercall { doorbell input type cap_id_t; enable_mask input uint64; ack_mask input uint64; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; diff --git a/hyp/interfaces/elf/include/elf.h b/hyp/interfaces/elf/include/elf.h index e2b5e8b..3fea0f6 100644 --- a/hyp/interfaces/elf/include/elf.h +++ b/hyp/interfaces/elf/include/elf.h @@ -13,8 +13,8 @@ typedef uint64_t Elf_Xword; typedef uint64_t Elf_Addr; typedef uint64_t Elf_Off; -#define R_TYPE(r_info) ((r_info)&0x7fffffff) -#define R_SYM(r_info) ((r_info) >> 32) +#define R_TYPE(r_info) ((r_info)&0x7fffffffU) +#define R_SYM(r_info) ((r_info) >> 32U) #define ELF_CLASS ELF_CLASS_64 @@ -40,38 +40,38 @@ typedef uint64_t Elf_Off; #define EI_ABIVERSION 8 #define EI_PAD 9 -#define ELF_CLASS_NONE 0 -#define ELF_CLASS_32 1 -#define ELF_CLASS_64 2 +#define ELF_CLASS_NONE 0U +#define ELF_CLASS_32 1U +#define ELF_CLASS_64 2U -#define ELF_DATA_NONE 0 -#define ELF_DATA_2LSB 1 -#define ELF_DATA_2MSB 2 +#define ELF_DATA_NONE 0U +#define ELF_DATA_2LSB 1U +#define ELF_DATA_2MSB 2U -#define EV_NONE 0 -#define EV_CURRENT 1 +#define EV_NONE 0U +#define EV_CURRENT 1U -#define ET_NONE 0 -#define ET_REL 1 -#define ET_EXEC 2 -#define ET_DYN 3 -#define ET_CORE 4 +#define ET_NONE 0U +#define ET_REL 1U +#define ET_EXEC 2U +#define ET_DYN 3U +#define ET_CORE 4U -#define EM_AARCH64 183 +#define EM_AARCH64 183U -#define PT_NULL 0 -#define PT_LOAD 1 -#define PT_DYNAMIC 2 -#define PT_INTERP 3 -#define PT_NOTE 4 -#define PT_SHLIB 5 -#define PT_PHDR 6 -#define PT_TLS 7 -#define PT_NUM 8 +#define PT_NULL 0U +#define PT_LOAD 1U +#define PT_DYNAMIC 2U +#define PT_INTERP 3U +#define PT_NOTE 4U +#define PT_SHLIB 5U +#define PT_PHDR 6U +#define PT_TLS 7U +#define PT_NUM 8U -#define PF_X 1 -#define PF_W 2 -#define PF_R 4 +#define PF_X 1U +#define PF_W 2U +#define PF_R 4U #define DT_NULL 0 #define DT_REL 17 @@ -81,12 +81,13 @@ typedef uint64_t Elf_Off; #define DT_CNT 19 // Architecture relocation types -#define R_AARCH64_NONE 0 -#define R_AARCH64_NULL 256 -#define R_AARCH64_RELATIVE 1027 +#define R_AARCH64_NONE 0U +#define R_AARCH64_NULL 256U +#define R_AARCH64_RELATIVE 1027U typedef struct { Elf_Sxword d_tag; + union { Elf_Xword d_val; Elf_Addr d_ptr; diff --git a/hyp/interfaces/globals/build.conf b/hyp/interfaces/globals/build.conf new file mode 100644 index 0000000..4cd0931 --- /dev/null +++ b/hyp/interfaces/globals/build.conf @@ -0,0 +1,5 @@ +# © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +types globals.tc diff --git a/hyp/interfaces/globals/globals.tc b/hyp/interfaces/globals/globals.tc new file mode 100644 index 0000000..e914fb8 --- /dev/null +++ b/hyp/interfaces/globals/globals.tc @@ -0,0 +1,6 @@ +// © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +define global_options bitfield<64> { +}; diff --git a/hyp/interfaces/globals/include/globals.h b/hyp/interfaces/globals/include/globals.h new file mode 100644 index 0000000..554eb82 --- /dev/null +++ b/hyp/interfaces/globals/include/globals.h @@ -0,0 +1,12 @@ +// © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +const global_options_t * +globals_get_options(void); + +void +globals_set_options(const global_options_t set); + +void +globals_clear_options(const global_options_t clear); diff --git a/hyp/interfaces/gpt/build.conf b/hyp/interfaces/gpt/build.conf new file mode 100644 index 0000000..7321770 --- /dev/null +++ b/hyp/interfaces/gpt/build.conf @@ -0,0 +1,6 @@ +# © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +events gpt.ev +types gpt.tc diff --git a/hyp/interfaces/gpt/gpt.ev b/hyp/interfaces/gpt/gpt.ev new file mode 100644 index 0000000..21868b3 --- /dev/null +++ b/hyp/interfaces/gpt/gpt.ev @@ -0,0 +1,29 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +interface gpt + +// Add an offset to a GPT value. +// The value can be left unmodified if not applicable. +selector_event gpt_value_add_offset + selector type: gpt_type_t + param value: gpt_value_t * + param offset: size_t + return: void + +// Return true if two values of the same type are equal. +selector_event gpt_values_equal + selector type: gpt_type_t + param x: gpt_value_t + param y: gpt_value_t + return: bool = false + +// Callback for the GPT type when performing a walk. +selector_event gpt_walk_callback + selector callback: gpt_callback_t + param entry: gpt_entry_t + param base: size_t + param size: size_t + param arg: gpt_arg_t + return: error_t = ERROR_ARGUMENT_INVALID diff --git a/hyp/interfaces/gpt/gpt.tc b/hyp/interfaces/gpt/gpt.tc new file mode 100644 index 0000000..80c9833 --- /dev/null +++ b/hyp/interfaces/gpt/gpt.tc @@ -0,0 +1,36 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +define gpt structure { +}; + +define gpt_type enumeration { + empty = 0; +}; + +define gpt_value union { +}; + +define gpt_entry structure { + type enumeration gpt_type; + value union gpt_value; +}; + +define gpt_lookup_result structure { + entry structure gpt_entry; + size size; +}; + +define gpt_callback enumeration { + reserved = 0; +}; + +define gpt_arg union { + raw uintptr; +}; + +define gpt_config bitfield<32> { + 7:0 max_bits type count_t; + 8 rcu_read bool; +}; diff --git a/hyp/interfaces/gpt/include/gpt.h b/hyp/interfaces/gpt/include/gpt.h new file mode 100644 index 0000000..bc0ccfe --- /dev/null +++ b/hyp/interfaces/gpt/include/gpt.h @@ -0,0 +1,82 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +// Note that the GPT is not thread-safe by default; the caller must use an +// external lock or some other protection to prevent concurrent calls to these +// APIs. If the rcu_read option is set in the GPT's config, read-only operations +// will be protected by RCU, and external locking is only required for write +// operations. + +// Initialise the GPT. +error_t +gpt_init(gpt_t *gpt, partition_t *partition, gpt_config_t config, + register_t allowed_types); + +// Destroy the GPT. +void +gpt_destroy(gpt_t *gpt); + +// Insert a range into the GPT. +// +// If expect_empty is true, the operation will fail if any part of the range is +// not found to be empty during insertion. Otherwise, any existing entries in +// the range will be overwritten. +error_t +gpt_insert(gpt_t *gpt, size_t base, size_t size, gpt_entry_t entry, + bool expect_empty); + +// Update a range in the GPT. +// +// The update will fail if all entries over the range do not match the given old +// entry. If successful, all old entries will be replaced with the new entry. +error_t +gpt_update(gpt_t *gpt, size_t base, size_t size, gpt_entry_t old_entry, + gpt_entry_t new_entry); + +// Remove a range from the GPT. +// +// This will fail if all entries over the range do not match the given entry. +error_t +gpt_remove(gpt_t *gpt, size_t base, size_t size, gpt_entry_t entry); + +// Clear a range in the GPT. +error_t +gpt_clear(gpt_t *gpt, size_t base, size_t size); + +// Clear the entire GPT. +void +gpt_clear_all(gpt_t *gpt); + +// Returns true if the GPT is empty. +bool +gpt_is_empty(gpt_t *gpt); + +// Lookup a range in the GPT. +// +// This function returns the entry found at the given base, and returns the size +// of this entry, which will be capped at max_size. +gpt_lookup_result_t +gpt_lookup(gpt_t *gpt, size_t base, size_t max_size); + +// Returns true if an entry is contiguous over a range in the GPT. +bool +gpt_is_contiguous(gpt_t *gpt, size_t base, size_t size, gpt_entry_t entry); + +// Walk over a range in the GPT and perform a callback for regions matching +// the given type. +error_t +gpt_walk(gpt_t *gpt, size_t base, size_t size, gpt_type_t type, + gpt_callback_t callback, gpt_arg_t arg); + +// Walk over the GPT and dump all contiguous ranges. +// +// This function is intended for debug use only. +void +gpt_dump_ranges(gpt_t *gpt); + +// Walk over the GPT and dump all levels. +// +// This function is intended for debug use only. +void +gpt_dump_levels(gpt_t *gpt); diff --git a/hyp/interfaces/hyp_aspace/build.conf b/hyp/interfaces/hyp_aspace/build.conf index 08fd955..337cd05 100644 --- a/hyp/interfaces/hyp_aspace/build.conf +++ b/hyp/interfaces/hyp_aspace/build.conf @@ -1,5 +1 @@ -# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - types hyp_aspace.tc diff --git a/hyp/interfaces/hyp_aspace/hyp_aspace.tc b/hyp/interfaces/hyp_aspace/hyp_aspace.tc index 0ff6fa2..2c3d87e 100644 --- a/hyp/interfaces/hyp_aspace/hyp_aspace.tc +++ b/hyp/interfaces/hyp_aspace/hyp_aspace.tc @@ -2,6 +2,12 @@ // // SPDX-License-Identifier: BSD-3-Clause +#if defined(ARCH_ARM_FEAT_VHE) && defined(ARCH_ARM_FEAT_PAN) +define ARCH_AARCH64_USE_PAN constant = 1; +#else +define ARCH_AARCH64_USE_PAN constant = 0; +#endif + define virt_range structure { base uintptr; size size; diff --git a/hyp/interfaces/hyp_aspace/include/hyp_aspace.h b/hyp/interfaces/hyp_aspace/include/hyp_aspace.h index b43b64b..6dde231 100644 --- a/hyp/interfaces/hyp_aspace/include/hyp_aspace.h +++ b/hyp/interfaces/hyp_aspace/include/hyp_aspace.h @@ -43,3 +43,13 @@ hyp_aspace_va_to_pa_el2_read(void *addr, paddr_t *pa, MAIR_ATTR_t *memattr, error_t hyp_aspace_va_to_pa_el2_write(void *addr, paddr_t *pa, MAIR_ATTR_t *memattr, vmsa_shareability_t *shareability); + +// Return the offset used for the physaccess mappings. For the CPUs that support +// PAN this is a compile-time constant offset, and for the older CPUs it is +// randomised on every boot. +uintptr_t +hyp_aspace_get_physaccess_offset(void); + +// Returns the base of the memory used for virtual address allocation +uintptr_t +hyp_aspace_get_alloc_base(void); diff --git a/hyp/interfaces/idle/include/idle.h b/hyp/interfaces/idle/include/idle.h index 429165a..54828a9 100644 --- a/hyp/interfaces/idle/include/idle.h +++ b/hyp/interfaces/idle/include/idle.h @@ -7,7 +7,7 @@ // Get the current CPU's idle thread. Must only be called in a cpulocal // critical section. thread_t * -idle_thread(void); +idle_thread(void) REQUIRE_PREEMPT_DISABLED; // Get the specified CPU's idle thread. thread_t * @@ -15,7 +15,7 @@ idle_thread_for(cpu_index_t cpu_index); // True when running in the current CPU's idle thread. bool -idle_is_current(void); +idle_is_current(void) REQUIRE_PREEMPT_DISABLED; bool idle_yield(void) REQUIRE_PREEMPT_DISABLED; diff --git a/hyp/interfaces/ipi/include/ipi.h b/hyp/interfaces/ipi/include/ipi.h index 8b8d113..b91d7d8 100644 --- a/hyp/interfaces/ipi/include/ipi.h +++ b/hyp/interfaces/ipi/include/ipi.h @@ -38,14 +38,14 @@ // // This implies a release barrier. void -ipi_others(ipi_reason_t ipi); +ipi_others(ipi_reason_t ipi) REQUIRE_PREEMPT_DISABLED; // Send the specified IPI to all online CPUs other than the caller, with low // priority. // // This implies a release barrier. void -ipi_others_relaxed(ipi_reason_t ipi); +ipi_others_relaxed(ipi_reason_t ipi) REQUIRE_PREEMPT_DISABLED; // Send the specified IPI to all online CPUs other than the caller, with low // priority, guaranteeing that idle CPUs will wake. @@ -54,7 +54,7 @@ ipi_others_relaxed(ipi_reason_t ipi); // // Do not use with the intention of waking a suspended CPU. void -ipi_others_idle(ipi_reason_t ipi); +ipi_others_idle(ipi_reason_t ipi) REQUIRE_PREEMPT_DISABLED; // Send the specified IPI to a single CPU. // @@ -92,9 +92,10 @@ ipi_one_idle(ipi_reason_t ipi, cpu_index_t cpu); // controllers can reliably cancel an IPI without handling it. // // This function may be safely called with preemption enabled, from any context. -// However, its result must be ignored if it is called in an IPI handler. +// However, its result must be ignored if it is called in an IPI handler or with +// preemption enabled. bool -ipi_clear(ipi_reason_t ipi); +ipi_clear(ipi_reason_t ipi) REQUIRE_PREEMPT_DISABLED; // Atomically check and clear the specified IPI, assuming it was a relaxed IPI. // @@ -108,7 +109,7 @@ ipi_clear(ipi_reason_t ipi); // can avoid the cost of interacting with the interrupt controller for IPIs that // are always, or mostly, raised using the _relaxed functions. bool -ipi_clear_relaxed(ipi_reason_t ipi); +ipi_clear_relaxed(ipi_reason_t ipi) REQUIRE_PREEMPT_DISABLED; // Immediately handle any relaxed IPIs. // diff --git a/hyp/interfaces/irq/include/irq.h b/hyp/interfaces/irq/include/irq.h index 4c0c586..a0d9a60 100644 --- a/hyp/interfaces/irq/include/irq.h +++ b/hyp/interfaces/irq/include/irq.h @@ -67,7 +67,7 @@ irq_deactivate(hwirq_t *hwirq); // Must be called from an RCU critical section. No reference is taken to the // result. hwirq_t * -irq_lookup_hwirq(irq_t irq); +irq_lookup_hwirq(irq_t irq) REQUIRE_RCU_READ; #if IRQ_HAS_MSI diff --git a/hyp/interfaces/log/build.conf b/hyp/interfaces/log/build.conf index 0dea298..808a782 100644 --- a/hyp/interfaces/log/build.conf +++ b/hyp/interfaces/log/build.conf @@ -3,4 +3,4 @@ # SPDX-License-Identifier: BSD-3-Clause types log.tc -events log.ev \ No newline at end of file +events log.ev diff --git a/hyp/interfaces/log/include/log.h b/hyp/interfaces/log/include/log.h index 2c10c91..bff9325 100644 --- a/hyp/interfaces/log/include/log.h +++ b/hyp/interfaces/log/include/log.h @@ -2,8 +2,16 @@ // // SPDX-License-Identifier: BSD-3-Clause +#if defined(PARASOFT_CYCLO) +#define LOG(tclass, id, ...) +#else #define LOG(tclass, id, ...) \ TRACE_EVENT(tclass, id, TRACE_ACTION_LOG, __VA_ARGS__) +#endif +#if defined(PARASOFT_CYCLO) +#define TRACE_AND_LOG(tclass, id, ...) +#else #define TRACE_AND_LOG(tclass, id, ...) \ TRACE_EVENT(tclass, id, TRACE_ACTION_TRACE_AND_LOG, __VA_ARGS__) +#endif diff --git a/hyp/interfaces/memdb/include/memdb.h b/hyp/interfaces/memdb/include/memdb.h index 618db9b..7d6de80 100644 --- a/hyp/interfaces/memdb/include/memdb.h +++ b/hyp/interfaces/memdb/include/memdb.h @@ -5,11 +5,14 @@ // FIXME: replace with a selector event typedef error_t (*memdb_fnptr)(paddr_t, size_t, void *); -error_t +void memdb_init(void); // Populate the memory database. If any entry from the range already has an // owner, return error and do not update the database. +// +// The partition argument is the partition to use for memdb node allocations, +// and typically should always be the hypervisor private partition. error_t memdb_insert(partition_t *partition, paddr_t start_addr, paddr_t end_addr, uintptr_t object, memdb_type_t obj_type); @@ -17,19 +20,22 @@ memdb_insert(partition_t *partition, paddr_t start_addr, paddr_t end_addr, // Change the ownership of the input address range. Checks if all entries of // range were pointing to previous object. If so, update all entries to point to // the new object. If not, return error. +// +// The partition argument is the partition to use for memdb node allocations, +// and typically should always be the hypervisor private partition. error_t memdb_update(partition_t *partition, paddr_t start_addr, paddr_t end_addr, uintptr_t object, memdb_type_t obj_type, uintptr_t prev_object, memdb_type_t prev_type); // Find the entry corresponding to the input address and return the object and -// type the entry is pointing to +// type the entry is pointing to. // // This function returns an RCU-protected reference and therefore, it needs to // be called in a RCU critical section and maintain it until we finish using the // returned object. memdb_obj_type_result_t -memdb_lookup(paddr_t addr); +memdb_lookup(paddr_t addr) REQUIRE_RCU_READ; // Check if all the entries from the input address range point to the object // passed as an argument diff --git a/hyp/interfaces/memdb/memdb.tc b/hyp/interfaces/memdb/memdb.tc index 172e789..4b37928 100644 --- a/hyp/interfaces/memdb/memdb.tc +++ b/hyp/interfaces/memdb/memdb.tc @@ -8,6 +8,8 @@ define memdb_type enumeration { PARTITION; ALLOCATOR; EXTENT; + // FIXME: // Memory which may not be mapped, e.g. converted to secure memory PARTITION_NOMAP; + TRACE; }; diff --git a/hyp/interfaces/memextent/include/memextent.h b/hyp/interfaces/memextent/include/memextent.h index 00486c0..8162594 100644 --- a/hyp/interfaces/memextent/include/memextent.h +++ b/hyp/interfaces/memextent/include/memextent.h @@ -25,6 +25,28 @@ error_t memextent_configure_derive(memextent_t *me, memextent_t *parent, size_t offset, size_t size, memextent_attrs_t attributes); +// Returns true if the memextent supports donation. +bool +memextent_supports_donation(memextent_t *me); + +// Donate memory to a child extent. +// +// This can only be used for sparse memory extents. If the memory extent has +// been derived the memory is taken from the parent extent, otherwise the memory +// is taken from the extent's partition. If reverse is true, the memory is +// donated back to the parent instead. +error_t +memextent_donate_child(memextent_t *me, size_t offset, size_t size, + bool reverse); + +// Donate memory to a sibling extent. +// +// This can only be used for sparse memory extents. Both from and to memory +// extents must have the same parent memory extent. +error_t +memextent_donate_sibling(memextent_t *from, memextent_t *to, size_t offset, + size_t size); + // Map a memory extent into a specified address space. The entire range is // mapped, except for any carveouts contained within the extent. // @@ -36,6 +58,19 @@ error_t memextent_map(memextent_t *me, addrspace_t *addrspace, vmaddr_t vm_base, memextent_mapping_attrs_t map_attrs); +// Map a portion of a memory extent into a specified address space. Only the +// range specified by offset and size is mapped, excluding any carveouts +// contained within this range. +// +// If there are any existing mappings in the affected region, they are replaced +// with the new mapping. There may still be in-progress EL2 operations using old +// mappings. These are RCU read operations and are guaranteed to complete (or +// start using the new mapping) by the end of the next grace period. +error_t +memextent_map_partial(memextent_t *me, addrspace_t *addrspace, vmaddr_t vm_base, + size_t offset, size_t size, + memextent_mapping_attrs_t map_attrs); + // Unmap a memory extent from a specified address space. The entire range is // unmapped, except for any carveouts contained within the extent. // @@ -45,7 +80,18 @@ memextent_map(memextent_t *me, addrspace_t *addrspace, vmaddr_t vm_base, error_t memextent_unmap(memextent_t *me, addrspace_t *addrspace, vmaddr_t vm_base); -// Unmap a memory extent from all address spaces. The entire range is unmmaped, +// Unmap a portion of a memory extent from a specified address space. Only the +// range specified by offset and size is unmapped, except for any carveouts +// contained within this range. +// +// There may still be in-progress EL2 operations using the removed mappings. +// These are RCU read operations and are guaranteed to complete (or fault due to +// the unmap) by the end of the next grace period. +error_t +memextent_unmap_partial(memextent_t *me, addrspace_t *addrspace, + vmaddr_t vm_base, size_t offset, size_t size); + +// Unmap a memory extent from all address spaces. The entire range is unmapped, // except for any carveouts contained within the extent. // // There may still be in-progress EL2 operations using the removed mappings. @@ -54,6 +100,10 @@ memextent_unmap(memextent_t *me, addrspace_t *addrspace, vmaddr_t vm_base); void memextent_unmap_all(memextent_t *me); +// Zero all owned regions of a memory extent in the given range. +error_t +memextent_zero_range(memextent_t *me, size_t offset, size_t size); + // Update the access rights on an existing mapping. // // There may still be in-progress EL2 operations using the old access rights. @@ -64,6 +114,24 @@ memextent_update_access(memextent_t *me, addrspace_t *addrspace, vmaddr_t vm_base, memextent_access_attrs_t access_attrs); +// Update the access rights on part of an existing mapping. +// +// There may still be in-progress EL2 operations using the old access rights. +// These are RCU read operations and are guaranteed to complete (or fault due to +// reduced access rights) by the end of the next grace period. +error_t +memextent_update_access_partial(memextent_t *me, addrspace_t *addrspace, + vmaddr_t vm_base, size_t offset, size_t size, + memextent_access_attrs_t access_attrs); + +// Returns true if a memextent is mapped in an addrspace. +// +// If exclusive is true, this function will return false if the memextent is +// mapped in any other addrspace, regardless of whether it is mapped in the +// given addrspace. +bool +memextent_is_mapped(memextent_t *me, addrspace_t *addrspace, bool exclusive); + // Helper function to check the combination memory type of the memory extent and // the mapping memtype. bool @@ -76,6 +144,75 @@ memextent_ptr_result_t memextent_derive(memextent_t *parent, paddr_t offset, size_t size, memextent_memtype_t memtype, pgtable_access_t access); -// FIXME: It should declare events for iterating through an extent's memory ranges -// and for handling read and write faults. -// return PGTABLE_VM_MEMTYPE_DEVICE_NGNRNE; +// Temporarily retain all of the memextent's mappings. +// +// This acquires references to all addrspaces the memextent has mappings in, +// preventing the addrspace from being destroyed while looking up mappings. +void +memextent_retain_mappings(memextent_t *me) REQUIRE_LOCK(me->lock) + ACQUIRE_LOCK(me->mappings); + +// Release a memextent's retained mappings. +// +// This frees any references acquired in memextent_retain_mappings(). +// If clear is true, all mappings will be cleared as well. +void +memextent_release_mappings(memextent_t *me, bool clear) REQUIRE_LOCK(me->lock) + RELEASE_LOCK(me->mappings); + +// Lookup a mapping in a memextent. +// +// The memextent's mappings must be retained when this is called. The supplied +// physical address and size must lie within the memextent, and the index must +// be less than MEMEXTENT_MAX_MAPS. +// +// If the returned addrspace is not NULL, the memextent has a mapping in the +// given range; otherwise the memextent is not mapped for this range. The +// returned size is valid for both of these cases, dictating the size of the +// mapping (or lack thereof). This size may be smaller than the given size if +// the range is only partially mapped. +memextent_mapping_t +memextent_lookup_mapping(memextent_t *me, paddr_t phys, size_t size, index_t i) + REQUIRE_LOCK(me->lock) REQUIRE_LOCK(me->mappings); + +// Claim and map a memextent for access in the hypervisor. +// +// The specified partition must be the owner of the object that the memextent +// is being attached to. Typically this is an address space or virtual device +// object. +// +// The specified extent must be a basic (not sparse) extent with normal memory +// type, no children, RW access, and no existing hypervisor attachment. While it +// remains attached, derivation of children or donation of memory to or from +// this extent will not be permitted. It may, however, be mapped in VM address +// spaces while attached, and existing VM address space mappings will not be +// removed by attachement. +// +// The specified virtual address range must be within a region allocated to the +// specified partition by hyp_aspace_allocate(). The specified size must be a +// nonzero multiple of the page size and no greater than the size of the +// memextent. +// +// If the memextent is provided through a hypervisor API, the caller should +// possess the MEMEXTENT_ATTACH right. +// +// If the call is successful, the entire specified address range will be mapped +// to the memextent. +error_t +memextent_attach(partition_t *owner, memextent_t *extent, uintptr_t hyp_va, + size_t size); + +// Detach a memextent from the hypervisor. +// +// The specified owner and extent must match a previous successful call to +// memextent_attach(). +void +memextent_detach(partition_t *owner, memextent_t *extent); + +// Find the offset of a specified physical access within a memextent. +// +// This is intended to be used for handling faults. If any part of the specified +// physical address range is outside the memextent (including in a gap in a +// sparse memextent), it will return an error. +size_result_t +memextent_get_offset_for_pa(memextent_t *me, paddr_t pa, size_t size); diff --git a/hyp/interfaces/memextent/memextent.ev b/hyp/interfaces/memextent/memextent.ev index 8bbc071..db3c3f7 100644 --- a/hyp/interfaces/memextent/memextent.ev +++ b/hyp/interfaces/memextent/memextent.ev @@ -4,21 +4,36 @@ interface memextent +selector_event memextent_activate + selector type: memextent_type_t + param me: memextent_t * + return: error_t = ERROR_MEMEXTENT_TYPE + selector_event memextent_activate_derive selector type: memextent_type_t param me: memextent_t * - return: error_t = ERROR_ARGUMENT_INVALID + return: error_t = ERROR_MEMEXTENT_TYPE + +selector_event memextent_supports_donation + selector type: memextent_type_t + param me: memextent_t * + return: bool = false -selector_event memextent_derive +selector_event memextent_donate_child selector type: memextent_type_t param me: memextent_t * - param parent: memextent_t * - param offsets: size_t * - param sizes: size_t * - param num_entries: count_t - param memtype: pgtable_vm_memtype_t - param access: pgtable_access_t - return: error_t = ERROR_ARGUMENT_INVALID + param phys: paddr_t + param size: size_t + param reverse: bool + return: error_t = ERROR_MEMEXTENT_TYPE + +selector_event memextent_donate_sibling + selector type: memextent_type_t + param from: memextent_t * + param to: memextent_t * + param phys: paddr_t + param size: size_t + return: error_t = ERROR_MEMEXTENT_TYPE selector_event memextent_map selector type: memextent_type_t @@ -26,14 +41,33 @@ selector_event memextent_map param addrspace: addrspace_t * param vm_base: vmaddr_t param map_attrs: memextent_mapping_attrs_t - return: error_t = ERROR_ARGUMENT_INVALID + return: error_t = ERROR_MEMEXTENT_TYPE + +selector_event memextent_map_partial + selector type: memextent_type_t + param extent: memextent_t * + param addrspace: addrspace_t * + param vm_base: vmaddr_t + param offset: size_t + param size: size_t + param map_attrs: memextent_mapping_attrs_t + return: error_t = ERROR_MEMEXTENT_TYPE selector_event memextent_unmap selector type: memextent_type_t param extent: memextent_t * param addrspace: addrspace_t * param vm_base: vmaddr_t - return: error_t = ERROR_ARGUMENT_INVALID + return: error_t = ERROR_MEMEXTENT_TYPE + +selector_event memextent_unmap_partial + selector type: memextent_type_t + param extent: memextent_t * + param addrspace: addrspace_t * + param vm_base: vmaddr_t + param offset: size_t + param size: size_t + return: error_t = ERROR_MEMEXTENT_TYPE selector_event memextent_unmap_all selector type: memextent_type_t @@ -46,7 +80,24 @@ selector_event memextent_update_access param addrspace: addrspace_t * param vm_base: vmaddr_t param access_attrs: memextent_access_attrs_t - return: error_t = ERROR_ARGUMENT_INVALID + return: error_t = ERROR_MEMEXTENT_TYPE + +selector_event memextent_update_access_partial + selector type: memextent_type_t + param extent: memextent_t * + param addrspace: addrspace_t * + param vm_base: vmaddr_t + param offset: size_t + param size: size_t + param access_attrs: memextent_access_attrs_t + return: error_t = ERROR_MEMEXTENT_TYPE + +selector_event memextent_is_mapped + selector type: memextent_type_t + param me: memextent_t * + param addrspace: addrspace_t * + param exclusive: bool + return: bool = false selector_event memextent_deactivate selector type: memextent_type_t @@ -57,3 +108,40 @@ selector_event memextent_cleanup selector type: memextent_type_t param extent: memextent_t * return: bool = false + +selector_event memextent_retain_mappings + selector type: memextent_type_t + param me: memextent_t * + +selector_event memextent_release_mappings + selector type: memextent_type_t + param me: memextent_t * + param clear: bool + +selector_event memextent_lookup_mapping + selector type: memextent_type_t + param me: memextent_t * + param phys: paddr_t + param size: size_t + param i: index_t + return: memextent_mapping_result_t = memextent_mapping_result_error(ERROR_MEMEXTENT_TYPE) + +selector_event memextent_attach + selector type: memextent_type_t + param me: memextent_t * + param hyp_va: uintptr_t + param size: size_t + param memtype: pgtable_hyp_memtype_t + return: error_t = ERROR_MEMEXTENT_TYPE + +selector_event memextent_detach + selector type: memextent_type_t + param me: memextent_t * + return: bool = false + +selector_event memextent_get_offset_for_pa + selector type: memextent_type_t + param extent: memextent_t * + param pa: paddr_t + param size: size_t + return: size_result_t = size_result_error(ERROR_MEMEXTENT_TYPE) diff --git a/hyp/interfaces/memextent/memextent.hvc b/hyp/interfaces/memextent/memextent.hvc index 9ab2aad..f85060f 100644 --- a/hyp/interfaces/memextent/memextent.hvc +++ b/hyp/interfaces/memextent/memextent.hvc @@ -2,10 +2,14 @@ // // SPDX-License-Identifier: BSD-3-Clause -define memextent_unmap_all hypercall { +define memextent_modify hypercall { call_num 0x30; memextent input type cap_id_t; - _res0 input uregister; + flags input bitfield memextent_modify_flags; + // The below arguments are only used for range operations; + // otherwise they are ignored for backwards compatibility. + offset input size; + size input size; error output enumeration error; }; @@ -15,7 +19,7 @@ define memextent_configure hypercall { phys_base input type paddr_t; size input size; attributes input bitfield memextent_attrs; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; @@ -26,6 +30,17 @@ define memextent_configure_derive hypercall { offset input size; size input size; attributes input bitfield memextent_attrs; - _res0 input uregister; + res0 input uregister; + error output enumeration error; +}; + +define memextent_donate hypercall { + call_num 0x61; + options input bitfield memextent_donate_options; + from input type cap_id_t; + to input type cap_id_t; + offset input size; + size input size; + res0 input uregister; error output enumeration error; }; diff --git a/hyp/interfaces/memextent/memextent.tc b/hyp/interfaces/memextent/memextent.tc index 6f7a63f..fce326f 100644 --- a/hyp/interfaces/memextent/memextent.tc +++ b/hyp/interfaces/memextent/memextent.tc @@ -2,13 +2,14 @@ // // SPDX-License-Identifier: BSD-3-Clause -define memextent_type enumeration { +define memextent_type public enumeration { BASIC = 0; + SPARSE = 1; }; extend memextent object { - type enumeration memextent_type; parent pointer object memextent; + type enumeration memextent_type; }; extend memextent_create structure module memextent { @@ -17,6 +18,7 @@ extend memextent_create structure module memextent { extend error enumeration { MEMEXTENT_MAPPINGS_FULL = 120; + MEMEXTENT_TYPE = 121; }; // Memory extents have a simplified memtype which is distinct from @@ -34,9 +36,10 @@ define memextent_memtype public enumeration(explicit) { define memextent_attrs public bitfield<32> { 2:0 access enumeration pgtable_access; 9:8 memtype enumeration memextent_memtype; + 17:16 type enumeration memextent_type; 31 append bool; // List append range // Reserved bits - 30:10,7:3 res_0 uregister(const) = 0; + 30:18,15:10,7:3 res_0 uregister(const) = 0; }; // Bitfield for mapping API attributes @@ -55,3 +58,35 @@ define memextent_access_attrs public bitfield<32> { // Reserved bits 31:8,7,3 res_0 uregister(const) = 0; }; + +define memextent_donate_type public enumeration(explicit) { + to_child = 0; // Donate from parent to child + to_parent = 1; // Donate from child to parent + to_sibling = 2; // Donate to extent with same parent +}; + +define memextent_donate_options public bitfield<32> { + 7:0 type enumeration memextent_donate_type; + 30:8 res_0 uregister(const) = 0; + 31 no_sync bool; +}; + +// Structure returned when looking up a mapping. +define memextent_mapping structure { + addrspace pointer object addrspace; + vbase type vmaddr_t; + size size; + attrs bitfield memextent_mapping_attrs; +}; + +define memextent_modify_op public enumeration(explicit) { + unmap_all = 0; // Remove all mappings from memextent + zero_range = 1; // Zero a range of the memextent + sync_all = 255; // Sync all previous memextent ops +}; + +define memextent_modify_flags public bitfield<32> { + 7:0 op enumeration memextent_modify_op; + 30:8 res_0 uregister(const) = 0; + 31 no_sync bool; +}; diff --git a/hyp/interfaces/msgqueue/msgqueue.hvc b/hyp/interfaces/msgqueue/msgqueue.hvc index b122fcd..228e712 100644 --- a/hyp/interfaces/msgqueue/msgqueue.hvc +++ b/hyp/interfaces/msgqueue/msgqueue.hvc @@ -7,7 +7,7 @@ define msgqueue_bind_send_virq hypercall { msgqueue input type cap_id_t; vic input type cap_id_t; virq input type virq_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; @@ -16,21 +16,21 @@ define msgqueue_bind_receive_virq hypercall { msgqueue input type cap_id_t; vic input type cap_id_t; virq input type virq_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; define msgqueue_unbind_send_virq hypercall { call_num 0x19; msgqueue input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; define msgqueue_unbind_receive_virq hypercall { call_num 0x1a; msgqueue input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; @@ -40,7 +40,7 @@ define msgqueue_send hypercall { size input size; data input type user_ptr_t; send_flags input bitfield msgqueue_send_flags; - _res0 input uregister; + res0 input uregister; error output enumeration error; not_full output bool; }; @@ -50,7 +50,7 @@ define msgqueue_receive hypercall { msgqueue input type cap_id_t; buffer input type user_ptr_t; buf_size input size; - _res0 input uregister; + res0 input uregister; error output enumeration error; size output size; not_empty output bool; @@ -59,7 +59,7 @@ define msgqueue_receive hypercall { define msgqueue_flush hypercall { call_num 0x1d; msgqueue input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; @@ -68,7 +68,7 @@ define msgqueue_configure_send hypercall { msgqueue input type cap_id_t; not_full_thres input type count_t; not_full_holdoff input type count_t; - _res1 input uregister; + res1 input uregister; error output enumeration error; }; @@ -77,7 +77,7 @@ define msgqueue_configure_receive hypercall { msgqueue input type cap_id_t; not_empty_thres input type count_t; not_empty_holdoff input type count_t; - _res1 input uregister; + res1 input uregister; error output enumeration error; }; @@ -85,6 +85,6 @@ define msgqueue_configure hypercall { call_num 0x21; msgqueue input type cap_id_t; create_info input bitfield msgqueue_create_info; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; diff --git a/hyp/interfaces/object/templates/object.ev.tmpl b/hyp/interfaces/object/templates/object.ev.tmpl index 8f671d9..590b15f 100644 --- a/hyp/interfaces/object/templates/object.ev.tmpl +++ b/hyp/interfaces/object/templates/object.ev.tmpl @@ -4,6 +4,7 @@ interface object +// FIXME: // Add rollback handlers #for obj in $object_list #set o = str(obj) diff --git a/hyp/interfaces/partition/include/partition.h b/hyp/interfaces/partition/include/partition.h index 8d7a0d1..127b845 100644 --- a/hyp/interfaces/partition/include/partition.h +++ b/hyp/interfaces/partition/include/partition.h @@ -124,3 +124,12 @@ partition_add_heap(partition_t *partition, paddr_t base, size_t size); // back to. error_t partition_map_and_add_heap(partition_t *partition, paddr_t base, size_t size); + +#if defined(PLATFORM_TRACE_STANDALONE_REGION) +// Map range and add memory to the partition's trace area +// +// The memory must have been allocated from the partition that is it is freed +// back to. +uintptr_result_t +partition_map_and_add_trace(partition_t *partition, paddr_t base, size_t size); +#endif diff --git a/hyp/interfaces/pgtable/include/pgtable.h b/hyp/interfaces/pgtable/include/pgtable.h index 6cc097d..acfd86b 100644 --- a/hyp/interfaces/pgtable/include/pgtable.h +++ b/hyp/interfaces/pgtable/include/pgtable.h @@ -63,7 +63,7 @@ bool pgtable_hyp_lookup(uintptr_t virt, paddr_t *mapped_base, size_t *mapped_size, pgtable_hyp_memtype_t *mapped_memtype, - pgtable_access_t *mapped_access); + pgtable_access_t *mapped_access); // Returns false if there is no mapping in the specified range. If a mapping // is found and can be efficiently determined to be the last mapping in the @@ -73,10 +73,10 @@ pgtable_hyp_lookup(uintptr_t virt, paddr_t *mapped_base, size_t *mapped_size, bool pgtable_hyp_lookup_range(uintptr_t virt_base, size_t virt_size, uintptr_t *mapped_virt, paddr_t *mapped_phys, - size_t *mapped_size, + size_t *mapped_size, pgtable_hyp_memtype_t *mapped_memtype, - pgtable_access_t *mapped_access, - bool *remainder_unmapped); + pgtable_access_t *mapped_access, + bool *remainder_unmapped); // Creates page table levels owned by the given partition which are able to // directly map entries covering the given size, but don't actually map @@ -91,19 +91,61 @@ extern opaque_lock_t pgtable_hyp_map_lock; void pgtable_hyp_start(void) ACQUIRE_LOCK(pgtable_hyp_map_lock); -// Creates a new mapping, assuming no existing mappings in the range. May -// use the given partition to allocate levels if needed. +// Creates a new mapping, possibly merging adjacent mappings into large blocks. +// +// An error will be returned if there are any existing mappings in the given +// region that are not exactly identical to the requested mapping. +// +// If merge_limit is nonzero, then this will attempt to merge page table levels +// that become congruent as a result of this operation into larger pages, as +// long as the new size is less than merge_limit. Any page table levels freed by +// this will be freed into the specified partition, so merge_limit should be no +// greater than preserved_prealloc would be for an unmap operation in the same +// region. +// +// Note that this operation may cause transient translation aborts or TLB +// conflict aborts in the affected range or within a merge_limit aligned region +// around it. The caller is responsible for not making calls with a nonzero +// merge_limit that might have those effects on the hypervisor code, the stack +// of any hypervisor thread, or any other address that may be touched during the +// handling of a transient hypervisor fault. error_t +pgtable_hyp_map_merge(partition_t *partition, uintptr_t virt, size_t size, + paddr_t phys, pgtable_hyp_memtype_t memtype, + pgtable_access_t access, vmsa_shareability_t shareability, + size_t merge_limit) REQUIRE_LOCK(pgtable_hyp_map_lock); + +// Creates a new mapping. No attempt will be made to merge adjacent mappings. +static inline error_t pgtable_hyp_map(partition_t *partition, uintptr_t virt, size_t size, paddr_t phys, pgtable_hyp_memtype_t memtype, pgtable_access_t access, vmsa_shareability_t shareability) - REQUIRE_LOCK(pgtable_hyp_map_lock); + REQUIRE_LOCK(pgtable_hyp_map_lock) +{ + return pgtable_hyp_map_merge(partition, virt, size, phys, memtype, + access, shareability, 0U); +} +// Creates a new mapping, replacing any existing mappings in the region, and +// possibly merging adjacent mappings into large blocks. The merge_limit +// argument has the same semantics as for @see pgtable_hyp_map_merge(). error_t +pgtable_hyp_remap_merge(partition_t *partition, uintptr_t virt, size_t size, + paddr_t phys, pgtable_hyp_memtype_t memtype, + pgtable_access_t access, + vmsa_shareability_t shareability, size_t merge_limit) + REQUIRE_LOCK(pgtable_hyp_map_lock); + +// Creates a new mapping, replacing any existing mappings in the region. +static inline error_t pgtable_hyp_remap(partition_t *partition, uintptr_t virt, size_t size, paddr_t phys, pgtable_hyp_memtype_t memtype, pgtable_access_t access, vmsa_shareability_t shareability) - REQUIRE_LOCK(pgtable_hyp_map_lock); + REQUIRE_LOCK(pgtable_hyp_map_lock) +{ + return pgtable_hyp_remap_merge(partition, virt, size, phys, memtype, + access, shareability, 0U); +} // Removes all mappings in the given range. Frees levels into the specified // partition's allocators, but only if they cannot be used to create mappings @@ -150,8 +192,8 @@ pgtable_vm_lookup_range(pgtable_vm_t *pgtable, vmaddr_t virt_base, size_t virt_size, vmaddr_t *mapped_virt, paddr_t *mapped_phys, size_t *mapped_size, pgtable_vm_memtype_t *mapped_memtype, - pgtable_access_t *mapped_vm_kernel_access, - pgtable_access_t *mapped_vm_user_access, + pgtable_access_t *mapped_vm_kernel_access, + pgtable_access_t *mapped_vm_user_access, bool *remainder_unmapped); extern opaque_lock_t pgtable_vm_map_lock; @@ -161,15 +203,22 @@ void pgtable_vm_start(pgtable_vm_t *pgtable) ACQUIRE_LOCK(pgtable) ACQUIRE_LOCK(pgtable_vm_map_lock); -// Creates a new mapping. If try_map is set, it returns an error if any existing -// mappings are present in the range. If try_map is false, any existing mappings -// in the specified range are removed or updated. +// Creates a new mapping. +// +// If try_map is true, it returns an error if any existing mappings are present +// in the range that are not exactly identical to the requested mapping. If +// try_map is false, any existing mappings in the specified range are removed or +// updated. +// +// If allow_merge is true, then any page table levels that become congruent as a +// result of this operation will be merged into larger pages. +// // pgtable_vm_start() must have been called before this call. error_t pgtable_vm_map(partition_t *partition, pgtable_vm_t *pgtable, vmaddr_t virt, size_t size, paddr_t phys, pgtable_vm_memtype_t memtype, pgtable_access_t vm_kernel_access, - pgtable_access_t vm_user_access, bool try_map) + pgtable_access_t vm_user_access, bool try_map, bool allow_merge) REQUIRE_LOCK(pgtable) REQUIRE_LOCK(pgtable_vm_map_lock); // Removes all mappings in the given range. pgtable_vm_start() must have been @@ -194,3 +243,19 @@ pgtable_vm_commit(pgtable_vm_t *pgtable) RELEASE_LOCK(pgtable) // Set VTCR and VTTBR registers with page table vtcr and vttbr bitfields values. void pgtable_vm_load_regs(pgtable_vm_t *vm_pgtable); + +// Validate page table access +bool +pgtable_access_check(pgtable_access_t access, pgtable_access_t access_check); + +// Mask a pagetable access +pgtable_access_t +pgtable_access_mask(pgtable_access_t access, pgtable_access_t access_mask); + +// Check if input page table accesses are equal +bool +pgtable_access_is_equal(pgtable_access_t access, pgtable_access_t access_check); + +// Get combined access +pgtable_access_t +pgtable_access_combine(pgtable_access_t access1, pgtable_access_t access2); diff --git a/hyp/interfaces/platform/include/platform_cpu.h b/hyp/interfaces/platform/include/platform_cpu.h index 21b3836..e4d9365 100644 --- a/hyp/interfaces/platform/include/platform_cpu.h +++ b/hyp/interfaces/platform/include/platform_cpu.h @@ -4,6 +4,11 @@ // Platform wrappers for SMP support. +// Check whether a cpu_index maps to cpu that exists. Note however, a CPU that +// exists may not be functional. +bool +platform_cpu_exists(cpu_index_t cpu); + // Power on the specified CPU. error_t platform_cpu_on(cpu_index_t cpu); @@ -54,14 +59,35 @@ platform_cpu_default_suspend(void) REQUIRE_PREEMPT_DISABLED; #endif #if defined(ARCH_ARM) -psci_mpidr_t -platform_cpu_index_to_mpidr(cpu_index_t cpu); +platform_mpidr_mapping_t +platform_cpu_get_mpidr_mapping(void); + +MPIDR_EL1_t +platform_cpu_map_index_to_mpidr(const platform_mpidr_mapping_t *mapping, + index_t index); + +index_t +platform_cpu_map_mpidr_to_index(const platform_mpidr_mapping_t *mapping, + MPIDR_EL1_t mpidr); + +bool +platform_cpu_map_mpidr_valid(const platform_mpidr_mapping_t *mapping, + MPIDR_EL1_t mpidr); + +MPIDR_EL1_t +platform_cpu_index_to_mpidr(index_t index); + +index_t +platform_cpu_mpidr_to_index(MPIDR_EL1_t mpidr); + +bool +platform_cpu_mpidr_valid(MPIDR_EL1_t mpidr); -cpu_index_result_t -platform_cpu_mpidr_to_index(psci_mpidr_t mpidr); +core_id_t +platform_cpu_get_coreid(MIDR_EL1_t midr); #endif -#if defined(ARCH_ARM_8_5_BTI) +#if defined(ARCH_ARM_FEAT_BTI) bool platform_cpu_bti_enabled(void); #endif diff --git a/hyp/interfaces/platform/include/platform_mem.h b/hyp/interfaces/platform/include/platform_mem.h index 0e7a3f9..10b9e7b 100644 --- a/hyp/interfaces/platform/include/platform_mem.h +++ b/hyp/interfaces/platform/include/platform_mem.h @@ -8,7 +8,7 @@ // minimal amount of memory provided by the hyp_partition to seed it). The // platform must provide the rest of the initial memory for the root partition. void -platform_add_root_heap(partition_t *partition); +platform_add_root_heap(partition_t *partition) REQUIRE_PREEMPT_DISABLED; // Return platform DDR (normal memory) information. // May only be called after platform_ram_probe(). @@ -19,3 +19,24 @@ platform_get_ram_info(void); // This function expects to be able map to the hyp pagetable. error_t platform_ram_probe(void); + +#if defined(ARCH_ARM) +// Check whether a platform VM page table is undergoing a break-before-make. +// +// This function is called while handling translation aborts in an address space +// that has the platform_pgtable flag set. It should return true if there is any +// possibility that an in-progress update of the VM page table has unmapped +// pages temporarily as part of an architecturally required break-before-make +// sequence while changing the block size of an existing mapping. +// +// If this function returns true, any translation abort in a read-only address +// space will be retried until either the fault is resolved or this function +// returns false. The platform code must therefore only return true from this +// function for a bounded period of time. +// +// Note that it is not necessary for the platform code to handle the race +// between completion of the break-before-make and reporting of a fault that +// occurred before it completed. That race will be handled by the caller. +bool +platform_pgtable_undergoing_bbm(void); +#endif diff --git a/hyp/interfaces/platform/include/platform_pmu.h b/hyp/interfaces/platform/include/platform_pmu.h index 0750cae..f4d84c4 100644 --- a/hyp/interfaces/platform/include/platform_pmu.h +++ b/hyp/interfaces/platform/include/platform_pmu.h @@ -3,7 +3,7 @@ // SPDX-License-Identifier: BSD-3-Clause void -platform_pmu_hw_irq_deactivate(void); +platform_pmu_hw_irq_deactivate(void) REQUIRE_PREEMPT_DISABLED; bool platform_pmu_is_hw_irq_pending(void); diff --git a/hyp/interfaces/platform/include/platform_psci.h b/hyp/interfaces/platform/include/platform_psci.h index fd2991a..481ea27 100644 --- a/hyp/interfaces/platform/include/platform_psci.h +++ b/hyp/interfaces/platform/include/platform_psci.h @@ -1,7 +1,6 @@ // © 2021 Qualcomm Innovation Center, Inc. All rights reserved. // // SPDX-License-Identifier: BSD-3-Clause - // Checks that the suspend state is valid // // This function checks that the pcpu supports the cpu and cluster level @@ -41,12 +40,30 @@ platform_psci_is_cpu_active(psci_cpu_state_t cpu_state); bool platform_psci_is_cpu_poweroff(psci_cpu_state_t cpu_state); +// Returns true if cluster state is in active state +bool +platform_psci_is_cluster_active(psci_cluster_state_L3_t cluster_state); + +// Returns the cluster indices +uint32_t +platform_psci_get_cluster_index(cpu_index_t cpu); + +// Returns the start index of children in hierarchy/counts based on level and +// cpu +error_t +platform_psci_get_index_by_level(cpu_index_t cpu, uint32_t *start_idx, + uint32_t *children_counts, uint32_t level); + #if !defined(PSCI_AFFINITY_LEVELS_NOT_SUPPORTED) || \ !PSCI_AFFINITY_LEVELS_NOT_SUPPORTED // Checks if cluster state corresponds to a power off state bool platform_psci_is_cluster_state_poweroff(psci_suspend_powerstate_t suspend_state); +// Returns true if cluster state is in active state +bool +platform_psci_is_cluster_active(psci_cluster_state_L3_t cluster_state); + // Checks if cluster state corresponds to a retention state bool platform_psci_is_cluster_state_retention( @@ -61,24 +78,37 @@ void platform_psci_set_cluster_state(psci_suspend_powerstate_t *suspend_state, psci_cluster_state_t cluster_state); -// Returns true if the last cpu bit is set the in suspend power state -bool -platform_psci_get_last_cpu(psci_suspend_powerstate_t suspend_state); +#if (PLATFORM_MAX_HIERARCHY == 2) +// Gets the system state from the suspend power state +psci_system_state_t +platform_psci_get_system_state(psci_suspend_powerstate_t suspend_state); -// Sets last cpu bit in suspend power state +// Sets system state to the stateid of the suspend power state void -platform_psci_set_last_cpu(psci_suspend_powerstate_t *suspend_state, - bool last_cpu); +platform_psci_set_system_state(psci_suspend_powerstate_t *suspend_state, + psci_system_state_t system_state); +#endif + +// Returns the suspend level of the last cpu +index_t +platform_psci_get_last_cpu_level(psci_suspend_powerstate_t suspend_state); + +// Sets the suspend level of the last cpu +void +platform_psci_set_last_cpu_level(psci_suspend_powerstate_t *suspend_state, + index_t last_cpu); + // Returns that shallowest state between two cluster states psci_cluster_state_t platform_psci_shallowest_cluster_state(psci_cluster_state_t state1, - psci_cluster_state_t state2); + uint16_t state2); -// Returns the deepest cluster state supported by a cpu +// Returns the deepest cluster-level suspend state supported by the system psci_cluster_state_t -platform_psci_deepest_cluster_state(cpu_index_t cpu); +platform_psci_deepest_cluster_state(void); // Returns the deepest cluster-level suspend state id supported by a cpu psci_suspend_powerstate_stateid_t platform_psci_deepest_cluster_level_stateid(cpu_index_t cpu); + #endif diff --git a/hyp/interfaces/platform/include/platform_timer_lp.h b/hyp/interfaces/platform/include/platform_timer_lp.h new file mode 100644 index 0000000..cd1aeb4 --- /dev/null +++ b/hyp/interfaces/platform/include/platform_timer_lp.h @@ -0,0 +1,25 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +void +platform_timer_lp_set_timeout(ticks_t timeout) REQUIRE_PREEMPT_DISABLED; + +ticks_t +platform_timer_lp_get_timeout(void); + +void +platform_timer_lp_cancel_timeout(void) REQUIRE_PREEMPT_DISABLED; + +uint32_t +platform_timer_lp_get_frequency(void); + +ticks_t +platform_timer_lp_get_current_ticks(void); + +void +platform_timer_lp_visibility(bool visible); + +void +platform_timer_lp_set_timeout_and_route(ticks_t timeout, cpu_index_t cpu_index) + REQUIRE_PREEMPT_DISABLED; diff --git a/hyp/interfaces/platform/platform.ev b/hyp/interfaces/platform/platform.ev index 1297253..74535c9 100644 --- a/hyp/interfaces/platform/platform.ev +++ b/hyp/interfaces/platform/platform.ev @@ -13,6 +13,8 @@ handled_event platform_ipi event platform_timer_expiry +event platform_timer_lp_expiry + event platform_pmu_counter_overflow event platform_watchdog_bark diff --git a/hyp/interfaces/platform/platform.tc b/hyp/interfaces/platform/platform.tc index 888adc4..27fcbe3 100644 --- a/hyp/interfaces/platform/platform.tc +++ b/hyp/interfaces/platform/platform.tc @@ -45,3 +45,12 @@ define platform_msi_controller structure { }; #endif // IRQ_HAS_MSI + +#if defined(ARCH_ARM) +define platform_mpidr_mapping structure { + aff_shift array(4) type count_t; + aff_mask array(4) uint8; + uniprocessor bool; + multi_thread bool; +}; +#endif diff --git a/hyp/interfaces/power/build.conf b/hyp/interfaces/power/build.conf index 516363b..bccc9c7 100644 --- a/hyp/interfaces/power/build.conf +++ b/hyp/interfaces/power/build.conf @@ -3,3 +3,4 @@ # SPDX-License-Identifier: BSD-3-Clause events power.ev +types power.tc diff --git a/hyp/interfaces/power/include/power.h b/hyp/interfaces/power/include/power.h index 4f6ddcf..f56d532 100644 --- a/hyp/interfaces/power/include/power.h +++ b/hyp/interfaces/power/include/power.h @@ -2,4 +2,27 @@ // // SPDX-License-Identifier: BSD-3-Clause -CPULOCAL_DECLARE_EXTERN(cpu_power_state_t, cpu_power_state); +// Obtain a pointer to the array of CPU states. +// +// This array may be modified by a remote CPU at any time. No synchronisation is +// available. Therefore this must only be used for debug purposes, e.g. to +// expose the array in crash dumps. +const cpu_power_state_array_t * +power_get_cpu_states_for_debug(void); + +// Vote to keep a CPU powered on. +// +// This may return ERROR_FAILURE if the specified CPU cannot be powered on for +// platform-specific reasons (such as the CPU being disabled by fuses or a +// hardware failure while powering it on), or ERROR_ARGUMENT_INVALID if the CPU +// index is out of range. +error_t +power_vote_cpu_on(cpu_index_t cpu); + +// Revoke an earlier vote to keep a CPU powered on. +// +// This must only be called after a successful power_vote_cpu_on(). Calling it +// on the only powered-on CPU may leave the system in an unrecoverable state; +// it is the caller's responsibility to avoid this. +void +power_vote_cpu_off(cpu_index_t cpu); diff --git a/hyp/interfaces/power/power.ev b/hyp/interfaces/power/power.ev index eae9884..6df8d8c 100644 --- a/hyp/interfaces/power/power.ev +++ b/hyp/interfaces/power/power.ev @@ -62,8 +62,8 @@ setup_event power_cpu_suspend // // The first_cpu argument is true if this is the first cpu to wake up. // However, beware that there is no synchronisation between this event and a -// subsequent power_cpu_suspend on another CPU, even with last_cpu == true. -// If synchronisation is required, use power_system_resume instead. +// subsequent power_cpu_suspend on another CPU, even with last_cpu set. If +// synchronisation is required, use power_system_resume instead. // // In many cases, handlers for this event will be the same as unwinders for // power_cpu_suspend; however, note that the was_poweroff argument has a diff --git a/hyp/interfaces/power/power.tc b/hyp/interfaces/power/power.tc new file mode 100644 index 0000000..a06fdee --- /dev/null +++ b/hyp/interfaces/power/power.tc @@ -0,0 +1,15 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +define cpu_power_state enumeration(explicit) { + off = 0; // never booted + cold_boot = 1; // in cold boot path, warm boot not yet reached + suspend = 2; // suspend; might either resume or warm boot + offline = 3; // voted off and called platform_cpu_off() + online = 4; // finished warm boot + started = 5; // voted on, but not yet finished warm boot +}; + +define cpu_power_state_array_t newtype + array(PLATFORM_MAX_CORES) enumeration cpu_power_state; diff --git a/hyp/interfaces/prng/prng.hvc b/hyp/interfaces/prng/prng.hvc index 1d549ad..14ddd0c 100644 --- a/hyp/interfaces/prng/prng.hvc +++ b/hyp/interfaces/prng/prng.hvc @@ -10,7 +10,7 @@ define prng_get_entropy hypercall { sensitve_return; call_num 0x57; num_bytes input type count_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; data0 output uint32; data1 output uint32; diff --git a/hyp/interfaces/psci/include/psci.h b/hyp/interfaces/psci/include/psci.h index 9e54533..3f3f7f5 100644 --- a/hyp/interfaces/psci/include/psci.h +++ b/hyp/interfaces/psci/include/psci.h @@ -2,5 +2,7 @@ // // SPDX-License-Identifier: BSD-3-Clause +// These flags are returned to userspace in vcpu_run results, so they are public +// API and must not be changed. #define PSCI_REQUEST_SYSTEM_RESET (1UL << 63) #define PSCI_REQUEST_SYSTEM_RESET2_64 (1UL << 62) diff --git a/hyp/interfaces/psci/psci.tc b/hyp/interfaces/psci/psci.tc index 1dac4dd..18ea4e5 100644 --- a/hyp/interfaces/psci/psci.tc +++ b/hyp/interfaces/psci/psci.tc @@ -78,13 +78,16 @@ define psci_suspend_powerstate_type enumeration(explicit) { powerdown = 1; }; -define vpm_mode enumeration(explicit) { - NONE = 0; - IDLE = 1; - PSCI = 2; -}; - define psci_mode enumeration(explicit) { PC = 0; OSI = 1; }; + +#if defined(INTERFACE_VCPU_RUN) +extend vcpu_run_state enumeration { + // VCPU made a PSCI_SYSTEM_RESET call to request a reset of the VM. + // This is otherwise equivalent to a power-off state, and resuming the + // VCPU will return it to a regular power-off state. + psci_system_reset = 0x100; +}; +#endif diff --git a/hyp/interfaces/qcbor/build.conf b/hyp/interfaces/qcbor/build.conf new file mode 100644 index 0000000..816c5af --- /dev/null +++ b/hyp/interfaces/qcbor/build.conf @@ -0,0 +1,3 @@ +# © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause diff --git a/hyp/interfaces/qcbor/include/qcbor.h b/hyp/interfaces/qcbor/include/qcbor.h new file mode 100644 index 0000000..0c5bdef --- /dev/null +++ b/hyp/interfaces/qcbor/include/qcbor.h @@ -0,0 +1,17 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#if defined(ARCH_ENDIAN_LITTLE) && ARCH_ENDIAN_LITTLE +#define USEFULBUF_CONFIG_LITTLE_ENDIAN 1 +#elif defined(ARCH_ENDIAN_BIG) && ARCH_ENDIAN_BIG +#define USEFULBUF_CONFIG_BIG_ENDIAN 1 +#endif + +#include "qcbor/qcbor_encode.h" + +typedef UsefulBuf useful_buff_t; +typedef UsefulBufC const_useful_buff_t; + +typedef QCBOREncodeContext qcbor_enc_ctxt_t; +typedef QCBORError qcbor_err_t; diff --git a/hyp/interfaces/qcbor/include/qcbor/UsefulBuf.h b/hyp/interfaces/qcbor/include/qcbor/UsefulBuf.h new file mode 100644 index 0000000..90958c9 --- /dev/null +++ b/hyp/interfaces/qcbor/include/qcbor/UsefulBuf.h @@ -0,0 +1,2432 @@ +/*============================================================================ + Copyright (c) 2016-2018, The Linux Foundation. + Copyright (c) 2018-2022, Laurence Lundblade. + Copyright (c) 2021, Arm Limited. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of The Linux Foundation nor the names of its + contributors, nor the name "Laurence Lundblade" may be used to + endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + =============================================================================*/ + +/*============================================================================ + FILE: UsefulBuf.h + + DESCRIPTION: General purpose input and output buffers + + EDIT HISTORY FOR FILE: + + This section contains comments describing changes made to the module. + Notice that changes are listed in reverse chronological order. + + when who what, where, why + -------- ---- -------------------------------------------------- + 4/11/2022 llundblade Add GetOutPlace and Advance to UsefulOutBuf. + 9/21/2021 llundbla Clarify UsefulOutBuf size calculation mode + 8/8/2021 dthaler/llundbla Work with C++ without compiler extensions + 5/11/2021 llundblade Improve comments and comment formatting. + 3/6/2021 mcr/llundblade Fix warnings related to --Wcast-qual + 2/17/2021 llundblade Add method to go from a pointer to an offset. + 1/25/2020 llundblade Add some casts so static anlyzers don't complain. + 5/21/2019 llundblade #define configs for efficient endianness handling. + 5/16/2019 llundblade Add UsefulOutBuf_IsBufferNULL(). + 3/23/2019 llundblade Big documentation & style update. No interface + change. + 3/6/2019 llundblade Add UsefulBuf_IsValue() + 12/17/2018 llundblade Remove const from UsefulBuf and UsefulBufC .len + 12/13/2018 llundblade Documentation improvements + 09/18/2018 llundblade Cleaner distinction between UsefulBuf and + UsefulBufC. + 02/02/18 llundbla Full support for integers in and out; fix pointer + alignment bug. Incompatible change: integers + in/out are now in network byte order. + 08/12/17 llundbla Added UsefulOutBuf_AtStart and UsefulBuf_Find + 06/27/17 llundbla Fix UsefulBuf_Compare() bug. Only affected + comparison for < or > for unequal length buffers. + Added UsefulBuf_Set() function. + 05/30/17 llundbla Functions for NULL UsefulBufs and const / unconst + 11/13/16 llundbla Initial Version. + + =============================================================================*/ + +#ifndef UsefulBuf_h_ +#define UsefulBuf_h_ + +/* + * Endianness Configuration + * + * This code is written so it will work correctly on big- and + * little-endian CPUs without configuration or any auto-detection of + * endianness. All code here will run correctly regardless of the + * endianness of the CPU it is running on. + * + * There are four C preprocessor macros that can be set with #define + * to explicitly configure endianness handling. Setting them can + * reduce code size a little and improve efficiency a little. + * + * Note that most of QCBOR is unaffected by this configuration. Its + * endianness handling is integrated with the code that handles + * alignment and preferred serialization. This configuration does + * affect QCBOR's (planned) implementation of integer arrays (tagged + * arrays) and use of the functions here to serialize or deserialize + * integers and floating-point values. + * + * Following is the recipe for configuring the endianness-related + * #defines. + * + * The first option is to not define anything. This will work fine + * with all CPUs, OS's and compilers. The code for encoding integers + * may be a little larger and slower. + * + * If your CPU is big-endian then define + * USEFULBUF_CONFIG_BIG_ENDIAN. This will give the most efficient code + * for big-endian CPUs. It will be small and efficient because there + * will be no byte swapping. + * + * Try defining USEFULBUF_CONFIG_HTON. This will work on most CPUs, + * OS's and compilers, but not all. On big-endian CPUs this should + * give the most efficient code, the same as + * USEFULBUF_CONFIG_BIG_ENDIAN does. On little-endian CPUs it should + * call the system-defined byte swapping method which is presumably + * implemented efficiently. In some cases, this will be a dedicated + * byte swap instruction like Intel's bswap. + * + * If USEFULBUF_CONFIG_HTON works and you know your CPU is + * little-endian, it is also good to define + * USEFULBUF_CONFIG_LITTLE_ENDIAN. + * + * if USEFULBUF_CONFIG_HTON doesn't work and you know your system is + * little-endian, try defining both USEFULBUF_CONFIG_LITTLE_ENDIAN and + * USEFULBUF_CONFIG_BSWAP. This should call the most efficient + * system-defined byte swap method. However, note + * https://hardwarebug.org/2010/01/14/beware-the-builtins/. Perhaps + * this is fixed now. Often hton() and ntoh() will call the built-in + * __builtin_bswapXX()() function, so this size issue could affect + * USEFULBUF_CONFIG_HTON. + * + * Last, run the tests. They must all pass. + * + * These #define config options affect the inline implementation of + * UsefulOutBuf_InsertUint64() and UsefulInputBuf_GetUint64(). They + * also affect the 16-, 32-bit, float and double versions of these + * functions. Since they are inline, the size effect is not in the + * UsefulBuf object code, but in the calling code. + * + * Summary: + * USEFULBUF_CONFIG_BIG_ENDIAN -- Force configuration to big-endian. + * USEFULBUF_CONFIG_LITTLE_ENDIAN -- Force to little-endian. + * USEFULBUF_CONFIG_HTON -- Use hton(), htonl(), ntohl()... to + * handle big and little-endian with system option. + * USEFULBUF_CONFIG_BSWAP -- With USEFULBUF_CONFIG_LITTLE_ENDIAN, + * use __builtin_bswapXX(). + * + * It is possible to run this code in environments where using floating point is + * not allowed. Defining USEFULBUF_DISABLE_ALL_FLOAT will disable all the code + * that is related to handling floating point types, along with related + * interfaces. This makes it possible to compile the code with the compile + * option -mgeneral-regs-only. + */ + +#if defined(USEFULBUF_CONFIG_BIG_ENDIAN) && \ + defined(USEFULBUF_CONFIG_LITTLE_ENDIAN) +#error "Cannot define both USEFULBUF_CONFIG_BIG_ENDIAN and USEFULBUF_CONFIG_LITTLE_ENDIAN" +#endif + +#include /* for size_t */ +#include /* for uint8_t, uint16_t.... */ +#include /* for strlen, memcpy, memmove, memset */ + +#ifdef USEFULBUF_CONFIG_HTON +#include /* for htons, htonl, htonll, ntohs... */ +#endif + +#ifdef __cplusplus +extern "C" { +#if 0 +} /* Keep editor indention formatting happy */ +#endif +#endif + +/** + * @file UsefulBuf.h + * + * The goal of this code is to make buffer and pointer manipulation + * easier and safer when working with binary data. + * + * The @ref UsefulBuf, @ref UsefulOutBuf and @ref UsefulInputBuf + * structures are used to represent buffers rather than ad hoc + * pointers and lengths. + * + * With these it is possible to write code that does little or no + * direct pointer manipulation for copying and formatting data. For + * example, the QCBOR encoder was written using these and has less + * pointer manipulation. + * + * While it is true that object code using these functions will be a + * little larger and slower than a white-knuckle clever use of + * pointers might be, but not by that much or enough to have an effect + * for most use cases. For security-oriented code this is highly + * worthwhile. Clarity, simplicity, reviewability and are more + * important. + * + * There are some extra sanity and double checks in this code to help + * catch coding errors and simple memory corruption. They are helpful, + * but not a substitute for proper code review, input validation and + * such. + * + * This code consists of a lot of inline functions and a few that are + * not. It should not generate very much object code, especially with + * the optimizer turned up to @c -Os or @c -O3. + */ + +/** + * @ref UsefulBufC and @ref UsefulBuf are simple data structures to + * hold a pointer and length for binary data. In C99 this data + * structure can be passed on the stack making a lot of code cleaner + * than carrying around a pointer and length as two parameters. + * + * This is also conducive to secure coding practice as the length is + * always carried with the pointer and the convention for handling a + * pointer and a length is clear. + * + * While it might be possible to write buffer and pointer code more + * efficiently in some use cases, the thought is that unless there is + * an extreme need for performance (e.g., you are building a + * gigabit-per-second IP router), it is probably better to have + * cleaner code you can be most certain about the security of. + * + * The non-const @ref UsefulBuf is usually used to refer an empty + * buffer to be filled in. The length is the size of the buffer. + * + * The const @ref UsefulBufC is usually used to refer to some data + * that has been filled in. The length is amount of valid data pointed + * to. + * + * A common use mode is to pass a @ref UsefulBuf to a function, the + * function puts some data in it, then the function returns a @ref + * UsefulBufC refering to the data. The @ref UsefulBuf is a non-const + * "in" parameter and the @ref UsefulBufC is a const "out" parameter + * so the constness stays correct. There is no single "in,out" + * parameter (if there was, it would have to be non-const). Note that + * the pointer returned in the @ref UsefulBufC usually ends up being + * the same pointer passed in as a @ref UsefulBuf, though this is not + * striclty required. + * + * A @ref UsefulBuf is null, it has no value, when @c ptr in it is + * @c NULL. + * + * There are functions and macros for the following: + * - Initializing + * - Create initialized const @ref UsefulBufC from compiler literals + * - Create initialized const @ref UsefulBufC from NULL-terminated string + * - Make an empty @ref UsefulBuf on the stack + * - Checking whether a @ref UsefulBuf is null, empty or both + * - Copying, copying with offset, copying head or tail + * - Comparing and finding substrings + * + * See also @ref UsefulOutBuf. It is a richer structure that has both + * the size of the valid data and the size of the buffer. + * + * @ref UsefulBuf is only 16 or 8 bytes on a 64- or 32-bit machine so + * it can go on the stack and be a function parameter or return value. + * + * Another way to look at it is this. C has the NULL-terminated string + * as a means for handling text strings, but no means or convention + * for binary strings. Other languages do have such means, Rust, an + * efficient compiled language, for example. + * + * @ref UsefulBuf is kind of like the Useful Pot Pooh gave Eeyore on + * his birthday. Eeyore's balloon fits beautifully, "it goes in and + * out like anything". + */ +typedef struct q_useful_buf_c { + const void *ptr; + size_t len; +} UsefulBufC; + +/** + * This non-const @ref UsefulBuf is typically used for some allocated + * memory that is to be filled in. The @c len is the amount of memory, + * not the length of the valid data in the buffer. + */ +typedef struct q_useful_buf { + void *ptr; + size_t len; +} UsefulBuf; + +/** + * A null @ref UsefulBufC is one that has no value in the same way a + * @c NULL pointer has no value. A @ref UsefulBufC is @c NULL when + * the @c ptr field is @c NULL. It doesn't matter what @c len is. See + * UsefulBuf_IsEmpty() for the distinction between null and empty. + */ +/* + * NULLUsefulBufC and few other macros have to be + * definied differently in C than C++ because there + * is no common construct for a literal structure. + * + * In C compound literals are used. + * + * In C++ list initalization is used. This only works + * in C++11 and later. + * + * Note that some popular C++ compilers can handle compound + * literals with on-by-default extensions, however + * this code aims for full correctness with strict + * compilers so they are not used. + */ +#ifdef __cplusplus +#define NULLUsefulBufC \ + { \ + NULL, 0 \ + } +#else +#define NULLUsefulBufC ((UsefulBufC){ NULL, 0 }) +#endif + +/** + * A null @ref UsefulBuf is one that has no memory associated the same + * way @c NULL points to nothing. It does not matter what @c len is. + **/ +#ifdef __cplusplus +#define NULLUsefulBuf \ + { \ + NULL, 0 \ + } +#else +#define NULLUsefulBuf ((UsefulBuf){ NULL, 0 }) +#endif + +/** + * @brief Check if a @ref UsefulBuf is @ref NULLUsefulBuf or not. + * + * @param[in] UB The UsefulBuf to check. + * + * @return 1 if it is @ref NULLUsefulBuf, 0 if not. + */ +static inline int +UsefulBuf_IsNULL(UsefulBuf UB); + +/** + * @brief Check if a @ref UsefulBufC is @ref NULLUsefulBufC or not. + * + * @param[in] UB The @ref UsefulBufC to check. + * + * @return 1 if it is @c NULLUsefulBufC, 0 if not. + */ +static inline int +UsefulBuf_IsNULLC(UsefulBufC UB); + +/** + * @brief Check if a @ref UsefulBuf is empty or not. + * + * @param[in] UB The @ref UsefulBuf to check. + * + * @return 1 if it is empty, 0 if not. + * + * An "empty" @ref UsefulBuf is one that has a value and can be + * considered to be set, but that value is of zero length. It is + * empty when @c len is zero. It doesn't matter what the @c ptr is. + * + * Many uses will not need to clearly distinguish a @c NULL @ref + * UsefulBuf from an empty one and can have the @c ptr @c NULL and the + * @c len 0. However if a use of @ref UsefulBuf needs to make a + * distinction then @c ptr should not be @c NULL when the @ref + * UsefulBuf is considered empty, but not @c NULL. + */ +static inline int +UsefulBuf_IsEmpty(UsefulBuf UB); + +/** + * @brief Check if a @ref UsefulBufC is empty or not. + * + * @param[in] UB The @ref UsefulBufC to check. + * + * @return 1 if it is empty, 0 if not. + */ +static inline int +UsefulBuf_IsEmptyC(UsefulBufC UB); + +/** + * @brief Check if a @ref UsefulBuf is @ref NULLUsefulBuf or empty. + * + * @param[in] UB The @ref UsefulBuf to check. + * + * @return 1 if it is either @ref NULLUsefulBuf or empty, 0 if not. + */ +static inline int +UsefulBuf_IsNULLOrEmpty(UsefulBuf UB); + +/** + * @brief Check if a @ref UsefulBufC is @ref NULLUsefulBufC or empty. + * + * @param[in] UB The @ref UsefulBufC to check. + * + * @return 1 if it is either @ref NULLUsefulBufC or empty, 0 if not. + */ +static inline int +UsefulBuf_IsNULLOrEmptyC(UsefulBufC UB); + +/** + * @brief Convert a non-const @ref UsefulBuf to a const @ref UsefulBufC. + * + * @param[in] UB The @ref UsefulBuf to convert. + * + * @return A @ref UsefulBufC struct. + */ +static inline UsefulBufC +UsefulBuf_Const(const UsefulBuf UB); + +/** + * @brief Convert a const @ref UsefulBufC to a non-const @ref UsefulBuf. + * + * @param[in] UBC The @ref UsefulBuf to convert. + * + * @return A non-const @ref UsefulBuf struct. + * + * Use of this is not necessary for the intended use mode of @ref + * UsefulBufC and @ref UsefulBuf. In that mode, the @ref UsefulBuf is + * created to describe a buffer that has not had any data put in + * it. Then the data is put in it. Then a @ref UsefulBufC is create + * to describe the part with the data in it. This goes from non-const + * to const, so this function is not needed. + * + * If the -Wcast-qual warning is enabled, this function can be used to + * avoid that warning. + */ +static inline UsefulBuf +UsefulBuf_Unconst(const UsefulBufC UBC); + +/** + * Convert a literal string to a @ref UsefulBufC. + * + * @c szString must be a literal string that @c sizeof() works on. + * This is better for literal strings than UsefulBuf_FromSZ() because + * it generates less code. It will not work on non-literal strings. + * + * The terminating \0 (NULL) is NOT included in the length! + */ +#ifdef __cplusplus +#define UsefulBuf_FROM_SZ_LITERAL(szString) \ + { \ + (szString), sizeof(szString) - 1 \ + } +#else +#define UsefulBuf_FROM_SZ_LITERAL(szString) \ + ((UsefulBufC){ (szString), sizeof(szString) - 1 }) +#endif + +/** + * Convert a literal byte array to a @ref UsefulBufC. + * + * @c pBytes must be a literal string that @c sizeof() works on. It + * will not work on non-literal arrays. + */ +#ifdef __cplusplus +#define UsefulBuf_FROM_BYTE_ARRAY_LITERAL(pBytes) \ + { \ + (pBytes), sizeof(pBytes) \ + } +#else +#define UsefulBuf_FROM_BYTE_ARRAY_LITERAL(pBytes) \ + ((UsefulBufC){ (pBytes), sizeof(pBytes) }) +#endif + +/** + * Make an automatic variable named @c name of type @ref UsefulBuf and + * point it to a stack variable of the given @c size. + */ +#define UsefulBuf_MAKE_STACK_UB(name, size) \ + uint8_t pBuf##name##_[(size)]; \ + UsefulBuf name = { pBuf##name##_, sizeof(pBuf##name##_) } + +/** + * Make a byte array in to a @ref UsefulBuf. This is usually used on + * stack variables or static variables. Also see @ref + * UsefulBuf_MAKE_STACK_UB. + */ +#ifdef __cplusplus +#define UsefulBuf_FROM_BYTE_ARRAY(pBytes) \ + { \ + (pBytes), sizeof(pBytes) \ + } +#else +#define UsefulBuf_FROM_BYTE_ARRAY(pBytes) \ + ((UsefulBuf){ (pBytes), sizeof(pBytes) }) +#endif + +/** + * @brief Convert a NULL-terminated string to a @ref UsefulBufC. + * + * @param[in] szString The string to convert. + * + * @return A @ref UsefulBufC struct. + * + * @c UsefulBufC.ptr points to the string so its lifetime must be + * maintained. + * + * The terminating \0 (NULL) is NOT included in the length. + */ +static inline UsefulBufC +UsefulBuf_FromSZ(const char *szString); + +/** + * @brief Copy one @ref UsefulBuf into another at an offset. + * + * @param[in] Dest Destination buffer to copy into. + * @param[in] uOffset The byte offset in @c Dest at which to copy to. + * @param[in] Src The bytes to copy. + * + * @return Pointer and length of the copy or @ref NULLUsefulBufC. + * + * This fails and returns @ref NULLUsefulBufC if @c offset is beyond the + * size of @c Dest. + * + * This fails and returns @ref NULLUsefulBufC if the @c Src length + * plus @c uOffset is greater than the length of @c Dest. + * + * The results are undefined if @c Dest and @c Src overlap. + * + * This assumes that there is valid data in @c Dest up to @c + * uOffset. The @ref UsefulBufC returned starts at the beginning of @c + * Dest and goes to @c Src.len @c + @c uOffset. + */ +UsefulBufC +UsefulBuf_CopyOffset(UsefulBuf Dest, size_t uOffset, const UsefulBufC Src); + +/** + * @brief Copy one @ref UsefulBuf into another. + * + * @param[in] Dest The destination buffer to copy into. + * @param[out] Src The source to copy from. + * + * @return Filled in @ref UsefulBufC on success, @ref NULLUsefulBufC + * on failure. + * + * This fails if @c Src.len is greater than @c Dest.len. + * + * Note that like @c memcpy(), the pointers are not checked and this + * will crash rather than return @ref NULLUsefulBufC if they are @c + * NULL or invalid. + * + * The results are undefined if @c Dest and @c Src overlap. + */ +static inline UsefulBufC +UsefulBuf_Copy(UsefulBuf Dest, const UsefulBufC Src); + +/** + * @brief Set all bytes in a @ref UsefulBuf to a value, for example to 0. + * + * @param[in] pDest The destination buffer to copy into. + * @param[in] value The value to set the bytes to. + * + * Note that like @c memset(), the pointer in @c pDest is not checked + * and this will crash if @c NULL or invalid. + */ +static inline UsefulBufC +UsefulBuf_Set(UsefulBuf pDest, uint8_t value); + +/** + * @brief Copy a pointer into a @ref UsefulBuf. + * + * @param[in,out] Dest The destination buffer to copy into. + * @param[in] ptr The source to copy from. + * @param[in] uLen Length of the source; amount to copy. + * + * @return Filled in @ref UsefulBufC on success, @ref NULLUsefulBufC + * on failure. + * + * This fails and returns @ref NULLUsefulBufC if @c uLen is greater + * than @c pDest->len. + * + * Note that like @c memcpy(), the pointers are not checked and this + * will crash, rather than return 1 if they are @c NULL or invalid. + */ +static inline UsefulBufC +UsefulBuf_CopyPtr(UsefulBuf Dest, const void *ptr, size_t uLen); + +/** + * @brief Returns a truncation of a @ref UsefulBufC. + * + * @param[in] UB The buffer to get the head of. + * @param[in] uAmount The number of bytes in the head. + * + * @return A @ref UsefulBufC that is the head of UB. + */ +static inline UsefulBufC +UsefulBuf_Head(UsefulBufC UB, size_t uAmount); + +/** + * @brief Returns bytes from the end of a @ref UsefulBufC. + * + * @param[in] UB The buffer to get the tail of. + * @param[in] uAmount The offset from the start where the tail is to begin. + * + * @return A @ref UsefulBufC that is the tail of @c UB or @ref NULLUsefulBufC + * if @c uAmount is greater than the length of the @ref UsefulBufC. + * + * If @c UB.ptr is @c NULL, but @c UB.len is not zero, then the result will + * be a @ref UsefulBufC with a @c NULL @c ptr and @c len with the length + * of the tail. + */ +static inline UsefulBufC +UsefulBuf_Tail(UsefulBufC UB, size_t uAmount); + +/** + * @brief Compare one @ref UsefulBufC to another. + * + * @param[in] UB1 The first buffer to compare. + * @param[in] UB2 The second buffer to compare. + * + * @return 0, positive or negative value. + * + * Returns a negative value if @c UB1 if is less than @c UB2. @c UB1 is + * less than @c UB2 if it is shorter or the first byte that is not the + * same is less. + * + * Returns 0 if the inputs are the same. + * + * Returns a positive value if @c UB2 is less than @c UB1. + * + * All that is of significance is that the result is positive, negative + * or 0. (This doesn't return the difference between the first + * non-matching byte like @c memcmp() ). + */ +int +UsefulBuf_Compare(const UsefulBufC UB1, const UsefulBufC UB2); + +/** + * @brief Find first byte that is not a particular byte value. + * + * @param[in] UB The destination buffer for byte comparison. + * @param[in] uValue The byte value to compare to. + * + * @return Offset of first byte that isn't @c uValue or + * @c SIZE_MAX if all bytes are @c uValue. + * + * Note that unlike most comparison functions, 0 + * does not indicate a successful comparison, so the + * test for match is: + * + * UsefulBuf_IsValue(...) == SIZE_MAX + * + * If @c UB is null or empty, there is no match + * and 0 is returned. + */ +size_t +UsefulBuf_IsValue(const UsefulBufC UB, uint8_t uValue); + +/** + * @brief Find one @ref UsefulBufC in another. + * + * @param[in] BytesToSearch Buffer to search through. + * @param[in] BytesToFind Buffer with bytes to be found. + * + * @return Position of found bytes or @c SIZE_MAX if not found. + */ +size_t +UsefulBuf_FindBytes(UsefulBufC BytesToSearch, UsefulBufC BytesToFind); + +/** + @brief Convert a pointer to an offset with bounds checking. + + @param[in] UB Pointer to the UsefulInputBuf. + @param[in] p Pointer to convert to offset. + + @return SIZE_MAX if @c p is out of range, the byte offset if not. +*/ +static inline size_t +UsefulBuf_PointerToOffset(UsefulBufC UB, const void *p); + +#ifndef USEFULBUF_DISABLE_DEPRECATED +/** Deprecated macro; use @ref UsefulBuf_FROM_SZ_LITERAL instead */ +#define SZLiteralToUsefulBufC(szString) UsefulBuf_FROM_SZ_LITERAL(szString) + +/** Deprecated macro; use UsefulBuf_MAKE_STACK_UB instead */ +#define MakeUsefulBufOnStack(name, size) \ + uint8_t __pBuf##name[(size)]; \ + UsefulBuf name = { __pBuf##name, sizeof(__pBuf##name) } + +/** Deprecated macro; use @ref UsefulBuf_FROM_BYTE_ARRAY_LITERAL instead */ +#define ByteArrayLiteralToUsefulBufC(pBytes) \ + UsefulBuf_FROM_BYTE_ARRAY_LITERAL(pBytes) + +/** Deprecated function; use UsefulBuf_Unconst() instead */ +static inline UsefulBuf +UsefulBufC_Unconst(const UsefulBufC UBC) +{ + UsefulBuf UB; + + // See UsefulBuf_Unconst() implementation for comment on pragmas +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" + UB.ptr = (void *)UBC.ptr; +#pragma GCC diagnostic pop + + UB.len = UBC.len; + + return UB; +} +#endif /* USEFULBUF_DISABLE_DEPRECATED */ + +#ifndef USEFULBUF_DISABLE_ALL_FLOAT +/** + * @brief Copy a @c float to a @c uint32_t. + * + * @param[in] f Float value to copy. + * + * @return A @c uint32_t with the float bits. + * + * Convenience function to avoid type punning, compiler warnings and + * such. The optimizer usually reduces this to a simple assignment. This + * is a crusty corner of C. + */ +static inline uint32_t +UsefulBufUtil_CopyFloatToUint32(float f); + +/** + * @brief Copy a @c double to a @c uint64_t. + * + * @param[in] d Double value to copy. + * + * @return A @c uint64_t with the double bits. + * + * Convenience function to avoid type punning, compiler warnings and + * such. The optimizer usually reduces this to a simple assignment. This + * is a crusty corner of C. + */ +static inline uint64_t +UsefulBufUtil_CopyDoubleToUint64(double d); + +/** + * @brief Copy a @c uint32_t to a @c float. + * + * @param[in] u32 Integer value to copy. + * + * @return The value as a @c float. + * + * Convenience function to avoid type punning, compiler warnings and + * such. The optimizer usually reduces this to a simple assignment. This + * is a crusty corner of C. + */ +static inline float +UsefulBufUtil_CopyUint32ToFloat(uint32_t u32); + +/** + * @brief Copy a @c uint64_t to a @c double. + * + * @param[in] u64 Integer value to copy. + * + * @return The value as a @c double. + * + * Convenience function to avoid type punning, compiler warnings and + * such. The optimizer usually reduces this to a simple assignment. This + * is a crusty corner of C. + */ +static inline double +UsefulBufUtil_CopyUint64ToDouble(uint64_t u64); +#endif /* USEFULBUF_DISABLE_ALL_FLOAT */ + +/** + * UsefulOutBuf is a structure and functions (an object) for + * serializing data into a buffer to encode for a network protocol or + * write data to a file. + * + * The main idea is that all the pointer manipulation is performed by + * @ref UsefulOutBuf functions so the caller doesn't have to do any + * pointer manipulation. The pointer manipulation is centralized. + * This code has been reviewed and written carefully so it + * spares the caller of much of this work and results in safer code + * with less effort. + * + * The @ref UsefulOutBuf methods that add data to the output buffer + * always check the length and will never write off the end of the + * output buffer. If an attempt to add data that will not fit is made, + * an internal error flag will be set and further attempts to add data + * will not do anything. + * + * There is no way to ever write off the end of that buffer when + * calling the @c UsefulOutBuf_AddXxx() and + * @c UsefulOutBuf_InsertXxx() functions. + * + * The functions to add data do not report success of failure. The + * caller only needs to check for an error in the final call, either + * UsefulOutBuf_OutUBuf() or UsefulOutBuf_CopyOut() to get the + * result. This makes the calling code cleaner. + * + * There is a utility function to get the error status anytime along + * the way for a special circumstance. There are functions to see how + * much room is left and see if some data will fit too, but their use + * is generally unnecessary. + * + * The general call flow is: + * + * - Initialize by calling @ref UsefulOutBuf_Init(). The output + * buffer given to it can be from the heap, stack or + * otherwise. @ref UsefulOutBuf_MakeOnStack is a convenience + * macro that makes a buffer on the stack and initializes it. + * + * - Call methods like UsefulOutBuf_InsertString(), + * UsefulOutBuf_AppendUint32() and UsefulOutBuf_InsertUsefulBuf() + * to output data. The append calls add data to the end of the + * valid data. The insert calls take a position argument. + * + * - Call UsefulOutBuf_OutUBuf() or UsefulOutBuf_CopyOut() to see + * there were no errors and to get the serialized output bytes. + * + * @ref UsefulOutBuf can be used in a mode to calculate the size of + * what would be output without actually outputting anything. This is + * useful to calculate the size of a buffer that is to be allocated to + * hold the output. See @ref SizeCalculateUsefulBuf. + * + * Methods like UsefulOutBuf_InsertUint64() always output in network + * byte order (big endian). + * + * The possible errors are: + * + * - The @ref UsefulOutBuf was not initialized or was corrupted. + * + * - An attempt was made to add data that will not fit. + * + * - An attempt was made to insert data at a position beyond the end of + * the buffer. + * + * - An attempt was made to insert data at a position beyond the valid + * data in the buffer. + * + * Some inexpensive simple sanity checks are performed before every + * data addition to guard against use of an uninitialized or corrupted + * UsefulOutBuf. + * + * @ref UsefulOutBuf has been used to create a CBOR encoder. The CBOR + * encoder has almost no pointer manipulation in it, is easier to + * read, and easier to review. + * + * A @ref UsefulOutBuf is small and can go on the stack: + * - 32 bytes (27 bytes plus alignment padding) on a 64-bit CPU + * - 16 bytes (15 bytes plus alignment padding) on a 32-bit CPU + */ +typedef struct useful_out_buf { + /* PRIVATE DATA STRUCTURE */ + UsefulBuf UB; /* Memory that is being output to */ + size_t data_len; /* length of the valid data, the insertion point */ + uint16_t magic; /* Used to detect corruption and lack + * of initialization */ + uint8_t err; + uint8_t padding_[5]; +} UsefulOutBuf; + +/** + * This is a @ref UsefulBuf value that can be passed to + * UsefulOutBuf_Init() to have it calculate the size of the output + * buffer needed. Pass this for @c Storage, call all the append and + * insert functions normally, then call UsefulOutBuf_OutUBuf(). The + * returned @ref UsefulBufC has the size. + * + * As one can see, this is just a NULL pointer and very large size. + * The NULL pointer tells UsefulOutputBuf to not copy any data. + */ +#ifdef __cplusplus +#define SizeCalculateUsefulBuf \ + { \ + NULL, SIZE_MAX \ + } +#else +#define SizeCalculateUsefulBuf ((UsefulBuf){ NULL, SIZE_MAX }) +#endif + +/** + * @brief Initialize and supply the actual output buffer. + * + * @param[out] pUOutBuf The @ref UsefulOutBuf to initialize. + * @param[in] Storage Buffer to output into. + * + * This initializes the @ref UsefulOutBuf with storage, sets the + * current position to the beginning of the buffer and clears the + * error state. + * + * See @ref SizeCalculateUsefulBuf for instructions on how to + * initialize a @ref UsefulOutBuf to calculate the size that would be + * output without actually outputting. + * + * This must be called before the @ref UsefulOutBuf is used. + */ +void +UsefulOutBuf_Init(UsefulOutBuf *pUOutBuf, UsefulBuf Storage); + +/** + * Convenience macro to make a @ref UsefulOutBuf on the stack and + * initialize it with a stack buffer of the given size. The variable + * will be named @c name. + */ +#define UsefulOutBuf_MakeOnStack(name, size) \ + uint8_t __pBuf##name[(size)]; \ + UsefulOutBuf name; \ + UsefulOutBuf_Init(&(name), (UsefulBuf){ __pBuf##name, (size) }); + +/** + * @brief Reset a @ref UsefulOutBuf for re use. + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf + * + * This sets the amount of data in the output buffer to none and + * clears the error state. + * + * The output buffer is still the same one and size as from the + * UsefulOutBuf_Init() call. + * + * This doesn't zero the data, just resets to 0 bytes of valid data. + */ +static inline void +UsefulOutBuf_Reset(UsefulOutBuf *pUOutBuf); + +/** + * @brief Returns position of end of data in the @ref UsefulOutBuf. + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * + * @return position of end of data. + * + * On a freshly initialized @ref UsefulOutBuf with no data added, this + * will return 0. After 10 bytes have been added, it will return 10 + * and so on. + * + * Generally, there is no need to call this for most uses of @ref + * UsefulOutBuf. + */ +static inline size_t +UsefulOutBuf_GetEndPosition(UsefulOutBuf *pUOutBuf); + +/** + * @brief Returns whether any data has been added to the @ref UsefulOutBuf. + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * + * @return 1 if output position is at start, 0 if not. + */ +static inline int +UsefulOutBuf_AtStart(UsefulOutBuf *pUOutBuf); + +/** + * @brief Inserts bytes into the @ref UsefulOutBuf. + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * @param[in] NewData The bytes to insert. + * @param[in] uPos Index in output buffer at which to insert. + * + * @c NewData is the pointer and length for the bytes to be added to + * the output buffer. There must be room in the output buffer for all + * of @c NewData or an error will occur. + * + * The insertion point must be between 0 and the current valid + * data. If not, an error will occur. Appending data to the output + * buffer is achieved by inserting at the end of the valid data. This + * can be retrieved by calling UsefulOutBuf_GetEndPosition(). + * + * When insertion is performed, the bytes between the insertion point + * and the end of data previously added to the output buffer are slid + * to the right to make room for the new data. + * + * Overlapping buffers are OK. @c NewData can point to data in the + * output buffer. + * + * If an error occurs, an error state is set in the @ref + * UsefulOutBuf. No error is returned. All subsequent attempts to add + * data will do nothing. + * + * The intended use is that all additions are made without checking + * for an error. The error will be taken into account when + * UsefulOutBuf_OutUBuf() returns @c NullUsefulBufC. + * UsefulOutBuf_GetError() can also be called to check for an error. + */ +void +UsefulOutBuf_InsertUsefulBuf(UsefulOutBuf *pUOutBuf, UsefulBufC NewData, + size_t uPos); + +/** + * @brief Insert a data buffer into the @ref UsefulOutBuf. + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * @param[in] pBytes Pointer to the bytes to insert + * @param[in] uLen Length of the bytes to insert + * @param[in] uPos Index in output buffer at which to insert + * + * See UsefulOutBuf_InsertUsefulBuf() for details. This is the same with + * the difference being a pointer and length is passed in rather than an + * @ref UsefulBufC. + */ +static inline void +UsefulOutBuf_InsertData(UsefulOutBuf *pUOutBuf, const void *pBytes, size_t uLen, + size_t uPos); + +/** + * @brief Insert a NULL-terminated string into the UsefulOutBuf. + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * @param[in] szString NULL-terminated string to insert. + * @param[in] uPos Index in output buffer at which to insert. + */ +static inline void +UsefulOutBuf_InsertString(UsefulOutBuf *pUOutBuf, const char *szString, + size_t uPos); + +/** + * @brief Insert a byte into the @ref UsefulOutBuf. + * + * @param[in] pUOutBuf Pointer to the UsefulOutBuf. + * @param[in] byte Bytes to insert. + * @param[in] uPos Index in output buffer at which to insert. + * + * See UsefulOutBuf_InsertUsefulBuf() for details. This is the same + * with the difference being a single byte is to be inserted. + */ +static inline void +UsefulOutBuf_InsertByte(UsefulOutBuf *pUOutBuf, uint8_t byte, size_t uPos); + +/** + * @brief Insert a 16-bit integer into the @ref UsefulOutBuf. + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * @param[in] uInteger16 Integer to insert. + * @param[in] uPos Index in output buffer at which to insert. + * + * See UsefulOutBuf_InsertUsefulBuf() for details. This is the same + * with the difference being a two-byte integer is to be inserted. + * + * The integer will be inserted in network byte order (big endian). + */ +static inline void +UsefulOutBuf_InsertUint16(UsefulOutBuf *pUOutBuf, uint16_t uInteger16, + size_t uPos); + +/** + * @brief Insert a 32-bit integer into the @ref UsefulOutBuf. + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * @param[in] uInteger32 Integer to insert. + * @param[in] uPos Index in output buffer at which to insert. + * + * See UsefulOutBuf_InsertUsefulBuf() for details. This is the same + * with the difference being a four-byte integer is to be inserted. + * + * The integer will be inserted in network byte order (big endian). + */ +static inline void +UsefulOutBuf_InsertUint32(UsefulOutBuf *pUOutBuf, uint32_t uInteger32, + size_t uPos); + +/** + * @brief Insert a 64-bit integer into the @ref UsefulOutBuf. + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * @param[in] uInteger64 Integer to insert. + * @param[in] uPos Index in output buffer at which to insert. + * + * See UsefulOutBuf_InsertUsefulBuf() for details. This is the same + * with the difference being an eight-byte integer is to be inserted. + * + * The integer will be inserted in network byte order (big endian). + */ +static inline void +UsefulOutBuf_InsertUint64(UsefulOutBuf *pUOutBuf, uint64_t uInteger64, + size_t uPos); + +#ifndef USEFULBUF_DISABLE_ALL_FLOAT +/** + * @brief Insert a @c float into the @ref UsefulOutBuf. + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * @param[in] f @c float to insert. + * @param[in] uPos Index in output buffer at which to insert. + * + * See UsefulOutBuf_InsertUsefulBuf() for details. This is the same + * with the difference being a @c float is to be inserted. + * + * The @c float will be inserted in network byte order (big endian). + */ +static inline void +UsefulOutBuf_InsertFloat(UsefulOutBuf *pUOutBuf, float f, size_t uPos); + +/** + * @brief Insert a @c double into the @ref UsefulOutBuf. + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * @param[in] d @c double to insert. + * @param[in] uPos Index in output buffer at which to insert. + * + * See UsefulOutBuf_InsertUsefulBuf() for details. This is the same + * with the difference being a @c double is to be inserted. + * + * The @c double will be inserted in network byte order (big endian). + */ +static inline void +UsefulOutBuf_InsertDouble(UsefulOutBuf *pUOutBuf, double d, size_t uPos); +#endif /* USEFULBUF_DISABLE_ALL_FLOAT */ + +/** + * @brief Append a @ref UsefulBuf into the @ref UsefulOutBuf. + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * @param[in] NewData The @ref UsefulBuf with the bytes to append. + * + * See UsefulOutBuf_InsertUsefulBuf() for details. This does the same + * with the insertion point at the end of the valid data. + */ +static inline void +UsefulOutBuf_AppendUsefulBuf(UsefulOutBuf *pUOutBuf, UsefulBufC NewData); + +/** + * @brief Append bytes to the @ref UsefulOutBuf. + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * @param[in] pBytes Pointer to bytes to append. + * @param[in] uLen Length of @c pBytes to append. + * + * See UsefulOutBuf_InsertData() for details. This does the same with + * the insertion point at the end of the valid data. + */ +static inline void +UsefulOutBuf_AppendData(UsefulOutBuf *pUOutBuf, const void *pBytes, + size_t uLen); + +/** + * @brief Append a NULL-terminated string to the @ref UsefulOutBuf + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * @param[in] szString NULL-terminated string to append. + */ +static inline void +UsefulOutBuf_AppendString(UsefulOutBuf *pUOutBuf, const char *szString); + +/** + * @brief Append a byte to the @ref UsefulOutBuf + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * @param[in] byte Bytes to append. + * + * See UsefulOutBuf_InsertByte() for details. This does the same + * with the insertion point at the end of the valid data. + */ +static inline void +UsefulOutBuf_AppendByte(UsefulOutBuf *pUOutBuf, uint8_t byte); + +/** + * @brief Append an integer to the @ref UsefulOutBuf + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * @param[in] uInteger16 Integer to append. + * + * See UsefulOutBuf_InsertUint16() for details. This does the same + * with the insertion point at the end of the valid data. + * + * The integer will be appended in network byte order (big endian). + */ +static inline void +UsefulOutBuf_AppendUint16(UsefulOutBuf *pUOutBuf, uint16_t uInteger16); + +/** + * @brief Append an integer to the @ref UsefulOutBuf + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * @param[in] uInteger32 Integer to append. + * + * See UsefulOutBuf_InsertUint32() for details. This does the same + * with the insertion point at the end of the valid data. + * + * The integer will be appended in network byte order (big endian). + */ +static inline void +UsefulOutBuf_AppendUint32(UsefulOutBuf *pUOutBuf, uint32_t uInteger32); + +/** + * @brief Append an integer to the @ref UsefulOutBuf + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * @param[in] uInteger64 Integer to append. + * + * See UsefulOutBuf_InsertUint64() for details. This does the same + * with the insertion point at the end of the valid data. + * + * The integer will be appended in network byte order (big endian). + */ +static inline void +UsefulOutBuf_AppendUint64(UsefulOutBuf *pUOutBuf, uint64_t uInteger64); + +#ifndef USEFULBUF_DISABLE_ALL_FLOAT +/** + * @brief Append a @c float to the @ref UsefulOutBuf + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * @param[in] f @c float to append. + * + * See UsefulOutBuf_InsertFloat() for details. This does the same with + * the insertion point at the end of the valid data. + * + * The float will be appended in network byte order (big endian). + */ +static inline void +UsefulOutBuf_AppendFloat(UsefulOutBuf *pUOutBuf, float f); + +/** + * @brief Append a @c double to the @ref UsefulOutBuf + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * @param[in] d @c double to append. + * + * See UsefulOutBuf_InsertDouble() for details. This does the same + * with the insertion point at the end of the valid data. + * + * The double will be appended in network byte order (big endian). + */ +static inline void +UsefulOutBuf_AppendDouble(UsefulOutBuf *pUOutBuf, double d); +#endif /* USEFULBUF_DISABLE_ALL_FLOAT */ + +/** + * @brief Returns the current error status. + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * + * @return 0 if all OK, 1 on error. + * + * This returns the error status since a call to either + * UsefulOutBuf_Reset() of UsefulOutBuf_Init(). Once a @ref UsefulOutBuf + * goes into the error state, it will stay until one of those + * functions is called. + * + * Possible error conditions are: + * - bytes to be inserted will not fit + * - insertion point is out of buffer or past valid data + * - current position is off end of buffer (probably corrupted or + * uninitialized) + * - detect corruption / uninitialized by bad magic number + */ +static inline int +UsefulOutBuf_GetError(UsefulOutBuf *pUOutBuf); + +/** + * @brief Returns number of bytes unused used in the output buffer. + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * + * @return Number of unused bytes or zero. + * + * Because of the error handling strategy and checks in + * UsefulOutBuf_InsertUsefulBuf() it is usually not necessary to use + * this. + */ +static inline size_t +UsefulOutBuf_RoomLeft(UsefulOutBuf *pUOutBuf); + +/** + *@brief Returns 1 if some number of bytes will fit in the @ref UsefulOutBuf. + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf + * @param[in] uLen Number of bytes for which to check + * + * @return 1 if @c uLen bytes will fit, 0 if not. + * + * Because of the error handling strategy and checks in + * UsefulOutBuf_InsertUsefulBuf() it is usually not necessary to use + * this. + */ +static inline int +UsefulOutBuf_WillItFit(UsefulOutBuf *pUOutBuf, size_t uLen); + +/** + * @brief Returns 1 if buffer given to UsefulOutBuf_Init() was @c NULL. + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf + * + * @return 1 if buffer given to UsefulOutBuf_Init() was @c NULL. + * + * Giving a @c NULL output buffer to UsefulOutBuf_Init() is used when + * just calculating the length of the encoded data. + */ +static inline int +UsefulOutBuf_IsBufferNULL(UsefulOutBuf *pUOutBuf); + +/** + * @brief Returns pointer and length of the output buffer not yet used. + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * + * @return pointer and length of output buffer not used. + * + * This is an escape that allows the caller to write directly + * to the output buffer without any checks. This doesn't + * change the output buffer or state. It just returns a pointer + * and length of the bytes remaining. + * + * This is useful to avoid having the bytes to be added all + * in a contiguous buffer. Its use can save memory. A good + * example is in the COSE encrypt implementation where + * the output of the symmetric cipher can go directly + * into the output buffer, rather than having to go into + * an intermediate buffer. + * + * See UsefulOutBuf_Advance() which is used to tell + * UsefulOutBuf how much was written. + * + * Warning: this bypasses the buffer safety provided by + * UsefulOutBuf! + */ +static inline UsefulBuf +UsefulOutBuf_GetOutPlace(UsefulOutBuf *pUOutBuf); + +/** + * @brief Advance the amount output assuming it was written by the caller. + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * @param[in] uAmount The amount to advance. + * + * This advances the position in the output buffer + * by \c uAmount. This assumes that the + * caller has written \c uAmount to the pointer obtained + * with UsefulOutBuf_GetOutPlace(). + * + * Warning: this bypasses the buffer safety provided by + * UsefulOutBuf! + */ +void +UsefulOutBuf_Advance(UsefulOutBuf *pUOutBuf, size_t uAmount); + +/** + * @brief Returns the resulting valid data in a UsefulOutBuf + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * + * @return The valid data in @ref UsefulOutBuf or + * @ref NULLUsefulBufC if there was an error adding data. + * + * The storage for the returned data is the @c Storage parameter + * passed to UsefulOutBuf_Init(). See also UsefulOutBuf_CopyOut(). + * + * This can be called anytime and many times to get intermediate + * results. It doesn't change the data or reset the current position, + * so further data can be added. + */ +UsefulBufC +UsefulOutBuf_OutUBuf(UsefulOutBuf *pUOutBuf); + +/** + * @brief Copies the valid data into a supplied buffer + * + * @param[in] pUOutBuf Pointer to the @ref UsefulOutBuf. + * @param[out] Dest The destination buffer to copy into. + * + * @return Pointer and length of copied data or @c NULLUsefulBufC + * if it will not fit in the @c Dest buffer or the error + * state was entered. + * + * This is the same as UsefulOutBuf_OutUBuf() except it copies the + * data to @c Dest. + */ +UsefulBufC +UsefulOutBuf_CopyOut(UsefulOutBuf *pUOutBuf, UsefulBuf Dest); + +/** + * @ref UsefulInputBuf is the counterpart to @ref UsefulOutBuf. It is + * for parsing data received. Initialize it with the data from the + * network. Then use the functions like UsefulInputBuf_GetBytes() to + * get data chunks of various types. A position cursor is maintained + * internally. + * + * As long as the functions here are used, there will never be any + * reference off the end of the given buffer (except + * UsefulInputBuf_SetBufferLength()). This is true even if they are + * called incorrectly, an attempt is made to seek off the end of the + * buffer or such. This makes it easier to write safe and correct + * code. For example, the QCBOR decoder implementation is safer and + * easier to review through its use of @ref UsefulInputBuf. + * + * @ref UsefulInputBuf maintains an internal error state. The + * intended use is fetching data chunks without any error checks until + * the end. If there was any error, such as an attempt to fetch data + * off the end, the error state is entered and no further data will be + * returned. In the error state the @c UsefulInputBuf_GetXxxx() + * functions return 0, or @c NULL or @ref NULLUsefulBufC. As long as + * null is not dereferenced, the error check can be put off until the + * end, simplifying the calling code. + * + * The integer and float parsing expects network byte order (big + * endian). Network byte order is what is used by TCP/IP, CBOR and + * most internet protocols. + * + * Lots of inline functions are used to keep code size down. The + * optimizer, particularly with the @c -Os or @c -O3, also reduces + * code size a lot. The only non-inline code is + * UsefulInputBuf_GetBytes(). It is less than 100 bytes so use of + * @ref UsefulInputBuf doesn't add much code for all the messy + * hard-to-get right issues with parsing binary protocols in C that it + * solves. + * + * The parse context size is: + * - 64-bit machine: 16 + 8 + 2 + 1 (+ 5 bytes padding to align) = 32 bytes + * - 32-bit machine: 8 + 4 + 2 + 1 (+ 1 byte padding to align) = 16 bytes + */ +typedef struct useful_input_buf { + /* PRIVATE DATA STRUCTURE */ + UsefulBufC UB; /* Data being parsed */ + size_t cursor; /* Current offset in data being parse */ + uint16_t magic; /* Check for corrupted or uninitialized UsefulInputBuf + */ + uint8_t err; /* Set request goes off end or magic number is bad */ + uint8_t padding_[5]; + +} UsefulInputBuf; + +#define UIB_MAGIC (0xB00F) + +#ifdef ENABLE_DECODE_ROUTINES + +/** + * @brief Initialize the @ref UsefulInputBuf structure before use. + * + * @param[in] pUInBuf Pointer to the @ref UsefulInputBuf. + * @param[in] UB The data to parse. + */ +static inline void +UsefulInputBuf_Init(UsefulInputBuf *pUInBuf, UsefulBufC UB); + +/** + * @brief Returns current position in input buffer. + * + * @param[in] pUInBuf Pointer to the @ref UsefulInputBuf. + * + * @return Integer position of the cursor. + * + * The position that the next bytes will be returned from. + */ +static size_t +UsefulInputBuf_Tell(UsefulInputBuf *pUInBuf); + +/** + * @brief Sets the current position in input buffer. + * + * @param[in] pUInBuf Pointer to the @ref UsefulInputBuf. + * @param[in] uPos Position to set to. + * + * If the position is off the end of the input buffer, the error state + * is entered. + * + * Seeking to a valid position in the buffer will not reset the error + * state. Only re-initialization will do that. + */ +static void +UsefulInputBuf_Seek(UsefulInputBuf *pUInBuf, size_t uPos); + +/** + * @brief Returns the number of bytes from the cursor to the end of the buffer, + * the unconsumed bytes. + * + * @param[in] pUInBuf Pointer to the @ref UsefulInputBuf. + * + * @return Number of bytes unconsumed or 0 on error. + * + * Returns 0 if the cursor is invalid or corruption of the + * @ref UsefulInputBuf structure is detected. + */ +static size_t +UsefulInputBuf_BytesUnconsumed(UsefulInputBuf *pUInBuf); + +/** + * @brief Check if there are unconsumed bytes. + * + * @param[in] pUInBuf Pointer to the @ref UsefulInputBuf. + * @param[in] uLen Number of bytes to check availability for. + * + * @return 1 if @c uLen bytes are available after the cursor, and 0 if not. + */ +static int +UsefulInputBuf_BytesAvailable(UsefulInputBuf *pUInBuf, size_t uLen); + +/** + * @brief Convert a pointer to an offset with bounds checking. + * + * @param[in] pUInBuf Pointer to the @ref UsefulInputBuf. + * @param[in] p Pointer to convert to offset. + * + * @return SIZE_MAX if @c p is out of range, the byte offset if not. + */ +static inline size_t +UsefulInputBuf_PointerToOffset(UsefulInputBuf *pUInBuf, const void *p); + +/** + * @brief Get pointer to bytes out of the input buffer. + * + * @param[in] pUInBuf Pointer to the @ref UsefulInputBuf. + * @param[in] uNum Number of bytes to get. + * + * @return Pointer to bytes. + * + * This consumes @c uNum bytes from the input buffer. This returns a + * pointer to the start of the @c uNum bytes. + * + * If there are not @c uNum bytes in the input buffer, @c NULL will be + * returned and the error state is entered. + * + * This advances the position cursor by @c uNum bytes. + */ +const void * +UsefulInputBuf_GetBytes(UsefulInputBuf *pUInBuf, size_t uNum); + +/** + * @brief Get @ref UsefulBuf out of the input buffer. + * + * @param[in] pUInBuf Pointer to the @ref UsefulInputBuf. + * @param[in] uNum Number of bytes to get. + * + * @return A @ref UsefulBufC with ptr and length of bytes consumed. + * + * This consumes @c uNum bytes from the input buffer and returns the + * pointer and length for them as a @ref UsefulBufC. The length + * returned will always be @c uNum. The position cursor is advanced by + * @c uNum bytes. + * + * If there are not @c uNum bytes in the input buffer, @ref + * NULLUsefulBufC will be returned and the error state is entered. + */ +static inline UsefulBufC +UsefulInputBuf_GetUsefulBuf(UsefulInputBuf *pUInBuf, size_t uNum); + +/** + * @brief Get a byte out of the input buffer. + * + * @param[in] pUInBuf Pointer to the @ref UsefulInputBuf. + * + * @return The byte. + * + * This consumes 1 byte from the input buffer, returns it and advances + * the position cursor by 1. + * + * If there is not 1 byte in the buffer, 0 will be returned for the + * byte and the error state is entered. To know if the 0 returned was + * in error or the real value, the error state must be checked. If + * possible, put this off until all values are retrieved to have + * smaller and simpler code, but if not possible + * UsefulInputBuf_GetError() can be called. Also, in the error state + * UsefulInputBuf_GetBytes() returns @c NULL *or the @c ptr from + * UsefulInputBuf_GetUsefulBuf() is @c NULL. + */ +static inline uint8_t +UsefulInputBuf_GetByte(UsefulInputBuf *pUInBuf); + +/** + * @brief Get a @c uint16_t out of the input buffer. + * + * @param[in] pUInBuf Pointer to the @ref UsefulInputBuf. + * + * @return The @c uint16_t. + * + * See UsefulInputBuf_GetByte(). This works the same, except it returns + * a @c uint16_t and two bytes are consumed. + * + * The input bytes are interpreted in network order (big endian). + */ +static inline uint16_t +UsefulInputBuf_GetUint16(UsefulInputBuf *pUInBuf); + +/** + * @brief Get a @c uint32_t out of the input buffer. + * + * @param[in] pUInBuf Pointer to the @ref UsefulInputBuf. + * + * @return The @c uint32_t. + * + * See UsefulInputBuf_GetByte(). This works the same, except it + * returns a @c uint32_t and four bytes are consumed. + * + * The input bytes are interpreted in network order (big endian). + */ +static uint32_t +UsefulInputBuf_GetUint32(UsefulInputBuf *pUInBuf); + +/** + * @brief Get a @c uint64_t out of the input buffer. + * + * @param[in] pUInBuf Pointer to the @ref UsefulInputBuf. + * + * @return The uint64_t. + * + * See UsefulInputBuf_GetByte(). This works the same, except it returns + * a @c uint64_t and eight bytes are consumed. + * + * The input bytes are interpreted in network order (big endian). + */ +static uint64_t +UsefulInputBuf_GetUint64(UsefulInputBuf *pUInBuf); + +#ifndef USEFULBUF_DISABLE_ALL_FLOAT +/** + * @brief Get a float out of the input buffer. + * + * @param[in] pUInBuf Pointer to the @ref UsefulInputBuf. + * + * @return The float. + * + * See UsefulInputBuf_GetByte(). This works the same, except it + * returns a float and four bytes are consumed. + * + * The input bytes are interpreted in network order (big endian). + */ +static float +UsefulInputBuf_GetFloat(UsefulInputBuf *pUInBuf); + +/** + * @brief Get a double out of the input buffer. + * + * @param[in] pUInBuf Pointer to the @ref UsefulInputBuf. + * + * @return The double. + * + * See UsefulInputBuf_GetByte(). This works the same, except it + * returns a double and eight bytes are consumed. + * + * The input bytes are interpreted in network order (big endian). + */ +static double +UsefulInputBuf_GetDouble(UsefulInputBuf *pUInBuf); +#endif /* USEFULBUF_DISABLE_ALL_FLOAT */ + +/** + * @brief Get the error status. + * + * @param[in] pUInBuf Pointer to the @ref UsefulInputBuf. + * + * @return 0 if not in the error state, 1 if in the error state. + * + * This returns whether the @ref UsefulInputBuf is in the + * error state or not. + * + * The error state is entered for one of these reasons: + * - Attempt to fetch data past the end of the buffer + * - Attempt to seek to a position past the end of the buffer + * - Attempt to get data from an uninitialized or corrupt instance + * of @ref UsefulInputBuf + * + * Once in the error state, it can only be cleared by calling + * UsefulInputBuf_Init(). + * + * For many use cases, it is possible to only call this once after all + * the @c UsefulInputBuf_GetXxxx() calls have been made. This is + * possible if no reference to the data returned are needed before the + * error state is checked. + * + * In some cases UsefulInputBuf_GetUsefulBuf() or + * UsefulInputBuf_GetBytes() can stand in for this because they return + * @c NULL if the error state has been entered. (The others can't stand + * in because they don't return a clearly distinct error value.) + */ +static int +UsefulInputBuf_GetError(UsefulInputBuf *pUInBuf); + +/** + * @brief Gets the input buffer length. + * + * @param[in] pUInBuf Pointer to the @ref UsefulInputBuf. + * + * @return The length of the input buffer. + * + * This returns the length of the input buffer set by + * UsefulInputBuf_Init() or UsefulInputBuf_SetBufferLength(). + */ +static inline size_t +UsefulInputBuf_GetBufferLength(UsefulInputBuf *pUInBuf); + +/** + * @brief Alters the input buffer length (use with caution). + * + * @param[in] pUInBuf Pointer to the @ref UsefulInputBuf. + * @param[in] uNewLen The new length of the input buffer. + * + * This alters the internal remembered length of the input buffer set + * when UsefulInputBuf_Init() was called. + * + * The new length given here should always be equal to or less than + * the length given when UsefulInputBuf_Init() was called. Making it + * larger allows @ref UsefulInputBuf to run off the input buffer. + * + * The typical use is to set a length shorter than that when + * initialized to constrain parsing. If + * UsefulInputBuf_GetBufferLength() was called before this, then the + * original length can be restored with another call to this. + * + * This should be used with caution. It is the only + * @ref UsefulInputBuf method that can violate the safety of input + * buffer parsing. + */ +static void +UsefulInputBuf_SetBufferLength(UsefulInputBuf *pUInBuf, size_t uNewLen); + +#endif /* ENABLE_DECODE_ROUTINES */ + +/*---------------------------------------------------------- + Inline implementations. + */ +static inline int +UsefulBuf_IsNULL(UsefulBuf UB) +{ + return !UB.ptr; +} + +static inline int +UsefulBuf_IsNULLC(UsefulBufC UB) +{ + return !UB.ptr; +} + +static inline int +UsefulBuf_IsEmpty(UsefulBuf UB) +{ + return !UB.len; +} + +static inline int +UsefulBuf_IsEmptyC(UsefulBufC UB) +{ + return !UB.len; +} + +static inline int +UsefulBuf_IsNULLOrEmpty(UsefulBuf UB) +{ + return UsefulBuf_IsEmpty(UB) || UsefulBuf_IsNULL(UB); +} + +static inline int +UsefulBuf_IsNULLOrEmptyC(UsefulBufC UB) +{ + return UsefulBuf_IsEmptyC(UB) || UsefulBuf_IsNULLC(UB); +} + +static inline UsefulBufC +UsefulBuf_Const(const UsefulBuf UB) +{ + UsefulBufC UBC; + UBC.ptr = UB.ptr; + UBC.len = UB.len; + + return UBC; +} + +static inline UsefulBuf +UsefulBuf_Unconst(const UsefulBufC UBC) +{ + UsefulBuf UB; + + /* -Wcast-qual is a good warning flag to use in general. This is + * the one place in UsefulBuf where it needs to be quieted. Since + * clang supports GCC pragmas, this works for clang too. */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" + UB.ptr = (void *)UBC.ptr; +#pragma GCC diagnostic pop + + UB.len = UBC.len; + + return UB; +} + +static inline UsefulBufC +UsefulBuf_FromSZ(const char *szString) +{ + UsefulBufC UBC; + UBC.ptr = szString; + UBC.len = strlen(szString); + return UBC; +} + +static inline UsefulBufC +UsefulBuf_Copy(UsefulBuf Dest, const UsefulBufC Src) +{ + return UsefulBuf_CopyOffset(Dest, 0, Src); +} + +static inline UsefulBufC +UsefulBuf_Set(UsefulBuf Dest, uint8_t value) +{ + memset(Dest.ptr, value, Dest.len); + + UsefulBufC UBC; + UBC.ptr = Dest.ptr; + UBC.len = Dest.len; + + return UBC; +} + +static inline UsefulBufC +UsefulBuf_CopyPtr(UsefulBuf Dest, const void *ptr, size_t len) +{ + UsefulBufC UBC; + UBC.ptr = ptr; + UBC.len = len; + return UsefulBuf_Copy(Dest, UBC); +} + +static inline UsefulBufC +UsefulBuf_Head(UsefulBufC UB, size_t uAmount) +{ + if (uAmount > UB.len) { + return NULLUsefulBufC; + } + UsefulBufC UBC; + + UBC.ptr = UB.ptr; + UBC.len = uAmount; + + return UBC; +} + +static inline UsefulBufC +UsefulBuf_Tail(UsefulBufC UB, size_t uAmount) +{ + UsefulBufC ReturnValue; + + if (uAmount > UB.len) { + ReturnValue = NULLUsefulBufC; + } else if (UB.ptr == NULL) { + ReturnValue.ptr = NULL; + ReturnValue.len = UB.len - uAmount; + } else { + ReturnValue.ptr = (const uint8_t *)UB.ptr + uAmount; + ReturnValue.len = UB.len - uAmount; + } + + return ReturnValue; +} + +static inline size_t +UsefulBuf_PointerToOffset(UsefulBufC UB, const void *p) +{ + if (UB.ptr == NULL) { + return SIZE_MAX; + } + + if (p < UB.ptr) { + /* given pointer is before start of buffer */ + return SIZE_MAX; + } + + // Cast to size_t (from ptrdiff_t) is OK because of check above + const size_t uOffset = + (size_t)((const uint8_t *)p - (const uint8_t *)UB.ptr); + + if (uOffset >= UB.len) { + /* given pointer is off the end of the buffer */ + return SIZE_MAX; + } + + return uOffset; +} + +#ifndef USEFULBUF_DISABLE_ALL_FLOAT +static inline uint32_t +UsefulBufUtil_CopyFloatToUint32(float f) +{ + uint32_t u32; + memcpy(&u32, &f, sizeof(uint32_t)); + return u32; +} + +static inline uint64_t +UsefulBufUtil_CopyDoubleToUint64(double d) +{ + uint64_t u64; + memcpy(&u64, &d, sizeof(uint64_t)); + return u64; +} + +static inline double +UsefulBufUtil_CopyUint64ToDouble(uint64_t u64) +{ + double d; + memcpy(&d, &u64, sizeof(uint64_t)); + return d; +} + +static inline float +UsefulBufUtil_CopyUint32ToFloat(uint32_t u32) +{ + float f; + memcpy(&f, &u32, sizeof(uint32_t)); + return f; +} +#endif /* USEFULBUF_DISABLE_ALL_FLOAT */ + +static inline void +UsefulOutBuf_Reset(UsefulOutBuf *pMe) +{ + pMe->data_len = 0; + pMe->err = 0; +} + +static inline size_t +UsefulOutBuf_GetEndPosition(UsefulOutBuf *pMe) +{ + return pMe->data_len; +} + +static inline int +UsefulOutBuf_AtStart(UsefulOutBuf *pMe) +{ + return 0 == pMe->data_len; +} + +static inline void +UsefulOutBuf_InsertData(UsefulOutBuf *pMe, const void *pBytes, size_t uLen, + size_t uPos) +{ + UsefulBufC Data = { pBytes, uLen }; + UsefulOutBuf_InsertUsefulBuf(pMe, Data, uPos); +} + +static inline void +UsefulOutBuf_InsertString(UsefulOutBuf *pMe, const char *szString, size_t uPos) +{ + UsefulBufC UBC; + UBC.ptr = szString; + UBC.len = strlen(szString); + + UsefulOutBuf_InsertUsefulBuf(pMe, UBC, uPos); +} + +static inline void +UsefulOutBuf_InsertByte(UsefulOutBuf *me, uint8_t byte, size_t uPos) +{ + UsefulOutBuf_InsertData(me, &byte, 1, uPos); +} + +static inline void +UsefulOutBuf_InsertUint16(UsefulOutBuf *me, uint16_t uInteger16, size_t uPos) +{ + /* See UsefulOutBuf_InsertUint64() for comments on this code */ + + const void *pBytes; + +#if defined(USEFULBUF_CONFIG_BIG_ENDIAN) + pBytes = &uInteger16; + +#elif defined(USEFULBUF_CONFIG_HTON) + uint16_t uTmp = htons(uInteger16); + pBytes = &uTmp; + +#elif defined(USEFULBUF_CONFIG_LITTLE_ENDIAN) && defined(USEFULBUF_CONFIG_BSWAP) + uint16_t uTmp = __builtin_bswap16(uInteger16); + pBytes = &uTmp; + +#else + uint8_t aTmp[2]; + + aTmp[0] = (uint8_t)((uInteger16 & 0xff00) >> 8); + aTmp[1] = (uint8_t)(uInteger16 & 0xff); + + pBytes = aTmp; +#endif + + UsefulOutBuf_InsertData(me, pBytes, 2, uPos); +} + +static inline void +UsefulOutBuf_InsertUint32(UsefulOutBuf *pMe, uint32_t uInteger32, size_t uPos) +{ + /* See UsefulOutBuf_InsertUint64() for comments on this code */ + + const void *pBytes; + +#if defined(USEFULBUF_CONFIG_BIG_ENDIAN) + pBytes = &uInteger32; + +#elif defined(USEFULBUF_CONFIG_HTON) + uint32_t uTmp = htonl(uInteger32); + pBytes = &uTmp; + +#elif defined(USEFULBUF_CONFIG_LITTLE_ENDIAN) && defined(USEFULBUF_CONFIG_BSWAP) + uint32_t uTmp = __builtin_bswap32(uInteger32); + + pBytes = &uTmp; + +#else + uint8_t aTmp[4]; + + aTmp[0] = (uint8_t)((uInteger32 & 0xff000000) >> 24); + aTmp[1] = (uint8_t)((uInteger32 & 0xff0000) >> 16); + aTmp[2] = (uint8_t)((uInteger32 & 0xff00) >> 8); + aTmp[3] = (uint8_t)(uInteger32 & 0xff); + + pBytes = aTmp; +#endif + + UsefulOutBuf_InsertData(pMe, pBytes, 4, uPos); +} + +static inline void +UsefulOutBuf_InsertUint64(UsefulOutBuf *pMe, uint64_t uInteger64, size_t uPos) +{ + const void *pBytes; + +#if defined(USEFULBUF_CONFIG_BIG_ENDIAN) + /* We have been told explicitly we are running on a big-endian + * machine. Network byte order is big endian, so just copy. There + * is no issue with alignment here because uInteger64 is always + * aligned (and it doesn't matter if pBytes is aligned). + */ + pBytes = &uInteger64; + +#elif defined(USEFULBUF_CONFIG_HTON) + /* Use system function to handle big- and little-endian. This works + * on both big- and little-endian machines, but hton() is not + * always available or in a standard place so it is not used by + * default. With some compilers and CPUs the code for this is very + * compact through use of a special swap instruction and on + * big-endian machines hton() will reduce to nothing. + */ + uint64_t uTmp = htonll(uInteger64); + + pBytes = &uTmp; + +#elif defined(USEFULBUF_CONFIG_LITTLE_ENDIAN) && defined(USEFULBUF_CONFIG_BSWAP) + /* Use built-in function for byte swapping. This usually compiles + * to an efficient special byte swap instruction. Unlike hton() it + * does not do this conditionally on the CPU endianness, so this + * code is also conditional on USEFULBUF_CONFIG_LITTLE_ENDIAN + */ + uint64_t uTmp = __builtin_bswap64(uInteger64); + + pBytes = &uTmp; + +#else + /* Default which works on every CPU with no dependency on anything + * from the CPU, compiler, libraries or OS. This always works, but + * it is usually a little larger and slower than hton(). + */ + uint8_t aTmp[8]; + + aTmp[0] = (uint8_t)((uInteger64 & 0xff00000000000000) >> 56); + aTmp[1] = (uint8_t)((uInteger64 & 0xff000000000000) >> 48); + aTmp[2] = (uint8_t)((uInteger64 & 0xff0000000000) >> 40); + aTmp[3] = (uint8_t)((uInteger64 & 0xff00000000) >> 32); + aTmp[4] = (uint8_t)((uInteger64 & 0xff000000) >> 24); + aTmp[5] = (uint8_t)((uInteger64 & 0xff0000) >> 16); + aTmp[6] = (uint8_t)((uInteger64 & 0xff00) >> 8); + aTmp[7] = (uint8_t)(uInteger64 & 0xff); + + pBytes = aTmp; +#endif + + /* Do the insert */ + UsefulOutBuf_InsertData(pMe, pBytes, sizeof(uint64_t), uPos); +} + +#ifndef USEFULBUF_DISABLE_ALL_FLOAT +static inline void +UsefulOutBuf_InsertFloat(UsefulOutBuf *pMe, float f, size_t uPos) +{ + UsefulOutBuf_InsertUint32(pMe, UsefulBufUtil_CopyFloatToUint32(f), + uPos); +} + +static inline void +UsefulOutBuf_InsertDouble(UsefulOutBuf *pMe, double d, size_t uPos) +{ + UsefulOutBuf_InsertUint64(pMe, UsefulBufUtil_CopyDoubleToUint64(d), + uPos); +} +#endif /* USEFULBUF_DISABLE_ALL_FLOAT */ + +static inline void +UsefulOutBuf_AppendUsefulBuf(UsefulOutBuf *pMe, UsefulBufC NewData) +{ + /* An append is just a insert at the end */ + UsefulOutBuf_InsertUsefulBuf(pMe, NewData, + UsefulOutBuf_GetEndPosition(pMe)); +} + +static inline void +UsefulOutBuf_AppendData(UsefulOutBuf *pMe, const void *pBytes, size_t uLen) +{ + UsefulBufC Data = { pBytes, uLen }; + UsefulOutBuf_AppendUsefulBuf(pMe, Data); +} + +static inline void +UsefulOutBuf_AppendString(UsefulOutBuf *pMe, const char *szString) +{ + UsefulBufC UBC; + UBC.ptr = szString; + UBC.len = strlen(szString); + + UsefulOutBuf_AppendUsefulBuf(pMe, UBC); +} + +static inline void +UsefulOutBuf_AppendByte(UsefulOutBuf *pMe, uint8_t byte) +{ + UsefulOutBuf_AppendData(pMe, &byte, 1); +} + +static inline void +UsefulOutBuf_AppendUint16(UsefulOutBuf *pMe, uint16_t uInteger16) +{ + UsefulOutBuf_InsertUint16(pMe, uInteger16, + UsefulOutBuf_GetEndPosition(pMe)); +} + +static inline void +UsefulOutBuf_AppendUint32(UsefulOutBuf *pMe, uint32_t uInteger32) +{ + UsefulOutBuf_InsertUint32(pMe, uInteger32, + UsefulOutBuf_GetEndPosition(pMe)); +} + +static inline void +UsefulOutBuf_AppendUint64(UsefulOutBuf *pMe, uint64_t uInteger64) +{ + UsefulOutBuf_InsertUint64(pMe, uInteger64, + UsefulOutBuf_GetEndPosition(pMe)); +} + +#ifndef USEFULBUF_DISABLE_ALL_FLOAT +static inline void +UsefulOutBuf_AppendFloat(UsefulOutBuf *pMe, float f) +{ + UsefulOutBuf_InsertFloat(pMe, f, UsefulOutBuf_GetEndPosition(pMe)); +} + +static inline void +UsefulOutBuf_AppendDouble(UsefulOutBuf *pMe, double d) +{ + UsefulOutBuf_InsertDouble(pMe, d, UsefulOutBuf_GetEndPosition(pMe)); +} +#endif /* USEFULBUF_DISABLE_ALL_FLOAT */ + +static inline int +UsefulOutBuf_GetError(UsefulOutBuf *pMe) +{ + return pMe->err; +} + +static inline size_t +UsefulOutBuf_RoomLeft(UsefulOutBuf *pMe) +{ + return pMe->UB.len - pMe->data_len; +} + +static inline int +UsefulOutBuf_WillItFit(UsefulOutBuf *pMe, size_t uLen) +{ + return uLen <= UsefulOutBuf_RoomLeft(pMe); +} + +static inline int +UsefulOutBuf_IsBufferNULL(UsefulOutBuf *pMe) +{ + return pMe->UB.ptr == NULL; +} + +static inline UsefulBuf +UsefulOutBuf_GetOutPlace(UsefulOutBuf *pUOutBuf) +{ + UsefulBuf R; + + R.len = UsefulOutBuf_RoomLeft(pUOutBuf); + if (R.len > 0 && pUOutBuf->UB.ptr != NULL) { + R.ptr = (uint8_t *)pUOutBuf->UB.ptr + pUOutBuf->data_len; + } else { + R.ptr = NULL; + } + + return R; +} + +#ifdef ENABLE_DECODE_ROUTINES + +static inline void +UsefulInputBuf_Init(UsefulInputBuf *pMe, UsefulBufC UB) +{ + pMe->cursor = 0; + pMe->err = 0; + pMe->magic = UIB_MAGIC; + pMe->UB = UB; +} + +static inline size_t +UsefulInputBuf_Tell(UsefulInputBuf *pMe) +{ + return pMe->cursor; +} + +static inline size_t +UsefulInputBuf_GetBufferLength(UsefulInputBuf *pMe) +{ + return pMe->UB.len; +} + +static inline void +UsefulInputBuf_Seek(UsefulInputBuf *pMe, size_t uPos) +{ + if (uPos > pMe->UB.len) { + pMe->err = 1; + } else { + pMe->cursor = uPos; + } +} + +static inline size_t +UsefulInputBuf_BytesUnconsumed(UsefulInputBuf *pMe) +{ + /* Code Reviewers: THIS FUNCTION DOES POINTER MATH */ + + /* Magic number is messed up. Either the structure got overwritten + * or was never initialized. + */ + if (pMe->magic != UIB_MAGIC) { + return 0; + } + + /* The cursor is off the end of the input buffer given. + * Presuming there are no bugs in this code, this should never happen. + * If it so, the struct was corrupted. The check is retained as + * as a defense in case there is a bug in this code or the struct is + * corrupted. + */ + if (pMe->cursor > pMe->UB.len) { + return 0; + } + + /* subtraction can't go negative because of check above */ + return pMe->UB.len - pMe->cursor; +} + +static inline int +UsefulInputBuf_BytesAvailable(UsefulInputBuf *pMe, size_t uLen) +{ + return UsefulInputBuf_BytesUnconsumed(pMe) >= uLen ? 1 : 0; +} + +static inline size_t +UsefulInputBuf_PointerToOffset(UsefulInputBuf *pUInBuf, const void *p) +{ + return UsefulBuf_PointerToOffset(pUInBuf->UB, p); +} + +static inline UsefulBufC +UsefulInputBuf_GetUsefulBuf(UsefulInputBuf *pMe, size_t uNum) +{ + const void *pResult = UsefulInputBuf_GetBytes(pMe, uNum); + if (!pResult) { + return NULLUsefulBufC; + } else { + UsefulBufC UBC; + UBC.ptr = pResult; + UBC.len = uNum; + return UBC; + } +} + +static inline uint8_t +UsefulInputBuf_GetByte(UsefulInputBuf *pMe) +{ + const void *pResult = UsefulInputBuf_GetBytes(pMe, sizeof(uint8_t)); + + /* The ternary operator is subject to integer promotion, because + * the operands are smaller than int, so cast back to uint8_t is + * needed to be completely explicit about types (for static + * analyzers). + */ + return (uint8_t)(pResult ? *(const uint8_t *)pResult : 0); +} + +static inline uint16_t +UsefulInputBuf_GetUint16(UsefulInputBuf *pMe) +{ + const uint8_t *pResult = + (const uint8_t *)UsefulInputBuf_GetBytes(pMe, sizeof(uint16_t)); + + if (!pResult) { + return 0; + } + + /* See UsefulInputBuf_GetUint64() for comments on this code */ +#if defined(USEFULBUF_CONFIG_BIG_ENDIAN) || defined(USEFULBUF_CONFIG_HTON) || \ + defined(USEFULBUF_CONFIG_BSWAP) + uint16_t uTmp; + memcpy(&uTmp, pResult, sizeof(uint16_t)); + +#if defined(USEFULBUF_CONFIG_BIG_ENDIAN) + return uTmp; + +#elif defined(USEFULBUF_CONFIG_HTON) + return ntohs(uTmp); + +#else + return __builtin_bswap16(uTmp); + +#endif + +#else + + /* The operations here are subject to integer promotion because the + * operands are smaller than int. They will be promoted to unsigned + * int for the shift and addition. The cast back to uint16_t is is + * needed to be completely explicit about types (for static + * analyzers). + */ + return (uint16_t)((pResult[0] << 8) + pResult[1]); + +#endif +} + +static inline uint32_t +UsefulInputBuf_GetUint32(UsefulInputBuf *pMe) +{ + const uint8_t *pResult = + (const uint8_t *)UsefulInputBuf_GetBytes(pMe, sizeof(uint32_t)); + + if (!pResult) { + return 0; + } + + /* See UsefulInputBuf_GetUint64() for comments on this code */ +#if defined(USEFULBUF_CONFIG_BIG_ENDIAN) || defined(USEFULBUF_CONFIG_HTON) || \ + defined(USEFULBUF_CONFIG_BSWAP) + uint32_t uTmp; + memcpy(&uTmp, pResult, sizeof(uint32_t)); + +#if defined(USEFULBUF_CONFIG_BIG_ENDIAN) + return uTmp; + +#elif defined(USEFULBUF_CONFIG_HTON) + return ntohl(uTmp); + +#else + return __builtin_bswap32(uTmp); + +#endif + +#else + return ((uint32_t)pResult[0] << 24) + ((uint32_t)pResult[1] << 16) + + ((uint32_t)pResult[2] << 8) + (uint32_t)pResult[3]; +#endif +} + +static inline uint64_t +UsefulInputBuf_GetUint64(UsefulInputBuf *pMe) +{ + const uint8_t *pResult = + (const uint8_t *)UsefulInputBuf_GetBytes(pMe, sizeof(uint64_t)); + + if (!pResult) { + return 0; + } + +#if defined(USEFULBUF_CONFIG_BIG_ENDIAN) || defined(USEFULBUF_CONFIG_HTON) || \ + defined(USEFULBUF_CONFIG_BSWAP) + /* pResult will probably not be aligned. This memcpy() moves the + * bytes into a temp variable safely for CPUs that can or can't do + * unaligned memory access. Many compilers will optimize the + * memcpy() into a simple move instruction. + */ + uint64_t uTmp; + memcpy(&uTmp, pResult, sizeof(uint64_t)); + +#if defined(USEFULBUF_CONFIG_BIG_ENDIAN) + /* We have been told expliclity this is a big-endian CPU. Since + * network byte order is big-endian, there is nothing to do. + */ + + return uTmp; + +#elif defined(USEFULBUF_CONFIG_HTON) + /* We have been told to use ntoh(), the system function to handle + * big- and little-endian. This works on both big- and + * little-endian machines, but ntoh() is not always available or in + * a standard place so it is not used by default. On some CPUs the + * code for this is very compact through use of a special swap + * instruction. + */ + + return ntohll(uTmp); + +#else + /* Little-endian (since it is not USEFULBUF_CONFIG_BIG_ENDIAN) and + * USEFULBUF_CONFIG_BSWAP (since it is not USEFULBUF_CONFIG_HTON). + * __builtin_bswap64() and friends are not conditional on CPU + * endianness so this must only be used on little-endian machines. + */ + + return __builtin_bswap64(uTmp); + +#endif + +#else + /* This is the default code that works on every CPU and every + * endianness with no dependency on ntoh(). This works on CPUs + * that either allow or do not allow unaligned access. It will + * always work, but usually is a little less efficient than ntoh(). + */ + + return ((uint64_t)pResult[0] << 56) + ((uint64_t)pResult[1] << 48) + + ((uint64_t)pResult[2] << 40) + ((uint64_t)pResult[3] << 32) + + ((uint64_t)pResult[4] << 24) + ((uint64_t)pResult[5] << 16) + + ((uint64_t)pResult[6] << 8) + (uint64_t)pResult[7]; +#endif +} + +#ifndef USEFULBUF_DISABLE_ALL_FLOAT +static inline float +UsefulInputBuf_GetFloat(UsefulInputBuf *pMe) +{ + uint32_t uResult = UsefulInputBuf_GetUint32(pMe); + + return uResult ? UsefulBufUtil_CopyUint32ToFloat(uResult) : 0; +} + +static inline double +UsefulInputBuf_GetDouble(UsefulInputBuf *pMe) +{ + uint64_t uResult = UsefulInputBuf_GetUint64(pMe); + + return uResult ? UsefulBufUtil_CopyUint64ToDouble(uResult) : 0; +} +#endif /* USEFULBUF_DISABLE_ALL_FLOAT */ + +static inline int +UsefulInputBuf_GetError(UsefulInputBuf *pMe) +{ + return pMe->err; +} + +static inline void +UsefulInputBuf_SetBufferLength(UsefulInputBuf *pMe, size_t uNewLen) +{ + pMe->UB.len = uNewLen; +} +#endif /* ENABLE_DECODE_ROUTINES */ + +#ifdef __cplusplus +} +#endif + +#endif /* UsefulBuf_h_ */ diff --git a/hyp/interfaces/qcbor/include/qcbor/qcbor_common.h b/hyp/interfaces/qcbor/include/qcbor/qcbor_common.h new file mode 100644 index 0000000..7564de2 --- /dev/null +++ b/hyp/interfaces/qcbor/include/qcbor/qcbor_common.h @@ -0,0 +1,575 @@ +/*============================================================================== + Copyright (c) 2016-2018, The Linux Foundation. + Copyright (c) 2018-2022, Laurence Lundblade. + Copyright (c) 2021, Arm Limited. + All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of The Linux Foundation nor the names of its + contributors, nor the name "Laurence Lundblade" may be used to + endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + =============================================================================*/ + +#ifndef qcbor_common_h +#define qcbor_common_h + +/** + @file qcbor_common.h + + This define indicates a version of QCBOR that supports spiffy decode, + the decode functions found in qcbor_spiffy_decode.h. + + Versions of QCBOR that support spiffy decode are backwards compatible + with previous versions, but there are a few minor exceptions such as + some aspects of tag handling that are different. This define can be + used handle these variances. +*/ +// #define QCBOR_SPIFFY_DECODE +#define QCBOR_DISABLE_EXP_AND_MANTISSA + +/* It was originally defined as QCBOR_CONFIG_DISABLE_EXP_AND_MANTISSA, + * but this is incosistent with all the other QCBOR_DISABLE_ + * #defines, so the name was changed and this was added for backwards + * compatibility + */ +#ifdef QCBOR_CONFIG_DISABLE_EXP_AND_MANTISSA +#define QCBOR_DISABLE_EXP_AND_MANTISSA +#endif + +/* If USEFULBUF_DISABLE_ALL_FLOATis defined then define + * QCBOR_DISABLE_FLOAT_HW_USE and QCBOR_DISABLE_PREFERRED_FLOAT + */ +#define USEFULBUF_DISABLE_ALL_FLOAT +#ifdef USEFULBUF_DISABLE_ALL_FLOAT +#ifndef QCBOR_DISABLE_FLOAT_HW_USE +#define QCBOR_DISABLE_FLOAT_HW_USE +#endif /* QCBOR_DISABLE_FLOAT_HW_USE */ +#ifndef QCBOR_DISABLE_PREFERRED_FLOAT +#define QCBOR_DISABLE_PREFERRED_FLOAT +#endif /* QCBOR_DISABLE_PREFERRED_FLOAT */ +#endif /* USEFULBUF_DISABLE_ALL_FLOAT */ + +/* Standard CBOR Major type for positive integers of various lengths */ +#define CBOR_MAJOR_TYPE_POSITIVE_INT 0 + +/* Standard CBOR Major type for negative integer of various lengths */ +#define CBOR_MAJOR_TYPE_NEGATIVE_INT 1 + +/* Standard CBOR Major type for an array of arbitrary 8-bit bytes. */ +#define CBOR_MAJOR_TYPE_BYTE_STRING 2 + +/* Standard CBOR Major type for a UTF-8 string. Note this is true 8-bit UTF8 + with no encoding and no NULL termination */ +#define CBOR_MAJOR_TYPE_TEXT_STRING 3 + +/* Standard CBOR Major type for an ordered array of other CBOR data items */ +#define CBOR_MAJOR_TYPE_ARRAY 4 + +/* Standard CBOR Major type for CBOR MAP. Maps an array of pairs. The + first item in the pair is the "label" (key, name or identfier) and the second + item is the value. */ +#define CBOR_MAJOR_TYPE_MAP 5 + +/* Standard CBOR major type for a tag number. This creates a CBOR "tag" that + * is the tag number and a data item that follows as the tag content. + * + * Note that this was called an optional tag in RFC 7049, but there's + * not really anything optional about it. It was misleading. It is + * renamed in RFC 8949. + */ +#define CBOR_MAJOR_TYPE_TAG 6 +#define CBOR_MAJOR_TYPE_OPTIONAL 6 + +/* Standard CBOR extra simple types like floats and the values true and false */ +#define CBOR_MAJOR_TYPE_SIMPLE 7 + +/* + These are special values for the AdditionalInfo bits that are part of + the first byte. Mostly they encode the length of the data item. + */ +#define LEN_IS_ONE_BYTE 24 +#define LEN_IS_TWO_BYTES 25 +#define LEN_IS_FOUR_BYTES 26 +#define LEN_IS_EIGHT_BYTES 27 +#define ADDINFO_RESERVED1 28 +#define ADDINFO_RESERVED2 29 +#define ADDINFO_RESERVED3 30 +#define LEN_IS_INDEFINITE 31 + +/* + 24 is a special number for CBOR. Integers and lengths + less than it are encoded in the same byte as the major type. + */ +#define CBOR_TWENTY_FOUR 24 + +/* + Tags that are used with CBOR_MAJOR_TYPE_OPTIONAL. These + are types defined in RFC 8949 and some additional ones + in the IANA CBOR tags registry. + */ +/** See QCBOREncode_AddDateString(). */ +#define CBOR_TAG_DATE_STRING 0 +/** See QCBOREncode_AddDateEpoch(). */ +#define CBOR_TAG_DATE_EPOCH 1 +/** See QCBOREncode_AddPositiveBignum(). */ +#define CBOR_TAG_POS_BIGNUM 2 +/** See QCBOREncode_AddNegativeBignum(). */ +#define CBOR_TAG_NEG_BIGNUM 3 +/** CBOR tag for a two-element array representing a fraction with a + mantissa and base-10 scaling factor. See QCBOREncode_AddDecimalFraction() + and @ref expAndMantissa. + */ +#define CBOR_TAG_DECIMAL_FRACTION 4 +/** CBOR tag for a two-element array representing a fraction with a + mantissa and base-2 scaling factor. See QCBOREncode_AddBigFloat() + and @ref expAndMantissa. */ +#define CBOR_TAG_BIGFLOAT 5 +/** Not Decoded by QCBOR. Tag for COSE format encryption with no recipient + identification. See [RFC 8152, COSE] + (https://tools.ietf.org/html/rfc8152). No API is provided for this + tag. */ +#define CBOR_TAG_COSE_ENCRYPT0 16 +#define CBOR_TAG_COSE_ENCRYPTO 16 +/** Not Decoded by QCBOR. Tag for COSE format MAC'd data with no recipient + identification. See [RFC 8152, COSE] + (https://tools.ietf.org/html/rfc8152). No API is provided for this + tag.*/ +#define CBOR_TAG_COSE_MAC0 17 +/** Tag for COSE format single signature signing. No API is provided + for this tag. See [RFC 8152, COSE] + (https://tools.ietf.org/html/rfc8152). */ +#define CBOR_TAG_COSE_SIGN1 18 +/** A hint that the following byte string should be encoded in + Base64URL when converting to JSON or similar text-based + representations. Call @c + QCBOREncode_AddTag(pCtx,CBOR_TAG_ENC_AS_B64URL) before the call to + QCBOREncode_AddBytes(). */ +#define CBOR_TAG_ENC_AS_B64URL 21 +/** A hint that the following byte string should be encoded in Base64 + when converting to JSON or similar text-based + representations. Call @c + QCBOREncode_AddTag(pCtx,CBOR_TAG_ENC_AS_B64) before the call to + QCBOREncode_AddBytes(). */ +#define CBOR_TAG_ENC_AS_B64 22 +/** A hint that the following byte string should be encoded in base-16 + format per [RFC 4648] (https://tools.ietf.org/html/rfc4648) when + converting to JSON or similar text-based + representations. Essentially, Base-16 encoding is the standard + case- insensitive hex encoding and may be referred to as + "hex". Call @c QCBOREncode_AddTag(pCtx,CBOR_TAG_ENC_AS_B16) before + the call to QCBOREncode_AddBytes(). */ +#define CBOR_TAG_ENC_AS_B16 23 +/** See QCBORDecode_EnterBstrWrapped()). */ +#define CBOR_TAG_CBOR 24 +/** See QCBOREncode_AddURI(). */ +#define CBOR_TAG_URI 32 +/** See QCBOREncode_AddB64URLText(). */ +#define CBOR_TAG_B64URL 33 +/** See QCBOREncode_AddB64Text(). */ +#define CBOR_TAG_B64 34 +/** See QCBOREncode_AddRegex(). */ +#define CBOR_TAG_REGEX 35 +/** See QCBOREncode_AddMIMEData(). */ +#define CBOR_TAG_MIME 36 +/** See QCBOREncode_AddBinaryUUID(). */ +#define CBOR_TAG_BIN_UUID 37 +/** The data is a CBOR Web Token per [RFC 8392] + (https://tools.ietf.org/html/rfc8932). No API is provided for this + tag. */ +#define CBOR_TAG_CWT 61 +/** Tag for COSE format encryption. See [RFC 8152, COSE] + (https://tools.ietf.org/html/rfc8152). No API is provided for this + tag. */ +#define CBOR_TAG_CBOR_SEQUENCE 63 +/** Not Decoded by QCBOR. Tag for COSE format encryption with recipient + identification. See [RFC 8152, COSE] + (https://tools.ietf.org/html/rfc8152). No API is provided for this + tag. */ +#define CBOR_TAG_COSE_ENCRYPT 96 +#define CBOR_TAG_ENCRYPT 96 +/** Not Decoded by QCBOR. Tag for COSE format MAC. See [RFC 8152, COSE] + (https://tools.ietf.org/html/rfc8152). No API is provided for this + tag. */ +#define CBOR_TAG_COSE_MAC 97 +#define CBOR_TAG_MAC 97 +/** Not Decoded by QCBOR. Tag for COSE format signed data. See [RFC 8152, COSE] + (https://tools.ietf.org/html/rfc8152). No API is provided for this + tag. */ +#define CBOR_TAG_COSE_SIGN 98 +#define CBOR_TAG_SIGN 98 +/** Tag for date counted by days from Jan 1 1970 per [RFC 8943] + (https://tools.ietf.org/html/rfc8943). See + QCBOREncode_AddTDaysEpoch(). */ +#define CBOR_TAG_DAYS_EPOCH 100 +/** Not Decoded by QCBOR. World geographic coordinates. See ISO 6709, [RFC 5870] + (https://tools.ietf.org/html/rfc5870) and WGS-84. No API is + provided for this tag. */ +#define CBOR_TAG_GEO_COORD 103 +/** Binary MIME.*/ +#define CBOR_TAG_BINARY_MIME 257 +/** Tag for date string without time or time zone per [RFC 8943] + (https://tools.ietf.org/html/rfc8943). See + QCBOREncode_AddTDaysString(). */ +#define CBOR_TAG_DAYS_STRING 1004 +/** The magic number, self-described CBOR. No API is provided for this + tag. */ +#define CBOR_TAG_CBOR_MAGIC 55799 + +/** The 16-bit invalid tag from the CBOR tags registry */ +#define CBOR_TAG_INVALID16 0xffff +/** The 32-bit invalid tag from the CBOR tags registry */ +#define CBOR_TAG_INVALID32 0xffffffff +/** The 64-bit invalid tag from the CBOR tags registry */ +#define CBOR_TAG_INVALID64 0xffffffffffffffff + +/* + Values for the 5 bits for items of major type 7 + */ +#define CBOR_SIMPLEV_FALSE 20 +#define CBOR_SIMPLEV_TRUE 21 +#define CBOR_SIMPLEV_NULL 22 +#define CBOR_SIMPLEV_UNDEF 23 +#define CBOR_SIMPLEV_ONEBYTE 24 +#define HALF_PREC_FLOAT 25 +#define SINGLE_PREC_FLOAT 26 +#define DOUBLE_PREC_FLOAT 27 +#define CBOR_SIMPLE_BREAK 31 +#define CBOR_SIMPLEV_RESERVED_START CBOR_SIMPLEV_ONEBYTE +#define CBOR_SIMPLEV_RESERVED_END CBOR_SIMPLE_BREAK + +/** + * Error codes returned by QCBOR Encoder and Decoder. + * + * The errors are grouped to keep the code size of + * QCBORDecode_IsNotWellFormedError() and + * QCBORDecode_IsUnrecoverableError() minimal. + * + * 1..19: Encode errors + * 20..: Decode errors + * 20-39: QCBORDecode_IsNotWellFormedError() + * 30..59: QCBORDecode_IsUnrecoverableError() + * 60..: Other decode errors + * + * Error renumbering may occur in the future when new error codes are + * added for new QCBOR features. + */ +typedef enum { + /** The encode or decode completely correctly. */ + QCBOR_SUCCESS = 0, + + /** The buffer provided for the encoded output when doing encoding + was too small and the encoded output will not fit. */ + QCBOR_ERR_BUFFER_TOO_SMALL = 1, + + /** During encoding, an attempt to create simple value between 24 + and 31. */ + QCBOR_ERR_ENCODE_UNSUPPORTED = 2, + + /** During encoding, the length of the encoded CBOR exceeded + QCBOR_MAX_ARRAY_OFFSET, which is slightly less than + @c UINT32_MAX. */ + QCBOR_ERR_BUFFER_TOO_LARGE = 3, + + /** During encoding, the array or map nesting was deeper than this + implementation can handle. Note that in the interest of code + size and memory use, this implementation has a hard limit on + array nesting. The limit is defined as the constant @ref + QCBOR_MAX_ARRAY_NESTING. */ + QCBOR_ERR_ARRAY_NESTING_TOO_DEEP = 4, + + /** During encoding, @c QCBOREncode_CloseXxx() called with a + different type than is currently open. */ + QCBOR_ERR_CLOSE_MISMATCH = 5, + + /** During encoding, the array or map had too many items in it. + This limit @ref QCBOR_MAX_ITEMS_IN_ARRAY, typically 65,535. */ + QCBOR_ERR_ARRAY_TOO_LONG = 6, + + /** During encoding, more arrays or maps were closed than + opened. This is a coding error on the part of the caller of the + encoder. */ + QCBOR_ERR_TOO_MANY_CLOSES = 7, + + /** During encoding the number of array or map opens was not + matched by the number of closes. Also occurs with opened + byte strings that are not closed. */ + QCBOR_ERR_ARRAY_OR_MAP_STILL_OPEN = 8, + + /** During encode, opening a byte string while a byte string is open + is not allowed. . */ + QCBOR_ERR_OPEN_BYTE_STRING = 9, + + /** Trying to cancel a byte string wrapping after items have been + added to it. */ + QCBOR_ERR_CANNOT_CANCEL = 10, + +#define QCBOR_START_OF_NOT_WELL_FORMED_ERRORS 20 + + /** During decoding, the CBOR is not well-formed because a simple + value between 0 and 31 is encoded in a two-byte integer rather + than one. */ + QCBOR_ERR_BAD_TYPE_7 = 20, + + /** During decoding, returned by QCBORDecode_Finish() if all the + inputs bytes have not been consumed. This is considered not + well-formed. */ + QCBOR_ERR_EXTRA_BYTES = 21, + + /** During decoding, some CBOR construct was encountered that this + decoder doesn't support, primarily this is the reserved + additional info values, 28 through 30. The CBOR is not + well-formed.*/ + QCBOR_ERR_UNSUPPORTED = 22, + + /** During decoding, the an array or map was not fully consumed. + Returned by QCBORDecode_Finish(). The CBOR is not + well-formed. */ + QCBOR_ERR_ARRAY_OR_MAP_UNCONSUMED = 23, + + /** During decoding, an integer type is encoded with a bad length + (that of an indefinite length string). The CBOR is not-well + formed. */ + QCBOR_ERR_BAD_INT = 24, + +#define QCBOR_START_OF_UNRECOVERABLE_DECODE_ERRORS 30 + + /** During decoding, one of the chunks in an indefinite-length + string is not of the type of the start of the string. The CBOR + is not well-formed. This error makes no further decoding + possible. */ + QCBOR_ERR_INDEFINITE_STRING_CHUNK = 30, + + /** During decoding, hit the end of the given data to decode. For + example, a byte string of 100 bytes was expected, but the end + of the input was hit before finding those 100 bytes. Corrupted + CBOR input will often result in this error. See also @ref + QCBOR_ERR_NO_MORE_ITEMS. The CBOR is not well-formed. This + error makes no further decoding possible. */ + QCBOR_ERR_HIT_END = 31, + + /** During decoding, a break occurred outside an indefinite-length + item. The CBOR is not well-formed. This error makes no further + decoding possible. */ + QCBOR_ERR_BAD_BREAK = 32, + +#define QCBOR_END_OF_NOT_WELL_FORMED_ERRORS 39 + + /** During decoding, the input is too large. It is greater than + QCBOR_MAX_DECODE_INPUT_SIZE. This is an implementation limit. + This error makes no further decoding possible. */ + QCBOR_ERR_INPUT_TOO_LARGE = 40, + + /** During decoding, the array or map nesting was deeper than this + implementation can handle. Note that in the interest of code + size and memory use, this implementation has a hard limit on + array nesting. The limit is defined as the constant @ref + QCBOR_MAX_ARRAY_NESTING. This error makes no further decoding + possible. */ + QCBOR_ERR_ARRAY_DECODE_NESTING_TOO_DEEP = 41, + + /** During decoding, the array or map had too many items in it. + This limit @ref QCBOR_MAX_ITEMS_IN_ARRAY, typically 65,534, + UINT16_MAX - 1. This error makes no further decoding + possible. */ + QCBOR_ERR_ARRAY_DECODE_TOO_LONG = 42, + + /** When decoding, a string's size is greater than what a size_t + can hold less 4. In all but some very strange situations this + is because of corrupt input CBOR and should be treated as + such. The strange situation is a CPU with a very small size_t + (e.g., a 16-bit CPU) and a large string (e.g., > 65KB). This + error makes no further decoding possible. */ + QCBOR_ERR_STRING_TOO_LONG = 43, + + /** Something is wrong with a decimal fraction or bigfloat such as + it not consisting of an array with two integers. This error + makes no further decoding possible. */ + QCBOR_ERR_BAD_EXP_AND_MANTISSA = 44, + + /** Unable to decode an indefinite-length string because no string + allocator was configured. See QCBORDecode_SetMemPool() or + QCBORDecode_SetUpAllocator(). This error makes no further + decoding possible. */ + QCBOR_ERR_NO_STRING_ALLOCATOR = 45, + + /** Error allocating space for a string, usually for an + indefinite-length string. This error makes no further decoding + possible. */ + QCBOR_ERR_STRING_ALLOCATE = 46, + + /** During decoding, the type of the label for a map entry is not + one that can be handled in the current decoding mode. Typically + this is because a label is not an intger or a string. This is + an implemation limit. */ + QCBOR_ERR_MAP_LABEL_TYPE = 47, + + /** When the built-in tag decoding encounters an unexpected type, + this error is returned. This error is unrecoverable because the + built-in tag decoding doesn't try to consume the unexpected + type. In previous versions of QCBOR this was considered a + recoverable error hence QCBOR_ERR_BAD_TAG_CONTENT. Going back + further, RFC 7049 use the name "optional tags". That name is no + longer used because "optional" was causing confusion. See + also @ref QCBOR_ERR_RECOVERABLE_BAD_TAG_CONTENT. */ + QCBOR_ERR_UNRECOVERABLE_TAG_CONTENT = 48, + QCBOR_ERR_BAD_TAG_CONTENT = 48, + QCBOR_ERR_BAD_OPT_TAG = 48, + + /** Indefinite length string handling is disabled and there is an + indefinite length string in the input CBOR. */ + QCBOR_ERR_INDEF_LEN_STRINGS_DISABLED = 49, + + /** Indefinite length arrays and maps handling are disabled and there is + an indefinite length map or array in the input CBOR. */ + QCBOR_ERR_INDEF_LEN_ARRAYS_DISABLED = 50, + +#define QCBOR_END_OF_UNRECOVERABLE_DECODE_ERRORS 59 + + /** More than @ref QCBOR_MAX_TAGS_PER_ITEM tags encountered for a + CBOR ITEM. @ref QCBOR_MAX_TAGS_PER_ITEM is a limit of this + implementation. During decoding, too many tags in the + caller-configured tag list, or not enough space in @ref + QCBORTagListOut. This error makes no further decoding + possible. */ + QCBOR_ERR_TOO_MANY_TAGS = 60, + + /** When decoding for a specific type, the type was not was + expected. */ + QCBOR_ERR_UNEXPECTED_TYPE = 61, + + /** Duplicate label in map detected. */ + QCBOR_ERR_DUPLICATE_LABEL = 62, + + /** During decoding, the buffer given to QCBORDecode_SetMemPool() + is either too small, smaller than + QCBOR_DECODE_MIN_MEM_POOL_SIZE or too large, larger than + UINT32_MAX. */ + QCBOR_ERR_MEM_POOL_SIZE = 63, + + /** During decoding, an integer smaller than INT64_MIN was received + (CBOR can represent integers smaller than INT64_MIN, but C + cannot). */ + QCBOR_ERR_INT_OVERFLOW = 64, + + /** During decoding, a date greater than +- 292 billion years from + Jan 1 1970 encountered during parsing. This is an + implementation limit. */ + QCBOR_ERR_DATE_OVERFLOW = 65, + + /** During decoding, @c QCBORDecode_ExitXxx() was called for a + different type than @c QCBORDecode_EnterXxx(). */ + QCBOR_ERR_EXIT_MISMATCH = 66, + + /** All well-formed data items have been consumed and there are no + more. If parsing a CBOR stream this indicates the non-error end + of the stream. If not parsing a CBOR stream / sequence, this + probably indicates that some data items expected are not + present. See also @ref QCBOR_ERR_HIT_END. */ + QCBOR_ERR_NO_MORE_ITEMS = 67, + + /** When finding an item by label, an item with the requested label + was not found. */ + QCBOR_ERR_LABEL_NOT_FOUND = 68, + + /** Number conversion failed because of sign. For example a + negative int64_t can't be converted to a uint64_t */ + QCBOR_ERR_NUMBER_SIGN_CONVERSION = 69, + + /** When converting a decoded number, the value is too large or to + small for the conversion target */ + QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW = 70, + + /** Trying to get an item by label when a map has not been + entered. */ + QCBOR_ERR_MAP_NOT_ENTERED = 71, + + /** A @ref QCBORItemCallback callback indicates processing should not + continue for some non-CBOR reason. */ + QCBOR_ERR_CALLBACK_FAIL = 72, + + /** This error code is deprecated. Instead, + @ref QCBOR_ERR_HALF_PRECISION_DISABLED, + @ref QCBOR_ERR_HW_FLOAT_DISABLED or @ref + QCBOR_ERR_ALL_FLOAT_DISABLED is returned depending on the specific + floating-point functionality that is disabled and the type of + floating-point input. */ + QCBOR_ERR_FLOAT_DATE_DISABLED = 73, + + /** Support for half-precision float decoding is disabled. */ + QCBOR_ERR_HALF_PRECISION_DISABLED = 74, + + /** Use of floating-point HW is disabled. This affects all type + conversions to and from double and float types. */ + QCBOR_ERR_HW_FLOAT_DISABLED = 75, + + /** Unable to complete operation because a floating-point value + that is a NaN (not a number), that is too large, too small, + infinity or -infinity was encountered in encoded CBOR. Usually + this because conversion of the float-point value was being + attempted. */ + QCBOR_ERR_FLOAT_EXCEPTION = 76, + + /** Floating point support is completely turned off, encoding/decoding + floating point numbers is not possible. */ + QCBOR_ERR_ALL_FLOAT_DISABLED = 77, + + /** Like @ref QCBOR_ERR_UNRECOVERABLE_TAG_CONTENT, but recoverable. + If an implementation decodes a tag and can and does consume the + whole tag contents when it is not the correct tag content, this + error can be returned. None of the built-in tag decoders do + this (to save object code). */ + QCBOR_ERR_RECOVERABLE_BAD_TAG_CONTENT = 78 + + /* This is stored in uint8_t; never add values > 255 */ +} QCBORError; + +/* Function for getting an error string from an error code */ +const char * +qcbor_err_to_str(QCBORError err); + +/** + The maximum nesting of arrays and maps when encoding or decoding. The + error @ref QCBOR_ERR_ARRAY_NESTING_TOO_DEEP will be returned on + encoding or QCBOR_ERR_ARRAY_DECODE_NESTING_TOO_DEEP on decoding if it is + exceeded. + */ +#define QCBOR_MAX_ARRAY_NESTING QCBOR_MAX_ARRAY_NESTING1 + +/** + * The maximum number of items in a single array or map when encoding of + * decoding. + */ +/* -1 because the value UINT16_MAX is used to track indefinite-length arrays */ +#define QCBOR_MAX_ITEMS_IN_ARRAY (UINT16_MAX - 1) + +/** + This is deprecated. See QCBORDecode_GetNthTag() and + QCBORDecode_GetNthTagOfLast() for tag handling. + + The maximum number of tags that can be in @ref QCBORTagListIn and passed to + QCBORDecode_SetCallerConfiguredTagList() + */ +#define QCBOR_MAX_CUSTOM_TAGS 16 + +#endif /* qcbor_common_h */ diff --git a/hyp/interfaces/qcbor/include/qcbor/qcbor_encode.h b/hyp/interfaces/qcbor/include/qcbor/qcbor_encode.h new file mode 100644 index 0000000..7e1e25b --- /dev/null +++ b/hyp/interfaces/qcbor/include/qcbor/qcbor_encode.h @@ -0,0 +1,3479 @@ +/*============================================================================== + Copyright (c) 2016-2018, The Linux Foundation. + Copyright (c) 2018-2021, Laurence Lundblade. + Copyright (c) 2021, Arm Limited. + All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of The Linux Foundation nor the names of its + contributors, nor the name "Laurence Lundblade" may be used to + endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + =============================================================================*/ + +#ifndef qcbor_encode_h +#define qcbor_encode_h + +#include + +#include "qcbor/qcbor_common.h" +#include "qcbor/qcbor_private.h" + +#ifdef __cplusplus +extern "C" { +#if 0 +} // Keep editor indention formatting happy +#endif +#endif + +/** + @file qcbor_encode.h + + @anchor Overview + + # QCBOR Overview + + This implements CBOR -- Concise Binary Object Representation as + defined in [RFC 8949] (https://tools.ietf.org/html/rfc8949). More + information is at http://cbor.io. This is a near-complete implementation of + the specification. [RFC 8742] (https://tools.ietf.org/html/rfc8742) CBOR + Sequences is also supported. Limitations are listed further down. + + See @ref Encoding for general discussion on encoding, + @ref BasicDecode for general discussion on the basic decode features + and @ref SpiffyDecode for general discussion on the easier-to-use + decoder functions. + + CBOR is intentionally designed to be translatable to JSON, but not + all CBOR can convert to JSON. See RFC 8949 for more info on how to + construct CBOR that is the most JSON friendly. + + The memory model for encoding and decoding is that encoded CBOR must + be in a contiguous buffer in memory. During encoding the caller must + supply an output buffer and if the encoding would go off the end of + the buffer an error is returned. During decoding the caller supplies + the encoded CBOR in a contiguous buffer and the decoder returns + pointers and lengths into that buffer for strings. + + This implementation does not require malloc. All data structures + passed in/out of the APIs can fit on the stack. + + Decoding of indefinite-length strings is a special case that requires + a "string allocator" to allocate memory into which the segments of + the string are coalesced. Without this, decoding will error out if an + indefinite-length string is encountered (indefinite-length maps and + arrays do not require the string allocator). A simple string + allocator called MemPool is built-in and will work if supplied with a + block of memory to allocate. The string allocator can optionally use + malloc() or some other custom scheme. + + Here are some terms and definitions: + + - "Item", "Data Item": An integer or string or such. The basic "thing" that + CBOR is about. An array is an item itself that contains some items. + + - "Array": An ordered sequence of items, the same as JSON. + + - "Map": A collection of label/value pairs. Each pair is a data + item. A JSON "object" is the same as a CBOR "map". + + - "Label": The data item in a pair in a map that names or identifies + the pair, not the value. This implementation refers to it as a + "label". JSON refers to it as the "name". The CBOR RFC refers to it + this as a "key". This implementation chooses label instead because + key is too easily confused with a cryptographic key. The COSE + standard, which uses CBOR, has also chosen to use the term "label" + rather than "key" for this same reason. + + - "Key": See "Label" above. + + - "Tag": A data item that is an explicitly labeled new data + type made up of the tagging integer and the tag content. + See @ref Tags-Overview and @ref Tag-Usage. + + - "Initial Byte": The first byte of an encoded item. Encoding and + decoding of this byte is taken care of by the implementation. + + - "Additional Info": In addition to the major type, all data items + have some other info. This is usually the length of the data but can + be several other things. Encoding and decoding of this is taken care + of by the implementation. + + CBOR has two mechanisms for tagging and labeling the data values like + integers and strings. For example, an integer that represents + someone's birthday in epoch seconds since Jan 1, 1970 could be + encoded like this: + + - First it is CBOR_MAJOR_TYPE_POSITIVE_INT (@ref QCBOR_TYPE_INT64), + the primitive positive integer. + + - Next it has a "tag" @ref CBOR_TAG_DATE_EPOCH indicating the integer + represents a date in the form of the number of seconds since Jan 1, + 1970. + + - Last it has a string "label" like "BirthDate" indicating the + meaning of the data. + + The encoded binary looks like this: + + a1 # Map of 1 item + 69 # Indicates text string of 9 bytes + 426972746844617465 # The text "BirthDate" + c1 # Tags next integer as epoch date + 1a # Indicates a 4-byte integer + 580d4172 # unsigned integer date 1477263730 + + Implementors using this API will primarily work with + labels. Generally, tags are only needed for making up new data + types. This implementation covers most of the data types defined in + the RFC using tags. It also, allows for the use of custom tags if + necessary. + + This implementation explicitly supports labels that are text strings + and integers. Text strings translate nicely into JSON objects and are + very readable. Integer labels are much less readable but can be very + compact. If they are in the range of 0 to 23, they take up only one + byte. + + CBOR allows a label to be any type of data including an array or a + map. It is possible to use this API to construct and parse such + labels, but it is not explicitly supported. + + @anchor Encoding + + ## Encoding + + A common encoding usage mode is to invoke the encoding twice. First + with the output buffer as @ref SizeCalculateUsefulBuf to compute the + length of the needed output buffer. The correct sized output buffer + is allocated. The encoder is invoked a second time with the allocated + output buffer. + + The double invocation is not required if the maximum output buffer + size can be predicted. This is usually possible for simple CBOR + structures. + + If a buffer too small to hold the encoded output is given, the error + @ref QCBOR_ERR_BUFFER_TOO_SMALL will be returned. Data will never be + written off the end of the output buffer no matter which functions + here are called or what parameters are passed to them. + + The encoding error handling is simple. The only possible errors are + trying to encode structures that are too large or too complex. There + are no internal malloc calls so there will be no failures for out of + memory. The error state is tracked internally, so there is no need + to check for errors when encoding. Only the return code from + QCBOREncode_Finish() need be checked as once an error happens, the + encoder goes into an error state and calls to it to add more data + will do nothing. An error check is not needed after every data item + is added. + + Encoding generally proceeds by calling QCBOREncode_Init(), calling + lots of @c QCBOREncode_AddXxx() functions and calling + QCBOREncode_Finish(). There are many @c QCBOREncode_AddXxx() + functions for various data types. The input buffers need only to be + valid during the @c QCBOREncode_AddXxx() calls as the data is copied + into the output buffer. + + There are three `Add` functions for each data type. The first / main + one for the type is for adding the data item to an array. The second + one's name ends in `ToMap`, is used for adding data items to maps and + takes a string argument that is its label in the map. The third one + ends in `ToMapN`, is also used for adding data items to maps, and + takes an integer argument that is its label in the map. + + The simplest aggregate type is an array, which is a simple ordered + set of items without labels the same as JSON arrays. Call + QCBOREncode_OpenArray() to open a new array, then various @c + QCBOREncode_AddXxx() functions to put items in the array and then + QCBOREncode_CloseArray(). Nesting to the limit @ref + QCBOR_MAX_ARRAY_NESTING is allowed. All opens must be matched by + closes or an encoding error will be returned. + + The other aggregate type is a map which does use labels. The `Add` + functions that end in `ToMap` and `ToMapN` are convenient ways to add + labeled data items to a map. You can also call any type of `Add` + function once to add a label of any time and then call any type of + `Add` again to add its value. + + Note that when you nest arrays or maps in a map, the nested array or + map has a label. + + Many CBOR-based protocols start with an array or map. This makes them + self-delimiting. No external length or end marker is needed to know + the end. It is also possible not start this way, in which case this + it is usually called a CBOR sequence which is described in + [RFC 8742] (https://tools.ietf.org/html/rfc8742). This encoder supports + either just by whether the first item added is an array, map or other. + + If QCBOR is compiled with QCBOR_DISABLE_ENCODE_USAGE_GUARDS defined, + the errors QCBOR_ERR_CLOSE_MISMATCH, QCBOR_ERR_ARRAY_TOO_LONG, + QCBOR_ERR_TOO_MANY_CLOSES, QCBOR_ERR_ARRAY_OR_MAP_STILL_OPEN, and + QCBOR_ERR_ENCODE_UNSUPPORTED will never be returned. It is up to the + caller to make sure that opened maps, arrays and byte-string wrapping + is closed correctly and that QCBOREncode_AddType7() is called + correctly. With this defined, it is easier to make a mistake when + authoring the encoding of a protocol that will output not well formed + CBOR, but as long as the calling code is correct, it is safe to + disable these checks. Bounds checking that prevents security issues + in the code is still enforced. This define reduces the size of + encoding object code by about 150 bytes. + + @anchor Tags-Overview + + ## Tags Overview + + Any CBOR data item can be made into a tag to add semantics, define a + new data type or such. Some tags are fully standardized and some are + just registered. Others are not registered and used in a proprietary + way. + + Encoding and decoding of many of the registered tags is fully + implemented by QCBOR. It is also possible to encode and decode tags + that are not directly supported. For many use cases the built-in tag + support should be adequate. + + For example, the registered epoch date tag is supported in encoding + by QCBOREncode_AddDateEpoch() and in decoding by @ref + QCBOR_TYPE_DATE_EPOCH and the @c epochDate member of @ref + QCBORItem. This is typical of the built-in tag support. There is an + API to encode data for it and a @c QCBOR_TYPE_XXX when it is decoded. + + Tags are registered in the [IANA CBOR Tags Registry] + (https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml). There + are roughly three options to create a new tag. First, a public + specification can be created and the new tag registered with IANA. + This is the most formal. Second, the new tag can be registered with + IANA with just a short description rather than a full specification. + These tags must be greater than 256. Third, a tag can be used without + any IANA registration, though the registry should be checked to see + that the new value doesn't collide with one that is registered. The + value of these tags must be 256 or larger. + + See also @ref CBORTags and @ref Tag-Usage + + The encoding side of tags not built-in is handled by + QCBOREncode_AddTag() and is relatively simple. Tag decoding is more + complex and mainly handled by QCBORDecode_GetNext(). Decoding of the + structure of tagged data not built-in (if there is any) has to be + implemented by the caller. + + @anchor Floating-Point + + ## Floating-Point + + By default QCBOR fully supports IEEE 754 floating-point: + - Encode/decode of double, single and half-precision + - CBOR preferred serialization of floating-point + - Floating-point epoch dates + + For the most part, the type double is used in the interface for + floating-point values. In the default configuration, all decoded + floating-point values are returned as a double. + + With CBOR preferred serialization, the encoder outputs the smallest + representation of the double or float that preserves precision. Zero, + NaN and infinity are always output as a half-precision, each taking + just 2 bytes. This reduces the number of bytes needed to encode + double and single-precision, especially if zero, NaN and infinity are + frequently used. + + To avoid use of preferred serialization in the standard configuration + when encoding, use QCBOREncode_AddDoubleNoPreferred() or + QCBOREncode_AddFloatNoPreferred(). + + This implementation of preferred floating-point serialization and + half-precision does not depend on the CPU having floating-point HW or + the compiler bringing in a (sometimes large) library to compensate + for lack of CPU support. This implementation uses shifts and masks + rather than floating-point functions. + + To reduce overall object code by about 900 bytes, define + QCBOR_DISABLE_PREFERRED_FLOAT. This will eliminate all support for + preferred serialization and half-precision. An error will be returned + when attempting to decode half-precision. A float will always be + encoded and decoded as 32-bits and a double will always be encoded + and decoded as 64 bits. + + Note that even if QCBOR_DISABLE_PREFERRED_FLOAT is not defined all + the float-point encoding object code can be avoided by never calling + any functions that encode double or float. Just not calling + floating-point functions will reduce object code by about 500 bytes. + + On CPUs that have no floating-point hardware, + QCBOR_DISABLE_FLOAT_HW_USE should be defined in most cases. If it is + not, then the compiler will bring in possibly large software + libraries to compensate. Defining QCBOR_DISABLE_FLOAT_HW_USE reduces + object code size on CPUs with floating-point hardware by a tiny + amount and eliminates the need for + + When QCBOR_DISABLE_FLOAT_HW_USE is defined, trying to decoding + floating-point dates will give error + @ref QCBOR_ERR_FLOAT_DATE_DISABLED and decoded single-precision + numbers will be returned as @ref QCBOR_TYPE_FLOAT instead of + converting them to double as usual. + + If both QCBOR_DISABLE_FLOAT_HW_USE and QCBOR_DISABLE_PREFERRED_FLOAT + are defined, then the only thing QCBOR can do is encode/decode a C + float type as 32-bits and a C double type as 64-bits. Floating-point + epoch dates will be unsupported. + + If USEFULBUF_DISABLE_ALL_FLOATis defined, then floating point support is + completely disabled. Decoding functions return @ref + QCBOR_ERR_ALL_FLOAT_DISABLED if a floating point value is encountered during + decoding. Functions that are encoding floating point values are not available. + + ## Limitations + + Summary Limits of this implementation: + - The entire encoded CBOR must fit into contiguous memory. + - Max size of encoded / decoded CBOR data is a few bytes less than @c + UINT32_MAX (4GB). + - Max array / map nesting level when encoding / decoding is + @ref QCBOR_MAX_ARRAY_NESTING (this is typically 15). + - Max items in an array or map when encoding / decoding is + @ref QCBOR_MAX_ITEMS_IN_ARRAY (typically 65,536). + - Does not directly support labels in maps other than text strings & integers. + - Does not directly support integer labels greater than @c INT64_MAX. + - Epoch dates limited to @c INT64_MAX (+/- 292 billion years). + - Exponents for bigfloats and decimal integers are limited to @c INT64_MAX. + - Tags on labels are ignored during decoding. + - The maximum tag nesting is @c QCBOR_MAX_TAGS_PER_ITEM (typically 4). + - Works only on 32- and 64-bit CPUs (modifications could make it work + on 16-bit CPUs). + + The public interface uses @c size_t for all lengths. Internally the + implementation uses 32-bit lengths by design to use less memory and + fit structures on the stack. This limits the encoded CBOR it can work + with to size @c UINT32_MAX (4GB) which should be enough. + + This implementation assumes two's compliment integer machines. @c + also requires this. It is possible to modify this + implementation for another integer representation, but all modern + machines seem to be two's compliment. + */ + +/** + The size of the buffer to be passed to QCBOREncode_EncodeHead(). It is one + byte larger than sizeof(uint64_t) + 1, the actual maximum size of the + head of a CBOR data item because QCBOREncode_EncodeHead() needs + one extra byte to work. + */ +#define QCBOR_HEAD_BUFFER_SIZE (sizeof(uint64_t) + 2) + +/** + Output the full CBOR tag. See @ref CBORTags, @ref Tag-Usage and + @ref Tags-Overview. + */ +#define QCBOR_ENCODE_AS_TAG 0 + +/** + Output only the 'borrowed' content format for the relevant tag. + See @ref CBORTags, @ref Tag-Usage and @ref Tags-Overview. + */ +#define QCBOR_ENCODE_AS_BORROWED 1 + +/** + QCBOREncodeContext is the data type that holds context for all the + encoding functions. It is less than 200 bytes, so it can go on the + stack. The contents are opaque, and the caller should not access + internal members. A context may be re used serially as long as it is + re initialized. + */ +typedef struct QCBOREncodeContext_s QCBOREncodeContext; + +/** + Initialize the encoder to prepare to encode some CBOR. + + @param[in,out] pCtx The encoder context to initialize. + @param[in] Storage The buffer into which the encoded result + will be written. + + Call this once at the start of an encoding of some CBOR. Then call + the many functions like QCBOREncode_AddInt64() and + QCBOREncode_AddText() to add the different data items. Finally, call + QCBOREncode_Finish() to get the pointer and length of the encoded + result. + + The primary purpose of this function is to give the pointer and + length of the output buffer into which the encoded CBOR will be + written. This is done with a @ref UsefulBuf structure, which is just + a pointer and length (it is equivalent to two parameters, one a + pointer and one a length, but a little prettier). + + The output buffer can be allocated any way (malloc, stack, + static). It is just some memory that QCBOR writes to. The length must + be the length of the allocated buffer. QCBOR will never write past + that length, but might write up to that length. If the buffer is too + small, encoding will go into an error state and not write anything + further. + + If allocating on the stack the convenience macro + UsefulBuf_MAKE_STACK_UB() can be used, but its use is not required. + + Since there is no reallocation or such, the output buffer must be + correctly sized when passed in here. It is OK, but wasteful if it is + too large. One way to pick the size is to figure out the maximum size + that will ever be needed and hard code a buffer of that size. + + Another way to do it is to have QCBOR calculate it for you. To do + this, pass @ref SizeCalculateUsefulBuf for @c Storage. + Then call all the functions to add the CBOR exactly as if + encoding for real. Finally, call QCBOREncode_FinishGetSize(). + Once the length is obtained, allocate a buffer of that + size, call QCBOREncode_Init() again with the real buffer. Call all + the add functions again and finally, QCBOREncode_Finish() to obtain + the final result. This uses twice the CPU time, but that is + usually not an issue. + + See QCBOREncode_Finish() for how the pointer and length for the + encoded CBOR is returned. + + For practical purposes QCBOR can't output encoded CBOR larger than + @c UINT32_MAX (4GB) even on 64-bit CPUs because the internal offsets + used to track the start of an array/map are 32 bits to reduce the + size of the encoding context. + + A @ref QCBOREncodeContext can be reused over and over as long as + QCBOREncode_Init() is called before each use. + */ +void +QCBOREncode_Init(QCBOREncodeContext *pCtx, UsefulBuf Storage); + +/** + @brief Add a signed 64-bit integer to the encoded output. + + @param[in] pCtx The encoding context to add the integer to. + @param[in] nNum The integer to add. + + The integer will be encoded and added to the CBOR output. + + This function figures out the size and the sign and encodes in the + correct minimal CBOR. Specifically, it will select CBOR major type 0 + or 1 based on sign and will encode to 1, 2, 4 or 8 bytes depending on + the value of the integer. Values less than 24 effectively encode to + one byte because they are encoded in with the CBOR major type. This + is a neat and efficient characteristic of CBOR that can be taken + advantage of when designing CBOR-based protocols. If integers like + tags can be kept between -23 and 23 they will be encoded in one byte + including the major type. + + If you pass a smaller int, say an @c int16_t or a small value, say + 100, the encoding will still be CBOR's most compact that can + represent the value. For example, CBOR always encodes the value 0 as + one byte, 0x00. The representation as 0x00 includes identification of + the type as an integer too as the major type for an integer is 0. See + [RFC 8949] (https://tools.ietf.org/html/rfc8949) Appendix A for more + examples of CBOR encoding. This compact encoding is also preferred + serialization CBOR as per section 34.1 in RFC 8949. + + There are no functions to add @c int16_t or @c int32_t because they + are not necessary because this always encodes to the smallest number + of bytes based on the value (If this code is running on a 32-bit + machine having a way to add 32-bit integers would reduce code size + some). + + If the encoding context is in an error state, this will do + nothing. If an error occurs when adding this integer, the internal + error flag will be set, and the error will be returned when + QCBOREncode_Finish() is called. + + See also QCBOREncode_AddUInt64(). + */ +void +QCBOREncode_AddInt64(QCBOREncodeContext *pCtx, int64_t nNum); + +static void +QCBOREncode_AddInt64ToMap(QCBOREncodeContext *pCtx, const char *szLabel, + int64_t uNum); + +static void +QCBOREncode_AddInt64ToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + int64_t uNum); + +/** + @brief Add an unsigned 64-bit integer to the encoded output. + + @param[in] pCtx The encoding context to add the integer to. + @param[in] uNum The integer to add. + + The integer will be encoded and added to the CBOR output. + + The only reason so use this function is for integers larger than @c + INT64_MAX and smaller than @c UINT64_MAX. Otherwise + QCBOREncode_AddInt64() will work fine. + + Error handling is the same as for QCBOREncode_AddInt64(). + */ +void +QCBOREncode_AddUInt64(QCBOREncodeContext *pCtx, uint64_t uNum); + +static void +QCBOREncode_AddUInt64ToMap(QCBOREncodeContext *pCtx, const char *szLabel, + uint64_t uNum); + +static void +QCBOREncode_AddUInt64ToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + uint64_t uNum); + +/** + @brief Add a UTF-8 text string to the encoded output. + + @param[in] pCtx The encoding context to add the text to. + @param[in] Text Pointer and length of text to add. + + The text passed in must be unencoded UTF-8 according to [RFC 3629] + (https://tools.ietf.org/html/rfc3629). There is no NULL + termination. The text is added as CBOR major type 3. + + If called with @c nBytesLen equal to 0, an empty string will be + added. When @c nBytesLen is 0, @c pBytes may be @c NULL. + + Note that the restriction of the buffer length to a @c uint32_t is + entirely intentional as this encoder is not capable of encoding + lengths greater. This limit to 4GB for a text string should not be a + problem. + + Text lines in Internet protocols (on the wire) are delimited by + either a CRLF or just an LF. Officially many protocols specify CRLF, + but implementations often work with either. CBOR type 3 text can be + either line ending, even a mixture of both. + + Operating systems usually have a line end convention. Windows uses + CRLF. Linux and MacOS use LF. Some applications on a given OS may + work with either and some may not. + + The majority of use cases and CBOR protocols using type 3 text will + work with either line ending. However, some use cases or protocols + may not work with either in which case translation to and/or from the + local line end convention, typically that of the OS, is necessary. + + QCBOR does no line ending translation for type 3 text when encoding + and decoding. + + Error handling is the same as QCBOREncode_AddInt64(). + */ +static void +QCBOREncode_AddText(QCBOREncodeContext *pCtx, UsefulBufC Text); + +static void +QCBOREncode_AddTextToMap(QCBOREncodeContext *pCtx, const char *szLabel, + UsefulBufC Text); + +static void +QCBOREncode_AddTextToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + UsefulBufC Text); + +/** + @brief Add a UTF-8 text string to the encoded output. + + @param[in] pCtx The encoding context to add the text to. + @param[in] szString Null-terminated text to add. + + This works the same as QCBOREncode_AddText(). + */ +static void +QCBOREncode_AddSZString(QCBOREncodeContext *pCtx, const char *szString); + +static void +QCBOREncode_AddSZStringToMap(QCBOREncodeContext *pCtx, const char *szLabel, + const char *szString); + +static void +QCBOREncode_AddSZStringToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + const char *szString); + +#ifndef USEFULBUF_DISABLE_ALL_FLOAT +/** + @brief Add a double-precision floating-point number to the encoded output. + + @param[in] pCtx The encoding context to add the double to. + @param[in] dNum The double-precision number to add. + + This encodes and outputs a floating-point number. CBOR major type 7 + is used. + + This implements preferred serialization, selectively encoding the + double-precision floating-point number as either double-precision, + single-precision or half-precision. Infinity, NaN and 0 are always + encoded as half-precision. If no precision will be lost in the + conversion to half-precision, then it will be converted and + encoded. If not and no precision will be lost in conversion to + single-precision, then it will be converted and encoded. If not, then + no conversion is performed, and it encoded as a double-precision. + + Half-precision floating-point numbers take up 2 bytes, half that of + single-precision, one quarter of double-precision + + This automatically reduces the size of encoded CBOR, maybe even by + four if most of values are 0, infinity or NaN. + + When decoded, QCBOR will usually return these values as + double-precision. + + It is possible to disable this preferred serialization when compiling + QCBOR. In that case, this functions the same as + QCBOREncode_AddDoubleNoPreferred(). + + Error handling is the same as QCBOREncode_AddInt64(). + + See also QCBOREncode_AddDoubleNoPreferred(), QCBOREncode_AddFloat() + and QCBOREncode_AddFloatNoPreferred() and @ref Floating-Point. + */ +void +QCBOREncode_AddDouble(QCBOREncodeContext *pCtx, double dNum); + +static void +QCBOREncode_AddDoubleToMap(QCBOREncodeContext *pCtx, const char *szLabel, + double dNum); + +static void +QCBOREncode_AddDoubleToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + double dNum); + +/** + @brief Add a single-precision floating-point number to the encoded output. + + @param[in] pCtx The encoding context to add the double to. + @param[in] fNum The single-precision number to add. + + This is identical to QCBOREncode_AddDouble() except the input is + single-precision. + + See also QCBOREncode_AddDouble(), QCBOREncode_AddDoubleNoPreferred(), + and QCBOREncode_AddFloatNoPreferred() and @ref Floating-Point. +*/ +void +QCBOREncode_AddFloat(QCBOREncodeContext *pCtx, float fNum); + +static void +QCBOREncode_AddFloatToMap(QCBOREncodeContext *pCtx, const char *szLabel, + float fNum); + +static void +QCBOREncode_AddFloatToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + float dNum); + +/** + @brief Add a double-precision floating-point number without preferred encoding. + + @param[in] pCtx The encoding context to add the double to. + @param[in] dNum The double-precision number to add. + + This always outputs the number as a 64-bit double-precision. + Preferred serialization is not used. + + Error handling is the same as QCBOREncode_AddInt64(). + + See also QCBOREncode_AddDouble(), QCBOREncode_AddFloat(), and + QCBOREncode_AddFloatNoPreferred() and @ref Floating-Point. +*/ +void +QCBOREncode_AddDoubleNoPreferred(QCBOREncodeContext *pCtx, double dNum); + +static void +QCBOREncode_AddDoubleNoPreferredToMap(QCBOREncodeContext *pCtx, + const char *szLabel, double dNum); + +static void +QCBOREncode_AddDoubleNoPreferredToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + double dNum); + +/** + @brief Add a single-precision floating-point number without preferred encoding. + + @param[in] pCtx The encoding context to add the double to. + @param[in] fNum The single-precision number to add. + + This always outputs the number as a 32-bit single-precision. + Preferred serialization is not used. + + Error handling is the same as QCBOREncode_AddInt64(). + + See also QCBOREncode_AddDouble(), QCBOREncode_AddFloat(), and + QCBOREncode_AddDoubleNoPreferred() and @ref Floating-Point. +*/ +void +QCBOREncode_AddFloatNoPreferred(QCBOREncodeContext *pCtx, float fNum); + +static void +QCBOREncode_AddFloatNoPreferredToMap(QCBOREncodeContext *pCtx, + const char *szLabel, float fNum); + +static void +QCBOREncode_AddFloatNoPreferredToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + float fNum); +#endif /* USEFULBUF_DISABLE_ALL_FLOAT */ + +/** + @brief Add an optional tag. + + @param[in] pCtx The encoding context to add the tag to. + @param[in] uTag The tag to add + + This outputs a CBOR major type 6 item that tags the next data item + that is output usually to indicate it is some new data type. + + For many of the common standard tags, a function to encode data using + it is provided and this is not needed. For example, + QCBOREncode_AddDateEpoch() already exists to output integers + representing dates with the right tag. + + The tag is applied to the next data item added to the encoded + output. That data item that is to be tagged can be of any major CBOR + type. Any number of tags can be added to a data item by calling this + multiple times before the data item is added. + + See @ref Tags-Overview for discussion of creating new non-standard + tags. See QCBORDecode_GetNext() for discussion of decoding custom + tags. +*/ +void +QCBOREncode_AddTag(QCBOREncodeContext *pCtx, uint64_t uTag); + +/** + @brief Add an epoch-based date. + + @param[in] pCtx The encoding context to add the date to. + @param[in] uTagRequirement Either @ref QCBOR_ENCODE_AS_TAG or @ref + QCBOR_ENCODE_AS_BORROWED. + @param[in] nDate Number of seconds since 1970-01-01T00:00Z in UTC time. + + As per RFC 8949 this is similar to UNIX/Linux/POSIX dates. This is + the most compact way to specify a date and time in CBOR. Note that + this is always UTC and does not include the time zone. Use + QCBOREncode_AddDateString() if you want to include the time zone. + + The preferred integer encoding rules apply here so the date will be encoded in + a minimal number of bytes. Until about the year 2106 these dates will + encode in 6 bytes -- one byte for the tag, one byte for the type and + 4 bytes for the integer. After that it will encode to 10 bytes. + + Negative values are supported for dates before 1970. + + If you care about leap-seconds and that level of accuracy, make sure + the system you are running this code on does it correctly. This code + just takes the value passed in. + + This implementation cannot encode fractional seconds using float or + double even though that is allowed by CBOR, but you can encode them + if you want to by calling QCBOREncode_AddTag() and QCBOREncode_AddDouble(). + + Error handling is the same as QCBOREncode_AddInt64(). + + See also QCBOREncode_AddTDaysEpoch(). + */ +static void +QCBOREncode_AddTDateEpoch(QCBOREncodeContext *pCtx, uint8_t uTagRequirement, + int64_t nDate); + +static void +QCBOREncode_AddTDateEpochToMapSZ(QCBOREncodeContext *pCtx, const char *szLabel, + uint8_t uTagRequirement, int64_t nDate); + +static void +QCBOREncode_AddTDateEpochToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + uint8_t uTagRequirement, int64_t nDate); + +static void +QCBOREncode_AddDateEpoch(QCBOREncodeContext *pCtx, int64_t nDate); + +static void +QCBOREncode_AddDateEpochToMap(QCBOREncodeContext *pCtx, const char *szLabel, + int64_t nDate); + +static void +QCBOREncode_AddDateEpochToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + int64_t nDate); + +/** + @brief Add an epoch-based day-count date. + + @param[in] pCtx The encoding context to add the date to. + @param[in] uTagRequirement Either @ref QCBOR_ENCODE_AS_TAG or + @ref QCBOR_ENCODE_AS_BORROWED. + @param[in] nDays Number of days before or after 1970-01-0. + + This date format is described in + [RFC 8943] (https://tools.ietf.org/html/rfc8943). + + The integer encoding rules apply here so the date will be encoded in + a minimal number of bytes. Until about the year 2149 these dates will + encode in 4 bytes -- one byte for the tag, one byte for the type and + 2 bytes for the integer. + + See also QCBOREncode_AddTDateEpoch(). + +*/ +static void +QCBOREncode_AddTDaysEpoch(QCBOREncodeContext *pCtx, uint8_t uTagRequirement, + int64_t nDays); + +static void +QCBOREncode_AddTDaysEpochToMapSZ(QCBOREncodeContext *pCtx, const char *szLabel, + uint8_t uTagRequirement, int64_t nDays); + +static void +QCBOREncode_AddTDaysEpochToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + uint8_t uTagRequirement, int64_t nDays); + +/** + @brief Add a byte string to the encoded output. + + @param[in] pCtx The encoding context to add the bytes to. + @param[in] Bytes Pointer and length of the input data. + + Simply adds the bytes to the encoded output as CBOR major type 2. + + If called with @c Bytes.len equal to 0, an empty string will be + added. When @c Bytes.len is 0, @c Bytes.ptr may be @c NULL. + + Error handling is the same as QCBOREncode_AddInt64(). + */ +static void +QCBOREncode_AddBytes(QCBOREncodeContext *pCtx, UsefulBufC Bytes); + +static void +QCBOREncode_AddBytesToMap(QCBOREncodeContext *pCtx, const char *szLabel, + UsefulBufC Bytes); + +static void +QCBOREncode_AddBytesToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + UsefulBufC Bytes); + +/** + @brief Set up to write a byte string value directly to encoded output. + + @param[in] pCtx The encoding context to add the bytes to. + @param[out] pPlace Pointer and length of place to write byte string value. + + QCBOREncode_AddBytes() is the normal way to encode a byte string. + This is for special cases and by passes some of the pointer safety. + + The purpose of this is to output the bytes that make up a byte string + value directly to the QCBOR output buffer so you don't need to have a + copy of it in memory. This is particularly useful if the byte string + is large, for example, the encrypted payload of a COSE_Encrypt + message. The payload encryption algorithm can output directly to the + encoded CBOR buffer, perhaps by making it the output buffer + for some function (e.g. symmetric encryption) or by multiple writes. + + The pointer in \c pPlace is where to start writing. Writing is just + copying bytes to the location by the pointer in \c pPlace. Writing + past the length in \c pPlace will be writing off the end of the + output buffer. + + If there is no room in the output buffer @ref NULLUsefulBuf will be + returned and there is no need to call QCBOREncode_CloseBytes(). + + The byte string must be closed by calling QCBOREncode_CloseBytes(). + + Warning: this bypasses some of the usual checks provided by QCBOR + against writing off the end of the encoded output buffer. + */ +void +QCBOREncode_OpenBytes(QCBOREncodeContext *pCtx, UsefulBuf *pPlace); + +static void +QCBOREncode_OpenBytesInMapSZ(QCBOREncodeContext *pCtx, const char *szLabel, + UsefulBuf *pPlace); + +static void +QCBOREncode_OpenBytesInMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + UsefulBuf *pPlace); + +/** + @brief Close out a byte string written directly to encoded output. + + @param[in] pCtx The encoding context to add the bytes to. + @param[out] uAmount The number of bytes written, the length of the byte + string. + + This closes out a call to QCBOREncode_OpenBytes(). This inserts a + CBOR header at the front of the byte string value to make it a + well-formed byte string. + + If there was no call to QCBOREncode_OpenBytes() then + @ref QCBOR_ERR_TOO_MANY_CLOSES is set. + */ +void +QCBOREncode_CloseBytes(QCBOREncodeContext *pCtx, size_t uAmount); + +/** + @brief Add a binary UUID to the encoded output. + + @param[in] pCtx The encoding context to add the UUID to. + @param[in] uTagRequirement Either @ref QCBOR_ENCODE_AS_TAG or @ref + QCBOR_ENCODE_AS_BORROWED. + @param[in] Bytes Pointer and length of the binary UUID. + + A binary UUID as defined in [RFC 4122] + (https://tools.ietf.org/html/rfc4122) is added to the output. + + It is output as CBOR major type 2, a binary string, with tag @ref + CBOR_TAG_BIN_UUID indicating the binary string is a UUID. + */ +static void +QCBOREncode_AddTBinaryUUID(QCBOREncodeContext *pCtx, uint8_t uTagRequirement, + UsefulBufC Bytes); + +static void +QCBOREncode_AddTBinaryUUIDToMapSZ(QCBOREncodeContext *pCtx, const char *szLabel, + uint8_t uTagRequirement, UsefulBufC Bytes); + +static void +QCBOREncode_AddTBinaryUUIDToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + uint8_t uTagRequirement, UsefulBufC Bytes); + +static void +QCBOREncode_AddBinaryUUID(QCBOREncodeContext *pCtx, UsefulBufC Bytes); + +static void +QCBOREncode_AddBinaryUUIDToMap(QCBOREncodeContext *pCtx, const char *szLabel, + UsefulBufC Bytes); + +static void +QCBOREncode_AddBinaryUUIDToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + UsefulBufC Bytes); + +/** + @brief Add a positive big number to the encoded output. + + @param[in] pCtx The encoding context to add the big number to. + @param[in] uTagRequirement Either @ref QCBOR_ENCODE_AS_TAG or @ref + QCBOR_ENCODE_AS_BORROWED. + @param[in] Bytes Pointer and length of the big number. + + Big numbers are integers larger than 64-bits. Their format is + described in [RFC 8949] (https://tools.ietf.org/html/rfc8949). + + It is output as CBOR major type 2, a binary string, with tag @ref + CBOR_TAG_POS_BIGNUM indicating the binary string is a positive big + number. + + Often big numbers are used to represent cryptographic keys, however, + COSE which defines representations for keys chose not to use this + particular type. + */ +static void +QCBOREncode_AddTPositiveBignum(QCBOREncodeContext *pCtx, + uint8_t uTagRequirement, UsefulBufC Bytes); + +static void +QCBOREncode_AddTPositiveBignumToMapSZ(QCBOREncodeContext *pCtx, + const char *szLabel, + uint8_t uTagRequirement, + UsefulBufC Bytes); + +static void +QCBOREncode_AddTPositiveBignumToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + uint8_t uTagRequirement, UsefulBufC Bytes); + +static void +QCBOREncode_AddPositiveBignum(QCBOREncodeContext *pCtx, UsefulBufC Bytes); + +static void +QCBOREncode_AddPositiveBignumToMap(QCBOREncodeContext *pCtx, + const char *szLabel, UsefulBufC Bytes); + +static void +QCBOREncode_AddPositiveBignumToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + UsefulBufC Bytes); + +/** + @brief Add a negative big number to the encoded output. + + @param[in] pCtx The encoding context to add the big number to. + @param[in] uTagRequirement Either @ref QCBOR_ENCODE_AS_TAG or @ref + QCBOR_ENCODE_AS_BORROWED. + @param[in] Bytes Pointer and length of the big number. + + Big numbers are integers larger than 64-bits. Their format is + described in [RFC 8949] (https://tools.ietf.org/html/rfc8949). + + It is output as CBOR major type 2, a binary string, with tag @ref + CBOR_TAG_NEG_BIGNUM indicating the binary string is a negative big + number. + + Often big numbers are used to represent cryptographic keys, however, + COSE which defines representations for keys chose not to use this + particular type. + */ +static void +QCBOREncode_AddTNegativeBignum(QCBOREncodeContext *pCtx, + uint8_t uTagRequirement, UsefulBufC Bytes); + +static void +QCBOREncode_AddTNegativeBignumToMapSZ(QCBOREncodeContext *pCtx, + const char *szLabel, + uint8_t uTagRequirement, + UsefulBufC Bytes); + +static void +QCBOREncode_AddTNegativeBignumToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + uint8_t uTagRequirement, UsefulBufC Bytes); + +static void +QCBOREncode_AddNegativeBignum(QCBOREncodeContext *pCtx, UsefulBufC Bytes); + +static void +QCBOREncode_AddNegativeBignumToMap(QCBOREncodeContext *pCtx, + const char *szLabel, UsefulBufC Bytes); + +static void +QCBOREncode_AddNegativeBignumToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + UsefulBufC Bytes); + +#ifndef QCBOR_DISABLE_EXP_AND_MANTISSA +/** + @brief Add a decimal fraction to the encoded output. + + @param[in] pCtx The encoding context to add the decimal fraction to. + @param[in] uTagRequirement Either @ref QCBOR_ENCODE_AS_TAG or @ref + QCBOR_ENCODE_AS_BORROWED. + @param[in] nMantissa The mantissa. + @param[in] nBase10Exponent The exponent. + + The value is nMantissa * 10 ^ nBase10Exponent. + + A decimal fraction is good for exact representation of some values + that can't be represented exactly with standard C (IEEE 754) + floating-point numbers. Much larger and much smaller numbers can + also be represented than floating-point because of the larger number + of bits in the exponent. + + The decimal fraction is conveyed as two integers, a mantissa and a + base-10 scaling factor. + + For example, 273.15 is represented by the two integers 27315 and -2. + + The exponent and mantissa have the range from @c INT64_MIN to + @c INT64_MAX for both encoding and decoding (CBOR allows @c -UINT64_MAX + to @c UINT64_MAX, but this implementation doesn't support this range to + reduce code size and interface complexity a little). + + CBOR Preferred encoding of the integers is used, thus they will be encoded + in the smallest number of bytes possible. + + See also QCBOREncode_AddDecimalFractionBigNum() for a decimal + fraction with arbitrarily large precision and QCBOREncode_AddBigFloat(). + + There is no representation of positive or negative infinity or NaN + (Not a Number). Use QCBOREncode_AddDouble() to encode them. + + See @ref expAndMantissa for decoded representation. + */ +static void +QCBOREncode_AddTDecimalFraction(QCBOREncodeContext *pCtx, + uint8_t uTagRequirement, int64_t nMantissa, + int64_t nBase10Exponent); + +static void +QCBOREncode_AddTDecimalFractionToMapSZ(QCBOREncodeContext *pCtx, + const char *szLabel, + uint8_t uTagRequirement, + int64_t nMantissa, + int64_t nBase10Exponent); + +static void +QCBOREncode_AddTDecimalFractionToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + uint8_t uTagRequirement, + int64_t nMantissa, + int64_t nBase10Exponent); + +static void +QCBOREncode_AddDecimalFraction(QCBOREncodeContext *pCtx, int64_t nMantissa, + int64_t nBase10Exponent); + +static void +QCBOREncode_AddDecimalFractionToMap(QCBOREncodeContext *pCtx, + const char *szLabel, int64_t nMantissa, + int64_t nBase10Exponent); + +static void +QCBOREncode_AddDecimalFractionToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + int64_t nMantissa, + int64_t nBase10Exponent); +/** + @brief Add a decimal fraction with a big number mantissa to the encoded output. + + @param[in] pCtx The encoding context to add the decimal fraction to. + @param[in] uTagRequirement Either @ref QCBOR_ENCODE_AS_TAG or @ref + QCBOR_ENCODE_AS_BORROWED. + @param[in] Mantissa The mantissa. + @param[in] bIsNegative false if mantissa is positive, true if negative. + @param[in] nBase10Exponent The exponent. + + This is the same as QCBOREncode_AddDecimalFraction() except the + mantissa is a big number (See QCBOREncode_AddPositiveBignum()) + allowing for arbitrarily large precision. + + See @ref expAndMantissa for decoded representation. + */ +static void +QCBOREncode_AddTDecimalFractionBigNum(QCBOREncodeContext *pCtx, + uint8_t uTagRequirement, + UsefulBufC Mantissa, bool bIsNegative, + int64_t nBase10Exponent); + +static void +QCBOREncode_AddTDecimalFractionBigNumToMapSZ( + QCBOREncodeContext *pCtx, const char *szLabel, uint8_t uTagRequirement, + UsefulBufC Mantissa, bool bIsNegative, int64_t nBase10Exponent); + +static void +QCBOREncode_AddTDecimalFractionBigNumToMapN( + QCBOREncodeContext *pCtx, int64_t nLabel, uint8_t uTagRequirement, + UsefulBufC Mantissa, bool bIsNegative, int64_t nBase10Exponent); + +static void +QCBOREncode_AddDecimalFractionBigNum(QCBOREncodeContext *pCtx, + UsefulBufC Mantissa, bool bIsNegative, + int64_t nBase10Exponent); + +static void +QCBOREncode_AddDecimalFractionBigNumToMapSZ(QCBOREncodeContext *pCtx, + const char *szLabel, + UsefulBufC Mantissa, + bool bIsNegative, + int64_t nBase10Exponent); + +static void +QCBOREncode_AddDecimalFractionBigNumToMapN(QCBOREncodeContext *pCtx, + int64_t nLabel, UsefulBufC Mantissa, + bool bIsNegative, + int64_t nBase10Exponent); + +/** + @brief Add a big floating-point number to the encoded output. + + @param[in] pCtx The encoding context to add the bigfloat to. + @param[in] uTagRequirement Either @ref QCBOR_ENCODE_AS_TAG or @ref + QCBOR_ENCODE_AS_BORROWED. + @param[in] nMantissa The mantissa. + @param[in] nBase2Exponent The exponent. + + The value is nMantissa * 2 ^ nBase2Exponent. + + "Bigfloats", as CBOR terms them, are similar to IEEE floating-point + numbers in having a mantissa and base-2 exponent, but they are not + supported by hardware or encoded the same. They explicitly use two + CBOR-encoded integers to convey the mantissa and exponent, each of which + can be 8, 16, 32 or 64 bits. With both the mantissa and exponent + 64 bits they can express more precision and a larger range than an + IEEE double floating-point number. See + QCBOREncode_AddBigFloatBigNum() for even more precision. + + For example, 1.5 would be represented by a mantissa of 3 and an + exponent of -1. + + The exponent and mantissa have the range from @c INT64_MIN to + @c INT64_MAX for both encoding and decoding (CBOR allows @c -UINT64_MAX + to @c UINT64_MAX, but this implementation doesn't support this range to + reduce code size and interface complexity a little). + + CBOR Preferred encoding of the integers is used, thus they will be encoded + in the smallest number of bytes possible. + + This can also be used to represent floating-point numbers in + environments that don't support IEEE 754. + + See @ref expAndMantissa for decoded representation. + */ +static void +QCBOREncode_AddTBigFloat(QCBOREncodeContext *pCtx, uint8_t uTagRequirement, + int64_t nMantissa, int64_t nBase2Exponent); + +static void +QCBOREncode_AddTBigFloatToMapSZ(QCBOREncodeContext *pCtx, const char *szLabel, + uint8_t uTagRequirement, int64_t nMantissa, + int64_t nBase2Exponent); + +static void +QCBOREncode_AddTBigFloatToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + uint8_t uTagRequirement, int64_t nMantissa, + int64_t nBase2Exponent); + +static void +QCBOREncode_AddBigFloat(QCBOREncodeContext *pCtx, int64_t nMantissa, + int64_t nBase2Exponent); + +static void +QCBOREncode_AddBigFloatToMap(QCBOREncodeContext *pCtx, const char *szLabel, + int64_t nMantissa, int64_t nBase2Exponent); + +static void +QCBOREncode_AddBigFloatToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + int64_t nMantissa, int64_t nBase2Exponent); + +/** + @brief Add a big floating-point number with a big number mantissa to + the encoded output. + + @param[in] pCtx The encoding context to add the bigfloat to. + @param[in] uTagRequirement Either @ref QCBOR_ENCODE_AS_TAG or @ref + QCBOR_ENCODE_AS_BORROWED. + @param[in] Mantissa The mantissa. + @param[in] bIsNegative false if mantissa is positive, true if negative. + @param[in] nBase2Exponent The exponent. + + This is the same as QCBOREncode_AddBigFloat() except the mantissa is + a big number (See QCBOREncode_AddPositiveBignum()) allowing for + arbitrary precision. + + See @ref expAndMantissa for decoded representation. + */ +static void +QCBOREncode_AddTBigFloatBigNum(QCBOREncodeContext *pCtx, + uint8_t uTagRequirement, UsefulBufC Mantissa, + bool bIsNegative, int64_t nBase2Exponent); + +static void +QCBOREncode_AddTBigFloatBigNumToMapSZ(QCBOREncodeContext *pCtx, + const char *szLabel, + uint8_t uTagRequirement, + UsefulBufC Mantissa, bool bIsNegative, + int64_t nBase2Exponent); + +static void +QCBOREncode_AddTBigFloatBigNumToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + uint8_t uTagRequirement, + UsefulBufC Mantissa, bool bIsNegative, + int64_t nBase2Exponent); + +static void +QCBOREncode_AddBigFloatBigNum(QCBOREncodeContext *pCtx, UsefulBufC Mantissa, + bool bIsNegative, int64_t nBase2Exponent); + +static void +QCBOREncode_AddBigFloatBigNumToMap(QCBOREncodeContext *pCtx, + const char *szLabel, UsefulBufC Mantissa, + bool bIsNegative, int64_t nBase2Exponent); + +static void +QCBOREncode_AddBigFloatBigNumToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + UsefulBufC Mantissa, bool bIsNegative, + int64_t nBase2Exponent); +#endif /* QCBOR_DISABLE_EXP_AND_MANTISSA */ + +/** + @brief Add a text URI to the encoded output. + + @param[in] pCtx The encoding context to add the URI to. + @param[in] uTagRequirement Either @ref QCBOR_ENCODE_AS_TAG or @ref + QCBOR_ENCODE_AS_BORROWED. + @param[in] URI Pointer and length of the URI. + + The format of URI must be per [RFC 3986] + (https://tools.ietf.org/html/rfc3986). + + It is output as CBOR major type 3, a text string, with tag @ref + CBOR_TAG_URI indicating the text string is a URI. + + A URI in a NULL-terminated string, @c szURI, can be easily added with + this code: + + QCBOREncode_AddURI(pCtx, UsefulBuf_FromSZ(szURI)); + */ +static void +QCBOREncode_AddTURI(QCBOREncodeContext *pCtx, uint8_t uTagRequirement, + UsefulBufC URI); + +static void +QCBOREncode_AddTURIToMapSZ(QCBOREncodeContext *pCtx, const char *szLabel, + uint8_t uTagRequirement, UsefulBufC URI); + +static void +QCBOREncode_AddTURIToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + uint8_t uTagRequirement, UsefulBufC URI); + +static void +QCBOREncode_AddURI(QCBOREncodeContext *pCtx, UsefulBufC URI); + +static void +QCBOREncode_AddURIToMap(QCBOREncodeContext *pCtx, const char *szLabel, + UsefulBufC URI); + +static void +QCBOREncode_AddURIToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + UsefulBufC URI); + +/** + @brief Add Base64-encoded text to encoded output. + + @param[in] pCtx The encoding context to add the base-64 text to. + @param[in] uTagRequirement Either @ref QCBOR_ENCODE_AS_TAG or @ref + QCBOR_ENCODE_AS_BORROWED. + @param[in] B64Text Pointer and length of the base-64 encoded text. + + The text content is Base64 encoded data per [RFC 4648] + (https://tools.ietf.org/html/rfc4648). + + It is output as CBOR major type 3, a text string, with tag @ref + CBOR_TAG_B64 indicating the text string is Base64 encoded. + */ +static void +QCBOREncode_AddTB64Text(QCBOREncodeContext *pCtx, uint8_t uTagRequirement, + UsefulBufC B64Text); + +static void +QCBOREncode_AddTB64TextToMapSZ(QCBOREncodeContext *pCtx, const char *szLabel, + uint8_t uTagRequirement, UsefulBufC B64Text); + +static void +QCBOREncode_AddTB64TextToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + uint8_t uTagRequirement, UsefulBufC B64Text); + +static void +QCBOREncode_AddB64Text(QCBOREncodeContext *pCtx, UsefulBufC B64Text); + +static void +QCBOREncode_AddB64TextToMap(QCBOREncodeContext *pCtx, const char *szLabel, + UsefulBufC B64Text); + +static void +QCBOREncode_AddB64TextToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + UsefulBufC B64Text); + +/** + @brief Add base64url encoded data to encoded output. + + @param[in] pCtx The encoding context to add the base64url to. + @param[in] uTagRequirement Either @ref QCBOR_ENCODE_AS_TAG or @ref + QCBOR_ENCODE_AS_BORROWED. + @param[in] B64Text Pointer and length of the base64url encoded text. + + The text content is base64URL encoded text as per [RFC 4648] + (https://tools.ietf.org/html/rfc4648). + + It is output as CBOR major type 3, a text string, with tag @ref + CBOR_TAG_B64URL indicating the text string is a Base64url encoded. + */ +static void +QCBOREncode_AddTB64URLText(QCBOREncodeContext *pCtx, uint8_t uTagRequirement, + UsefulBufC B64Text); + +static void +QCBOREncode_AddTB64URLTextToMapSZ(QCBOREncodeContext *pCtx, const char *szLabel, + uint8_t uTagRequirement, UsefulBufC B64Text); + +static void +QCBOREncode_AddTB64URLTextToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + uint8_t uTagRequirement, UsefulBufC B64Text); + +static void +QCBOREncode_AddB64URLText(QCBOREncodeContext *pCtx, UsefulBufC B64Text); + +static void +QCBOREncode_AddB64URLTextToMap(QCBOREncodeContext *pCtx, const char *szLabel, + UsefulBufC B64Text); + +static void +QCBOREncode_AddB64URLTextToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + UsefulBufC B64Text); + +/** + @brief Add Perl Compatible Regular Expression. + + @param[in] pCtx The encoding context to add the regular expression to. + @param[in] uTagRequirement Either @ref QCBOR_ENCODE_AS_TAG or @ref + QCBOR_ENCODE_AS_BORROWED. + @param[in] Regex Pointer and length of the regular expression. + + The text content is Perl Compatible Regular + Expressions (PCRE) / JavaScript syntax [ECMA262]. + + It is output as CBOR major type 3, a text string, with tag @ref + CBOR_TAG_REGEX indicating the text string is a regular expression. + */ +static void +QCBOREncode_AddTRegex(QCBOREncodeContext *pCtx, uint8_t uTagRequirement, + UsefulBufC Regex); + +static void +QCBOREncode_AddTRegexToMapSZ(QCBOREncodeContext *pCtx, const char *szLabel, + uint8_t uTagRequirement, UsefulBufC Regex); + +static void +QCBOREncode_AddTRegexToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + uint8_t uTagRequirement, UsefulBufC Regex); + +static void +QCBOREncode_AddRegex(QCBOREncodeContext *pCtx, UsefulBufC Regex); + +static void +QCBOREncode_AddRegexToMap(QCBOREncodeContext *pCtx, const char *szLabel, + UsefulBufC Regex); + +static void +QCBOREncode_AddRegexToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + UsefulBufC Regex); + +/** + @brief MIME encoded data to the encoded output. + + @param[in] pCtx The encoding context to add the MIME data to. + @param[in] uTagRequirement Either @ref QCBOR_ENCODE_AS_TAG or + @ref QCBOR_ENCODE_AS_BORROWED. + @param[in] MIMEData Pointer and length of the MIME data. + + The text content is in MIME format per [RFC 2045] + (https://tools.ietf.org/html/rfc2045) including the headers. + + It is output as CBOR major type 2, a binary string, with tag @ref + CBOR_TAG_BINARY_MIME indicating the string is MIME data. This + outputs tag 257, not tag 36, as it can carry any type of MIME binary, + 7-bit, 8-bit, quoted-printable and base64 where tag 36 cannot. + + Previous versions of QCBOR, those before spiffy decode, output tag + 36. Decoding supports both tag 36 and 257. (if the old behavior with + tag 36 is needed, copy the inline functions below and change the tag + number). + + See also QCBORDecode_GetMIMEMessage() and + @ref QCBOR_TYPE_BINARY_MIME. + + This does no translation of line endings. See QCBOREncode_AddText() + for a discussion of line endings in CBOR. + */ +static void +QCBOREncode_AddTMIMEData(QCBOREncodeContext *pCtx, uint8_t uTagRequirement, + UsefulBufC MIMEData); + +static void +QCBOREncode_AddTMIMEDataToMapSZ(QCBOREncodeContext *pCtx, const char *szLabel, + uint8_t uTagRequirement, UsefulBufC MIMEData); + +static void +QCBOREncode_AddTMIMEDataToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + uint8_t uTagRequirement, UsefulBufC MIMEData); + +static void +QCBOREncode_AddMIMEData(QCBOREncodeContext *pCtx, UsefulBufC MIMEData); + +static void +QCBOREncode_AddMIMEDataToMap(QCBOREncodeContext *pCtx, const char *szLabel, + UsefulBufC MIMEData); + +static void +QCBOREncode_AddMIMEDataToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + UsefulBufC MIMEData); + +/** + @brief Add an RFC 3339 date string + + @param[in] pCtx The encoding context to add the date to. + @param[in] uTagRequirement Either @ref QCBOR_ENCODE_AS_TAG or @ref + QCBOR_ENCODE_AS_BORROWED. + @param[in] szDate Null-terminated string with date to add. + + The string szDate should be in the form of [RFC 3339] + (https://tools.ietf.org/html/rfc3339) as defined by section 3.3 in + [RFC 4287] (https://tools.ietf.org/html/rfc4287). This is as + described in section 3.4.1 in [RFC 8949] + (https://tools.ietf.org/html/rfc8949). + + Note that this function doesn't validate the format of the date string + at all. If you add an incorrect format date string, the generated + CBOR will be incorrect and the receiver may not be able to handle it. + + Error handling is the same as QCBOREncode_AddInt64(). + + See also QCBOREncode_AddTDayString(). + */ +static void +QCBOREncode_AddTDateString(QCBOREncodeContext *pCtx, uint8_t uTagRequirement, + const char *szDate); + +static void +QCBOREncode_AddTDateStringToMapSZ(QCBOREncodeContext *pCtx, const char *szLabel, + uint8_t uTagRequirement, const char *szDate); + +static void +QCBOREncode_AddTDateStringToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + uint8_t uTagRequirement, const char *szDate); + +static void +QCBOREncode_AddDateString(QCBOREncodeContext *pCtx, const char *szDate); + +static void +QCBOREncode_AddDateStringToMap(QCBOREncodeContext *pCtx, const char *szLabel, + const char *szDate); + +static void +QCBOREncode_AddDateStringToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + const char *szDate); + +/** + @brief Add a date-only string. + + @param[in] pCtx The encoding context to add the date to. + @param[in] uTagRequirement Either @ref QCBOR_ENCODE_AS_TAG or + @ref QCBOR_ENCODE_AS_BORROWED. + @param[in] szDate Null-terminated string with date to add. + + This date format is described in + [RFC 8943] (https://tools.ietf.org/html/rfc8943), but that mainly + references RFC 3339. The string szDate must be in the forrm + specified the ABNF for a full-date in + [RFC 3339] (https://tools.ietf.org/html/rfc3339). Examples of this + are "1985-04-12" and "1937-01-01". The time and the time zone are + never included. + + Note that this function doesn't validate the format of the date + string at all. If you add an incorrect format date string, the + generated CBOR will be incorrect and the receiver may not be able to + handle it. + + Error handling is the same as QCBOREncode_AddInt64(). + + See also QCBOREncode_AddTDateString(). + */ +static void +QCBOREncode_AddTDaysString(QCBOREncodeContext *pCtx, uint8_t uTagRequirement, + const char *szDate); + +static void +QCBOREncode_AddTDaysStringToMapSZ(QCBOREncodeContext *pCtx, const char *szLabel, + uint8_t uTagRequirement, const char *szDate); + +static void +QCBOREncode_AddTDaysStringToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + uint8_t uTagRequirement, const char *szDate); + +/** + @brief Add a standard Boolean. + + @param[in] pCtx The encoding context to add the Boolean to. + @param[in] b true or false from @c . + + Adds a Boolean value as CBOR major type 7. + + Error handling is the same as QCBOREncode_AddInt64(). + */ +static void +QCBOREncode_AddBool(QCBOREncodeContext *pCtx, bool b); + +static void +QCBOREncode_AddBoolToMap(QCBOREncodeContext *pCtx, const char *szLabel, bool b); + +static void +QCBOREncode_AddBoolToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, bool b); + +/** + @brief Add a NULL to the encoded output. + + @param[in] pCtx The encoding context to add the NULL to. + + Adds the NULL value as CBOR major type 7. + + This NULL doesn't have any special meaning in CBOR such as a + terminating value for a string or an empty value. + + Error handling is the same as QCBOREncode_AddInt64(). + */ +static void +QCBOREncode_AddNULL(QCBOREncodeContext *pCtx); + +static void +QCBOREncode_AddNULLToMap(QCBOREncodeContext *pCtx, const char *szLabel); + +static void +QCBOREncode_AddNULLToMapN(QCBOREncodeContext *pCtx, int64_t nLabel); + +/** + @brief Add an "undef" to the encoded output. + + @param[in] pCtx The encoding context to add the "undef" to. + + Adds the undef value as CBOR major type 7. + + Note that this value will not translate to JSON. + + This Undef doesn't have any special meaning in CBOR such as a + terminating value for a string or an empty value. + + Error handling is the same as QCBOREncode_AddInt64(). + */ +static void +QCBOREncode_AddUndef(QCBOREncodeContext *pCtx); + +static void +QCBOREncode_AddUndefToMap(QCBOREncodeContext *pCtx, const char *szLabel); + +static void +QCBOREncode_AddUndefToMapN(QCBOREncodeContext *pCtx, int64_t nLabel); + +/** + @brief Indicates that the next items added are in an array. + + @param[in] pCtx The encoding context to open the array in. + + Arrays are the basic CBOR aggregate or structure type. Call this + function to start or open an array. Then call the various @c + QCBOREncode_AddXxx() functions to add the items that go into the + array. Then call QCBOREncode_CloseArray() when all items have been + added. The data items in the array can be of any type and can be of + mixed types. + + Nesting of arrays and maps is allowed and supported just by calling + QCBOREncode_OpenArray() again before calling + QCBOREncode_CloseArray(). While CBOR has no limit on nesting, this + implementation does in order to keep it smaller and simpler. The + limit is @ref QCBOR_MAX_ARRAY_NESTING. This is the max number of + times this can be called without calling + QCBOREncode_CloseArray(). QCBOREncode_Finish() will return @ref + QCBOR_ERR_ARRAY_NESTING_TOO_DEEP when it is called as this function + just sets an error state and returns no value when this occurs. + + If you try to add more than @ref QCBOR_MAX_ITEMS_IN_ARRAY items to a + single array or map, @ref QCBOR_ERR_ARRAY_TOO_LONG will be returned + when QCBOREncode_Finish() is called. + + An array itself must have a label if it is being added to a map. + Note that array elements do not have labels (but map elements do). + + An array itself may be tagged by calling QCBOREncode_AddTag() before this call. + */ +static void +QCBOREncode_OpenArray(QCBOREncodeContext *pCtx); + +static void +QCBOREncode_OpenArrayInMap(QCBOREncodeContext *pCtx, const char *szLabel); + +static void +QCBOREncode_OpenArrayInMapN(QCBOREncodeContext *pCtx, int64_t nLabel); + +/** + @brief Close an open array. + + @param[in] pCtx The encoding context to close the array in. + + The closes an array opened by QCBOREncode_OpenArray(). It reduces + nesting level by one. All arrays (and maps) must be closed before + calling QCBOREncode_Finish(). + + When an error occurs as a result of this call, the encoder records + the error and enters the error state. The error will be returned when + QCBOREncode_Finish() is called. + + If this has been called more times than QCBOREncode_OpenArray(), then + @ref QCBOR_ERR_TOO_MANY_CLOSES will be returned when QCBOREncode_Finish() + is called. + + If this is called and it is not an array that is currently open, @ref + QCBOR_ERR_CLOSE_MISMATCH will be returned when QCBOREncode_Finish() + is called. + */ +static void +QCBOREncode_CloseArray(QCBOREncodeContext *pCtx); + +/** + @brief Indicates that the next items added are in a map. + + @param[in] pCtx The encoding context to open the map in. + + See QCBOREncode_OpenArray() for more information, particularly error + handling. + + CBOR maps are an aggregate type where each item in the map consists + of a label and a value. They are similar to JSON objects. + + The value can be any CBOR type including another map. + + The label can also be any CBOR type, but in practice they are + typically, integers as this gives the most compact output. They might + also be text strings which gives readability and translation to JSON. + + Every @c QCBOREncode_AddXxx() call has one version that ends with @c + InMap for adding items to maps with string labels and one that ends + with @c InMapN that is for adding with integer labels. + + RFC 8949 uses the term "key" instead of "label". + + If you wish to use map labels that are neither integer labels nor + text strings, then just call the QCBOREncode_AddXxx() function + explicitly to add the label. Then call it again to add the value. + + See the [RFC 8949] (https://tools.ietf.org/html/rfc8949) for a lot + more information on creating maps. + */ +static void +QCBOREncode_OpenMap(QCBOREncodeContext *pCtx); + +static void +QCBOREncode_OpenMapInMap(QCBOREncodeContext *pCtx, const char *szLabel); + +static void +QCBOREncode_OpenMapInMapN(QCBOREncodeContext *pCtx, int64_t nLabel); + +/** + @brief Close an open map. + + @param[in] pCtx The encoding context to close the map in . + + This closes a map opened by QCBOREncode_OpenMap(). It reduces nesting + level by one. + + When an error occurs as a result of this call, the encoder records + the error and enters the error state. The error will be returned when + QCBOREncode_Finish() is called. + + If this has been called more times than QCBOREncode_OpenMap(), + then @ref QCBOR_ERR_TOO_MANY_CLOSES will be returned when + QCBOREncode_Finish() is called. + + If this is called and it is not a map that is currently open, @ref + QCBOR_ERR_CLOSE_MISMATCH will be returned when QCBOREncode_Finish() + is called. + */ +static void +QCBOREncode_CloseMap(QCBOREncodeContext *pCtx); + +/** + @brief Indicate start of encoded CBOR to be wrapped in a bstr. + + @param[in] pCtx The encoding context to open the bstr-wrapped CBOR in. + + All added encoded items between this call and a call to + QCBOREncode_CloseBstrWrap2() will be wrapped in a bstr. They will + appear in the final output as a byte string. That byte string will + contain encoded CBOR. This increases nesting level by one. + + The typical use case is for encoded CBOR that is to be + cryptographically hashed, as part of a [RFC 8152, COSE] + (https://tools.ietf.org/html/rfc8152) implementation. The + wrapping byte string is taken as input by the hash function + (which is why it is returned by QCBOREncode_CloseBstrWrap2()). + It is also easy to recover on decoding with standard CBOR + decoders. + + Using QCBOREncode_BstrWrap() and QCBOREncode_CloseBstrWrap2() avoids + having to encode the items first in one buffer (e.g., the COSE + payload) and then add that buffer as a bstr to another encoding + (e.g. the COSE to-be-signed bytes, the @c Sig_structure) potentially + halving the memory needed. + + CBOR by nature must be decoded item by item in order from the start. + By wrapping some CBOR in a byte string, the decoding of that wrapped + CBOR can be skipped. This is another use of wrapping, perhaps + because the CBOR is large and deeply nested. Perhaps APIs for + handling one defined CBOR message that is being embedded in another + only take input as a byte string. Perhaps the desire is to be able + to decode the out layer even in the wrapped has errors. + */ +static void +QCBOREncode_BstrWrap(QCBOREncodeContext *pCtx); + +static void +QCBOREncode_BstrWrapInMap(QCBOREncodeContext *pCtx, const char *szLabel); + +static void +QCBOREncode_BstrWrapInMapN(QCBOREncodeContext *pCtx, int64_t nLabel); + +/** + @brief Close a wrapping bstr. + + @param[in] pCtx The encoding context to close of bstr wrapping in. + @param[in] bIncludeCBORHead Include the encoded CBOR head of the bstr + as well as the bytes in @c pWrappedCBOR. + @param[out] pWrappedCBOR A @ref UsefulBufC containing wrapped bytes. + + The closes a wrapping bstr opened by QCBOREncode_BstrWrap(). It reduces + nesting level by one. + + A pointer and length of the enclosed encoded CBOR is returned in @c + *pWrappedCBOR if it is not @c NULL. The main purpose of this is so + this data can be hashed (e.g., with SHA-256) as part of a [RFC 8152, + COSE] (https://tools.ietf.org/html/rfc8152) + implementation. **WARNING**, this pointer and length should be used + right away before any other calls to @c QCBOREncode_CloseXxx() as + they will move data around and the pointer and length will no longer + be to the correct encoded CBOR. + + When an error occurs as a result of this call, the encoder records + the error and enters the error state. The error will be returned when + QCBOREncode_Finish() is called. + + If this has been called more times than QCBOREncode_BstrWrap(), then + @ref QCBOR_ERR_TOO_MANY_CLOSES will be returned when + QCBOREncode_Finish() is called. + + If this is called and it is not a wrapping bstr that is currently + open, @ref QCBOR_ERR_CLOSE_MISMATCH will be returned when + QCBOREncode_Finish() is called. + + QCBOREncode_CloseBstrWrap() is a deprecated version of this function + that is equivalent to the call with @c bIncludeCBORHead @c true. + */ +void +QCBOREncode_CloseBstrWrap2(QCBOREncodeContext *pCtx, bool bIncludeCBORHead, + UsefulBufC *pWrappedCBOR); + +static void +QCBOREncode_CloseBstrWrap(QCBOREncodeContext *pCtx, UsefulBufC *pWrappedCBOR); + +/** + * @brief Cancel byte string wrapping. + * + * @param[in] pCtx The encoding context. + * + * This cancels QCBOREncode_BstrWrap() making tghe encoding as if it + * were never called. + * + * WARNING: This does not work on QCBOREncode_BstrWrapInMap() + * or QCBOREncode_BstrWrapInMapN() and there is no error detection + * of an attempt at their use. + * + * This only works if nothing has been added into the wrapped byte + * string. If something has been added, this sets the error + * @ref QCBOR_ERR_CANNOT_CANCEL. + */ +void +QCBOREncode_CancelBstrWrap(QCBOREncodeContext *pCtx); + +/** + @brief Add some already-encoded CBOR bytes. + + @param[in] pCtx The encoding context to add the already-encode CBOR to. + @param[in] Encoded The already-encoded CBOR to add to the context. + + The encoded CBOR being added must be fully conforming CBOR. It must + be complete with no arrays or maps that are incomplete. While this + encoder doesn't ever produce indefinite lengths, it is OK for the + raw CBOR added here to have indefinite lengths. + + The raw CBOR added here is not checked in anyway. If it is not + conforming or has open arrays or such, the final encoded CBOR + will probably be wrong or not what was intended. + + If the encoded CBOR being added here contains multiple items, they + must be enclosed in a map or array. At the top level the raw + CBOR must be a single data item. + */ +static void +QCBOREncode_AddEncoded(QCBOREncodeContext *pCtx, UsefulBufC Encoded); + +static void +QCBOREncode_AddEncodedToMap(QCBOREncodeContext *pCtx, const char *szLabel, + UsefulBufC Encoded); + +static void +QCBOREncode_AddEncodedToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + UsefulBufC Encoded); + +/** + @brief Get the encoded result. + + @param[in] pCtx The context to finish encoding with. + @param[out] pEncodedCBOR Structure in which the pointer and length of the + encoded CBOR is returned. + + @retval QCBOR_SUCCESS Encoded CBOR is returned. + + @retval QCBOR_ERR_TOO_MANY_CLOSES Nesting error + + @retval QCBOR_ERR_CLOSE_MISMATCH Nesting error + + @retval QCBOR_ERR_ARRAY_OR_MAP_STILL_OPEN Nesting error + + @retval QCBOR_ERR_BUFFER_TOO_LARGE Encoded output buffer size + + @retval QCBOR_ERR_BUFFER_TOO_SMALL Encoded output buffer size + + @retval QCBOR_ERR_ARRAY_NESTING_TOO_DEEP Implementation limit + + @retval QCBOR_ERR_ARRAY_TOO_LONG Implementation limit + + On success, the pointer and length of the encoded CBOR are returned + in @c *pEncodedCBOR. The pointer is the same pointer that was passed + in to QCBOREncode_Init(). Note that it is not const when passed to + QCBOREncode_Init(), but it is const when returned here. The length + will be smaller than or equal to the length passed in when + QCBOREncode_Init() as this is the length of the actual result, not + the size of the buffer it was written to. + + If a @c NULL was passed for @c Storage.ptr when QCBOREncode_Init() + was called, @c NULL will be returned here, but the length will be + that of the CBOR that would have been encoded. + + Encoding errors primarily manifest here as most other encoding function + do no return an error. They just set the error state in the encode + context after which no encoding function does anything. + + Three types of errors manifest here. The first type are nesting + errors where the number of @c QCBOREncode_OpenXxx() calls do not + match the number @c QCBOREncode_CloseXxx() calls. The solution is to + fix the calling code. + + The second type of error is because the buffer given is either too + small or too large. The remedy is to give a correctly sized buffer. + + The third type are due to limits in this implementation. @ref + QCBOR_ERR_ARRAY_NESTING_TOO_DEEP can be worked around by encoding the + CBOR in two (or more) phases and adding the CBOR from the first phase + to the second with @c QCBOREncode_AddEncoded(). + + If an error is returned, the buffer may have partially encoded + incorrect CBOR in it and it should not be used. Likewise, the length + may be incorrect and should not be used. + + Note that the error could have occurred in one of the many @c + QCBOREncode_AddXxx() calls long before QCBOREncode_Finish() was + called. This error handling reduces the CBOR implementation size but + makes debugging harder. + + This may be called multiple times. It will always return the same. It + can also be interleaved with calls to QCBOREncode_FinishGetSize(). + + QCBOREncode_GetErrorState() can be called to get the current + error state in order to abort encoding early as an optimization, but + calling it is is never required. + */ +QCBORError +QCBOREncode_Finish(QCBOREncodeContext *pCtx, UsefulBufC *pEncodedCBOR); + +/** + @brief Get the encoded CBOR and error status. + + @param[in] pCtx The context to finish encoding with. + @param[out] uEncodedLen The length of the encoded or potentially + encoded CBOR in bytes. + + @return The same errors as QCBOREncode_Finish(). + + This functions the same as QCBOREncode_Finish(), but only returns the + size of the encoded output. + */ +QCBORError +QCBOREncode_FinishGetSize(QCBOREncodeContext *pCtx, size_t *uEncodedLen); + +/** + @brief Indicate whether output buffer is NULL or not. + + @param[in] pCtx The encoding context. + + @return 1 if the output buffer is @c NULL. + + Sometimes a @c NULL input buffer is given to QCBOREncode_Init() so + that the size of the generated CBOR can be calculated without + allocating a buffer for it. This returns 1 when the output buffer is + NULL and 0 when it is not. +*/ +static int +QCBOREncode_IsBufferNULL(QCBOREncodeContext *pCtx); + +/** + @brief Get the encoding error state. + + @param[in] pCtx The encoding context. + + @return One of @ref QCBORError. See return values from + QCBOREncode_Finish() + + Normally encoding errors need only be handled at the end of encoding + when QCBOREncode_Finish() is called. This can be called to get the + error result before finish should there be a need to halt encoding + before QCBOREncode_Finish() is called. +*/ +static QCBORError +QCBOREncode_GetErrorState(QCBOREncodeContext *pCtx); + +/** + Encode the "head" of a CBOR data item. + + @param buffer Buffer to output the encoded head to; must be + @ref QCBOR_HEAD_BUFFER_SIZE bytes in size. + @param uMajorType One of CBOR_MAJOR_TYPE_XX. + @param uMinLen The minimum number of bytes to encode uNumber. Almost + always this is 0 to use preferred minimal encoding. If this is 4, then even the + values 0xffff and smaller will be encoded in 4 bytes. This is used primarily + when encoding a float or double put into uNumber as the leading zero bytes for + them must be encoded. + @param uNumber The numeric argument part of the CBOR head. + @return Pointer and length of the encoded head or + @ref NULLUsefulBufC if the output buffer is too small. + + Callers do not to need to call this for normal CBOR encoding. Note that it + doesn't even take a @ref QCBOREncodeContext argument. + + This encodes the major type and argument part of a data item. The + argument is an integer that is usually either the value or the length + of the data item. + + This is exposed in the public interface to allow hashing of some CBOR + data types, bstr in particular, a chunk at a time so the full CBOR + doesn't have to be encoded in a contiguous buffer. + + For example, if you have a 100,000 byte binary blob in a buffer that + needs to be a bstr encoded and then hashed. You could allocate a + 100,010 byte buffer and encode it normally. Alternatively, you can + encode the head in a 10 byte buffer with this function, hash that and + then hash the 100,000 bytes using the same hash context. + + See also QCBOREncode_AddBytesLenOnly(); + */ +UsefulBufC +QCBOREncode_EncodeHead(UsefulBuf buffer, uint8_t uMajorType, uint8_t uMinLen, + uint64_t uNumber); + +/* ========================================================================= + BEGINNING OF PRIVATE INLINE IMPLEMENTATION + ========================================================================= */ + +/** + @brief Semi-private method to add a buffer full of bytes to encoded output + + @param[in] pCtx The encoding context to add the integer to. + @param[in] uMajorType The CBOR major type of the bytes. + @param[in] Bytes The bytes to add. + + Use QCBOREncode_AddText() or QCBOREncode_AddBytes() or + QCBOREncode_AddEncoded() instead. They are inline functions that call + this and supply the correct major type. This function is public to + make the inline functions work to keep the overall code size down and + because the C language has no way to make it private. + + If this is called the major type should be @c + CBOR_MAJOR_TYPE_TEXT_STRING, @c CBOR_MAJOR_TYPE_BYTE_STRING or @c + CBOR_MAJOR_NONE_TYPE_RAW. The last one is special for adding + already-encoded CBOR. + */ +void +QCBOREncode_AddBuffer(QCBOREncodeContext *pCtx, uint8_t uMajorType, + UsefulBufC Bytes); + +/** + @brief Semi-private method to open a map, array or bstr-wrapped CBOR + + @param[in] pCtx The context to add to. + @param[in] uMajorType The major CBOR type to close + + Call QCBOREncode_OpenArray(), QCBOREncode_OpenMap() or + QCBOREncode_BstrWrap() instead of this. + */ +void +QCBOREncode_OpenMapOrArray(QCBOREncodeContext *pCtx, uint8_t uMajorType); + +/** + @brief Semi-private method to open a map, array with indefinite length + + @param[in] pCtx The context to add to. + @param[in] uMajorType The major CBOR type to close + + Call QCBOREncode_OpenArrayIndefiniteLength() or + QCBOREncode_OpenMapIndefiniteLength() instead of this. + */ +void +QCBOREncode_OpenMapOrArrayIndefiniteLength(QCBOREncodeContext *pCtx, + uint8_t uMajorType); + +/** + @brief Semi-private method to close a map, array or bstr wrapped CBOR + + @param[in] pCtx The context to add to. + @param[in] uMajorType The major CBOR type to close. + + Call QCBOREncode_CloseArray() or QCBOREncode_CloseMap() instead of this. + */ +void +QCBOREncode_CloseMapOrArray(QCBOREncodeContext *pCtx, uint8_t uMajorType); + +/** + @brief Semi-private method to close a map, array with indefinite length + + @param[in] pCtx The context to add to. + @param[in] uMajorType The major CBOR type to close. + + Call QCBOREncode_CloseArrayIndefiniteLength() or + QCBOREncode_CloseMapIndefiniteLength() instead of this. + */ +void +QCBOREncode_CloseMapOrArrayIndefiniteLength(QCBOREncodeContext *pCtx, + uint8_t uMajorType); + +/** + @brief Semi-private method to add simple types. + + @param[in] pCtx The encoding context to add the simple value to. + @param[in] uMinLen Minimum encoding size for uNum. Usually 0. + @param[in] uNum One of CBOR_SIMPLEV_FALSE through _UNDEF or other. + + This is used to add simple types like true and false. + + Call QCBOREncode_AddBool(), QCBOREncode_AddNULL(), + QCBOREncode_AddUndef() instead of this. + + This function can add simple values that are not defined by CBOR + yet. This expansion point in CBOR should not be used unless they are + standardized. + + Error handling is the same as QCBOREncode_AddInt64(). + */ +void +QCBOREncode_AddType7(QCBOREncodeContext *pCtx, uint8_t uMinLen, uint64_t uNum); + +/** + @brief Semi-private method to add bigfloats and decimal fractions. + + @param[in] pCtx The encoding context to add the value to. + @param[in] uTag The type 6 tag indicating what this is to be. + @param[in] BigNumMantissa Is @ref NULLUsefulBufC if mantissa is an + @c int64_t or the actual big number mantissa + if not. + @param[in] bBigNumIsNegative This is @c true if the big number is negative. + @param[in] nMantissa The @c int64_t mantissa if it is not a big + number. + @param[in] nExponent The exponent. + + This outputs either the @ref CBOR_TAG_DECIMAL_FRACTION or @ref + CBOR_TAG_BIGFLOAT tag. if @c uTag is @ref CBOR_TAG_INVALID64, then + this outputs the "borrowed" content format. + + The tag content output by this is an array with two members, the + exponent and then the mantissa. The mantissa can be either a big + number or an @c int64_t. + + This implementation cannot output an exponent further from 0 than + @c INT64_MAX. + + To output a mantissa that is between INT64_MAX and UINT64_MAX from 0, + it must be as a big number. + + Typically, QCBOREncode_AddDecimalFraction(), QCBOREncode_AddBigFloat(), + QCBOREncode_AddDecimalFractionBigNum() or QCBOREncode_AddBigFloatBigNum() + is called instead of this. + */ +void +QCBOREncode_AddExponentAndMantissa(QCBOREncodeContext *pCtx, uint64_t uTag, + UsefulBufC BigNumMantissa, + bool bBigNumIsNegative, int64_t nMantissa, + int64_t nExponent); + +/** + @brief Semi-private method to add only the type and length of a byte string. + + @param[in] pCtx The context to initialize. + @param[in] Bytes Pointer and length of the input data. + + This is the same as QCBOREncode_AddBytes() except it only adds the + CBOR encoding for the type and the length. It doesn't actually add + the bytes. You can't actually produce correct CBOR with this and the + rest of this API. It is only used for a special case where + the valid CBOR is created manually by putting this type and length in + and then adding the actual bytes. In particular, when only a hash of + the encoded CBOR is needed, where the type and header are hashed + separately and then the bytes is hashed. This makes it possible to + implement COSE Sign1 with only one copy of the payload in the output + buffer, rather than two, roughly cutting memory use in half. + + This is only used for this odd case, but this is a supported + tested function. + + See also QCBOREncode_EncodeHead(). +*/ +static inline void +QCBOREncode_AddBytesLenOnly(QCBOREncodeContext *pCtx, UsefulBufC Bytes); + +static inline void +QCBOREncode_AddBytesLenOnlyToMap(QCBOREncodeContext *pCtx, const char *szLabel, + UsefulBufC Bytes); + +static inline void +QCBOREncode_AddBytesLenOnlyToMapN(QCBOREncodeContext *pCtx, int64_t nLabel, + UsefulBufC Bytes); + +static inline void +QCBOREncode_AddInt64ToMap(QCBOREncodeContext *pMe, const char *szLabel, + int64_t uNum) +{ + // Use _AddBuffer() because _AddSZString() is defined below, not above + QCBOREncode_AddBuffer(pMe, CBOR_MAJOR_TYPE_TEXT_STRING, + UsefulBuf_FromSZ(szLabel)); + QCBOREncode_AddInt64(pMe, uNum); +} + +static inline void +QCBOREncode_AddInt64ToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + int64_t uNum) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddInt64(pMe, uNum); +} + +static inline void +QCBOREncode_AddUInt64ToMap(QCBOREncodeContext *pMe, const char *szLabel, + uint64_t uNum) +{ + // Use _AddBuffer() because _AddSZString() is defined below, not above + QCBOREncode_AddBuffer(pMe, CBOR_MAJOR_TYPE_TEXT_STRING, + UsefulBuf_FromSZ(szLabel)); + QCBOREncode_AddUInt64(pMe, uNum); +} + +static inline void +QCBOREncode_AddUInt64ToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + uint64_t uNum) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddUInt64(pMe, uNum); +} + +static inline void +QCBOREncode_AddText(QCBOREncodeContext *pMe, UsefulBufC Text) +{ + QCBOREncode_AddBuffer(pMe, CBOR_MAJOR_TYPE_TEXT_STRING, Text); +} + +static inline void +QCBOREncode_AddTextToMap(QCBOREncodeContext *pMe, const char *szLabel, + UsefulBufC Text) +{ + QCBOREncode_AddText(pMe, UsefulBuf_FromSZ(szLabel)); + QCBOREncode_AddText(pMe, Text); +} + +static inline void +QCBOREncode_AddTextToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + UsefulBufC Text) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddText(pMe, Text); +} + +inline static void +QCBOREncode_AddSZString(QCBOREncodeContext *pMe, const char *szString) +{ + QCBOREncode_AddText(pMe, UsefulBuf_FromSZ(szString)); +} + +static inline void +QCBOREncode_AddSZStringToMap(QCBOREncodeContext *pMe, const char *szLabel, + const char *szString) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddSZString(pMe, szString); +} + +static inline void +QCBOREncode_AddSZStringToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + const char *szString) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddSZString(pMe, szString); +} + +#ifndef USEFULBUF_DISABLE_ALL_FLOAT +static inline void +QCBOREncode_AddDoubleToMap(QCBOREncodeContext *pMe, const char *szLabel, + double dNum) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddDouble(pMe, dNum); +} + +static inline void +QCBOREncode_AddDoubleToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + double dNum) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddDouble(pMe, dNum); +} + +static inline void +QCBOREncode_AddFloatToMap(QCBOREncodeContext *pMe, const char *szLabel, + float dNum) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddFloat(pMe, dNum); +} + +static inline void +QCBOREncode_AddFloatToMapN(QCBOREncodeContext *pMe, int64_t nLabel, float fNum) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddFloat(pMe, fNum); +} + +static inline void +QCBOREncode_AddDoubleNoPreferredToMap(QCBOREncodeContext *pMe, + const char *szLabel, double dNum) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddDoubleNoPreferred(pMe, dNum); +} + +static inline void +QCBOREncode_AddDoubleNoPreferredToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + double dNum) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddDoubleNoPreferred(pMe, dNum); +} + +static inline void +QCBOREncode_AddFloatNoPreferredToMap(QCBOREncodeContext *pMe, + const char *szLabel, float dNum) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddFloatNoPreferred(pMe, dNum); +} + +static inline void +QCBOREncode_AddFloatNoPreferredToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + float dNum) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddFloatNoPreferred(pMe, dNum); +} +#endif /* USEFULBUF_DISABLE_ALL_FLOAT */ + +static inline void +QCBOREncode_AddTDateEpoch(QCBOREncodeContext *pMe, uint8_t uTag, int64_t nDate) +{ + if (uTag == QCBOR_ENCODE_AS_TAG) { + QCBOREncode_AddTag(pMe, CBOR_TAG_DATE_EPOCH); + } + QCBOREncode_AddInt64(pMe, nDate); +} + +static inline void +QCBOREncode_AddTDateEpochToMapSZ(QCBOREncodeContext *pMe, const char *szLabel, + uint8_t uTag, int64_t nDate) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddTDateEpoch(pMe, uTag, nDate); +} + +static inline void +QCBOREncode_AddTDateEpochToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + uint8_t uTag, int64_t nDate) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddTDateEpoch(pMe, uTag, nDate); +} + +static inline void +QCBOREncode_AddDateEpoch(QCBOREncodeContext *pMe, int64_t nDate) +{ + QCBOREncode_AddTDateEpoch(pMe, QCBOR_ENCODE_AS_TAG, nDate); +} + +static inline void +QCBOREncode_AddDateEpochToMap(QCBOREncodeContext *pMe, const char *szLabel, + int64_t nDate) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddDateEpoch(pMe, nDate); +} + +static inline void +QCBOREncode_AddDateEpochToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + int64_t nDate) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddDateEpoch(pMe, nDate); +} + +static inline void +QCBOREncode_AddTDaysEpoch(QCBOREncodeContext *pMe, uint8_t uTag, int64_t nDays) +{ + if (uTag == QCBOR_ENCODE_AS_TAG) { + QCBOREncode_AddTag(pMe, CBOR_TAG_DAYS_EPOCH); + } + QCBOREncode_AddInt64(pMe, nDays); +} + +static inline void +QCBOREncode_AddTDaysEpochToMapSZ(QCBOREncodeContext *pMe, const char *szLabel, + uint8_t uTag, int64_t nDays) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddTDaysEpoch(pMe, uTag, nDays); +} + +static inline void +QCBOREncode_AddTDaysEpochToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + uint8_t uTag, int64_t nDays) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddTDaysEpoch(pMe, uTag, nDays); +} + +static inline void +QCBOREncode_AddBytes(QCBOREncodeContext *pMe, UsefulBufC Bytes) +{ + QCBOREncode_AddBuffer(pMe, CBOR_MAJOR_TYPE_BYTE_STRING, Bytes); +} + +static inline void +QCBOREncode_AddBytesToMap(QCBOREncodeContext *pMe, const char *szLabel, + UsefulBufC Bytes) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddBytes(pMe, Bytes); +} + +static inline void +QCBOREncode_AddBytesToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + UsefulBufC Bytes) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddBytes(pMe, Bytes); +} + +static inline void +QCBOREncode_OpenBytesInMapSZ(QCBOREncodeContext *pMe, const char *szLabel, + UsefulBuf *pPlace) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_OpenBytes(pMe, pPlace); +} + +static inline void +QCBOREncode_OpenBytesInMapN(QCBOREncodeContext *pMe, int64_t nLabel, + UsefulBuf *pPlace) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_OpenBytes(pMe, pPlace); +} + +static inline void +QCBOREncode_AddBytesLenOnly(QCBOREncodeContext *pMe, UsefulBufC Bytes) +{ + QCBOREncode_AddBuffer(pMe, CBOR_MAJOR_NONE_TYPE_BSTR_LEN_ONLY, Bytes); +} + +static inline void +QCBOREncode_AddBytesLenOnlyToMap(QCBOREncodeContext *pMe, const char *szLabel, + UsefulBufC Bytes) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddBytesLenOnly(pMe, Bytes); +} + +static inline void +QCBOREncode_AddBytesLenOnlyToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + UsefulBufC Bytes) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddBytesLenOnly(pMe, Bytes); +} + +static inline void +QCBOREncode_AddTBinaryUUID(QCBOREncodeContext *pMe, uint8_t uTagRequirement, + UsefulBufC Bytes) +{ + if (uTagRequirement == QCBOR_ENCODE_AS_TAG) { + QCBOREncode_AddTag(pMe, CBOR_TAG_BIN_UUID); + } + QCBOREncode_AddBytes(pMe, Bytes); +} + +static inline void +QCBOREncode_AddTBinaryUUIDToMapSZ(QCBOREncodeContext *pMe, const char *szLabel, + uint8_t uTagRequirement, UsefulBufC Bytes) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddTBinaryUUID(pMe, uTagRequirement, Bytes); +} + +static inline void +QCBOREncode_AddTBinaryUUIDToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + uint8_t uTagRequirement, UsefulBufC Bytes) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddTBinaryUUID(pMe, uTagRequirement, Bytes); +} + +static inline void +QCBOREncode_AddBinaryUUID(QCBOREncodeContext *pMe, UsefulBufC Bytes) +{ + QCBOREncode_AddTBinaryUUID(pMe, QCBOR_ENCODE_AS_TAG, Bytes); +} + +static inline void +QCBOREncode_AddBinaryUUIDToMap(QCBOREncodeContext *pMe, const char *szLabel, + UsefulBufC Bytes) +{ + QCBOREncode_AddTBinaryUUIDToMapSZ(pMe, szLabel, QCBOR_ENCODE_AS_TAG, + Bytes); +} + +static inline void +QCBOREncode_AddBinaryUUIDToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + UsefulBufC Bytes) +{ + QCBOREncode_AddTBinaryUUIDToMapN(pMe, nLabel, QCBOR_ENCODE_AS_TAG, + Bytes); +} + +static inline void +QCBOREncode_AddTPositiveBignum(QCBOREncodeContext *pMe, uint8_t uTagRequirement, + UsefulBufC Bytes) +{ + if (uTagRequirement == QCBOR_ENCODE_AS_TAG) { + QCBOREncode_AddTag(pMe, CBOR_TAG_POS_BIGNUM); + } + QCBOREncode_AddBytes(pMe, Bytes); +} + +static inline void +QCBOREncode_AddTPositiveBignumToMapSZ(QCBOREncodeContext *pMe, + const char *szLabel, + uint8_t uTagRequirement, UsefulBufC Bytes) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddTPositiveBignum(pMe, uTagRequirement, Bytes); +} + +static inline void +QCBOREncode_AddTPositiveBignumToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + uint8_t uTagRequirement, UsefulBufC Bytes) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddTPositiveBignum(pMe, uTagRequirement, Bytes); +} + +static inline void +QCBOREncode_AddPositiveBignum(QCBOREncodeContext *pMe, UsefulBufC Bytes) +{ + QCBOREncode_AddTPositiveBignum(pMe, QCBOR_ENCODE_AS_TAG, Bytes); +} + +static inline void +QCBOREncode_AddPositiveBignumToMap(QCBOREncodeContext *pMe, const char *szLabel, + UsefulBufC Bytes) +{ + QCBOREncode_AddTPositiveBignumToMapSZ(pMe, szLabel, QCBOR_ENCODE_AS_TAG, + Bytes); +} + +static inline void +QCBOREncode_AddPositiveBignumToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + UsefulBufC Bytes) +{ + QCBOREncode_AddTPositiveBignumToMapN(pMe, nLabel, QCBOR_ENCODE_AS_TAG, + Bytes); +} + +static inline void +QCBOREncode_AddTNegativeBignum(QCBOREncodeContext *pMe, uint8_t uTagRequirement, + UsefulBufC Bytes) +{ + if (uTagRequirement == QCBOR_ENCODE_AS_TAG) { + QCBOREncode_AddTag(pMe, CBOR_TAG_NEG_BIGNUM); + } + QCBOREncode_AddBytes(pMe, Bytes); +} + +static inline void +QCBOREncode_AddTNegativeBignumToMapSZ(QCBOREncodeContext *pMe, + const char *szLabel, + uint8_t uTagRequirement, UsefulBufC Bytes) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddTNegativeBignum(pMe, uTagRequirement, Bytes); +} + +static inline void +QCBOREncode_AddTNegativeBignumToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + uint8_t uTagRequirement, UsefulBufC Bytes) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddTNegativeBignum(pMe, uTagRequirement, Bytes); +} + +static inline void +QCBOREncode_AddNegativeBignum(QCBOREncodeContext *pMe, UsefulBufC Bytes) +{ + QCBOREncode_AddTNegativeBignum(pMe, QCBOR_ENCODE_AS_TAG, Bytes); +} + +static inline void +QCBOREncode_AddNegativeBignumToMap(QCBOREncodeContext *pMe, const char *szLabel, + UsefulBufC Bytes) +{ + QCBOREncode_AddTNegativeBignumToMapSZ(pMe, szLabel, QCBOR_ENCODE_AS_TAG, + Bytes); +} + +static inline void +QCBOREncode_AddNegativeBignumToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + UsefulBufC Bytes) +{ + QCBOREncode_AddTNegativeBignumToMapN(pMe, nLabel, QCBOR_ENCODE_AS_TAG, + Bytes); +} + +#ifndef QCBOR_DISABLE_EXP_AND_MANTISSA + +static inline void +QCBOREncode_AddTDecimalFraction(QCBOREncodeContext *pMe, + uint8_t uTagRequirement, int64_t nMantissa, + int64_t nBase10Exponent) +{ + uint64_t uTag; + if (uTagRequirement == QCBOR_ENCODE_AS_TAG) { + uTag = CBOR_TAG_DECIMAL_FRACTION; + } else { + uTag = CBOR_TAG_INVALID64; + } + QCBOREncode_AddExponentAndMantissa(pMe, uTag, NULLUsefulBufC, false, + nMantissa, nBase10Exponent); +} + +static inline void +QCBOREncode_AddTDecimalFractionToMapSZ(QCBOREncodeContext *pMe, + const char *szLabel, + uint8_t uTagRequirement, + int64_t nMantissa, + int64_t nBase10Exponent) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddTDecimalFraction(pMe, uTagRequirement, nMantissa, + nBase10Exponent); +} + +static inline void +QCBOREncode_AddTDecimalFractionToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + uint8_t uTagRequirement, + int64_t nMantissa, + int64_t nBase10Exponent) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddTDecimalFraction(pMe, uTagRequirement, nMantissa, + nBase10Exponent); +} + +static inline void +QCBOREncode_AddDecimalFraction(QCBOREncodeContext *pMe, int64_t nMantissa, + int64_t nBase10Exponent) +{ + QCBOREncode_AddTDecimalFraction(pMe, QCBOR_ENCODE_AS_TAG, nMantissa, + nBase10Exponent); +} + +static inline void +QCBOREncode_AddDecimalFractionToMap(QCBOREncodeContext *pMe, + const char *szLabel, int64_t nMantissa, + int64_t nBase10Exponent) +{ + QCBOREncode_AddTDecimalFractionToMapSZ( + pMe, szLabel, QCBOR_ENCODE_AS_TAG, nMantissa, nBase10Exponent); +} + +static inline void +QCBOREncode_AddDecimalFractionToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + int64_t nMantissa, int64_t nBase10Exponent) +{ + QCBOREncode_AddTDecimalFractionToMapN(pMe, nLabel, QCBOR_ENCODE_AS_TAG, + nMantissa, nBase10Exponent); +} + +static inline void +QCBOREncode_AddTDecimalFractionBigNum(QCBOREncodeContext *pMe, + uint8_t uTagRequirement, + UsefulBufC Mantissa, bool bIsNegative, + int64_t nBase10Exponent) +{ + uint64_t uTag; + if (uTagRequirement == QCBOR_ENCODE_AS_TAG) { + uTag = CBOR_TAG_DECIMAL_FRACTION; + } else { + uTag = CBOR_TAG_INVALID64; + } + QCBOREncode_AddExponentAndMantissa(pMe, uTag, Mantissa, bIsNegative, 0, + nBase10Exponent); +} + +static inline void +QCBOREncode_AddTDecimalFractionBigNumToMapSZ( + QCBOREncodeContext *pMe, const char *szLabel, uint8_t uTagRequirement, + UsefulBufC Mantissa, bool bIsNegative, int64_t nBase10Exponent) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddTDecimalFractionBigNum(pMe, uTagRequirement, Mantissa, + bIsNegative, nBase10Exponent); +} + +static inline void +QCBOREncode_AddTDecimalFractionBigNumToMapN( + QCBOREncodeContext *pMe, int64_t nLabel, uint8_t uTagRequirement, + UsefulBufC Mantissa, bool bIsNegative, int64_t nBase10Exponent) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddTDecimalFractionBigNum(pMe, uTagRequirement, Mantissa, + bIsNegative, nBase10Exponent); +} + +static inline void +QCBOREncode_AddDecimalFractionBigNum(QCBOREncodeContext *pMe, + UsefulBufC Mantissa, bool bIsNegative, + int64_t nBase10Exponent) +{ + QCBOREncode_AddTDecimalFractionBigNum(pMe, QCBOR_ENCODE_AS_TAG, + Mantissa, bIsNegative, + nBase10Exponent); +} + +static inline void +QCBOREncode_AddDecimalFractionBigNumToMapSZ(QCBOREncodeContext *pMe, + const char *szLabel, + UsefulBufC Mantissa, + bool bIsNegative, + int64_t nBase10Exponent) +{ + QCBOREncode_AddTDecimalFractionBigNumToMapSZ(pMe, szLabel, + QCBOR_ENCODE_AS_TAG, + Mantissa, bIsNegative, + nBase10Exponent); +} + +static inline void +QCBOREncode_AddDecimalFractionBigNumToMapN(QCBOREncodeContext *pMe, + int64_t nLabel, UsefulBufC Mantissa, + bool bIsNegative, + int64_t nBase2Exponent) +{ + QCBOREncode_AddTDecimalFractionBigNumToMapN(pMe, nLabel, + QCBOR_ENCODE_AS_TAG, + Mantissa, bIsNegative, + nBase2Exponent); +} + +static inline void +QCBOREncode_AddTBigFloat(QCBOREncodeContext *pMe, uint8_t uTagRequirement, + int64_t nMantissa, int64_t nBase2Exponent) +{ + uint64_t uTag; + if (uTagRequirement == QCBOR_ENCODE_AS_TAG) { + uTag = CBOR_TAG_BIGFLOAT; + } else { + uTag = CBOR_TAG_INVALID64; + } + QCBOREncode_AddExponentAndMantissa(pMe, uTag, NULLUsefulBufC, false, + nMantissa, nBase2Exponent); +} + +static inline void +QCBOREncode_AddTBigFloatToMapSZ(QCBOREncodeContext *pMe, const char *szLabel, + uint8_t uTagRequirement, int64_t nMantissa, + int64_t nBase2Exponent) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddTBigFloat(pMe, uTagRequirement, nMantissa, + nBase2Exponent); +} + +static inline void +QCBOREncode_AddTBigFloatToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + uint8_t uTagRequirement, int64_t nMantissa, + int64_t nBase2Exponent) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddTBigFloat(pMe, uTagRequirement, nMantissa, + nBase2Exponent); +} + +static inline void +QCBOREncode_AddBigFloat(QCBOREncodeContext *pMe, int64_t nMantissa, + int64_t nBase2Exponent) +{ + QCBOREncode_AddTBigFloat(pMe, QCBOR_ENCODE_AS_TAG, nMantissa, + nBase2Exponent); +} + +static inline void +QCBOREncode_AddBigFloatToMap(QCBOREncodeContext *pMe, const char *szLabel, + int64_t nMantissa, int64_t nBase2Exponent) +{ + QCBOREncode_AddTBigFloatToMapSZ(pMe, szLabel, QCBOR_ENCODE_AS_TAG, + nMantissa, nBase2Exponent); +} + +static inline void +QCBOREncode_AddBigFloatToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + int64_t nMantissa, int64_t nBase2Exponent) +{ + QCBOREncode_AddTBigFloatToMapN(pMe, nLabel, QCBOR_ENCODE_AS_TAG, + nMantissa, nBase2Exponent); +} + +static inline void +QCBOREncode_AddTBigFloatBigNum(QCBOREncodeContext *pMe, uint8_t uTagRequirement, + UsefulBufC Mantissa, bool bIsNegative, + int64_t nBase2Exponent) +{ + uint64_t uTag; + if (uTagRequirement == QCBOR_ENCODE_AS_TAG) { + uTag = CBOR_TAG_BIGFLOAT; + } else { + uTag = CBOR_TAG_INVALID64; + } + QCBOREncode_AddExponentAndMantissa(pMe, uTag, Mantissa, bIsNegative, 0, + nBase2Exponent); +} + +static inline void +QCBOREncode_AddTBigFloatBigNumToMapSZ(QCBOREncodeContext *pMe, + const char *szLabel, + uint8_t uTagRequirement, + UsefulBufC Mantissa, bool bIsNegative, + int64_t nBase2Exponent) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddTBigFloatBigNum(pMe, uTagRequirement, Mantissa, + bIsNegative, nBase2Exponent); +} + +static inline void +QCBOREncode_AddTBigFloatBigNumToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + uint8_t uTagRequirement, + UsefulBufC Mantissa, bool bIsNegative, + int64_t nBase2Exponent) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddTBigFloatBigNum(pMe, uTagRequirement, Mantissa, + bIsNegative, nBase2Exponent); +} + +static inline void +QCBOREncode_AddBigFloatBigNum(QCBOREncodeContext *pMe, UsefulBufC Mantissa, + bool bIsNegative, int64_t nBase2Exponent) +{ + QCBOREncode_AddTBigFloatBigNum(pMe, QCBOR_ENCODE_AS_TAG, Mantissa, + bIsNegative, nBase2Exponent); +} + +static inline void +QCBOREncode_AddBigFloatBigNumToMap(QCBOREncodeContext *pMe, const char *szLabel, + UsefulBufC Mantissa, bool bIsNegative, + int64_t nBase2Exponent) +{ + QCBOREncode_AddTBigFloatBigNumToMapSZ(pMe, szLabel, QCBOR_ENCODE_AS_TAG, + Mantissa, bIsNegative, + nBase2Exponent); +} + +static inline void +QCBOREncode_AddBigFloatBigNumToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + UsefulBufC Mantissa, bool bIsNegative, + int64_t nBase2Exponent) +{ + QCBOREncode_AddTBigFloatBigNumToMapN(pMe, nLabel, QCBOR_ENCODE_AS_TAG, + Mantissa, bIsNegative, + nBase2Exponent); +} +#endif /* QCBOR_DISABLE_EXP_AND_MANTISSA */ + +static inline void +QCBOREncode_AddTURI(QCBOREncodeContext *pMe, uint8_t uTagRequirement, + UsefulBufC URI) +{ + if (uTagRequirement == QCBOR_ENCODE_AS_TAG) { + QCBOREncode_AddTag(pMe, CBOR_TAG_URI); + } + QCBOREncode_AddText(pMe, URI); +} + +static inline void +QCBOREncode_AddTURIToMapSZ(QCBOREncodeContext *pMe, const char *szLabel, + uint8_t uTagRequirement, UsefulBufC URI) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddTURI(pMe, uTagRequirement, URI); +} + +static inline void +QCBOREncode_AddTURIToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + uint8_t uTagRequirement, UsefulBufC URI) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddTURI(pMe, uTagRequirement, URI); +} + +static inline void +QCBOREncode_AddURI(QCBOREncodeContext *pMe, UsefulBufC URI) +{ + QCBOREncode_AddTURI(pMe, QCBOR_ENCODE_AS_TAG, URI); +} + +static inline void +QCBOREncode_AddURIToMap(QCBOREncodeContext *pMe, const char *szLabel, + UsefulBufC URI) +{ + QCBOREncode_AddTURIToMapSZ(pMe, szLabel, QCBOR_ENCODE_AS_TAG, URI); +} + +static inline void +QCBOREncode_AddURIToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + UsefulBufC URI) +{ + QCBOREncode_AddTURIToMapN(pMe, nLabel, QCBOR_ENCODE_AS_TAG, URI); +} + +static inline void +QCBOREncode_AddTB64Text(QCBOREncodeContext *pMe, uint8_t uTagRequirement, + UsefulBufC B64Text) +{ + if (uTagRequirement == QCBOR_ENCODE_AS_TAG) { + QCBOREncode_AddTag(pMe, CBOR_TAG_B64); + } + QCBOREncode_AddText(pMe, B64Text); +} + +static inline void +QCBOREncode_AddTB64TextToMapSZ(QCBOREncodeContext *pMe, const char *szLabel, + uint8_t uTagRequirement, UsefulBufC B64Text) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddTB64Text(pMe, uTagRequirement, B64Text); +} + +static inline void +QCBOREncode_AddTB64TextToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + uint8_t uTagRequirement, UsefulBufC B64Text) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddTB64Text(pMe, uTagRequirement, B64Text); +} + +static inline void +QCBOREncode_AddB64Text(QCBOREncodeContext *pMe, UsefulBufC B64Text) +{ + QCBOREncode_AddTB64Text(pMe, QCBOR_ENCODE_AS_TAG, B64Text); +} + +static inline void +QCBOREncode_AddB64TextToMap(QCBOREncodeContext *pMe, const char *szLabel, + UsefulBufC B64Text) +{ + QCBOREncode_AddTB64TextToMapSZ(pMe, szLabel, QCBOR_ENCODE_AS_TAG, + B64Text); +} + +static inline void +QCBOREncode_AddB64TextToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + UsefulBufC B64Text) +{ + QCBOREncode_AddTB64TextToMapN(pMe, nLabel, QCBOR_ENCODE_AS_TAG, + B64Text); +} + +static inline void +QCBOREncode_AddTB64URLText(QCBOREncodeContext *pMe, uint8_t uTagRequirement, + UsefulBufC B64Text) +{ + if (uTagRequirement == QCBOR_ENCODE_AS_TAG) { + QCBOREncode_AddTag(pMe, CBOR_TAG_B64URL); + } + QCBOREncode_AddText(pMe, B64Text); +} + +static inline void +QCBOREncode_AddTB64URLTextToMapSZ(QCBOREncodeContext *pMe, const char *szLabel, + uint8_t uTagRequirement, UsefulBufC B64Text) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddTB64URLText(pMe, uTagRequirement, B64Text); +} + +static inline void +QCBOREncode_AddTB64URLTextToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + uint8_t uTagRequirement, UsefulBufC B64Text) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddTB64URLText(pMe, uTagRequirement, B64Text); +} + +static inline void +QCBOREncode_AddB64URLText(QCBOREncodeContext *pMe, UsefulBufC B64Text) +{ + QCBOREncode_AddTB64URLText(pMe, QCBOR_ENCODE_AS_TAG, B64Text); +} + +static inline void +QCBOREncode_AddB64URLTextToMap(QCBOREncodeContext *pMe, const char *szLabel, + UsefulBufC B64Text) +{ + QCBOREncode_AddTB64URLTextToMapSZ(pMe, szLabel, QCBOR_ENCODE_AS_TAG, + B64Text); +} + +static inline void +QCBOREncode_AddB64URLTextToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + UsefulBufC B64Text) +{ + QCBOREncode_AddTB64URLTextToMapN(pMe, nLabel, QCBOR_ENCODE_AS_TAG, + B64Text); +} + +static inline void +QCBOREncode_AddTRegex(QCBOREncodeContext *pMe, uint8_t uTagRequirement, + UsefulBufC Bytes) +{ + if (uTagRequirement == QCBOR_ENCODE_AS_TAG) { + QCBOREncode_AddTag(pMe, CBOR_TAG_REGEX); + } + QCBOREncode_AddText(pMe, Bytes); +} + +static inline void +QCBOREncode_AddTRegexToMapSZ(QCBOREncodeContext *pMe, const char *szLabel, + uint8_t uTagRequirement, UsefulBufC Bytes) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddTRegex(pMe, uTagRequirement, Bytes); +} + +static inline void +QCBOREncode_AddTRegexToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + uint8_t uTagRequirement, UsefulBufC Bytes) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddTRegex(pMe, uTagRequirement, Bytes); +} + +static inline void +QCBOREncode_AddRegex(QCBOREncodeContext *pMe, UsefulBufC Bytes) +{ + QCBOREncode_AddTRegex(pMe, QCBOR_ENCODE_AS_TAG, Bytes); +} + +static inline void +QCBOREncode_AddRegexToMap(QCBOREncodeContext *pMe, const char *szLabel, + UsefulBufC Bytes) +{ + QCBOREncode_AddTRegexToMapSZ(pMe, szLabel, QCBOR_ENCODE_AS_TAG, Bytes); +} + +static inline void +QCBOREncode_AddRegexToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + UsefulBufC Bytes) +{ + QCBOREncode_AddTRegexToMapN(pMe, nLabel, QCBOR_ENCODE_AS_TAG, Bytes); +} + +static inline void +QCBOREncode_AddTMIMEData(QCBOREncodeContext *pMe, uint8_t uTagRequirement, + UsefulBufC MIMEData) +{ + if (uTagRequirement == QCBOR_ENCODE_AS_TAG) { + QCBOREncode_AddTag(pMe, CBOR_TAG_BINARY_MIME); + } + QCBOREncode_AddBytes(pMe, MIMEData); +} + +static inline void +QCBOREncode_AddTMIMEDataToMapSZ(QCBOREncodeContext *pMe, const char *szLabel, + uint8_t uTagRequirement, UsefulBufC MIMEData) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddTMIMEData(pMe, uTagRequirement, MIMEData); +} + +static inline void +QCBOREncode_AddTMIMEDataToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + uint8_t uTagRequirement, UsefulBufC MIMEData) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddTMIMEData(pMe, uTagRequirement, MIMEData); +} + +static inline void +QCBOREncode_AddMIMEData(QCBOREncodeContext *pMe, UsefulBufC MIMEData) +{ + QCBOREncode_AddTMIMEData(pMe, QCBOR_ENCODE_AS_TAG, MIMEData); +} + +static inline void +QCBOREncode_AddMIMEDataToMap(QCBOREncodeContext *pMe, const char *szLabel, + UsefulBufC MIMEData) +{ + QCBOREncode_AddTMIMEDataToMapSZ(pMe, szLabel, QCBOR_ENCODE_AS_TAG, + MIMEData); +} + +static inline void +QCBOREncode_AddMIMEDataToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + UsefulBufC MIMEData) +{ + QCBOREncode_AddTMIMEDataToMapN(pMe, nLabel, QCBOR_ENCODE_AS_TAG, + MIMEData); +} + +static inline void +QCBOREncode_AddTDateString(QCBOREncodeContext *pMe, uint8_t uTagRequirement, + const char *szDate) +{ + if (uTagRequirement == QCBOR_ENCODE_AS_TAG) { + QCBOREncode_AddTag(pMe, CBOR_TAG_DATE_STRING); + } + QCBOREncode_AddSZString(pMe, szDate); +} + +static inline void +QCBOREncode_AddTDateStringToMapSZ(QCBOREncodeContext *pMe, const char *szLabel, + uint8_t uTagRequirement, const char *szDate) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddTDateString(pMe, uTagRequirement, szDate); +} + +static inline void +QCBOREncode_AddTDateStringToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + uint8_t uTagRequirement, const char *szDate) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddTDateString(pMe, uTagRequirement, szDate); +} + +static inline void +QCBOREncode_AddDateString(QCBOREncodeContext *pMe, const char *szDate) +{ + QCBOREncode_AddTDateString(pMe, QCBOR_ENCODE_AS_TAG, szDate); +} + +static inline void +QCBOREncode_AddDateStringToMap(QCBOREncodeContext *pMe, const char *szLabel, + const char *szDate) +{ + QCBOREncode_AddTDateStringToMapSZ(pMe, szLabel, QCBOR_ENCODE_AS_TAG, + szDate); +} + +static inline void +QCBOREncode_AddDateStringToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + const char *szDate) +{ + QCBOREncode_AddTDateStringToMapN(pMe, nLabel, QCBOR_ENCODE_AS_TAG, + szDate); +} + +static inline void +QCBOREncode_AddTDaysString(QCBOREncodeContext *pMe, uint8_t uTagRequirement, + const char *szDate) +{ + if (uTagRequirement == QCBOR_ENCODE_AS_TAG) { + QCBOREncode_AddTag(pMe, CBOR_TAG_DAYS_STRING); + } + QCBOREncode_AddSZString(pMe, szDate); +} + +static inline void +QCBOREncode_AddTDaysStringToMapSZ(QCBOREncodeContext *pMe, const char *szLabel, + uint8_t uTagRequirement, const char *szDate) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddTDaysString(pMe, uTagRequirement, szDate); +} + +static inline void +QCBOREncode_AddTDaysStringToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + uint8_t uTagRequirement, const char *szDate) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddTDaysString(pMe, uTagRequirement, szDate); +} + +static inline void +QCBOREncode_AddSimple(QCBOREncodeContext *pMe, uint64_t uNum) +{ + QCBOREncode_AddType7(pMe, 0, uNum); +} + +static inline void +QCBOREncode_AddSimpleToMap(QCBOREncodeContext *pMe, const char *szLabel, + uint8_t uSimple) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddSimple(pMe, uSimple); +} + +static inline void +QCBOREncode_AddSimpleToMapN(QCBOREncodeContext *pMe, int nLabel, + uint8_t uSimple) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddSimple(pMe, uSimple); +} + +static inline void +QCBOREncode_AddBool(QCBOREncodeContext *pMe, bool b) +{ + uint8_t uSimple = CBOR_SIMPLEV_FALSE; + if (b) { + uSimple = CBOR_SIMPLEV_TRUE; + } + QCBOREncode_AddSimple(pMe, uSimple); +} + +static inline void +QCBOREncode_AddBoolToMap(QCBOREncodeContext *pMe, const char *szLabel, bool b) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddBool(pMe, b); +} + +static inline void +QCBOREncode_AddBoolToMapN(QCBOREncodeContext *pMe, int64_t nLabel, bool b) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddBool(pMe, b); +} + +static inline void +QCBOREncode_AddNULL(QCBOREncodeContext *pMe) +{ + QCBOREncode_AddSimple(pMe, CBOR_SIMPLEV_NULL); +} + +static inline void +QCBOREncode_AddNULLToMap(QCBOREncodeContext *pMe, const char *szLabel) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddNULL(pMe); +} + +static inline void +QCBOREncode_AddNULLToMapN(QCBOREncodeContext *pMe, int64_t nLabel) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddNULL(pMe); +} + +static inline void +QCBOREncode_AddUndef(QCBOREncodeContext *pMe) +{ + QCBOREncode_AddSimple(pMe, CBOR_SIMPLEV_UNDEF); +} + +static inline void +QCBOREncode_AddUndefToMap(QCBOREncodeContext *pMe, const char *szLabel) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddUndef(pMe); +} + +static inline void +QCBOREncode_AddUndefToMapN(QCBOREncodeContext *pMe, int64_t nLabel) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddUndef(pMe); +} + +static inline void +QCBOREncode_OpenArray(QCBOREncodeContext *pMe) +{ + QCBOREncode_OpenMapOrArray(pMe, CBOR_MAJOR_TYPE_ARRAY); +} + +static inline void +QCBOREncode_OpenArrayInMap(QCBOREncodeContext *pMe, const char *szLabel) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_OpenArray(pMe); +} + +static inline void +QCBOREncode_OpenArrayInMapN(QCBOREncodeContext *pMe, int64_t nLabel) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_OpenArray(pMe); +} + +static inline void +QCBOREncode_CloseArray(QCBOREncodeContext *pMe) +{ + QCBOREncode_CloseMapOrArray(pMe, CBOR_MAJOR_TYPE_ARRAY); +} + +static inline void +QCBOREncode_OpenMap(QCBOREncodeContext *pMe) +{ + QCBOREncode_OpenMapOrArray(pMe, CBOR_MAJOR_TYPE_MAP); +} + +static inline void +QCBOREncode_OpenMapInMap(QCBOREncodeContext *pMe, const char *szLabel) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_OpenMap(pMe); +} + +static inline void +QCBOREncode_OpenMapInMapN(QCBOREncodeContext *pMe, int64_t nLabel) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_OpenMap(pMe); +} + +static inline void +QCBOREncode_CloseMap(QCBOREncodeContext *pMe) +{ + QCBOREncode_CloseMapOrArray(pMe, CBOR_MAJOR_TYPE_MAP); +} + +static inline void +QCBOREncode_OpenArrayIndefiniteLength(QCBOREncodeContext *pMe) +{ + QCBOREncode_OpenMapOrArrayIndefiniteLength( + pMe, CBOR_MAJOR_NONE_TYPE_ARRAY_INDEFINITE_LEN); +} + +static inline void +QCBOREncode_OpenArrayIndefiniteLengthInMap(QCBOREncodeContext *pMe, + const char *szLabel) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_OpenArrayIndefiniteLength(pMe); +} + +static inline void +QCBOREncode_OpenArrayIndefiniteLengthInMapN(QCBOREncodeContext *pMe, + int64_t nLabel) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_OpenArrayIndefiniteLength(pMe); +} + +static inline void +QCBOREncode_CloseArrayIndefiniteLength(QCBOREncodeContext *pMe) +{ + QCBOREncode_CloseMapOrArrayIndefiniteLength( + pMe, CBOR_MAJOR_NONE_TYPE_ARRAY_INDEFINITE_LEN); +} + +static inline void +QCBOREncode_OpenMapIndefiniteLength(QCBOREncodeContext *pMe) +{ + QCBOREncode_OpenMapOrArrayIndefiniteLength( + pMe, CBOR_MAJOR_NONE_TYPE_MAP_INDEFINITE_LEN); +} + +static inline void +QCBOREncode_OpenMapIndefiniteLengthInMap(QCBOREncodeContext *pMe, + const char *szLabel) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_OpenMapIndefiniteLength(pMe); +} + +static inline void +QCBOREncode_OpenMapIndefiniteLengthInMapN(QCBOREncodeContext *pMe, + int64_t nLabel) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_OpenMapIndefiniteLength(pMe); +} + +static inline void +QCBOREncode_CloseMapIndefiniteLength(QCBOREncodeContext *pMe) +{ + QCBOREncode_CloseMapOrArrayIndefiniteLength( + pMe, CBOR_MAJOR_NONE_TYPE_MAP_INDEFINITE_LEN); +} + +static inline void +QCBOREncode_BstrWrap(QCBOREncodeContext *pMe) +{ + QCBOREncode_OpenMapOrArray(pMe, CBOR_MAJOR_TYPE_BYTE_STRING); +} + +static inline void +QCBOREncode_BstrWrapInMap(QCBOREncodeContext *pMe, const char *szLabel) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_BstrWrap(pMe); +} + +static inline void +QCBOREncode_BstrWrapInMapN(QCBOREncodeContext *pMe, int64_t nLabel) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_BstrWrap(pMe); +} + +static inline void +QCBOREncode_CloseBstrWrap(QCBOREncodeContext *pMe, UsefulBufC *pWrappedCBOR) +{ + QCBOREncode_CloseBstrWrap2(pMe, true, pWrappedCBOR); +} + +static inline void +QCBOREncode_AddEncoded(QCBOREncodeContext *pMe, UsefulBufC Encoded) +{ + QCBOREncode_AddBuffer(pMe, CBOR_MAJOR_NONE_TYPE_RAW, Encoded); +} + +static inline void +QCBOREncode_AddEncodedToMap(QCBOREncodeContext *pMe, const char *szLabel, + UsefulBufC Encoded) +{ + QCBOREncode_AddSZString(pMe, szLabel); + QCBOREncode_AddEncoded(pMe, Encoded); +} + +static inline void +QCBOREncode_AddEncodedToMapN(QCBOREncodeContext *pMe, int64_t nLabel, + UsefulBufC Encoded) +{ + QCBOREncode_AddInt64(pMe, nLabel); + QCBOREncode_AddEncoded(pMe, Encoded); +} + +static inline int +QCBOREncode_IsBufferNULL(QCBOREncodeContext *pMe) +{ + return UsefulOutBuf_IsBufferNULL(&(pMe->OutBuf)); +} + +static inline QCBORError +QCBOREncode_GetErrorState(QCBOREncodeContext *pMe) +{ + if (UsefulOutBuf_GetError(&(pMe->OutBuf))) { + // Items didn't fit in the buffer. + // This check catches this condition for all the appends and + // inserts so checks aren't needed when the appends and inserts + // are performed. And of course UsefulBuf will never overrun the + // input buffer given to it. No complex analysis of the error + // handling in this file is needed to know that is true. Just + // read the UsefulBuf code. + pMe->uError = QCBOR_ERR_BUFFER_TOO_SMALL; + // QCBOR_ERR_BUFFER_TOO_SMALL masks other errors, but that is + // OK. Once the caller fixes this, they'll be unmasked. + } + + return (QCBORError)pMe->uError; +} + +/* ======================================================================== + END OF PRIVATE INLINE IMPLEMENTATION + ======================================================================== */ + +#ifdef __cplusplus +} +#endif + +#endif /* qcbor_encode_h */ diff --git a/hyp/interfaces/qcbor/include/qcbor/qcbor_private.h b/hyp/interfaces/qcbor/include/qcbor/qcbor_private.h new file mode 100644 index 0000000..1f4ab89 --- /dev/null +++ b/hyp/interfaces/qcbor/include/qcbor/qcbor_private.h @@ -0,0 +1,347 @@ +/*============================================================================== + Copyright (c) 2016-2018, The Linux Foundation. + Copyright (c) 2018-2021, Laurence Lundblade. + Copyright (c) 2021, Arm Limited. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of The Linux Foundation nor the names of its + contributors, nor the name "Laurence Lundblade" may be used to + endorse or promote products derived from this software without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + =============================================================================*/ + +#ifndef qcbor_private_h +#define qcbor_private_h + +#include + +#include "UsefulBuf.h" + +#ifdef __cplusplus +extern "C" { +#if 0 +} // Keep editor indention formatting happy +#endif +#endif + +/* + The maxium nesting of arrays and maps when encoding or decoding. + (Further down in the file there is a definition that refers to this + that is public. This is done this way so there can be a nice + separation of public and private parts in this file. +*/ +#define QCBOR_MAX_ARRAY_NESTING1 15 // Do not increase this over 255 + +/* The largest offset to the start of an array or map. It is slightly + less than UINT32_MAX so the error condition can be tested on 32-bit machines. + UINT32_MAX comes from uStart in QCBORTrackNesting being a uin32_t. + + This will cause trouble on a machine where size_t is less than 32-bits. + */ +#define QCBOR_MAX_ARRAY_OFFSET (UINT32_MAX - 100) + +/* The number of tags that are 16-bit or larger that can be handled + in a decode. + */ +#define QCBOR_NUM_MAPPED_TAGS 4 + +/* The number of tags (of any size) recorded for an individual item. */ +#define QCBOR_MAX_TAGS_PER_ITEM1 4 + +/* + Convenience macro for selecting the proper return value in case floating + point feature(s) are disabled. + + The macros: + + FLOAT_ERR_CODE_NO_FLOAT(x) Can be used when disabled floating point should + result error, and all other cases should return + 'x'. + + The below macros always return QCBOR_ERR_ALL_FLOAT_DISABLED when all floating + point is disabled. + + FLOAT_ERR_CODE_NO_HALF_PREC(x) Can be used when disabled preferred float + results in error, and all other cases should + return 'x'. + FLOAT_ERR_CODE_NO_FLOAT_HW(x) Can be used when disabled hardware floating + point results in error, and all other cases + should return 'x'. + FLOAT_ERR_CODE_NO_HALF_PREC_NO_FLOAT_HW(x) Can be used when either disabled + preferred float or disabling + hardware floating point results in + error, and all other cases should + return 'x'. + */ +#ifdef USEFULBUF_DISABLE_ALL_FLOAT +#define FLOAT_ERR_CODE_NO_FLOAT(x) QCBOR_ERR_ALL_FLOAT_DISABLED +#define FLOAT_ERR_CODE_NO_HALF_PREC(x) QCBOR_ERR_ALL_FLOAT_DISABLED +#define FLOAT_ERR_CODE_NO_FLOAT_HW(x) QCBOR_ERR_ALL_FLOAT_DISABLED +#define FLOAT_ERR_CODE_NO_HALF_PREC_NO_FLOAT_HW(x) QCBOR_ERR_ALL_FLOAT_DISABLED +#else /* USEFULBUF_DISABLE_ALL_FLOAT*/ +#define FLOAT_ERR_CODE_NO_FLOAT(x) x +#ifdef QCBOR_DISABLE_PREFERRED_FLOAT +#define FLOAT_ERR_CODE_NO_HALF_PREC(x) QCBOR_ERR_HALF_PRECISION_DISABLED +#define FLOAT_ERR_CODE_NO_HALF_PREC_NO_FLOAT_HW(x) \ + QCBOR_ERR_HALF_PRECISION_DISABLED +#else /* QCBOR_DISABLE_PREFERRED_FLOAT */ +#define FLOAT_ERR_CODE_NO_HALF_PREC(x) x +#ifdef QCBOR_DISABLE_FLOAT_HW_USE +#define FLOAT_ERR_CODE_NO_HALF_PREC_NO_FLOAT_HW(x) QCBOR_ERR_HW_FLOAT_DISABLED +#else +#define FLOAT_ERR_CODE_NO_HALF_PREC_NO_FLOAT_HW(x) x +#endif +#endif /* QCBOR_DISABLE_PREFERRED_FLOAT */ +#ifdef QCBOR_DISABLE_FLOAT_HW_USE +#define FLOAT_ERR_CODE_NO_FLOAT_HW(x) QCBOR_ERR_HW_FLOAT_DISABLED +#else /* QCBOR_DISABLE_FLOAT_HW_USE */ +#define FLOAT_ERR_CODE_NO_FLOAT_HW(x) x +#endif /* QCBOR_DISABLE_FLOAT_HW_USE */ +#endif /*USEFULBUF_DISABLE_ALL_FLOAT*/ + +/* + PRIVATE DATA STRUCTURE + + Holds the data for tracking array and map nesting during encoding. Pairs up + with the Nesting_xxx functions to make an "object" to handle nesting encoding. + + uStart is a uint32_t instead of a size_t to keep the size of this + struct down so it can be on the stack without any concern. It would be about + double if size_t was used instead. + + Size approximation (varies with CPU/compiler): + 64-bit machine: (15 + 1) * (4 + 2 + 1 + 1 pad) + 8 = 136 bytes + 32-bit machine: (15 + 1) * (4 + 2 + 1 + 1 pad) + 4 = 132 bytes +*/ +typedef struct QCBORTrackNesting_s { + // PRIVATE DATA STRUCTURE + struct { + // See function QCBOREncode_OpenMapOrArray() for details on how + // this works + uint32_t uStart; // uStart is the byte position where the array + // starts + uint16_t uCount; // Number of items in the arrary or map; counts + // items in a map, not pairs of items + uint8_t uMajorType; // Indicates if item is a map or an array + uint8_t padding_[1]; + } pArrays[QCBOR_MAX_ARRAY_NESTING1 + 1], // stored state for the nesting + // levels + *pCurrentNesting; // the current nesting level +} QCBORTrackNesting; + +/* + PRIVATE DATA STRUCTURE + + Context / data object for encoding some CBOR. Used by all encode functions to + form a public "object" that does the job of encdoing. + + Size approximation (varies with CPU/compiler): + 64-bit machine: 27 + 1 (+ 4 padding) + 136 = 32 + 136 = 168 bytes + 32-bit machine: 15 + 1 + 132 = 148 bytes +*/ +struct QCBOREncodeContext_s { + // PRIVATE DATA STRUCTURE + UsefulOutBuf OutBuf; // Pointer to output buffer, its length and + // position in it + uint8_t uError; // Error state, always from QCBORError enum + uint8_t padding_[7]; + QCBORTrackNesting nesting; // Keep track of array and map nesting +}; + +/* + PRIVATE DATA STRUCTURE + + Holds the data for array and map nesting for decoding work. This + structure and the DecodeNesting_Xxx() functions in qcbor_decode.c + form an "object" that does the work for arrays and maps. All access + to this structure is through DecodeNesting_Xxx() functions. + + 64-bit machine size + 128 = 16 * 8 for the two unions + 64 = 16 * 4 for the uLevelType, 1 byte padded to 4 bytes for alignment + 16 = 16 bytes for two pointers + 208 TOTAL + + 32-bit machine size is 200 bytes + */ +typedef struct QCBORDecodeNesting_s { + // PRIVATE DATA STRUCTURE + struct nesting_decode_level { + /* + This keeps tracking info for each nesting level. There are two + main types of levels: + 1) Byte count tracking. This is for the top level input CBOR + which might be a single item or a CBOR sequence and byte + string wrapped encoded CBOR. + 2) Item tracking. This is for maps and arrays. + + uLevelType has value QCBOR_TYPE_BYTE_STRING for 1) and + QCBOR_TYPE_MAP or QCBOR_TYPE_ARRAY or QCBOR_TYPE_MAP_AS_ARRAY + for 2). + + Item tracking is either for definite or indefinite-length + maps/arrays. For definite lengths, the total count and items + unconsumed are tracked. For indefinite-length, uTotalCount is + QCBOR_COUNT_INDICATES_INDEFINITE_LENGTH (UINT16_MAX) and there + is no per-item count of members. For indefinite-length maps and + arrays, uCountCursor is UINT16_MAX if not consumed and zero if + it is consumed in the pre-order traversal. Additionally, if + entered in bounded mode, uCountCursor is + QCBOR_COUNT_INDICATES_ZERO_LENGTH to indicate it is empty. + + This also records whether a level is bounded or not. All + byte-count tracked levels (the top-level sequence and + bstr-wrapped CBOR) are bounded. Maps and arrays may or may not + be bounded. They are bounded if they were Entered() and not if + they were traversed with GetNext(). They are marked as bounded + by uStartOffset not being UINT32_MAX. + */ + /* + If uLevelType can put in a separately indexed array, the union/ + struct will be 8 bytes rather than 9 and a lot of wasted + padding for alignment will be saved. + */ + uint8_t uLevelType; + uint8_t padding_[3]; + + union { + struct { +#define QCBOR_COUNT_INDICATES_INDEFINITE_LENGTH UINT16_MAX +#define QCBOR_COUNT_INDICATES_ZERO_LENGTH UINT16_MAX - 1 + uint16_t uCountTotal; + uint16_t uCountCursor; +#define QCBOR_NON_BOUNDED_OFFSET UINT32_MAX + uint32_t uStartOffset; + } ma; /* for maps and arrays */ + + struct { + /* The end of the input before the bstr was + * entered so that it can be restored when the + * bstr is exited. */ + uint32_t uSavedEndOffset; + /* The beginning of the bstr so that it can be + * rewound. */ + uint32_t uBstrStartOffset; + } bs; /* for top-level sequence and bstr-wrapped CBOR */ + } u; + } pLevels[QCBOR_MAX_ARRAY_NESTING1 + 1], *pCurrent, *pCurrentBounded; + + /* + pCurrent is for item-by-item pre-order traversal. + + pCurrentBounded points to the current bounding level or is NULL if + there isn't one. + + pCurrent must always be below pCurrentBounded as the pre-order + traversal is always bounded by the bounding level. + + When a bounded level is entered, the pre-order traversal is set to + the first item in the bounded level. When a bounded level is + exited, the pre-order traversl is set to the next item after the + map, array or bstr. This may be more than one level up, or even + the end of the input CBOR. + */ +} QCBORDecodeNesting; + +typedef struct { + // PRIVATE DATA STRUCTURE + void *pAllocateCxt; + UsefulBuf (*pfAllocator)(void *pAllocateCxt, void *pOldMem, + size_t uNewSize); +} QCBORInternalAllocator; + +/* + PRIVATE DATA STRUCTURE + + The decode context. This data structure plus the public QCBORDecode_xxx + functions form an "object" that does CBOR decoding. + + Size approximation (varies with CPU/compiler): + 64-bit machine: 32 + 1 + 1 + 6 bytes padding + 72 + 16 + 8 + 8 = 144 bytes + 32-bit machine: 16 + 1 + 1 + 2 bytes padding + 68 + 8 + 8 + 4 = 108 bytes + */ +struct QCBORDecodeContext_s { + // PRIVATE DATA STRUCTURE + UsefulInputBuf InBuf; + + QCBORDecodeNesting nesting; + + // If a string allocator is configured for indefinite-length + // strings, it is configured here. + QCBORInternalAllocator StringAllocator; + + // These are special for the internal MemPool allocator. + // They are not used otherwise. We tried packing these + // in the MemPool itself, but there are issues + // with memory alignment. + uint32_t uMemPoolSize; + uint32_t uMemPoolFreeOffset; + + // A cached offset to the end of the current map + // 0 if no value is cached. +#define QCBOR_MAP_OFFSET_CACHE_INVALID UINT32_MAX + uint32_t uMapEndOffsetCache; + + uint8_t uDecodeMode; + uint8_t bStringAllocateAll; + uint8_t padding_[2]; + QCBORError uLastError; // QCBORError stuffed into a uint8_t + uint8_t padding_1[4]; + + /* See MapTagNumber() for description of how tags are mapped. */ + uint64_t auMappedTags[QCBOR_NUM_MAPPED_TAGS]; + + uint16_t uLastTags[QCBOR_MAX_TAGS_PER_ITEM1]; +}; + +// Used internally in the impementation here +// Must not conflict with any of the official CBOR types +#define CBOR_MAJOR_NONE_TYPE_RAW 9 +#define CBOR_MAJOR_NONE_TAG_LABEL_REORDER 10 +#define CBOR_MAJOR_NONE_TYPE_BSTR_LEN_ONLY 11 +#define CBOR_MAJOR_NONE_TYPE_OPEN_BSTR 12 + +// Add this to types to indicate they are to be encoded as indefinite lengths +#define QCBOR_INDEFINITE_LEN_TYPE_MODIFIER 0x80 +#define CBOR_MAJOR_NONE_TYPE_ARRAY_INDEFINITE_LEN \ + CBOR_MAJOR_TYPE_ARRAY + QCBOR_INDEFINITE_LEN_TYPE_MODIFIER +#define CBOR_MAJOR_NONE_TYPE_MAP_INDEFINITE_LEN \ + CBOR_MAJOR_TYPE_MAP + QCBOR_INDEFINITE_LEN_TYPE_MODIFIER +#define CBOR_MAJOR_NONE_TYPE_SIMPLE_BREAK \ + CBOR_MAJOR_TYPE_SIMPLE + QCBOR_INDEFINITE_LEN_TYPE_MODIFIER + +/* Value of QCBORItem.val.string.len when the string length is + * indefinite. Used temporarily in the implementation and never + * returned in the public interface. + */ +#define QCBOR_STRING_LENGTH_INDEFINITE SIZE_MAX + +/* The number of elements in a C array of a particular type */ +#define C_ARRAY_COUNT(array, type) (sizeof(array) / sizeof(type)) + +#ifdef __cplusplus +} +#endif + +#endif /* qcbor_private_h */ diff --git a/hyp/interfaces/rcu/build.conf b/hyp/interfaces/rcu/build.conf index b5e3db1..f64616f 100644 --- a/hyp/interfaces/rcu/build.conf +++ b/hyp/interfaces/rcu/build.conf @@ -4,3 +4,4 @@ events rcu.ev types rcu.tc +macros rcu_attrs.h diff --git a/hyp/interfaces/rcu/include/rcu.h b/hyp/interfaces/rcu/include/rcu.h index 458fb76..f49d2a9 100644 --- a/hyp/interfaces/rcu/include/rcu.h +++ b/hyp/interfaces/rcu/include/rcu.h @@ -11,7 +11,7 @@ // // The caller must not assume that this function disables preemption. void -rcu_read_start(void); +rcu_read_start(void) ACQUIRE_RCU_READ; // End a read-side critical section. // @@ -20,7 +20,7 @@ rcu_read_start(void); // queued on any CPU after the start of the critical section are permitted to // run. void -rcu_read_finish(void); +rcu_read_finish(void) RELEASE_RCU_READ; // Enqueue a write-side update. // @@ -52,3 +52,10 @@ rcu_sync(void); // be running while blocked and/or on a CPU it does not have affinity to. bool rcu_sync_killable(void); + +// Check for pending updates on the calling CPU. +// +// If this call returns true, the CPU should not be allowed to enter a low power +// state or power itself off. +bool +rcu_has_pending_updates(void) REQUIRE_PREEMPT_DISABLED; diff --git a/hyp/interfaces/rcu/include/rcu_attrs.h b/hyp/interfaces/rcu/include/rcu_attrs.h new file mode 100644 index 0000000..3b0b287 --- /dev/null +++ b/hyp/interfaces/rcu/include/rcu_attrs.h @@ -0,0 +1,7 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#define ACQUIRE_RCU_READ ACQUIRE_READ(rcu_read) +#define RELEASE_RCU_READ RELEASE_READ(rcu_read) +#define REQUIRE_RCU_READ REQUIRE_READ(rcu_read) diff --git a/hyp/interfaces/rcu/rcu.tc b/hyp/interfaces/rcu/rcu.tc index f69ebd0..2429e06 100644 --- a/hyp/interfaces/rcu/rcu.tc +++ b/hyp/interfaces/rcu/rcu.tc @@ -2,6 +2,8 @@ // // SPDX-License-Identifier: BSD-3-Clause +define rcu_read global structure opaque_lock; + define rcu_update_class enumeration { }; diff --git a/hyp/interfaces/scheduler/hypercalls.hvc b/hyp/interfaces/scheduler/hypercalls.hvc index 2f497c5..8b62f93 100644 --- a/hyp/interfaces/scheduler/hypercalls.hvc +++ b/hyp/interfaces/scheduler/hypercalls.hvc @@ -6,6 +6,6 @@ define scheduler_yield hypercall { call_num 0x3b; control input bitfield scheduler_yield_control; arg1 input uregister; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; diff --git a/hyp/interfaces/scheduler/include/scheduler.h b/hyp/interfaces/scheduler/include/scheduler.h index fe40bdf..677cb6f 100644 --- a/hyp/interfaces/scheduler/include/scheduler.h +++ b/hyp/interfaces/scheduler/include/scheduler.h @@ -22,7 +22,7 @@ scheduler_schedule(void); // The scheduler run is not guaranteed to until the next return to userspace or // the idle thread. void -scheduler_trigger(void); +scheduler_trigger(void) REQUIRE_PREEMPT_DISABLED; // Run the scheduler and possibly switch to a different thread, with a hint // that the current thread wants to yield the CPU even if it is still @@ -140,18 +140,15 @@ scheduler_unblock(thread_t *thread, scheduler_block_t block) bool scheduler_is_blocked(const thread_t *thread, scheduler_block_t block); -// Return true if a thread is not blocked for any reason. +// Return true if a thread is available for scheduling. // // The caller must either be the specified thread, or hold a reference to the -// specified thread, or be in an RCU read-side critical section. +// specified thread, or be in an RCU read-side critical section. The caller must +// also hold the scheduling lock for the thread (see scheduler_lock()). // -// Note that this function is inherently racy: if the specified thread might -// be blocked for any reason by a third party, or if all current blocks might be -// removed by third parties, then it may return an incorrect value. It is the -// caller's responsibility to guarantee that such races do not occur, typically -// by calling scheduler_lock(). +// This function may ignore some block flags if the thread has been killed. bool -scheduler_is_runnable(const thread_t *thread); +scheduler_is_runnable(const thread_t *thread) REQUIRE_SCHEDULER_LOCK(thread); // Wait until a specified thread is not running. // @@ -204,7 +201,7 @@ scheduler_unpin(thread_t *thread) REQUIRE_SCHEDULER_LOCK(thread); // This function does not take a reference to the returned thread, so it must be // called from an RCU read-side critical section. thread_t * -scheduler_get_primary_vcpu(cpu_index_t cpu); +scheduler_get_primary_vcpu(cpu_index_t cpu) REQUIRE_RCU_READ; // Returns the configured affinity of a thread. // diff --git a/hyp/interfaces/scheduler/scheduler.ev b/hyp/interfaces/scheduler/scheduler.ev index b326108..269e035 100644 --- a/hyp/interfaces/scheduler/scheduler.ev +++ b/hyp/interfaces/scheduler/scheduler.ev @@ -39,20 +39,89 @@ event scheduler_selected_thread param thread: thread_t * param can_idle: bool * -// This event is triggered when the affinity of a thread is explicitly -// changed. The scheduler lock for the thread is held by the caller. -// If this is too restrictive for a module, handlers can request the -// next event to be triggered after a grace period. +// Prepare to change a thread's affinity. +// +// This event is for cases where we may want to deny certain affinity changes, +// e.g. the new CPU doesn't support a feature required by the affected VCPU. +// +// This event is triggered prior to the scheduler_affinity_changed event. The +// targeted thread has not yet been blocked and may be running either locally +// or remotely. The thread's scheduler lock is held. +// +// If this event returns an error then the affinity change is aborted. +setup_event scheduler_set_affinity_prepare + param thread: thread_t * + param prev_cpu: cpu_index_t + param next_cpu: cpu_index_t + return: error_t = OK + success: OK + +// Change a thread's affinity. +// +// This event is triggered after a thread is blocked to change affinity. The +// scheduler lock for the thread is held by the caller, and the thread is not +// current on any CPU (though it may be the previous thread in a +// thread_context_switch_post event). +// +// If it is necessary to take actions with no lock held, the need_sync +// parameter can be set to true to trigger a scheduler_affinity_changed_sync +// event after after a grace period. +// +// This event must not fail. If a module needs to prevent affinity changes in +// some cases, it must be checked in a scheduler_set_affinity_prepare handler, +// which will be called prior to this event. Note that the scheduler lock may +// be dropped between these events. event scheduler_affinity_changed param thread: thread_t * param prev_cpu: cpu_index_t param next_cpu: cpu_index_t param need_sync: bool * -// This event is triggered when the affinity of a thread is explicitly -// changed, if requested by the previous event. It is triggered after -// a grace period, with no locks held. +// Clean up after changing a thread's affinity. +// +// This event is triggered after the affinity of a thread is explicitly +// changed, if requested by a handler for the scheduler_affinity_changed +// event. It is triggered after a grace period, with no locks held. +// +// Modules that handle this event must not assume that they were responsible +// for triggering it; a different module may have triggered the event. event scheduler_affinity_changed_sync param thread: thread_t * param prev_cpu: cpu_index_t param next_cpu: cpu_index_t + +// This event is triggered just before the scheduler schedules the next thread. +// "yielded_from" is the value of CPULOCAL(yielded_from) for this CPU, +// "schedtime" is the start time of the previous scheduler run, and "curticks" +// is the start time of this scheduler run. +event scheduler_schedule + param current: thread_t * + param yielded_from: thread_t * + param schedtime: ticks_t + param curticks: ticks_t + +// This event is triggered before the scheduler sets a given block flag on a +// thread, if that flag is not already set. +// +// The was_runnable argument is true if the thread was not blocked for any +// reason prior to setting the block flag (after excluding killable block +// flags, if the thread has been killed). +// +// The scheduler lock for the thread is held by the caller. +event scheduler_blocked + param thread: thread_t * + param block: scheduler_block_t + param was_runnable: bool + +// This event is triggered after the scheduler clears a given block flag on a +// thread, if that flag was not already clear. +// +// The now_runnable argument is true if the thread will not be blocked for any +// reason after to this event (after excluding killable block flags, if the +// thread has been killed). +// +// The scheduler lock for the thread is held by the caller. +event scheduler_unblocked + param thread: thread_t * + param block: scheduler_block_t + param now_runnable: bool diff --git a/hyp/interfaces/slat/aarch64/slat.tc b/hyp/interfaces/slat/aarch64/slat.tc index 8cf8d97..317867c 100644 --- a/hyp/interfaces/slat/aarch64/slat.tc +++ b/hyp/interfaces/slat/aarch64/slat.tc @@ -5,3 +5,5 @@ // Second Level Address Translation (SLAT) Base Module define vmaddr_t public newtype uint64; + +define VMADDR_INVALID constant type vmaddr_t = -1; diff --git a/hyp/interfaces/smccc/include/smccc.ev.h b/hyp/interfaces/smccc/include/smccc.ev.h index 4083e92..5b8f7c0 100644 --- a/hyp/interfaces/smccc/include/smccc.ev.h +++ b/hyp/interfaces/smccc/include/smccc.ev.h @@ -3,31 +3,33 @@ // SPDX-License-Identifier: BSD-3-Clause #define SMCCC_ARCH_FUNCTION_32(fn, feat, h, ...) \ - subscribe smccc_call_fast_32_arch[SMCCC_ARCH_FUNCTION_##fn]; \ + subscribe smccc_call_fast_32_arch[( \ + smccc_function_t)SMCCC_ARCH_FUNCTION_##fn]; \ handler smccc_##h(__VA_ARGS__); \ exclude_preempt_disabled.subscribe \ smccc_arch_features_fast32[SMCCC_ARCH_FUNCTION_##fn]; \ constant feat. #define SMCCC_ARCH_FUNCTION_64(fn, feat, h, ...) \ - subscribe smccc_call_fast_64_arch[SMCCC_ARCH_FUNCTION_##fn]; \ + subscribe smccc_call_fast_64_arch[( \ + smccc_function_t)SMCCC_ARCH_FUNCTION_##fn]; \ handler smccc_##h(__VA_ARGS__); \ exclude_preempt_disabled.subscribe \ smccc_arch_features_fast64[SMCCC_ARCH_FUNCTION_##fn]; \ constant feat. -#define SMCCC_STANDARD_HYP_FUNCTION_32(fn, feat, h, ...) \ - subscribe smccc_call_fast_32_standard_hyp \ - [SMCCC_STANDARD_HYP_FUNCTION_##fn]; \ - handler smccc_##h(__VA_ARGS__); \ - exclude_preempt_disabled.subscribe smccc_standard_hyp_features_fast32 \ - [SMCCC_STANDARD_HYP_FUNCTION_##fn]; \ +#define SMCCC_STANDARD_HYP_FUNCTION_32(fn, feat, h, ...) \ + subscribe smccc_call_fast_32_standard_hyp[( \ + smccc_function_t)SMCCC_STANDARD_HYP_FUNCTION_##fn]; \ + handler smccc_##h(__VA_ARGS__); \ + exclude_preempt_disabled.subscribe smccc_standard_hyp_features_fast32 \ + [SMCCC_STANDARD_HYP_FUNCTION_##fn]; \ constant feat. -#define SMCCC_STANDARD_HYP_FUNCTION_64(fn, feat, h, ...) \ - subscribe smccc_call_fast_64_standard_hyp \ - [SMCCC_STANDARD_HYP_FUNCTION_##fn]; \ - handler smccc_##h(__VA_ARGS__); \ - exclude_preempt_disabled.subscribe smccc_standard_hyp_features_fast64 \ - [SMCCC_STANDARD_HYP_FUNCTION_##fn]; \ +#define SMCCC_STANDARD_HYP_FUNCTION_64(fn, feat, h, ...) \ + subscribe smccc_call_fast_64_standard_hyp[( \ + smccc_function_t)SMCCC_STANDARD_HYP_FUNCTION_##fn]; \ + handler smccc_##h(__VA_ARGS__); \ + exclude_preempt_disabled.subscribe smccc_standard_hyp_features_fast64 \ + [SMCCC_STANDARD_HYP_FUNCTION_##fn]; \ constant feat. diff --git a/hyp/interfaces/smccc/include/smccc_platform.h b/hyp/interfaces/smccc/include/smccc_platform.h new file mode 100644 index 0000000..291e4b8 --- /dev/null +++ b/hyp/interfaces/smccc/include/smccc_platform.h @@ -0,0 +1,6 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +bool +smccc_handle_smc_platform_call(register_t args[7], bool is_hvc); diff --git a/hyp/interfaces/smccc/smccc.tc b/hyp/interfaces/smccc/smccc.tc index e3d8de0..6e7f2ad 100644 --- a/hyp/interfaces/smccc/smccc.tc +++ b/hyp/interfaces/smccc/smccc.tc @@ -10,9 +10,9 @@ // Note that the new features in v1.3 relative to v1.1 are all optional // (extra arg / return registers, the SMCCC_ARCH_SOC_ID function, and // the sve_not_live hint bit in the function ID). -define SMCCC_VERSION constant uint32 = 0x10003; +define SMCCC_VERSION public constant uint32 = 0x10003; -define smccc_interface_id enumeration(explicit) { +define smccc_interface_id public enumeration(explicit) { ARCH = 0; CPU = 1; SIP = 2; @@ -22,9 +22,9 @@ define smccc_interface_id enumeration(explicit) { VENDOR_HYP = 6; }; -define smccc_function_t newtype uint16; +define smccc_function_t public newtype uint16; -define smccc_function_id bitfield<32> { +define smccc_function_id public bitfield<32> { 15:0 function type smccc_function_t; 16 sve_live_state_hint bool; // from SMCCC v1.3+ 23:17 res0 uint32(const); @@ -33,15 +33,26 @@ define smccc_function_id bitfield<32> { 31 is_fast bool; }; +define smccc_vendor_hyp_function_class public enumeration(explicit) { + PLATFORM_CALL = 0b00; + HYPERCALL = 0b10; + SERVICE = 0b11; +}; + +define smccc_vendor_hyp_function_id public bitfield<16> { + 15:14 call_class enumeration smccc_vendor_hyp_function_class; + 13:0 function uint16; +}; + define smccc_client_id bitfield<32> { 15:0 client_id uint16; 31:16 secure_os_id uint16; }; -define SMCCC_UNKNOWN_FUNCTION64 constant uint64 = -1; -define SMCCC_UNKNOWN_FUNCTION32 constant uint32 = -1; +define SMCCC_UNKNOWN_FUNCTION64 public constant uint64 = -1; +define SMCCC_UNKNOWN_FUNCTION32 public constant uint32 = -1; -define smccc_arch_function enumeration(explicit) { +define smccc_arch_function public enumeration(explicit) { VERSION = 0; ARCH_FEATURES = 1; ARCH_SOC_ID = 2; @@ -49,7 +60,21 @@ define smccc_arch_function enumeration(explicit) { ARCH_WORKAROUND_1 = 0x8000; }; -define smccc_standard_hyp_function enumeration(explicit) { +define smccc_standard_hyp_function public enumeration(explicit) { + CALL_COUNT = 0xff00; CALL_UID = 0xff01; REVISION = 0xff03; }; + +// Gunyah SMCCC UUID: c1d58fcd-a453-5fdb-9265-ce36673d5f14 +define SMCCC_GUNYAH_UID0 public constant uregister = 0xcd8fd5c1U; +define SMCCC_GUNYAH_UID1 public constant uregister = 0xdb5f53a4U; +define SMCCC_GUNYAH_UID2 public constant uregister = 0x36ce6592U; +define SMCCC_GUNYAH_UID3 public constant uregister = 0x145f3d67U; + +// function id bits 13:0 +define smccc_vendor_hyp_function public enumeration(explicit) { + CALL_COUNT = 0x3f00; + CALL_UID = 0x3f01; + REVISION = 0x3f03; +}; diff --git a/hyp/interfaces/spinlock/include/spinlock.h b/hyp/interfaces/spinlock/include/spinlock.h index edd6bd8..caa41a7 100644 --- a/hyp/interfaces/spinlock/include/spinlock.h +++ b/hyp/interfaces/spinlock/include/spinlock.h @@ -45,5 +45,5 @@ spinlock_release_nopreempt(spinlock_t *lock) RELEASE_SPINLOCK_NP(lock); // // This might only be a static check, especially in non-debug builds. void -assert_spinlock_held(spinlock_t *lock) REQUIRE_LOCK(lock) +assert_spinlock_held(const spinlock_t *lock) REQUIRE_LOCK(lock) REQUIRE_PREEMPT_DISABLED; diff --git a/hyp/interfaces/spinlock/spinlock.ev b/hyp/interfaces/spinlock/spinlock.ev index 0a12b6c..7fac5ee 100644 --- a/hyp/interfaces/spinlock/spinlock.ev +++ b/hyp/interfaces/spinlock/spinlock.ev @@ -30,4 +30,4 @@ event spinlock_released param lock: spinlock_t * event spinlock_assert_held - param lock: spinlock_t * + param lock: const spinlock_t * diff --git a/hyp/interfaces/tests/build.conf b/hyp/interfaces/tests/build.conf new file mode 100644 index 0000000..703ef4a --- /dev/null +++ b/hyp/interfaces/tests/build.conf @@ -0,0 +1,6 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +events tests.ev +types tests.tc diff --git a/hyp/interfaces/tests/tests.ev b/hyp/interfaces/tests/tests.ev new file mode 100644 index 0000000..0292c47 --- /dev/null +++ b/hyp/interfaces/tests/tests.ev @@ -0,0 +1,21 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +interface tests + +event tests_init + +handled_event tests_start + +#if defined(INTERFACE_TESTS) +selector_event tests_run + selector test_id: tests_run_id_t + param arg0 : uint64_t + param arg1 : uint64_t + param arg2 : uint64_t + param arg3 : uint64_t + param arg4 : uint64_t + param arg5 : uint64_t + return: error_t = ERROR_UNIMPLEMENTED +#endif diff --git a/hyp/interfaces/tests/tests.tc b/hyp/interfaces/tests/tests.tc new file mode 100644 index 0000000..bf8a944 --- /dev/null +++ b/hyp/interfaces/tests/tests.tc @@ -0,0 +1,6 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +define tests_run_id enumeration { +}; diff --git a/hyp/interfaces/thread/include/thread.h b/hyp/interfaces/thread/include/thread.h index 5ec7bc3..430a497 100644 --- a/hyp/interfaces/thread/include/thread.h +++ b/hyp/interfaces/thread/include/thread.h @@ -22,10 +22,14 @@ thread_get_self(void); // stops running. If the switch fails, the reference will be released // immediately before returning. // +// The second argument is the absolute time at which the scheduler made the +// decision to run this thread. It is not used directly by this module, but is +// passed to context switch event handlers for use in time accounting. +// // This function will fail if the specified thread is already running on another // CPU. The scheduler is responsible for guaranteeing this. error_t -thread_switch_to(thread_t *thread) REQUIRE_PREEMPT_DISABLED; +thread_switch_to(thread_t *thread, ticks_t curticks) REQUIRE_PREEMPT_DISABLED; // Kill a thread. This marks it as exiting, sends an interrupt to any CPU that // is currently running it, and switches to it on the current CPU if it is not @@ -43,11 +47,26 @@ thread_kill(thread_t *thread); // Return true if the specified thread has had thread_kill() called on it. // +// This function has relaxed memory semantics. If the thread may be running on a +// remote CPU, or may have been killed by a remote CPU, it is the caller's +// responsibility to ensure that the memory access is ordered correctly. +// // The caller must either be the specified thread, or hold a reference to the // specified thread, or be in an RCU read-side critical section. bool thread_is_dying(const thread_t *thread); +// Return true if the specified thread has exited. +// +// This function has relaxed memory semantics. If the thread may be running on a +// remote CPU, or may have just exited on a remote CPU, it is the caller's +// responsibility to ensure that the memory access is ordered correctly. +// +// The caller must either hold a reference to the specified thread, or be in an +// RCU read-side critical section. +bool +thread_has_exited(const thread_t *thread); + // Block until a specified thread has exited. // // The caller must not be the specified thread, and must hold a reference to diff --git a/hyp/interfaces/thread/include/thread_init.h b/hyp/interfaces/thread/include/thread_init.h index 431fafc..4393d9e 100644 --- a/hyp/interfaces/thread/include/thread_init.h +++ b/hyp/interfaces/thread/include/thread_init.h @@ -6,4 +6,4 @@ // Switch to the idle thread at the end of the boot sequence. noreturn void -thread_boot_set_idle(void); +thread_boot_set_idle(void) REQUIRE_PREEMPT_DISABLED; diff --git a/hyp/interfaces/thread/thread.ev b/hyp/interfaces/thread/thread.ev index 96dd4d9..0361d49 100644 --- a/hyp/interfaces/thread/thread.ev +++ b/hyp/interfaces/thread/thread.ev @@ -39,6 +39,7 @@ event thread_save_state // executed. setup_event thread_context_switch_pre param next: thread_t * + param curticks: ticks_t return: error_t = OK success: OK @@ -59,6 +60,8 @@ setup_event thread_context_switch_pre // thread is also the idle thread. event thread_context_switch_post param prev: thread_t * + param curticks: ticks_t + param prevticks: ticks_t // Return the thread's main function. // diff --git a/hyp/interfaces/thread/thread.tc b/hyp/interfaces/thread/thread.tc index da2ffb9..6d1a263 100644 --- a/hyp/interfaces/thread/thread.tc +++ b/hyp/interfaces/thread/thread.tc @@ -9,7 +9,6 @@ define THREAD_STACK_MAP_ALIGN_BITS constant type count_t = THREAD_STACK_MAX_BITS define THREAD_STACK_MAP_ALIGN constant type count_t = 1 << THREAD_STACK_MAP_ALIGN_BITS; define thread_kind enumeration { - none = 0; }; define thread_state enumeration { diff --git a/hyp/interfaces/timer/include/timer_queue.h b/hyp/interfaces/timer/include/timer_queue.h index 70acc83..3462d63 100644 --- a/hyp/interfaces/timer/include/timer_queue.h +++ b/hyp/interfaces/timer/include/timer_queue.h @@ -4,6 +4,7 @@ // TODO: Add functions that work on other CPUs' queues. Important for migrating // schedulers. +// FIXME: // Initialise a timer object void @@ -44,4 +45,4 @@ timer_convert_ticks_to_ns(ticks_t ticks); // Get next timeout from cpu local queue ticks_t -timer_queue_get_next_timeout(void); +timer_queue_get_next_timeout(void) REQUIRE_PREEMPT_DISABLED; diff --git a/hyp/interfaces/timer/timer.tc b/hyp/interfaces/timer/timer.tc index 07b2f3e..a9df792 100644 --- a/hyp/interfaces/timer/timer.tc +++ b/hyp/interfaces/timer/timer.tc @@ -2,8 +2,12 @@ // // SPDX-License-Identifier: BSD-3-Clause -define ticks_t newtype uint64; +define ticks_t public newtype uint64; define nanoseconds_t public newtype uint64; +define microseconds_t public newtype uint64; + +define TIMER_NANOSECS_IN_SECOND constant type nanoseconds_t = 1000000000; +define TIMER_MICROSECS_IN_SECOND constant type microseconds_t = 1000000; define timer structure { }; diff --git a/hyp/interfaces/timer_lp/build.conf b/hyp/interfaces/timer_lp/build.conf new file mode 100644 index 0000000..e69de29 diff --git a/hyp/interfaces/trace/include/trace.h b/hyp/interfaces/trace/include/trace.h index 11ab38d..9eeb317 100644 --- a/hyp/interfaces/trace/include/trace.h +++ b/hyp/interfaces/trace/include/trace.h @@ -16,9 +16,10 @@ #include -#define TRACE_ID(id) (TRACE_ID_##id) -#define TRACE_CLASS(tclass) (TRACE_CLASS_##tclass) -#define TRACE_CLASS_BITS(tclass) (1U << TRACE_CLASS_##tclass) +#define TRACE_ID(id) (TRACE_ID_##id) +#define TRACE_CLASS(tclass) (TRACE_CLASS_##tclass) +#define TRACE_CLASS_BITS(tclass) \ + ((register_t)1 << (index_t)TRACE_CLASS_##tclass) #define TRACE_FUNC_I(id, action, a0, a1, a2, a3, a4, a5, n, ...) \ TRACE_ADD##n(TRACE_ID(id), action, a0, a1, a2, a3, a4, a5, __VA_ARGS__) @@ -30,17 +31,20 @@ extern register_t trace_public_class_flags; #define TRACE_MAYBE(classes, X) \ do { \ - register_t enabled_ = atomic_load_explicit( \ + register_t class_enabled_ = atomic_load_explicit( \ &hyp_trace.enabled_class_flags, memory_order_relaxed); \ - if (compiler_unexpected((enabled_ & classes) != 0)) { \ + if (compiler_unexpected((class_enabled_ & classes) != 0U)) { \ X; \ } \ } while (0) // Used for single class trace +#if defined(PARASOFT_CYCLO) +#define TRACE(tclass, id, ...) +#else #define TRACE(tclass, id, ...) \ TRACE_EVENT(tclass, id, TRACE_ACTION_TRACE, __VA_ARGS__) - +#endif #define TRACE_LOCAL(tclass, id, ...) \ TRACE_EVENT(tclass, id, TRACE_ACTION_TRACE_LOCAL, __VA_ARGS__) @@ -99,4 +103,14 @@ trace_get_class_flags(void); // It stops using the trace boot buffer and starts using a dynamically allocated // trace buffer of bigger size void -trace_init(partition_t *partition, size_t size); +trace_init(partition_t *partition, size_t size) REQUIRE_PREEMPT_DISABLED; + +#if defined(PLATFORM_TRACE_STANDALONE_REGION) +// Use pre-allocated memory for trace buffer +// +// It stops using the trace boot buffer and starts using a pre-allocated +// trace buffer of bigger size +void +trace_single_region_init(partition_t *partition, paddr_t base, size_t size) + REQUIRE_PREEMPT_DISABLED; +#endif diff --git a/hyp/interfaces/trace/include/trace_helpers.h b/hyp/interfaces/trace/include/trace_helpers.h index 7683e58..be542ff 100644 --- a/hyp/interfaces/trace/include/trace_helpers.h +++ b/hyp/interfaces/trace/include/trace_helpers.h @@ -4,13 +4,15 @@ #define TRACE_SET_CLASS(bitmap, trace_class) \ do { \ - (bitmap) |= 1U << (TRACE_CLASS_##trace_class); \ + (bitmap) |= (register_t)1U \ + << ((index_t)TRACE_CLASS_##trace_class); \ } while (0) #define TRACE_CLEAR_CLASS(bitmap, trace_class) \ do { \ - (bitmap) &= (~(1U << (TRACE_CLASS_##trace_class))); \ + (bitmap) &= (~((register_t)1U \ + << ((index_t)TRACE_CLASS_##trace_class))); \ } while (0) #define TRACE_CLASS_ENABLED(bitmap, trace_class) \ - ((bitmap) & (1U << (TRACE_CLASS_##trace_class))) + ((bitmap) & ((register_t)1U << ((index_t)TRACE_CLASS_##trace_class))) diff --git a/hyp/interfaces/trace/trace.hvc b/hyp/interfaces/trace/trace.hvc index 74eaf97..a6cf0ad 100644 --- a/hyp/interfaces/trace/trace.hvc +++ b/hyp/interfaces/trace/trace.hvc @@ -6,7 +6,7 @@ define trace_update_class_flags hypercall { call_num 0x3f; set_flags input uregister; clear_flags input uregister; - _res0 input uregister; + res0 input uregister; error output enumeration error; flags output uregister; }; diff --git a/hyp/interfaces/trace/trace.tc b/hyp/interfaces/trace/trace.tc index a9100f9..9119e70 100644 --- a/hyp/interfaces/trace/trace.tc +++ b/hyp/interfaces/trace/trace.tc @@ -7,9 +7,9 @@ // The number of enumerators in trace_class enumeration should be less than 32 // for _core_ classes to support 32-bit architectures. -define trace_class enumeration() { - ERROR; // Critical errors - DEBUG; // Debugging information +define trace_class public enumeration(explicit) { + ERROR = 0; // Critical errors + DEBUG = 1; // Debugging information }; define trace_id enumeration(explicit) { diff --git a/hyp/interfaces/util/include/assert.h b/hyp/interfaces/util/include/assert.h index 66a79bf..474cce4 100644 --- a/hyp/interfaces/util/include/assert.h +++ b/hyp/interfaces/util/include/assert.h @@ -35,7 +35,12 @@ panic(const char *str); _Noreturn void assert_failed(const char *file, int line, const char *func, const char *err); #define assert(x) \ - (assert_if_const(x) \ - ? (void)0 \ - : assert_failed(__BUILD_FILE__, __LINE__, __func__, #x)) + (assert_if_const(x) ? (void)0 \ + : assert_failed(__FILE__, __LINE__, __func__, #x)) +#endif + +#if defined(VERBOSE) && VERBOSE +#define assert_debug assert +#else +#define assert_debug(x) (void)assert_if_const(x) #endif diff --git a/hyp/interfaces/util/include/attributes.h b/hyp/interfaces/util/include/attributes.h index 52c4982..ca29685 100644 --- a/hyp/interfaces/util/include/attributes.h +++ b/hyp/interfaces/util/include/attributes.h @@ -12,7 +12,12 @@ // This rule does not apply to language constructs that have an effect that is // similar or equivalent to an attribute, such as _Noreturn or _Alignas. -// Don't inline the function. This is used to mark cold functions for which +// Mark a function as cold. This is used to mark cold functions for +// which internal inlining or size increasing optimizations would be +// a waste of space. +#define COLD __attribute__((cold)) + +// Don't inline this function. This is used to mark functions for which // inlining would be a waste of space and/or would make debugging inconvenient. #define NOINLINE __attribute__((noinline)) diff --git a/hyp/interfaces/util/include/bitmap.h b/hyp/interfaces/util/include/bitmap.h index a0e4f04..c1c640d 100644 --- a/hyp/interfaces/util/include/bitmap.h +++ b/hyp/interfaces/util/include/bitmap.h @@ -81,15 +81,15 @@ bitmap__atomic_get_word(const _Atomic register_t *bitmap, index_t word) { \ index_t w = 0; \ register_t r = 0; \ - while ((r != 0) || ((w * BITMAP_WORD_BITS) < n)) { \ - if (r == 0) { \ + while ((r != 0U) || ((w * BITMAP_WORD_BITS) < n)) { \ + if (r == 0U) { \ r = g((b), (w)); \ w++; \ } \ - if (r != 0) { \ + if (r != 0U) { \ index_t i = compiler_ctz(r); \ r &= ~(register_t)1 << i; \ - i += ((w - 1) * BITMAP_WORD_BITS); \ + i += ((w - 1U) * BITMAP_WORD_BITS); \ if (i >= n) { \ r = 0; \ } else { diff --git a/hyp/interfaces/util/include/compiler.h b/hyp/interfaces/util/include/compiler.h index 2e00d98..80da8b5 100644 --- a/hyp/interfaces/util/include/compiler.h +++ b/hyp/interfaces/util/include/compiler.h @@ -79,3 +79,15 @@ __asm__(".error"); // the static determination can be made after inlining by LTO. #define compiler_sizeof_object(ptr) __builtin_object_size((ptr), 1) #define compiler_sizeof_container(ptr) __builtin_object_size((ptr), 0) + +// Mark a break statement as unreachable. This expected to be used at the end of +// switch cases to comply with MISRA rule 16.3, which requires a break statement +// to end the case regardless of whether it is reachable. + +// clang-format off +#define compiler_unreachable_break \ + _Pragma("clang diagnostic push") \ + _Pragma("clang diagnostic ignored \"-Wunreachable-code\"") \ + break \ + _Pragma("clang diagnostic pop") +// clang-format on diff --git a/hyp/interfaces/util/include/enum.h b/hyp/interfaces/util/include/enum.h new file mode 100644 index 0000000..976aa1c --- /dev/null +++ b/hyp/interfaces/util/include/enum.h @@ -0,0 +1,8 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#define ENUM_FOREACH(name, i) \ + static_assert(name##__MIN >= 0, "negative enum"); \ + static_assert(name##__MAX >= name##__MIN, "invalid enum"); \ + for (index_t i = (index_t)name##__MIN; i <= (index_t)name##__MAX; i++) diff --git a/hyp/interfaces/util/include/string.h b/hyp/interfaces/util/include/string.h index 666b255..06e832f 100644 --- a/hyp/interfaces/util/include/string.h +++ b/hyp/interfaces/util/include/string.h @@ -29,13 +29,14 @@ _Static_assert(__STDC_HOSTED__ == 0, // Define size_t and NULL #include +#define memscpy(s1, s1_size, s2, s2_size) \ + ((void)memcpy(s1, s2, \ + ((s1_size) < (s2_size)) ? (s1_size) : (s2_size)), \ + ((s1_size) < (s2_size)) ? (s1_size) : (s2_size)) + extern void * memcpy(void *restrict s1, const void *restrict s2, size_t n); -extern size_t -memscpy(void *restrict s1, size_t s1_size, const void *restrict s2, - size_t s2_size); - extern void * memmove(void *s1, const void *s2, size_t n); @@ -47,7 +48,7 @@ typedef size_t rsize_t; // A secure memset, guaranteed not to be optimized out extern errno_t -memset_s(void *dest, rsize_t destsz, int c, rsize_t n); +memset_s(void *s, rsize_t smax, int c, rsize_t n); extern size_t strlen(const char *str); diff --git a/hyp/interfaces/util/include/types/bitmap.h b/hyp/interfaces/util/include/types/bitmap.h index 2b168b9..cfce772 100644 --- a/hyp/interfaces/util/include/types/bitmap.h +++ b/hyp/interfaces/util/include/types/bitmap.h @@ -5,7 +5,7 @@ // Bitmap definitions for the type DSL. This is also included by C code to // get BITMAP_NUM_WORDS(). -#define BITMAP_NUM_WORDS(x) (((x) + BITMAP_WORD_BITS - 1) / BITMAP_WORD_BITS) +#define BITMAP_NUM_WORDS(x) (((x) + BITMAP_WORD_BITS - 1U) / BITMAP_WORD_BITS) #if defined(__TYPED_DSL__) #define BITMAP(bits, ...) \ diff --git a/hyp/interfaces/util/include/util.h b/hyp/interfaces/util/include/util.h index 525256f..c188ff2 100644 --- a/hyp/interfaces/util/include/util.h +++ b/hyp/interfaces/util/include/util.h @@ -9,7 +9,7 @@ #define util_bit(b) ((uintmax_t)1U << (b)) #define util_sbit(b) ((intmax_t)1 << (b)) -#define util_mask(n) (util_bit(n) - 1) +#define util_mask(n) (util_bit(n) - 1U) #define util_max(x, y) (((x) > (y)) ? (x) : (y)) #define util_min(x, y) (((x) < (y)) ? (x) : (y)) @@ -18,9 +18,13 @@ #define util_is_p2_or_zero(x) (((x) & ((x)-1U)) == 0U) #define util_is_p2(x) (((x) != 0U) && util_is_p2_or_zero(x)) #define util_is_baligned(x, a) (assert(util_is_p2(a)), (((x) & ((a)-1U)) == 0U)) -#define util_is_p2aligned(x, b) (((x) & (util_bit(b) - 1)) == 0U) +#define util_is_p2aligned(x, b) (((x) & (util_bit(b) - 1U)) == 0U) #define util_add_overflows(a, b) ((a) > ~(__typeof__((a) + (b)))(b)) +// This version can be used in static asserts +#define util_is_baligned_assert(x, a) \ + (util_is_p2(a) && (((x) & ((a)-1U)) == 0U)) + // Align up or down to bytes (which must be a power of two) #if defined(__TYPED_DSL__) #define util_balign_down(x, a) ((x) & ~((a)-1U)) @@ -32,7 +36,7 @@ // Align up or down to a power-of-two size (in bits) #define util_p2align_down(x, b) \ - (assert((sizeof(x) * 8) > (b)), (((x) >> (b)) << (b))) + (assert((sizeof(x) * 8U) > (b)), (((x) >> (b)) << (b))) #define util_p2align_up(x, b) util_p2align_down((x) + util_bit(b) - 1U, b) // Generate an identifier that can be declared inside a macro without diff --git a/hyp/interfaces/vcpu/aarch64/vcpu.tc b/hyp/interfaces/vcpu/aarch64/vcpu.tc new file mode 100644 index 0000000..705997d --- /dev/null +++ b/hyp/interfaces/vcpu/aarch64/vcpu.tc @@ -0,0 +1,9 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +extend vcpu_register_set enumeration { + X = 0; // index 0-30 + PC = 1; // index 0 + SP_EL = 2; // index 0-1 +}; diff --git a/hyp/interfaces/vcpu/armv8/traps.ev b/hyp/interfaces/vcpu/armv8/traps.ev index f5b99c6..383a2aa 100644 --- a/hyp/interfaces/vcpu/armv8/traps.ev +++ b/hyp/interfaces/vcpu/armv8/traps.ev @@ -3,6 +3,7 @@ // SPDX-License-Identifier: BSD-3-Clause // Add the events for the traps coming from AArch32 +// FIXME: interface vcpu @@ -21,8 +22,9 @@ handled_event vcpu_trap_wfi handled_event vcpu_trap_fp_enabled param esr: ESR_EL2_t + return: vcpu_trap_result_t = VCPU_TRAP_RESULT_UNHANDLED -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) handled_event vcpu_trap_pauth #endif @@ -45,18 +47,18 @@ handled_event vcpu_trap_sysreg_write param iss: ESR_EL2_ISS_MSR_MRS_t return: vcpu_trap_result_t = VCPU_TRAP_RESULT_UNHANDLED -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) handled_event vcpu_trap_eret param esr: ESR_EL2_t #endif handled_event vcpu_trap_pf_abort_guest param esr: ESR_EL2_t - param ipa: vmaddr_t + param ipa: vmaddr_result_t param far: FAR_EL2_t return: vcpu_trap_result_t = VCPU_TRAP_RESULT_UNHANDLED -#if defined(ARCH_ARM_8_2_SVE) +#if defined(ARCH_ARM_FEAT_SVE) handled_event vcpu_trap_sve_access return: vcpu_trap_result_t = VCPU_TRAP_RESULT_UNHANDLED #endif @@ -65,7 +67,7 @@ handled_event vcpu_trap_pc_alignment_fault handled_event vcpu_trap_data_abort_guest param esr: ESR_EL2_t - param ipa: vmaddr_t + param ipa: vmaddr_result_t param far: FAR_EL2_t return: vcpu_trap_result_t = VCPU_TRAP_RESULT_UNHANDLED @@ -93,7 +95,6 @@ handled_event vcpu_trap_brk_instruction_guest handled_event vcpu_trap_serror param iss: ESR_EL2_ISS_SERROR_t - return: vcpu_trap_result_t = VCPU_TRAP_RESULT_UNHANDLED // AArch32 traps #if ARCH_AARCH64_32BIT_EL0 diff --git a/hyp/interfaces/vcpu/armv8/vcpu.tc b/hyp/interfaces/vcpu/armv8/vcpu.tc new file mode 100644 index 0000000..cc124eb --- /dev/null +++ b/hyp/interfaces/vcpu/armv8/vcpu.tc @@ -0,0 +1,7 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +extend platform_cpu_features bitfield { + auto scxt_disable bool = 1; +}; diff --git a/hyp/interfaces/vcpu/build.conf b/hyp/interfaces/vcpu/build.conf index 4540adc..be913c1 100644 --- a/hyp/interfaces/vcpu/build.conf +++ b/hyp/interfaces/vcpu/build.conf @@ -5,3 +5,5 @@ types vcpu.tc events vcpu.ev arch_events armv8 traps.ev +arch_types armv8 vcpu.tc +arch_types aarch64 vcpu.tc diff --git a/hyp/interfaces/vcpu/include/vcpu.h b/hyp/interfaces/vcpu/include/vcpu.h index 04ddf92..9f1e54c 100644 --- a/hyp/interfaces/vcpu/include/vcpu.h +++ b/hyp/interfaces/vcpu/include/vcpu.h @@ -9,7 +9,7 @@ error_t vcpu_configure(thread_t *thread, vcpu_option_flags_t vcpu_options); -// Set a VCPU's initial execution state, including its entry point and context. +// Set a VCPU's initial execution state and start execution. // // The target thread must be a VCPU, its scheduling lock must be held by the // caller, and it must be currently blocked by the SCHEDULER_BLOCK_VCPU_OFF @@ -19,18 +19,32 @@ vcpu_configure(thread_t *thread, vcpu_option_flags_t vcpu_options); // // If the target VCPU has ever run, it must have called vcpu_poweroff() before // this function is called on it. -bool -vcpu_poweron(thread_t *vcpu, paddr_t entry_point, register_t context) - REQUIRE_SCHEDULER_LOCK(vcpu); +bool_result_t +vcpu_poweron(thread_t *vcpu, vmaddr_result_t entry_point, + register_result_t context) REQUIRE_SCHEDULER_LOCK(vcpu); -// Tear down the current thread's VCPU execution state. +// Halt execution of the current VCPU and tear down its execution state. // // The caller must be a runnable VCPU. This function will block the caller with // the SCHEDULER_BLOCK_VCPU_OFF flag and yield. It does not return on success; // if the thread is re-activated by a call to vcpu_poweron(), it will jump // directly to the new userspace context. +// +// If the last_vcpu argument is true, the caller must be the only powered-on +// VCPU attached to a VPM group, or else not attached to a VPM group. Otherwise, +// it must be attached to a VPM group with at least one other powered-on VCPU. +// If the last_vcpu argument is not correct, this call may return ERROR_DENIED. +// This check is not performed if the force argument is true. error_t -vcpu_poweroff(void); +vcpu_poweroff(bool last_vcpu, bool force); + +// Halt the current thread's VCPU execution state. +// +// This function will raise the VCPU's halt VIRQ, notifying its handler VM +// that the VCPU has halted. It can be called by any module that has +// already blocked a VCPU in a way that might need handling by RM. +noreturn void +vcpu_halted(void); // Suspend a VCPU. // @@ -143,3 +157,10 @@ vcpu_gpr_read(thread_t *thread, uint8_t reg_num); // Update the VCPU's general purpose registers void vcpu_gpr_write(thread_t *thread, uint8_t reg_num, register_t value); + +error_t +vcpu_bind_virq(thread_t *vcpu, vic_t *vic, virq_t virq, + vcpu_virq_type_t virq_type); + +error_t +vcpu_unbind_virq(thread_t *vcpu, vcpu_virq_type_t virq_type); diff --git a/hyp/interfaces/vcpu/vcpu.ev b/hyp/interfaces/vcpu/vcpu.ev index 74b878f..2777dd5 100644 --- a/hyp/interfaces/vcpu/vcpu.ev +++ b/hyp/interfaces/vcpu/vcpu.ev @@ -8,57 +8,68 @@ interface vcpu // // This event is triggered with the scheduler lock for the specified VCPU held // by the caller. The VCPU is blocked by the SCHEDULER_BLOCK_VCPU_OFF flag. -event vcpu_poweron +// +// The valid returns are OK or ERROR_ARGUMENT_INVALID or errors forwarded +// from the PSCI SMC cpu_on call. +setup_event vcpu_poweron param vcpu: thread_t * + return: error_t = OK + success: OK -// Triggered when a VCPU's state is torn down by a call to vcpu_poweroff(). +// Triggered when a VCPU halts itself by calling vcpu_poweroff(). // // This event is triggered with the scheduler lock for the current VCPU held // by the caller. The VCPU will be blocked by the SCHEDULER_BLOCK_VCPU_OFF // flag after the event completes. // -// The argument force is to override 'soft errors' such as PSCI preventing -// poweroff of the last vCPU in a group. +// The last_vcpu argument indicates whether this VCPU is expected to be the +// only powered-on VCPU in the VM. If the force argument is not set, handlers +// may assume or assert that the last_vcpu flag is correct, and return +// ERROR_DENIED otherwise. // -// Additionally, the only valid error returns for vcpu_poweroff handlers are -// OK or ERROR_DENIED. +// If the force argument is not set, handlers must return OK. No result other +// than OK or ERROR_DENIED is permitted. setup_event vcpu_poweroff param current: thread_t * + param last_vcpu: bool param force: bool return: error_t = OK success: OK -// Triggered when a VCPU has completed power off sequence. +// Triggered when a VCPU has stopped execution due to a power-off or halt. // -// This event is triggered without the scheduler lock for the current VCPU -// held by the caller. -event vcpu_poweredoff +// This event is triggered in the context of the VCPU that is stopping, with +// preemption disabled after marking it blocked. Handlers must not enable +// preemption, because the thread will not resume if preempted. +// +// If the VCPU starts again after this event, there will be a vcpu_started +// event with the warm_reset parameter set to false. +event vcpu_stopped // Triggered when the current VCPU is requesting entry to a virtual low-power // state. Entry to the low-power state is denied if any registered handler // returns an error. The error code may be passed on to the VCPU. // -// This event is triggered with the scheduler lock for the current VCPU held -// by the caller. +// This event is triggered with preemption disabled. // // Handlers that only want to check for pending wakeup events and return // ERROR_BUSY should register for vcpu_pending_wakeup instead. // // The valid errors for vcpu_suspend are OK, ERROR_DENIED, or ERROR_BUSY. setup_event vcpu_suspend + param current: thread_t * return: error_t = OK success: OK // Triggered when a VCPU has been woken from a virtual low-power state. // -// This event is triggered with the scheduler lock for the current VCPU -// held by the caller. Note that this is is triggered by the resuming VCPU, -// not directly by the call to vcpu_resume(). Also, note that the scheduler -// lock will have been dropped and re-acquired after the vcpu_suspend event. +// This event is triggered with preemption disabled. Note that this is +// triggered by the resuming VCPU, not directly by the call to vcpu_resume(). // // In many cases, handlers for this event will be the same as unwinders for // vcpu_suspend. event vcpu_resume + param current: thread_t * // Triggered when a VCPU is simulating a warm reset. // @@ -126,7 +137,29 @@ setup_event vcpu_block_start // Triggered when vcpu_block_finish() is called. event vcpu_block_finish -// Triggered when vcpu_thread_start() is called +// Triggered when a VCPU starts execution after a power-off or halt state. // -// This event is triggered when the VCPU returns from powerdown +// The warm_reset argument is true if the VCPU is completing a warm reset. If +// it is false, then the VCPU is either starting for the first time, or is +// starting after a previous vcpu_stopped event. If it is true, the VCPU had +// been running previously (or was in power-off suspend) and has not executed +// a vcpu_poweroff event. event vcpu_started + param warm_reset: bool + +// Triggered when vcpu_bind_virq() / vcpu_unbind_virq() are called. +// +// Handlers must be registered by any module that registers a VCPU IRQ type. +// They must call vic_bind_*() / vic_unbind_sync(), respectively, with the +// appropriate VIRQ source object and triggering type. +selector_event vcpu_bind_virq + selector type: vcpu_virq_type_t + param vcpu: thread_t * + param vic: vic_t * + param virq: virq_t + return: error_t = ERROR_ARGUMENT_INVALID + +selector_event vcpu_unbind_virq + selector type: vcpu_virq_type_t + param vcpu: thread_t * + return: error_t = ERROR_ARGUMENT_INVALID diff --git a/hyp/interfaces/vcpu/vcpu.tc b/hyp/interfaces/vcpu/vcpu.tc index e84e6e3..7685171 100644 --- a/hyp/interfaces/vcpu/vcpu.tc +++ b/hyp/interfaces/vcpu/vcpu.tc @@ -7,7 +7,11 @@ extend cap_rights_thread bitfield { 1 affinity bool; 2 priority bool; 3 timeslice bool; + 5 bind_virq bool; + 6 state bool; 7 lifecycle bool; + 8 write_context bool; + 9 disable bool; }; define vcpu_trap_result enumeration { @@ -16,3 +20,22 @@ define vcpu_trap_result enumeration { retry; fault; }; + +define vcpu_virq_type public enumeration(explicit) { + vcpu_run_wakeup = 1; +}; + +define vcpu_poweroff_flags public bitfield<64> { + 0 last_vcpu bool; + others unknown=0; +}; + +define vcpu_register_set public enumeration(explicit) { + // All values are arch-specific +}; + +define vcpu_poweron_flags public bitfield<64> { + 0 preserve_entry_point bool; + 1 preserve_context bool; + others unknown=0; +}; diff --git a/hyp/interfaces/vcpu_power/build.conf b/hyp/interfaces/vcpu_power/build.conf new file mode 100644 index 0000000..816c5af --- /dev/null +++ b/hyp/interfaces/vcpu_power/build.conf @@ -0,0 +1,3 @@ +# © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause diff --git a/hyp/interfaces/vcpu_run/build.conf b/hyp/interfaces/vcpu_run/build.conf new file mode 100644 index 0000000..de6ed89 --- /dev/null +++ b/hyp/interfaces/vcpu_run/build.conf @@ -0,0 +1,7 @@ +# © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +hypercalls vcpu_run.hvc +events vcpu_run.ev +types vcpu_run.tc diff --git a/hyp/interfaces/vcpu_run/include/vcpu_run.h b/hyp/interfaces/vcpu_run/include/vcpu_run.h new file mode 100644 index 0000000..d774ca8 --- /dev/null +++ b/hyp/interfaces/vcpu_run/include/vcpu_run.h @@ -0,0 +1,15 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +// Return true if vcpu_run is enabled for the specified VCPU. +// +// Returns false if vcpu_run is disabled or the specified thread is not a VCPU. +// +// Note that this is all-or-nothing for any given VCPU: either it is scheduled +// exclusively by calling the vcpu_run hypercall, or else it is scheduled by the +// EL2 scheduler and any attempt to call vcpu_run on it will fail. +// +// The caller must hold the specified thread's scheduler lock. +bool +vcpu_run_is_enabled(const thread_t *vcpu) REQUIRE_SCHEDULER_LOCK(vcpu); diff --git a/hyp/interfaces/vcpu_run/vcpu_run.ev b/hyp/interfaces/vcpu_run/vcpu_run.ev new file mode 100644 index 0000000..09b43bd --- /dev/null +++ b/hyp/interfaces/vcpu_run/vcpu_run.ev @@ -0,0 +1,15 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +interface vcpu_run + +// Triggered after vcpu_run_is_enabled() becomes true for a VCPU. The VCPU's +// scheduler lock is held by the caller. +event vcpu_run_enabled + param vcpu: thread_t * + +// Triggered before vcpu_run_is_enabled() becomes false for a VCPU. The VCPU's +// scheduler lock is held by the caller. +event vcpu_run_disabled + param vcpu: thread_t * diff --git a/hyp/interfaces/vcpu_run/vcpu_run.hvc b/hyp/interfaces/vcpu_run/vcpu_run.hvc new file mode 100644 index 0000000..aeb4983 --- /dev/null +++ b/hyp/interfaces/vcpu_run/vcpu_run.hvc @@ -0,0 +1,28 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +define vcpu_run hypercall { + call_num 0x65; + cap_id input type cap_id_t; + resume_data_0 input type register_t; + resume_data_1 input type register_t; + resume_data_2 input type register_t; + res0 input uregister; + error output enumeration error; + vcpu_state output enumeration vcpu_run_state; + state_data_0 output type register_t; + state_data_1 output type register_t; + state_data_2 output type register_t; +}; + +define vcpu_run_check hypercall { + call_num 0x68; + cap_id input type cap_id_t; + res0 input uregister; + error output enumeration error; + vcpu_state output enumeration vcpu_run_state; + state_data_0 output type register_t; + state_data_1 output type register_t; + state_data_2 output type register_t; +}; diff --git a/hyp/interfaces/vcpu_run/vcpu_run.tc b/hyp/interfaces/vcpu_run/vcpu_run.tc new file mode 100644 index 0000000..638d0b3 --- /dev/null +++ b/hyp/interfaces/vcpu_run/vcpu_run.tc @@ -0,0 +1,36 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +define vcpu_run_state public enumeration(explicit) { + // VCPU is ready to run on the next vcpu_run hypercall. + ready = 0; + + // VCPU is sleeping until an interrupt arrives. The wakeup IRQ will be + // asserted when that occurs. + // + // If the platform implements PSCI and the VCPU has called + // PSCI_CPU_SUSPEND, the first state data word will be the requested + // suspend state. Otherwise, it will be 0 (e.g. if the VCPU is + // executing a WFI instruction). + expects_wakeup = 1; + + // VCPU is powered off and cannot execute until another VCPU triggers + // a power-on event. The wakeup IRQ will be asserted when that occurs. + // The first state data word will be a vcpu_run_poweroff_flags bitmap. + powered_off = 2; + + // VCPU is blocked in EL2 for an unspecified reason. This state is + // normally transient, and the EL1 caller should retry after yielding. + blocked = 3; + + // VCPU has an unrecoverable fault. + fault = 6; +}; + +// Type of the first state data word for VCPU_RUN_STATE_POWERED_OFF. +define vcpu_run_poweroff_flags public bitfield<32> { + // True if the VCPU has permanently exited and cannot be powered on. + 0 exited bool; + others unknown = 0; +}; diff --git a/hyp/interfaces/vdevice/build.conf b/hyp/interfaces/vdevice/build.conf index fcc75fd..07ded41 100644 --- a/hyp/interfaces/vdevice/build.conf +++ b/hyp/interfaces/vdevice/build.conf @@ -2,5 +2,5 @@ # # SPDX-License-Identifier: BSD-3-Clause -arch_events armv8 vdevice.ev +events vdevice.ev types vdevice.tc diff --git a/hyp/interfaces/vdevice/include/vdevice.h b/hyp/interfaces/vdevice/include/vdevice.h new file mode 100644 index 0000000..abb2f17 --- /dev/null +++ b/hyp/interfaces/vdevice/include/vdevice.h @@ -0,0 +1,55 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +// Configure a vdevice that is backed by a physical memory extent. +// +// The given memextent is presumed to be mapped (either before or after this +// call) with reduced permissions, typically read-only, in the guest's address +// space. Any permission fault received for this memextent will be forwarded to +// the access handler for the vdevice. The vdevice's type must be set before +// calling this function. +// +// The caller should ensure that the memextent meets any requirements it has for +// size, memory type / cache attributes, permissions, etc. Normally this would +// be done by calling memextent_attach(). +error_t +vdevice_attach_phys(vdevice_t *vdevice, memextent_t *memextent); + +// Tear down a vdevice's attachment to a physical memory extent. This must only +// be called after receiving an OK result from vdevice_attach_phys(). +// +// Note that calls to the access handler are not guaranteed to be complete until +// an RCU grace period has elapsed after calling this function. If the access +// handler makes use of a pointer to or mapping of the memextent, the caller +// should not release or unmap the memextent until a grace period has elapsed. +void +vdevice_detach_phys(vdevice_t *vdevice, memextent_t *memextent); + +// Configure a vdevice that is not backed by physical memory. +// +// After this call succeeds, any translation faults in the specified range will +// be forwarded to the access handler for the vdevice. The vdevice's type must +// be set before calling this function. +// +// The given address range in the addrspace is presumed to not be mapped to any +// physical memextent. If such a mapping exists or is created later, it may +// shadow the device. +// +// The caller is responsible for ensuring that calls to this function are +// serialised for each device. Note that multiple calls are not generally useful +// because only one attachment is allowed. +// +// This function will retain a reference to the specified address space. +error_t +vdevice_attach_vmaddr(vdevice_t *vdevice, addrspace_t *addrspace, vmaddr_t ipa, + size_t size); + +// Tear down a vdevice's attachment to a guest address range. This must only +// be called after receiving an OK result from vdevice_attach_vmaddr(). +// +// Note that calls to the access handler are not guaranteed to be complete and +// it is not safe to call vdevice_attach_vmaddr() again until an RCU grace +// period has elapsed after calling this function. +void +vdevice_detach_vmaddr(vdevice_t *vdevice); diff --git a/hyp/interfaces/vdevice/armv8/vdevice.ev b/hyp/interfaces/vdevice/vdevice.ev similarity index 65% rename from hyp/interfaces/vdevice/armv8/vdevice.ev rename to hyp/interfaces/vdevice/vdevice.ev index f41cc8d..528cc5d 100644 --- a/hyp/interfaces/vdevice/armv8/vdevice.ev +++ b/hyp/interfaces/vdevice/vdevice.ev @@ -9,12 +9,13 @@ handled_event vdevice_access_fixed_addr param access_size: size_t param value: register_t * param is_write: bool + return: vcpu_trap_result_t = VCPU_TRAP_RESULT_UNHANDLED selector_event vdevice_access - selector_type: vdevice_type_t - param vdevice: vdevice_ptr_t - param pa: paddr_t - param ipa: vmaddr_t + selector type_: vdevice_type_t + param vdevice: vdevice_t * + param offset: size_t param access_size: size_t param value: register_t * param is_write: bool + return: vcpu_trap_result_t = VCPU_TRAP_RESULT_UNHANDLED diff --git a/hyp/interfaces/vdevice/vdevice.tc b/hyp/interfaces/vdevice/vdevice.tc index 94409ec..dcfce1b 100644 --- a/hyp/interfaces/vdevice/vdevice.tc +++ b/hyp/interfaces/vdevice/vdevice.tc @@ -6,6 +6,6 @@ define vdevice_type enumeration { NONE = 0; }; -define vdevice_ptr union { - null uintptr; +define vdevice structure { + type enumeration vdevice_type; }; diff --git a/hyp/interfaces/vectors/armv8/traps.ev b/hyp/interfaces/vectors/armv8/traps.ev index c160490..4f1050b 100644 --- a/hyp/interfaces/vectors/armv8/traps.ev +++ b/hyp/interfaces/vectors/armv8/traps.ev @@ -27,7 +27,7 @@ handled_event vectors_trap_sp_alignment_fault_el2 handled_event vectors_trap_brk_el2 param esr: ESR_EL2_t -#if defined(ARCH_ARM_8_3_PAUTH) && defined(ARCH_ARM_8_3_FPAC) +#if defined(ARCH_ARM_FEAT_PAuth) && defined(ARCH_ARM_FEAT_FPAC) handled_event vectors_trap_pauth_failed_el2 param iss: ESR_EL2_t #endif diff --git a/hyp/interfaces/vet/build.conf b/hyp/interfaces/vet/build.conf new file mode 100644 index 0000000..3491b85 --- /dev/null +++ b/hyp/interfaces/vet/build.conf @@ -0,0 +1,5 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +types vet.tc diff --git a/hyp/interfaces/vet/include/vet.h b/hyp/interfaces/vet/include/vet.h new file mode 100644 index 0000000..73da4a6 --- /dev/null +++ b/hyp/interfaces/vet/include/vet.h @@ -0,0 +1,119 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +// The Virtual Embedded Trace (VET) interface. + +// The vet_ordering variable is used as an artificial assembly ordering +// dependency for modules implementing this API. It orders individual asm +// statements with respect to each other in a way that is lighter weight than a +// full "memory" clobber. +extern asm_ordering_dummy_t vet_ordering; + +// Flush data for trace unit. +// +// Since a HW trace unit may have delays in transferring the trace byte stream +// to system infrastructure, we may need to explicitly flush it to ensure the +// trace stream is observable (mostly the trace buffer unit). +void +vet_flush_trace(thread_t *self); + +// Disable trace unit. +// +// Trace unit should be configured to not generate additional trace data after +// disabling. +void +vet_disable_trace(void); + +// Enable trace unit. +void +vet_enable_trace(void); + +// Save current trace unit's thread context. +// +// After thread context is saved, access to the trace unit registers is +// disabled. +// +// The implementation depends on the configured policy. This can save all +// registers or just control the trace's enable/disable. +void +vet_save_trace_thread_context(thread_t *self) REQUIRE_PREEMPT_DISABLED; + +// Restore a thread's trace buffer unit context. +// +// This reverses the actions of vet_save_trace_thread_context. +void +vet_restore_trace_thread_context(thread_t *self) REQUIRE_PREEMPT_DISABLED; + +// Save trace unit context for local CPU before suspend. +// +// Note that this may modify the trace unit state, so an aborted suspend must +// be followed by a call to vet_restore_trace_power_context(). +void +vet_save_trace_power_context(bool may_poweroff) REQUIRE_PREEMPT_DISABLED; + +// Restore trace unit context for local CPU after resume or aborted suspend. +void +vet_restore_trace_power_context(bool was_poweroff) REQUIRE_PREEMPT_DISABLED; + +// Flush data in the trace buffer unit. +// +// After this flush, all data pending in the trace buffer should be committed +// to memory. The implementation should ensure that this completes in finite +// time. If the trace buffer is located in memory with normal non-cacheable or +// device memory attributes, the write of trace data reaches the endpoint of +// that location in finite time. +void +vet_flush_buffer(thread_t *self); + +// Disable trace buffer unit. +// +// After disabling the trace buffer, it still host software stack's +// responsibility to check if all data is written out to the buffer. +void +vet_disable_buffer(void); + +// Enable trace buffer unit. +void +vet_enable_buffer(void); + +// Save trace buffer unit thread context before power-off. +// +// Similar to vet_save_trace_thread_context, this may save trace buffer +// registers / information. However, it does not change any configuration +// and does not need to be called for non-poweroff suspends. +void +vet_save_buffer_thread_context(thread_t *self) REQUIRE_PREEMPT_DISABLED; + +// Restore trace buffer unit thread context after power-off. +// +// This must be called when resuming from a power-off state. It need not be +// called when resuming from a retention state or aborting a power-off suspend. +void +vet_restore_buffer_thread_context(thread_t *self) REQUIRE_PREEMPT_DISABLED; + +// Save trace buffer context for local CPU before power-off. +// +// This does not need to save any information which is already saved by thread +// context. +// NOTE: if register access is disabled, then we need to enable it before save/ +// restore of the context. +void +vet_save_buffer_power_context(void) REQUIRE_PREEMPT_DISABLED; + +// Restore trace buffer context for local CPU after power-on. +void +vet_restore_buffer_power_context(void) REQUIRE_PREEMPT_DISABLED; + +// Update trace unit status for the current thread. +// +// This function checks the thread's current usage of trace infrastructure +// to guide the subsequent context-switch behaviour such as saving context. +void +vet_update_trace_unit_status(thread_t *self); + +// Update trace buffer status for the current thread +// +// Similar to vet_update_trace_unit_status() for the trace buffer. +void +vet_update_trace_buffer_status(thread_t *self); diff --git a/hyp/interfaces/vet/vet.tc b/hyp/interfaces/vet/vet.tc new file mode 100644 index 0000000..02b47bd --- /dev/null +++ b/hyp/interfaces/vet/vet.tc @@ -0,0 +1,7 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +extend platform_cpu_features bitfield { + auto trace_disable bool = 1; +}; diff --git a/hyp/interfaces/vgic/build.conf b/hyp/interfaces/vgic/build.conf new file mode 100644 index 0000000..e69de29 diff --git a/hyp/interfaces/vgic/include/vgic.h b/hyp/interfaces/vgic/include/vgic.h new file mode 100644 index 0000000..5ec0010 --- /dev/null +++ b/hyp/interfaces/vgic/include/vgic.h @@ -0,0 +1,6 @@ +// © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +const platform_mpidr_mapping_t * +vgic_get_mpidr_mapping(const vic_t *vic); diff --git a/hyp/interfaces/vic/include/vic.h b/hyp/interfaces/vic/include/vic.h index 391abff..ba11ac9 100644 --- a/hyp/interfaces/vic/include/vic.h +++ b/hyp/interfaces/vic/include/vic.h @@ -41,6 +41,10 @@ // the object that contains the VIRQ source structure), or else be in an RCU // read-side critical section. +// Get a pointer to a thread's vic +vic_t * +vic_get_vic(const thread_t *vcpu); + // Exclusively claim a shared VIRQ on the specified VIC. // // This prevents the VIRQ being claimed by any other source, and allows calls diff --git a/hyp/interfaces/vic/vic.hvc b/hyp/interfaces/vic/vic.hvc index 1b680c1..f520853 100644 --- a/hyp/interfaces/vic/vic.hvc +++ b/hyp/interfaces/vic/vic.hvc @@ -7,14 +7,14 @@ define hwirq_bind_virq hypercall { hwirq input type cap_id_t; vic input type cap_id_t; virq input type virq_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; define hwirq_unbind_virq hypercall { call_num 0x27; hwirq input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; @@ -35,7 +35,7 @@ define vic_attach_vcpu hypercall { vic input type cap_id_t; vcpu input type cap_id_t; index input type index_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; @@ -47,6 +47,6 @@ define vic_bind_msi_source hypercall { // Some platforms may not implement any MSI sources, or multiple types // of MSI source. For GICv3/v4, this is typically a virtual ITS. msi_source input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; diff --git a/hyp/interfaces/vic/vic.tc b/hyp/interfaces/vic/vic.tc index dbcb545..a34f6c6 100644 --- a/hyp/interfaces/vic/vic.tc +++ b/hyp/interfaces/vic/vic.tc @@ -20,6 +20,7 @@ extend hyp_api_flags0 bitfield { #endif define vic_option_flags public bitfield<64> { - 0 max_msis_valid bool = 1; - 63:1 res0_0 uregister = 0; + 0 max_msis_valid bool = 1; + 1 disable_default_addr bool = 1; + 63:2 res0_0 uregister = 0; }; diff --git a/hyp/interfaces/virq/virq.ev b/hyp/interfaces/virq/virq.ev index 98e219f..03f05c0 100644 --- a/hyp/interfaces/virq/virq.ev +++ b/hyp/interfaces/virq/virq.ev @@ -13,18 +13,18 @@ interface virq // delivery is possible, e.g. for IPC objects. // // However, the result of the handler will not necessarily affect the VIRQ -// source's state. Tn particular, after the handler returns false, clearing -// the interrupt's pending state may fail due to a concurrent virq_assert() -// call, or any other concurrent VIRQ or VIC operation, or spuriously. +// source's state. Changing the state may fail due to any concurrent VIRQ or +// VIC operation, or spuriously. In this case, the event will be triggered +// again, unless the concurrent operation made it redundant. // // The VIC implementation must guarantee that as soon as practical after an -// EOI by the VM for any VIRQ that has been asserted by its VIRQ source with -// edge_only=false, this event will be triggered repeatedly for that source -// until at least one of the following has occurred: +// EOI by the VM for any VIRQ that has been delivered in level-triggering mode +// after being asserted by its VIRQ source, this event will be triggered +// repeatedly for that source until at least one of the following has +// occurred: // // 1. the handler for this event returns false, and the level-pending flag is -// successfully cleared (rather than remaining set for any of the reasons -// listed above); +// successfully cleared; // // 2. the handler for this event returns true (which is the default when // there is no registered handler); or @@ -35,17 +35,19 @@ interface virq // acknowledges the IRQ but before EOI), this event may not be triggered. // // The event may not be triggered in a timely manner, or at all, after an EOI -// for an edge-triggered delivery. +// of a VIRQ that was delivered in edge-triggering mode. // // In addition to the guaranteed triggers above, this event may be triggered // at any other time while the interrupt is marked pending for level-triggered // delivery. Any memory reads executed by the handler are guaranteed to be -// ordered after the read that determines that the interrupt is marked pending. -// -// The reasserted argument is true if the virq_assert() function has been -// called more recently than the last acknowledgement of this VIRQ by the VM. -// Any memory reads executed by the handler are guaranteed to be ordered after -// the read that determines the value of the reasserted argument. +// ordered after the read that determines that the interrupt is marked pending +// for level-triggered delivery. +// +// The reasserted argument is true if the source has been asserted more +// recently than the last acknowledgement of its VIRQ by the VM. This may +// include a previous true result from this handler. Any memory reads executed +// by the handler are guaranteed to be ordered after the read that determines +// the value of the reasserted argument. // // If the reasserted argument is false, then a false return from the handler is // guaranteed to take effect only if it is ordered before any virq_assert(), diff --git a/hyp/interfaces/virq/virq.tc b/hyp/interfaces/virq/virq.tc index d246536..78b9151 100644 --- a/hyp/interfaces/virq/virq.tc +++ b/hyp/interfaces/virq/virq.tc @@ -13,9 +13,9 @@ define virq_trigger enumeration { define virq_source structure { virq type virq_t; trigger enumeration virq_trigger; - is_private bool; // RCU-protected pointer to the targeted controller. vic pointer(atomic) object vic; + is_private bool; }; extend error enumeration { diff --git a/hyp/interfaces/virtio_mmio/build.conf b/hyp/interfaces/virtio_mmio/build.conf new file mode 100644 index 0000000..ee6b29c --- /dev/null +++ b/hyp/interfaces/virtio_mmio/build.conf @@ -0,0 +1,7 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +first_class_object virtio_mmio +types virtio_mmio.tc +hypercalls virtio_mmio.hvc diff --git a/hyp/interfaces/virtio_mmio/virtio_mmio.hvc b/hyp/interfaces/virtio_mmio/virtio_mmio.hvc new file mode 100644 index 0000000..e3c1e7a --- /dev/null +++ b/hyp/interfaces/virtio_mmio/virtio_mmio.hvc @@ -0,0 +1,116 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +define virtio_mmio_configure hypercall { + call_num 0x49; + virtio_mmio input type cap_id_t; + memextent input type cap_id_t; + vqs_num input type count_t; + res0 input uregister; + error output enumeration error; +}; + +define virtio_mmio_backend_bind_virq hypercall { + call_num 0x4a; + virtio_mmio input type cap_id_t; + vic input type cap_id_t; + virq input type virq_t; + res0 input uregister; + error output enumeration error; +}; + +define virtio_mmio_backend_unbind_virq hypercall { + call_num 0x4b; + virtio_mmio input type cap_id_t; + res0 input uregister; + error output enumeration error; +}; + +define virtio_mmio_frontend_bind_virq hypercall { + call_num 0x4c; + virtio_mmio input type cap_id_t; + vic input type cap_id_t; + virq input type virq_t; + res0 input uregister; + error output enumeration error; +}; + +define virtio_mmio_frontend_unbind_virq hypercall { + call_num 0x4d; + virtio_mmio input type cap_id_t; + res0 input uregister; + error output enumeration error; +}; + +define virtio_mmio_backend_assert_virq hypercall { + call_num 0x4e; + virtio_mmio input type cap_id_t; + interrupt_status input uint32; + res0 input uregister; + error output enumeration error; +}; + +define virtio_mmio_backend_set_dev_features hypercall { + call_num 0x4f; + virtio_mmio input type cap_id_t; + sel input uint32; + dev_feat input uint32; + res0 input uregister; + error output enumeration error; +}; + +define virtio_mmio_backend_set_queue_num_max hypercall { + call_num 0x50; + virtio_mmio input type cap_id_t; + sel input uint32; + queue_num_max input uint32; + res0 input uregister; + error output enumeration error; +}; + +define virtio_mmio_backend_get_drv_features hypercall { + call_num 0x51; + virtio_mmio input type cap_id_t; + sel input uint32; + res0 input uregister; + error output enumeration error; + drv_feat output uint32; +}; + +define virtio_mmio_backend_get_queue_info hypercall { + call_num 0x52; + virtio_mmio input type cap_id_t; + sel input uint32; + res0 input uregister; + error output enumeration error; + queue_num output uint32; + queue_ready output uint32; + queue_desc output uint64; + queue_drv output uint64; + queue_dev output uint64; +}; + +define virtio_mmio_backend_get_notification hypercall { + call_num 0x53; + virtio_mmio input type cap_id_t; + res0 input uregister; + error output enumeration error; + vqs_bitmap output type register_t; + reason output bitfield virtio_mmio_notify_reason; +}; + +define virtio_mmio_backend_acknowledge_reset hypercall { + call_num 0x54; + virtio_mmio input type cap_id_t; + res0 input uregister; + error output enumeration error; +}; + +define virtio_mmio_backend_update_status hypercall { + call_num 0x55; + virtio_mmio input type cap_id_t; + val input uint32; + res0 input uregister; + error output enumeration error; +}; diff --git a/hyp/interfaces/virtio_mmio/virtio_mmio.tc b/hyp/interfaces/virtio_mmio/virtio_mmio.tc new file mode 100644 index 0000000..72c3a1c --- /dev/null +++ b/hyp/interfaces/virtio_mmio/virtio_mmio.tc @@ -0,0 +1,19 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#if defined(HYPERCALLS) +extend hyp_api_flags0 bitfield { + delete virtio_mmio; + 9 virtio_mmio bool = 1; +}; +#endif + +define virtio_mmio_notify_reason public bitfield<64> { + 0 new_buffer bool = 0; + 1 reset_rqst bool = 0; + 2 interrupt_ack bool = 0; + 3 driver_ok bool = 0; + 4 failed bool = 0; + others unknown = 0; +}; diff --git a/hyp/interfaces/vpm/include/vpm.h b/hyp/interfaces/vpm/include/vpm.h index 007240b..fe1614f 100644 --- a/hyp/interfaces/vpm/include/vpm.h +++ b/hyp/interfaces/vpm/include/vpm.h @@ -3,17 +3,23 @@ // SPDX-License-Identifier: BSD-3-Clause #define vpm__vcpus_state_foreach(cpu_index, cpu_state, vcpus_state, i) \ - cpu_state = vcpus_state & PSCI_VCPUS_STATE_PER_VCPU_MASK; \ + cpu_state = \ + (psci_cpu_state_t)(uint32_t)(vcpus_state & \ + PSCI_VCPUS_STATE_PER_VCPU_MASK); \ cpu_index = 0; \ for (index_t i = 0; i < PSCI_VCPUS_STATE_MAX_INDEX; \ i += PSCI_VCPUS_STATE_PER_VCPU_BITS, cpu_index++, \ - cpu_state = (vcpus_state >> i) & \ - PSCI_VCPUS_STATE_PER_VCPU_MASK) + cpu_state = \ + (psci_cpu_state_t)(uint32_t)((vcpus_state >> i) & \ + PSCI_VCPUS_STATE_PER_VCPU_MASK)) #define vpm_vcpus_state_foreach(cpu_index, cpu_state, vcpus_state) \ vpm__vcpus_state_foreach((cpu_index), (cpu_state), (vcpus_state), \ util_cpp_unique_ident(i)) +error_t +vpm_group_configure(vpm_group_t *vpm_group, vpm_group_option_flags_t flags); + error_t vpm_attach(vpm_group_t *pg, thread_t *thread, index_t index); diff --git a/hyp/interfaces/vpm/vpm.hvc b/hyp/interfaces/vpm/vpm.hvc index 8ac6d06..d089cf9 100644 --- a/hyp/interfaces/vpm/vpm.hvc +++ b/hyp/interfaces/vpm/vpm.hvc @@ -2,12 +2,20 @@ // // SPDX-License-Identifier: BSD-3-Clause +define vpm_group_configure hypercall { + call_num 0x66; + vpm_group input type cap_id_t; + flags input type vpm_group_option_flags; + res0 input uregister; + error output enumeration error; +}; + define vpm_group_attach_vcpu hypercall { call_num 0x3c; vpm_group input type cap_id_t; vcpu input type cap_id_t; index input type index_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; @@ -16,21 +24,21 @@ define vpm_group_bind_virq hypercall { vpm_group input type cap_id_t; vic input type cap_id_t; virq input type virq_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; define vpm_group_unbind_virq hypercall { call_num 0x44; vpm_group input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; define vpm_group_get_state hypercall { call_num 0x45; vpm_group input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; vpm_state output uregister; }; diff --git a/hyp/interfaces/vpm/vpm.tc b/hyp/interfaces/vpm/vpm.tc index ae2264b..0ea2389 100644 --- a/hyp/interfaces/vpm/vpm.tc +++ b/hyp/interfaces/vpm/vpm.tc @@ -9,7 +9,18 @@ extend hyp_api_flags0 bitfield { }; #endif -define vpm_state enumeration(explicit) { +extend cap_rights_vpm_group bitfield { + 0 attach_vcpu bool; + 1 bind_virq bool; + 2 query bool; +}; + +define vpm_group_option_flags public bitfield<64> { + 0 no_aggregation bool; + others unknown=0; +}; + +define vpm_state public enumeration(explicit) { no_state = 0; // Invalid / non existent running = 1; // VPM is active cpus_suspended = 2; // VPM is suspended after a CPU_SUSPEND call diff --git a/hyp/interfaces/vrtc/build.conf b/hyp/interfaces/vrtc/build.conf new file mode 100644 index 0000000..62f2647 --- /dev/null +++ b/hyp/interfaces/vrtc/build.conf @@ -0,0 +1,6 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +first_class_object vrtc +hypercalls vrtc.hvc diff --git a/hyp/interfaces/vrtc/vrtc.hvc b/hyp/interfaces/vrtc/vrtc.hvc new file mode 100644 index 0000000..304905c --- /dev/null +++ b/hyp/interfaces/vrtc/vrtc.hvc @@ -0,0 +1,37 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +define partition_create_vrtc hypercall { + vendor_hyp_call; + src_partition input type cap_id_t; + cspace input type cap_id_t; + res0 input uregister; + error output enumeration error; + new_cap output type cap_id_t; +}; + +define vrtc_configure hypercall { + vendor_hyp_call; + vrtc input type cap_id_t; + ipa input type vmaddr_t; + res0 input uregister; + error output enumeration error; +}; + +define vrtc_set_time_base hypercall { + vendor_hyp_call; + vrtc input type cap_id_t; + time_base input type nanoseconds_t; + sys_timer_ref input type ticks_t; + res0 input uregister; + error output enumeration error; +}; + +define vrtc_attach_addrspace hypercall { + vendor_hyp_call; + vrtc input type cap_id_t; + addrspace input type cap_id_t; + res0 input uregister; + error output enumeration error; +}; diff --git a/hyp/ipc/doorbell/src/doorbell.c b/hyp/ipc/doorbell/src/doorbell.c index 9c71afd..261fe22 100644 --- a/hyp/ipc/doorbell/src/doorbell.c +++ b/hyp/ipc/doorbell/src/doorbell.c @@ -113,7 +113,7 @@ doorbell_mask(doorbell_t *doorbell, doorbell_flags_t new_enable_mask, // Deassert if new mask disables all currently asserted flags (void)virq_clear(&doorbell->source); - } else if (!was_asserted & now_asserted) { + } else if (!was_asserted && now_asserted) { // Assert if new mask enables flags that are already set (void)virq_assert(&doorbell->source, false); doorbell->flags &= ~doorbell->ack_mask; diff --git a/hyp/ipc/doorbell/src/hypercalls.c b/hyp/ipc/doorbell/src/hypercalls.c index 39d8365..371e3e7 100644 --- a/hyp/ipc/doorbell/src/hypercalls.c +++ b/hyp/ipc/doorbell/src/hypercalls.c @@ -73,7 +73,7 @@ hypercall_doorbell_send_result_t hypercall_doorbell_send(cap_id_t doorbell_cap, uint64_t new_flags) { hypercall_doorbell_send_result_t ret = { 0 }; - cspace_t *cspace = cspace_get_self(); + cspace_t *cspace = cspace_get_self(); doorbell_ptr_result_t p = cspace_lookup_doorbell( cspace, doorbell_cap, CAP_RIGHTS_DOORBELL_SEND); @@ -101,7 +101,7 @@ hypercall_doorbell_receive_result_t hypercall_doorbell_receive(cap_id_t doorbell_cap, uint64_t clear_flags) { hypercall_doorbell_receive_result_t ret = { 0 }; - cspace_t *cspace = cspace_get_self(); + cspace_t *cspace = cspace_get_self(); doorbell_ptr_result_t p = cspace_lookup_doorbell( cspace, doorbell_cap, CAP_RIGHTS_DOORBELL_RECEIVE); diff --git a/hyp/ipc/msgqueue/include/msgqueue_common.h b/hyp/ipc/msgqueue/include/msgqueue_common.h index ebb17fc..eeeed0a 100644 --- a/hyp/ipc/msgqueue/include/msgqueue_common.h +++ b/hyp/ipc/msgqueue/include/msgqueue_common.h @@ -1,5 +1,4 @@ -// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. -// +// 2020 Qualcomm Innovation Center, Inc. All rights reserved. // SPDX-License-Identifier: BSD-3-Clause // Configure the message queue. diff --git a/hyp/ipc/msgqueue/msgqueue.tc b/hyp/ipc/msgqueue/msgqueue.tc index 403d47a..6ec1dd6 100644 --- a/hyp/ipc/msgqueue/msgqueue.tc +++ b/hyp/ipc/msgqueue/msgqueue.tc @@ -16,7 +16,7 @@ extend cap_rights_msgqueue bitfield { }; extend msgqueue object { - buf pointer char; + buf pointer uint8; count type count_t; queue_size size; max_msg_size size; diff --git a/hyp/ipc/msgqueue/src/hypercalls.c b/hyp/ipc/msgqueue/src/hypercalls.c index b8e9c6c..1489965 100644 --- a/hyp/ipc/msgqueue/src/hypercalls.c +++ b/hyp/ipc/msgqueue/src/hypercalls.c @@ -130,7 +130,7 @@ hypercall_msgqueue_send(cap_id_t msgqueue_cap, size_t size, user_ptr_t data, msgqueue_send_flags_t send_flags) { hypercall_msgqueue_send_result_t ret = { 0 }; - cspace_t *cspace = cspace_get_self(); + cspace_t *cspace = cspace_get_self(); msgqueue_ptr_result_t p = cspace_lookup_msgqueue( cspace, msgqueue_cap, CAP_RIGHTS_MSGQUEUE_SEND); @@ -157,7 +157,7 @@ hypercall_msgqueue_receive(cap_id_t msgqueue_cap, user_ptr_t buffer, size_t buf_size) { hypercall_msgqueue_receive_result_t ret = { 0 }; - cspace_t *cspace = cspace_get_self(); + cspace_t *cspace = cspace_get_self(); msgqueue_ptr_result_t p = cspace_lookup_msgqueue( cspace, msgqueue_cap, CAP_RIGHTS_MSGQUEUE_RECEIVE); @@ -253,7 +253,7 @@ hypercall_msgqueue_configure(cap_id_t msgqueue_cap, msgqueue_create_info_t create_info) { error_t err; - cspace_t *cspace = cspace_get_self(); + cspace_t *cspace = cspace_get_self(); object_type_t type; object_ptr_result_t o = cspace_lookup_object_any( diff --git a/hyp/ipc/msgqueue/src/msgqueue.c b/hyp/ipc/msgqueue/src/msgqueue.c index a647989..b10d6ce 100644 --- a/hyp/ipc/msgqueue/src/msgqueue.c +++ b/hyp/ipc/msgqueue/src/msgqueue.c @@ -202,7 +202,7 @@ msgqueue_handle_object_activate_msgqueue(msgqueue_t *msgqueue) goto out; } - msgqueue->buf = res.r; + msgqueue->buf = (uint8_t *)res.r; msgqueue->count = 0U; msgqueue->queue_size = queue_size; msgqueue->head = 0U; diff --git a/hyp/ipc/msgqueue/src/msgqueue_common.c b/hyp/ipc/msgqueue/src/msgqueue_common.c index aaaf9ca..1ad3443 100644 --- a/hyp/ipc/msgqueue/src/msgqueue_common.c +++ b/hyp/ipc/msgqueue/src/msgqueue_common.c @@ -56,11 +56,12 @@ msgqueue_send_msg(msgqueue_t *msgqueue, size_t size, kernel_or_gvaddr_t msg, } } - (void)memcpy(msgqueue->buf + msgqueue->tail, &size, sizeof(size_t)); + (void)memcpy(msgqueue->buf + msgqueue->tail, (uint8_t *)&size, + sizeof(size_t)); msgqueue->count++; // Update tail value - msgqueue->tail += msgqueue->max_msg_size + sizeof(size_t); + msgqueue->tail += (count_t)(msgqueue->max_msg_size + sizeof(size_t)); if (msgqueue->tail == msgqueue->queue_size) { msgqueue->tail = 0U; @@ -104,7 +105,8 @@ msgqueue_receive_msg(msgqueue_t *msgqueue, kernel_or_gvaddr_t buffer, goto out; } - memcpy(&size, msgqueue->buf + msgqueue->head, sizeof(size_t)); + (void)memcpy((uint8_t *)&size, msgqueue->buf + msgqueue->head, + sizeof(size_t)); // Dequeue message from the head of the queue void *hyp_va = @@ -130,7 +132,8 @@ msgqueue_receive_msg(msgqueue_t *msgqueue, kernel_or_gvaddr_t buffer, msgqueue->count--; // Update head value - msgqueue->head += msgqueue->max_msg_size + sizeof(size_t); + msgqueue->head += (count_t)(msgqueue->max_msg_size + sizeof(size_t)); + assert(msgqueue->head <= msgqueue->queue_size); if (msgqueue->head == msgqueue->queue_size) { msgqueue->head = 0U; @@ -168,7 +171,8 @@ msgqueue_flush_queue(msgqueue_t *msgqueue) (void)virq_clear(&msgqueue->rcv_source); } - memset(msgqueue->buf, 0, msgqueue->queue_size); + (void)memset_s(msgqueue->buf, msgqueue->queue_size, 0, + msgqueue->queue_size); msgqueue->count = 0U; msgqueue->head = 0U; msgqueue->tail = 0U; diff --git a/hyp/mem/addrspace/aarch64/addrspace.ev b/hyp/mem/addrspace/aarch64/addrspace.ev new file mode 100644 index 0000000..dc48ae1 --- /dev/null +++ b/hyp/mem/addrspace/aarch64/addrspace.ev @@ -0,0 +1,21 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module addrspace + +#if defined(INTERFACE_VCPU_RUN) +subscribe vdevice_access_fixed_addr + priority last + +subscribe vcpu_run_check + require_scheduler_lock(vcpu) + +subscribe vcpu_run_resume[VCPU_RUN_STATE_ADDRSPACE_VMMIO_READ] + handler addrspace_handle_vcpu_run_resume_read(vcpu, resume_data_0) + require_scheduler_lock(vcpu) + +subscribe vcpu_run_resume[VCPU_RUN_STATE_ADDRSPACE_VMMIO_WRITE] + handler addrspace_handle_vcpu_run_resume_write(vcpu) + require_scheduler_lock(vcpu) +#endif diff --git a/hyp/mem/addrspace/aarch64/addrspace.tc b/hyp/mem/addrspace/aarch64/addrspace.tc new file mode 100644 index 0000000..e31aea6 --- /dev/null +++ b/hyp/mem/addrspace/aarch64/addrspace.tc @@ -0,0 +1,21 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#if defined(INTERFACE_VCPU_RUN) +extend scheduler_block enumeration { + addrspace_vmmio_access; +}; + +extend vcpu_run_state enumeration { + addrspace_vmmio_read = 4; + addrspace_vmmio_write = 5; +}; + +extend thread object module addrspace { + vmmio_access_ipa type vmaddr_t; + vmmio_access_size size; + vmmio_access_value uregister; + vmmio_access_write bool; +}; +#endif diff --git a/hyp/mem/addrspace/aarch64/src/lookup.c b/hyp/mem/addrspace/aarch64/src/lookup.c index 57c09a7..dbffa16 100644 --- a/hyp/mem/addrspace/aarch64/src/lookup.c +++ b/hyp/mem/addrspace/aarch64/src/lookup.c @@ -4,76 +4,109 @@ #include #include +#include #include #include #include +#include #include +// Check whether an address range is within the address space. +error_t +addrspace_check_range(addrspace_t *addrspace, vmaddr_t base, size_t size) +{ + error_t err; + + if ((size != 0U) && util_add_overflows(base, size - 1U)) { + err = ERROR_ADDR_OVERFLOW; + goto out; + } + + count_t bits = addrspace->vm_pgtable.control.address_bits; + // Ensure that the value used for shift operation is within the limit 0 + // to sizeof(type) -1 + assert(bits < (sizeof(vmaddr_t) * (size_t)CHAR_BIT)); + if (base >= util_bit(bits)) { + err = ERROR_ADDR_INVALID; + } else if ((size != 0U) && ((base + size - 1U) >= util_bit(bits))) { + err = ERROR_ARGUMENT_SIZE; + } else { + err = OK; + } + +out: + return err; +} + +#if defined(INTERFACE_VCPU) paddr_result_t addrspace_va_to_pa_read(gvaddr_t addr) { - bool success; - paddr_t pa = 0U; + paddr_result_t ret; thread_t *thread = thread_get_self(); assert(thread->kind == THREAD_KIND_VCPU); PAR_EL1_base_t saved_par = register_PAR_EL1_base_read_ordered(&asm_ordering); - - __asm__ volatile("at S12E1R, %[addr] ;" - "isb ;" + __asm__ volatile("at S12E1R, %[addr]" : "+m"(asm_ordering) : [addr] "r"(addr)); - + asm_context_sync_ordered(&asm_ordering); PAR_EL1_t par = { .base = register_PAR_EL1_base_read_ordered(&asm_ordering), }; - success = !PAR_EL1_base_get_F(&par.base); + register_PAR_EL1_base_write_ordered(saved_par, &asm_ordering); - if (success) { - pa = PAR_EL1_F0_get_PA(&par.f0); + if (!PAR_EL1_base_get_F(&par.base)) { + paddr_t pa = PAR_EL1_F0_get_PA(&par.f0); pa |= (gvaddr_t)addr & 0xfffU; + ret = paddr_result_ok(pa); + } else if (PAR_EL1_F1_get_S(&par.f1)) { + // Stage 2 fault + ret = paddr_result_error(ERROR_DENIED); + } else { + // Stage 1 fault + ret = paddr_result_error(ERROR_ADDR_INVALID); } - register_PAR_EL1_base_write_ordered(saved_par, &asm_ordering); - - return success ? paddr_result_ok(pa) - : paddr_result_error(ERROR_ADDR_INVALID); + return ret; } vmaddr_result_t addrspace_va_to_ipa_read(gvaddr_t addr) { - bool success; - vmaddr_t ipa = 0U; + vmaddr_result_t ret; thread_t *thread = thread_get_self(); assert(thread->kind == THREAD_KIND_VCPU); PAR_EL1_base_t saved_par = register_PAR_EL1_base_read_ordered(&asm_ordering); - - __asm__ volatile("at S1E1R, %[addr] ;" - "isb ;" + __asm__ volatile("at S1E1R, %[addr]" : "+m"(asm_ordering) : [addr] "r"(addr)); - + asm_context_sync_ordered(&asm_ordering); PAR_EL1_t par = { .base = register_PAR_EL1_base_read_ordered(&asm_ordering), }; - success = !PAR_EL1_base_get_F(&par.base); + register_PAR_EL1_base_write_ordered(saved_par, &asm_ordering); - if (success) { - ipa = PAR_EL1_F0_get_PA(&par.f0); + if (!PAR_EL1_base_get_F(&par.base)) { + vmaddr_t ipa = PAR_EL1_F0_get_PA(&par.f0); ipa |= (vmaddr_t)addr & 0xfffU; + ret = vmaddr_result_ok(ipa); + } else if (PAR_EL1_F1_get_S(&par.f1)) { + // Stage 2 fault (on a S1 page table walk access) + ret = vmaddr_result_error(ERROR_DENIED); + } else { + // Stage 1 fault + ret = vmaddr_result_error(ERROR_ADDR_INVALID); } - register_PAR_EL1_base_write_ordered(saved_par, &asm_ordering); - - return success ? vmaddr_result_ok(ipa) - : vmaddr_result_error(ERROR_ADDR_INVALID); + return ret; } +#endif diff --git a/hyp/mem/addrspace/aarch64/src/vmmio.c b/hyp/mem/addrspace/aarch64/src/vmmio.c new file mode 100644 index 0000000..461910d --- /dev/null +++ b/hyp/mem/addrspace/aarch64/src/vmmio.c @@ -0,0 +1,108 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#if defined(INTERFACE_VCPU_RUN) +#include +#include +#include +#include +#include +#include + +#include "event_handlers.h" + +vcpu_trap_result_t +addrspace_handle_vdevice_access_fixed_addr(vmaddr_t ipa, size_t access_size, + register_t *value, bool is_write) +{ + thread_t *current = thread_get_self(); + + vcpu_trap_result_t ret = VCPU_TRAP_RESULT_UNHANDLED; + + scheduler_lock(current); + if (vcpu_run_is_enabled(current)) { + addrspace_t *addrspace = current->addrspace; + + // We need to call gpt_lookup() in an RCU critical section to + // ensure that levels aren't freed while it is accessing them, + // but we can end the critical section immediately afterwards + // since we are not dereferencing anything. + rcu_read_start(); + gpt_lookup_result_t result = + gpt_lookup(&addrspace->vmmio_ranges, ipa, access_size); + rcu_read_finish(); + + if ((result.size == access_size) && + (result.entry.type == GPT_TYPE_VMMIO_RANGE)) { + current->addrspace_vmmio_access_ipa = ipa; + current->addrspace_vmmio_access_size = access_size; + current->addrspace_vmmio_access_value = + is_write ? *value : 0U; + current->addrspace_vmmio_access_write = is_write; + + scheduler_block(current, + SCHEDULER_BLOCK_ADDRSPACE_VMMIO_ACCESS); + scheduler_unlock_nopreempt(current); + (void)scheduler_schedule(); + scheduler_lock_nopreempt(current); + + if (!is_write) { + *value = current->addrspace_vmmio_access_value; + } + + ret = VCPU_TRAP_RESULT_EMULATED; + } + } + scheduler_unlock(current); + + return ret; +} + +vcpu_run_state_t +addrspace_handle_vcpu_run_check(const thread_t *vcpu, register_t *state_data_0, + register_t *state_data_1, + register_t *state_data_2) +{ + vcpu_run_state_t ret; + + if (scheduler_is_blocked(vcpu, + SCHEDULER_BLOCK_ADDRSPACE_VMMIO_ACCESS)) { + *state_data_0 = (register_t)vcpu->addrspace_vmmio_access_ipa; + *state_data_1 = (register_t)vcpu->addrspace_vmmio_access_size; + *state_data_2 = (register_t)vcpu->addrspace_vmmio_access_value; + ret = vcpu->addrspace_vmmio_access_write + ? VCPU_RUN_STATE_ADDRSPACE_VMMIO_WRITE + : VCPU_RUN_STATE_ADDRSPACE_VMMIO_READ; + } else { + ret = VCPU_RUN_STATE_BLOCKED; + } + + return ret; +} + +error_t +addrspace_handle_vcpu_run_resume_read(thread_t *vcpu, register_t resume_data_0) +{ + assert(scheduler_is_blocked(vcpu, + SCHEDULER_BLOCK_ADDRSPACE_VMMIO_ACCESS) && + !vcpu->addrspace_vmmio_access_write); + vcpu->addrspace_vmmio_access_value = resume_data_0; + (void)scheduler_unblock(vcpu, SCHEDULER_BLOCK_ADDRSPACE_VMMIO_ACCESS); + return OK; +} + +error_t +addrspace_handle_vcpu_run_resume_write(thread_t *vcpu) +{ + assert(scheduler_is_blocked(vcpu, + SCHEDULER_BLOCK_ADDRSPACE_VMMIO_ACCESS) && + vcpu->addrspace_vmmio_access_write); + (void)scheduler_unblock(vcpu, SCHEDULER_BLOCK_ADDRSPACE_VMMIO_ACCESS); + return OK; +} + +#endif // INTERFACE_VCPU_RUN diff --git a/hyp/mem/addrspace/addrspace.ev b/hyp/mem/addrspace/addrspace.ev index 3f1ed46..6a4a608 100644 --- a/hyp/mem/addrspace/addrspace.ev +++ b/hyp/mem/addrspace/addrspace.ev @@ -6,10 +6,17 @@ module addrspace subscribe boot_cold_init() +#if defined(INTERFACE_VCPU) subscribe object_activate_thread subscribe object_deactivate_thread +subscribe thread_load_state + handler addrspace_context_switch_load() + +subscribe thread_get_stack_base[THREAD_KIND_VCPU](thread) +#endif + subscribe object_create_addrspace(addrspace_create) unwinder addrspace_unwind_object_create_addrspace(addrspace_create) @@ -19,10 +26,8 @@ subscribe object_activate_addrspace priority last subscribe object_deactivate_addrspace + priority last -subscribe thread_load_state - handler addrspace_context_switch_load() - -subscribe rootvm_init(root_thread, root_cspace, env_data) - -subscribe thread_get_stack_base[THREAD_KIND_VCPU](thread) +#if defined(MODULE_VM_ROOTVM) +subscribe rootvm_init(root_thread, root_cspace, qcbor_enc_ctxt) +#endif diff --git a/hyp/mem/addrspace/addrspace.tc b/hyp/mem/addrspace/addrspace.tc index b4c8b7c..4c5d649 100644 --- a/hyp/mem/addrspace/addrspace.tc +++ b/hyp/mem/addrspace/addrspace.tc @@ -4,13 +4,30 @@ #include -define ROOT_VM_VMID constant type count_t = 255; +define ROOT_VM_VMID constant type vmid_t = 255; define ADDRSPACE_MAX_THREADS constant type count_t = PLATFORM_MAX_CORES; +#if defined(INTERFACE_VCPU_RUN) +// This upper bound exists to prevent an unprivileged VM with the ability to +// add VMMIO ranges to its own address space executing a denial of service +// attack by adding many small ranges to the GPT and exhausting the partition +// heap. In order to safely reduce this count when a range is removed, we also +// require that removed ranges are identical to added ranges. +define ADDRSPACE_MAX_VMMIO_RANGES constant type count_t = 128; +#endif + extend cap_rights_addrspace bitfield { - 0 attach bool; - 1 map bool; + 0 attach bool; + 1 map bool; + 2 lookup bool; + 3 add_vmmio_range bool; +}; + +define addrspace_information_area structure { + hyp_va pointer structure addrspace_info_area_layout; + ipa type vmaddr_t; + me pointer object memextent; }; extend addrspace object { @@ -18,9 +35,24 @@ extend addrspace object { pgtable_lock structure spinlock; vm_pgtable structure pgtable_vm; vmid type vmid_t; - vm_read_only bool; - stack_range structure virt_range; + read_only bool; + platform_pgtable bool; + hyp_va_range structure virt_range; stack_bitmap BITMAP(ADDRSPACE_MAX_THREADS, atomic); + info_area structure addrspace_information_area; +#if defined(INTERFACE_VCPU_RUN) + vmmio_range_lock structure spinlock; + vmmio_ranges structure gpt; + vmmio_range_count type count_t; +#endif +}; + +extend gpt_type enumeration { + vmmio_range; +}; + +extend gpt_value union { + vmmio_range_base type vmaddr_t; }; extend thread object { @@ -28,10 +60,6 @@ extend thread object { stack_map_index type index_t; }; -extend boot_env_data structure { - addrspace_capid type cap_id_t; -}; - extend trace_ids bitfield { 15:0 vmid type vmid_t; }; diff --git a/hyp/mem/addrspace/armv8/abort.ev b/hyp/mem/addrspace/armv8/abort.ev new file mode 100644 index 0000000..c61232c --- /dev/null +++ b/hyp/mem/addrspace/armv8/abort.ev @@ -0,0 +1,13 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module addrspace + +#if defined(INTERFACE_VCPU) +subscribe vcpu_trap_pf_abort_guest(esr, ipa, far) + priority last + +subscribe vcpu_trap_data_abort_guest(esr, ipa, far) + priority last +#endif diff --git a/hyp/mem/addrspace/armv8/src/abort.c b/hyp/mem/addrspace/armv8/src/abort.c new file mode 100644 index 0000000..14dc93b --- /dev/null +++ b/hyp/mem/addrspace/armv8/src/abort.c @@ -0,0 +1,195 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#if defined(INTERFACE_VCPU) +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include "event_handlers.h" + +static bool +addrspace_undergoing_bbm(addrspace_t *addrspace) +{ + bool ret; + + if (addrspace->platform_pgtable) { + ret = platform_pgtable_undergoing_bbm(); + } else { +#if (CPU_PGTABLE_BBM_LEVEL == 0) && !defined(PLATFORM_PGTABLE_AVOID_BBM) + // We use break-before-make for block splits and merges, + // which might affect addresses outside the operation range + // and therefore might cause faults that should be hidden. + if (!spinlock_trylock(&addrspace->pgtable_lock)) { + ret = true; + } else { + spinlock_release(&addrspace->pgtable_lock); + ret = false; + } +#else + // Break-before-make is only used when changing the output + // address or cache attributes, which shouldn't happen while + // the affected pages are being accessed. + ret = false; +#endif + } + + return ret; +} + +static vcpu_trap_result_t +addrspace_handle_guest_tlb_conflict(vmaddr_result_t ipa, FAR_EL2_t far, + bool s1ptw) +{ + // If this fault was not on a stage 1 PT walk, the ipa argument is not + // valid, because the architecture allows the TLB to avoid caching it. + // We can do a lookup on the VA to try to find it. This may fail if the + // CPU caches S1-only translations and the conflict is in that cache. + // + // For a fault on a stage 1 PT walk, the ipa argument is always valid. + if (!s1ptw) { + ipa = addrspace_va_to_ipa_read( + FAR_EL2_get_VirtualAddress(&far)); + } else { + assert(ipa.e == OK); + } + + asm_ordering_dummy_t tlbi_s2_ordering; + if (ipa.e == OK) { + // If the IPA is valid, the conflict may have been between S2 + // TLB entries, so flush the IPA from the S2 TLB. Note that if + // our IPA lookup above failed, the conflict must be in S1+S2 or + // S1-only entries, so no S2 flush is needed. + vmsa_tlbi_ipa_input_t ipa_input = vmsa_tlbi_ipa_input_default(); + vmsa_tlbi_ipa_input_set_IPA(&ipa_input, ipa.r); + __asm__ volatile( + "tlbi IPAS2E1, %[VA]" + : "=m"(tlbi_s2_ordering) + : [VA] "r"(vmsa_tlbi_ipa_input_raw(ipa_input))); + } + + // Regardless of whether the IPA is valid, there is always a possibility + // that the conflict was on S1+S2 or S1-only entries. So we always flush + // by VA. If the fault was on a stage 1 page table walk, the fault may + // have been on a cached next-level entry, so we flush those too. + asm_ordering_dummy_t tlbi_s1_ordering; + vmsa_tlbi_vaa_input_t va_input = vmsa_tlbi_vaa_input_default(); + vmsa_tlbi_vaa_input_set_VA(&va_input, FAR_EL2_get_VirtualAddress(&far)); + if (s1ptw) { + __asm__ volatile("tlbi VAAE1, %[VA]" + : "=m"(tlbi_s1_ordering) + : [VA] "r"(vmsa_tlbi_vaa_input_raw(va_input))); + } else { + __asm__ volatile("tlbi VAALE1, %[VA]" + : "=m"(tlbi_s1_ordering) + : [VA] "r"(vmsa_tlbi_vaa_input_raw(va_input))); + } + + __asm__ volatile("dsb nsh" ::"m"(tlbi_s1_ordering), + "m"(tlbi_s2_ordering)); + + return VCPU_TRAP_RESULT_RETRY; +} + +// Retry faults if they may have been caused by break before make during block +// splits in the direct physical access region +static vcpu_trap_result_t +addrspace_handle_guest_translation_fault(FAR_EL2_t far) +{ + vcpu_trap_result_t ret; + + uintptr_t addr = FAR_EL2_get_VirtualAddress(&far); + + thread_t *current = thread_get_self(); + assert(current != NULL); + + addrspace_t *addrspace = current->addrspace; + assert(addrspace != NULL); + + rcu_read_start(); + if (!addrspace_undergoing_bbm(addrspace)) { + // There is no BBM in progress, but there might have been when + // the fault occurred. Perform a lookup to see whether the + // accessed address is now mapped in S2. + // + // If the accessed address no longer faults in stage 2, we can + // just retry the faulting access. Otherwise we can consider the + // fault to be fatal, because there is no BBM operation still in + // progress. + ret = (addrspace_va_to_pa_read(addr).e != ERROR_DENIED) + ? VCPU_TRAP_RESULT_RETRY + : VCPU_TRAP_RESULT_UNHANDLED; + } else { + // A map operation is in progress, so retry until it finishes. + // Note that we might get stuck here if the page table is + // corrupt! + ret = VCPU_TRAP_RESULT_RETRY; + } + rcu_read_finish(); + + return ret; +} + +vcpu_trap_result_t +addrspace_handle_vcpu_trap_data_abort_guest(ESR_EL2_t esr, vmaddr_result_t ipa, + FAR_EL2_t far) +{ + vcpu_trap_result_t ret = VCPU_TRAP_RESULT_UNHANDLED; + + ESR_EL2_ISS_DATA_ABORT_t iss = + ESR_EL2_ISS_DATA_ABORT_cast(ESR_EL2_get_ISS(&esr)); + iss_da_ia_fsc_t fsc = ESR_EL2_ISS_DATA_ABORT_get_DFSC(&iss); + + if (fsc == ISS_DA_IA_FSC_TLB_CONFLICT) { + ret = addrspace_handle_guest_tlb_conflict( + ipa, far, ESR_EL2_ISS_DATA_ABORT_get_S1PTW(&iss)); + } + + // Only translation faults can be caused by BBM + if ((fsc == ISS_DA_IA_FSC_TRANSLATION_1) || + (fsc == ISS_DA_IA_FSC_TRANSLATION_2) || + (fsc == ISS_DA_IA_FSC_TRANSLATION_3)) { + ret = addrspace_handle_guest_translation_fault(far); + } + + return ret; +} + +vcpu_trap_result_t +addrspace_handle_vcpu_trap_pf_abort_guest(ESR_EL2_t esr, vmaddr_result_t ipa, + FAR_EL2_t far) +{ + vcpu_trap_result_t ret = VCPU_TRAP_RESULT_UNHANDLED; + + ESR_EL2_ISS_INST_ABORT_t iss = + ESR_EL2_ISS_INST_ABORT_cast(ESR_EL2_get_ISS(&esr)); + iss_da_ia_fsc_t fsc = ESR_EL2_ISS_INST_ABORT_get_IFSC(&iss); + + if (fsc == ISS_DA_IA_FSC_TLB_CONFLICT) { + ret = addrspace_handle_guest_tlb_conflict( + ipa, far, ESR_EL2_ISS_INST_ABORT_get_S1PTW(&iss)); + } + + // Only translation faults can be caused by BBM + if ((fsc == ISS_DA_IA_FSC_TRANSLATION_1) || + (fsc == ISS_DA_IA_FSC_TRANSLATION_2) || + (fsc == ISS_DA_IA_FSC_TRANSLATION_3)) { + ret = addrspace_handle_guest_translation_fault(far); + } + + return ret; +} +#else +extern char unused; +#endif diff --git a/hyp/mem/addrspace/build.conf b/hyp/mem/addrspace/build.conf index dcdfcef..c651f06 100644 --- a/hyp/mem/addrspace/build.conf +++ b/hyp/mem/addrspace/build.conf @@ -6,4 +6,8 @@ interface addrspace events addrspace.ev types addrspace.tc source addrspace.c hypercalls.c -arch_source aarch64 lookup.c +arch_source aarch64 lookup.c vmmio.c +arch_events aarch64 addrspace.ev +arch_types aarch64 addrspace.tc +arch_events armv8 abort.ev +arch_source armv8 abort.c diff --git a/hyp/mem/addrspace/src/addrspace.c b/hyp/mem/addrspace/src/addrspace.c index f7e628b..3432fd8 100644 --- a/hyp/mem/addrspace/src/addrspace.c +++ b/hyp/mem/addrspace/src/addrspace.c @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -12,16 +13,24 @@ #include #include #include +#include #include #include +#include +#include #include #include +#include #include #include +#include #include #include +#include #include +#include + #include "event_handlers.h" // FIXME: This file contains architecture specific details and should be @@ -30,14 +39,17 @@ extern VTTBR_EL2_t hlos_vm_vttbr; // FIXME: Limit VMIDs to reduce bitmap size -#if defined(ARCH_ARM_8_1_VMID16) -#define NUM_VMIDS 256 +#if defined(ARCH_ARM_FEAT_VMID16) +#define NUM_VMIDS 256U #else -#define NUM_VMIDS 256 +#define NUM_VMIDS 256U #endif static _Atomic BITMAP_DECLARE(NUM_VMIDS, addrspace_vmids); +static_assert(ADDRSPACE_INFO_AREA_LAYOUT_SIZE <= MAX_VM_INFO_AREA_SIZE, + "Address space information area too small"); + void addrspace_handle_boot_cold_init(void) { @@ -47,12 +59,13 @@ addrspace_handle_boot_cold_init(void) assert(!already_set); } +#if defined(INTERFACE_VCPU) void addrspace_context_switch_load(void) { thread_t *thread = thread_get_self(); - if (thread->kind == THREAD_KIND_VCPU) { + if (compiler_expected(thread->kind == THREAD_KIND_VCPU)) { pgtable_vm_load_regs(&thread->addrspace->vm_pgtable); } } @@ -111,6 +124,12 @@ addrspace_attach_thread(addrspace_t *addrspace, thread_t *thread) return ret; } +addrspace_t * +addrspace_get_self(void) +{ + return thread_get_self()->addrspace; +} + error_t addrspace_handle_object_activate_thread(thread_t *thread) { @@ -133,9 +152,33 @@ addrspace_handle_object_deactivate_thread(thread_t *thread) } } +uintptr_t +addrspace_handle_thread_get_stack_base(thread_t *thread) +{ + assert(thread != NULL); + assert(thread->kind == THREAD_KIND_VCPU); + assert(thread->addrspace != NULL); + + virt_range_t *range = &thread->addrspace->hyp_va_range; + + // Align the starting base to the next boundary to ensure we have guard + // pages before the first stack mapping. + uintptr_t base = + util_balign_up(range->base + 1U, THREAD_STACK_MAP_ALIGN); + + base += (uintptr_t)thread->stack_map_index * THREAD_STACK_MAP_ALIGN; + + assert((base + THREAD_STACK_MAX_SIZE) < + (range->base + (range->size - 1U))); + + return base; +} +#endif + +#if defined(MODULE_VM_ROOTVM) void addrspace_handle_rootvm_init(thread_t *root_thread, cspace_t *root_cspace, - boot_env_data_t *env_data) + qcbor_enc_ctxt_t *qcbor_enc_ctxt) { addrspace_create_t as_params = { NULL }; @@ -147,21 +190,12 @@ addrspace_handle_rootvm_init(thread_t *root_thread, cspace_t *root_cspace, } addrspace_t *root_addrspace = addrspace_ret.r; - vmid_t vmid; - -#if defined(ROOTVM_IS_HLOS) && ROOTVM_IS_HLOS - assert(vcpu_option_flags_get_hlos_vm(&root_thread->vcpu_options)); - - vmid = (vmid_t)VTTBR_EL2_get_VMID(&hlos_vm_vttbr); -#else assert(!vcpu_option_flags_get_hlos_vm(&root_thread->vcpu_options)); - vmid = ROOT_VM_VMID; -#endif - spinlock_acquire(&root_addrspace->header.lock); + // FIXME: // Root VM address space could be smaller - if (addrspace_configure(root_addrspace, vmid) != OK) { + if (addrspace_configure(root_addrspace, ROOT_VM_VMID) != OK) { spinlock_release(&root_addrspace->header.lock); panic("Error configuring addrspace"); } @@ -175,7 +209,10 @@ addrspace_handle_rootvm_init(thread_t *root_thread, cspace_t *root_cspace, panic("Error create addrspace cap id."); } - env_data->addrspace_capid = capid_ret.r; + assert(qcbor_enc_ctxt != NULL); + + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "addrspace_capid", + capid_ret.r); if (object_activate_addrspace(root_addrspace) != OK) { panic("Error activating addrspace"); @@ -186,54 +223,57 @@ addrspace_handle_rootvm_init(thread_t *root_thread, cspace_t *root_cspace, if (ret != OK) { panic("Error attaching root addrspace to root thread."); } - -#if defined(ROOTVM_IS_HLOS) && ROOTVM_IS_HLOS - // Attach all of the secondary root VM threads to the address space - for (cpu_index_t i = 0; cpulocal_index_valid(i); i++) { - thread_t *thread; - if (i == root_thread->scheduler_affinity) { - continue; - } - cap_id_t thread_cap = env_data->psci_secondary_vcpus[i]; - object_type_t type; - object_ptr_result_t o = cspace_lookup_object_any( - root_cspace, thread_cap, - CAP_RIGHTS_GENERIC_OBJECT_ACTIVATE, &type); - if ((o.e != OK) || (type != OBJECT_TYPE_THREAD)) { - panic("VIC couldn't attach root VM thread"); - } - thread = o.r.thread; - - // Attach thread to root addrspace - if (addrspace_attach_thread(root_thread->addrspace, thread) != - OK) { - panic("Error attaching addrspace to secondary VCPU"); - } - - object_put_thread(thread); - } -#endif } +#endif error_t addrspace_handle_object_create_addrspace(addrspace_create_t params) { + error_t ret; + addrspace_t *addrspace = params.addrspace; assert(addrspace != NULL); spinlock_init(&addrspace->mapping_list_lock); spinlock_init(&addrspace->pgtable_lock); - list_init(&addrspace->mapping_list); +#if defined(INTERFACE_VCPU_RUN) + spinlock_init(&addrspace->vmmio_range_lock); + gpt_config_t gpt_config = gpt_config_default(); + gpt_config_set_max_bits(&gpt_config, GPT_MAX_SIZE_BITS); + gpt_config_set_rcu_read(&gpt_config, true); + ret = gpt_init(&addrspace->vmmio_ranges, addrspace->header.partition, + gpt_config, util_bit(GPT_TYPE_VMMIO_RANGE)); + if (ret != OK) { + goto out; + } +#endif + + addrspace->info_area.ipa = VMADDR_INVALID; + addrspace->info_area.me = NULL; + + // Allocate some hypervisor address space for the addrspace object use, + // including kernel stacks of attached threads and vm_info_page. + + // Stack region size, including start and end guard regions. + size_t stack_area_size = + THREAD_STACK_MAP_ALIGN * (ADDRSPACE_MAX_THREADS + 2U); + size_t alloc_size = + stack_area_size + + util_balign_up(MAX_VM_INFO_AREA_SIZE, PGTABLE_HYP_PAGE_SIZE); - // Allocate some hypervisor address space for the kernel stacks of - // attached threads. - size_t aspace_size = - THREAD_STACK_MAP_ALIGN * (ADDRSPACE_MAX_THREADS + 1U); - virt_range_result_t stack_range = hyp_aspace_allocate(aspace_size); - if (stack_range.e == OK) { - addrspace->stack_range = stack_range.r; + virt_range_result_t alloc_range = hyp_aspace_allocate(alloc_size); + if (alloc_range.e == OK) { + addrspace->hyp_va_range = alloc_range.r; + + addrspace->info_area.hyp_va = + (addrspace_info_area_layout_t *)(alloc_range.r.base + + stack_area_size); } - return stack_range.e; + ret = alloc_range.e; +#if defined(INTERFACE_VCPU_RUN) +out: +#endif + return ret; } void @@ -241,8 +281,12 @@ addrspace_handle_object_cleanup_addrspace(addrspace_t *addrspace) { assert(addrspace != NULL); +#if defined(INTERFACE_VCPU_RUN) + gpt_destroy(&addrspace->vmmio_ranges); +#endif + hyp_aspace_deallocate(addrspace->header.partition, - addrspace->stack_range); + addrspace->hyp_va_range); } void @@ -258,7 +302,7 @@ addrspace_configure(addrspace_t *addrspace, vmid_t vmid) assert(addrspace != NULL); - if ((vmid == 0U) || vmid >= NUM_VMIDS) { + if ((vmid == 0U) || (vmid >= NUM_VMIDS)) { ret = ERROR_ARGUMENT_INVALID; } else { addrspace->vmid = vmid; @@ -267,6 +311,52 @@ addrspace_configure(addrspace_t *addrspace, vmid_t vmid) return ret; } +error_t +addrspace_configure_info_area(addrspace_t *addrspace, memextent_t *info_area_me, + vmaddr_t ipa) +{ + error_t ret = OK; + + assert(addrspace != NULL); + + size_t size = info_area_me->size; + assert(size != 0); + +#if (ADDRSPACE_INFO_AREA_LAYOUT_SIZE != 0) + if ((size < (size_t)ADDRSPACE_INFO_AREA_LAYOUT_SIZE) || + (size > (size_t)MAX_VM_INFO_AREA_SIZE)) +#else + if (size > (size_t)MAX_VM_INFO_AREA_SIZE) +#endif + { + ret = ERROR_ARGUMENT_INVALID; + goto out; + } + + if (!util_is_baligned(ipa, PGTABLE_HYP_PAGE_SIZE) || + util_add_overflows(ipa, size) || + ((ipa + size) > util_bit(PLATFORM_VM_ADDRESS_SPACE_BITS))) { + ret = ERROR_ADDR_INVALID; + goto out; + } + + if ((info_area_me->type != MEMEXTENT_TYPE_BASIC) || + (!pgtable_access_check(info_area_me->access, PGTABLE_ACCESS_RW)) || + (info_area_me->memtype != MEMEXTENT_MEMTYPE_ANY)) { + ret = ERROR_ARGUMENT_INVALID; + goto out; + } + + addrspace->info_area.ipa = ipa; + if (addrspace->info_area.me != NULL) { + object_put_memextent(addrspace->info_area.me); + } + addrspace->info_area.me = object_get_memextent_additional(info_area_me); + +out: + return ret; +} + error_t addrspace_handle_object_activate_addrspace(addrspace_t *addrspace) { @@ -274,24 +364,53 @@ addrspace_handle_object_activate_addrspace(addrspace_t *addrspace) assert(addrspace != NULL); + // FIXME: bool already_set = bitmap_atomic_test_and_set( addrspace_vmids, addrspace->vmid, memory_order_relaxed); if (already_set) { ret = ERROR_BUSY; - goto out; + goto out_busy; } partition_t *partition = addrspace->header.partition; ret = pgtable_vm_init(partition, &addrspace->vm_pgtable, addrspace->vmid); + if (ret != OK) { + goto out_vmid_dealloc; + } + + if (addrspace->info_area.me != NULL) { + vmaddr_t ipa = addrspace->info_area.ipa; + uintptr_t va = (uintptr_t)addrspace->info_area.hyp_va; + size_t size = addrspace->info_area.me->size; + + // Ensure the IPA is within VM's range + ret = addrspace_check_range(addrspace, ipa, size); + if (ret != OK) { + goto out_vmid_dealloc; + } + + // Attach the extent to the addrspace object. + ret = memextent_attach(partition, addrspace->info_area.me, va, + size); + if (ret != OK) { + object_put_memextent(addrspace->info_area.me); + addrspace->info_area.me = NULL; + goto out_vmid_dealloc; + } + + assert(va != 0u); + (void)memset_s((uint8_t *)va, size, 0, size); + } + +out_vmid_dealloc: if (ret != OK) { // Undo the vmid allocation (void)bitmap_atomic_test_and_clear( addrspace_vmids, addrspace->vmid, memory_order_relaxed); addrspace->vmid = 0U; } - -out: +out_busy: return ret; } @@ -300,38 +419,270 @@ addrspace_handle_object_deactivate_addrspace(addrspace_t *addrspace) { assert(addrspace != NULL); - if (!addrspace->vm_read_only) { + if (addrspace->info_area.me != NULL) { + memextent_detach(addrspace->header.partition, + addrspace->info_area.me); + + object_put_memextent(addrspace->info_area.me); + addrspace->info_area.me = NULL; + } + + if (!addrspace->read_only) { pgtable_vm_destroy(addrspace->header.partition, &addrspace->vm_pgtable); } - bool already_cleared = bitmap_atomic_test_and_clear( + bool set = bitmap_atomic_test_and_clear( addrspace_vmids, addrspace->vm_pgtable.control.vmid, memory_order_relaxed); - if (already_cleared) { + if (!set) { panic("VMID bitmap never set or already cleared."); } addrspace->vmid = 0U; } -uintptr_t -addrspace_handle_thread_get_stack_base(thread_t *thread) +error_t +addrspace_map(addrspace_t *addrspace, vmaddr_t vbase, size_t size, paddr_t phys, + pgtable_vm_memtype_t memtype, pgtable_access_t kernel_access, + pgtable_access_t user_access) { - assert(thread != NULL); - assert(thread->kind == THREAD_KIND_VCPU); - assert(thread->addrspace != NULL); + error_t err = trigger_addrspace_map_event(addrspace, vbase, size, phys, + memtype, kernel_access, + user_access); + if (err != ERROR_UNIMPLEMENTED) { + goto out; + } - virt_range_t *range = &thread->addrspace->stack_range; + if (addrspace->read_only) { + err = ERROR_DENIED; + goto out; + } - // Align the starting base to the next boundary to ensure we have guard - // pages before the first stack mapping. - uintptr_t base = - util_balign_up(range->base + 1U, THREAD_STACK_MAP_ALIGN); + spinlock_acquire(&addrspace->pgtable_lock); + pgtable_vm_start(&addrspace->vm_pgtable); - base += thread->stack_map_index * THREAD_STACK_MAP_ALIGN; + // We do not set the try_map option; we expect the caller to know if it + // is overwriting an existing mapping. + err = pgtable_vm_map(addrspace->header.partition, + &addrspace->vm_pgtable, vbase, size, phys, memtype, + kernel_access, user_access, false, false); - assert((base + THREAD_STACK_MAX_SIZE) < - (range->base + (range->size - 1U))); + pgtable_vm_commit(&addrspace->vm_pgtable); + spinlock_release(&addrspace->pgtable_lock); - return base; +out: + return err; +} + +error_t +addrspace_unmap(addrspace_t *addrspace, vmaddr_t vbase, size_t size, + paddr_t phys) +{ + error_t err = + trigger_addrspace_unmap_event(addrspace, vbase, size, phys); + if (err != ERROR_UNIMPLEMENTED) { + goto out; + } + + if (addrspace->read_only) { + err = ERROR_DENIED; + goto out; + } + + spinlock_acquire(&addrspace->pgtable_lock); + pgtable_vm_start(&addrspace->vm_pgtable); + + // Unmap only if the physical address is matching. + pgtable_vm_unmap_matching(addrspace->header.partition, + &addrspace->vm_pgtable, vbase, phys, size); + err = OK; + + pgtable_vm_commit(&addrspace->vm_pgtable); + spinlock_release(&addrspace->pgtable_lock); + +out: + return err; +} + +addrspace_lookup_result_t +addrspace_lookup(addrspace_t *addrspace, vmaddr_t vbase, size_t size) +{ + addrspace_lookup_result_t ret; + + assert(addrspace != NULL); + + if (size == 0U) { + ret = addrspace_lookup_result_error(ERROR_ARGUMENT_SIZE); + goto out; + } + + if (util_add_overflows(vbase, size - 1U)) { + ret = addrspace_lookup_result_error(ERROR_ADDR_OVERFLOW); + goto out; + } + + if (!util_is_baligned(vbase, PGTABLE_VM_PAGE_SIZE) || + !util_is_baligned(size, PGTABLE_VM_PAGE_SIZE)) { + ret = addrspace_lookup_result_error(ERROR_ARGUMENT_ALIGNMENT); + goto out; + } + + bool first_lookup = true; + paddr_t lookup_phys = 0U; + size_t lookup_size = 0U; + pgtable_vm_memtype_t lookup_memtype = PGTABLE_VM_MEMTYPE_NORMAL_WB; + pgtable_access_t lookup_kernel_access = PGTABLE_ACCESS_NONE; + pgtable_access_t lookup_user_access = PGTABLE_ACCESS_NONE; + + spinlock_acquire(&addrspace->pgtable_lock); + + bool mapped = true; + size_t mapped_size = 0U; + for (size_t offset = 0U; offset < size; offset += mapped_size) { + vmaddr_t curr = vbase + offset; + paddr_t mapped_phys; + pgtable_vm_memtype_t mapped_memtype; + pgtable_access_t mapped_kernel_access, mapped_user_access; + + mapped = pgtable_vm_lookup(&addrspace->vm_pgtable, curr, + &mapped_phys, &mapped_size, + &mapped_memtype, + &mapped_kernel_access, + &mapped_user_access); + if (mapped) { + size_t mapping_offset = curr & (mapped_size - 1U); + mapped_phys += mapping_offset; + mapped_size = util_min(mapped_size - mapping_offset, + size - offset); + + if (first_lookup) { + lookup_phys = mapped_phys; + lookup_size = mapped_size; + lookup_memtype = mapped_memtype; + lookup_kernel_access = mapped_kernel_access; + lookup_user_access = mapped_user_access; + first_lookup = false; + } else if (((lookup_phys + lookup_size) == + mapped_phys) && + (lookup_memtype == mapped_memtype) && + (lookup_kernel_access == + mapped_kernel_access) && + (lookup_user_access == mapped_user_access)) { + lookup_size += mapped_size; + } else { + // Mapped range no longer contiguous, end the + // lookup. + break; + } + } else { + break; + } + } + + spinlock_release(&addrspace->pgtable_lock); + + if (first_lookup) { + ret = addrspace_lookup_result_error(ERROR_ADDR_INVALID); + goto out; + } + + addrspace_lookup_t lookup = { + .phys = lookup_phys, + .size = lookup_size, + .memtype = lookup_memtype, + .kernel_access = lookup_kernel_access, + .user_access = lookup_user_access, + }; + + ret = addrspace_lookup_result_ok(lookup); + +out: + return ret; +} + +error_t +addrspace_add_vmmio_range(addrspace_t *addrspace, vmaddr_t base, size_t size) +{ + error_t ret; + +#if defined(INTERFACE_VCPU_RUN) + if (size == 0U) { + ret = ERROR_ARGUMENT_SIZE; + goto out; + } + + if (util_add_overflows(base, size)) { + ret = ERROR_ADDR_OVERFLOW; + goto out; + } + + spinlock_acquire(&addrspace->vmmio_range_lock); + + if (addrspace->vmmio_range_count == ADDRSPACE_MAX_VMMIO_RANGES) { + ret = ERROR_NORESOURCES; + goto out_locked; + } + + gpt_entry_t entry = (gpt_entry_t){ + .type = GPT_TYPE_VMMIO_RANGE, + .value.vmmio_range_base = base, + }; + + ret = gpt_insert(&addrspace->vmmio_ranges, base, size, entry, true); + + if (ret == OK) { + addrspace->vmmio_range_count++; + } + +out_locked: + spinlock_release(&addrspace->vmmio_range_lock); +out: +#else // !INTERFACE_VCPU_RUN + (void)addrspace; + (void)base; + (void)size; + ret = ERROR_UNIMPLEMENTED; +#endif + return ret; +} + +error_t +addrspace_remove_vmmio_range(addrspace_t *addrspace, vmaddr_t base, size_t size) +{ + error_t ret; + +#if defined(INTERFACE_VCPU_RUN) + if (size == 0U) { + ret = ERROR_ARGUMENT_SIZE; + goto out; + } + + if (util_add_overflows(base, size)) { + ret = ERROR_ADDR_OVERFLOW; + goto out; + } + + spinlock_acquire(&addrspace->vmmio_range_lock); + + gpt_entry_t entry = (gpt_entry_t){ + .type = GPT_TYPE_VMMIO_RANGE, + .value.vmmio_range_base = base, + }; + + ret = gpt_remove(&addrspace->vmmio_ranges, base, size, entry); + + if (ret == OK) { + assert(addrspace->vmmio_range_count > 0U); + addrspace->vmmio_range_count--; + } + + spinlock_release(&addrspace->vmmio_range_lock); +out: +#else // !INTERFACE_VCPU_RUN + (void)addrspace; + (void)base; + (void)size; + ret = ERROR_UNIMPLEMENTED; +#endif + return ret; } diff --git a/hyp/mem/addrspace/src/hypercalls.c b/hyp/mem/addrspace/src/hypercalls.c index 08e8b4c..522f514 100644 --- a/hyp/mem/addrspace/src/hypercalls.c +++ b/hyp/mem/addrspace/src/hypercalls.c @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: BSD-3-Clause +#include #include #include @@ -11,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -24,7 +26,7 @@ error_t hypercall_addrspace_attach_thread(cap_id_t addrspace_cap, cap_id_t thread_cap) { error_t ret; - cspace_t *cspace = cspace_get_self(); + cspace_t *cspace = cspace_get_self(); object_type_t type; object_ptr_result_t o = cspace_lookup_object_any( @@ -90,14 +92,39 @@ hypercall_addrspace_attach_vdma(cap_id_t addrspace_cap, cap_id_t dma_device_cap, return err; } +error_t +hypercall_addrspace_attach_vdevice(cap_id_t addrspace_cap, cap_id_t vdevice_cap, + index_t index, vmaddr_t vbase, size_t size) +{ + error_t err; + cspace_t *cspace = cspace_get_self(); + + addrspace_ptr_result_t addrspace_r = cspace_lookup_addrspace( + cspace, addrspace_cap, CAP_RIGHTS_ADDRSPACE_MAP); + if (compiler_unexpected(addrspace_r.e != OK)) { + err = addrspace_r.e; + goto out; + } + + err = trigger_addrspace_attach_vdevice_event(addrspace_r.r, vdevice_cap, + index, vbase, size); + + object_put_addrspace(addrspace_r.r); +out: + return err; +} + error_t hypercall_addrspace_map(cap_id_t addrspace_cap, cap_id_t memextent_cap, - vmaddr_t vbase, memextent_mapping_attrs_t map_attrs) + vmaddr_t vbase, memextent_mapping_attrs_t map_attrs, + addrspace_map_flags_t map_flags, size_t offset, + size_t size) { error_t ret; cspace_t *cspace = cspace_get_self(); - if (memextent_mapping_attrs_get_res_0(&map_attrs) != 0U) { + if ((memextent_mapping_attrs_get_res_0(&map_attrs) != 0U) || + (addrspace_map_flags_get_res0_0(&map_flags) != 0U)) { ret = ERROR_ARGUMENT_INVALID; goto out; } @@ -120,8 +147,14 @@ hypercall_addrspace_map(cap_id_t addrspace_cap, cap_id_t memextent_cap, memextent_t *memextent = m.r; - ret = memextent_map(memextent, addrspace, vbase, map_attrs); - if (ret == OK) { + if (addrspace_map_flags_get_partial(&map_flags)) { + ret = memextent_map_partial(memextent, addrspace, vbase, offset, + size, map_attrs); + } else { + ret = memextent_map(memextent, addrspace, vbase, map_attrs); + } + + if ((ret == OK) && !addrspace_map_flags_get_no_sync(&map_flags)) { // Wait for completion of EL2 operations using manual lookups rcu_sync(); } @@ -135,11 +168,17 @@ hypercall_addrspace_map(cap_id_t addrspace_cap, cap_id_t memextent_cap, error_t hypercall_addrspace_unmap(cap_id_t addrspace_cap, cap_id_t memextent_cap, - vmaddr_t vbase) + vmaddr_t vbase, addrspace_map_flags_t map_flags, + size_t offset, size_t size) { error_t ret; cspace_t *cspace = cspace_get_self(); + if (addrspace_map_flags_get_res0_0(&map_flags) != 0U) { + ret = ERROR_ARGUMENT_INVALID; + goto out; + } + addrspace_ptr_result_t c = cspace_lookup_addrspace( cspace, addrspace_cap, CAP_RIGHTS_ADDRSPACE_MAP); if (compiler_unexpected(c.e != OK)) { @@ -158,8 +197,14 @@ hypercall_addrspace_unmap(cap_id_t addrspace_cap, cap_id_t memextent_cap, memextent_t *memextent = m.r; - ret = memextent_unmap(memextent, addrspace, vbase); - if (ret == OK) { + if (addrspace_map_flags_get_partial(&map_flags)) { + ret = memextent_unmap_partial(memextent, addrspace, vbase, + offset, size); + } else { + ret = memextent_unmap(memextent, addrspace, vbase); + } + + if ((ret == OK) && !addrspace_map_flags_get_no_sync(&map_flags)) { // Wait for completion of EL2 operations using manual lookups rcu_sync(); } @@ -174,12 +219,15 @@ hypercall_addrspace_unmap(cap_id_t addrspace_cap, cap_id_t memextent_cap, error_t hypercall_addrspace_update_access(cap_id_t addrspace_cap, cap_id_t memextent_cap, vmaddr_t vbase, - memextent_access_attrs_t access_attrs) + memextent_access_attrs_t access_attrs, + addrspace_map_flags_t map_flags, + size_t offset, size_t size) { error_t ret; cspace_t *cspace = cspace_get_self(); - if (memextent_access_attrs_get_res_0(&access_attrs) != 0U) { + if ((memextent_access_attrs_get_res_0(&access_attrs) != 0U) || + (addrspace_map_flags_get_res0_0(&map_flags) != 0U)) { ret = ERROR_ARGUMENT_INVALID; goto out; } @@ -202,9 +250,16 @@ hypercall_addrspace_update_access(cap_id_t addrspace_cap, memextent_t *memextent = m.r; - ret = memextent_update_access(memextent, addrspace, vbase, - access_attrs); - if (ret == OK) { + if (addrspace_map_flags_get_partial(&map_flags)) { + ret = memextent_update_access_partial(memextent, addrspace, + vbase, offset, size, + access_attrs); + } else { + ret = memextent_update_access(memextent, addrspace, vbase, + access_attrs); + } + + if ((ret == OK) && !addrspace_map_flags_get_no_sync(&map_flags)) { // Wait for completion of EL2 operations using manual lookups rcu_sync(); } @@ -220,7 +275,7 @@ error_t hypercall_addrspace_configure(cap_id_t addrspace_cap, vmid_t vmid) { error_t err; - cspace_t *cspace = cspace_get_self(); + cspace_t *cspace = cspace_get_self(); object_type_t type; object_ptr_result_t o = cspace_lookup_object_any( @@ -252,3 +307,155 @@ hypercall_addrspace_configure(cap_id_t addrspace_cap, vmid_t vmid) out: return err; } + +hypercall_addrspace_lookup_result_t +hypercall_addrspace_lookup(cap_id_t addrspace_cap, cap_id_t memextent_cap, + vmaddr_t vbase, size_t size) +{ + hypercall_addrspace_lookup_result_t ret = { .error = OK }; + cspace_t *cspace = cspace_get_self(); + + addrspace_ptr_result_t a = cspace_lookup_addrspace( + cspace, addrspace_cap, CAP_RIGHTS_ADDRSPACE_LOOKUP); + if (compiler_unexpected(a.e != OK)) { + ret.error = a.e; + goto out; + } + + addrspace_t *addrspace = a.r; + + memextent_ptr_result_t m = cspace_lookup_memextent( + cspace, memextent_cap, CAP_RIGHTS_MEMEXTENT_LOOKUP); + if (compiler_unexpected(m.e != OK)) { + ret.error = m.e; + goto out_addrspace_release; + } + + memextent_t *memextent = m.r; + + addrspace_lookup_result_t lookup_ret = + addrspace_lookup(addrspace, vbase, size); + if (lookup_ret.e != OK) { + ret.error = lookup_ret.e; + goto out_memextent_release; + } + + // Determine if the memextent owns the range of memory returned by the + // lookup. + paddr_t phys_start = lookup_ret.r.phys; + paddr_t phys_end = phys_start + (lookup_ret.r.size - 1U); + if (!memdb_is_ownership_contiguous(phys_start, phys_end, + (uintptr_t)memextent, + MEMDB_TYPE_EXTENT)) { + ret.error = ERROR_MEMDB_NOT_OWNER; + goto out_memextent_release; + } + + assert((phys_start >= memextent->phys_base) && + (phys_end <= (memextent->phys_base + (memextent->size - 1U)))); + + memextent_mapping_attrs_t map_attrs = memextent_mapping_attrs_default(); + memextent_mapping_attrs_set_memtype(&map_attrs, lookup_ret.r.memtype); + memextent_mapping_attrs_set_user_access(&map_attrs, + lookup_ret.r.user_access); + memextent_mapping_attrs_set_kernel_access(&map_attrs, + lookup_ret.r.kernel_access); + + ret.offset = phys_start - memextent->phys_base; + ret.size = lookup_ret.r.size; + ret.map_attrs = map_attrs; + +out_memextent_release: + object_put_memextent(memextent); +out_addrspace_release: + object_put_addrspace(addrspace); +out: + return ret; +} + +error_t +hypercall_addrspace_configure_info_area(cap_id_t addrspace_cap, + cap_id_t info_area_me_cap, vmaddr_t ipa) +{ + error_t err; + cspace_t *cspace = cspace_get_self(); + object_type_t type; + + object_ptr_result_t o = cspace_lookup_object_any( + cspace, addrspace_cap, CAP_RIGHTS_GENERIC_OBJECT_ACTIVATE, + &type); + if (compiler_unexpected(o.e != OK)) { + err = o.e; + goto out_bad_cap; + } + if (type != OBJECT_TYPE_ADDRSPACE) { + err = ERROR_CSPACE_WRONG_OBJECT_TYPE; + goto out_addrspace_release; + } + addrspace_t *target_as = o.r.addrspace; + + memextent_ptr_result_t m = cspace_lookup_memextent( + cspace, info_area_me_cap, CAP_RIGHTS_MEMEXTENT_ATTACH); + if (compiler_unexpected(m.e != OK)) { + err = m.e; + goto out_addrspace_release; + } + memextent_t *info_area_me = m.r; + + spinlock_acquire(&target_as->header.lock); + if (atomic_load_relaxed(&target_as->header.state) == + OBJECT_STATE_INIT) { + err = addrspace_configure_info_area(target_as, info_area_me, + ipa); + } else { + err = ERROR_OBJECT_STATE; + } + spinlock_release(&target_as->header.lock); + + object_put_memextent(info_area_me); +out_addrspace_release: + object_put(type, o.r); +out_bad_cap: + return err; +} + +error_t +hypercall_addrspace_configure_vmmio(cap_id_t addrspace_cap, vmaddr_t vbase, + size_t size, + addrspace_vmmio_configure_op_t op) +{ + error_t err; + cspace_t *cspace = cspace_get_self(); + + addrspace_ptr_result_t o = cspace_lookup_addrspace_any( + cspace, addrspace_cap, CAP_RIGHTS_ADDRSPACE_ADD_VMMIO_RANGE); + if (compiler_unexpected(o.e != OK)) { + err = o.e; + goto out; + } + + addrspace_t *target_as = o.r; + + object_state_t state = atomic_load_relaxed(&target_as->header.state); + if ((state != OBJECT_STATE_INIT) && (state != OBJECT_STATE_ACTIVE)) { + err = ERROR_OBJECT_STATE; + goto out_ref; + } + + switch (op) { + case ADDRSPACE_VMMIO_CONFIGURE_OP_ADD: + err = addrspace_add_vmmio_range(target_as, vbase, size); + break; + case ADDRSPACE_VMMIO_CONFIGURE_OP_REMOVE: + err = addrspace_remove_vmmio_range(target_as, vbase, size); + break; + default: + err = ERROR_UNIMPLEMENTED; + break; + } + +out_ref: + object_put_addrspace(o.r); +out: + return err; +} diff --git a/hyp/mem/addrspace_null/addrspace.tc b/hyp/mem/addrspace_null/addrspace.tc deleted file mode 100644 index e45c027..0000000 --- a/hyp/mem/addrspace_null/addrspace.tc +++ /dev/null @@ -1,15 +0,0 @@ -// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. -// -// SPDX-License-Identifier: BSD-3-Clause - -extend addrspace object { - mapping_list_lock structure spinlock; - pgtable_lock structure spinlock; - vm_pgtable structure pgtable_vm; - vmid type vmid_t; - vm_read_only bool; -}; - -extend thread object { - addrspace pointer object addrspace; -}; diff --git a/hyp/mem/allocator_boot/src/bootmem.c b/hyp/mem/allocator_boot/src/bootmem.c index 5af0ad4..3917cb2 100644 --- a/hyp/mem/allocator_boot/src/bootmem.c +++ b/hyp/mem/allocator_boot/src/bootmem.c @@ -23,11 +23,24 @@ allocator_boot_handle_boot_runtime_first_init(void) { assert((uintptr_t)&heap_private_end > (uintptr_t)&heap_private_start); - void *base = &heap_private_start; - size_t size = (size_t)(&heap_private_end - &heap_private_start); + static_assert( + PLATFORM_HEAP_PRIVATE_SIZE <= PLATFORM_RW_DATA_SIZE, + "PLATFORM_HEAP_PRIVATE_SIZE must be <= PLATFORM_RW_DATA_SIZE"); + static_assert(PLATFORM_RW_DATA_SIZE >= 0x200000, + "PLATFORM_RW_DATA_SIZE must be >= 2MB"); + + // We only give heap within the first 2MB RW page to the bootmem. We + // will map the rest of the heap during the hyp_aspace init. + void *base = &heap_private_start; + uintptr_t map_end = util_balign_up((uintptr_t)base, 0x200000U); + uintptr_t end = util_min( + map_end, (uintptr_t)base + (size_t)PLATFORM_HEAP_PRIVATE_SIZE); + + assert((uintptr_t)base < end); + size_t size = end - (uintptr_t)base; assert(base != NULL); - assert(size >= 0x1000); + assert(size >= 0x1000U); bootmem_allocator.pool_base = (uint8_t *)base; bootmem_allocator.pool_size = size; @@ -71,7 +84,7 @@ bootmem_allocate_remaining(size_t *size) assert(bootmem_allocator.alloc_offset <= bootmem_allocator.pool_size); size_t free = bootmem_allocator.pool_size - bootmem_allocator.alloc_offset; - if (free == 0) { + if (free == 0U) { return void_ptr_result_error(ERROR_NOMEM); } *size = free; diff --git a/hyp/mem/allocator_list/allocator.ev b/hyp/mem/allocator_list/allocator.ev new file mode 100644 index 0000000..89e0592 --- /dev/null +++ b/hyp/mem/allocator_list/allocator.ev @@ -0,0 +1,8 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module allocator_list + +subscribe allocator_add_ram_range + priority last diff --git a/hyp/mem/allocator_list/allocator.tc b/hyp/mem/allocator_list/allocator.tc index df47a31..fcf23d6 100644 --- a/hyp/mem/allocator_list/allocator.tc +++ b/hyp/mem/allocator_list/allocator.tc @@ -10,4 +10,6 @@ define allocator_node structure { extend allocator structure { heap pointer structure allocator_node; lock structure spinlock; + total_size size; + alloc_size size; }; diff --git a/hyp/mem/allocator_list/build.conf b/hyp/mem/allocator_list/build.conf index 368caf4..679e8c5 100644 --- a/hyp/mem/allocator_list/build.conf +++ b/hyp/mem/allocator_list/build.conf @@ -3,5 +3,7 @@ # SPDX-License-Identifier: BSD-3-Clause interface allocator + +events allocator.ev types allocator.tc source freelist.c diff --git a/hyp/mem/allocator_list/src/freelist.c b/hyp/mem/allocator_list/src/freelist.c index 24bebbc..67ae553 100644 --- a/hyp/mem/allocator_list/src/freelist.c +++ b/hyp/mem/allocator_list/src/freelist.c @@ -91,13 +91,15 @@ #include #include +#include "event_handlers.h" + // Maximum supported heap allocation size or alignment size. We filter out // really large allocations so we can avoid having to think about corner-cases // causing overflow. #define MAX_ALLOC_SIZE (256UL * 1024UL * 1024UL) #define MAX_ALIGNMENT_SIZE (16UL * 1024UL * 1024UL) -#define NODE_HEADER_SIZE (sizeof(struct allocator_node)) +#define NODE_HEADER_SIZE (sizeof(allocator_node_t)) // Minimum allocation size from the heap. #define HEAP_MIN_ALLOC NODE_HEADER_SIZE @@ -107,7 +109,7 @@ #if defined(ALLOCATOR_DEBUG) #define OVERFLOW_DEBUG #define OVERFLOW_REDZONE_SIZE NODE_HEADER_SIZE -//#define DEBUG_PRINT +// #define DEBUG_PRINT #endif // --------------------------------------- @@ -213,6 +215,7 @@ list_add(allocator_node_t **head, allocator_node_t *node, size_t size) (uint64_t)current) { // 7. Merge with current node->size = size + current->size; + node->next = current->next; previous->next = node; } else { goto out; @@ -224,7 +227,7 @@ list_add(allocator_node_t **head, allocator_node_t *node, size_t size) if (((uint64_t)previous + previous->size) == (uint64_t)node) { // 8. Merge with previous - previous->size += previous->size + size; + previous->size += size; } else if (((uint64_t)previous + previous->size) < (uint64_t)node) { // 9. Append node to list @@ -249,37 +252,39 @@ list_add(allocator_node_t **head, allocator_node_t *node, size_t size) return ret; } -error_t NOINLINE -allocator_heap_add_memory(allocator_t *allocator, void *addr, size_t size) +static error_t NOINLINE +allocator_heap_add_memory(allocator_t *allocator, uintptr_t addr, size_t size) { allocator_node_t *block; error_t ret = OK; - assert(addr != NULL); + assert(addr != 0U); // Check input arguments - if (!util_is_baligned((uintptr_t)addr, NODE_HEADER_SIZE)) { - uintptr_t new_addr = - util_balign_up((uintptr_t)addr, NODE_HEADER_SIZE); - size -= (new_addr - (uintptr_t)addr); - addr = (void *)new_addr; + if (!util_is_baligned(addr, NODE_HEADER_SIZE)) { + uintptr_t new_addr = util_balign_up(addr, NODE_HEADER_SIZE); + size -= (new_addr - addr); + addr = new_addr; } if (!util_is_baligned(size, NODE_HEADER_SIZE)) { size = util_balign_down(size, NODE_HEADER_SIZE); } - if (util_add_overflows((uint64_t)addr, size)) { + if (util_add_overflows(addr, size)) { ret = ERROR_ADDR_OVERFLOW; } else if (size < (2UL * NODE_HEADER_SIZE)) { ret = ERROR_ARGUMENT_SIZE; } else { // FIXME: Check if added memory is in kernel address space - block = (allocator_node_t *)(uintptr_t)addr; + block = (allocator_node_t *)addr; // Add memory to the freelist spinlock_acquire(&allocator->lock); ret = list_add(&allocator->heap, block, size); + if (ret == OK) { + allocator->total_size += size; + } spinlock_release(&allocator->lock); } @@ -287,6 +292,18 @@ allocator_heap_add_memory(allocator_t *allocator, void *addr, size_t size) return ret; } +error_t +allocator_list_handle_allocator_add_ram_range(partition_t *owner, + paddr_t phys_base, + uintptr_t virt_base, size_t size) +{ + assert(owner != NULL); + + (void)phys_base; + + return allocator_heap_add_memory(&owner->allocator, virt_base, size); +} + // Cases: // 1 .-----------------------. // | | @@ -425,7 +442,7 @@ allocate_block(allocator_node_t **head, size_t size, size_t alignment) return ret; } -void_ptr_result_t NOINLINE +void_ptr_result_t allocator_allocate_object(allocator_t *allocator, size_t size, size_t alignment) { void_ptr_result_t ret; @@ -473,7 +490,7 @@ allocator_allocate_object(allocator_t *allocator, size_t size, size_t alignment) goto error; } - // TODO: Update allocation total (increment +1) + allocator->alloc_size += size; #if defined(ALLOCATOR_DEBUG) char *data = (char *)ret.r; @@ -516,6 +533,10 @@ list_remove(allocator_node_t **head, allocator_node_t *remove, } } +// TODO: Exported only for test code currently +error_t +allocator_heap_remove_memory(allocator_t *allocator, void *obj, size_t size); + // Returns -1 if addresses are still being used and therefore cannot be freed. error_t NOINLINE allocator_heap_remove_memory(allocator_t *allocator, void *obj, size_t size) @@ -525,9 +546,8 @@ allocator_heap_remove_memory(allocator_t *allocator, void *obj, size_t size) assert(obj != NULL); assert(allocator->heap != NULL); - allocator_node_t *before_prev = NULL; - allocator_node_t *previous = NULL; - allocator_node_t *current = allocator->heap; + allocator_node_t *previous = NULL; + allocator_node_t *current = allocator->heap; uint64_t object_location; uint64_t current_location; uint64_t previous_location; @@ -539,9 +559,8 @@ allocator_heap_remove_memory(allocator_t *allocator, void *obj, size_t size) while (((uint64_t)obj > (uint64_t)current) && (current != NULL)) { assert((uint64_t)previous < (uint64_t)obj); - before_prev = previous; - previous = current; - current = current->next; + previous = current; + current = current->next; } object_location = (uint64_t)obj; @@ -578,39 +597,26 @@ allocator_heap_remove_memory(allocator_t *allocator, void *obj, size_t size) } if ((previous_location + previous->size) == aligned_alloc_end) { - // Divide previous into 2 nodes & remove second one + // Reduce size of previous + previous->size -= size; + } else { + // Divide previous into 3 nodes & remove middle one allocator_node_t *new; - new = (allocator_node_t *)object_location; - new->next = current; - new->size = size; + new = (allocator_node_t *)aligned_alloc_end; + new->next = current; + new->size = previous_location + previous->size - + aligned_alloc_end; + previous->next = new; - previous->size -= previous->size - size; - list_remove(&allocator->heap, previous, before_prev); - } else { - // Divide previous into 3 nodes & remove middle one - allocator_node_t *new_current; - size_t first_size = - aligned_alloc_end - previous_location; - allocator_node_t *remove; - - new_current = (allocator_node_t *)aligned_alloc_end; - new_current->next = current; - new_current->size = previous->size - size - first_size; - - remove = (allocator_node_t *)object_location; - remove->next = new_current; - remove->size = size; - - previous->next = remove; - previous->size = first_size; - list_remove(&allocator->heap, remove, previous); + previous->size = object_location - previous_location; } } else { goto out; } ret = OK; + allocator->total_size -= size; out: spinlock_release(&allocator->lock); @@ -741,7 +747,7 @@ allocator_deallocate_object(allocator_t *allocator, void *object, size_t size) deallocate_block(&allocator->heap, object, size); CHECK_HEAP(allocator->heap); - // TODO: Update allocation total (decrease -1) + allocator->alloc_size -= size; spinlock_release(&allocator->lock); @@ -753,6 +759,9 @@ allocator_init(allocator_t *allocator) { assert(allocator->heap == NULL); + allocator->total_size = 0UL; + allocator->alloc_size = 0UL; + spinlock_init(&allocator->lock); return OK; } diff --git a/hyp/mem/allocator_list/test/tests.c b/hyp/mem/allocator_list/test/tests.c new file mode 100644 index 0000000..0e98e23 --- /dev/null +++ b/hyp/mem/allocator_list/test/tests.c @@ -0,0 +1,346 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include "allocator.h" +#include "freelist.h" + +#define MEM_POOL_SIZE (1024 * 1024) // 1MB + +// Maximum supported heap allocation size or alignment size. We filter out +// really large allocations so we can avoid having to think about corner-cases +// causing overflow. +#define MAX_ALLOC_SIZE (256UL * 1024UL * 1024UL) +#define MAX_ALIGNMENT_SIZE (16UL * 1024UL * 1024UL) + +#define NODE_HEADER_SIZE ((size_t)sizeof(allocator_node_t)) + +// Minimum allocation size from the heap. +#define HEAP_MIN_ALLOC NODE_HEADER_SIZE +#define HEAP_MIN_ALIGN NODE_HEADER_SIZE + +// -------------- DEBUGGING -------------- +// FIXME: remove eventually +#define HEAP_DEBUG +#define OVERFLOW_DEBUG +// #define DEBUG_PRINT +// --------------------------------------- + +#if defined(HEAP_DEBUG) +#define CHECK_HEAP(x) check_heap_consistency(x) +#else +#define CHECK_HEAP(x) +#endif + +void +print_free_blocks(struct node *head) +{ + struct node *current = head; + int count = 0; + + printf("\n----------- FREE BLOCKS ----------\n"); + + while (current != NULL) { + printf("%d pointer: %p, size %zu\n", count, current, + current->size); + current = current->next; + count++; + } + + printf("----------------------------------\n\n"); +} + +void * +give_mem_to_heap(allocator_t *allocator, size_t size, size_t alignment) +{ + int ret = 0; + void *block = malloc(size); + + printf("Give memory to heap from block, pointer: %p, size %zu\n", block, + size); + ret = allocator_heap_add_memory(allocator, block, size); + if (!ret) { + printf("Memory added to heap, pointer: %p, size %zu\n", + *&allocator->heap, allocator->heap->size); + } + + print_free_blocks(allocator->heap); + + return block; +} + +void * +alloc_obj(allocator_t *allocator, size_t size, size_t alignment) +{ + void *object = allocator_allocate_object(allocator, size, alignment); + + print_free_blocks(allocator->heap); + + return object; +} + +void +dealloc_objs(allocator_t *allocator, int order, void *object, size_t size, + void *object2, size_t size2, void *object3, size_t size3) +{ + switch (order) { + case 0: + // 1st -> 2nd -> 3rd + if (object != NULL) { + allocator_deallocate_object(allocator, object, size); + print_free_blocks(allocator->heap); + } + + if (object2 != NULL) { + allocator_deallocate_object(allocator, object2, size2); + print_free_blocks(allocator->heap); + } + + if (object3 != NULL) { + allocator_deallocate_object(allocator, object3, size3); + print_free_blocks(allocator->heap); + } + break; + + case 1: + // 3rd -> 2nd -> 1st + if (object3 != NULL) { + allocator_deallocate_object(allocator, object3, size3); + print_free_blocks(allocator->heap); + } + + if (object2 != NULL) { + allocator_deallocate_object(allocator, object2, size2); + print_free_blocks(allocator->heap); + } + + if (object != NULL) { + allocator_deallocate_object(allocator, object, size); + print_free_blocks(allocator->heap); + } + break; + + case 2: + default: + // 1st -> 3rd -> 2nd + if (object != NULL) { + printf("Free: %p, size: %zu\n", object, size); + allocator_deallocate_object(allocator, object, size); + print_free_blocks(allocator->heap); + } + + if (object3 != NULL) { + printf("Free: %p, size: %zu\n", object3, size3); + allocator_deallocate_object(allocator, object3, size3); + print_free_blocks(allocator->heap); + } + + if (object2 != NULL) { + printf("Free: %p, size: %zu\n", object2, size2); + allocator_deallocate_object(allocator, object2, size2); + print_free_blocks(allocator->heap); + } + break; + } +} + +void +remove_from_heap(allocator_t *allocator, void *block, size_t size) +{ + int ret; + + if (block != NULL) { + ret = allocator_heap_remove_memory(allocator, block, size); + } + + if (!ret) { + printf("Memory removed from heap. size: %zu\n", size); + } + print_free_blocks(allocator->heap); +} + +// Test 1: +// - Give 1 chunk of memory to the heap of pool_size passed +// - Allocate objects of passed sizes +// - Free all the objects in order specied in 'order' variable. +// - Remove pool from heap +static void +test1(int order, size_t alignment, size_t pool_size, size_t size, size_t size2, + size_t size3) +{ + allocator_t allocator; + + allocator.heap = NULL; + + // ---------------- Giving memory to heap --------------------- + void *block = give_mem_to_heap(&allocator, pool_size, alignment); + + // ---------------- Allocating object from heap --------------- + void *object = alloc_obj(&allocator, size, alignment); + void *object2 = alloc_obj(&allocator, size2, alignment); + void *object3 = alloc_obj(&allocator, size3, alignment); + + // ---------------- Deallocating object to heap ---------------- + dealloc_objs(&allocator, order, object, size, object2, size2, object3, + size3); + + // ---------------- Removing memory to heap -------------------- + remove_from_heap(&allocator, block, pool_size); +} + +// Test 2: +// - Give 3 chunks of memory to the heap of pool_size passed +// - Allocate objects of passed sizes +// - Free all the objects in order specied in 'order' variable. +// - Remove all pools from heap +static void +test2(int order, size_t alignment, size_t pool_size, size_t pool_size2, + size_t pool_size3, size_t size, size_t size2, size_t size3) +{ + allocator_t allocator; + + allocator.heap = NULL; + + // ---------------- Giving memory to heap --------------------- + void *block = give_mem_to_heap(&allocator, pool_size, alignment); + void *block2 = give_mem_to_heap(&allocator, pool_size2, alignment); + void *block3 = give_mem_to_heap(&allocator, pool_size3, alignment); + + // ---------------- Allocating object from heap --------------- + void *object = alloc_obj(&allocator, size, alignment); + void *object2 = alloc_obj(&allocator, size2, alignment); + void *object3 = alloc_obj(&allocator, size3, alignment); + + // ---------------- Deallocating object to heap ---------------- + dealloc_objs(&allocator, order, object, size, object2, size2, object3, + size3); + + // ---------------- Removing memory to heap -------------------- + remove_from_heap(&allocator, block, pool_size); + remove_from_heap(&allocator, block2, pool_size2); + remove_from_heap(&allocator, block3, pool_size3); +} + +void +values_selection(int order, size_t alignment, size_t pool_size, + size_t pool_size2, size_t pool_size3, size_t size, + size_t size2, size_t size3, int test) +{ +} + +void +tests_choice(int test) +{ + int order = 2; + size_t alignment = sizeof(void *); + size_t pool_size = MEM_POOL_SIZE; + size_t pool_size2 = 2UL * NODE_HEADER_SIZE; + size_t pool_size3 = 4UL * NODE_HEADER_SIZE; + size_t size = MEM_POOL_SIZE / 2; + size_t size2 = 48; + size_t size3 = MEM_POOL_SIZE / 2 - 48; + int default_val = 1; + + printf("Do you want to you the fault test? Yes(1), No(0): "); + scanf("%d", &default_val); + + if (test == 1) { + // Default: + // Allocate 3 objects emptying the free list + // Deallocate in this order: 1st -> 3rd -> 2nd + // so that we can check that when the 2nd object is + // freed there is a merge of all free blocks + printf("--------- Test 1 ---------\n"); + if (!default_val) { + printf("Order 0) 1->2->3 | 1) 3->2->1 | 2) 1->3->2: "); + scanf("%d", &order); + printf("Alignment: "); + scanf("%zu", &alignment); + printf("Pool size: "); + scanf("%zu", &pool_size); + printf("Size: "); + scanf("%zu", &size); + printf("Size2: "); + scanf("%zu", &size2); + printf("Size3: "); + scanf("%zu", &size3); + } + +#if defined(OVERFLOW_DEBUG) + // Extra 2*NODE_HEADER_SIZE for overflow checks + pool_size = pool_size + (6 * NODE_HEADER_SIZE); +#endif + test1(order, alignment, pool_size, size, size2, size3); + + } else { + // Default: + // - Allocate an object that consumes 1st pool + // - Allocate an object that does not fit in next pool + // but has go the 3rd one + // - Allocate a smaller object from 2nd pool + // (now first) and needs alignment + order = 1; + pool_size = MEM_POOL_SIZE; + size = MEM_POOL_SIZE; + size2 = 36; + size3 = 10; + + printf("--------- Test 2 ---------\n"); + if (!default_val) { + printf("Order 0) 1->2->3 | 1) 3->2->1 | 2) 1->3->2: "); + scanf("%d", &order); + printf("Alignment: "); + scanf("%zu", &alignment); + printf("Pool size: "); + scanf("%zu", &pool_size); + printf("Pool size2: "); + scanf("%zu", &pool_size2); + printf("Pool size3: "); + scanf("%zu", &pool_size3); + printf("Size: "); + scanf("%zu", &size); + printf("Size2: "); + scanf("%zu", &size2); + printf("Size3: "); + scanf("%zu", &size3); + } + +#if defined(OVERFLOW_DEBUG) + // Extra 2*NODE_HEADER_SIZE for overflow checks + pool_size = pool_size + (2 * NODE_HEADER_SIZE); + pool_size2 = pool_size2 + (2 * NODE_HEADER_SIZE); + pool_size3 = pool_size3 + (2 * NODE_HEADER_SIZE); +#endif + test2(order, alignment, pool_size, pool_size2, pool_size3, size, + size2, size3); + } +} + +int +main() +{ + int option = 0; + + while (option != 3) { + printf("Test 1: "); + printf("- Allocates 3 objects from one pool\n"); + printf(" - Deallocates in specified order)\n"); + + printf("Test 2: "); + printf("- Allocates 3 objects from three pools\n"); + printf(" - Deallocates in specified order:\n"); + + printf("Choose which test to execute 1 or 2 or exit (3): "); + + scanf("%d", &option); + + if (option != 3) { + tests_choice(option); + } + } + + return 0; +} diff --git a/hyp/mem/hyp_aspace/armv8/hyp_aspace.ev b/hyp/mem/hyp_aspace/armv8/hyp_aspace.ev index eaac9ae..fdf6e4e 100644 --- a/hyp/mem/hyp_aspace/armv8/hyp_aspace.ev +++ b/hyp/mem/hyp_aspace/armv8/hyp_aspace.ev @@ -4,14 +4,10 @@ module hyp_aspace -#if ARCH_AARCH64_USE_PAN subscribe partition_add_ram_range(phys_base, size) - unwinder hyp_aspace_handle_partition_remove_ram_range(phys_base, size) public + unwinder(phys_base, size) subscribe partition_remove_ram_range(phys_base, size) - unwinder hyp_aspace_handle_partition_add_ram_range(phys_base, size) public -#endif + unwinder(phys_base, size) -#if CPU_PGTABLE_BLOCK_SPLIT_LEVEL == 0 subscribe vectors_trap_data_abort_el2 -#endif diff --git a/hyp/mem/hyp_aspace/armv8/src/hyp_aspace.c b/hyp/mem/hyp_aspace/armv8/src/hyp_aspace.c index 65a597c..ce90e45 100644 --- a/hyp/mem/hyp_aspace/armv8/src/hyp_aspace.c +++ b/hyp/mem/hyp_aspace/armv8/src/hyp_aspace.c @@ -32,42 +32,44 @@ #define HYP_ASPACE_ALLOCATE_BITS (25U) #define HYP_ASPACE_ALLOCATE_SIZE (size_t) util_bit(HYP_ASPACE_ALLOCATE_BITS) +#define BITS_2MiB 21 +#define SIZE_2MiB (size_t) util_bit(BITS_2MiB) + static spinlock_t hyp_aspace_direct_lock; static const uintptr_t hyp_aspace_direct_end = - util_bit(HYP_ASPACE_MAP_DIRECT_BITS) - 1; + util_bit(HYP_ASPACE_MAP_DIRECT_BITS) - 1U; static spinlock_t hyp_aspace_alloc_lock; static _Atomic register_t *hyp_aspace_regions; -#if defined(ARCH_ARM_8_1_VHE) -static const uintptr_t hyp_aspace_alloc_base = -util_bit(HYP_ASPACE_HIGH_BITS); -static const uintptr_t hyp_aspace_alloc_end = ~(uintptr_t)0U; +#if defined(ARCH_ARM_FEAT_VHE) +static const uintptr_t hyp_aspace_alloc_base = + 0U - util_bit(HYP_ASPACE_HIGH_BITS); +static const uintptr_t hyp_aspace_alloc_end = ~(uintptr_t)0U; #else +// The upper half of the address space (256GB) is reserved for the randomised +// constant-offset mappings. The lower half is shared between the direct maps +// and the VA allocator (64GB and 192GB respectively). static const uintptr_t hyp_aspace_alloc_base = util_bit(HYP_ASPACE_MAP_DIRECT_BITS); -static const uintptr_t hyp_aspace_alloc_end = util_bit(HYP_ASPACE_BITS) - 1; +static const uintptr_t hyp_aspace_alloc_end = + util_bit(HYP_ASPACE_LOWER_HALF_BITS) - 1; #endif static const size_t hyp_aspace_total_size = hyp_aspace_alloc_end - hyp_aspace_alloc_base + 1U; static const size_t hyp_aspace_num_regions = hyp_aspace_total_size / HYP_ASPACE_ALLOCATE_SIZE; -#if CPU_PGTABLE_BLOCK_SPLIT_LEVEL == 0 -static _Atomic bool hyp_aspace_direct_unmap; -#endif - extern const char image_virt_start; extern const char image_virt_last; extern const char image_phys_start; extern const char image_phys_last; -static const uintptr_t virt_start = (paddr_t)&image_virt_start; -static const uintptr_t virt_end = (paddr_t)&image_virt_last; +static const uintptr_t virt_start = (uintptr_t)&image_virt_start; +static const uintptr_t virt_end = (uintptr_t)&image_virt_last; static const paddr_t hyp_phys_start = (paddr_t)&image_phys_start; static const paddr_t hyp_phys_last = (paddr_t)&image_phys_last; -#if !defined(ARCH_ARM_8_1_VHE) -static_assert(!ARCH_AARCH64_USE_PAN, "PAN is not useful when VHE is disabled"); -#endif +static uintptr_t hyp_aspace_physaccess_offset; void hyp_aspace_handle_boot_cold_init(void) @@ -77,42 +79,64 @@ hyp_aspace_handle_boot_cold_init(void) partition_t *hyp_partition = partition_get_private(); + // First, map the kernel image, assuming that all of the initial page + // tables are within its physical memory. This should be sufficient to + // allow partition_phys_access_begin to work, so we can do other page + // table operations with the private partition. + #if ARCH_AARCH64_USE_PAN // Congruent (constant offset) mappings to support physical address // access (partition_phys_*). - // // Access rights are set to PGTABLE_ACCESS_NONE, which creates mappings // that can only be accessed with PSTATE.PAN cleared. - // - // First, map the kernel image, assuming that all of the initial page - // tables are within its physical memory. This should be sufficient to - // allow partition_phys_access_begin to work, so we can do other page - // table operations with the private partition. + hyp_aspace_physaccess_offset = HYP_ASPACE_PHYSACCESS_OFFSET; + pgtable_access_t access = PGTABLE_ACCESS_NONE; +#else + // The upper half of the address space (256GB to 512MB) is reserved for + // the randomised constant-offset mappings. + + // Generate a random number in the rage of 1/4th of the address space + // (between 0 and 128GB) with the lower 21 bits (2MB) cleared. Then add + // it to the base of the physaccess offset which is at half-point of + // the address space (256GB). This will give us a random physaccess + // offset between half and 3/4th of the address space (256GB-384GB). + + uint64_result_t prng_ret = prng_get64(); + if (prng_ret.e != OK) { + panic("Failed to get a random number"); + } + + // Clear the lower 21 bits (2MB) of the random number + hyp_aspace_physaccess_offset = prng_ret.r & ~((1UL << 21) - 1); + hyp_aspace_physaccess_offset &= HYP_ASPACE_PHYSACCESS_OFFSET_RND_MAX; + hyp_aspace_physaccess_offset += HYP_ASPACE_PHYSACCESS_OFFSET_BASE; + pgtable_access_t access = PGTABLE_ACCESS_RW; +#endif + size_t phys_size = (size_t)(hyp_phys_last - hyp_phys_start + 1U); pgtable_hyp_start(); error_t err = pgtable_hyp_map( hyp_partition, - (uintptr_t)hyp_phys_start + HYP_ASPACE_PHYSACCESS_OFFSET, + (uintptr_t)hyp_phys_start + hyp_aspace_physaccess_offset, phys_size, hyp_phys_start, PGTABLE_HYP_MEMTYPE_WRITEBACK, - PGTABLE_ACCESS_NONE, VMSA_SHAREABILITY_INNER_SHAREABLE); + access, VMSA_SHAREABILITY_INNER_SHAREABLE); assert(err == OK); pgtable_hyp_commit(); -#else // !ARCH_AARCH64_USE_PAN -#error Non-PAN physical address access is not yet implemented -#endif // Allocate the bitmap used for region allocations. size_t bitmap_size = BITMAP_NUM_WORDS(hyp_aspace_num_regions) * sizeof(register_t); void_ptr_result_t alloc_ret = partition_alloc( hyp_partition, bitmap_size, alignof(register_t)); - - assert(alloc_ret.e == OK); - memset(alloc_ret.r, 0, bitmap_size); + if (alloc_ret.e != OK) { + panic("Unable to alloc aspace regions"); + } hyp_aspace_regions = alloc_ret.r; + (void)memset_s(hyp_aspace_regions, bitmap_size, 0, bitmap_size); + assert((virt_start >= hyp_aspace_alloc_base) && (virt_end <= hyp_aspace_alloc_end)); @@ -125,9 +149,30 @@ hyp_aspace_handle_boot_cold_init(void) for (index_t i = start_bit; i <= end_bit; i++) { bitmap_atomic_set(hyp_aspace_regions, i, memory_order_relaxed); } + + // map any remaining memory past the first 2MB of RW data which was + // mapped by the assembly boot code. + if ((size_t)PLATFORM_HEAP_PRIVATE_SIZE > SIZE_2MiB) { + size_t remaining_size = + (size_t)PLATFORM_HEAP_PRIVATE_SIZE - SIZE_2MiB; + uintptr_t remaining_virt = + (virt_end + 1U) - + ((size_t)PLATFORM_RW_DATA_SIZE - 0x200000U); + paddr_t remaining_phys = + (hyp_phys_last + 1U) - + ((size_t)PLATFORM_RW_DATA_SIZE - 0x200000U); + + pgtable_hyp_start(); + err = pgtable_hyp_map(hyp_partition, remaining_virt, + remaining_size, remaining_phys, + PGTABLE_HYP_MEMTYPE_WRITEBACK, + PGTABLE_ACCESS_RW, + VMSA_SHAREABILITY_INNER_SHAREABLE); + assert(err == OK); + pgtable_hyp_commit(); + } } -#if ARCH_AARCH64_USE_PAN error_t hyp_aspace_handle_partition_add_ram_range(paddr_t phys_base, size_t size) { @@ -137,26 +182,27 @@ hyp_aspace_handle_partition_add_ram_range(paddr_t phys_base, size_t size) assert(util_is_baligned(size, PGTABLE_HYP_PAGE_SIZE)); if (util_add_overflows(phys_base, size - 1U) || - ((phys_base + size - 1U) >= (1UL << HYP_ASPACE_MAP_DIRECT_BITS))) { + ((phys_base + size - 1U) >= util_bit(HYP_ASPACE_MAP_DIRECT_BITS))) { LOG(ERROR, WARN, "Failed to add high memory: {:x}..{:x}\n", phys_base, phys_base + size - 1U); return ERROR_ADDR_INVALID; } + spinlock_acquire(&hyp_aspace_direct_lock); pgtable_hyp_start(); -#if CPU_PGTABLE_BLOCK_SPLIT_LEVEL == 0 - atomic_exchange_explicit(&hyp_aspace_direct_unmap, true, - memory_order_acquire); -#endif - error_t err = pgtable_hyp_remap( - hyp_partition, - (uintptr_t)phys_base + HYP_ASPACE_PHYSACCESS_OFFSET, size, - phys_base, PGTABLE_HYP_MEMTYPE_WRITEBACK, PGTABLE_ACCESS_NONE, - VMSA_SHAREABILITY_INNER_SHAREABLE); -#if CPU_PGTABLE_BLOCK_SPLIT_LEVEL == 0 - atomic_store_release(&hyp_aspace_direct_unmap, false); +#if ARCH_AARCH64_USE_PAN + pgtable_access_t access = PGTABLE_ACCESS_NONE; +#else + pgtable_access_t access = PGTABLE_ACCESS_RW; #endif + uintptr_t virt = phys_base + hyp_aspace_physaccess_offset; + error_t err = + pgtable_hyp_remap_merge(hyp_partition, virt, size, phys_base, + PGTABLE_HYP_MEMTYPE_WRITEBACK, access, + VMSA_SHAREABILITY_INNER_SHAREABLE, + util_bit(HYP_ASPACE_MAP_DIRECT_BITS)); pgtable_hyp_commit(); + spinlock_release(&hyp_aspace_direct_lock); return err; } @@ -166,33 +212,44 @@ hyp_aspace_handle_partition_remove_ram_range(paddr_t phys_base, size_t size) { partition_t *hyp_partition = partition_get_private(); - char *virt = (void *)(phys_base + HYP_ASPACE_PHYSACCESS_OFFSET); - assert(util_is_baligned(phys_base, PGTABLE_HYP_PAGE_SIZE)); assert(util_is_baligned(size, PGTABLE_HYP_PAGE_SIZE)); // Remap the memory as DEVICE so that no speculative reads occur. + spinlock_acquire(&hyp_aspace_direct_lock); pgtable_hyp_start(); -#if CPU_PGTABLE_BLOCK_SPLIT_LEVEL == 0 - atomic_exchange_explicit(&hyp_aspace_direct_unmap, true, - memory_order_acquire); -#endif - pgtable_hyp_remap(hyp_partition, (uintptr_t)virt, size, phys_base, - PGTABLE_HYP_MEMTYPE_DEVICE, PGTABLE_ACCESS_RW, - VMSA_SHAREABILITY_INNER_SHAREABLE); -#if CPU_PGTABLE_BLOCK_SPLIT_LEVEL == 0 - atomic_store_release(&hyp_aspace_direct_unmap, false); -#endif + uintptr_t virt = phys_base + hyp_aspace_physaccess_offset; + (void)pgtable_hyp_remap_merge(hyp_partition, virt, size, phys_base, + PGTABLE_HYP_MEMTYPE_DEVICE, + PGTABLE_ACCESS_RW, + VMSA_SHAREABILITY_INNER_SHAREABLE, + util_bit(HYP_ASPACE_MAP_DIRECT_BITS)); pgtable_hyp_commit(); + spinlock_release(&hyp_aspace_direct_lock); // Clean the memory range being removed to ensure no future write-backs // occur. No need to remap since speculative reads after the cache // clean won't be written back. - CACHE_CLEAN_INVALIDATE_RANGE(virt, size); + CACHE_CLEAN_INVALIDATE_RANGE((char *)virt, size); return OK; } -#endif // ARCH_AARCH64_USE_PAN + +void +hyp_aspace_unwind_partition_add_ram_range(paddr_t phys_base, size_t size) +{ + error_t ret = + hyp_aspace_handle_partition_remove_ram_range(phys_base, size); + assert(ret == OK); +} + +void +hyp_aspace_unwind_partition_remove_ram_range(paddr_t phys_base, size_t size) +{ + error_t ret = + hyp_aspace_handle_partition_add_ram_range(phys_base, size); + assert(ret == OK); +} static bool reserve_range(index_t start_bit, index_t end_bit, index_t *fail_bit) @@ -218,6 +275,18 @@ reserve_range(index_t start_bit, index_t end_bit, index_t *fail_bit) return success; } +uintptr_t +hyp_aspace_get_physaccess_offset(void) +{ + return hyp_aspace_physaccess_offset; +} + +uintptr_t +hyp_aspace_get_alloc_base(void) +{ + return hyp_aspace_alloc_base; +} + virt_range_result_t hyp_aspace_allocate(size_t min_size) { @@ -273,12 +342,20 @@ hyp_aspace_allocate(size_t min_size) ret = virt_range_result_ok( (virt_range_t){ .base = virt, .size = size }); + partition_t *hyp_partition = partition_get_private(); // Preallocate shared page table levels before mapping spinlock_acquire(&hyp_aspace_alloc_lock); for (size_t offset = 0U; offset < size; offset += HYP_ASPACE_ALLOCATE_SIZE) { - pgtable_hyp_preallocate(partition_get_private(), virt + offset, - HYP_ASPACE_ALLOCATE_SIZE); + ret.e = pgtable_hyp_preallocate(hyp_partition, virt + offset, + HYP_ASPACE_ALLOCATE_SIZE); + if (ret.e != OK) { + virt_range_t vr = { .base = (virt + offset), + .size = HYP_ASPACE_ALLOCATE_SIZE }; + spinlock_release(&hyp_aspace_alloc_lock); + hyp_aspace_deallocate(hyp_partition, vr); + goto out; + } } spinlock_release(&hyp_aspace_alloc_lock); @@ -363,8 +440,9 @@ hyp_aspace_map_direct(paddr_t phys, size_t size, pgtable_access_t access, spinlock_acquire(&hyp_aspace_direct_lock); pgtable_hyp_start(); - err = pgtable_hyp_map(partition_get_private(), virt, size, phys, - memtype, access, share); + err = pgtable_hyp_map_merge(partition_get_private(), virt, size, phys, + memtype, access, share, + util_bit(HYP_ASPACE_MAP_DIRECT_BITS)); pgtable_hyp_commit(); spinlock_release(&hyp_aspace_direct_lock); @@ -392,15 +470,8 @@ hyp_aspace_unmap_direct(paddr_t phys, size_t size) spinlock_acquire(&hyp_aspace_direct_lock); pgtable_hyp_start(); -#if CPU_PGTABLE_BLOCK_SPLIT_LEVEL == 0 - atomic_exchange_explicit(&hyp_aspace_direct_unmap, true, - memory_order_acquire); -#endif pgtable_hyp_unmap(partition_get_private(), virt, size, PGTABLE_HYP_UNMAP_PRESERVE_NONE); -#if CPU_PGTABLE_BLOCK_SPLIT_LEVEL == 0 - atomic_store_release(&hyp_aspace_direct_unmap, false); -#endif pgtable_hyp_commit(); spinlock_release(&hyp_aspace_direct_lock); @@ -408,9 +479,8 @@ hyp_aspace_unmap_direct(paddr_t phys, size_t size) return err; } -#if CPU_PGTABLE_BLOCK_SPLIT_LEVEL == 0 -// Retry faults if they may have been caused by break before make during block -// splits in the direct physical access region +// Retry faults if they may have been caused by TLB conflicts or break before +// make during block splits or merges in the direct physical access region. bool hyp_aspace_handle_vectors_trap_data_abort_el2(ESR_EL2_t esr) { @@ -419,7 +489,15 @@ hyp_aspace_handle_vectors_trap_data_abort_el2(ESR_EL2_t esr) ESR_EL2_ISS_DATA_ABORT_cast(ESR_EL2_get_ISS(&esr)); iss_da_ia_fsc_t fsc = ESR_EL2_ISS_DATA_ABORT_get_DFSC(&iss); - // Only translation faults can be caused by BBM + FAR_EL2_t far = register_FAR_EL2_read_ordered(&asm_ordering); + uintptr_t addr = FAR_EL2_get_VirtualAddress(&far); + +#if (CPU_PGTABLE_BBM_LEVEL < 2) && !defined(PLATFORM_PGTABLE_AVOID_BBM) + // If the FEAT_BBM level is 0, then block splits and merges will do + // break before make, and we might get transient translation faults. + // If the FEAT_BBM level is 1, then splits and merges will temporarily + // set the nT bit in the block PTE while flushing the TLBs; the CPU is + // allowed to treat this the same as an invalid entry. if ((fsc != ISS_DA_IA_FSC_TRANSLATION_1) && (fsc != ISS_DA_IA_FSC_TRANSLATION_2) && (fsc != ISS_DA_IA_FSC_TRANSLATION_3)) { @@ -427,18 +505,18 @@ hyp_aspace_handle_vectors_trap_data_abort_el2(ESR_EL2_t esr) } // Only handle faults that are in the direct access region - FAR_EL2_t far = register_FAR_EL2_read_ordered(&asm_ordering); - uintptr_t addr = FAR_EL2_get_VirtualAddress(&far); - if (addr > (HYP_ASPACE_PHYSACCESS_OFFSET + + if (addr > (hyp_aspace_physaccess_offset + util_bit(HYP_ASPACE_MAP_DIRECT_BITS) - 1)) { goto out; } - if (!atomic_load_acquire(&hyp_aspace_direct_unmap)) { + if (spinlock_trylock(&hyp_aspace_direct_lock)) { + spinlock_release(&hyp_aspace_direct_lock); + // There is no map in progress. Perform a lookup to see whether // the accessed address is now mapped. - PAR_EL1_RAW_t saved_par = - register_PAR_EL1_RAW_read_ordered(&asm_ordering); + PAR_EL1_base_t saved_par = + register_PAR_EL1_base_read_ordered(&asm_ordering); if (ESR_EL2_ISS_DATA_ABORT_get_WnR(&iss)) { __asm__ volatile("at S1E2W, %[addr] ;" "isb ;" @@ -450,14 +528,14 @@ hyp_aspace_handle_vectors_trap_data_abort_el2(ESR_EL2_t esr) : "+m"(asm_ordering) : [addr] "r"(addr)); } - PAR_EL1_RAW_t par_raw = - register_PAR_EL1_RAW_read_ordered(&asm_ordering); - register_PAR_EL1_RAW_write_ordered(saved_par, &asm_ordering); + PAR_EL1_base_t par_raw = + register_PAR_EL1_base_read_ordered(&asm_ordering); + register_PAR_EL1_base_write_ordered(saved_par, &asm_ordering); // If the accessed address is now mapped, we can just return // from the fault. Otherwise we can consider the fault to be // fatal, because there is no BBM operation still in progress. - PAR_EL1_F0_t par = PAR_EL1_F0_cast(PAR_EL1_RAW_raw(par_raw)); + PAR_EL1_F0_t par = PAR_EL1_F0_cast(PAR_EL1_base_raw(par_raw)); handled = !PAR_EL1_F0_get_F(&par); } else { // A map operation is in progress, so retry until it finishes. @@ -465,11 +543,27 @@ hyp_aspace_handle_vectors_trap_data_abort_el2(ESR_EL2_t esr) // corrupt! handled = true; } +#else + // If the FEAT_BBM level is 2 we do block splits and merges without BBM + // or the nT bit. So we might get TLB conflicts. If one occurs, we must + // flush the TLB and retry. We don't need to broadcast the TLB flush, + // because the operation causing the fault should do that. + if (fsc == ISS_DA_IA_FSC_TLB_CONFLICT) { + vmsa_tlbi_va_input_t input = vmsa_tlbi_va_input_default(); + vmsa_tlbi_va_input_set_VA(&input, addr); + + __asm__ volatile("tlbi VAE2, %[VA]; dsb nsh" + : "+m"(asm_ordering) + : [VA] "r"(vmsa_tlbi_va_input_raw(input))); + + handled = true; + goto out; + } +#endif out: return handled; } -#endif // CPU_PGTABLE_BLOCK_SPLIT_LEVEL == 0 lookup_result_t hyp_aspace_is_mapped(uintptr_t virt, size_t size, pgtable_access_t access) @@ -517,7 +611,7 @@ hyp_aspace_is_mapped(uintptr_t virt, size_t size, pgtable_access_t access) first_lookup = false; } - have_access = ((curr_access & access) == access); + have_access = pgtable_access_check(curr_access, access); direct = direct && (curr == phys); contiguous = contiguous && have_access; diff --git a/hyp/mem/hyp_aspace/hyp_aspace.tc b/hyp/mem/hyp_aspace/hyp_aspace.tc index f2a40b9..9e2ff52 100644 --- a/hyp/mem/hyp_aspace/hyp_aspace.tc +++ b/hyp/mem/hyp_aspace/hyp_aspace.tc @@ -2,14 +2,25 @@ // // SPDX-License-Identifier: BSD-3-Clause -#if defined(ARCH_ARM_8_1_VHE) +define HYP_ASPACE_MAP_DIRECT_BITS constant type count_t = + PLATFORM_PHYS_ADDRESS_BITS; + +#if defined(ARCH_ARM_FEAT_VHE) define HYP_ASPACE_LOW_BITS constant type count_t = HYP_ASPACE_MAP_DIRECT_BITS + 1; define HYP_ASPACE_HIGH_BITS constant type count_t = 39; define HYP_ASPACE_PHYSACCESS_OFFSET constant uintptr = (1 << HYP_ASPACE_MAP_DIRECT_BITS); #else -define HYP_ASPACE_BITS constant type count_t = HYP_ASPACE_MAP_DIRECT_BITS + 2; +define HYP_ASPACE_LOW_BITS constant type count_t = 39; +// Reserve the upper half of the address space for the randomised +// constant-offset mappings +define HYP_ASPACE_LOWER_HALF_BITS constant type count_t = + HYP_ASPACE_LOW_BITS - 1; +define HYP_ASPACE_PHYSACCESS_OFFSET_BASE constant uintptr = + (1 << (HYP_ASPACE_LOW_BITS - 1)); +define HYP_ASPACE_PHYSACCESS_OFFSET_RND_MAX constant uintptr = + (1 << (HYP_ASPACE_LOW_BITS - 2)) - 1; #endif define lookup_result bitfield<32> { diff --git a/hyp/mem/memdb/memdb.ev b/hyp/mem/memdb/memdb.ev index 0c88d28..30a717b 100644 --- a/hyp/mem/memdb/memdb.ev +++ b/hyp/mem/memdb/memdb.ev @@ -12,7 +12,7 @@ subscribe boot_cold_init() priority 10 subscribe partition_add_ram_range(owner, phys_base, size) - unwinder memdb_handle_partition_remove_ram_range(owner, phys_base, size) public + unwinder subscribe partition_remove_ram_range(owner, phys_base, size) - unwinder memdb_handle_partition_add_ram_range(owner, phys_base, size) public + unwinder diff --git a/hyp/mem/memdb/memdb.tc b/hyp/mem/memdb/memdb.tc index b37a04d..6563372 100644 --- a/hyp/mem/memdb/memdb.tc +++ b/hyp/mem/memdb/memdb.tc @@ -55,5 +55,5 @@ define memdb_op enumeration { }; extend trace_class enumeration { - MEMDB; + MEMDB = 7; }; diff --git a/hyp/mem/memdb/memdb_tests.ev b/hyp/mem/memdb/memdb_tests.ev index c1467b8..ee79162 100644 --- a/hyp/mem/memdb/memdb_tests.ev +++ b/hyp/mem/memdb/memdb_tests.ev @@ -9,4 +9,5 @@ subscribe tests_init subscribe tests_start priority last + require_preempt_disabled #endif diff --git a/hyp/mem/memdb/memdb_tests.tc b/hyp/mem/memdb/memdb_tests.tc index e848c40..6d8db20 100644 --- a/hyp/mem/memdb/memdb_tests.tc +++ b/hyp/mem/memdb/memdb_tests.tc @@ -16,6 +16,7 @@ define memdb_range structure { define memdb_data structure { ranges array(MEMDB_RANGES_NUM) structure memdb_range; ranges_count type count_t; + ranges_index type index_t; }; #endif diff --git a/hyp/mem/memdb/src/memdb.c b/hyp/mem/memdb/src/memdb.c index cdab30d..21df4be 100644 --- a/hyp/mem/memdb/src/memdb.c +++ b/hyp/mem/memdb/src/memdb.c @@ -81,9 +81,9 @@ #endif #define MEMDB_BITS_PER_ENTRY_MASK util_mask(MEMDB_BITS_PER_ENTRY) -#define ADDR_SIZE (sizeof(paddr_t) * CHAR_BIT) +#define ADDR_SIZE (sizeof(paddr_t) * (size_t)CHAR_BIT) // levels + 1 for root -#define MAX_LEVELS (ADDR_SIZE / MEMDB_BITS_PER_ENTRY) + 1 +#define MAX_LEVELS (ADDR_SIZE / MEMDB_BITS_PER_ENTRY) + 1U #if (defined(VERBOSE) && VERBOSE) || defined(UNIT_TESTS) #define MEMDB_DEBUG 1 @@ -97,14 +97,14 @@ extern const char image_phys_last; static const paddr_t phys_start = (paddr_t)&image_phys_start; static const paddr_t phys_end = (paddr_t)&image_phys_last; -typedef struct start_path { +typedef struct start_path_s { memdb_level_t *levels[MAX_LEVELS]; index_t indexes[MAX_LEVELS]; count_t count; } start_path_t; -typedef struct locked_levels { - spinlock_t *locks[MAX_LEVELS]; +typedef struct locked_levels_s { + spinlock_t *locks[MAX_LEVELS]; _Atomic memdb_entry_t *entries[MAX_LEVELS]; count_t count; uint8_t pad_end_[4]; @@ -141,7 +141,8 @@ get_next_index(paddr_t addr, count_t *shifts) *shifts -= MEMDB_BITS_PER_ENTRY; - return ((addr >> *shifts) & MEMDB_BITS_PER_ENTRY_MASK); + return ((count_t)(addr >> *shifts) & + (count_t)MEMDB_BITS_PER_ENTRY_MASK); } static void @@ -182,7 +183,8 @@ init_level(memdb_level_t *level, allocator_t *allocator, memdb_type_t type, for (index_t i = 0; i < MEMDB_NUM_ENTRIES; i++) { // Guard shifts of 64 (ADDR_SIZE) means there is no guard. atomic_entry_write(&level->level[i], memory_order_relaxed, - MEMDB_TYPE_LEVEL, ADDR_SIZE, type, obj); + (paddr_t)MEMDB_TYPE_LEVEL, + (count_t)ADDR_SIZE, type, obj); } } @@ -269,9 +271,9 @@ unlock_levels(locked_levels_t *locked_levels) LOCK_IMPL assert(locked_levels->count != 0U); - for (count_t i = (locked_levels->count - 1); i > 0; i--) { + for (count_t i = (locked_levels->count - 1U); i > 0U; i--) { memdb_entry_t entry = atomic_load_explicit( - locked_levels->entries[i - 1], memory_order_relaxed); + locked_levels->entries[i - 1U], memory_order_relaxed); memdb_level_t *level = (memdb_level_t *)entry.next; #if defined(MEMDB_DEBUG) @@ -295,8 +297,8 @@ unlock_levels(locked_levels_t *locked_levels) LOCK_IMPL assert(cont_level); #endif if (optimize) { - atomic_entry_read(&level->level[0], &guard, - &guard_shifts, &type, &next); + (void)atomic_entry_read(&level->level[0], &guard, + &guard_shifts, &type, &next); res = are_all_entries_same(level, next, MEMDB_NUM_ENTRIES, type, 0, @@ -304,7 +306,7 @@ unlock_levels(locked_levels_t *locked_levels) LOCK_IMPL if (res) { // Update parent and deallocate level. atomic_entry_write( - locked_levels->entries[i - 1], + locked_levels->entries[i - 1U], memory_order_relaxed, guard, guard_shifts, type, next); @@ -331,7 +333,8 @@ unlock_levels(locked_levels_t *locked_levels) LOCK_IMPL static paddr_t calculate_address(paddr_t addr, count_t shifts, index_t index) { - paddr_t result = util_p2align_down(addr, shifts + MEMDB_BITS_PER_ENTRY); + paddr_t result = util_p2align_down( + addr, ((paddr_t)shifts + MEMDB_BITS_PER_ENTRY)); assert(index < util_bit(MEMDB_BITS_PER_ENTRY)); @@ -366,19 +369,19 @@ fill_level_entries(memdb_level_t *level, uintptr_t object, memdb_type_t type, goto end_function; } atomic_entry_write(&level->level[i], memory_order_relaxed, 0, - ADDR_SIZE, type, object); + (count_t)ADDR_SIZE, type, object); } end_function: if (ret != OK) { if (failed_index > start_index) { *last_success_addr = calculate_address( - addr, shifts, failed_index - 1); + addr, shifts, failed_index - 1U); } } else { if ((op != MEMDB_OP_ROLLBACK) && (start_index != end_index)) { *last_success_addr = - calculate_address(addr, shifts, end_index - 1); + calculate_address(addr, shifts, end_index - 1U); } } out: @@ -436,8 +439,8 @@ create_n_levels(allocator_t *allocator, memdb_level_t **level, bool start, comparison = util_mask(*shifts); } - atomic_entry_read(&(*level)->level[*index], &level_guard, - &level_guard_shifts, &level_type, &level_next); + (void)atomic_entry_read(&(*level)->level[*index], &level_guard, + &level_guard_shifts, &level_type, &level_next); // Create levels and update parent entry to point to new level. while ((*shifts != limit) && @@ -456,7 +459,7 @@ create_n_levels(allocator_t *allocator, memdb_level_t **level, bool start, memdb_level_t *next_level = res.r; level_guard = 0; - level_guard_shifts = ADDR_SIZE; + level_guard_shifts = (count_t)ADDR_SIZE; level_type = MEMDB_TYPE_LEVEL; level_next = (uintptr_t)next_level; @@ -474,8 +477,8 @@ create_n_levels(allocator_t *allocator, memdb_level_t **level, bool start, *common_level_shifts = *shifts; } - if ((start_path->count == 0) || - (start_path->levels[start_path->count - 1] != + if ((start_path->count == 0U) || + (start_path->levels[start_path->count - 1U] != *level)) { start_path->levels[start_path->count] = *level; start_path->indexes[start_path->count] = *index; @@ -523,15 +526,15 @@ go_down_levels(memdb_level_t *first_level, memdb_level_t **level, paddr_t addr, uintptr_t level_next; error_t ret = OK; - atomic_entry_read(&(*level)->level[*index], &level_guard, - &level_guard_shifts, &level_type, &level_next); + (void)atomic_entry_read(&(*level)->level[*index], &level_guard, + &level_guard_shifts, &level_type, &level_next); // We need to go down the levels until we find an empty entry or we run // out of remaining bits. In the former case, return error since the // address already has an owner. while ((level_type == MEMDB_TYPE_LEVEL) && (*shifts != 0U)) { if (start) { - paddr_t level_shifts = *shifts + MEMDB_BITS_PER_ENTRY; + count_t level_shifts = *shifts + MEMDB_BITS_PER_ENTRY; if ((start) && (op == MEMDB_OP_ROLLBACK) && (level_shifts != ADDR_SIZE) && @@ -541,8 +544,8 @@ go_down_levels(memdb_level_t *first_level, memdb_level_t **level, paddr_t addr, *common_level_shifts = *shifts; } - if ((start_path->count == 0) || - (start_path->levels[start_path->count - 1] != + if ((start_path->count == 0U) || + (start_path->levels[start_path->count - 1U] != *level)) { start_path->levels[start_path->count] = *level; start_path->indexes[start_path->count] = *index; @@ -563,7 +566,7 @@ go_down_levels(memdb_level_t *first_level, memdb_level_t **level, paddr_t addr, last_shifts = level_guard_shifts; level_guard = 0; - level_guard_shifts = ADDR_SIZE; + level_guard_shifts = (count_t)ADDR_SIZE; memdb_level_t *level_aux = *level; ret = create_n_levels( @@ -585,12 +588,13 @@ go_down_levels(memdb_level_t *first_level, memdb_level_t **level, paddr_t addr, (uintptr_t)last_level); if (start && (*level != level_aux)) { - paddr_t level_shifts = + count_t level_shifts = *shifts + MEMDB_BITS_PER_ENTRY; if ((start) && (op == MEMDB_OP_ROLLBACK) && - (level_shifts != ADDR_SIZE) && + (level_shifts != + (count_t)ADDR_SIZE) && ((*last_success_addr >> level_shifts) == (addr >> level_shifts))) { @@ -618,7 +622,7 @@ go_down_levels(memdb_level_t *first_level, memdb_level_t **level, paddr_t addr, tmp_cmn); if ((aux_shifts + level_guard_shifts) != - ADDR_SIZE) { + (count_t)ADDR_SIZE) { index_t new_index; last_shifts = level_guard_shifts + @@ -650,14 +654,14 @@ go_down_levels(memdb_level_t *first_level, memdb_level_t **level, paddr_t addr, (uintptr_t)last_level); if (start && (*level != level_aux)) { - paddr_t level_shifts = + count_t level_shifts = *shifts + MEMDB_BITS_PER_ENTRY; if ((start) && (op == MEMDB_OP_ROLLBACK) && (level_shifts != - ADDR_SIZE) && + (count_t)ADDR_SIZE) && ((*last_success_addr >> level_shifts) == (addr >> level_shifts))) { @@ -708,9 +712,9 @@ go_down_levels(memdb_level_t *first_level, memdb_level_t **level, paddr_t addr, *level = (memdb_level_t *)level_next; *index = get_next_index(addr, shifts); - atomic_entry_read(&(*level)->level[*index], &level_guard, - &level_guard_shifts, &level_type, - &level_next); + (void)atomic_entry_read(&(*level)->level[*index], &level_guard, + &level_guard_shifts, &level_type, + &level_next); } if ((level_type != prev_type) || (level_next != prev_object) || @@ -779,7 +783,7 @@ add_address(allocator_t *allocator, uintptr_t object, memdb_type_t type, } level_guard = 0; - level_guard_shifts = ADDR_SIZE; + level_guard_shifts = (count_t)ADDR_SIZE; level_type = type; level_next = object; @@ -805,7 +809,7 @@ add_address(allocator_t *allocator, uintptr_t object, memdb_type_t type, } if ((start_path.count == 0U) || - (start_path.levels[start_path.count - 1] != level)) { + (start_path.levels[start_path.count - 1U] != level)) { start_path.levels[start_path.count] = level; start_path.indexes[start_path.count] = index; start_path.count++; @@ -816,12 +820,12 @@ add_address(allocator_t *allocator, uintptr_t object, memdb_type_t type, common_level_shifts = first_level_shifts - MEMDB_BITS_PER_ENTRY; } - count = start_path.count - 1; + count = start_path.count - 1U; // Fill entries from start_index+1 to MEMDB_NUM_ENTRIES in start path // levels while (start_path.levels[count] != common_level) { - index_t start_index = start_path.indexes[count] + 1; + index_t start_index = start_path.indexes[count] + 1U; level = start_path.levels[count]; @@ -837,11 +841,11 @@ add_address(allocator_t *allocator, uintptr_t object, memdb_type_t type, if ((op == MEMDB_OP_ROLLBACK) && (count != 0U)) { level = start_path.levels[count]; - index_t start_index = start_path.indexes[count] + 1; + index_t start_index = start_path.indexes[count] + 1U; index_t end_index = - ((*last_success_addr >> common_level_shifts) & - MEMDB_BITS_PER_ENTRY_MASK) + - 1; + (index_t)(((*last_success_addr >> common_level_shifts) & + MEMDB_BITS_PER_ENTRY_MASK) + + 1U); // Fill intermediate entries of new common level ret = fill_level_entries(level, object, type, prev_object, @@ -897,13 +901,13 @@ add_address_range(allocator_t *allocator, paddr_t start_addr, paddr_t end_addr, memdb_type_t level_type; uintptr_t level_next; - atomic_entry_read(&common_level->level[start_index], - &level_guard, &level_guard_shifts, - &level_type, &level_next); + (void)atomic_entry_read(&common_level->level[start_index], + &level_guard, &level_guard_shifts, + &level_type, &level_next); if ((level_type == prev_type) && (level_next == prev_object)) { atomic_entry_write(&common_level->level[start_index], - memory_order_relaxed, 0, ADDR_SIZE, - type, object); + memory_order_relaxed, 0, + (count_t)ADDR_SIZE, type, object); } else { ret = ERROR_MEMDB_NOT_OWNER; } @@ -934,7 +938,7 @@ add_address_range(allocator_t *allocator, paddr_t start_addr, paddr_t end_addr, // Fill first level intermediate entries between start end end ret = fill_level_entries(common_level, object, type, prev_object, - prev_type, start_index + 1, end_index, + prev_type, start_index + 1U, end_index, start_addr, last_success_addr, end_shifts, op); if (ret != OK) { goto end_function; @@ -966,17 +970,20 @@ compare_adjust_bits(count_t guard_shifts, count_t shifts, ret = ERROR_ADDR_INVALID; goto end_function; } + } else { + // No need to adjust the guard_shifts value } if (insert) { - if ((guard_shifts + *extra_guard_shifts) != ADDR_SIZE) { + if ((guard_shifts + *extra_guard_shifts) != + (count_t)ADDR_SIZE) { tmp_guard = guard >> *extra_guard_shifts; } else { tmp_guard = 0; } } - if ((shifts + *extra_shifts) != ADDR_SIZE) { + if ((shifts + *extra_shifts) != (count_t)ADDR_SIZE) { tmp_cmn = addr >> (shifts + *extra_shifts); } else { tmp_cmn = 0; @@ -1010,7 +1017,7 @@ compare_adjust_bits(count_t guard_shifts, count_t shifts, } else { tmp_guard = 0; } - if ((shifts + *extra_shifts) != ADDR_SIZE) { + if ((shifts + *extra_shifts) != (count_t)ADDR_SIZE) { tmp_cmn = addr >> (shifts + *extra_shifts); } else { tmp_cmn = 0; @@ -1054,9 +1061,9 @@ add_extra_shifts_update(allocator_t *allocator, count_t *shifts, lock_level(level, index, locked_levels); } - atomic_entry_read(&level->level[index], &level_guard, - &level_guard_shifts, &level_type, - &level_next); + (void)atomic_entry_read(&level->level[index], &level_guard, + &level_guard_shifts, &level_type, + &level_next); // If entry has guard, it must match with common bits. ret = check_guard(level_guard_shifts, level_guard, end_addr, @@ -1086,13 +1093,16 @@ add_extra_shifts_update(allocator_t *allocator, count_t *shifts, locked_levels->entries[1]; locked_levels->locks[0] = locked_levels->locks[1]; - locked_levels->entries[1] = 0; - locked_levels->locks[1] = 0; - locked_levels->count = 1; + locked_levels->entries[1] = + (_Atomic(memdb_entry_t) *)NULL; + locked_levels->locks[1] = (spinlock_t *)NULL; + locked_levels->count = 1; } else if (locking) { // Current level needs to be locked, so all // next levels also need to be. lock_taken = true; + } else { + // Nothing to do } level = (memdb_level_t *)level_next; @@ -1232,8 +1242,8 @@ create_intermediate_level(allocator_t *allocator, paddr_t start_addr, // Set guard equal to common bits and create level. - atomic_entry_read(&level->level[index], &lower_guard, - &lower_guard_shifts, &lower_type, &lower_next); + (void)atomic_entry_read(&level->level[index], &lower_guard, + &lower_guard_shifts, &lower_type, &lower_next); paddr_t lower_addr = lower_guard << lower_guard_shifts; @@ -1259,7 +1269,7 @@ create_intermediate_level(allocator_t *allocator, paddr_t start_addr, if (level_guard_shifts == (level_shifts - MEMDB_BITS_PER_ENTRY)) { // No guard if no levels are skipped. - level_guard_shifts = ADDR_SIZE; + level_guard_shifts = (count_t)ADDR_SIZE; level_guard = 0U; } @@ -1277,7 +1287,7 @@ create_intermediate_level(allocator_t *allocator, paddr_t start_addr, if (lower_guard_shifts == (shifts - MEMDB_BITS_PER_ENTRY)) { // No guard if no levels are skipped. - lower_guard_shifts = ADDR_SIZE; + lower_guard_shifts = (count_t)ADDR_SIZE; lower_guard = 0U; } @@ -1306,7 +1316,7 @@ add_extra_shifts(allocator_t *allocator, count_t shifts, count_t extra_shifts, error_t ret = OK; while (rem_cmn_shifts != shifts) { - index_t llevel_index = locked_levels->count - 1; + index_t llevel_index = locked_levels->count - 1U; count_t level_shifts = rem_cmn_shifts; index_t index = get_next_index(start_addr, &rem_cmn_shifts); @@ -1318,9 +1328,9 @@ add_extra_shifts(allocator_t *allocator, count_t shifts, count_t extra_shifts, lock_level(level, index, locked_levels); } - atomic_entry_read(&level->level[index], &level_guard, - &level_guard_shifts, &level_type, - &level_next); + (void)atomic_entry_read(&level->level[index], &level_guard, + &level_guard_shifts, &level_type, + &level_next); if (level_type != MEMDB_TYPE_NOTYPE) { if ((!lock_taken) && @@ -1339,9 +1349,10 @@ add_extra_shifts(allocator_t *allocator, count_t shifts, count_t extra_shifts, locked_levels->entries[1]; locked_levels->locks[0] = locked_levels->locks[1]; - locked_levels->entries[1] = 0; - locked_levels->locks[1] = 0; - locked_levels->count = 1; + locked_levels->entries[1] = + (_Atomic(memdb_entry_t) *)NULL; + locked_levels->locks[1] = (spinlock_t *)NULL; + locked_levels->count = 1; } else { // Current level needs to hold lock, so all // next levels also. @@ -1480,8 +1491,8 @@ find_common_level(paddr_t start_addr, paddr_t end_addr, } } - atomic_entry_read(&memdb.root, &guard, &guard_shifts, &root_type, - &next); + (void)atomic_entry_read(&memdb.root, &guard, &guard_shifts, &root_type, + &next); if ((!first) && (root_type == MEMDB_TYPE_NOTYPE)) { ret = ERROR_MEMDB_EMPTY; @@ -1577,7 +1588,7 @@ add_range(allocator_t *allocator, paddr_t start_addr, paddr_t end_addr, if (init_error != OK) { ret = init_error; - goto unlock_levels; + goto unlock_levels_l; } ret = add_address_range(allocator, start_addr, end_addr, common_level, @@ -1588,14 +1599,15 @@ add_range(allocator_t *allocator, paddr_t start_addr, paddr_t end_addr, if ((ret != OK) && (start_addr <= last_success_addr) && (last_success_addr != (paddr_t)-1)) { // Rolling back the entries to old owner. - add_address_range(allocator, start_addr, last_success_addr, - common_level, shifts, prev_object, prev_type, - object, obj_type, end_locked_levels, - &start_locked_levels, &last_success_addr, - MEMDB_OP_ROLLBACK); + (void)add_address_range(allocator, start_addr, + last_success_addr, common_level, shifts, + prev_object, prev_type, object, + obj_type, end_locked_levels, + &start_locked_levels, + &last_success_addr, MEMDB_OP_ROLLBACK); } -unlock_levels: +unlock_levels_l: if (start_locked_levels.count != 0U) { unlock_levels(&start_locked_levels); } @@ -1618,8 +1630,8 @@ check_address(memdb_level_t *first_level, memdb_level_t **level, paddr_t addr, uintptr_t level_next; error_t ret = OK; - atomic_entry_read(&(*level)->level[*index], &level_guard, - &level_guard_shifts, &level_type, &level_next); + (void)atomic_entry_read(&(*level)->level[*index], &level_guard, + &level_guard_shifts, &level_type, &level_next); // We need to go down the levels until we find an empty entry or we run // out of remaining bits. In the former case, return error since the @@ -1640,7 +1652,7 @@ check_address(memdb_level_t *first_level, memdb_level_t **level, paddr_t addr, bool res = false; if (start) { - start_index = *index + 1; + start_index = *index + 1U; end_index = MEMDB_NUM_ENTRIES; } else { start_index = 0; @@ -1659,9 +1671,9 @@ check_address(memdb_level_t *first_level, memdb_level_t **level, paddr_t addr, *level = (memdb_level_t *)level_next; *index = get_next_index(addr, shifts); - atomic_entry_read(&(*level)->level[*index], &level_guard, - &level_guard_shifts, &level_type, - &level_next); + (void)atomic_entry_read(&(*level)->level[*index], &level_guard, + &level_guard_shifts, &level_type, + &level_next); } if ((op == MEMDB_OP_CONTIGUOUSNESS) && @@ -1699,8 +1711,8 @@ memdb_insert(partition_t *partition, paddr_t start_addr, paddr_t end_addr, allocator_t *allocator = &partition->allocator; - atomic_entry_read(&memdb.root, &guard, &guard_shifts, &root_type, - &next); + (void)atomic_entry_read(&memdb.root, &guard, &guard_shifts, &root_type, + &next); if (root_type == MEMDB_TYPE_NOTYPE) { first_entry = true; @@ -1760,7 +1772,7 @@ memdb_insert(partition_t *partition, paddr_t start_addr, paddr_t end_addr, if (ret == OK) { TRACE(MEMDB, INFO, "memdb_insert: {:#x}..{:#x} - obj({:#x}) - type({:d})", - start_addr, end_addr, object, obj_type); + start_addr, end_addr, object, (register_t)obj_type); #if defined(MEMDB_DEBUG) // Check that the range was added correctly @@ -1776,7 +1788,8 @@ memdb_insert(partition_t *partition, paddr_t start_addr, paddr_t end_addr, } else { TRACE(MEMDB, INFO, "memdb: Error inserting {:#x}..{:#x} - obj({:#x}) - type({:d}), err = {:d}", - start_addr, end_addr, object, obj_type, (register_t)ret); + start_addr, end_addr, object, (register_t)obj_type, + (register_t)ret); } return ret; @@ -1818,7 +1831,7 @@ memdb_update(partition_t *partition, paddr_t start_addr, paddr_t end_addr, if (ret == OK) { TRACE(MEMDB, INFO, "memdb_update: {:#x}..{:#x} - obj({:#x}) - type({:d})", - start_addr, end_addr, object, obj_type); + start_addr, end_addr, object, (register_t)obj_type); #if defined(MEMDB_DEBUG) // Check that the range was added correctly @@ -1834,7 +1847,8 @@ memdb_update(partition_t *partition, paddr_t start_addr, paddr_t end_addr, } else { TRACE(MEMDB, INFO, "memdb: Error updating {:#x}..{:#x} - obj({:#x}) - type({:d}), err = {:d}", - start_addr, end_addr, object, obj_type, (register_t)ret); + start_addr, end_addr, object, (register_t)obj_type, + (register_t)ret); } return ret; @@ -1866,7 +1880,8 @@ memdb_is_ownership_contiguous(paddr_t start_addr, paddr_t end_addr, assert((start_addr != 0U) || (~end_addr != 0U)); ret = find_common_level(start_addr, end_addr, &common_level, &shifts, - NULL, object, type, 0, 0, NULL, false, false); + NULL, object, type, 0, MEMDB_TYPE_LEVEL, NULL, + false, false); if (ret != OK) { ret_bool = false; goto end_function; @@ -1891,7 +1906,7 @@ memdb_is_ownership_contiguous(paddr_t start_addr, paddr_t end_addr, // Check first level intermediate entries between start end end ret_bool = are_all_entries_same(common_level, object, MEMDB_NUM_ENTRIES, - type, start_index + 1, end_index); + type, start_index + 1U, end_index); if (!ret_bool) { goto end_function; } @@ -1923,12 +1938,12 @@ memdb_lookup(paddr_t addr) count_t guard_shifts; memdb_type_t root_type; uintptr_t next; - memdb_level_t *level; + memdb_level_t *level; index_t index; bool start = true; - atomic_entry_read(&memdb.root, &guard, &guard_shifts, &root_type, - &next); + (void)atomic_entry_read(&memdb.root, &guard, &guard_shifts, &root_type, + &next); if (root_type == MEMDB_TYPE_NOTYPE) { ret.e = ERROR_MEMDB_EMPTY; @@ -1947,7 +1962,8 @@ memdb_lookup(paddr_t addr) // Go down levels until we get to input address // Dummy start argument, does not affect lookup. ret.e = check_address((memdb_level_t *)next, &level, addr, &index, - &guard_shifts, MEMDB_OP_LOOKUP, start, 0, 0); + &guard_shifts, MEMDB_OP_LOOKUP, start, 0, + MEMDB_TYPE_LEVEL); if (ret.e != OK) { ret.r.type = MEMDB_TYPE_NOTYPE; ret.r.object = 0; @@ -2007,25 +2023,30 @@ memdb_do_walk(uintptr_t object, memdb_type_t type, memdb_fnptr fn, void *arg, break; } - atomic_entry_read(&level->level[index], &guard, - &guard_shifts, &next_type, &next); + (void)atomic_entry_read(&level->level[index], &guard, + &guard_shifts, &next_type, + &next); + size_t size; if (guard_shifts != ADDR_SIZE) { - if (next_type == MEMDB_TYPE_NOTYPE) { - // FIXME: handle bad entry. - } else { - assert(next_type == MEMDB_TYPE_LEVEL); - } + assert(next_type == MEMDB_TYPE_LEVEL); + base = guard << guard_shifts; + size = util_bit(guard_shifts); + } else { + size = util_bit(shifts); } - if ((next_type == type) && (next == object)) { - // If entry points to the object, meaning this - // address is owned by the object, we add it to - // the pending address and size to be added to - // the range. The range will be added when the - // ownership stops being contiguous. + // Skip entry if it is before the start address. + if (!all_memdb && ((base + size - 1U) < start_addr)) { + index++; + continue; + } - size_t size = util_bit(shifts); + if ((next_type == type) && (next == object)) { + // Entry points to the object. If the entry is + // contiguous with the current pending range, we + // add this entry to it; otherwise we flush it + // and start a new one. if (!all_memdb) { if (base < start_addr) { @@ -2033,22 +2054,19 @@ memdb_do_walk(uintptr_t object, memdb_type_t type, memdb_fnptr fn, void *arg, base = start_addr; } - if ((base + size - 1) > end_addr) { - size -= (base + size - 1) - + if ((base + size - 1U) > end_addr) { + size -= (base + size - 1U) - end_addr; } } - if (pending_size != 0U) { - assert((pending_base + pending_size) == - base); + assert((pending_base + pending_size) <= base); + + if ((pending_base + pending_size) == base) { pending_size += size; - } else { - pending_base = base; - pending_size = size; + index++; + continue; } - index++; - } else if (next_type == MEMDB_TYPE_LEVEL) { // We move down to the next level and iterate // through all its entries. We save current @@ -2059,7 +2077,7 @@ memdb_do_walk(uintptr_t object, memdb_type_t type, memdb_fnptr fn, void *arg, covered_stack[count] = covered_bits; shifts_stack[count] = shifts; levels[count] = level; - index_stack[count] = index + 1; + index_stack[count] = index + 1U; count++; if (guard_shifts == ADDR_SIZE) { @@ -2076,21 +2094,23 @@ memdb_do_walk(uintptr_t object, memdb_type_t type, memdb_fnptr fn, void *arg, level = (memdb_level_t *)next; index = 0; - + continue; } else { - // Entry does not point to object. Add range if - // it is pending to be added. - if (pending_size != 0U) { - ret = fn(pending_base, pending_size, - arg); - if (ret != OK) { - goto error; - } - pending_base = 0; - pending_size = 0; + // Entry does not point to object. + base = 0U; + size = 0U; + } + + if (pending_size != 0U) { + ret = fn(pending_base, pending_size, arg); + if (ret != OK) { + goto error; } - index++; } + + pending_base = base; + pending_size = size; + index++; } } while (count > 0U); @@ -2113,37 +2133,43 @@ memdb_range_walk(uintptr_t object, memdb_type_t type, paddr_t start_addr, { error_t ret = OK; paddr_t covered_bits = 0; + count_t guard_shifts; + paddr_t guard; + memdb_type_t next_type; + uintptr_t next; count_t shifts; memdb_level_t *common_level = NULL; rcu_read_start(); - memdb_entry_t root_entry = - atomic_load_explicit(&memdb.root, memory_order_relaxed); - if ((memdb_entry_info_get_type(&root_entry.info)) == - MEMDB_TYPE_NOTYPE) { + (void)atomic_entry_read(&memdb.root, &guard, &guard_shifts, &next_type, + &next); + + if (next_type == MEMDB_TYPE_NOTYPE) { ret = ERROR_MEMDB_EMPTY; goto error; } + assert(next_type == MEMDB_TYPE_LEVEL); assert((start_addr != end_addr) && (start_addr < end_addr)); assert((start_addr != 0U) || (~end_addr != 0U)); ret = find_common_level(start_addr, end_addr, &common_level, &shifts, - NULL, object, type, 0, 0, NULL, false, false); + NULL, object, type, 0, MEMDB_TYPE_LEVEL, NULL, + false, false); if (ret != OK) { - goto error; + // Start from the root level. + common_level = (memdb_level_t *)next; + shifts = guard_shifts; } - memdb_level_t *level = common_level; - if (shifts == ADDR_SIZE) { - covered_bits = 0; - } else { + if (shifts != ADDR_SIZE) { covered_bits = start_addr >> shifts; } - ret = memdb_do_walk(object, type, fn, arg, level, covered_bits, shifts, - start_addr, end_addr, false); + ret = memdb_do_walk(object, type, fn, arg, common_level, covered_bits, + shifts, start_addr, end_addr, false); + error: rcu_read_finish(); @@ -2162,12 +2188,12 @@ memdb_walk(uintptr_t object, memdb_type_t type, memdb_fnptr fn, void *arg) paddr_t guard; memdb_type_t next_type; uintptr_t next; - count_t shifts = ADDR_SIZE - MEMDB_BITS_PER_ENTRY; + count_t shifts = (count_t)(ADDR_SIZE - MEMDB_BITS_PER_ENTRY); rcu_read_start(); - atomic_entry_read(&memdb.root, &guard, &guard_shifts, &next_type, - &next); + (void)atomic_entry_read(&memdb.root, &guard, &guard_shifts, &next_type, + &next); if (next_type == MEMDB_TYPE_NOTYPE) { ret = ERROR_MEMDB_EMPTY; @@ -2192,14 +2218,12 @@ memdb_walk(uintptr_t object, memdb_type_t type, memdb_fnptr fn, void *arg) return ret; } -error_t +void memdb_init(void) { - atomic_entry_write(&memdb.root, memory_order_relaxed, 0, ADDR_SIZE, - MEMDB_TYPE_NOTYPE, 0); + atomic_entry_write(&memdb.root, memory_order_relaxed, 0, + (count_t)ADDR_SIZE, MEMDB_TYPE_NOTYPE, 0); spinlock_init(&memdb.lock); - - return OK; } void @@ -2255,6 +2279,7 @@ memdb_handle_partition_add_ram_range(partition_t *owner, paddr_t phys_base, assert(size > 0U); assert(!util_add_overflows(phys_base, size - 1U)); + // FIXME: // We should use memdb_insert() once this is safe to do so. error_t err = memdb_update(hyp_partition, phys_base, phys_base + (size - 1U), (uintptr_t)owner, @@ -2279,6 +2304,7 @@ memdb_handle_partition_remove_ram_range(partition_t *owner, paddr_t phys_base, assert(size > 0U); assert(!util_add_overflows(phys_base, size - 1U)); + // FIXME: // We should use memdb_insert() once this is safe to do so. error_t err = memdb_update(hyp_partition, phys_base, phys_base + (size - 1U), (uintptr_t)owner, diff --git a/hyp/mem/memdb/src/memdb_tests.c b/hyp/mem/memdb/src/memdb_tests.c index 188babf..7a1d0f1 100644 --- a/hyp/mem/memdb/src/memdb_tests.c +++ b/hyp/mem/memdb/src/memdb_tests.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -54,7 +55,7 @@ memdb_test_add_free_range(paddr_t base, size_t size, void *arg) first_entry = false; } - if (first_entry == false) { + if (!first_entry) { index++; } if (index >= MEMDB_RANGES_NUM) { @@ -98,7 +99,7 @@ get_inserted_ranges(memdb_data_t *memdb_data, uintptr_t object, LOG(DEBUG, INFO, "<<< BUG!! range {:#x}..{:#x} should be contiguouos", start_addr, end_addr); - assert(cont == true); + assert(cont); } } } @@ -160,9 +161,9 @@ memdb_test1(void) // Use addresses in: (0x300000000..0x5FFFFFFFFF) - partition_t *root_partition = partition_get_root(); - partition_t *hyp_partition = partition_get_private(); - allocator_t *root_allocator = &root_partition->allocator; + partition_t *root_partition = partition_get_root(); + partition_t *hyp_partition = partition_get_private(); + allocator_t *root_allocator = &root_partition->allocator; paddr_t start_addr = 0U; paddr_t end_addr = 0U; uintptr_t obj; @@ -174,7 +175,7 @@ memdb_test1(void) bool cont; void_ptr_result_t alloc_ret; - memdb_data_t *memdb_data; + memdb_data_t *memdb_data; size_t memdb_data_size = sizeof(*memdb_data); alloc_ret = partition_alloc(hyp_partition, memdb_data_size, @@ -199,10 +200,12 @@ memdb_test1(void) bool is_range_used = is_range_in_memdb(memdb_data, start_addr, end_addr); - assert(is_range_used == false); + assert(!is_range_used); + rcu_read_start(); res = memdb_lookup(start_addr); assert((res.e != OK) || (res.r.type == MEMDB_TYPE_NOTYPE)); + rcu_read_finish(); start_addr = 0x3000000000; end_addr = 0x300003FFFF; @@ -228,10 +231,12 @@ memdb_test1(void) // Lookup an address from root_partition that I know it is not // explicitly in an entry since it is in a skipped level due to a guard; paddr_t addr = 0x3000100000; - res = memdb_lookup(addr); + rcu_read_start(); + res = memdb_lookup(addr); assert(res.e == OK); assert(res.r.object == obj); assert(res.r.type == type); + rcu_read_finish(); // Update ownership of range in skipped levels @@ -323,7 +328,7 @@ memdb_test1(void) cont = memdb_is_ownership_contiguous(start_addr, end_addr, prev_obj, prev_type); - assert(cont == true); + assert(cont); // Check if rollback left everything correct check_ranges_in_memdb(memdb_data); @@ -358,7 +363,7 @@ memdb_test1(void) cont = memdb_is_ownership_contiguous(start_addr, end_addr, prev_obj, prev_type); - assert(cont == true); + assert(cont); start_addr = 0x3040000000; end_addr = 0x30FFFFFFFF; @@ -366,7 +371,7 @@ memdb_test1(void) type = MEMDB_TYPE_ALLOCATOR; cont = memdb_is_ownership_contiguous(start_addr, end_addr, obj, type); - assert(cont == true); + assert(cont); partition_free(hyp_partition, memdb_data, memdb_data_size); } @@ -393,7 +398,7 @@ memdb_test_insert_update(memdb_data_t *test_data, paddr_t start, paddr_t end) partition_t *hyp_partition = partition_get_private(); void_ptr_result_t alloc_ret; - memdb_data_t *memdb_data; + memdb_data_t *memdb_data; size_t memdb_data_size = sizeof(*memdb_data); alloc_ret = partition_alloc(hyp_partition, memdb_data_size, @@ -412,10 +417,12 @@ memdb_test_insert_update(memdb_data_t *test_data, paddr_t start, paddr_t end) check_ranges_in_memdb(memdb_data); bool is_range_used = is_range_in_memdb(memdb_data, start, end); - assert(is_range_used == false); + assert(!is_range_used); + rcu_read_start(); memdb_obj_type_result_t res = memdb_lookup(start); assert((res.e != OK) || (res.r.type == MEMDB_TYPE_NOTYPE)); + rcu_read_finish(); for (index_t i = 0; i < test_data->ranges_count; i++) { paddr_t start_addr = test_data->ranges[i].base; @@ -447,14 +454,16 @@ memdb_test_insert_update(memdb_data_t *test_data, paddr_t start, paddr_t end) } assert(err == OK); + rcu_read_start(); res = memdb_lookup(start_addr); assert(res.e == OK); assert(res.r.object == obj); assert(res.r.type == type); + rcu_read_finish(); bool cont = memdb_is_ownership_contiguous(start_addr, end_addr, obj, type); - assert(cont == true); + assert(cont); // Failure cases: err = memdb_insert(root_partition, start_addr, end_addr, obj, @@ -478,7 +487,7 @@ memdb_test_insert_update(memdb_data_t *test_data, paddr_t start, paddr_t end) // Verify that the failure cases did not modify the memdb cont = memdb_is_ownership_contiguous(start_addr, end_addr, obj, type); - assert(cont == true); + assert(cont); } // Check all ranges in memdb to see if everything has been done @@ -495,7 +504,7 @@ memdb_test2(void) partition_t *hyp_partition = partition_get_private(); void_ptr_result_t alloc_ret; - memdb_data_t *test_data; + memdb_data_t *test_data; size_t test_data_size = sizeof(*test_data); alloc_ret = partition_alloc(hyp_partition, test_data_size, @@ -538,7 +547,7 @@ memdb_test3(void) partition_t *hyp_partition = partition_get_private(); void_ptr_result_t alloc_ret; - memdb_data_t *test_data; + memdb_data_t *test_data; size_t test_data_size = sizeof(*test_data); alloc_ret = partition_alloc(hyp_partition, test_data_size, @@ -591,7 +600,7 @@ memdb_test4(void) partition_t *hyp_partition = partition_get_private(); void_ptr_result_t alloc_ret; - memdb_data_t *test_data; + memdb_data_t *test_data; size_t test_data_size = sizeof(*test_data); alloc_ret = partition_alloc(hyp_partition, test_data_size, @@ -652,7 +661,7 @@ memdb_test5(void) partition_t *hyp_partition = partition_get_private(); void_ptr_result_t alloc_ret; - memdb_data_t *test_data; + memdb_data_t *test_data; size_t test_data_size = sizeof(*test_data); alloc_ret = partition_alloc(hyp_partition, test_data_size, @@ -706,7 +715,7 @@ memdb_test_update(memdb_data_t *test_data, paddr_t start, paddr_t end, partition_t *hyp_partition = partition_get_private(); void_ptr_result_t alloc_ret; - memdb_data_t *memdb_data; + memdb_data_t *memdb_data; size_t memdb_data_size = sizeof(*memdb_data); alloc_ret = partition_alloc(hyp_partition, memdb_data_size, @@ -723,10 +732,12 @@ memdb_test_update(memdb_data_t *test_data, paddr_t start, paddr_t end, check_ranges_in_memdb(memdb_data); bool is_range_used = is_range_in_memdb(memdb_data, start, end); - assert(is_range_used == false); + assert(!is_range_used); + rcu_read_start(); memdb_obj_type_result_t res = memdb_lookup(start); assert((res.e != OK) || (res.r.type == MEMDB_TYPE_NOTYPE)); + rcu_read_finish(); LOG(DEBUG, INFO, "<<< Adding range: {:#x}-{:#x}", start, end); @@ -757,14 +768,16 @@ memdb_test_update(memdb_data_t *test_data, paddr_t start, paddr_t end, uintptr_t obj = test_data->ranges[i].obj; memdb_type_t type = test_data->ranges[i].type; + rcu_read_start(); res = memdb_lookup(start_addr); assert(res.e == OK); assert(res.r.object == obj); assert(res.r.type == type); + rcu_read_finish(); bool cont = memdb_is_ownership_contiguous(start_addr, end_addr, obj, type); - assert(cont == true); + assert(cont); } // Check all ranges in memdb to see if everything has been done @@ -784,14 +797,16 @@ memdb_test_update(memdb_data_t *test_data, paddr_t start, paddr_t end, assert(err == OK); } + rcu_read_start(); res = memdb_lookup(start); assert(res.e == OK); assert(res.r.object == initial_obj); assert(res.r.type == initial_type); + rcu_read_finish(); bool cont = memdb_is_ownership_contiguous(start, end, initial_obj, initial_type); - assert(cont == true); + assert(cont); // Check all ranges in memdb to see if everything has been done // correctly @@ -812,7 +827,7 @@ memdb_test0(void) partition_t *hyp_partition = partition_get_private(); void_ptr_result_t alloc_ret; - memdb_data_t *test_data; + memdb_data_t *test_data; size_t test_data_size = sizeof(*test_data); alloc_ret = partition_alloc(hyp_partition, test_data_size, @@ -856,13 +871,15 @@ memdb_test0(void) MEMDB_TYPE_PARTITION); assert(err == OK); + rcu_read_start(); memdb_obj_type_result_t res = memdb_lookup(start_addr); assert(res.e == OK); + rcu_read_finish(); bool cont = memdb_is_ownership_contiguous(start_addr, end_addr2, (uintptr_t)root_partition, MEMDB_TYPE_PARTITION); - assert(cont == true); + assert(cont); paddr_t start_addr3 = 0x380000000; paddr_t end_addr3 = 0x3D4FFFFFF; @@ -871,13 +888,15 @@ memdb_test0(void) (uintptr_t)root_partition, MEMDB_TYPE_PARTITION); assert(err == OK); + rcu_read_start(); res = memdb_lookup(start_addr3); assert(res.e == OK); + rcu_read_finish(); cont = memdb_is_ownership_contiguous(start_addr3, end_addr2, (uintptr_t)root_partition, MEMDB_TYPE_PARTITION); - assert(cont == true); + assert(cont); } static void @@ -885,8 +904,8 @@ memdb_test6(void) { LOG(DEBUG, INFO, " Start TEST 6:"); - partition_t *root_partition = partition_get_root(); - partition_t *hyp_partition = partition_get_private(); + partition_t *root_partition = partition_get_root(); + partition_t *hyp_partition = partition_get_private(); paddr_t start_addr = 0U; paddr_t end_addr = 0U; uintptr_t obj; @@ -897,7 +916,7 @@ memdb_test6(void) memdb_obj_type_result_t res; void_ptr_result_t alloc_ret; - memdb_data_t *memdb_data; + memdb_data_t *memdb_data; size_t memdb_data_size = sizeof(*memdb_data); alloc_ret = partition_alloc(hyp_partition, memdb_data_size, @@ -916,10 +935,12 @@ memdb_test6(void) bool is_range_used = is_range_in_memdb(memdb_data, start_addr, end_addr); - assert(is_range_used == false); + assert(!is_range_used); + rcu_read_start(); res = memdb_lookup(start_addr); assert((res.e != OK) || (res.r.type == MEMDB_TYPE_NOTYPE)); + rcu_read_finish(); start_addr = 0x2000000000000; end_addr = 0x201ffffffffff; @@ -995,8 +1016,8 @@ memdb_test7(void) { LOG(DEBUG, INFO, " Start TEST 7:"); - partition_t *root_partition = partition_get_root(); - partition_t *hyp_partition = partition_get_private(); + partition_t *root_partition = partition_get_root(); + partition_t *hyp_partition = partition_get_private(); paddr_t start_addr = 0U; paddr_t end_addr = 0U; uintptr_t obj; @@ -1007,7 +1028,7 @@ memdb_test7(void) memdb_obj_type_result_t res; void_ptr_result_t alloc_ret; - memdb_data_t *memdb_data; + memdb_data_t *memdb_data; size_t memdb_data_size = sizeof(*memdb_data); alloc_ret = partition_alloc(hyp_partition, memdb_data_size, @@ -1026,10 +1047,12 @@ memdb_test7(void) bool is_range_used = is_range_in_memdb(memdb_data, start_addr, end_addr); - assert(is_range_used == false); + assert(!is_range_used); + rcu_read_start(); res = memdb_lookup(start_addr); assert((res.e != OK) || (res.r.type == MEMDB_TYPE_NOTYPE)); + rcu_read_finish(); start_addr = 0x3000000000000; end_addr = 0x300001fffffff; @@ -1105,8 +1128,8 @@ memdb_test8(void) { LOG(DEBUG, INFO, " Start TEST 8:"); - partition_t *root_partition = partition_get_root(); - partition_t *hyp_partition = partition_get_private(); + partition_t *root_partition = partition_get_root(); + partition_t *hyp_partition = partition_get_private(); paddr_t start_addr = 0U; paddr_t end_addr = 0U; uintptr_t obj; @@ -1117,7 +1140,7 @@ memdb_test8(void) memdb_obj_type_result_t res; void_ptr_result_t alloc_ret; - memdb_data_t *memdb_data; + memdb_data_t *memdb_data; size_t memdb_data_size = sizeof(*memdb_data); alloc_ret = partition_alloc(hyp_partition, memdb_data_size, @@ -1136,10 +1159,12 @@ memdb_test8(void) bool is_range_used = is_range_in_memdb(memdb_data, start_addr, end_addr); - assert(is_range_used == false); + assert(!is_range_used); + rcu_read_start(); res = memdb_lookup(start_addr); assert((res.e != OK) || (res.r.type == MEMDB_TYPE_NOTYPE)); + rcu_read_finish(); start_addr = 0x4000000000000; end_addr = 0x400001fffffff; @@ -1270,8 +1295,8 @@ memdb_test9(void) { LOG(DEBUG, INFO, " Start TEST 9:"); - partition_t *root_partition = partition_get_root(); - partition_t *hyp_partition = partition_get_private(); + partition_t *root_partition = partition_get_root(); + partition_t *hyp_partition = partition_get_private(); paddr_t start_addr = 0U; paddr_t end_addr = 0U; uintptr_t obj; @@ -1280,7 +1305,7 @@ memdb_test9(void) memdb_obj_type_result_t res; void_ptr_result_t alloc_ret; - memdb_data_t *memdb_data; + memdb_data_t *memdb_data; size_t memdb_data_size = sizeof(*memdb_data); alloc_ret = partition_alloc(hyp_partition, memdb_data_size, @@ -1321,10 +1346,12 @@ memdb_test9(void) bool is_range_used = is_range_in_memdb(memdb_data, start_addr, end_addr); - assert(is_range_used == false); + assert(!is_range_used); + rcu_read_start(); res = memdb_lookup(start_addr); assert((res.e != OK) || (res.r.type == MEMDB_TYPE_NOTYPE)); + rcu_read_finish(); obj = (uintptr_t)root_partition; type = MEMDB_TYPE_PARTITION; @@ -1340,32 +1367,100 @@ memdb_test9(void) partition_free(root_partition, dummy_region, dummy_size); // Verify that the address does not exist in the memdb + rcu_read_start(); res = memdb_lookup(start_addr); assert((res.e != OK) || (res.r.type == MEMDB_TYPE_NOTYPE)); + rcu_read_finish(); err = memdb_insert(root_partition, start_addr, end_addr, obj, type); assert(err == OK); } +static error_t +verify_range(paddr_t base, size_t size, void *arg) +{ + memdb_data_t *memdb_data = (memdb_data_t *)arg; + index_t i = memdb_data->ranges_index; + + assert(i < memdb_data->ranges_count); + assert(base == memdb_data->ranges[i].base); + assert(size == memdb_data->ranges[i].size); + + memdb_data->ranges_index++; + + return OK; +} + +static void +memdb_test10(void) +{ + LOG(DEBUG, INFO, " Start TEST 10:"); + + error_t err; + partition_t *hyp_partition = partition_get_private(); + partition_t *fake_partition = (partition_t *)0x123124; + + void_ptr_result_t alloc_ret; + memdb_data_t *memdb_data; + size_t memdb_data_size = sizeof(*memdb_data); + + alloc_ret = partition_alloc(hyp_partition, memdb_data_size, + alignof(*memdb_data)); + if (alloc_ret.e != OK) { + panic("memdb_test: allocate memdb_data failed"); + } + + memdb_data = (memdb_data_t *)alloc_ret.r; + memset(memdb_data, 0, memdb_data_size); + + paddr_t base0 = 0x1082800000U; + size_t size0 = 0x55800000U; + + err = memdb_insert(hyp_partition, base0, base0 + size0 - 1U, + (uintptr_t)fake_partition, MEMDB_TYPE_PARTITION); + assert(err == OK); + + paddr_t base1 = 0x10D8200000U; + size_t size1 = 0xE0000U; + + err = memdb_insert(hyp_partition, base1, base1 + size1 - 1U, + (uintptr_t)fake_partition, MEMDB_TYPE_PARTITION); + assert(err == OK); + + memdb_data->ranges[0].base = base0; + memdb_data->ranges[0].size = size0; + memdb_data->ranges[1].base = base1; + memdb_data->ranges[1].size = size1; + memdb_data->ranges_count = 2U; + + err = memdb_walk((uintptr_t)fake_partition, MEMDB_TYPE_PARTITION, + verify_range, memdb_data); + assert(err == OK); + + assert(memdb_data->ranges_index == memdb_data->ranges_count); + + partition_free(hyp_partition, memdb_data, memdb_data_size); +} + bool memdb_handle_tests_start(void) { bool wait_all_cores_end = true; bool wait_all_cores_start = true; - spinlock_acquire(&test_memdb_spinlock); + spinlock_acquire_nopreempt(&test_memdb_spinlock); test_memdb_count++; - spinlock_release(&test_memdb_spinlock); + spinlock_release_nopreempt(&test_memdb_spinlock); // Wait until all cores have reached this point to start. while (wait_all_cores_start) { - spinlock_acquire(&test_memdb_spinlock); + spinlock_acquire_nopreempt(&test_memdb_spinlock); if (test_memdb_count == (PLATFORM_MAX_CORES)) { wait_all_cores_start = false; } - spinlock_release(&test_memdb_spinlock); + spinlock_release_nopreempt(&test_memdb_spinlock); } if (cpulocal_get_index() != 0U) { @@ -1402,9 +1497,12 @@ memdb_handle_tests_start(void) // Test handling of out of memory error memdb_test9(); - spinlock_acquire(&test_memdb_spinlock); + // Test walk over two ranges with empty space from guard. + memdb_test10(); + + spinlock_acquire_nopreempt(&test_memdb_spinlock); test_memdb_count++; - spinlock_release(&test_memdb_spinlock); + spinlock_release_nopreempt(&test_memdb_spinlock); LOG(DEBUG, INFO, "Memdb tests successfully finished "); @@ -1412,13 +1510,13 @@ memdb_handle_tests_start(void) // Make all threads wait for test to end while (wait_all_cores_end) { - spinlock_acquire(&test_memdb_spinlock); + spinlock_acquire_nopreempt(&test_memdb_spinlock); if (test_memdb_count == PLATFORM_MAX_CORES + 1) { wait_all_cores_end = false; } - spinlock_release(&test_memdb_spinlock); + spinlock_release_nopreempt(&test_memdb_spinlock); } return false; diff --git a/hyp/mem/memdb/tests/test.c b/hyp/mem/memdb/tests/test.c new file mode 100644 index 0000000..ed6b66d --- /dev/null +++ b/hyp/mem/memdb/tests/test.c @@ -0,0 +1,2961 @@ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define MEMDB_BITS_PER_ENTRY_MASK ((1UL << MEMDB_BITS_PER_ENTRY) - 1) +#define ADDR_SIZE (sizeof(paddr_t) * CHAR_BIT) +// levels + 1 for root +#define MAX_LEVELS (ADDR_SIZE / MEMDB_BITS_PER_ENTRY) + 1 + +partition_t partition; + +void +print_level(memdb_level_t *level) +{ + if (level == NULL) { + printf("Empty database.\n"); + } + + printf("Level lock: %p\n", &level->lock); + + for (int i = 0; i < MEMDB_NUM_ENTRIES; i++) { + memdb_entry_t tmp_entry = atomic_load_explicit( + &level->level[i], memory_order_relaxed); + memdb_type_t tmp_type = + memdb_entry_info_get_type(&tmp_entry.info); + count_t tmp_shifts = + memdb_entry_info_get_shifts(&tmp_entry.info); + uint64_t tmp_guard = + memdb_entry_info_get_guard(&tmp_entry.info); + + if (tmp_type != MEMDB_TYPE_NOTYPE) { + // printf("| %d ", tmp_type); + uint32_t tt = tmp_entry.info.bf[0] & 0x7; + printf("| %d ", tt); + if (tmp_shifts != ADDR_SIZE) { + printf("guard_shifts: %d ", tmp_shifts); + printf("guard: %#lx ", tmp_guard); + } + } else { + printf("| - "); + } + } + printf("|\n"); + + for (int i = 0; i < MEMDB_NUM_ENTRIES; i++) { + memdb_entry_t tmp_entry = atomic_load_explicit( + &level->level[i], memory_order_relaxed); + memdb_type_t tmp_type = + memdb_entry_info_get_type(&tmp_entry.info); + void *next = &tmp_entry.next; + + if (tmp_type != MEMDB_TYPE_NOTYPE) { + if (tmp_type == MEMDB_TYPE_LEVEL) { + printf("----- Level below index: %d -----\n", + i); + print_level(next); + printf("---------------------------------\n"); + } + } + } +} + +void +print_memdb(void) +{ +#if 0 + // To print the database we need to declare memdb_t memdb in memdb.h not + // in mem_ownership_db.c + memdb_entry_t tmp_root = + atomic_load_explicit(&memdb.root, memory_order_relaxed); + uintptr_t * next = &tmp_root.next; + memdb_type_t tmp_type = memdb_entry_info_get_type(&tmp_root.info); + count_t tmp_shifts = memdb_entry_info_get_shifts(&tmp_root.info); + paddr_t tmp_guard = memdb_entry_info_get_guard(&tmp_root.info); + + if (next == NULL) { + printf("Empty memory database\n"); + return; + } else { + printf("------- Memory Ownership Database -------\n"); + printf("root guard: %#lx\n", tmp_guard); + printf("root guard shifts: %d\n", tmp_shifts); + printf("root type: %d\n", tmp_type); + printf("root lock pointer: %p\n", &memdb.lock); + } + + memdb_level_t *level = (memdb_level_t *)next; + + printf("\n"); + + if (level == NULL) { + printf("Empty database"); + } else { + print_level(level); + } + + printf("\n-----------------------------------------\n\n"); +#endif +} + +void +print_memdb_empty(void) +{ + // print_memdb(); +} + +// Insert two ranges in database +int +test1() +{ + partition_t partition; + int ret = 0; + size_t alignment = sizeof(void *); + size_t pool_size = 4096 * 100; + + partition.allocator.heap = NULL; + + /* ---------------- Giving memory to heap --------------------- */ + uintptr_t block = (uintptr_t)malloc(pool_size); + + ret = allocator_heap_add_memory(&partition.allocator, (void *)block, + pool_size); + if (!ret) { + printf("Memory added to heap\n"); + } + + /* ---------------- Init memory database --------------------- */ + ret = memdb_init(); + + if (!ret) { + printf("Mem db init correct!\n"); + } else { + printf("Error init!\n"); + } + + size_t obj_size = 1024; + memdb_type_t type = MEMDB_TYPE_PARTITION; + + size_t obj_size2 = 4096; + memdb_type_t type2 = MEMDB_TYPE_ALLOCATOR; + + /* ---------- Allocate object. partition for example ---------------- */ + void_ptr_result_t res; + res = allocator_allocate_object(&partition.allocator, obj_size, + alignment); + uintptr_t object = (uintptr_t)res.r; + + if (res.e != OK) { + printf("Object allocation failed\n"); + } else { + printf("Object allocation SUCCESS\n"); + } + + uint64_t start_addr = (uint64_t)object; + uint64_t end_addr = (uint64_t)object + (obj_size - 1); + + /* ---------- Allocate object. hypervisor for example ---------------- + */ + res = allocator_allocate_object(&partition.allocator, obj_size2, + alignment); + uintptr_t object2 = (uintptr_t)res.r; + if (res.e != OK) { + printf("\nObject allocation failed\n"); + } else { + printf("\nObject allocation SUCCESS\n"); + } + + uint64_t start_addr2 = (uint64_t)object2; + uint64_t end_addr2 = (uint64_t)object2 + (obj_size2 - 1); + + /* ---------------- Insert object in database --------------------- */ + printf("\nstart_addr: %#lx, end_addr: %#lx\n", start_addr, end_addr); + + ret = memdb_insert(&partition, start_addr, end_addr, (uintptr_t)object, + type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + printf("\nstart_addr2: %#lx, end_addr2: %#lx\n\n", start_addr2, + end_addr2); + + ret = memdb_insert(&partition, start_addr2, end_addr2, + (uintptr_t)object2, type2); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } +error: + return ret; +} + +// Insert one range in database and do two updates. +int +test2() +{ + partition_t partition; + int ret = 0; + size_t alignment = sizeof(void *); + + memdb_type_t type = MEMDB_TYPE_PARTITION; + memdb_type_t type1 = MEMDB_TYPE_ALLOCATOR; + memdb_type_t type2 = MEMDB_TYPE_EXTENT; + + size_t pool_size = 4096 * 100; + size_t obj_size1 = 4096; + size_t obj_size2 = 1024; + + partition.allocator.heap = NULL; + + /* ---------------- Giving memory to heap --------------------- */ + uintptr_t block = (uintptr_t)malloc(pool_size); + + ret = allocator_heap_add_memory(&partition.allocator, (void *)block, + pool_size); + if (!ret) { + printf("Memory added to heap\n"); + } + + /* ---------- Allocate object. partition for example ---------------- */ + void_ptr_result_t res; + res = allocator_allocate_object(&partition.allocator, obj_size1, + alignment); + uintptr_t object1 = (uintptr_t)res.r; + if (res.e != OK) { + printf("Object allocation failed\n"); + } else { + printf("Object allocation SUCCESS\n"); + } + + uint64_t start_addr1 = (uint64_t)object1; + uint64_t end_addr1 = (uint64_t)object1 + (obj_size1 - 1); + + /* ---------- Allocate object. hypervisor for example ---------------- + */ + res = allocator_allocate_object(&partition.allocator, obj_size2, + alignment); + uintptr_t object2 = (uintptr_t)res.r; + if (res.e != OK) { + printf("\nObject allocation failed\n"); + } else { + printf("\nObject allocation SUCCESS\n"); + } + + uint64_t start_addr2 = (uint64_t)object2; + uint64_t end_addr2 = (uint64_t)object2 + (obj_size2 - 1); + + /* ---------------- Init memory database --------------------- */ + ret = memdb_init(); + + if (!ret) { + printf("Mem db init correct!\n"); + } else { + printf("Error init!\n"); + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addr = (uint64_t)block; + uint64_t end_addr = (uint64_t)block + (pool_size - 1); + + printf("\nstart_addr: %#lx, end_addr: %#lx\n", start_addr, end_addr); + + ret = memdb_insert(&partition, start_addr, end_addr, (uintptr_t)block, + type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ---------------- Update database --------------------- */ + + printf("\nstart_addr1: %#lx, end_addr1: %#lx\n", start_addr1, + end_addr1); + + ret = memdb_update(&partition, start_addr1, end_addr1, object1, type1, + block, type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ---------------- Update database --------------------- */ + + printf("\nstart_addr2: %#lx, end_addr2: %#lx\n", start_addr2, + end_addr2); + + ret = memdb_update(&partition, start_addr2, end_addr2, object2, type2, + block, type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } +error: + return ret; +} + +// One insertion, two updates and update back to the initial state +int +test3() +{ + partition_t partition; + int ret = 0; + size_t alignment = sizeof(void *); + + memdb_type_t type = MEMDB_TYPE_PARTITION; + memdb_type_t type1 = MEMDB_TYPE_ALLOCATOR; + memdb_type_t type2 = MEMDB_TYPE_EXTENT; + + size_t pool_size = 4096 * 100; + size_t obj_size1 = 4096; + size_t obj_size2 = 1024; + + partition.allocator.heap = NULL; + + /* ---------------- Giving memory to heap --------------------- */ + uintptr_t block = (uintptr_t)malloc(pool_size); + + ret = allocator_heap_add_memory(&partition.allocator, (void *)block, + pool_size); + if (!ret) { + printf("Memory added to heap\n"); + } + + /* ---------- Allocate object. partition for example ---------------- */ + void_ptr_result_t res; + res = allocator_allocate_object(&partition.allocator, obj_size1, + alignment); + uintptr_t object1 = (uintptr_t)res.r; + + if (res.e != OK) { + printf("Object allocation failed\n"); + goto error; + } else { + printf("Object allocation SUCCESS\n"); + } + + uint64_t start_addr1 = (uint64_t)object1; + uint64_t end_addr1 = (uint64_t)object1 + (obj_size1 - 1); + + /* ---------- Allocate object. hypervisor for example ---------------- + */ + res = allocator_allocate_object(&partition.allocator, obj_size2, + alignment); + uintptr_t object2 = (uintptr_t)res.r; + if (res.e != OK) { + printf("\nObject allocation failed\n"); + goto error; + } else { + printf("\nObject allocation SUCCESS\n"); + } + + uint64_t start_addr2 = (uint64_t)object2; + uint64_t end_addr2 = (uint64_t)object2 + (obj_size2 - 1); + + /* ---------------- Init memory database --------------------- */ + ret = memdb_init(); + + if (!ret) { + printf("Mem db init correct!\n"); + } else { + printf("Error init!\n"); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addr = (uint64_t)block; + uint64_t end_addr = (uint64_t)block + (pool_size - 1); + + printf("\nstart_addr: %#lx, end_addr: %#lx\n", start_addr, end_addr); + + ret = memdb_insert(&partition, start_addr, end_addr, (uintptr_t)block, + type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ---------------- Update database --------------------- */ + + printf("\nstart_addr1: %#lx, end_addr1: %#lx\n", start_addr1, + end_addr1); + + ret = memdb_update(&partition, start_addr1, end_addr1, object1, type1, + block, type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ---------------- Update database --------------------- */ + + printf("\nstart_addr2: %#lx, end_addr2: %#lx\n", start_addr2, + end_addr2); + + ret = memdb_update(&partition, start_addr2, end_addr2, object2, type2, + block, type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ---------------- Update back database --------------------- */ + + printf("\nstart_addr1: %#lx, end_addr1: %#lx\n", start_addr1, + end_addr1); + + ret = memdb_update(&partition, start_addr1, end_addr1, block, type, + object1, type1); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update back SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update back FAILED\n\n"); + goto error; + } + + /* ---------------- Update back database --------------------- */ + + printf("\nstart_addr2: %#lx, end_addr2: %#lx\n", start_addr2, + end_addr2); + + ret = memdb_update(&partition, start_addr2, end_addr2, block, type, + object2, type2); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update back SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update back FAILED\n\n"); + goto error; + } +error: + return ret; +} + +// Two insertion, 2 updates, 2 updates back to state after insertions +int +test4() +{ + partition_t partition; + int ret = 0; + size_t alignment = 4096; + + memdb_type_t type = MEMDB_TYPE_PARTITION; + memdb_type_t type1 = MEMDB_TYPE_ALLOCATOR; + memdb_type_t type2 = MEMDB_TYPE_EXTENT; + + size_t pool_size = 4096 * 100; + size_t pool_size2 = 1024; + size_t obj_size1 = 4096; + size_t obj_size2 = 1024; + + partition.allocator.heap = NULL; + + /* ---------------- Giving memory to heap --------------------- */ + uintptr_t block = (uintptr_t)aligned_alloc(alignment, pool_size); + + ret = allocator_heap_add_memory(&partition.allocator, (void *)block, + pool_size); + if (!ret) { + printf("Memory added to heap\n"); + } + + /* ----------------- Hypervisor memory ------------------------ */ + uintptr_t block2 = (uintptr_t)aligned_alloc(alignment, pool_size2); + + /* ---------- Allocate object. partition for example ---------------- */ + void_ptr_result_t res; + res = allocator_allocate_object(&partition.allocator, obj_size1, + alignment); + uintptr_t object1 = (uintptr_t)res.r; + if (res.e != OK) { + printf("Object allocation failed\n"); + goto error; + } else { + printf("Object allocation SUCCESS\n"); + } + + uint64_t start_addr1 = (uint64_t)object1; + uint64_t end_addr1 = (uint64_t)object1 + (obj_size1 - 1); + + /* ---------- Allocate object. hypervisor for example ---------------- + */ + res = allocator_allocate_object(&partition.allocator, obj_size2, + alignment); + + uintptr_t object2 = (uintptr_t)res.r; + if (res.e != OK) { + printf("\nObject allocation failed\n"); + goto error; + } else { + printf("\nObject allocation SUCCESS\n"); + } + + uint64_t start_addr2 = (uint64_t)object2; + uint64_t end_addr2 = (uint64_t)object2 + (obj_size2 - 1); + + /* ---------------- Init memory database --------------------- */ + ret = memdb_init(); + + if (!ret) { + printf("Mem db init correct!\n"); + } else { + printf("Error init!\n"); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addr = (uint64_t)block; + uint64_t end_addr = (uint64_t)block + (pool_size - 1); + + printf("\nstart_addr: %#lx, end_addr: %#lx\n", start_addr, end_addr); + printf("\nnew type: %d\n", type); + + ret = memdb_insert(&partition, start_addr, end_addr, (uintptr_t)block, + type); + if (!ret) { + printf("\nmemdb_insert SUCCESS\n\n"); + print_memdb_empty(); + } else { + printf("\nmemdb_insert FAILED\n\n"); + print_memdb_empty(); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addrh = (uint64_t)block2; + uint64_t end_addrh = (uint64_t)block2 + (pool_size2 - 1); + + printf("\nstart_addr: %#lx, end_addr: %#lx\n", start_addrh, end_addrh); + printf("\nnew type: %d\n", MEMDB_TYPE_ALLOCATOR); + + ret = memdb_insert(&partition, start_addrh, end_addrh, + (uintptr_t)block2, MEMDB_TYPE_ALLOCATOR); + + if (!ret) { + printf("\nmemdb_insert SUCCESS\n\n"); + print_memdb_empty(); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ---------------- Update database --------------------- */ + + printf("\nstart_addr1: %#lx, end_addr1: %#lx\n", start_addr1, + end_addr1); + printf("\nnew type: %d old type: %d\n", type1, type); + + ret = memdb_update(&partition, start_addr1, end_addr1, object1, type1, + block, type); + + if (!ret) { + printf("\nmemdb_update SUCCESS\n\n"); + print_memdb_empty(); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ---------------- Update database --------------------- */ + printf("\nstart_addr2: %#lx, end_addr2: %#lx\n", start_addr2, + end_addr2); + printf("\nnew type: %d old type: %d\n", type2, type); + + ret = memdb_update(&partition, start_addr2, end_addr2, object2, type2, + block, type); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ---------------- Update back database --------------------- */ + + printf("\nstart_addr1: %#lx, end_addr1: %#lx\n", start_addr1, + end_addr1); + printf("\nnew type: %d old type: %d\n", type, type1); + + ret = memdb_update(&partition, start_addr1, end_addr1, block, type, + object1, type1); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update back SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update back FAILED\n\n"); + goto error; + } + + /* ---------------- Update back database --------------------- */ + + printf("\nstart_addr2: %#lx, end_addr2: %#lx\n", start_addr2, + end_addr2); + printf("\nnew type: %d old type: %d\n", type, type2); + + ret = memdb_update(&partition, start_addr2, end_addr2, block, type, + object2, type2); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update back SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update back FAILED\n\n"); + goto error; + } +error: + return ret; +} + +// 1 insertion, 2 updates, 2 checks of contiguouness (1 should succeed and the +// other one fail) +int +test5() +{ + partition_t partition; + int ret = 0; + size_t alignment = sizeof(void *); + + memdb_type_t type = MEMDB_TYPE_PARTITION; + memdb_type_t type1 = MEMDB_TYPE_ALLOCATOR; + memdb_type_t type2 = MEMDB_TYPE_EXTENT; + + size_t pool_size = 4096 * 100; + size_t obj_size1 = 4096; + size_t obj_size2 = 1024; + + partition.allocator.heap = NULL; + + /* ---------------- Giving memory to heap --------------------- */ + uintptr_t block = (uintptr_t)malloc(pool_size); + + ret = allocator_heap_add_memory(&partition.allocator, (void *)block, + pool_size); + if (!ret) { + printf("Memory added to heap\n"); + } + + /* ---------- Allocate object. partition for example ---------------- */ + void_ptr_result_t res; + res = allocator_allocate_object(&partition.allocator, obj_size1, + alignment); + + uintptr_t object1 = (uintptr_t)res.r; + if (res.e != OK) { + printf("Object allocation failed\n"); + } else { + printf("Object allocation SUCCESS\n"); + } + + uint64_t start_addr1 = (uint64_t)object1; + uint64_t end_addr1 = (uint64_t)object1 + (obj_size1 - 1); + + /* ---------- Allocate object. hypervisor for example ---------------- + */ + res = allocator_allocate_object(&partition.allocator, obj_size2, + alignment); + + uintptr_t object2 = (uintptr_t)res.r; + if (res.e != OK) { + printf("\nObject allocation failed\n"); + } else { + printf("\nObject allocation SUCCESS\n"); + } + + uint64_t start_addr2 = (uint64_t)object2; + uint64_t end_addr2 = (uint64_t)object2 + (obj_size2 - 1); + + /* ---------------- Init memory database --------------------- */ + ret = memdb_init(); + + if (!ret) { + printf("Mem db init correct!\n"); + } else { + printf("Error init!\n"); + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addr = (uint64_t)block; + uint64_t end_addr = (uint64_t)block + (pool_size - 1); + + printf("\nstart_addr: %#lx, end_addr: %#lx\n", start_addr, end_addr); + + ret = memdb_insert(&partition, start_addr, end_addr, (uintptr_t)block, + type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ---------------- Update database --------------------- */ + + printf("\nstart_addr1: %#lx, end_addr1: %#lx\n", start_addr1, + end_addr1); + + ret = memdb_update(&partition, start_addr1, end_addr1, object1, type1, + block, type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ---------------- Update database --------------------- */ + + printf("\nstart_addr2: %#lx, end_addr2: %#lx\n", start_addr2, + end_addr2); + + ret = memdb_update(&partition, start_addr2, end_addr2, object2, type2, + block, type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ------------------- Is contiguous ? -------------------- */ + // Should succeed: + printf("\nIs start_addr1: %#lx, end_addr1: %#lx contiguous??\n", + start_addr1, end_addr1); + + bool ans = memdb_is_ownership_contiguous(start_addr1, end_addr1, + object1, type1); + if (ans) { + printf("\nmemdb_is_ownership_contiguous SUCCESS\n\n"); + } else { + printf("\nmemdb_is_ownership_contiguous FAILED\n\n"); + goto error; + } + + /* ------------------- Is contiguous ? -------------------- */ + // Should fail: + printf("\nIs start_addr1: %#lx, end_addr1: %#lx contiguous??\n", + start_addr1 - 1, end_addr1); + + ans = memdb_is_ownership_contiguous(start_addr1 - 1, end_addr1, object1, + type1); + if (ans) { + printf("\nmemdb_is_ownership_contiguous SUCCESS\n\n"); + ret = -1; + goto error; + } else { + printf("\nmemdb_is_ownership_contiguous FAILED as expected.\n\n"); + ret = 0; + } +error: + return ret; +} + +// 1 insertion, 2 updates and 2 lookups +int +test6() +{ + partition_t partition; + int ret = 0; + size_t alignment = 16; + + memdb_type_t type = MEMDB_TYPE_PARTITION; + memdb_type_t type1 = MEMDB_TYPE_ALLOCATOR; + memdb_type_t type2 = MEMDB_TYPE_EXTENT; + + size_t pool_size = 4096 * 100; + size_t obj_size1 = 4096; + size_t obj_size2 = 1024; + + partition.allocator.heap = NULL; + + /* ---------------- Giving memory to heap --------------------- */ + uintptr_t block = (uintptr_t)malloc(pool_size); + + ret = allocator_heap_add_memory(&partition.allocator, (void *)block, + pool_size); + if (!ret) { + printf("Memory added to heap\n"); + } + + /* ---------- Allocate object. partition for example ---------------- */ + void_ptr_result_t res; + res = allocator_allocate_object(&partition.allocator, obj_size1, + alignment); + + uintptr_t object1 = (uintptr_t)res.r; + if (res.e != OK) { + printf("Object allocation failed\n"); + } else { + printf("Object allocation SUCCESS\n"); + } + + uint64_t start_addr1 = (uint64_t)object1; + uint64_t end_addr1 = (uint64_t)object1 + (obj_size1 - 1); + + /* ---------- Allocate object. hypervisor for example ---------------- + */ + res = allocator_allocate_object(&partition.allocator, obj_size2, + alignment); + + uintptr_t object2 = (uintptr_t)res.r; + if (res.e != OK) { + printf("\nObject allocation failed\n"); + } else { + printf("\nObject allocation SUCCESS\n"); + } + + uint64_t start_addr2 = (uint64_t)object2; + uint64_t end_addr2 = (uint64_t)object2 + (obj_size2 - 1); + + /* ---------------- Init memory database --------------------- */ + ret = memdb_init(); + + if (!ret) { + printf("Mem db init correct!\n"); + } else { + printf("Error init!\n"); + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addr = (uint64_t)block; + uint64_t end_addr = (uint64_t)block + (pool_size - 1); + + printf("\nstart_addr: %#lx, end_addr: %#lx\n", start_addr, end_addr); + + ret = memdb_insert(&partition, start_addr, end_addr, (uintptr_t)block, + type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ---------------- Update database --------------------- */ + + printf("\nstart_addr1: %#lx, end_addr1: %#lx\n", start_addr1, + end_addr1); + + ret = memdb_update(&partition, start_addr1, end_addr1, object1, type1, + block, type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ---------------- Update database --------------------- */ + + printf("\nstart_addr2: %#lx, end_addr2: %#lx\n", start_addr2, + end_addr2); + + ret = memdb_update(&partition, start_addr2, end_addr2, object2, type2, + block, type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ----------------- lookup --------------------- */ + printf("\nLooking for start_addr1: %#lx\n", start_addr1); + memdb_obj_type_result_t tot_res; + + tot_res = memdb_lookup(start_addr1); + ret = tot_res.e; + if (tot_res.e == OK) { + printf("\nmemdb_lookup SUCCESS. type: %d\n\n", tot_res.r.type); + } else { + printf("\nmemdb_lookup FAILED\n\n"); + goto error; + } +error: + return ret; +} + +// 2 insertions, 2 updates, 2 lookups +int +test7() +{ + partition_t partition; + int ret = 0; + size_t alignment = sizeof(void *); + + memdb_type_t type = MEMDB_TYPE_PARTITION; + memdb_type_t type1 = MEMDB_TYPE_ALLOCATOR; + memdb_type_t type2 = MEMDB_TYPE_EXTENT; + + size_t pool_size = 4096 * 100; + size_t pool_size2 = 1024; + size_t obj_size1 = 4096; + size_t obj_size2 = 1024; + + partition.allocator.heap = NULL; + + /* ---------------- Giving memory to heap --------------------- */ + uintptr_t block = (uintptr_t)malloc(pool_size); + + ret = allocator_heap_add_memory(&partition.allocator, (void *)block, + pool_size); + if (!ret) { + printf("Memory added to heap\n"); + } + + /* ----------------- Hypervisor memory ------------------------ */ + uintptr_t block2 = (uintptr_t)malloc(pool_size2); + + /* ---------- Allocate object. partition for example ---------------- */ + void_ptr_result_t res; + res = allocator_allocate_object(&partition.allocator, obj_size1, + alignment); + + uintptr_t object1 = (uintptr_t)res.r; + if (res.e != OK) { + printf("Object allocation failed\n"); + goto error; + } else { + printf("Object allocation SUCCESS\n"); + } + + uint64_t start_addr1 = (uint64_t)object1; + uint64_t end_addr1 = (uint64_t)object1 + (obj_size1 - 1); + + /* ---------- Allocate object. hypervisor for example ---------------- + */ + res = allocator_allocate_object(&partition.allocator, obj_size2, + alignment); + + uintptr_t object2 = (uintptr_t)res.r; + if (res.e != OK) { + printf("\nObject allocation failed\n"); + goto error; + } else { + printf("\nObject allocation SUCCESS\n"); + } + + uint64_t start_addr2 = (uint64_t)object2; + uint64_t end_addr2 = (uint64_t)object2 + (obj_size2 - 1); + + /* ---------------- Init memory database --------------------- */ + ret = memdb_init(); + + if (!ret) { + printf("Mem db init correct!\n"); + } else { + printf("Error init!\n"); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addr = (uint64_t)block; + uint64_t end_addr = (uint64_t)block + (pool_size - 1); + + printf("\nstart_addr: %#lx, end_addr: %#lx\n", start_addr, end_addr); + printf("\nnew type: %d\n", type); + + ret = memdb_insert(&partition, start_addr, end_addr, (uintptr_t)block, + type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + printf("\nmemdb_insert FAILED\n\n"); + print_memdb_empty(); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addrh = (uint64_t)block2; + uint64_t end_addrh = (uint64_t)block2 + (pool_size2 - 1); + + printf("\nstart_addr: %#lx, end_addr: %#lx\n", start_addrh, end_addrh); + printf("\nnew type: %d\n", MEMDB_TYPE_ALLOCATOR); + + ret = memdb_insert(&partition, start_addrh, end_addrh, + (uintptr_t)block2, MEMDB_TYPE_ALLOCATOR); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ---------------- Update database --------------------- */ + + printf("\nstart_addr1: %#lx, end_addr1: %#lx\n", start_addr1, + end_addr1); + printf("\nnew type: %d old type: %d\n", type1, type); + + ret = memdb_update(&partition, start_addr1, end_addr1, object1, type1, + block, type); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ---------------- Update database --------------------- */ + + printf("\nstart_addr2: %#lx, end_addr2: %#lx\n", start_addr2, + end_addr2); + printf("\nnew type: %d old type: %d\n", type2, type); + + ret = memdb_update(&partition, start_addr2, end_addr2, object2, type2, + block, type); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ----------------- lookup --------------------- */ + printf("\nLooking for start_addr1: %#lx\n", start_addr1); + memdb_obj_type_result_t tot_res; + + tot_res = memdb_lookup(start_addr1); + ret = tot_res.e; + if (tot_res.e == OK) { + printf("\nmemdb_lookup SUCCESS. type: %d\n\n", tot_res.r.type); + } else { + printf("\nmemdb_lookup FAILED\n\n"); + goto error; + } + + /* ----------------- lookup --------------------- */ + printf("\nLooking for start_addrh: %#lx\n", start_addr1); + tot_res = memdb_lookup(start_addrh); + ret = tot_res.e; + if (tot_res.e == OK) { + printf("\nmemdb_lookup SUCCESS. type: %d\n\n", tot_res.r.type); + } else { + printf("\nmemdb_lookup FAILED\n\n"); + goto error; + } +error: + return ret; +} + +// 2 insertions, 2 updates, 2 updates back +// (Address ranges with 64 guard) +int +test8() +{ + partition_t partition; + int ret = 0; + size_t alignment = 4096; + + memdb_type_t type = MEMDB_TYPE_PARTITION; + memdb_type_t type1 = MEMDB_TYPE_ALLOCATOR; + memdb_type_t type2 = MEMDB_TYPE_EXTENT; + + size_t pool_size = 4096 * 100; + size_t pool_size2 = 1024; + size_t obj_size1 = 4096; + size_t obj_size2 = 1024; + + partition.allocator.heap = NULL; + + /* ---------------- Giving memory to heap --------------------- */ + uintptr_t block = (uintptr_t)aligned_alloc(alignment, pool_size); + + ret = allocator_heap_add_memory(&partition.allocator, (void *)block, + pool_size); + if (!ret) { + printf("Memory added to heap\n"); + } + + /* ----------------- Hypervisor memory ------------------------ */ + uintptr_t block2 = (uintptr_t)aligned_alloc(alignment, pool_size2); + + /* ---------- Allocate object. partition for example ---------------- */ + void_ptr_result_t res; + res = allocator_allocate_object(&partition.allocator, obj_size1, + alignment); + + uintptr_t object1 = (uintptr_t)res.r; + if (res.e != OK) { + printf("Object allocation failed\n"); + goto error; + } else { + printf("Object allocation SUCCESS\n"); + } + + uint64_t start_addr1 = (uint64_t)object1; + uint64_t end_addr1 = (uint64_t)object1 + (obj_size1 - 1); + + /* ---------- Allocate object. hypervisor for example ---------------- + */ + res = allocator_allocate_object(&partition.allocator, obj_size2, + alignment); + + uintptr_t object2 = (uintptr_t)res.r; + if (res.e != OK) { + printf("\nObject allocation failed\n"); + goto error; + } else { + printf("\nObject allocation SUCCESS\n"); + } + + uint64_t start_addr2 = (uint64_t)object2; + uint64_t end_addr2 = (uint64_t)object2 + (obj_size2 - 1); + + /* ---------------- Init memory database --------------------- */ + ret = memdb_init(); + + if (!ret) { + printf("Mem db init correct!\n"); + } else { + printf("Error init!\n"); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addr = 0; + uint64_t end_addr = 0; + end_addr = (~(end_addr & 0) - 4096); + + printf("\nstart_addr: %#lx, end_addr: %#lx\n", start_addr, end_addr); + printf("\nnew type: %d\n", type); + + ret = memdb_insert(&partition, start_addr, end_addr, (uintptr_t)block, + type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addrh = 0; + uint64_t end_addrh = 0; + + start_addrh = (~(start_addrh & 0) - 4096) + 1; + end_addrh = (~(end_addrh & 0)); + + printf("\nstart_addr: %#lx, end_addr: %#lx\n", start_addrh, end_addrh); + printf("\nnew type: %d\n", MEMDB_TYPE_ALLOCATOR); + + ret = memdb_insert(&partition, start_addrh, end_addrh, + (uintptr_t)block2, MEMDB_TYPE_ALLOCATOR); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ---------------- Update database --------------------- */ + start_addr1 = start_addr + 6; + end_addr1 = end_addr - (4096000000000000000); + + printf("\nstart_addr1: %#lx, end_addr1: %#lx\n", start_addr1, + end_addr1); + printf("\nnew type: %d old type: %d\n", type1, type); + + ret = memdb_update(&partition, start_addr1, end_addr1, object1, type1, + block, type); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED. ret = %d\n\n", ret); + goto error; + } + + /* ---------------- Update database --------------------- */ + start_addr2 = end_addr1 + 1; + end_addr2 = end_addr; + + printf("\nstart_addr2: %#lx, end_addr2: %#lx\n", start_addr2, + end_addr2); + printf("\nnew type: %d old type: %d\n", type2, type); + + ret = memdb_update(&partition, start_addr2, end_addr2, object2, type2, + block, type); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ---------------- Update back database --------------------- */ + + printf("\nstart_addr1: %#lx, end_addr1: %#lx\n", start_addr1, + end_addr1); + printf("\nnew type: %d old type: %d\n", type, type1); + + ret = memdb_update(&partition, start_addr1, end_addr1, block, type, + object1, type1); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update back SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update back FAILED\n\n"); + goto error; + } + + /* ---------------- Update back database --------------------- */ + + printf("\nstart_addr2: %#lx, end_addr2: %#lx\n", start_addr2, + end_addr2); + printf("\nnew type: %d old type: %d\n", type, type2); + + ret = memdb_update(&partition, start_addr2, end_addr2, block, type, + object2, type2); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update back SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update back FAILED\n\n"); + goto error; + } +error: + return ret; +} + +// Rollback. (2 insertion, 2 updates (last one rolled back)) +int +test9() +{ + partition_t partition; + int ret = 0; + size_t alignment = 4096; + + memdb_type_t type = MEMDB_TYPE_PARTITION; + memdb_type_t type1 = MEMDB_TYPE_ALLOCATOR; + memdb_type_t type2 = MEMDB_TYPE_EXTENT; + + size_t pool_size = 4096 * 100; + size_t pool_size2 = 1024; + size_t obj_size1 = 4096; + size_t obj_size2 = 1024; + + partition.allocator.heap = NULL; + + /* ---------------- Giving memory to heap --------------------- */ + uintptr_t block = (uintptr_t)aligned_alloc(alignment, pool_size); + + ret = allocator_heap_add_memory(&partition.allocator, (void *)block, + pool_size); + if (!ret) { + printf("Memory added to heap\n"); + } + + /* ----------------- Hypervisor memory ------------------------ */ + uintptr_t block2 = (uintptr_t)aligned_alloc(alignment, pool_size2); + + /* ---------- Allocate object. partition for example ---------------- */ + void_ptr_result_t res; + res = allocator_allocate_object(&partition.allocator, obj_size1, + alignment); + + uintptr_t object1 = (uintptr_t)res.r; + if (res.e != OK) { + printf("Object allocation failed\n"); + goto error; + } else { + printf("Object allocation SUCCESS\n"); + } + + uint64_t start_addr1 = (uint64_t)object1; + uint64_t end_addr1 = (uint64_t)object1 + (obj_size1 - 1); + + /* ---------- Allocate object. hypervisor for example ---------------- + */ + res = allocator_allocate_object(&partition.allocator, obj_size2, + alignment); + + uintptr_t object2 = (uintptr_t)res.r; + if (res.e != OK) { + printf("\nObject allocation failed\n"); + goto error; + } else { + printf("\nObject allocation SUCCESS\n"); + } + + uint64_t start_addr2 = (uint64_t)object2; + uint64_t end_addr2 = (uint64_t)object2 + (obj_size2 - 1); + + /* ---------------- Init memory database --------------------- */ + ret = memdb_init(); + + if (!ret) { + printf("Mem db init correct!\n"); + } else { + printf("Error init!\n"); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addr = 139944292126720; + uint64_t end_addr = 139944292536319; + + printf("\nstart_addr: %#lx, end_addr: %#lx\n", start_addr, end_addr); + printf("\nnew type: %d\n", type); + + ret = memdb_insert(&partition, start_addr, end_addr, (uintptr_t)block, + type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addrh = (uint64_t)block2; + uint64_t end_addrh = (uint64_t)block2 + (pool_size2 - 1); + + printf("\nstart_addr: %#lx, end_addr: %#lx\n", start_addrh, end_addrh); + printf("\nnew type: %d\n", MEMDB_TYPE_ALLOCATOR); + + ret = memdb_insert(&partition, start_addrh, end_addrh, + (uintptr_t)block2, MEMDB_TYPE_ALLOCATOR); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED: %d\n\n", ret); + goto error; + } + + /* ---------------- Update database --------------------- */ + start_addr1 = start_addr + (4096 * 4); + end_addr1 = end_addr; + + printf("\nstart_addr1: %#lx, end_addr1: %#lx\n", start_addr1, + end_addr1); + printf("\nnew type: %d old type: %d\n", type1, type); + + ret = memdb_update(&partition, start_addr1, end_addr1, object1, type1, + block, type); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ---------------- Update database --------------------- */ + start_addr2 = start_addr; + end_addr2 = start_addr1; + + printf("\nstart_addr2: %#lx, end_addr2: %#lx\n", start_addr2, + end_addr2); + printf("\nnew type: %d old type: %d\n", type2, type); + + ret = memdb_update(&partition, start_addr2, end_addr2, object2, type2, + block, type); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS, should not have succeeded!!!\n\n"); + ret = -1; + goto error; + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED as expected (rollback).\n\n"); + ret = 0; + } + +error: + return ret; +} + +// Insert a second range that has to check a guard in the end +// path. Guard matches +int +test10() +{ + partition_t partition; + int ret = 0; + size_t alignment = 4096; + + memdb_type_t type = MEMDB_TYPE_PARTITION; + + size_t pool_size = 4096 * 100; + size_t pool_size2 = 1024; + size_t obj_size1 = 4096; + + partition.allocator.heap = NULL; + + /* ---------------- Giving memory to heap --------------------- */ + uintptr_t block = (uintptr_t)aligned_alloc(alignment, pool_size); + + ret = allocator_heap_add_memory(&partition.allocator, (void *)block, + pool_size); + if (!ret) { + printf("Memory added to heap\n"); + } + + /* ----------------- Hypervisor memory ------------------------ */ + uintptr_t block2 = (uintptr_t)aligned_alloc(alignment, pool_size2); + + /* ---------- Allocate object. partition for example ---------------- */ + void_ptr_result_t res; + res = allocator_allocate_object(&partition.allocator, obj_size1, + alignment); + if (res.e != OK) { + printf("Object allocation failed\n"); + goto error; + } else { + printf("Object allocation SUCCESS\n"); + } + + /* ---------------- Init memory database --------------------- */ + ret = memdb_init(); + + if (!ret) { + printf("Mem db init correct!\n"); + } else { + printf("Error init!\n"); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addrh = 0xfffffffffffff000; + uint64_t end_addrh = 0xffffffffffffffff; + + printf("\nstart_addrh: %#lx, end_addrh: %#lx\n", start_addrh, + end_addrh); + printf("\nnew type: %d\n", MEMDB_TYPE_ALLOCATOR); + + ret = memdb_insert(&partition, start_addrh, end_addrh, + (uintptr_t)block2, MEMDB_TYPE_ALLOCATOR); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addr = 0x0; + uint64_t end_addr = 0xffffffffffffefff; + + printf("\nstart_addr: %#lx, end_addr: %#lx\n", start_addr, end_addr); + printf("\nnew type: %d\n", type); + + ret = memdb_insert(&partition, start_addr, end_addr, (uintptr_t)block, + type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ----------------- lookup --------------------- */ + uint64_t addr = 0xfffffffffffeeffe; + + printf("\nLooking for addr: %#lx\n", addr); + memdb_obj_type_result_t tot_res; + + tot_res = memdb_lookup(addr); + ret = tot_res.e; + if (tot_res.e == OK) { + printf("\nmemdb_lookup SUCCESS. type: %d\n\n", tot_res.r.type); + } else { + printf("\nmemdb_lookup FAILED\n\n"); + goto error; + } + +error: + return ret; +} + +// Insert a range with a root guard and then insert another range that remove +// root guard +int +test11() +{ + partition_t partition; + int ret = 0; + size_t alignment = sizeof(void *); + size_t pool_size = 4096 * 100; + + partition.allocator.heap = NULL; + + /* ---------------- Giving memory to heap --------------------- */ + uintptr_t block = (uintptr_t)malloc(pool_size); + + ret = allocator_heap_add_memory(&partition.allocator, (void *)block, + pool_size); + if (!ret) { + printf("Memory added to heap\n"); + } + + /* ---------------- Init memory database --------------------- */ + ret = memdb_init(); + + if (!ret) { + printf("Mem db init correct!\n"); + } else { + printf("Error init!\n"); + } + + size_t obj_size = 1024; + memdb_type_t type = MEMDB_TYPE_PARTITION; + + size_t obj_size2 = 4096; + memdb_type_t type2 = MEMDB_TYPE_ALLOCATOR; + + /* ---------- Allocate object. partition for example ---------------- */ + void_ptr_result_t res; + res = allocator_allocate_object(&partition.allocator, obj_size, + alignment); + + uintptr_t object = (uintptr_t)res.r; + if (res.e != OK) { + printf("Object allocation failed\n"); + } else { + printf("Object allocation SUCCESS\n"); + } + + uint64_t start_addr = (uint64_t)object; + uint64_t end_addr = (uint64_t)object + (obj_size - 1); + + /* ---------- Allocate object. hypervisor for example ---------------- + */ + res = allocator_allocate_object(&partition.allocator, obj_size2, + alignment); + + uintptr_t object2 = (uintptr_t)res.r; + if (res.e != OK) { + printf("\nObject allocation failed\n"); + } else { + printf("\nObject allocation SUCCESS\n"); + } + + uint64_t start_addr2 = (uint64_t)object2; + uint64_t end_addr2 = (uint64_t)object2 + (obj_size2 - 1); + + /* ---------------- Insert object in database --------------------- */ + start_addr = 0; + end_addr = 15; + + printf("\nstart_addr: %#lx, end_addr: %#lx\n", start_addr, end_addr); + + ret = memdb_insert(&partition, start_addr, end_addr, (uintptr_t)object, + type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + start_addr2 = ~(start_addr2 & 0) - 15; + end_addr2 = ~(end_addr2 & 0); + + printf("\nstart_addr2: %#lx, end_addr2: %#lx\n\n", start_addr2, + end_addr2); + + ret = memdb_insert(&partition, start_addr2, end_addr2, + (uintptr_t)object2, type2); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } +error: + return ret; +} + +// 2 non contiguous insertions and 1 update that should fail due to +// contiguousness +int +test12() +{ + partition_t partition; + int ret = 0; + size_t alignment = 4096; + + memdb_type_t type = MEMDB_TYPE_PARTITION; + + size_t pool_size = 4096 * 100; + size_t obj_size1 = 4096; + size_t obj_size2 = 1024; + + partition.allocator.heap = NULL; + + /* ---------------- Giving memory to heap --------------------- */ + uintptr_t block = (uintptr_t)aligned_alloc(alignment, pool_size); + + ret = allocator_heap_add_memory(&partition.allocator, (void *)block, + pool_size); + if (!ret) { + printf("Memory added to heap\n"); + } + + /* ----------------- Hypervisor memory ------------------------ */ + // uintptr_t block2 = (uintptr_t)aligned_alloc(alignment, pool_size2); + + /* ---------- Allocate object. partition for example ---------------- */ + void_ptr_result_t res; + res = allocator_allocate_object(&partition.allocator, obj_size1, + alignment); + + uintptr_t object1 = (uintptr_t)res.r; + if (res.e != OK) { + printf("Object allocation failed\n"); + goto error; + } else { + printf("Object allocation SUCCESS\n"); + } + + uint64_t start_addr1 = (uint64_t)object1; + uint64_t end_addr1 = (uint64_t)object1 + (obj_size1 - 1); + + /* ---------- Allocate object. hypervisor for example ---------------- + */ + res = allocator_allocate_object(&partition.allocator, obj_size2, + alignment); + if (res.e != OK) { + printf("\nObject allocation failed\n"); + goto error; + } else { + printf("\nObject allocation SUCCESS\n"); + } + + /* ---------------- Init memory database --------------------- */ + ret = memdb_init(); + + if (!ret) { + printf("Mem db init correct!\n"); + } else { + printf("Error init!\n"); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addr = 0x4000; + uint64_t end_addr = 0x7fff; + + printf("\nstart_addr: %#lx, end_addr: %#lx\n", start_addr, end_addr); + printf("\nnew type: %d\n", type); + + ret = memdb_insert(&partition, start_addr, end_addr, (uintptr_t)block, + type); + if (!ret) { + printf("\nmemdb_insert SUCCESS\n\n"); + print_memdb_empty(); + } else { + printf("\nmemdb_insert FAILED\n\n"); + print_memdb_empty(); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addrh = 0x1380; + uint64_t end_addrh = 0x13ff; + + printf("\nstart_addr: %#lx, end_addr: %#lx\n", start_addrh, end_addrh); + printf("\nnew type: %d\n", MEMDB_TYPE_ALLOCATOR); + + ret = memdb_insert(&partition, start_addrh, end_addrh, (uintptr_t)block, + type); + + if (!ret) { + printf("\nmemdb_insert SUCCESS\n\n"); + print_memdb_empty(); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ---------------- Update database --------------------- */ + start_addr1 = 0x1380; + end_addr1 = 0x7fff; + + printf("\nstart_addr1: %#lx, end_addr1: %#lx\n", start_addr1, + end_addr1); + printf("\nnew type: %d old type: %d\n", 4, type); + + ret = memdb_update(&partition, start_addr1, end_addr1, object1, 4, + block, type); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS. should have failed!!\n\n"); + ret = -1; + goto error; + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED as expected\n\n"); + ret = 0; + } + +error: + return ret; +} + +// Insert a second range that has to check a guard in the end +// path. Guard partially matches +int +test13() +{ + partition_t partition; + int ret = 0; + size_t alignment = 4096; + + memdb_type_t type = MEMDB_TYPE_PARTITION; + + size_t pool_size = 4096 * 100; + size_t pool_size2 = 1024; + size_t obj_size1 = 4096; + + partition.allocator.heap = NULL; + + /* ---------------- Giving memory to heap --------------------- */ + uintptr_t block = (uintptr_t)aligned_alloc(alignment, pool_size); + + ret = allocator_heap_add_memory(&partition.allocator, (void *)block, + pool_size); + if (!ret) { + printf("Memory added to heap\n"); + } + + /* ----------------- Hypervisor memory ------------------------ */ + uintptr_t block2 = (uintptr_t)aligned_alloc(alignment, pool_size2); + + /* ---------- Allocate object. partition for example ---------------- */ + void_ptr_result_t res; + res = allocator_allocate_object(&partition.allocator, obj_size1, + alignment); + if (res.e != OK) { + printf("Object allocation failed\n"); + goto error; + } else { + printf("Object allocation SUCCESS\n"); + } + + /* ---------------- Init memory database --------------------- */ + ret = memdb_init(); + + if (!ret) { + printf("Mem db init correct!\n"); + } else { + printf("Error init!\n"); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addrh = 0xfffffffffffff000; + uint64_t end_addrh = 0xffffffffffffffff; + + printf("\nstart_addrh: %#lx, end_addrh: %#lx\n", start_addrh, + end_addrh); + printf("\nnew type: %d\n", MEMDB_TYPE_ALLOCATOR); + + ret = memdb_insert(&partition, start_addrh, end_addrh, + (uintptr_t)block2, MEMDB_TYPE_ALLOCATOR); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addr = 0x0; + uint64_t end_addr = 0xfffffffffffeeffe; + + printf("\nstart_addr: %#lx, end_addr: %#lx\n", start_addr, end_addr); + printf("\nnew type: %d\n", type); + + ret = memdb_insert(&partition, start_addr, end_addr, (uintptr_t)block, + type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ----------------- lookup --------------------- */ + uint64_t addr = 0xfffffffffffeeffe; + + printf("\nLooking for addr: %#lx\n", addr); + memdb_obj_type_result_t tot_res; + + tot_res = memdb_lookup(addr); + ret = tot_res.e; + if (tot_res.e == OK) { + printf("\nmemdb_lookup SUCCESS. type: %d\n\n", tot_res.r.type); + } else { + printf("\nmemdb_lookup FAILED\n\n"); + goto error; + } + +error: + return ret; +} + +paddr_t returned_base; +size_t returned_size; + +error_t +add_free_range(paddr_t base, size_t size, void *arg) +{ + (void)arg; + // uint64_t end_addr = base + (size - 1); + // printf("add_free_range: base: %#lx - size: %lu - end_addr: %#lx\n", + // base, size, end_addr); + printf("add_free_range: base: %#lx - size: %#lx\n", base, size); + + returned_base = base; + returned_size = size; + + return OK; +} + +// 2 insertions, 2 updates, 2 lookups, 2 memwalk with guards +int +test14() +{ + partition_t partition; + int ret = 0; + size_t alignment = sizeof(void *); + + memdb_type_t type = MEMDB_TYPE_PARTITION; + memdb_type_t type1 = MEMDB_TYPE_ALLOCATOR; + memdb_type_t type2 = MEMDB_TYPE_EXTENT; + + size_t pool_size = 4096 * 100; + size_t pool_size2 = 1024; + size_t obj_size1 = 4096; + size_t obj_size2 = 1024; + + partition.allocator.heap = NULL; + + /* ---------------- Giving memory to heap --------------------- */ + uintptr_t block = (uintptr_t)malloc(pool_size); + + ret = allocator_heap_add_memory(&partition.allocator, (void *)block, + pool_size); + if (!ret) { + printf("Memory added to heap\n"); + } + + /* ----------------- Hypervisor memory ------------------------ */ + uintptr_t block2 = (uintptr_t)malloc(pool_size2); + + /* ---------- Allocate object. partition for example ---------------- */ + void_ptr_result_t res; + res = allocator_allocate_object(&partition.allocator, obj_size1, + alignment); + + uintptr_t object1 = (uintptr_t)res.r; + if (res.e != OK) { + printf("Object allocation failed\n"); + goto error; + } else { + printf("Object allocation SUCCESS\n"); + } + + uint64_t start_addr1 = (uint64_t)object1; + uint64_t end_addr1 = (uint64_t)object1 + (obj_size1 - 1); + + /* ---------- Allocate object. hypervisor for example ---------------- + */ + res = allocator_allocate_object(&partition.allocator, obj_size2, + alignment); + + uintptr_t object2 = (uintptr_t)res.r; + if (res.e != OK) { + printf("\nObject allocation failed\n"); + goto error; + } else { + printf("\nObject allocation SUCCESS\n"); + } + + uint64_t start_addr2 = (uint64_t)object2; + uint64_t end_addr2 = (uint64_t)object2 + (obj_size2 - 1); + + /* ---------------- Init memory database --------------------- */ + ret = memdb_init(); + + if (!ret) { + printf("Mem db init correct!\n"); + } else { + printf("Error init!\n"); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addr = (uint64_t)block; + uint64_t end_addr = (uint64_t)block + (pool_size - 1); + uint64_t range_size = (end_addr - start_addr + 1); + + printf("\nstart_addr: %#lx, end_addr: %#lx, size: %#lx\n", start_addr, + end_addr, range_size); + printf("\nnew type: %d\n", type); + + ret = memdb_insert(&partition, start_addr, end_addr, block, type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + printf("\nmemdb_insert FAILED\n\n"); + print_memdb_empty(); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addrh = (uint64_t)block2; + uint64_t end_addrh = (uint64_t)block2 + (pool_size2 - 1); + uint64_t range_size2 = (end_addrh - start_addrh + 1); + + printf("\nstart_addr: %#lx, end_addr: %#lx, size: %#lx\n", start_addrh, + end_addrh, (end_addrh - start_addrh + 1)); + printf("\nnew type: %d\n", type1); + + ret = memdb_insert(&partition, start_addrh, end_addrh, block2, type1); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ---------------- MEMDB WALK --------------------- */ + printf("Mem walk to match type: %d\n", type); + void *arg = NULL; + error_t resl = memdb_walk(block, type, add_free_range, arg); + if ((resl == OK) && (returned_base == start_addr) && + (returned_size == range_size)) { + printf("memdb walk SUCCESS\n\n"); + } else { + printf("memdb walk FAILED\n"); + ret = resl; + if (!(returned_base == start_addr)) { + printf("returned_base: %#lx - start_addr: %#lx\n", + returned_base, start_addr); + ret = -1; + } + if (!(returned_size == range_size)) { + printf("returned_size: %#lx - size: %lx\n\n", + returned_size, range_size); + ret = -1; + } + goto error; + } + + /* ---------------- Update database --------------------- */ + + printf("\nstart_addr1: %#lx, end_addr1: %#lx, size: %#lx\n", + start_addr1, end_addr1, obj_size1); + printf("\nnew type: %d old type: %d\n", type1, type); + uint64_t range_size3 = (end_addr1 - start_addr1 + 1); + + ret = memdb_update(&partition, start_addr1, end_addr1, block2, type1, + block, type); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ---------------- Update database --------------------- */ + + printf("\nstart_addr2: %#lx, end_addr2: %#lx, size: %#lx\n", + start_addr2, end_addr2, obj_size2); + printf("\nnew type: %d old type: %d\n", type2, type); + + ret = memdb_update(&partition, start_addr2, end_addr2, object2, type2, + block, type); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ---------------- MEMDB WALK --------------------- */ + printf("Mem walk to match type: %d\n", type1); + resl = memdb_walk(block2, type1, add_free_range, arg); + + if ((resl == OK) && + ((returned_base == start_addrh) || + (returned_base == start_addr1)) && + ((returned_size == range_size2) || + (returned_size == range_size3))) { + printf("memdb walk SUCCESS\n\n"); + } else { + printf("memdb walk FAILED\n\n"); + ret = resl; + if (!(returned_base == start_addrh)) { + printf("returned_base: %#lx - start_addr: %#lx\n", + returned_base, start_addrh); + ret = -1; + } + if (!(returned_size == range_size2)) { + printf("returned_size: %#lx - size: %#lx\n", + returned_size, range_size2); + ret = -1; + } + goto error; + } + + /* ----------------- lookup --------------------- */ + printf("\nLooking for start_addr1: %#lx\n", start_addr1); + memdb_obj_type_result_t tot_res; + + tot_res = memdb_lookup(start_addr1); + ret = tot_res.e; + if (tot_res.e == OK) { + printf("\nmemdb_lookup SUCCESS. type: %d\n\n", tot_res.r.type); + } else { + printf("\nmemdb_lookup FAILED\n\n"); + goto error; + } + + /* ----------------- lookup --------------------- */ + printf("\nLooking for start_addrh: %#lx\n", start_addr1); + tot_res = memdb_lookup(start_addrh); + ret = tot_res.e; + if (tot_res.e == OK) { + printf("\nmemdb_lookup SUCCESS. type: %d\n\n", tot_res.r.type); + } else { + printf("\nmemdb_lookup FAILED\n\n"); + goto error; + } +error: + return ret; +} + +// 2 insertions, 2 updates, 2 lookups, 2 memwalk without guard +int +test15() +{ + partition_t partition; + int ret = 0; + size_t alignment = sizeof(void *); + + memdb_type_t type = MEMDB_TYPE_PARTITION; + memdb_type_t type1 = MEMDB_TYPE_ALLOCATOR; + memdb_type_t type2 = MEMDB_TYPE_EXTENT; + + size_t pool_size = 4096 * 100; + size_t pool_size2 = 1024; + size_t obj_size1 = 4096; + size_t obj_size2 = 1024; + + partition.allocator.heap = NULL; + + /* ---------------- Giving memory to heap --------------------- */ + uintptr_t block = (uintptr_t)malloc(pool_size); + + ret = allocator_heap_add_memory(&partition.allocator, (void *)block, + pool_size); + if (!ret) { + printf("Memory added to heap\n"); + } + + /* ----------------- Hypervisor memory ------------------------ */ + uintptr_t block2 = (uintptr_t)malloc(pool_size2); + + /* ---------- Allocate object. partition for example ---------------- */ + void_ptr_result_t res; + res = allocator_allocate_object(&partition.allocator, obj_size1, + alignment); + + uintptr_t object1 = (uintptr_t)res.r; + if (res.e != OK) { + printf("Object allocation failed\n"); + goto error; + } else { + printf("Object allocation SUCCESS\n"); + } + + uint64_t start_addr1 = (uint64_t)object1; + uint64_t end_addr1 = (uint64_t)object1 + (obj_size1 - 1); + + /* ---------- Allocate object. hypervisor for example ---------------- + */ + res = allocator_allocate_object(&partition.allocator, obj_size2, + alignment); + + uintptr_t object2 = (uintptr_t)res.r; + if (res.e != OK) { + printf("\nObject allocation failed\n"); + goto error; + } else { + printf("\nObject allocation SUCCESS\n"); + } + + uint64_t start_addr2 = (uint64_t)object2; + uint64_t end_addr2 = (uint64_t)object2 + (obj_size2 - 1); + + /* ---------------- Init memory database --------------------- */ + ret = memdb_init(); + + if (!ret) { + printf("Mem db init correct!\n"); + } else { + printf("Error init!\n"); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addr = 0x4; + uint64_t end_addr = 0xfffffffffffeeffe; + uint64_t range_size = (end_addr - start_addr + 1); + + printf("\nstart_addr: %#lx, end_addr: %#lx, size: %#lx\n", start_addr, + end_addr, (end_addr - start_addr + 1)); + printf("\nnew type: %d\n", type); + + ret = memdb_insert(&partition, start_addr, end_addr, block, type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + printf("\nmemdb_insert FAILED\n\n"); + print_memdb_empty(); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addrh = 0xffffffffffffff00; + uint64_t end_addrh = 0xffffffffffffffff; + uint64_t range_size2 = (end_addrh - start_addrh + 1); + + printf("\nstart_addr: %#lx, end_addr: %#lx, size: %#lx\n", start_addrh, + end_addrh, (end_addrh - start_addrh + 1)); + printf("\nnew type: %d\n", type1); + + ret = memdb_insert(&partition, start_addrh, end_addrh, block2, type1); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ---------------- MEMDB WALK --------------------- */ + printf("Mem walk to match type: %d\n", type); + void *arg = NULL; + error_t resl = memdb_walk(block, type, add_free_range, arg); + if ((resl == OK) && (returned_base == start_addr) && + (returned_size == range_size)) { + printf("memdb walk SUCCESS\n\n"); + } else { + printf("memdb walk FAILED\n\n"); + if (!(returned_base == start_addr)) { + printf("returned_base: %#lx start_addr: %#lx\n", + returned_base, start_addr); + } + if (!(returned_size == range_size)) { + printf("returned_size: %#lx, range_size: %#lx\n", + returned_size, range_size); + } + ret = -1; + goto error; + } + + /* ---------------- Update database --------------------- */ + + printf("\nstart_addr1: %#lx, end_addr1: %#lx, size: %#lx\n", + start_addr1, end_addr1, obj_size1); + printf("\nnew type: %d old type: %d\n", type1, type); + + ret = memdb_update(&partition, start_addr1, end_addr1, block2, type1, + block, type); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ---------------- Update database --------------------- */ + + printf("\nstart_addr2: %#lx, end_addr2: %#lx, size: %#lx\n", + start_addr2, end_addr2, obj_size2); + printf("\nnew type: %d old type: %d\n", type2, type); + + ret = memdb_update(&partition, start_addr2, end_addr2, object2, type2, + block, type); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ---------------- MEMDB WALK --------------------- */ + printf("Mem walk to match type: %d\n", type1); + resl = memdb_walk(block2, type1, add_free_range, arg); + if ((resl == OK) && (returned_base == start_addrh) && + (returned_size == range_size2)) { + printf("memdb walk SUCCESS\n\n"); + } else { + printf("memdb walk FAILED\n\n"); + if (!(returned_base == start_addrh)) { + printf("returned_base: %#lx start_addr: %#lx\n", + returned_base, start_addrh); + } + if (!(returned_size == range_size2)) { + printf("returned_size: %#lx, range_size: %#lx\n", + returned_size, range_size2); + } + ret = -1; + + goto error; + } + + /* ----------------- lookup --------------------- */ + printf("\nLooking for start_addr1: %#lx\n", start_addr1); + memdb_obj_type_result_t tot_res; + + tot_res = memdb_lookup(start_addr1); + ret = tot_res.e; + if (tot_res.e == OK) { + printf("\nmemdb_lookup SUCCESS. type: %d\n\n", tot_res.r.type); + } else { + printf("\nmemdb_lookup FAILED\n\n"); + goto error; + } + + /* ----------------- lookup --------------------- */ + printf("\nLooking for start_addrh: %#lx\n", start_addr1); + tot_res = memdb_lookup(start_addrh); + ret = tot_res.e; + if (tot_res.e == OK) { + printf("\nmemdb_lookup SUCCESS. type: %d\n\n", tot_res.r.type); + } else { + printf("\nmemdb_lookup FAILED\n\n"); + goto error; + } +error: + return ret; +} + +// 2 insertions, 2 updates, 2 lookups, 2 memwalk RANGE with guards +int +test16() +{ + partition_t partition; + int ret = 0; + size_t alignment = sizeof(void *); + + memdb_type_t type = MEMDB_TYPE_PARTITION; + memdb_type_t type1 = MEMDB_TYPE_ALLOCATOR; + memdb_type_t type2 = MEMDB_TYPE_EXTENT; + + size_t pool_size = 4096 * 100; + size_t pool_size2 = 1024; + size_t obj_size1 = 4096; + size_t obj_size2 = 1024; + + partition.allocator.heap = NULL; + + /* ---------------- Giving memory to heap --------------------- */ + uintptr_t block = (uintptr_t)malloc(pool_size); + + ret = allocator_heap_add_memory(&partition.allocator, (void *)block, + pool_size); + if (!ret) { + printf("Memory added to heap\n"); + } + + /* ----------------- Hypervisor memory ------------------------ */ + uintptr_t block2 = (uintptr_t)malloc(pool_size2); + + /* ---------- Allocate object. partition for example ---------------- */ + void_ptr_result_t res; + res = allocator_allocate_object(&partition.allocator, obj_size1, + alignment); + + uintptr_t object1 = (uintptr_t)res.r; + if (res.e != OK) { + printf("Object allocation failed\n"); + goto error; + } else { + printf("Object allocation SUCCESS\n"); + } + + uint64_t start_addr1 = (uint64_t)object1; + uint64_t end_addr1 = (uint64_t)object1 + (obj_size1 - 1); + + /* ---------- Allocate object. hypervisor for example ---------------- + */ + res = allocator_allocate_object(&partition.allocator, obj_size2, + alignment); + + uintptr_t object2 = (uintptr_t)res.r; + if (res.e != OK) { + printf("\nObject allocation failed\n"); + goto error; + } else { + printf("\nObject allocation SUCCESS\n"); + } + + uint64_t start_addr2 = (uint64_t)object2; + uint64_t end_addr2 = (uint64_t)object2 + (obj_size2 - 1); + + /* ---------------- Init memory database --------------------- */ + ret = memdb_init(); + + if (!ret) { + printf("Mem db init correct!\n"); + } else { + printf("Error init!\n"); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addr = (uint64_t)block; + uint64_t end_addr = (uint64_t)block + (pool_size - 1); + uint64_t range_size = (end_addr - start_addr + 1); + + printf("\nstart_addr: %#lx, end_addr: %#lx, size: %#lx\n", start_addr, + end_addr, range_size); + printf("\nnew type: %d\n", type); + + ret = memdb_insert(&partition, start_addr, end_addr, block, type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + printf("\nmemdb_insert FAILED\n\n"); + print_memdb_empty(); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addrh = (uint64_t)block2; + uint64_t end_addrh = (uint64_t)block2 + (pool_size2 - 1); + uint64_t range_size2 = (end_addrh - start_addrh + 1); + + printf("\nstart_addr: %#lx, end_addr: %#lx, size: %#lx\n", start_addrh, + end_addrh, (end_addrh - start_addrh + 1)); + printf("\nnew type: %d\n", type1); + + ret = memdb_insert(&partition, start_addrh, end_addrh, block2, type1); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ---------------- MEMDB RANGE WALK --------------------- */ + printf("Mem range walk to match type: %d\n", type); + void *arg = NULL; + error_t resl = memdb_range_walk(block, type, start_addr, end_addr, + add_free_range, arg); + if ((resl == OK) && (returned_base == start_addr) && + (returned_size == range_size)) { + printf("memdb range walk SUCCESS\n\n"); + } else { + printf("memdb range walk FAILED\n"); + ret = resl; + if (!(returned_base == start_addr)) { + printf("returned_base: %#lx - start_addr: %#lx\n", + returned_base, start_addr); + ret = -1; + } + if (!(returned_size == range_size)) { + printf("returned_size: %#lx - size: %lx\n\n", + returned_size, range_size); + ret = -1; + } + goto error; + } + + /* ---------------- Update database --------------------- */ + + printf("\nstart_addr1: %#lx, end_addr1: %#lx, size: %#lx\n", + start_addr1, end_addr1, obj_size1); + printf("\nnew type: %d old type: %d\n", type1, type); + uint64_t range_size3 = (end_addr1 - start_addr1 + 1); + + ret = memdb_update(&partition, start_addr1, end_addr1, block2, type1, + block, type); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ---------------- Update database --------------------- */ + + printf("\nstart_addr2: %#lx, end_addr2: %#lx, size: %#lx\n", + start_addr2, end_addr2, obj_size2); + printf("\nnew type: %d old type: %d\n", type2, type); + + ret = memdb_update(&partition, start_addr2, end_addr2, object2, type2, + block, type); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ---------------- MEMDB RANGE WALK --------------------- */ + printf("Mem range walk to match type: %d\n", type1); + resl = memdb_range_walk(block2, type1, start_addrh, end_addrh, + add_free_range, arg); + + if ((resl == OK) && (returned_base == start_addrh) && + (returned_size == range_size2)) { + printf("memdb range walk SUCCESS\n\n"); + } else { + printf("memdb range walk FAILED\n\n"); + ret = resl; + if (!(returned_base == start_addrh)) { + printf("returned_base: %#lx - start_addr: %#lx\n", + returned_base, start_addrh); + ret = -1; + } + if (!(returned_size == range_size2)) { + printf("returned_size: %#lx - size: %#lx\n", + returned_size, range_size2); + ret = -1; + } + goto error; + } + + /* ---------------- MEMDB RANGE WALK --------------------- */ + printf("Mem range walk to match type: %d\n", type1); + resl = memdb_range_walk(block2, type1, start_addr1, end_addr1, + add_free_range, arg); + + if ((resl == OK) && (returned_base == start_addr1) && + (returned_size == range_size3)) { + printf("memdb range walk SUCCESS\n\n"); + } else { + printf("memdb range walk FAILED\n\n"); + ret = resl; + if (!(returned_base == start_addr1)) { + printf("returned_base: %#lx - start_addr: %#lx\n", + returned_base, start_addr1); + ret = -1; + } + if (!(returned_size == range_size3)) { + printf("returned_size: %#lx - size: %#lx\n", + returned_size, range_size3); + ret = -1; + } + goto error; + } + + /* ----------------- lookup --------------------- */ + printf("\nLooking for start_addr1: %#lx\n", start_addr1); + memdb_obj_type_result_t tot_res; + + tot_res = memdb_lookup(start_addr1); + ret = tot_res.e; + if (tot_res.e == OK) { + printf("\nmemdb_lookup SUCCESS. type: %d\n\n", tot_res.r.type); + } else { + printf("\nmemdb_lookup FAILED\n\n"); + goto error; + } + + /* ----------------- lookup --------------------- */ + printf("\nLooking for start_addrh: %#lx\n", start_addr1); + tot_res = memdb_lookup(start_addrh); + ret = tot_res.e; + if (tot_res.e == OK) { + printf("\nmemdb_lookup SUCCESS. type: %d\n\n", tot_res.r.type); + } else { + printf("\nmemdb_lookup FAILED\n\n"); + goto error; + } +error: + return ret; +} + +// 2 insertions, 2 updates, 2 lookups, 2 memwalk RANGE without guard +int +test17() +{ + partition_t partition; + int ret = 0; + size_t alignment = sizeof(void *); + + memdb_type_t type = MEMDB_TYPE_PARTITION; + memdb_type_t type1 = MEMDB_TYPE_ALLOCATOR; + memdb_type_t type2 = MEMDB_TYPE_EXTENT; + + size_t pool_size = 4096 * 100; + size_t pool_size2 = 1024; + size_t obj_size1 = 4096; + size_t obj_size2 = 1024; + + partition.allocator.heap = NULL; + + /* ---------------- Giving memory to heap --------------------- */ + uintptr_t block = (uintptr_t)malloc(pool_size); + + ret = allocator_heap_add_memory(&partition.allocator, (void *)block, + pool_size); + if (!ret) { + printf("Memory added to heap\n"); + } + + /* ----------------- Hypervisor memory ------------------------ */ + uintptr_t block2 = (uintptr_t)malloc(pool_size2); + + /* ---------- Allocate object. partition for example ---------------- */ + void_ptr_result_t res; + res = allocator_allocate_object(&partition.allocator, obj_size1, + alignment); + + uintptr_t object1 = (uintptr_t)res.r; + if (res.e != OK) { + printf("Object allocation failed\n"); + goto error; + } else { + printf("Object allocation SUCCESS\n"); + } + + uint64_t start_addr1 = (uint64_t)object1; + uint64_t end_addr1 = (uint64_t)object1 + (obj_size1 - 1); + + /* ---------- Allocate object. hypervisor for example ---------------- + */ + res = allocator_allocate_object(&partition.allocator, obj_size2, + alignment); + + uintptr_t object2 = (uintptr_t)res.r; + if (res.e != OK) { + printf("\nObject allocation failed\n"); + goto error; + } else { + printf("\nObject allocation SUCCESS\n"); + } + + uint64_t start_addr2 = (uint64_t)object2; + uint64_t end_addr2 = (uint64_t)object2 + (obj_size2 - 1); + + /* ---------------- Init memory database --------------------- */ + ret = memdb_init(); + + if (!ret) { + printf("Mem db init correct!\n"); + } else { + printf("Error init!\n"); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addr = 0x4; + uint64_t end_addr = 0xfffffffffffeeffe; + uint64_t range_size = (end_addr - start_addr + 1); + + printf("\nstart_addr: %#lx, end_addr: %#lx, size: %#lx\n", start_addr, + end_addr, (end_addr - start_addr + 1)); + printf("\nnew type: %d\n", type); + + ret = memdb_insert(&partition, start_addr, end_addr, block, type); + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + printf("\nmemdb_insert FAILED\n\n"); + print_memdb_empty(); + goto error; + } + + /* ---------------- Insert object in database --------------------- */ + uint64_t start_addrh = 0xffffffffffffff00; + uint64_t end_addrh = 0xffffffffffffffff; + uint64_t range_size2 = (end_addrh - start_addrh + 1); + + printf("\nstart_addr: %#lx, end_addr: %#lx, size: %#lx\n", start_addrh, + end_addrh, (end_addrh - start_addrh + 1)); + printf("\nnew type: %d\n", type1); + + ret = memdb_insert(&partition, start_addrh, end_addrh, block2, type1); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_insert SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_insert FAILED\n\n"); + goto error; + } + + /* ---------------- MEMDB RANGE WALK --------------------- */ + printf("Mem range walk to match type: %d\n", type); + void *arg = NULL; + error_t resl = memdb_range_walk(block, type, start_addr, end_addr, + add_free_range, arg); + if ((resl == OK) && (returned_base == start_addr) && + (returned_size == range_size)) { + printf("memdb range walk SUCCESS\n\n"); + } else { + printf("memdb range walk FAILED\n\n"); + if (!(returned_base == start_addr)) { + printf("returned_base: %#lx start_addr: %#lx\n", + returned_base, start_addr); + } + if (!(returned_size == range_size)) { + printf("returned_size: %#lx, range_size: %#lx\n", + returned_size, range_size); + } + ret = -1; + goto error; + } + + /* ---------------- Update database --------------------- */ + + printf("\nstart_addr1: %#lx, end_addr1: %#lx, size: %#lx\n", + start_addr1, end_addr1, obj_size1); + printf("\nnew type: %d old type: %d\n", type1, type); + uint64_t range_size3 = (end_addr1 - start_addr1 + 1); + + ret = memdb_update(&partition, start_addr1, end_addr1, block2, type1, + block, type); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ---------------- Update database --------------------- */ + + printf("\nstart_addr2: %#lx, end_addr2: %#lx, size: %#lx\n", + start_addr2, end_addr2, obj_size2); + printf("\nnew type: %d old type: %d\n", type2, type); + + ret = memdb_update(&partition, start_addr2, end_addr2, object2, type2, + block, type); + + if (!ret) { + print_memdb_empty(); + printf("\nmemdb_update SUCCESS\n\n"); + } else { + print_memdb_empty(); + printf("\nmemdb_update FAILED\n\n"); + goto error; + } + + /* ---------------- MEMDB RANGE WALK --------------------- */ + printf("Mem walk to match type: %d\n", type1); + resl = memdb_range_walk(block2, type1, start_addrh, end_addrh, + add_free_range, arg); + if ((resl == OK) && (returned_base == start_addrh) && + (returned_size == range_size2)) { + printf("memdb range walk SUCCESS\n\n"); + } else { + printf("memdb range walk FAILED\n\n"); + if (!(returned_base == start_addrh)) { + printf("returned_base: %#lx start_addr: %#lx\n", + returned_base, start_addrh); + } + if (!(returned_size == range_size2)) { + printf("returned_size: %#lx, range_size: %#lx\n", + returned_size, range_size2); + } + ret = -1; + + goto error; + } + + /* ---------------- MEMDB RANGE WALK --------------------- */ + printf("Mem walk to match type: %d\n", type1); + resl = memdb_range_walk(block2, type1, start_addr1, end_addr1, + add_free_range, arg); + if ((resl == OK) && (returned_base == start_addr1) && + (returned_size == range_size3)) { + printf("memdb range walk SUCCESS\n\n"); + } else { + printf("memdb range walk FAILED\n\n"); + if (!(returned_base == start_addr1)) { + printf("returned_base: %#lx start_addr: %#lx\n", + returned_base, start_addr1); + } + if (!(returned_size == range_size3)) { + printf("returned_size: %#lx, range_size: %#lx\n", + returned_size, range_size3); + } + ret = -1; + + goto error; + } + + /* ----------------- lookup --------------------- */ + printf("\nLooking for start_addr1: %#lx\n", start_addr1); + memdb_obj_type_result_t tot_res; + + tot_res = memdb_lookup(start_addr1); + ret = tot_res.e; + if (tot_res.e == OK) { + printf("\nmemdb_lookup SUCCESS. type: %d\n\n", tot_res.r.type); + } else { + printf("\nmemdb_lookup FAILED\n\n"); + goto error; + } + + /* ----------------- lookup --------------------- */ + printf("\nLooking for start_addrh: %#lx\n", start_addr1); + tot_res = memdb_lookup(start_addrh); + ret = tot_res.e; + if (tot_res.e == OK) { + printf("\nmemdb_lookup SUCCESS. type: %d\n\n", tot_res.r.type); + } else { + printf("\nmemdb_lookup FAILED\n\n"); + goto error; + } +error: + return ret; +} + +#define NUM_TESTS 17 + +int (*func_ptr[NUM_TESTS])(void) = { + test1, // Insert two ranges in db + test2, // One insertion and two updates + test3, // One insertion, two updates, and two updates back to state + // after insertion + test4, // 2 insertions, 2 updates, 2 updates back to state after + // insertions + test5, // 1 insertion, 2 updates, 2 check of contiguousness (1 must + // succeed and the other one fail) + test6, // 1 insertion, 2 updates, 2 lookups + test7, // 2 insertions, 2 updates, 2 lookups + test8, // Address ranges with 64 guard (2 insertions, 2 updates, 2 + // updates back) + test9, // Rollback. (2 insertion, 2 updates (last one rolled back)) + test10, // Insert a second range that has to check a guard in the end + // path. Guard matches + test11, // Insert a range with a root guard and then insert another + // range that remove root guard + test12, // 2 non contiguous insertions and 1 update that should fail due + // to contiguousness + test13, // Insert a second range that has to check a guard in the end + // path. Guard partially matches + test14, // 2 insertions, 2 updates, 2 lookups, 2 mem walk with GUARDS + test15, // 2 insertions, 2 updates, 2 lookups, 2 mem walk without guards + test16, // 2 insertions, 2 updates, 2 lookups, 2 mem RANGE walk with + // GUARDS + test17, // 2 insertions, 2 updates, 2 lookups, 2 mem RANGE walk without + // guards +}; + +int +main() +{ + int test = 0; + int ret = 0; + + switch (test) { + case 0: + for (int i = 0; i < NUM_TESTS; i++) { + printf("\n\n_____________________________________________________ TEST %d ________________________________________________\n\n", + i + 1); + ret = func_ptr[i](); + if (ret != 0) { + printf("FAILED test: %d\n", i + 1); + break; + } + } + if (ret == 0) { + printf("All %d tests passed!\n", NUM_TESTS); + } + break; + + case 1: + test1(); + break; + case 2: + test2(); + break; + case 3: + test3(); + break; + case 4: + test4(); + break; + case 5: + // is memory contiguous test + test5(); + break; + case 6: + // lookup test + test6(); + break; + case 7: + // lookup test + test7(); + break; + case 8: + // Address ranges with 64 guard + test8(); + break; + case 9: + // Rollback test + test9(); + break; + case 10: + // Insert a second range that has to check a guard in the end + // path. Guard matches + test10(); + break; + case 11: + test11(); + break; + case 12: + test12(); + break; + case 13: + // Insert a second range that has to check a guard in the end + // path. Guard partially matches + test13(); + break; + } +} diff --git a/hyp/mem/memextent/memextent.ev b/hyp/mem/memextent/memextent.ev index 5e417eb..491b03a 100644 --- a/hyp/mem/memextent/memextent.ev +++ b/hyp/mem/memextent/memextent.ev @@ -15,16 +15,14 @@ subscribe object_deactivate_memextent(memextent) subscribe object_cleanup_memextent(memextent) -subscribe object_deactivate_addrspace(addrspace) - // BASIC memory extent +subscribe memextent_activate[MEMEXTENT_TYPE_BASIC] + handler memextent_activate_basic(me) + subscribe memextent_activate_derive[MEMEXTENT_TYPE_BASIC] handler memextent_activate_derive_basic(me) -subscribe memextent_derive[MEMEXTENT_TYPE_BASIC] - handler memextent_derive_basic(me, parent, offsets, sizes, memtype, access) - subscribe memextent_map[MEMEXTENT_TYPE_BASIC] handler memextent_map_basic(extent, addrspace, vm_base, map_attrs) @@ -38,8 +36,35 @@ subscribe memextent_update_access[MEMEXTENT_TYPE_BASIC] handler memextent_update_access_basic(extent, addrspace, vm_base, access_attrs) +subscribe memextent_is_mapped[MEMEXTENT_TYPE_BASIC] + handler memextent_is_mapped_basic(me, addrspace, exclusive) + subscribe memextent_deactivate[MEMEXTENT_TYPE_BASIC] handler memextent_deactivate_basic(extent) subscribe memextent_cleanup[MEMEXTENT_TYPE_BASIC] handler memextent_cleanup_basic(extent) + +subscribe memextent_retain_mappings[MEMEXTENT_TYPE_BASIC] + handler memextent_retain_mappings_basic(me) + +subscribe memextent_release_mappings[MEMEXTENT_TYPE_BASIC] + handler memextent_release_mappings_basic(me, clear) + +subscribe memextent_lookup_mapping[MEMEXTENT_TYPE_BASIC] + handler memextent_lookup_mapping_basic(me, phys, size, i) + +subscribe memextent_attach[MEMEXTENT_TYPE_BASIC] + handler memextent_attach_basic(me, hyp_va, size, memtype) + +subscribe memextent_detach[MEMEXTENT_TYPE_BASIC] + handler memextent_detach_basic(me) + +subscribe object_create_addrspace + handler memextent_create_addrspace_basic + +subscribe object_deactivate_addrspace + handler memextent_deactivate_addrspace_basic + +subscribe memextent_get_offset_for_pa[MEMEXTENT_TYPE_BASIC] + handler memextent_get_offset_for_pa_basic(extent, pa, size) diff --git a/hyp/mem/memextent/memextent.tc b/hyp/mem/memextent/memextent.tc index fb44fab..0dbb790 100644 --- a/hyp/mem/memextent/memextent.tc +++ b/hyp/mem/memextent/memextent.tc @@ -2,30 +2,37 @@ // // SPDX-License-Identifier: BSD-3-Clause -extend addrspace object { - mapping_list structure list; -}; +define MEMEXTENT_MAX_MAPS constant type count_t = 4; extend cap_rights_memextent bitfield { 0 map bool; 1 derive bool; 2 attach bool; + 3 lookup bool; + 4 donate bool; }; -define memextent_max_maps constant = 4; +extend addrspace object { + basic_mapping_list structure list; +}; -define memextent_arg structure { +define memextent_basic_arg structure { me pointer object memextent; - map array(memextent_max_maps) pointer structure memextent_mapping; + map array(MEMEXTENT_MAX_MAPS) pointer structure memextent_basic_mapping; failed_address type paddr_t; }; -define memextent_mapping structure { +define memextent_basic_mapping structure { // RCU-protected addrspace pointer addrspace pointer(atomic) object addrspace; - attrs bitfield memextent_mapping_attrs; mapping_list_node structure list_node(contained); vbase type vmaddr_t; + attrs bitfield memextent_mapping_attrs; + retained bool; +}; + +define memextent_map_ptr union(lockable) { + basic pointer structure memextent_basic_mapping; }; extend memextent object { @@ -36,9 +43,11 @@ extend memextent object { access enumeration pgtable_access; children_list structure list; children_list_node structure list_node(contained); - mappings array(memextent_max_maps) structure memextent_mapping; + mappings union memextent_map_ptr; active bool; device_mem bool; + attached_address uintptr; + attached_size size; }; #if defined(HYPERCALLS) diff --git a/hyp/mem/memextent/memextent_tests.ev b/hyp/mem/memextent/memextent_tests.ev index beead96..a8a8060 100644 --- a/hyp/mem/memextent/memextent_tests.ev +++ b/hyp/mem/memextent/memextent_tests.ev @@ -11,5 +11,6 @@ subscribe tests_init subscribe tests_start handler tests_memextent() + require_preempt_disabled #endif diff --git a/hyp/mem/memextent/src/hypercalls.c b/hyp/mem/memextent/src/hypercalls.c index c5529ee..feb0cc7 100644 --- a/hyp/mem/memextent/src/hypercalls.c +++ b/hyp/mem/memextent/src/hypercalls.c @@ -19,11 +19,19 @@ #include error_t -hypercall_memextent_unmap_all(cap_id_t memextent_cap) +hypercall_memextent_modify(cap_id_t memextent_cap, + memextent_modify_flags_t flags, size_t offset, + size_t size) { error_t err = OK; cspace_t *cspace = cspace_get_self(); + // FIXME: + if (memextent_modify_flags_get_res_0(&flags) != 0U) { + err = ERROR_ARGUMENT_INVALID; + goto out; + } + memextent_ptr_result_t m = cspace_lookup_memextent( cspace, memextent_cap, CAP_RIGHTS_MEMEXTENT_MAP); if (compiler_unexpected(m.e != OK)) { @@ -32,13 +40,25 @@ hypercall_memextent_unmap_all(cap_id_t memextent_cap) } memextent_t *memextent = m.r; + bool need_sync = !memextent_modify_flags_get_no_sync(&flags); + + memextent_modify_op_t op = memextent_modify_flags_get_op(&flags); + if (op == MEMEXTENT_MODIFY_OP_UNMAP_ALL) { + memextent_unmap_all(memextent); + } else if ((op == MEMEXTENT_MODIFY_OP_ZERO_RANGE) && !need_sync) { + err = memextent_zero_range(memextent, offset, size); + } else if (op == MEMEXTENT_MODIFY_OP_SYNC_ALL) { + err = need_sync ? OK : ERROR_ARGUMENT_INVALID; + } else { + err = ERROR_ARGUMENT_INVALID; + } - memextent_unmap_all(memextent); - // Wait for completion of EL2 operations using manual lookups - rcu_sync(); + if ((err == OK) && need_sync) { + // Wait for completion of EL2 operations using manual lookups + rcu_sync(); + } object_put_memextent(memextent); - out: return err; } @@ -48,7 +68,7 @@ hypercall_memextent_configure(cap_id_t memextent_cap, paddr_t phys_base, size_t size, memextent_attrs_t attributes) { error_t err; - cspace_t *cspace = cspace_get_self(); + cspace_t *cspace = cspace_get_self(); object_type_t type; object_ptr_result_t o = cspace_lookup_object_any( @@ -89,7 +109,7 @@ hypercall_memextent_configure_derive(cap_id_t memextent_cap, memextent_attrs_t attributes) { error_t err; - cspace_t *cspace = cspace_get_self(); + cspace_t *cspace = cspace_get_self(); object_type_t type; memextent_ptr_result_t m = cspace_lookup_memextent( @@ -134,6 +154,128 @@ hypercall_memextent_configure_derive(cap_id_t memextent_cap, out: return err; } + +static error_t +hypercall_memextent_donate_child(cap_id_t parent_cap, cap_id_t child_cap, + size_t offset, size_t size, bool reverse) +{ + error_t err = OK; + cspace_t *cspace = cspace_get_self(); + + memextent_ptr_result_t child = cspace_lookup_memextent( + cspace, child_cap, CAP_RIGHTS_MEMEXTENT_DONATE); + if (compiler_unexpected(child.e != OK)) { + err = child.e; + goto out; + } + + // We don't actually need a reference to the parent for the donate; the + // child already has a reference. So after sanity checking the provided + // parent cap we can immediately drop the reference. + if (child.r->parent != NULL) { + memextent_ptr_result_t m = cspace_lookup_memextent( + cspace, parent_cap, CAP_RIGHTS_MEMEXTENT_DONATE); + if (compiler_unexpected(m.e != OK)) { + err = m.e; + goto out_child_release; + } + + if (child.r->parent != m.r) { + err = ERROR_ARGUMENT_INVALID; + } + + object_put_memextent(m.r); + } else { + partition_ptr_result_t p = cspace_lookup_partition( + cspace, parent_cap, CAP_RIGHTS_PARTITION_DONATE); + if (compiler_unexpected(p.e != OK)) { + err = p.e; + goto out_child_release; + } + + if (child.r->header.partition != p.r) { + err = ERROR_ARGUMENT_INVALID; + } + + object_put_partition(p.r); + } + + if (err == OK) { + err = memextent_donate_child(child.r, offset, size, reverse); + } + +out_child_release: + object_put_memextent(child.r); +out: + return err; +} + +static error_t +hypercall_memextent_donate_sibling(cap_id_t from, cap_id_t to, size_t offset, + size_t size) +{ + error_t err; + cspace_t *cspace = cspace_get_self(); + + memextent_ptr_result_t m1 = cspace_lookup_memextent( + cspace, from, CAP_RIGHTS_MEMEXTENT_DONATE); + if (compiler_unexpected(m1.e != OK)) { + err = m1.e; + goto out; + } + + memextent_ptr_result_t m2 = cspace_lookup_memextent( + cspace, to, CAP_RIGHTS_MEMEXTENT_DONATE); + if (compiler_unexpected(m2.e != OK)) { + err = m2.e; + goto out_m1_release; + } + + err = memextent_donate_sibling(m1.r, m2.r, offset, size); + + object_put_memextent(m2.r); +out_m1_release: + object_put_memextent(m1.r); +out: + return err; +} + +error_t +hypercall_memextent_donate(memextent_donate_options_t options, cap_id_t from, + cap_id_t to, size_t offset, size_t size) +{ + error_t err; + + if (memextent_donate_options_get_res_0(&options) != 0U) { + err = ERROR_ARGUMENT_INVALID; + goto out; + } + + memextent_donate_type_t type = + memextent_donate_options_get_type(&options); + if (type == MEMEXTENT_DONATE_TYPE_TO_CHILD) { + err = hypercall_memextent_donate_child(from, to, offset, size, + false); + } else if (type == MEMEXTENT_DONATE_TYPE_TO_PARENT) { + err = hypercall_memextent_donate_child(to, from, offset, size, + true); + } else if (type == MEMEXTENT_DONATE_TYPE_TO_SIBLING) { + err = hypercall_memextent_donate_sibling(from, to, offset, + size); + } else { + err = ERROR_ARGUMENT_INVALID; + } + + if ((err == OK) && !memextent_donate_options_get_no_sync(&options)) { + // The donation may have caused addrspace mappings to change. + // Wait for completion of EL2 operations using manual lookups. + rcu_sync(); + } + +out: + return err; +} + #else extern int unused; #endif diff --git a/hyp/mem/memextent/src/memextent.c b/hyp/mem/memextent/src/memextent.c index 971edac..db6c3c8 100644 --- a/hyp/mem/memextent/src/memextent.c +++ b/hyp/mem/memextent/src/memextent.c @@ -4,8 +4,7 @@ #include #include - -#include +#include #include #include @@ -13,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +23,9 @@ #include #include +#include +#include + #include "event_handlers.h" error_t @@ -38,35 +41,21 @@ memextent_handle_object_create_memextent(memextent_create_t params) return OK; } -error_t -memextent_configure(memextent_t *me, paddr_t phys_base, size_t size, - memextent_attrs_t attributes) +static bool +memextent_validate_attrs(memextent_type_t type, memextent_memtype_t memtype, + pgtable_access_t access) { - error_t ret = OK; + bool ret = true; - assert(me != NULL); - - // The address range must not wrap around the end of the address space - if ((size == 0U) || util_add_overflows(phys_base, size - 1U)) { - ret = ERROR_ARGUMENT_INVALID; - goto out; - } - - if (!util_is_baligned(phys_base, PGTABLE_VM_PAGE_SIZE) || - !util_is_baligned(size, PGTABLE_VM_PAGE_SIZE)) { - ret = ERROR_ARGUMENT_ALIGNMENT; - goto out; - } - - // Only basic memory extents are implemented - if ((memextent_attrs_get_res_0(&attributes) != 0U) || - (memextent_attrs_get_append(&attributes))) { - ret = ERROR_ARGUMENT_INVALID; + switch (type) { + case MEMEXTENT_TYPE_BASIC: + case MEMEXTENT_TYPE_SPARSE: + break; + default: + ret = false; goto out; } - // Validate arguments - memextent_memtype_t memtype = memextent_attrs_get_memtype(&attributes); switch (memtype) { case MEMEXTENT_MEMTYPE_ANY: case MEMEXTENT_MEMTYPE_DEVICE: @@ -80,10 +69,10 @@ memextent_configure(memextent_t *me, paddr_t phys_base, size_t size, case MEMEXTENT_MEMTYPE_CACHED: #endif default: - ret = ERROR_ARGUMENT_INVALID; + ret = false; goto out; } - pgtable_access_t access = memextent_attrs_get_access(&attributes); + switch (access) { case PGTABLE_ACCESS_X: case PGTABLE_ACCESS_W: @@ -94,11 +83,49 @@ memextent_configure(memextent_t *me, paddr_t phys_base, size_t size, break; case PGTABLE_ACCESS_NONE: default: + ret = false; + break; + } + +out: + return ret; +} + +error_t +memextent_configure(memextent_t *me, paddr_t phys_base, size_t size, + memextent_attrs_t attributes) +{ + error_t ret = OK; + + assert(me != NULL); + + // The address range must not wrap around the end of the address space + if ((size == 0U) || util_add_overflows(phys_base, size - 1U)) { ret = ERROR_ARGUMENT_INVALID; goto out; } - me->type = MEMEXTENT_TYPE_BASIC; + if (!util_is_baligned(phys_base, PGTABLE_VM_PAGE_SIZE) || + !util_is_baligned(size, PGTABLE_VM_PAGE_SIZE)) { + ret = ERROR_ARGUMENT_ALIGNMENT; + goto out; + } + + if ((memextent_attrs_get_res_0(&attributes) != 0U) || + (memextent_attrs_get_append(&attributes))) { + ret = ERROR_ARGUMENT_INVALID; + goto out; + } + + memextent_type_t type = memextent_attrs_get_type(&attributes); + memextent_memtype_t memtype = memextent_attrs_get_memtype(&attributes); + pgtable_access_t access = memextent_attrs_get_access(&attributes); + if (!memextent_validate_attrs(type, memtype, access)) { + ret = ERROR_ARGUMENT_INVALID; + goto out; + } + + me->type = type; me->phys_base = phys_base; me->size = size; me->memtype = memtype; @@ -113,6 +140,7 @@ memextent_configure(memextent_t *me, paddr_t phys_base, size_t size, return ret; } +// FIXME: error_t memextent_configure_derive(memextent_t *me, memextent_t *parent, size_t offset, size_t size, memextent_attrs_t attributes) @@ -143,55 +171,34 @@ memextent_configure_derive(memextent_t *me, memextent_t *parent, size_t offset, goto out; } - // Validate arguments - memextent_memtype_t memtype = memextent_attrs_get_memtype(&attributes); - switch (memtype) { - case MEMEXTENT_MEMTYPE_ANY: - case MEMEXTENT_MEMTYPE_DEVICE: - case MEMEXTENT_MEMTYPE_UNCACHED: -#if defined(ARCH_AARCH64_USE_S2FWB) - case MEMEXTENT_MEMTYPE_CACHED: -#endif - break; -#if !defined(ARCH_AARCH64_USE_S2FWB) - // Without S2FWB, we cannot force cached mappings - case MEMEXTENT_MEMTYPE_CACHED: -#endif - default: + if ((memextent_attrs_get_res_0(&attributes) != 0U) || + (memextent_attrs_get_append(&attributes))) { ret = ERROR_ARGUMENT_INVALID; goto out; } - pgtable_access_t access = memextent_attrs_get_access(&attributes); - switch (access) { - case PGTABLE_ACCESS_X: - case PGTABLE_ACCESS_W: - case PGTABLE_ACCESS_R: - case PGTABLE_ACCESS_RX: - case PGTABLE_ACCESS_RW: - case PGTABLE_ACCESS_RWX: - break; - case PGTABLE_ACCESS_NONE: - default: + memextent_type_t type = memextent_attrs_get_type(&attributes); + memextent_memtype_t memtype = memextent_attrs_get_memtype(&attributes); + pgtable_access_t access = memextent_attrs_get_access(&attributes); + if (!memextent_validate_attrs(type, memtype, access)) { ret = ERROR_ARGUMENT_INVALID; goto out; } - if ((parent->access & access) != access) { + if (!pgtable_access_check(parent->access, access)) { ret = ERROR_ARGUMENT_INVALID; goto out; } - // Only basic memory extents are implemented - if ((memextent_attrs_get_res_0(&attributes) != 0U) || - (memextent_attrs_get_append(&attributes))) { + if ((parent->memtype != MEMEXTENT_MEMTYPE_ANY) && + (parent->memtype != memtype)) { ret = ERROR_ARGUMENT_INVALID; goto out; } paddr_t phys_base = parent->phys_base + offset; - me->type = MEMEXTENT_TYPE_BASIC; + me->type = type; me->phys_base = phys_base; me->size = size; me->memtype = memtype; @@ -252,44 +259,7 @@ memextent_handle_object_activate_memextent(memextent_t *me) goto out; } - // memextent should have been zero initialized - for (index_t i = 0; i < util_array_size(me->mappings); i++) { - assert(atomic_load_relaxed( - &me->mappings[i].addrspace) == NULL); - } - - partition_t *partition = me->header.partition; - - partition_t *hyp_partition = partition_get_private(); - - if (me->device_mem) { - assert(me->memtype == MEMEXTENT_MEMTYPE_DEVICE); - - ret = memdb_insert(hyp_partition, me->phys_base, - me->phys_base + (me->size - 1U), - (uintptr_t)me, MEMDB_TYPE_EXTENT); - } else { - ret = memdb_update(hyp_partition, me->phys_base, - me->phys_base + (me->size - 1U), - (uintptr_t)me, MEMDB_TYPE_EXTENT, - (uintptr_t)partition, - MEMDB_TYPE_PARTITION); - - if (ret == ERROR_MEMDB_NOT_OWNER) { - // We might have failed to take ownership - // because a previously deleted memextent has - // not yet been cleaned up, so wait for an RCU - // grace period and then retry. If it still - // fails after that, there's a real conflict. - rcu_sync(); - ret = memdb_update( - hyp_partition, me->phys_base, - me->phys_base + (me->size - 1U), - (uintptr_t)me, MEMDB_TYPE_EXTENT, - (uintptr_t)partition, - MEMDB_TYPE_PARTITION); - } - } + ret = trigger_memextent_activate_event(me->type, me); } if (ret == OK) { @@ -299,12 +269,104 @@ memextent_handle_object_activate_memextent(memextent_t *me) return ret; } +bool +memextent_supports_donation(memextent_t *me) +{ + return trigger_memextent_supports_donation_event(me->type, me); +} + +static bool +extent_range_valid(memextent_t *me, paddr_t phys, size_t size) +{ + assert(!util_add_overflows(phys, size - 1U)); + + return (me->phys_base <= phys) && + ((me->phys_base + (me->size - 1U)) >= (phys + (size - 1U))); +} + error_t -memextent_map(memextent_t *extent, addrspace_t *addrspace, vmaddr_t vm_base, - memextent_mapping_attrs_t map_attrs) +memextent_donate_child(memextent_t *me, size_t offset, size_t size, + bool reverse) { error_t ret; + if (!util_is_baligned(offset, PGTABLE_VM_PAGE_SIZE) || + !util_is_baligned(size, PGTABLE_VM_PAGE_SIZE)) { + ret = ERROR_ARGUMENT_ALIGNMENT; + goto out; + } + + if (util_add_overflows(me->phys_base, offset)) { + ret = ERROR_ARGUMENT_INVALID; + goto out; + } + + paddr_t phys = me->phys_base + offset; + + if ((size == 0U) || util_add_overflows(phys, size - 1U)) { + ret = ERROR_ARGUMENT_SIZE; + goto out; + } + + if (!extent_range_valid(me, phys, size)) { + ret = ERROR_ARGUMENT_INVALID; + goto out; + } + + ret = trigger_memextent_donate_child_event(me->type, me, phys, size, + reverse); + +out: + return ret; +} + +error_t +memextent_donate_sibling(memextent_t *from, memextent_t *to, size_t offset, + size_t size) +{ + error_t ret; + + if (!util_is_baligned(offset, PGTABLE_VM_PAGE_SIZE) || + !util_is_baligned(size, PGTABLE_VM_PAGE_SIZE)) { + ret = ERROR_ARGUMENT_ALIGNMENT; + goto out; + } + + if (util_add_overflows(from->phys_base, offset)) { + ret = ERROR_ARGUMENT_INVALID; + goto out; + } + + paddr_t phys = from->phys_base + offset; + + if ((size == 0U) || util_add_overflows(phys, size - 1U)) { + ret = ERROR_ARGUMENT_SIZE; + goto out; + } + + if (!extent_range_valid(from, phys, size) || + !extent_range_valid(to, phys, size)) { + ret = ERROR_ARGUMENT_INVALID; + goto out; + } + + if ((from == to) || (from->parent == NULL) || + (from->parent != to->parent)) { + ret = ERROR_ARGUMENT_INVALID; + goto out; + } + + ret = trigger_memextent_donate_sibling_event(from->type, from, to, phys, + size); + +out: + return ret; +} + +static bool +memextent_check_map_attrs(memextent_t *extent, + memextent_mapping_attrs_t map_attrs) +{ pgtable_access_t access_user = memextent_mapping_attrs_get_user_access(&map_attrs); pgtable_access_t access_kernel = @@ -312,25 +374,28 @@ memextent_map(memextent_t *extent, addrspace_t *addrspace, vmaddr_t vm_base, pgtable_vm_memtype_t memtype = memextent_mapping_attrs_get_memtype(&map_attrs); - // Check validity of access rights and memtype + return (pgtable_access_check(extent->access, access_user)) && + pgtable_access_check(extent->access, access_kernel) && + memextent_check_memtype(extent->memtype, memtype); +} - if (((access_user & ~extent->access) != 0U) || - ((access_kernel & ~extent->access) != 0U)) { - ret = ERROR_ARGUMENT_INVALID; - goto out; - } +error_t +memextent_map(memextent_t *extent, addrspace_t *addrspace, vmaddr_t vm_base, + memextent_mapping_attrs_t map_attrs) +{ + error_t ret; if (!util_is_baligned(vm_base, PGTABLE_VM_PAGE_SIZE)) { ret = ERROR_ARGUMENT_ALIGNMENT; goto out; } - if (!memextent_check_memtype(extent->memtype, memtype)) { + if (!memextent_check_map_attrs(extent, map_attrs)) { ret = ERROR_ARGUMENT_INVALID; goto out; } - if (addrspace->vm_read_only) { + if (addrspace->read_only) { ret = ERROR_DENIED; } else { ret = trigger_memextent_map_event( @@ -341,6 +406,49 @@ memextent_map(memextent_t *extent, addrspace_t *addrspace, vmaddr_t vm_base, return ret; } +error_t +memextent_map_partial(memextent_t *extent, addrspace_t *addrspace, + vmaddr_t vm_base, size_t offset, size_t size, + memextent_mapping_attrs_t map_attrs) +{ + error_t ret; + + if (!util_is_baligned(vm_base, PGTABLE_VM_PAGE_SIZE) || + !util_is_baligned(offset, PGTABLE_VM_PAGE_SIZE) || + !util_is_baligned(size, PGTABLE_VM_PAGE_SIZE)) { + ret = ERROR_ARGUMENT_ALIGNMENT; + goto out; + } + + if ((size == 0U) || util_add_overflows(offset, size - 1U) || + util_add_overflows(vm_base, size - 1U)) { + ret = ERROR_ARGUMENT_SIZE; + goto out; + } + + if ((offset + (size - 1U)) >= extent->size) { + ret = ERROR_ARGUMENT_SIZE; + goto out; + } + + if (!memextent_check_map_attrs(extent, map_attrs)) { + ret = ERROR_ARGUMENT_INVALID; + goto out; + } + + if (addrspace->read_only) { + ret = ERROR_DENIED; + } else { + ret = trigger_memextent_map_partial_event(extent->type, extent, + addrspace, vm_base, + offset, size, + map_attrs); + } + +out: + return ret; +} + error_t memextent_unmap(memextent_t *extent, addrspace_t *addrspace, vmaddr_t vm_base) { @@ -351,7 +459,7 @@ memextent_unmap(memextent_t *extent, addrspace_t *addrspace, vmaddr_t vm_base) goto out; } - if (addrspace->vm_read_only) { + if (addrspace->read_only) { ret = ERROR_DENIED; } else { ret = trigger_memextent_unmap_event(extent->type, extent, @@ -362,26 +470,127 @@ memextent_unmap(memextent_t *extent, addrspace_t *addrspace, vmaddr_t vm_base) return ret; } +error_t +memextent_unmap_partial(memextent_t *extent, addrspace_t *addrspace, + vmaddr_t vm_base, size_t offset, size_t size) +{ + error_t ret; + + if (!util_is_baligned(vm_base, PGTABLE_VM_PAGE_SIZE) || + !util_is_baligned(offset, PGTABLE_VM_PAGE_SIZE) || + !util_is_baligned(size, PGTABLE_VM_PAGE_SIZE)) { + ret = ERROR_ARGUMENT_ALIGNMENT; + goto out; + } + + if ((size == 0U) || util_add_overflows(offset, size - 1U) || + util_add_overflows(vm_base, size - 1U)) { + ret = ERROR_ARGUMENT_SIZE; + goto out; + } + + if ((offset + (size - 1U)) >= extent->size) { + ret = ERROR_ARGUMENT_SIZE; + goto out; + } + + if (addrspace->read_only) { + ret = ERROR_DENIED; + } else { + ret = trigger_memextent_unmap_partial_event( + extent->type, extent, addrspace, vm_base, offset, size); + } + +out: + return ret; +} + void memextent_unmap_all(memextent_t *extent) { - trigger_memextent_unmap_all_event(extent->type, extent); + if (!trigger_memextent_unmap_all_event(extent->type, extent)) { + panic("Invalid memory extent unmap all!"); + } +} + +static error_t +memextent_do_zero(paddr_t base, size_t size, void *arg) +{ + void *addr = partition_phys_map(base, size); + partition_phys_access_enable(addr); + + (void)memset_s(addr, size, 0, size); + CACHE_CLEAN_RANGE((uint8_t *)addr, size); + + partition_phys_access_disable(addr); + partition_phys_unmap(addr, base, size); + + (void)arg; + + return OK; } error_t -memextent_update_access(memextent_t *extent, addrspace_t *addrspace, - vmaddr_t vm_base, memextent_access_attrs_t access_attrs) +memextent_zero_range(memextent_t *extent, size_t offset, size_t size) { - error_t ret; + error_t err; + + if ((extent->memtype == MEMEXTENT_MEMTYPE_DEVICE) || + !pgtable_access_check(extent->access, PGTABLE_ACCESS_W)) { + err = ERROR_DENIED; + goto out; + } + + if (!util_is_baligned(offset, PGTABLE_VM_PAGE_SIZE) || + !util_is_baligned(size, PGTABLE_VM_PAGE_SIZE)) { + err = ERROR_ARGUMENT_ALIGNMENT; + goto out; + } + + if (util_add_overflows(extent->phys_base, offset)) { + err = ERROR_ARGUMENT_INVALID; + goto out; + } + + paddr_t phys = extent->phys_base + offset; + + if ((size == 0U) || util_add_overflows(phys, size - 1U)) { + err = ERROR_ARGUMENT_SIZE; + goto out; + } + + if (!extent_range_valid(extent, phys, size)) { + err = ERROR_ARGUMENT_INVALID; + goto out; + } + + err = memdb_range_walk((uintptr_t)extent, MEMDB_TYPE_EXTENT, phys, + phys + size - 1U, memextent_do_zero, NULL); +out: + return err; +} + +static bool +memextent_check_access_attrs(memextent_t *extent, + memextent_access_attrs_t access_attrs) +{ pgtable_access_t access_user = memextent_access_attrs_get_user_access(&access_attrs); pgtable_access_t access_kernel = memextent_access_attrs_get_kernel_access(&access_attrs); - // Check validity of access rights - if (((access_user & ~extent->access) != 0U) || - ((access_kernel & ~extent->access) != 0U)) { + return (pgtable_access_check(extent->access, access_user) && + pgtable_access_check(extent->access, access_kernel)); +} + +error_t +memextent_update_access(memextent_t *extent, addrspace_t *addrspace, + vmaddr_t vm_base, memextent_access_attrs_t access_attrs) +{ + error_t ret; + + if (!memextent_check_access_attrs(extent, access_attrs)) { ret = ERROR_ARGUMENT_INVALID; goto out; } @@ -391,7 +600,7 @@ memextent_update_access(memextent_t *extent, addrspace_t *addrspace, goto out; } - if (addrspace->vm_read_only) { + if (addrspace->read_only) { ret = ERROR_DENIED; } else { ret = trigger_memextent_update_access_event( @@ -402,36 +611,72 @@ memextent_update_access(memextent_t *extent, addrspace_t *addrspace, return ret; } -void -memextent_handle_object_deactivate_memextent(memextent_t *memextent) +error_t +memextent_update_access_partial(memextent_t *extent, addrspace_t *addrspace, + vmaddr_t vm_base, size_t offset, size_t size, + memextent_access_attrs_t access_attrs) { - trigger_memextent_deactivate_event(memextent->type, memextent); + error_t ret; + + if (!memextent_check_access_attrs(extent, access_attrs)) { + ret = ERROR_ARGUMENT_INVALID; + goto out; + } + + if (!util_is_baligned(vm_base, PGTABLE_VM_PAGE_SIZE) || + !util_is_baligned(offset, PGTABLE_VM_PAGE_SIZE) || + !util_is_baligned(size, PGTABLE_VM_PAGE_SIZE)) { + ret = ERROR_ARGUMENT_ALIGNMENT; + goto out; + } + + if ((size == 0U) || util_add_overflows(offset, size - 1U) || + util_add_overflows(vm_base, size - 1U)) { + ret = ERROR_ARGUMENT_SIZE; + goto out; + } + + if ((offset + (size - 1U)) >= extent->size) { + ret = ERROR_ARGUMENT_SIZE; + goto out; + } + + if (addrspace->read_only) { + ret = ERROR_DENIED; + } else { + ret = trigger_memextent_update_access_partial_event( + extent->type, extent, addrspace, vm_base, offset, size, + access_attrs); + } + +out: + return ret; } -void -memextent_handle_object_deactivate_addrspace(addrspace_t *addrspace) +bool +memextent_is_mapped(memextent_t *me, addrspace_t *addrspace, bool exclusive) { + assert(me != NULL); assert(addrspace != NULL); - spinlock_acquire(&addrspace->mapping_list_lock); - - list_t *list = &addrspace->mapping_list; - memextent_mapping_t *map = NULL; + return trigger_memextent_is_mapped_event(me->type, me, addrspace, + exclusive); +} - // Remove all mappings from addrspace - list_foreach_container_maydelete (map, list, memextent_mapping, - mapping_list_node) { - atomic_store_relaxed(&map->addrspace, NULL); - list_delete_node(list, &map->mapping_list_node); +void +memextent_handle_object_deactivate_memextent(memextent_t *memextent) +{ + if (!trigger_memextent_deactivate_event(memextent->type, memextent)) { + panic("Invalid memory extent deactivate!"); } - - spinlock_release(&addrspace->mapping_list_lock); } void memextent_handle_object_cleanup_memextent(memextent_t *memextent) { - trigger_memextent_cleanup_event(memextent->type, memextent); + if (!trigger_memextent_cleanup_event(memextent->type, memextent)) { + panic("Invalid memory extent cleanup!"); + } if (memextent->parent != NULL) { object_put_memextent(memextent->parent); @@ -439,9 +684,16 @@ memextent_handle_object_cleanup_memextent(memextent_t *memextent) } } +size_result_t +memextent_get_offset_for_pa(memextent_t *memextent, paddr_t pa, size_t size) +{ + return trigger_memextent_get_offset_for_pa_event(memextent->type, + memextent, pa, size); +} + #if defined(ARCH_AARCH64_USE_S2FWB) -#if !defined(ARCH_ARM_8_4_S2FWB) -#error S2FWB requires ARCH_ARM_8_4_S2FWB +#if !defined(ARCH_ARM_FEAT_S2FWB) +#error S2FWB requires ARCH_ARM_FEAT_S2FWB #endif #error S2FWB support not implemented #endif @@ -471,16 +723,17 @@ memextent_check_memtype(memextent_memtype_t extent_type, success = true; } break; - case PGTABLE_VM_MEMTYPE_NORMAL_WB: + case PGTABLE_VM_MEMTYPE_NORMAL_WB: { #if defined(ARCH_AARCH64_USE_S2FWB) if ((extent_type == MEMEXTENT_MEMTYPE_ANY) || - (extent_type == MEMEXTENT_MEMTYPE_CACHED)) { + (extent_type == MEMEXTENT_MEMTYPE_CACHED)) #else - if (extent_type == MEMEXTENT_MEMTYPE_ANY) { + if (extent_type == MEMEXTENT_MEMTYPE_ANY) #endif + { success = true; } - break; + } break; case PGTABLE_VM_MEMTYPE_NORMAL_WT: case PGTABLE_VM_MEMTYPE_NORMAL_OWT_IWB: case PGTABLE_VM_MEMTYPE_NORMAL_OWB_INC: @@ -494,6 +747,7 @@ memextent_check_memtype(memextent_memtype_t extent_type, break; default: success = false; + break; } return success; @@ -512,7 +766,7 @@ memextent_derive(memextent_t *parent, paddr_t offset, size_t size, goto out; } - memextent_t *me = me_ret.r; + memextent_t *me = me_ret.r; memextent_attrs_t attrs = memextent_attrs_default(); memextent_attrs_set_access(&attrs, access); memextent_attrs_set_memtype(&attrs, memtype); @@ -522,6 +776,7 @@ memextent_derive(memextent_t *parent, paddr_t offset, size_t size, me_ret.e = memextent_configure_derive(me, parent, offset, size, attrs); if (me_ret.e != OK) { spinlock_release(&me->header.lock); + me_ret.r = NULL; object_put_memextent(me); goto out; } @@ -530,8 +785,91 @@ memextent_derive(memextent_t *parent, paddr_t offset, size_t size, me_ret.e = object_activate_memextent(me); if (me_ret.e != OK) { object_put_memextent(me); + me_ret.r = NULL; } out: return me_ret; } + +void +memextent_retain_mappings(memextent_t *me) LOCK_IMPL +{ + (void)trigger_memextent_retain_mappings_event(me->type, me); +} + +void +memextent_release_mappings(memextent_t *me, bool clear) LOCK_IMPL +{ + (void)trigger_memextent_release_mappings_event(me->type, me, clear); +} + +memextent_mapping_t +memextent_lookup_mapping(memextent_t *me, paddr_t phys, size_t size, index_t i) +{ + memextent_mapping_result_t ret; + + ret = trigger_memextent_lookup_mapping_event(me->type, me, phys, size, + i); + assert(ret.e == OK); + + return ret.r; +} + +error_t +memextent_attach(partition_t *owner, memextent_t *me, uintptr_t hyp_va, + size_t size) +{ + assert(owner != NULL); + assert(me != NULL); + + error_t ret; + + if (owner != me->header.partition) { + ret = ERROR_DENIED; + goto out; + } + + if (!pgtable_access_check(me->access, PGTABLE_ACCESS_RW)) { + ret = ERROR_DENIED; + goto out; + } + + if (me->size < size) { + ret = ERROR_ARGUMENT_SIZE; + goto out; + } + + pgtable_hyp_memtype_t memtype; + switch (me->memtype) { + case MEMEXTENT_MEMTYPE_CACHED: + case MEMEXTENT_MEMTYPE_ANY: + memtype = PGTABLE_HYP_MEMTYPE_WRITEBACK; + break; + case MEMEXTENT_MEMTYPE_DEVICE: + memtype = PGTABLE_HYP_MEMTYPE_DEVICE; + break; + case MEMEXTENT_MEMTYPE_UNCACHED: + memtype = PGTABLE_HYP_MEMTYPE_WRITECOMBINE; + break; + default: + ret = ERROR_ARGUMENT_INVALID; + goto out; + } + + ret = trigger_memextent_attach_event(me->type, me, hyp_va, size, + memtype); +out: + return ret; +} + +void +memextent_detach(partition_t *owner, memextent_t *me) +{ + assert(owner != NULL); + assert(me != NULL); + assert(owner == me->header.partition); + + bool handled = trigger_memextent_detach_event(me->type, me); + assert(handled); +} diff --git a/hyp/mem/memextent/src/memextent_basic.c b/hyp/mem/memextent/src/memextent_basic.c index cd3a692..f2846d9 100644 --- a/hyp/mem/memextent/src/memextent_basic.c +++ b/hyp/mem/memextent/src/memextent_basic.c @@ -6,6 +6,9 @@ #include #include +#include + +#include #include #include #include @@ -24,68 +27,164 @@ #include "event_handlers.h" -// Needs to be called holding a reference to the addrspace to be used static error_t -memextent_do_map(memextent_t *me, memextent_mapping_t *map, size_t offset, - size_t size) +allocate_mappings(memextent_t *me) { - assert((me != NULL) && (map != NULL)); + error_t ret = OK; + partition_t *partition = me->header.partition; + const size_t alloc_size = + sizeof(memextent_basic_mapping_t) * MEMEXTENT_MAX_MAPS; + const size_t alloc_align = alignof(memextent_basic_mapping_t); + + void_ptr_result_t alloc_ret = + partition_alloc(partition, alloc_size, alloc_align); + if (alloc_ret.e != OK) { + ret = alloc_ret.e; + goto out; + } + + (void)memset_s(alloc_ret.r, alloc_size, 0, alloc_size); + + me->mappings.basic = alloc_ret.r; + +out: + return ret; +} - addrspace_t *const s = atomic_load_consume(&map->addrspace); - assert((s != NULL) && !s->vm_read_only); +static void +free_mappings(memextent_t *me) +{ + partition_t *partition = me->header.partition; + const size_t alloc_size = + sizeof(memextent_basic_mapping_t) * MEMEXTENT_MAX_MAPS; - spinlock_acquire(&s->pgtable_lock); + assert(me->mappings.basic != NULL); + (void)partition_free(partition, me->mappings.basic, alloc_size); + + me->mappings.basic = NULL; +} + +// Needs to be called holding a reference to the addrspace to be used +static error_t +memextent_do_map(memextent_t *me, memextent_basic_mapping_t *map, size_t offset, + size_t size) +{ + assert((me != NULL) && (map != NULL)); assert((size > 0U) && (size <= me->size)); assert(!util_add_overflows(me->phys_base, offset)); assert(!util_add_overflows(map->vbase, offset)); assert(!util_add_overflows(me->phys_base + offset, size - 1U)); assert(!util_add_overflows(map->vbase + offset, size - 1U)); - pgtable_vm_start(&s->vm_pgtable); + addrspace_t *const s = atomic_load_relaxed(&map->addrspace); + assert((s != NULL) && !s->read_only); - // We do not set the try_map option, as we want to do the mapping even - // if the specified range has already been mapped - error_t ret = pgtable_vm_map( - s->header.partition, &s->vm_pgtable, map->vbase + offset, size, - me->phys_base + offset, + return addrspace_map( + s, map->vbase + offset, size, me->phys_base + offset, memextent_mapping_attrs_get_memtype(&map->attrs), memextent_mapping_attrs_get_kernel_access(&map->attrs), - memextent_mapping_attrs_get_user_access(&map->attrs), false); + memextent_mapping_attrs_get_user_access(&map->attrs)); +} - pgtable_vm_commit(&s->vm_pgtable); +// Needs to be called holding a reference to the addrspace to be used +static void +memextent_remove_map_from_addrspace_list(memextent_basic_mapping_t *map) +{ + assert(map != NULL); + addrspace_t *as = atomic_load_relaxed(&map->addrspace); + assert(as != NULL); + + spinlock_acquire(&as->mapping_list_lock); + (void)list_delete_node(&as->basic_mapping_list, + &map->mapping_list_node); + spinlock_release(&as->mapping_list_lock); + + atomic_store_relaxed(&map->addrspace, NULL); +} + +error_t +memextent_activate_basic(memextent_t *me) +{ + error_t ret; + partition_t *hyp_partition = partition_get_private(); + + assert(me != NULL); + assert(hyp_partition != NULL); + + ret = allocate_mappings(me); if (ret != OK) { - goto error; + goto out; } -error: - spinlock_release(&s->pgtable_lock); + if (me->device_mem) { + assert(me->memtype == MEMEXTENT_MEMTYPE_DEVICE); + + ret = memdb_insert(hyp_partition, me->phys_base, + me->phys_base + (me->size - 1U), + (uintptr_t)me, MEMDB_TYPE_EXTENT); + } else { + partition_t *partition = me->header.partition; + assert(partition != NULL); + + ret = memdb_update(hyp_partition, me->phys_base, + me->phys_base + (me->size - 1U), + (uintptr_t)me, MEMDB_TYPE_EXTENT, + (uintptr_t)partition, MEMDB_TYPE_PARTITION); + + if (ret == ERROR_MEMDB_NOT_OWNER) { + // We might have failed to take ownership + // because a previously deleted memextent has + // not yet been cleaned up, so wait for an RCU + // grace period and then retry. If it still + // fails after that, there's a real conflict. + rcu_sync(); + ret = memdb_update(hyp_partition, me->phys_base, + me->phys_base + (me->size - 1U), + (uintptr_t)me, MEMDB_TYPE_EXTENT, + (uintptr_t)partition, + MEMDB_TYPE_PARTITION); + } + } + if (ret != OK) { + free_mappings(me); + } + +out: return ret; } error_t memextent_activate_derive_basic(memextent_t *me) { - error_t ret = OK; + error_t ret = OK; + partition_t *hyp_partition = partition_get_private(); assert(me != NULL); assert(me->parent != NULL); - assert(me->parent->type == MEMEXTENT_TYPE_BASIC); + + ret = allocate_mappings(me); + if (ret != OK) { + goto out; + } bool retried = false; while (1) { spinlock_acquire(&me->parent->lock); + if (me->parent->attached_size != 0U) { + ret = ERROR_BUSY; + goto out_locked_parent; + } + // Take the mapping lock before the memdb update, because we // haven't set up the mapping pointers yet. We do that after the // memdb update so we don't have to undo them if the memdb // update fails. spinlock_acquire_nopreempt(&me->lock); - partition_t *hyp_partition = partition_get_private(); - ret = memdb_update(hyp_partition, me->phys_base, me->phys_base + (me->size - 1U), (uintptr_t)me, MEMDB_TYPE_EXTENT, @@ -107,45 +206,32 @@ memextent_activate_derive_basic(memextent_t *me) retried = true; } - size_t offset = me->phys_base - me->parent->phys_base; + memextent_retain_mappings(me->parent); - for (index_t i = 0; i < util_array_size(me->mappings); i++) { - memextent_mapping_t *map = &me->mappings[i]; - const memextent_mapping_t *parent_map = - &me->parent->mappings[i]; + for (index_t i = 0U; (i < MEMEXTENT_MAX_MAPS); i++) { + memextent_basic_mapping_t *map = &me->mappings.basic[i]; - // RCU protects ->addrspace - rcu_read_start(); - addrspace_t *as = atomic_load_consume(&parent_map->addrspace); - if (as == NULL) { - memset((void *)map, 0U, sizeof(map)); - rcu_read_finish(); - continue; + memextent_mapping_t parent_map = memextent_lookup_mapping( + me->parent, me->phys_base, me->size, i); + if (parent_map.size != me->size) { + // The parent is partially mapped over the child's + // range; we cannot handle this with a basic memextent. + ret = ERROR_DENIED; + break; } - assert(!util_add_overflows(parent_map->vbase, offset)); - - vmaddr_t vbase = parent_map->vbase + offset; - - assert(!util_add_overflows(vbase, me->size - 1U)); - - // Take a reference to the address space to ensure that - // we don't race with its destruction. - if (!object_get_addrspace_safe(as)) { - // Either there is no mapping, or the address space is - // in the process of being deleted. - memset((void *)map, 0U, sizeof(map)); - rcu_read_finish(); + addrspace_t *as = parent_map.addrspace; + if (as == NULL) { continue; } - rcu_read_finish(); - *map = *parent_map; - - map->vbase = vbase; + atomic_store_relaxed(&map->addrspace, as); + map->vbase = parent_map.vbase; + map->attrs = parent_map.attrs; spinlock_acquire_nopreempt(&as->mapping_list_lock); - list_insert_at_head(&as->mapping_list, &map->mapping_list_node); + list_insert_at_head(&as->basic_mapping_list, + &map->mapping_list_node); spinlock_release_nopreempt(&as->mapping_list_lock); pgtable_access_t access_user = @@ -155,62 +241,93 @@ memextent_activate_derive_basic(memextent_t *me) // Reduce access rights on the map memextent_mapping_attrs_set_user_access( - &map->attrs, access_user & me->access); + &map->attrs, + pgtable_access_mask(access_user, me->access)); memextent_mapping_attrs_set_kernel_access( - &map->attrs, access_kernel & me->access); + &map->attrs, + pgtable_access_mask(access_kernel, me->access)); // If accesses are the same then mapping can be inherited from // parent, if not, remap memextent to update access. - if (memextent_mapping_attrs_raw(map->attrs) == - memextent_mapping_attrs_raw(parent_map->attrs)) { - object_put_addrspace(as); - continue; + if (!memextent_mapping_attrs_is_equal(map->attrs, + parent_map.attrs)) { + ret = memextent_do_map(me, map, 0, me->size); + if (ret != OK) { + memextent_remove_map_from_addrspace_list(map); + break; + } } + } - ret = memextent_do_map(me, map, 0, me->size); - if (ret != OK) { - panic("unhandled memextent remap failure"); + if (ret != OK) { + // Revert any remappings that were made. + error_t err; + for (index_t i = 0; i < MEMEXTENT_MAX_MAPS; i++) { + memextent_basic_mapping_t *map = &me->mappings.basic[i]; + + addrspace_t *as = atomic_load_relaxed(&map->addrspace); + if (as == NULL) { + continue; + } + + memextent_mapping_t parent_map = + memextent_lookup_mapping( + me->parent, me->phys_base, me->size, i); + assert(as == parent_map.addrspace); + + if (!memextent_mapping_attrs_is_equal( + map->attrs, parent_map.attrs)) { + map->attrs = parent_map.attrs; + + err = memextent_do_map(me, map, 0, me->size); + assert(err == OK); + } + + memextent_remove_map_from_addrspace_list(map); } - object_put_addrspace(as); + // Revert the earlier memdb update. + err = memdb_update(hyp_partition, me->phys_base, + me->phys_base + (me->size - 1U), + (uintptr_t)me->parent, MEMDB_TYPE_EXTENT, + (uintptr_t)me, MEMDB_TYPE_EXTENT); + assert(err == OK); } + memextent_release_mappings(me->parent, false); + list_insert_at_head(&me->parent->children_list, &me->children_list_node); out_locked: spinlock_release_nopreempt(&me->lock); +out_locked_parent: spinlock_release(&me->parent->lock); + if (ret != OK) { + free_mappings(me); + } + +out: return ret; } // Needs to be called holding a reference to the addrspace to be used static void -memextent_do_unmap(memextent_t *me, memextent_mapping_t *map, size_t offset, - size_t size) +memextent_do_unmap(memextent_t *me, memextent_basic_mapping_t *map, + size_t offset, size_t size) { assert((me != NULL) && (map != NULL)); - - addrspace_t *const s = atomic_load_consume(&map->addrspace); - assert((s != NULL) && !s->vm_read_only); - - spinlock_acquire(&s->pgtable_lock); - assert((size > 0U) && (size <= me->size)); assert(!util_add_overflows(map->vbase, offset)); assert(!util_add_overflows(map->vbase + offset, size - 1U)); - pgtable_vm_start(&s->vm_pgtable); + addrspace_t *const s = atomic_load_relaxed(&map->addrspace); + assert((s != NULL) && !s->read_only); - // Unmap only matching physical addresses - pgtable_vm_unmap_matching(s->header.partition, &s->vm_pgtable, - map->vbase + offset, me->phys_base + offset, - size); - - pgtable_vm_commit(&s->vm_pgtable); - - spinlock_release(&s->pgtable_lock); + error_t err = addrspace_unmap(s, map->vbase + offset, size, + me->phys_base + offset); + assert(err == OK); } static error_t @@ -218,20 +335,20 @@ memextent_map_range(paddr_t base, size_t size, void *arg) { error_t ret = OK; - if ((size == 0U) && (util_add_overflows(base, size - 1))) { + if ((size == 0U) || (util_add_overflows(base, size - 1))) { ret = ERROR_ARGUMENT_SIZE; goto error; } assert(arg != NULL); - memextent_arg_t *args = (memextent_arg_t *)arg; + memextent_basic_arg_t *args = (memextent_basic_arg_t *)arg; assert((args->me != NULL) && (args->map[0] != NULL)); size_t offset = base - args->me->phys_base; - ret = memextent_do_map(args->me, args->map[0], offset, size); + ret = memextent_do_map(args->me, args->map[0], offset, size); if (ret != OK) { args->failed_address = base; } @@ -245,14 +362,14 @@ memextent_unmap_range(paddr_t base, size_t size, void *arg) { error_t ret = OK; - if ((size == 0U) && (util_add_overflows(base, size - 1))) { + if ((size == 0U) || (util_add_overflows(base, size - 1))) { ret = ERROR_ARGUMENT_SIZE; goto error; } assert(arg != NULL); - memextent_arg_t *args = (memextent_arg_t *)arg; + memextent_basic_arg_t *args = (memextent_basic_arg_t *)arg; assert((args->me != NULL) && (args->map[0] != NULL)); @@ -274,7 +391,8 @@ memextent_map_basic(memextent_t *me, addrspace_t *addrspace, vmaddr_t vm_base, { assert((me != NULL) && (addrspace != NULL)); - error_t ret = OK; + error_t ret = OK; + bool mappings_full = true; if (util_add_overflows(vm_base, me->size - 1U)) { ret = ERROR_ADDR_OVERFLOW; @@ -283,12 +401,15 @@ memextent_map_basic(memextent_t *me, addrspace_t *addrspace, vmaddr_t vm_base, spinlock_acquire(&me->lock); - bool mappings_full = true; - memextent_mapping_t *map = NULL; - for (index_t i = 0; i < util_array_size(me->mappings); i++) { - map = &me->mappings[i]; + memextent_basic_mapping_t *map = NULL; + for (index_t i = 0; i < MEMEXTENT_MAX_MAPS; i++) { + map = &me->mappings.basic[i]; - if (atomic_load_relaxed(&map->addrspace) == NULL) { + // The mapping may have been used by a now deactivated + // addrspace; use a load-acquire to ensure we observe the + // removal from the addrspace's mapping list in + // memextent_deactivate_addrspace_basic(). + if (atomic_load_acquire(&map->addrspace) == NULL) { mappings_full = false; break; } @@ -306,34 +427,25 @@ memextent_map_basic(memextent_t *me, addrspace_t *addrspace, vmaddr_t vm_base, pgtable_vm_memtype_t memtype = memextent_mapping_attrs_get_memtype(&map_attrs); - // Take a reference to the address space to ensure that - // we don't race with its destruction. - if (!object_get_addrspace_safe(addrspace)) { - ret = ERROR_OBJECT_STATE; - goto out_locked; - } - // Add mapping to address space's list spinlock_acquire_nopreempt(&addrspace->mapping_list_lock); - list_insert_at_head(&addrspace->mapping_list, &map->mapping_list_node); + list_insert_at_head(&addrspace->basic_mapping_list, + &map->mapping_list_node); spinlock_release_nopreempt(&addrspace->mapping_list_lock); atomic_store_relaxed(&map->addrspace, addrspace); map->vbase = vm_base; memextent_mapping_attrs_set_memtype(&map->attrs, memtype); - memextent_mapping_attrs_set_user_access(&map->attrs, - access_user & me->access); - memextent_mapping_attrs_set_kernel_access(&map->attrs, - access_kernel & me->access); + memextent_mapping_attrs_set_user_access(&map->attrs, access_user); + memextent_mapping_attrs_set_kernel_access(&map->attrs, access_kernel); if (list_is_empty(&me->children_list)) { ret = memextent_do_map(me, map, 0, me->size); goto out_mapping_recorded; } - memextent_arg_t arg = { me, { NULL }, 0 }; - arg.map[0] = map; + memextent_basic_arg_t arg = { me, { map }, 0 }; // Walk through the memory extent physical range and map the contiguous // ranges it owns. @@ -344,61 +456,39 @@ memextent_map_basic(memextent_t *me, addrspace_t *addrspace, vmaddr_t vm_base, // If a range failed to be mapped, we need to rollback and unmap the // ranges that have already been mapped if ((ret != OK) && (arg.failed_address != me->phys_base)) { - memdb_range_walk((uintptr_t)me, MEMDB_TYPE_EXTENT, - me->phys_base, arg.failed_address - 1U, - memextent_unmap_range, (void *)&arg); + (void)memdb_range_walk((uintptr_t)me, MEMDB_TYPE_EXTENT, + me->phys_base, arg.failed_address - 1U, + memextent_unmap_range, (void *)&arg); } out_mapping_recorded: // If mapping failed, clear the map structure. if (ret != OK) { spinlock_acquire_nopreempt(&addrspace->mapping_list_lock); - list_delete_node(&addrspace->mapping_list, - &map->mapping_list_node); + (void)list_delete_node(&addrspace->basic_mapping_list, + &map->mapping_list_node); spinlock_release_nopreempt(&addrspace->mapping_list_lock); - memset((void *)map, 0U, sizeof(map)); + atomic_store_relaxed(&map->addrspace, NULL); } - object_put_addrspace(addrspace); - out_locked: spinlock_release(&me->lock); out: return ret; } -// Needs to be called holding a reference to the addrspace to be used -static void -memextent_remove_map_from_addrspace_list(memextent_mapping_t **mapping) -{ - assert(*mapping != NULL); - - memextent_mapping_t *map = *mapping; - addrspace_t *as = atomic_load_consume(&map->addrspace); - - assert(as != NULL); - - spinlock_acquire(&as->mapping_list_lock); - list_delete_node(&as->mapping_list, &map->mapping_list_node); - spinlock_release(&as->mapping_list_lock); - - memset((void *)map, 0U, sizeof(map)); - - *mapping = map; -} - error_t memextent_unmap_basic(memextent_t *me, addrspace_t *addrspace, vmaddr_t vm_base) { assert((me != NULL) && (addrspace != NULL)); - error_t ret = OK; - bool addrspace_not_mapped = true; - memextent_mapping_t *map = NULL; + error_t ret = OK; + bool addrspace_not_mapped = true; spinlock_acquire(&me->lock); - for (index_t i = 0; i < util_array_size(me->mappings); i++) { - map = &me->mappings[i]; + memextent_basic_mapping_t *map = NULL; + for (index_t i = 0; i < MEMEXTENT_MAX_MAPS; i++) { + map = &me->mappings.basic[i]; if ((atomic_load_relaxed(&map->addrspace) == addrspace) && (map->vbase == vm_base)) { @@ -412,18 +502,10 @@ memextent_unmap_basic(memextent_t *me, addrspace_t *addrspace, vmaddr_t vm_base) goto out; } - // Take a reference to the address space to ensure that - // we don't race with its destruction. - if (!object_get_addrspace_safe(addrspace)) { - ret = ERROR_OBJECT_STATE; - goto out; - } - if (list_is_empty(&me->children_list)) { memextent_do_unmap(me, map, 0, me->size); } else { - memextent_arg_t arg = { me, { NULL }, 0 }; - arg.map[0] = map; + memextent_basic_arg_t arg = { me, { map }, 0 }; // Walk through the memory extent physical range and unmap the // contiguous ranges it owns. @@ -434,9 +516,7 @@ memextent_unmap_basic(memextent_t *me, addrspace_t *addrspace, vmaddr_t vm_base) } assert(ret == OK); - memextent_remove_map_from_addrspace_list(&map); - object_put_addrspace(addrspace); - + memextent_remove_map_from_addrspace_list(map); out: spinlock_release(&me->lock); return ret; @@ -447,16 +527,17 @@ memextent_unmap_all_basic(memextent_t *me) { assert(me != NULL); - memextent_arg_t arg = { NULL, { NULL }, 0 }; - index_t index = 0; + memextent_basic_arg_t arg = { me, { NULL }, 0 }; + index_t index = 0; spinlock_acquire(&me->lock); // RCU protects ->addrspace rcu_read_start(); - for (index_t j = 0; j < util_array_size(me->mappings); j++) { - addrspace_t *addrspace = - atomic_load_consume(&me->mappings[j].addrspace); + for (index_t j = 0; j < MEMEXTENT_MAX_MAPS; j++) { + memextent_basic_mapping_t *map = &me->mappings.basic[j]; + + addrspace_t *addrspace = atomic_load_consume(&map->addrspace); if (addrspace != NULL) { // Take a reference to the address space to ensure that // we don't race with its destruction. @@ -465,24 +546,19 @@ memextent_unmap_all_basic(memextent_t *me) } if (list_is_empty(&me->children_list)) { - memextent_do_unmap(me, &me->mappings[j], 0, - me->size); + memextent_do_unmap(me, map, 0, me->size); + memextent_remove_map_from_addrspace_list(map); object_put_addrspace(addrspace); - continue; } else { - arg.map[index] = &me->mappings[j]; + arg.map[index] = map; index++; } } } rcu_read_finish(); - if (list_is_empty(&me->children_list)) { - goto out; - } - if (index != 0U) { - arg.me = me; + assert(!list_is_empty(&me->children_list)); // Walk through the memory extent physical range and unmap the // contiguous ranges it owns. @@ -495,15 +571,17 @@ memextent_unmap_all_basic(memextent_t *me) // Remove mapping from their corresponding address space's list for (index_t j = 0; j < index; j++) { - memextent_mapping_t *map = arg.map[j]; - memextent_remove_map_from_addrspace_list(&map); + memextent_basic_mapping_t *map = arg.map[j]; + assert(map != NULL); - object_put_addrspace( - atomic_load_consume(&map->addrspace)); + addrspace_t *as = atomic_load_relaxed(&map->addrspace); + assert(as != NULL); + + memextent_remove_map_from_addrspace_list(map); + object_put_addrspace(as); } } -out: spinlock_release(&me->lock); return true; @@ -519,12 +597,12 @@ memextent_update_access_basic(memextent_t *me, addrspace_t *addrspace, error_t ret = OK; bool addrspace_not_mapped = true; - memextent_mapping_t *map = NULL; + memextent_basic_mapping_t *map = NULL; spinlock_acquire(&me->lock); - for (index_t j = 0; j < util_array_size(me->mappings); j++) { - map = &me->mappings[j]; + for (index_t j = 0; j < MEMEXTENT_MAX_MAPS; j++) { + map = &me->mappings.basic[j]; if ((atomic_load_relaxed(&map->addrspace) == addrspace) && (map->vbase == vm_base)) { @@ -538,26 +616,44 @@ memextent_update_access_basic(memextent_t *me, addrspace_t *addrspace, goto out; } - // Take a reference to the address space to ensure that - // we don't race with its destruction. - if (!object_get_addrspace_safe(addrspace)) { - ret = ERROR_OBJECT_STATE; - goto out; - } + memextent_mapping_attrs_t old_attrs = map->attrs; pgtable_access_t access_user = memextent_access_attrs_get_user_access(&access_attrs); pgtable_access_t access_kernel = memextent_access_attrs_get_kernel_access(&access_attrs); - memextent_mapping_attrs_set_user_access(&map->attrs, - access_user & me->access); - memextent_mapping_attrs_set_kernel_access(&map->attrs, - access_kernel & me->access); + memextent_mapping_attrs_set_user_access(&map->attrs, access_user); + memextent_mapping_attrs_set_kernel_access(&map->attrs, access_kernel); - ret = memextent_do_map(me, map, 0, me->size); + if (list_is_empty(&me->children_list)) { + ret = memextent_do_map(me, map, 0, me->size); + if (ret != OK) { + // Restore the old mapping attributes. + map->attrs = old_attrs; + } + } else { + memextent_basic_arg_t arg = { me, { map }, 0 }; - object_put_addrspace(addrspace); + // Walk through the memory extent physical range and remap the + // contiguous ranges it owns with the new mapping attributes. + ret = memdb_range_walk((uintptr_t)me, MEMDB_TYPE_EXTENT, + me->phys_base, + me->phys_base + (me->size - 1U), + memextent_map_range, (void *)&arg); + + // If a range failed to be remapped, we need to rollback and + // remap the modified ranges with the original attributes. + if (ret != OK) { + map->attrs = old_attrs; + if (arg.failed_address != me->phys_base) { + (void)memdb_range_walk( + (uintptr_t)me, MEMDB_TYPE_EXTENT, + me->phys_base, arg.failed_address - 1U, + memextent_map_range, (void *)&arg); + } + } + } out: spinlock_release(&me->lock); @@ -565,6 +661,32 @@ memextent_update_access_basic(memextent_t *me, addrspace_t *addrspace, return ret; } +bool +memextent_is_mapped_basic(memextent_t *me, addrspace_t *addrspace, + bool exclusive) +{ + bool ret = false; + + for (index_t i = 0; i < MEMEXTENT_MAX_MAPS; i++) { + memextent_basic_mapping_t *map = &me->mappings.basic[i]; + + addrspace_t *as = atomic_load_relaxed(&map->addrspace); + if (as == addrspace) { + ret = true; + } else if (as != NULL) { + ret = false; + } else { + continue; + } + + if (ret != exclusive) { + break; + } + } + + return ret; +} + // Revert mappings of extent to the parent, assuming that the extent has no // children. static void @@ -574,82 +696,104 @@ memextent_revert_mappings(memextent_t *me) memextent_t *parent = me->parent; - size_t offset = me->phys_base - parent->phys_base; + memextent_mapping_t child_maps[MEMEXTENT_MAX_MAPS] = { 0 }; + memextent_mapping_t parent_maps[MEMEXTENT_MAX_MAPS] = { 0 }; spinlock_acquire(&parent->lock); + spinlock_acquire_nopreempt(&me->lock); - BITMAP_DECLARE(util_array_size(parent->mappings), - parent_matched) = { 0 }; + memextent_retain_mappings(me); + memextent_retain_mappings(parent); - for (index_t j = 0; j < util_array_size(me->mappings); j++) { - // RCU protects ->addrspace - rcu_read_start(); - memextent_mapping_t *map = &me->mappings[j]; - addrspace_t *s = atomic_load_consume(&map->addrspace); + for (index_t i = 0; i < MEMEXTENT_MAX_MAPS; i++) { + child_maps[i] = memextent_lookup_mapping(me, me->phys_base, + me->size, i); + } - // Take a reference to the address space to ensure that - // we don't race with its destruction. - if ((s == NULL) || !object_get_addrspace_safe(s)) { - // Nothing to do. - rcu_read_finish(); - continue; - } - rcu_read_finish(); + size_t offset = 0U; + while (offset < me->size) { + paddr_t phys = me->phys_base + offset; + size_t size = me->size - offset; + + bool child_match[MEMEXTENT_MAX_MAPS] = { 0 }; + bool parent_match[MEMEXTENT_MAX_MAPS] = { 0 }; - bool matched = false; + for (index_t i = 0; i < MEMEXTENT_MAX_MAPS; i++) { + parent_maps[i] = + memextent_lookup_mapping(parent, phys, size, i); - for (index_t i = 0; i < util_array_size(parent->mappings); - i++) { - memextent_mapping_t *p_map = &parent->mappings[i]; + memextent_mapping_t *pmap = &parent_maps[i]; - if (atomic_load_relaxed(&p_map->addrspace) != s) { + // We only want to revert the range covered by the + // parent's smallest mapping (or unmapped range). + size = util_min(pmap->size, size); + + if (pmap->addrspace == NULL) { continue; } - paddr_t p_vbase = p_map->vbase + offset; - if (p_vbase == map->vbase) { - bitmap_set(parent_matched, i); - matched = true; + for (index_t j = 0; j < MEMEXTENT_MAX_MAPS; j++) { + memextent_mapping_t *cmap = &child_maps[j]; - // Revert attributes if they have changed. - if (memextent_mapping_attrs_raw(p_map->attrs) != - memextent_mapping_attrs_raw(map->attrs)) { - memextent_do_map(parent, p_map, offset, - me->size); + if ((cmap->addrspace == NULL) || + (cmap->addrspace != pmap->addrspace)) { + continue; } - memextent_remove_map_from_addrspace_list(&map); + bool vbase_match = cmap->vbase == pmap->vbase; + bool attrs_match = + memextent_mapping_attrs_is_equal( + cmap->attrs, pmap->attrs); + + // We only need to unmap the child's mapping if + // the vbase does not match. If vbase matches + // but attrs don't, applying the parent's + // mapping will overwrite the child's. + parent_match[i] = vbase_match && attrs_match; + child_match[j] = vbase_match; } } - if (!matched) { - // The parent does not have this mapping; remove it. - memextent_do_unmap(me, map, 0, me->size); - memextent_remove_map_from_addrspace_list(&map); - } + for (index_t i = 0; i < MEMEXTENT_MAX_MAPS; i++) { + memextent_mapping_t *cmap = &child_maps[i]; + memextent_mapping_t *pmap = &parent_maps[i]; - object_put_addrspace(s); - } + if ((cmap->addrspace != NULL) && !child_match[i]) { + error_t err = addrspace_unmap(cmap->addrspace, + cmap->vbase, size, + phys); + assert(err == OK); + } - BITMAP_FOREACH_CLEAR_BEGIN(i, parent_matched, - util_array_size(parent->mappings)) - // RCU protects ->addrspace - rcu_read_start(); - memextent_mapping_t *p_map = &parent->mappings[i]; - addrspace_t *addrspace = atomic_load_consume(&p_map->addrspace); - if ((addrspace == NULL) || - (!object_get_addrspace_safe(addrspace))) { - rcu_read_finish(); - continue; + if ((pmap->addrspace != NULL) && !parent_match[i]) { + pgtable_vm_memtype_t memtype = + memextent_mapping_attrs_get_memtype( + &pmap->attrs); + pgtable_access_t kernel_access = + memextent_mapping_attrs_get_kernel_access( + &pmap->attrs); + pgtable_access_t user_access = + memextent_mapping_attrs_get_user_access( + &pmap->attrs); + + error_t err = addrspace_map(pmap->addrspace, + pmap->vbase, size, + phys, memtype, + kernel_access, + user_access); + if (err != OK) { + panic("Failed revert to parent mapping"); + } + } } - rcu_read_finish(); - // Revert the mapping - memextent_do_map(parent, p_map, offset, me->size); + offset += size; + } - object_put_addrspace(addrspace); - BITMAP_FOREACH_CLEAR_END + memextent_release_mappings(parent, false); + memextent_release_mappings(me, true); + spinlock_release_nopreempt(&me->lock); spinlock_release(&parent->lock); } @@ -664,7 +808,7 @@ memextent_deactivate_basic(memextent_t *me) if (me->parent != NULL) { memextent_revert_mappings(me); } else { - memextent_unmap_all_basic(me); + (void)memextent_unmap_all_basic(me); } return true; @@ -681,7 +825,7 @@ memextent_cleanup_basic(memextent_t *me) } // release ownership of the range - void *new_owner; + void *new_owner; memdb_type_t new_type; memextent_t *parent = me->parent; @@ -704,11 +848,188 @@ memextent_cleanup_basic(memextent_t *me) // Remove extent from parent's children list if (parent != NULL) { spinlock_acquire(&parent->lock); - list_delete_node(&parent->children_list, - &me->children_list_node); + (void)list_delete_node(&parent->children_list, + &me->children_list_node); spinlock_release(&parent->lock); } + free_mappings(me); + out: return true; } + +bool +memextent_retain_mappings_basic(memextent_t *me) +{ + assert(me != NULL); + + // RCU protects ->addrspace + rcu_read_start(); + for (index_t i = 0; i < MEMEXTENT_MAX_MAPS; i++) { + memextent_basic_mapping_t *map = &me->mappings.basic[i]; + + addrspace_t *as = atomic_load_consume(&map->addrspace); + if ((as != NULL) && object_get_addrspace_safe(as)) { + map->retained = true; + } + } + rcu_read_finish(); + + return true; +} + +bool +memextent_release_mappings_basic(memextent_t *me, bool clear) +{ + assert(me != NULL); + + for (index_t i = 0; i < MEMEXTENT_MAX_MAPS; i++) { + memextent_basic_mapping_t *map = &me->mappings.basic[i]; + + if (!map->retained) { + continue; + } + + addrspace_t *as = atomic_load_relaxed(&map->addrspace); + assert(as != NULL); + + if (clear) { + memextent_remove_map_from_addrspace_list(map); + } + + object_put_addrspace(as); + map->retained = false; + } + + return true; +} + +memextent_mapping_result_t +memextent_lookup_mapping_basic(memextent_t *me, paddr_t phys, size_t size, + index_t i) +{ + assert(me != NULL); + assert(i < MEMEXTENT_MAX_MAPS); + assert((phys >= me->phys_base) && + ((phys + (size - 1U)) <= (me->phys_base + (me->size - 1U)))); + + memextent_mapping_t ret = { + .size = size, + }; + + memextent_basic_mapping_t *map = &me->mappings.basic[i]; + + if (map->retained) { + addrspace_t *as = atomic_load_relaxed(&map->addrspace); + assert(as != NULL); + + ret.addrspace = as; + ret.vbase = map->vbase + (phys - me->phys_base); + ret.attrs = map->attrs; + } + + return memextent_mapping_result_ok(ret); +} + +error_t +memextent_create_addrspace_basic(addrspace_create_t params) +{ + addrspace_t *addrspace = params.addrspace; + assert(addrspace != NULL); + + list_init(&addrspace->basic_mapping_list); + + return OK; +} + +error_t +memextent_attach_basic(memextent_t *me, uintptr_t hyp_va, size_t size, + pgtable_hyp_memtype_t memtype) +{ + error_t ret; + + assert(me != NULL); + + spinlock_acquire(&me->lock); + + if (!list_is_empty(&me->children_list)) { + ret = ERROR_BUSY; + goto out_locked; + } + + pgtable_hyp_start(); + ret = pgtable_hyp_map(me->header.partition, hyp_va, size, me->phys_base, + memtype, PGTABLE_ACCESS_RW, + VMSA_SHAREABILITY_INNER_SHAREABLE); + pgtable_hyp_commit(); + + if (ret == OK) { + me->attached_address = hyp_va; + me->attached_size = size; + } + +out_locked: + spinlock_release(&me->lock); + + return ret; +} + +bool +memextent_detach_basic(memextent_t *me) +{ + assert(me != NULL); + + spinlock_acquire(&me->lock); + assert(me->attached_size != 0); + + pgtable_hyp_start(); + pgtable_hyp_unmap(me->header.partition, me->attached_address, + me->attached_size, me->attached_size); + pgtable_hyp_commit(); + + me->attached_size = 0; + spinlock_release(&me->lock); + + return true; +} + +void +memextent_deactivate_addrspace_basic(addrspace_t *addrspace) +{ + assert(addrspace != NULL); + + spinlock_acquire(&addrspace->mapping_list_lock); + + list_t *list = &addrspace->basic_mapping_list; + + // Remove all mappings from addrspace + memextent_basic_mapping_t *map = NULL; + list_foreach_container_maydelete (map, list, memextent_basic_mapping, + mapping_list_node) { + (void)list_delete_node(list, &map->mapping_list_node); + // We use a store-release to ensure that this list deletion is + // observed before using this mapping for another addrspace in + // memextent_map_basic(). + atomic_store_release(&map->addrspace, NULL); + } + + spinlock_release(&addrspace->mapping_list_lock); +} + +size_result_t +memextent_get_offset_for_pa_basic(memextent_t *me, paddr_t pa, size_t size) +{ + size_result_t ret; + + if (util_add_overflows(pa, size - 1U)) { + ret = size_result_error(ERROR_ADDR_OVERFLOW); + } else if ((pa < me->phys_base) || + ((pa + size - 1U) > (me->phys_base + me->size - 1U))) { + ret = size_result_error(ERROR_ADDR_INVALID); + } else { + ret = size_result_ok(pa - me->phys_base); + } + + return ret; +} diff --git a/hyp/mem/memextent/src/memextent_tests.c b/hyp/mem/memextent/src/memextent_tests.c index 2e1447a..7fc8df5 100644 --- a/hyp/mem/memextent/src/memextent_tests.c +++ b/hyp/mem/memextent/src/memextent_tests.c @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -23,7 +24,7 @@ #include #include -#include +#include #include "event_handlers.h" @@ -43,9 +44,14 @@ extern void pgtable_vm_dump(pgtable_vm_t *pgtable); #endif +rcu_update_status_t +partition_destroy_memextent(rcu_entry_t *entry); + void tests_memextent_init(void) { + spinlock_init(&test_memextent_spinlock); + partition = partition_get_root(); addrspace_ptr_result_t ret; @@ -68,24 +74,22 @@ tests_memextent_init(void) as2 = ret.r; - spinlock_init(&as->mapping_list_lock); - spinlock_init(&as->pgtable_lock); - list_init(&as->mapping_list); - spinlock_init(&as2->mapping_list_lock); - spinlock_init(&as2->pgtable_lock); - list_init(&as2->mapping_list); - // Dummy vmids - ret.e = pgtable_vm_init(partition, &as->vm_pgtable, 65U); - if (ret.e != OK) { - panic("Error pgtable_vm_init as"); + if (addrspace_configure(as, 65U) != OK) { + panic("Failed addrspace configuration"); } - ret.e = pgtable_vm_init(partition, &as2->vm_pgtable, 66U); - if (ret.e != OK) { - panic("Error pgtable_vm_init as 2"); + + if (addrspace_configure(as2, 66U) != OK) { + panic("Failed addrspace 2 configuration"); + } + + if (object_activate_addrspace(as) != OK) { + panic("Failed addrspace activation"); } - tests_memextent_count = 0; + if (object_activate_addrspace(as2) != OK) { + panic("Failed addrspace 2 activation"); + } } static error_t @@ -353,27 +357,24 @@ tests_memextent_test1(paddr_t phys_base) // Uncomment to make the destruction now and be able to see the pgtable // update instead of doing it sometime later by the rcu_update -#if 1 - trigger_memextent_deactivate_event(me_dd2->type, me_dd2); - trigger_memextent_deactivate_event(me_dd->type, me_dd); - trigger_memextent_deactivate_event(me_d2->type, me_d2); - trigger_memextent_deactivate_event(me_d->type, me_d); - trigger_memextent_deactivate_event(me2->type, me2); - trigger_memextent_deactivate_event(me->type, me); - - partition_free(partition, me_dd2, sizeof(memextent_t)); - partition_free(partition, me_dd, sizeof(memextent_t)); - partition_free(partition, me_d2, sizeof(memextent_t)); - partition_free(partition, me_d, sizeof(memextent_t)); - partition_free(partition, me2, sizeof(memextent_t)); - partition_free(partition, me, sizeof(memextent_t)); - - object_put_partition(partition); - object_put_partition(partition); - object_put_partition(partition); - object_put_partition(partition); - object_put_partition(partition); - object_put_partition(partition); +#if 0 + trigger_object_deactivate_memextent_event(me_dd2); + (void)partition_destroy_memextent(&me_dd2->header.rcu_entry); + + trigger_object_deactivate_memextent_event(me_dd); + (void)partition_destroy_memextent(&me_dd->header.rcu_entry); + + trigger_object_deactivate_memextent_event(me_d2); + (void)partition_destroy_memextent(&me_d2->header.rcu_entry); + + trigger_object_deactivate_memextent_event(me_d); + (void)partition_destroy_memextent(&me_d->header.rcu_entry); + + trigger_object_deactivate_memextent_event(me2); + (void)partition_destroy_memextent(&me2->header.rcu_entry); + + trigger_object_deactivate_memextent_event(me); + (void)partition_destroy_memextent(&me->header.rcu_entry); #else object_put_memextent(me_dd2); object_put_memextent(me_dd); @@ -489,27 +490,23 @@ tests_memextent_test2(paddr_t phys_base) // Uncomment to make the destruction now and be able to see the pgtable // update instead of doing it sometime later by the rcu_update -#if 1 - trigger_memextent_deactivate_event(me_d->type, me_d); - partition_free(partition, me_d, sizeof(memextent_t)); - object_put_partition(partition); +#if 0 + trigger_object_deactivate_memextent_event(me_d); + (void)partition_destroy_memextent(&me_d->header.rcu_entry); #if !defined(NDEBUG) LOG(DEBUG, INFO, "+--------------- deactivate me_d pgtable 1:\n"); pgtable_vm_dump(&as->vm_pgtable); #endif - trigger_memextent_deactivate_event(me->type, me); - partition_free(partition, me, sizeof(memextent_t)); - object_put_partition(partition); + trigger_object_deactivate_memextent_event(me); + (void)partition_destroy_memextent(&me->header.rcu_entry); #if !defined(NDEBUG) LOG(DEBUG, INFO, "+--------------- deactivate me pgtable 1:\n"); pgtable_vm_dump(&as->vm_pgtable); #endif - - trigger_memextent_deactivate_event(me2->type, me2); - partition_free(partition, me2, sizeof(memextent_t)); - object_put_partition(partition); + trigger_object_deactivate_memextent_event(me2); + (void)partition_destroy_memextent(&me2->header.rcu_entry); #if !defined(NDEBUG) LOG(DEBUG, INFO, "+--------------- deactivate me2 pgtable 1:\n"); @@ -530,19 +527,19 @@ tests_memextent(void) bool wait_all_cores_end = true; bool wait_all_cores_start = true; - spinlock_acquire(&test_memextent_spinlock); + spinlock_acquire_nopreempt(&test_memextent_spinlock); tests_memextent_count++; - spinlock_release(&test_memextent_spinlock); + spinlock_release_nopreempt(&test_memextent_spinlock); // Wait until all cores have reached this point to start. while (wait_all_cores_start) { - spinlock_acquire(&test_memextent_spinlock); + spinlock_acquire_nopreempt(&test_memextent_spinlock); if (tests_memextent_count == (PLATFORM_MAX_CORES)) { wait_all_cores_start = false; } - spinlock_release(&test_memextent_spinlock); + spinlock_release_nopreempt(&test_memextent_spinlock); } if (cpulocal_get_index() != 0U) { @@ -558,22 +555,22 @@ tests_memextent(void) phys_base = tests_find_free_range(); tests_memextent_test2(phys_base); - spinlock_acquire(&test_memextent_spinlock); + spinlock_acquire_nopreempt(&test_memextent_spinlock); tests_memextent_count++; - spinlock_release(&test_memextent_spinlock); + spinlock_release_nopreempt(&test_memextent_spinlock); LOG(DEBUG, INFO, "Memextent tests finished"); wait: // Make all threads wait for test to end while (wait_all_cores_end) { - spinlock_acquire(&test_memextent_spinlock); + spinlock_acquire_nopreempt(&test_memextent_spinlock); if (tests_memextent_count == PLATFORM_MAX_CORES + 1) { wait_all_cores_end = false; } - spinlock_release(&test_memextent_spinlock); + spinlock_release_nopreempt(&test_memextent_spinlock); } return false; diff --git a/hyp/mem/memextent_sparse/build.conf b/hyp/mem/memextent_sparse/build.conf new file mode 100644 index 0000000..760d100 --- /dev/null +++ b/hyp/mem/memextent_sparse/build.conf @@ -0,0 +1,9 @@ +# © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +base_module hyp/mem/memextent +base_module hyp/misc/gpt +events memextent_sparse.ev +types memextent_sparse.tc +source memextent_sparse.c memextent_tests.c diff --git a/hyp/mem/memextent_sparse/memextent_sparse.ev b/hyp/mem/memextent_sparse/memextent_sparse.ev new file mode 100644 index 0000000..14088ff --- /dev/null +++ b/hyp/mem/memextent_sparse/memextent_sparse.ev @@ -0,0 +1,80 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module memextent_sparse + +subscribe gpt_value_add_offset[GPT_TYPE_MEMEXTENT_MAPPING] + handler memextent_mapping_add_offset(value, offset) + +subscribe gpt_values_equal[GPT_TYPE_MEMEXTENT_MAPPING] + handler memextent_mappings_equal(x, y) + +subscribe memextent_activate[MEMEXTENT_TYPE_SPARSE] + handler memextent_activate_sparse(me) + +subscribe memextent_activate_derive[MEMEXTENT_TYPE_SPARSE] + handler memextent_activate_derive_sparse(me) + +subscribe memextent_supports_donation[MEMEXTENT_TYPE_SPARSE] + handler memextent_supports_donation_sparse() + +subscribe memextent_donate_child[MEMEXTENT_TYPE_SPARSE] + handler memextent_donate_child_sparse(me, phys, size, reverse) + +subscribe memextent_donate_sibling[MEMEXTENT_TYPE_SPARSE] + handler memextent_donate_sibling_sparse(from, to, phys, size) + +subscribe memextent_map[MEMEXTENT_TYPE_SPARSE] + handler memextent_map_sparse(extent, addrspace, vm_base, map_attrs) + +subscribe memextent_map_partial[MEMEXTENT_TYPE_SPARSE] + handler memextent_map_partial_sparse(extent, addrspace, vm_base, offset, size, map_attrs) + +subscribe memextent_unmap[MEMEXTENT_TYPE_SPARSE] + handler memextent_unmap_sparse(extent, addrspace, vm_base) + +subscribe memextent_unmap_partial[MEMEXTENT_TYPE_SPARSE] + handler memextent_unmap_partial_sparse(extent, addrspace, vm_base, offset, size) + +subscribe memextent_unmap_all[MEMEXTENT_TYPE_SPARSE] + handler memextent_unmap_all_sparse(extent) + +subscribe memextent_update_access[MEMEXTENT_TYPE_SPARSE] + handler memextent_update_access_sparse(extent, addrspace, vm_base, access_attrs) + +subscribe memextent_update_access_partial[MEMEXTENT_TYPE_SPARSE] + handler memextent_update_access_partial_sparse(extent, addrspace, vm_base, + offset, size, access_attrs) + +subscribe memextent_is_mapped[MEMEXTENT_TYPE_SPARSE] + handler memextent_is_mapped_sparse(me, addrspace, exclusive) + +subscribe memextent_deactivate[MEMEXTENT_TYPE_SPARSE] + handler memextent_deactivate_sparse(extent) + +subscribe memextent_cleanup[MEMEXTENT_TYPE_SPARSE] + handler memextent_cleanup_sparse(extent) + +subscribe memextent_retain_mappings[MEMEXTENT_TYPE_SPARSE] + handler memextent_retain_mappings_sparse(me) + +subscribe memextent_release_mappings[MEMEXTENT_TYPE_SPARSE] + handler memextent_release_mappings_sparse(me, clear) + +subscribe memextent_lookup_mapping[MEMEXTENT_TYPE_SPARSE] + handler memextent_lookup_mapping_sparse(me, phys, size, i) + +subscribe object_create_addrspace + handler memextent_create_addrspace_sparse + +subscribe object_deactivate_addrspace + handler memextent_deactivate_addrspace_sparse + +#if defined(UNIT_TESTS) +subscribe tests_init + handler tests_memextent_sparse_init() + +subscribe tests_start + handler tests_memextent_sparse_start() +#endif diff --git a/hyp/mem/memextent_sparse/memextent_sparse.tc b/hyp/mem/memextent_sparse/memextent_sparse.tc new file mode 100644 index 0000000..2fff552 --- /dev/null +++ b/hyp/mem/memextent_sparse/memextent_sparse.tc @@ -0,0 +1,54 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +define GPT_PHYS_BITS constant type count_t = 44; +define GPT_VBASE_BITS constant type count_t = 48; + +extend addrspace object { + sparse_mapping_list structure list; +}; + +define memextent_gpt_map bitfield<64> { + auto vbase type vmaddr_t; + auto<8> memtype enumeration pgtable_vm_memtype; + auto<3> user_access enumeration pgtable_access; + auto<3> kernel_access enumeration pgtable_access; + // For unmap operations, we only care about comparing vbases, so this + // flag indicates that we should ignore the mapping attributes. + auto ignore_attrs bool; +}; + +extend gpt_type enumeration { + memextent_mapping; +}; + +extend gpt_value union { + me_map bitfield memextent_gpt_map; +}; + +define memextent_sparse_arg structure { + addrspace pointer object addrspace; + vbase type vmaddr_t; + pbase type paddr_t; + memtype enumeration pgtable_vm_memtype; + user_access enumeration pgtable_access; + kernel_access enumeration pgtable_access; + fail_addr type paddr_t; +}; + +define memextent_gpt_arg structure { + addrspace pointer object addrspace; +}; + +define memextent_sparse_mapping structure { + // RCU-protected addrspace pointer + addrspace pointer(atomic) object addrspace; + mapping_list_node structure list_node(contained); + gpt structure gpt; + retained bool; +}; + +extend memextent_map_ptr union { + sparse pointer structure memextent_sparse_mapping; +}; diff --git a/hyp/mem/memextent_sparse/src/memextent_sparse.c b/hyp/mem/memextent_sparse/src/memextent_sparse.c new file mode 100644 index 0000000..d1a5166 --- /dev/null +++ b/hyp/mem/memextent_sparse/src/memextent_sparse.c @@ -0,0 +1,1519 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "event_handlers.h" + +void +memextent_mapping_add_offset(gpt_value_t *value, size_t offset) +{ + vmaddr_t vbase = memextent_gpt_map_get_vbase(&value->me_map); + + memextent_gpt_map_set_vbase(&value->me_map, vbase + offset); +} + +bool +memextent_mappings_equal(gpt_value_t x, gpt_value_t y) +{ + bool ret; + + if (memextent_gpt_map_get_ignore_attrs(&x.me_map) || + memextent_gpt_map_get_ignore_attrs(&y.me_map)) { + // We only need to check if the vbases are equal. + ret = memextent_gpt_map_get_vbase(&x.me_map) == + memextent_gpt_map_get_vbase(&y.me_map); + } else { + ret = memextent_gpt_map_is_equal(x.me_map, y.me_map); + } + + return ret; +} + +static error_t +allocate_sparse_mappings(memextent_t *me) +{ + error_t ret = OK; + partition_t *partition = me->header.partition; + const size_t alloc_size = + sizeof(memextent_sparse_mapping_t) * MEMEXTENT_MAX_MAPS; + const size_t alloc_align = alignof(memextent_sparse_mapping_t); + + void_ptr_result_t alloc_ret = + partition_alloc(partition, alloc_size, alloc_align); + if (alloc_ret.e != OK) { + ret = alloc_ret.e; + goto out; + } + + (void)memset_s(alloc_ret.r, alloc_size, 0, alloc_size); + + me->mappings.sparse = alloc_ret.r; + + for (index_t i = 0U; i < MEMEXTENT_MAX_MAPS; i++) { + gpt_config_t config = gpt_config_default(); + gpt_config_set_max_bits(&config, GPT_PHYS_BITS); + + ret = gpt_init(&me->mappings.sparse[i].gpt, partition, config, + util_bit(GPT_TYPE_MEMEXTENT_MAPPING)); + assert(ret == OK); + } + +out: + return ret; +} + +static void +free_sparse_mappings(memextent_t *me) +{ + partition_t *partition = me->header.partition; + const size_t alloc_size = + sizeof(memextent_sparse_mapping_t) * MEMEXTENT_MAX_MAPS; + + assert(me->mappings.sparse != NULL); + + for (index_t i = 0U; i < MEMEXTENT_MAX_MAPS; i++) { + gpt_destroy(&me->mappings.sparse[i].gpt); + } + + (void)partition_free(partition, me->mappings.sparse, alloc_size); + + me->mappings.sparse = NULL; +} + +static error_t +insert_gpt_mapping(memextent_sparse_mapping_t *map, paddr_t phys, size_t size, + vmaddr_t vbase, memextent_mapping_attrs_t attrs) +{ + assert(map != NULL); + + pgtable_vm_memtype_t memtype = + memextent_mapping_attrs_get_memtype(&attrs); + pgtable_access_t user_access = + memextent_mapping_attrs_get_user_access(&attrs); + pgtable_access_t kernel_access = + memextent_mapping_attrs_get_kernel_access(&attrs); + + memextent_gpt_map_t gpt_map = memextent_gpt_map_default(); + memextent_gpt_map_set_vbase(&gpt_map, vbase); + memextent_gpt_map_set_memtype(&gpt_map, memtype); + memextent_gpt_map_set_user_access(&gpt_map, user_access); + memextent_gpt_map_set_kernel_access(&gpt_map, kernel_access); + + gpt_entry_t gpt_entry = { + .type = GPT_TYPE_MEMEXTENT_MAPPING, + .value = { .me_map = gpt_map }, + }; + + return gpt_insert(&map->gpt, phys, size, gpt_entry, true); +} + +static error_t +remove_gpt_mapping(memextent_sparse_mapping_t *map, paddr_t phys, size_t size, + vmaddr_t vbase) +{ + assert(map != NULL); + + memextent_gpt_map_t gpt_map = memextent_gpt_map_default(); + memextent_gpt_map_set_vbase(&gpt_map, vbase); + memextent_gpt_map_set_ignore_attrs(&gpt_map, true); + + gpt_entry_t gpt_entry = { + .type = GPT_TYPE_MEMEXTENT_MAPPING, + .value = { .me_map = gpt_map }, + }; + + return gpt_remove(&map->gpt, phys, size, gpt_entry); +} + +static error_t +update_gpt_mapping(memextent_sparse_mapping_t *map, paddr_t phys, size_t size, + memextent_gpt_map_t old_gpt_map, + memextent_gpt_map_t new_gpt_map) +{ + assert(map != NULL); + assert(!memextent_gpt_map_get_ignore_attrs(&old_gpt_map)); + assert(!memextent_gpt_map_get_ignore_attrs(&new_gpt_map)); + + gpt_entry_t old_gpt_entry = { + .type = GPT_TYPE_MEMEXTENT_MAPPING, + .value = { .me_map = old_gpt_map }, + }; + + gpt_entry_t new_gpt_entry = { + .type = GPT_TYPE_MEMEXTENT_MAPPING, + .value = { .me_map = new_gpt_map }, + }; + + return gpt_update(&map->gpt, phys, size, old_gpt_entry, new_gpt_entry); +} + +static void +delete_sparse_mapping(memextent_sparse_mapping_t *map, addrspace_t *addrspace) + REQUIRE_PREEMPT_DISABLED +{ + assert(atomic_load_relaxed(&map->addrspace) == addrspace); + assert(gpt_is_empty(&map->gpt)); + + spinlock_acquire_nopreempt(&addrspace->mapping_list_lock); + (void)list_delete_node(&addrspace->basic_mapping_list, + &map->mapping_list_node); + spinlock_release_nopreempt(&addrspace->mapping_list_lock); + + atomic_store_relaxed(&map->addrspace, NULL); +} + +static bool +apply_access_mask(memextent_t *me, memextent_mapping_attrs_t *attrs) +{ + pgtable_access_t old_user_access = + memextent_mapping_attrs_get_user_access(attrs); + pgtable_access_t old_kernel_access = + memextent_mapping_attrs_get_kernel_access(attrs); + + pgtable_access_t new_user_access = + pgtable_access_mask(old_user_access, me->access); + pgtable_access_t new_kernel_access = + pgtable_access_mask(old_kernel_access, me->access); + + memextent_mapping_attrs_set_user_access(attrs, new_user_access); + memextent_mapping_attrs_set_kernel_access(attrs, new_kernel_access); + + return (old_user_access != new_user_access) || + (old_kernel_access != new_kernel_access); +} + +static error_t +add_sparse_mapping(memextent_t *me, addrspace_t *addrspace, paddr_t phys, + size_t size, vmaddr_t vbase, memextent_mapping_attrs_t attrs) + REQUIRE_SPINLOCK(me->lock) +{ + error_t err = OK; + bool mapped = false; + + assert(me != NULL); + assert(addrspace != NULL); + + memextent_sparse_mapping_t *empty_map = NULL; + + // First, try to use an existing mapping with matching addrspace. + for (index_t i = 0U; !mapped && (i < MEMEXTENT_MAX_MAPS); i++) { + memextent_sparse_mapping_t *map = &me->mappings.sparse[i]; + + addrspace_t *as = atomic_load_relaxed(&map->addrspace); + if (as == addrspace) { + err = insert_gpt_mapping(map, phys, size, vbase, attrs); + if (err == OK) { + mapped = true; + } else if (err == ERROR_BUSY) { + // There is an overlapping entry in this + // mapping's GPT, but we can try again with a + // different mapping. + err = OK; + } else { + // Unexpected GPT error. + break; + } + } else if ((as == NULL) && (empty_map == NULL)) { + empty_map = map; + } else { + // Mapping in use by another addrspace, or we have + // already found an earlier empty mapping, continue. + } + } + + if (mapped || (err != OK)) { + goto out; + } + + if (empty_map == NULL) { + err = ERROR_MEMEXTENT_MAPPINGS_FULL; + goto out; + } + + // We need an acquire fence as the empty mapping may have been cleared + // without the memextent lock if the previous addrspace was destroyed. + // This synchronises the earlier relaxed load of map->addrspace with the + // store-release in memextent_deactivate_addrspace_sparse(). + atomic_thread_fence(memory_order_acquire); + + err = insert_gpt_mapping(empty_map, phys, size, vbase, attrs); + if (err != OK) { + goto out; + } + + spinlock_acquire_nopreempt(&addrspace->mapping_list_lock); + list_insert_at_head(&addrspace->sparse_mapping_list, + &empty_map->mapping_list_node); + spinlock_release_nopreempt(&addrspace->mapping_list_lock); + atomic_store_relaxed(&empty_map->addrspace, addrspace); + +out: + return err; +} + +static error_t +remove_sparse_mapping(memextent_t *me, addrspace_t *addrspace, paddr_t phys, + size_t size, vmaddr_t vbase) REQUIRE_SPINLOCK(me->lock) +{ + error_t err = OK; + bool unmapped = false; + + assert(me != NULL); + assert(addrspace != NULL); + + for (index_t i = 0U; !unmapped && (i < MEMEXTENT_MAX_MAPS); i++) { + memextent_sparse_mapping_t *map = &me->mappings.sparse[i]; + + addrspace_t *as = atomic_load_relaxed(&map->addrspace); + if (as != addrspace) { + continue; + } + + err = remove_gpt_mapping(map, phys, size, vbase); + if (err == OK) { + unmapped = true; + if (gpt_is_empty(&map->gpt)) { + delete_sparse_mapping(map, addrspace); + } + } else if (err == ERROR_BUSY) { + // The entry was not found in this mapping's GPT, but + // may be in another mapping. + err = OK; + } else { + // Unexpected GPT error. + goto out; + } + } + + if (!unmapped) { + err = ERROR_ADDR_INVALID; + } + +out: + return err; +} + +static error_t +memextent_map_range_sparse(paddr_t phys, size_t size, void *arg) +{ + error_t err; + + assert(size != 0U); + assert(!util_add_overflows(phys, size - 1U)); + assert(arg != NULL); + + memextent_sparse_arg_t *me_arg = (memextent_sparse_arg_t *)arg; + + size_t offset = phys - me_arg->pbase; + + err = addrspace_map(me_arg->addrspace, me_arg->vbase + offset, size, + phys, me_arg->memtype, me_arg->kernel_access, + me_arg->user_access); + if (err != OK) { + me_arg->fail_addr = phys; + } + + return err; +} + +static error_t +memextent_unmap_range_sparse(paddr_t phys, size_t size, void *arg) +{ + assert(size != 0U); + assert(!util_add_overflows(phys, size - 1U)); + assert(arg != NULL); + + memextent_sparse_arg_t *me_arg = (memextent_sparse_arg_t *)arg; + + size_t offset = phys - me_arg->pbase; + + return addrspace_unmap(me_arg->addrspace, me_arg->vbase + offset, size, + phys); +} + +static error_t +do_as_map(addrspace_t *as, vmaddr_t vbase, size_t size, paddr_t phys, + memextent_mapping_attrs_t attrs) +{ + assert(as != NULL); + + pgtable_vm_memtype_t memtype = + memextent_mapping_attrs_get_memtype(&attrs); + pgtable_access_t kernel_access = + memextent_mapping_attrs_get_kernel_access(&attrs); + pgtable_access_t user_access = + memextent_mapping_attrs_get_user_access(&attrs); + + return addrspace_map(as, vbase, size, phys, memtype, kernel_access, + user_access); +} + +static error_t +apply_mappings(memextent_t *me, paddr_t phys, size_t size, bool unmap, + size_t *fail_offset) REQUIRE_SPINLOCK(me->lock) + REQUIRE_LOCK(me->mappings) +{ + error_t err = OK; + + memextent_mapping_t maps[MEMEXTENT_MAX_MAPS] = { 0 }; + + size_t offset = 0U; + while (offset < size) { + paddr_t curr_phys = phys + offset; + size_t curr_size = size - offset; + + for (index_t i = 0U; i < MEMEXTENT_MAX_MAPS; i++) { + maps[i] = memextent_lookup_mapping(me, curr_phys, + curr_size, i); + // For each iteration, we only want to transfer the + // range covered by the smallest mapping (or unmapped + // range). + curr_size = util_min(maps[i].size, curr_size); + } + + index_t fail_idx = 0U; + for (index_t i = 0U; i < MEMEXTENT_MAX_MAPS; i++) { + if (maps[i].addrspace == NULL) { + continue; + } + + if (unmap) { + err = addrspace_unmap(maps[i].addrspace, + maps[i].vbase, curr_size, + curr_phys); + } else { + err = do_as_map(maps[i].addrspace, + maps[i].vbase, curr_size, + curr_phys, maps[i].attrs); + } + + if (err != OK) { + fail_idx = i; + break; + } + } + if (unmap && (err != OK)) { + break; + } + + if (err != OK) { + if (fail_offset != NULL) { + *fail_offset = offset; + } else { + // If fail_offset wasn't provided then we assume + // the caller cannot recover from the error. + panic("Failed to apply sparse mappings"); + } + + for (index_t i = 0U; i < fail_idx; i++) { + if (maps[i].addrspace == NULL) { + continue; + } + + error_t revert_err; + if (unmap) { + revert_err = do_as_map( + maps[i].addrspace, + maps[i].vbase, curr_size, + curr_phys, maps[i].attrs); + } else { + revert_err = addrspace_unmap( + maps[i].addrspace, + maps[i].vbase, curr_size, + curr_phys); + } + + if (revert_err != OK) { + panic("Failed to revert sparse mappings"); + } + } + + break; + } + + offset += curr_size; + } + + return err; +} + +static error_t +do_mapping_transfer(memextent_t *x, memextent_t *y, paddr_t phys, size_t size, + size_t *fail_offset) REQUIRE_SPINLOCK(x->lock) + REQUIRE_SPINLOCK(y->lock) REQUIRE_LOCK(x->mappings) + REQUIRE_LOCK(y->mappings) +{ + error_t err = OK; + + memextent_mapping_t x_mappings[MEMEXTENT_MAX_MAPS] = { 0 }; + memextent_mapping_t y_mappings[MEMEXTENT_MAX_MAPS] = { 0 }; + + size_t offset = 0U; + while (offset < size) { + paddr_t curr_phys = phys + offset; + size_t curr_size = size - offset; + + bool x_match[MEMEXTENT_MAX_MAPS] = { 0 }; + bool y_match[MEMEXTENT_MAX_MAPS] = { 0 }; + + for (index_t i = 0U; i < MEMEXTENT_MAX_MAPS; i++) { + x_mappings[i] = memextent_lookup_mapping(x, curr_phys, + curr_size, i); + y_mappings[i] = memextent_lookup_mapping(y, curr_phys, + curr_size, i); + + // For each iteration, we only want to transfer the + // range covered by the smallest mapping (or unmapped + // range). + curr_size = util_min(x_mappings[i].size, curr_size); + curr_size = util_min(y_mappings[i].size, curr_size); + } + + for (index_t i = 0U; i < MEMEXTENT_MAX_MAPS; i++) { + memextent_mapping_t *xmap = &x_mappings[i]; + if (xmap->addrspace == NULL) { + continue; + } + + for (index_t j = 0U; j < MEMEXTENT_MAX_MAPS; j++) { + memextent_mapping_t *ymap = &y_mappings[j]; + if (xmap->addrspace != ymap->addrspace) { + continue; + } + + bool vbase_match = xmap->vbase == ymap->vbase; + bool attrs_match = + memextent_mapping_attrs_is_equal( + xmap->attrs, ymap->attrs); + + // We only need to unmap from x if the vbase + // does not match. If the vbases match but the + // attrs don't, applying y's mapping will + // overwrite the mapping from x. + x_match[i] = vbase_match; + y_match[j] = vbase_match && attrs_match; + } + } + + index_t x_idx = 0U; + index_t y_idx = 0U; + for (index_t i = 0U; i < MEMEXTENT_MAX_MAPS; i++) { + memextent_mapping_t *xmap = &x_mappings[i]; + memextent_mapping_t *ymap = &y_mappings[i]; + + if ((xmap->addrspace != NULL) && !x_match[i]) { + err = addrspace_unmap(xmap->addrspace, + xmap->vbase, curr_size, + curr_phys); + if (err != OK) { + break; + } + } + + x_idx++; + + if ((ymap->addrspace != NULL) && !y_match[i]) { + err = do_as_map(ymap->addrspace, ymap->vbase, + curr_size, curr_phys, + ymap->attrs); + if (err != OK) { + break; + } + } + + y_idx++; + } + + if (err != OK) { + if (fail_offset != NULL) { + *fail_offset = offset; + } else { + // If fail_offset wasn't provided then we assume + // the caller cannot recover from the error. + panic("Failed to do sparse mapping transfer"); + } + + for (index_t i = 0U; i < MEMEXTENT_MAX_MAPS; i++) { + memextent_mapping_t *xmap = &x_mappings[i]; + memextent_mapping_t *ymap = &y_mappings[i]; + + error_t revert_err = OK; + + if ((i < x_idx) && (xmap->addrspace != NULL) && + !x_match[i]) { + revert_err = do_as_map(xmap->addrspace, + xmap->vbase, + curr_size, + curr_phys, + xmap->attrs); + } + + if ((revert_err == OK) && (i < y_idx) && + (ymap->addrspace != NULL) && !y_match[i]) { + revert_err = addrspace_unmap( + ymap->addrspace, ymap->vbase, + curr_size, curr_phys); + } + + if (revert_err != OK) { + panic("Failed to revert mapping transfer"); + } + } + + break; + } + + offset += curr_size; + } + + return err; +} + +static error_t +update_memdb_partition_and_extent(memextent_t *me, paddr_t phys, size_t size, + bool to_partition) REQUIRE_SPINLOCK(me->lock) +{ + error_t ret; + partition_t *hyp_partition = partition_get_private(); + + assert(hyp_partition != NULL); + assert(me != NULL); + assert(!util_add_overflows(phys, size - 1U)); + + partition_t *parent_partition = me->header.partition; + + assert(parent_partition != NULL); + + uintptr_t object, prev_object; + memdb_type_t type, prev_type; + + if (to_partition) { + object = (uintptr_t)parent_partition; + type = MEMDB_TYPE_PARTITION; + prev_object = (uintptr_t)me; + prev_type = MEMDB_TYPE_EXTENT; + } else { + object = (uintptr_t)me; + type = MEMDB_TYPE_EXTENT; + prev_object = (uintptr_t)parent_partition; + prev_type = MEMDB_TYPE_PARTITION; + } + + paddr_t end = phys + (size - 1U); + + ret = memdb_update(hyp_partition, phys, end, object, type, prev_object, + prev_type); + if (ret == ERROR_MEMDB_NOT_OWNER) { + // We might have failed to take ownership because a previously + // deleted memextent has not yet been cleaned up, so wait for a + // RCU grace period and then retry. If it still fails after + // that, there's a real conflict. + spinlock_release(&me->lock); + rcu_sync(); + spinlock_acquire(&me->lock); + ret = memdb_update(hyp_partition, phys, end, object, type, + prev_object, prev_type); + } + + return ret; +} + +static error_t +update_memdb_two_extents(memextent_t *from, memextent_t *to, paddr_t phys, + size_t size, bool from_locked_first) + REQUIRE_SPINLOCK(from->lock) REQUIRE_SPINLOCK(to->lock) +{ + error_t ret; + partition_t *hyp_partition = partition_get_private(); + + assert(hyp_partition != NULL); + assert(from != NULL); + assert(to != NULL); + assert(!util_add_overflows(phys, size - 1U)); + + paddr_t end = phys + (size - 1U); + + ret = memdb_update(hyp_partition, phys, end, (uintptr_t)to, + MEMDB_TYPE_EXTENT, (uintptr_t)from, + MEMDB_TYPE_EXTENT); + if (ret == ERROR_MEMDB_NOT_OWNER) { + // We might have failed to take ownership because a previously + // deleted memextent has not yet been cleaned up, so wait for a + // RCU grace period and then retry. If it still fails after + // that, there's a real conflict. + if (from_locked_first) { + spinlock_release_nopreempt(&to->lock); + spinlock_release(&from->lock); + } else { + spinlock_release_nopreempt(&from->lock); + spinlock_release(&to->lock); + } + + rcu_sync(); + + if (from_locked_first) { + spinlock_acquire(&from->lock); + spinlock_acquire_nopreempt(&to->lock); + } else { + spinlock_acquire(&to->lock); + spinlock_acquire_nopreempt(&from->lock); + } + + ret = memdb_update(hyp_partition, phys, end, (uintptr_t)to, + MEMDB_TYPE_EXTENT, (uintptr_t)from, + MEMDB_TYPE_EXTENT); + } + + return ret; +} + +static error_t +get_phys_range(paddr_t phys, size_t size, void *arg) +{ + phys_range_result_t *ret = (phys_range_result_t *)arg; + assert(ret != NULL); + + phys_range_t range = { + .base = phys, + .size = size, + }; + + *ret = phys_range_result_ok(range); + + return ERROR_RETRY; +} + +static phys_range_result_t +lookup_phys_range(memextent_t *me, size_t *offset) +{ + phys_range_result_t ret = phys_range_result_error(ERROR_FAILURE); + + assert(offset != NULL); + assert(*offset < me->size); + + paddr_t start = me->phys_base + *offset; + paddr_t end = me->phys_base + (me->size - 1U); + + error_t err = memdb_range_walk((uintptr_t)me, MEMDB_TYPE_EXTENT, start, + end, get_phys_range, &ret); + assert((err == OK) || (ret.e == OK)); + + if (ret.e == OK) { + *offset = ret.r.base + ret.r.size - me->phys_base; + } + + return ret; +} + +error_t +memextent_activate_sparse(memextent_t *me) +{ + error_t ret; + partition_t *hyp_partition = partition_get_private(); + + assert(me != NULL); + assert(hyp_partition != NULL); + + ret = allocate_sparse_mappings(me); + if (ret != OK) { + goto out; + } + + if (me->device_mem) { + assert(me->memtype == MEMEXTENT_MEMTYPE_DEVICE); + + ret = memdb_insert(hyp_partition, me->phys_base, + me->phys_base + (me->size - 1U), + (uintptr_t)me, MEMDB_TYPE_EXTENT); + if (ret != OK) { + free_sparse_mappings(me); + } + } else { + // Memory will be added to the memextent after + // activation; there is nothing to do now. + } + +out: + return ret; +} + +error_t +memextent_activate_derive_sparse(memextent_t *me) +{ + error_t ret; + partition_t *hyp_partition = partition_get_private(); + + assert(me != NULL); + assert(me->parent != NULL); + + ret = allocate_sparse_mappings(me); + if (ret != OK) { + goto out; + } + + spinlock_acquire(&me->parent->lock); + spinlock_acquire_nopreempt(&me->lock); + + if (me->parent->attached_size != 0U) { + ret = ERROR_BUSY; + goto out_locked; + } + + bool transfer = !memextent_supports_donation(me->parent); + if (transfer) { + // The parent does not support donation, so we need to transfer + // ownership of the memextent's entire range now. + ret = update_memdb_two_extents(me->parent, me, me->phys_base, + me->size, true); + if (ret != OK) { + goto out_locked; + } + } + + memextent_retain_mappings(me->parent); + + bool access_changed = false; + for (index_t i = 0U; i < MEMEXTENT_MAX_MAPS; i++) { + size_t offset = 0U; + while (offset < me->size) { + paddr_t phys = me->phys_base + offset; + size_t size = me->size - offset; + + memextent_mapping_t parent_map = + memextent_lookup_mapping(me->parent, phys, size, + i); + offset += parent_map.size; + + if (parent_map.addrspace == NULL) { + continue; + } + + memextent_mapping_attrs_t attrs = parent_map.attrs; + + if (apply_access_mask(me, &attrs)) { + access_changed = true; + } + + ret = add_sparse_mapping(me, parent_map.addrspace, phys, + parent_map.size, + parent_map.vbase, attrs); + if (ret != OK) { + break; + } + } + } + + if ((ret == OK) && transfer && access_changed) { + // The child memextent has modified the mappings of memory it + // now owns. Ensure these mappings changes are applied. + size_t fail_offset = 0U; + + memextent_retain_mappings(me); + + ret = do_mapping_transfer(me->parent, me, me->phys_base, + me->size, &fail_offset); + if (ret != OK) { + // Revert mapping changes. + (void)do_mapping_transfer(me, me->parent, me->phys_base, + fail_offset, NULL); + } + + memextent_release_mappings(me, ret != OK); + } + + memextent_release_mappings(me->parent, false); + + if (ret == OK) { + list_insert_at_head(&me->parent->children_list, + &me->children_list_node); + } else if (transfer) { + error_t err = memdb_update(hyp_partition, me->phys_base, + me->phys_base + (me->size - 1U), + (uintptr_t)me->parent, + MEMDB_TYPE_EXTENT, (uintptr_t)me, + MEMDB_TYPE_EXTENT); + assert(err == OK); + } else { + // Nothing to do. + } + +out_locked: + spinlock_release_nopreempt(&me->lock); + spinlock_release(&me->parent->lock); + + if (ret != OK) { + free_sparse_mappings(me); + } + +out: + return ret; +} + +bool +memextent_supports_donation_sparse(void) +{ + return true; +} + +static error_t +donate_memextents_common(memextent_t *from, memextent_t *to, paddr_t phys, + size_t size, bool lock_from_first) +{ + error_t ret; + + if (lock_from_first) { + spinlock_acquire(&from->lock); + spinlock_acquire_nopreempt(&to->lock); + } else { + spinlock_acquire(&to->lock); + spinlock_acquire_nopreempt(&from->lock); + } + + ret = update_memdb_two_extents(from, to, phys, size, lock_from_first); + if (ret != OK) { + goto out; + } + + size_t fail_offset = 0U; + + memextent_retain_mappings(from); + memextent_retain_mappings(to); + + ret = do_mapping_transfer(from, to, phys, size, &fail_offset); + if (ret != OK) { + (void)do_mapping_transfer(from, to, phys, fail_offset, NULL); + } + + memextent_release_mappings(to, false); + memextent_release_mappings(from, false); + +out: + if (lock_from_first) { + spinlock_release_nopreempt(&to->lock); + spinlock_release(&from->lock); + } else { + spinlock_release_nopreempt(&from->lock); + spinlock_release(&to->lock); + } + + return ret; +} + +error_t +memextent_donate_child_sparse(memextent_t *me, paddr_t phys, size_t size, + bool reverse) +{ + error_t ret; + + assert(me != NULL); + + if (me->parent != NULL) { + if (me->parent->type != MEMEXTENT_TYPE_SPARSE) { + ret = ERROR_ARGUMENT_INVALID; + goto out; + } + + // The parent extent is always locked first. + if (reverse) { + ret = donate_memextents_common(me, me->parent, phys, + size, false); + } else { + ret = donate_memextents_common(me->parent, me, phys, + size, true); + } + + goto out; + } + + spinlock_acquire(&me->lock); + + ret = update_memdb_partition_and_extent(me, phys, size, reverse); + if (ret != OK) { + goto unlock_me; + } + + size_t fail_offset = 0U; + memextent_retain_mappings(me); + + ret = apply_mappings(me, phys, size, reverse, &fail_offset); + if (ret != OK) { + (void)apply_mappings(me, phys, fail_offset, true, NULL); + } + + memextent_release_mappings(me, false); + +unlock_me: + spinlock_release(&me->lock); +out: + return ret; +} + +error_t +memextent_donate_sibling_sparse(memextent_t *from, memextent_t *to, + paddr_t phys, size_t size) +{ + error_t ret; + + assert(from != to); + assert(from->parent == to->parent); + + if (to->type != MEMEXTENT_TYPE_SPARSE) { + ret = ERROR_ARGUMENT_INVALID; + goto out; + } + + // To prevent deadlocks, we need to obtain the memextents' locks in a + // consistent order. Lock the child at the lower address first. + ret = donate_memextents_common(from, to, phys, size, from < to); + +out: + return ret; +} + +error_t +memextent_map_sparse(memextent_t *me, addrspace_t *addrspace, vmaddr_t vm_base, + memextent_mapping_attrs_t map_attrs) +{ + return memextent_map_partial_sparse(me, addrspace, vm_base, 0U, + me->size, map_attrs); +} + +error_t +memextent_map_partial_sparse(memextent_t *me, addrspace_t *addrspace, + vmaddr_t vm_base, size_t offset, size_t size, + memextent_mapping_attrs_t map_attrs) +{ + error_t ret; + + assert(!util_add_overflows(offset, size - 1U)); + assert(!util_add_overflows(vm_base, size - 1U)); + + if (vm_base + (size - 1U) >= util_bit(GPT_VBASE_BITS)) { + ret = ERROR_ADDR_INVALID; + goto out; + } + + paddr_t phys = me->phys_base + offset; + + spinlock_acquire(&me->lock); + + ret = add_sparse_mapping(me, addrspace, phys, size, vm_base, map_attrs); + if (ret != OK) { + goto out_locked; + } + + pgtable_vm_memtype_t memtype = + memextent_mapping_attrs_get_memtype(&map_attrs); + pgtable_access_t user_access = + memextent_mapping_attrs_get_user_access(&map_attrs); + pgtable_access_t kernel_access = + memextent_mapping_attrs_get_kernel_access(&map_attrs); + + memextent_sparse_arg_t arg = { + .addrspace = addrspace, + .vbase = vm_base, + .pbase = phys, + .memtype = memtype, + .user_access = user_access, + .kernel_access = kernel_access, + }; + + ret = memdb_range_walk((uintptr_t)me, MEMDB_TYPE_EXTENT, phys, + phys + (size - 1U), memextent_map_range_sparse, + &arg); + if (ret != OK) { + error_t err; + + if (arg.fail_addr != phys) { + // Unmap any ranges that were mapped in the memdb walk. + err = memdb_range_walk((uintptr_t)me, MEMDB_TYPE_EXTENT, + phys, arg.fail_addr - 1U, + memextent_unmap_range_sparse, + &arg); + assert(err == OK); + } + + err = remove_sparse_mapping(me, addrspace, phys, size, vm_base); + assert(err == OK); + } + +out_locked: + spinlock_release(&me->lock); +out: + return ret; +} + +error_t +memextent_unmap_sparse(memextent_t *me, addrspace_t *addrspace, + vmaddr_t vm_base) +{ + return memextent_unmap_partial_sparse(me, addrspace, vm_base, 0U, + me->size); +} + +error_t +memextent_unmap_partial_sparse(memextent_t *me, addrspace_t *addrspace, + vmaddr_t vm_base, size_t offset, size_t size) +{ + error_t ret; + + assert(!util_add_overflows(offset, size - 1U)); + assert(!util_add_overflows(vm_base, size - 1U)); + + if (vm_base + (size - 1U) >= util_bit(GPT_VBASE_BITS)) { + ret = ERROR_ADDR_INVALID; + goto out; + } + + paddr_t phys = me->phys_base + offset; + + spinlock_acquire(&me->lock); + + ret = remove_sparse_mapping(me, addrspace, phys, size, vm_base); + if (ret != OK) { + goto out_locked; + } + + memextent_sparse_arg_t arg = { + .addrspace = addrspace, + .vbase = vm_base, + .pbase = phys, + }; + + ret = memdb_range_walk((uintptr_t)me, MEMDB_TYPE_EXTENT, phys, + phys + (size - 1U), memextent_unmap_range_sparse, + &arg); + assert(ret == OK); + +out_locked: + spinlock_release(&me->lock); +out: + return ret; +} + +bool +memextent_unmap_all_sparse(memextent_t *me) +{ + spinlock_acquire(&me->lock); + memextent_retain_mappings(me); + + size_t offset = 0U; + while (offset < me->size) { + phys_range_result_t range = lookup_phys_range(me, &offset); + if (range.e != OK) { + break; + } + + error_t err = apply_mappings(me, range.r.base, range.r.size, + true, NULL); + assert(err == OK); + } + + memextent_release_mappings(me, true); + spinlock_release(&me->lock); + + return true; +} + +error_t +memextent_update_access_sparse(memextent_t *me, addrspace_t *addrspace, + vmaddr_t vm_base, + memextent_access_attrs_t access_attrs) +{ + return memextent_update_access_partial_sparse(me, addrspace, vm_base, 0, + me->size, access_attrs); +} + +error_t +memextent_update_access_partial_sparse(memextent_t *me, addrspace_t *addrspace, + vmaddr_t vm_base, size_t offset, + size_t size, + memextent_access_attrs_t access_attrs) +{ + error_t ret; + paddr_t phys = me->phys_base + offset; + + assert(!util_add_overflows(offset, size - 1U)); + assert(!util_add_overflows(vm_base, size - 1U)); + + if (vm_base + (size - 1U) >= util_bit(GPT_VBASE_BITS)) { + ret = ERROR_ADDR_INVALID; + goto out; + } + + spinlock_acquire(&me->lock); + + memextent_sparse_mapping_t *update_map = NULL; + memextent_gpt_map_t old_gpt_map = memextent_gpt_map_default(); + + for (index_t i = 0U; i < MEMEXTENT_MAX_MAPS; i++) { + memextent_sparse_mapping_t *map = &me->mappings.sparse[i]; + + addrspace_t *as = atomic_load_relaxed(&map->addrspace); + if (as != addrspace) { + continue; + } + + // We need to keep the existing memtype when updating access. + // Perform a lookup on the first page of the mapping so we know + // what it is. If the memtype isn't consistent for the range + // then the GPT update will detect this and return an error. + gpt_lookup_result_t lookup_ret = + gpt_lookup(&map->gpt, phys, PGTABLE_VM_PAGE_SIZE); + if (lookup_ret.entry.type == GPT_TYPE_EMPTY) { + continue; + } + + assert(lookup_ret.entry.type == GPT_TYPE_MEMEXTENT_MAPPING); + + old_gpt_map = lookup_ret.entry.value.me_map; + if (memextent_gpt_map_get_vbase(&old_gpt_map) == vm_base) { + break; + } + } + + if (update_map == NULL) { + ret = ERROR_ADDR_INVALID; + goto out_locked; + } + + pgtable_access_t new_user_access = + memextent_access_attrs_get_user_access(&access_attrs); + pgtable_access_t new_kernel_access = + memextent_access_attrs_get_kernel_access(&access_attrs); + + memextent_gpt_map_t new_gpt_map = old_gpt_map; + memextent_gpt_map_set_user_access(&new_gpt_map, new_user_access); + memextent_gpt_map_set_kernel_access(&new_gpt_map, new_kernel_access); + + ret = update_gpt_mapping(update_map, phys, size, old_gpt_map, + new_gpt_map); + if (ret != OK) { + goto out_locked; + } + + memextent_sparse_arg_t arg = { + .addrspace = addrspace, + .vbase = vm_base, + .pbase = phys, + .memtype = memextent_gpt_map_get_memtype(&new_gpt_map), + .user_access = new_user_access, + .kernel_access = new_kernel_access, + }; + + ret = memdb_range_walk((uintptr_t)me, MEMDB_TYPE_EXTENT, phys, + phys + (size - 1U), memextent_map_range_sparse, + &arg); + if (ret != OK) { + error_t err; + + if (arg.fail_addr != phys) { + // Revert any access changes applied to the addrspace. + arg.user_access = + memextent_gpt_map_get_user_access(&old_gpt_map); + arg.kernel_access = memextent_gpt_map_get_kernel_access( + &old_gpt_map); + + err = memdb_range_walk((uintptr_t)me, MEMDB_TYPE_EXTENT, + phys, arg.fail_addr - 1U, + memextent_map_range_sparse, + &arg); + assert(err == OK); + } + + // Revert the GPT update. + err = update_gpt_mapping(update_map, phys, size, new_gpt_map, + old_gpt_map); + assert(err == OK); + } + +out_locked: + spinlock_release(&me->lock); +out: + return ret; +} + +bool +memextent_is_mapped_sparse(memextent_t *me, addrspace_t *addrspace, + bool exclusive) +{ + bool ret = false; + + for (index_t i = 0; i < MEMEXTENT_MAX_MAPS; i++) { + memextent_sparse_mapping_t *map = &me->mappings.sparse[i]; + + addrspace_t *as = atomic_load_relaxed(&map->addrspace); + if (as == addrspace) { + ret = true; + } else if (as != NULL) { + ret = false; + } else { + continue; + } + + if (ret != exclusive) { + break; + } + } + + return ret; +} + +bool +memextent_deactivate_sparse(memextent_t *me) +{ + assert(me != NULL); + + // There should be no children by this time + assert(list_is_empty(&me->children_list)); + + if (me->parent == NULL) { + (void)memextent_unmap_all_sparse(me); + goto out; + } + + spinlock_acquire(&me->parent->lock); + spinlock_acquire_nopreempt(&me->lock); + + memextent_retain_mappings(me->parent); + memextent_retain_mappings(me); + + size_t offset = 0U; + while (offset < me->size) { + phys_range_result_t range = lookup_phys_range(me, &offset); + if (range.e != OK) { + break; + } + + error_t err = do_mapping_transfer(me, me->parent, range.r.base, + range.r.size, NULL); + assert(err == OK); + } + + memextent_release_mappings(me->parent, false); + memextent_release_mappings(me, true); + + spinlock_release_nopreempt(&me->lock); + spinlock_release(&me->parent->lock); + +out: + return true; +} + +static error_t +memextent_return_range(paddr_t base, size_t size, void *arg) +{ + partition_t *hyp_partition = partition_get_private(); + + assert(hyp_partition != NULL); + assert(size != 0U); + assert(!util_add_overflows(base, size - 1U)); + assert(arg != NULL); + + memextent_t *me = (memextent_t *)arg; + uintptr_t parent; + memdb_type_t parent_type; + + if (me->parent != NULL) { + parent = (uintptr_t)me->parent; + parent_type = MEMDB_TYPE_EXTENT; + } else { + parent = (uintptr_t)me->header.partition; + parent_type = MEMDB_TYPE_PARTITION; + } + + return memdb_update(hyp_partition, base, base + (size - 1U), parent, + parent_type, (uintptr_t)me, MEMDB_TYPE_EXTENT); +} + +bool +memextent_cleanup_sparse(memextent_t *me) +{ + assert(me != NULL); + + if (!me->active) { + goto out; + } + + // Walk over the memextent's range and donate any memory still + // owned by the extent back to the parent. + error_t err = memdb_range_walk((uintptr_t)me, MEMDB_TYPE_EXTENT, + me->phys_base, + me->phys_base + (me->size - 1U), + memextent_return_range, me); + assert(err == OK); + + memextent_t *parent = me->parent; + if (parent != NULL) { + // Remove extent from parent's list of children. + spinlock_acquire(&parent->lock); + (void)list_delete_node(&parent->children_list, + &me->children_list_node); + spinlock_release(&parent->lock); + } + + free_sparse_mappings(me); + +out: + return true; +} + +bool +memextent_retain_mappings_sparse(memextent_t *me) REQUIRE_SPINLOCK(me->lock) +{ + assert(me != NULL); + + rcu_read_start(); + for (index_t i = 0U; i < MEMEXTENT_MAX_MAPS; i++) { + memextent_sparse_mapping_t *map = &me->mappings.sparse[i]; + + addrspace_t *as = atomic_load_consume(&map->addrspace); + if ((as != NULL) && object_get_addrspace_safe(as)) { + map->retained = true; + } + } + rcu_read_finish(); + + return true; +} + +bool +memextent_release_mappings_sparse(memextent_t *me, bool clear) + REQUIRE_SPINLOCK(me->lock) +{ + assert(me != NULL); + + for (index_t i = 0U; i < MEMEXTENT_MAX_MAPS; i++) { + memextent_sparse_mapping_t *map = &me->mappings.sparse[i]; + + if (!map->retained) { + continue; + } + + addrspace_t *as = atomic_load_relaxed(&map->addrspace); + assert(as != NULL); + + if (clear) { + gpt_clear_all(&map->gpt); + delete_sparse_mapping(map, as); + } + + object_put_addrspace(as); + map->retained = false; + } + + return true; +} + +memextent_mapping_result_t +memextent_lookup_mapping_sparse(memextent_t *me, paddr_t phys, size_t size, + index_t i) +{ + assert(me != NULL); + assert(i < MEMEXTENT_MAX_MAPS); + assert((phys >= me->phys_base) && + ((phys + (size - 1U)) <= (me->phys_base + (me->size - 1U)))); + + memextent_mapping_t ret = { + .size = size, + }; + + memextent_sparse_mapping_t *map = &me->mappings.sparse[i]; + + if (!map->retained) { + goto out; + } + + addrspace_t *as = atomic_load_relaxed(&map->addrspace); + assert(as != NULL); + + gpt_lookup_result_t lookup = gpt_lookup(&map->gpt, phys, size); + + ret.size = lookup.size; + + if (lookup.entry.type == GPT_TYPE_EMPTY) { + goto out; + } + + assert(lookup.entry.type == GPT_TYPE_MEMEXTENT_MAPPING); + + memextent_gpt_map_t gpt_map = lookup.entry.value.me_map; + assert(!memextent_gpt_map_get_ignore_attrs(&gpt_map)); + + memextent_mapping_attrs_t attrs = memextent_mapping_attrs_default(); + memextent_mapping_attrs_set_memtype( + &attrs, memextent_gpt_map_get_memtype(&gpt_map)); + memextent_mapping_attrs_set_user_access( + &attrs, memextent_gpt_map_get_user_access(&gpt_map)); + memextent_mapping_attrs_set_kernel_access( + &attrs, memextent_gpt_map_get_kernel_access(&gpt_map)); + + ret.addrspace = as; + ret.vbase = memextent_gpt_map_get_vbase(&gpt_map); + ret.attrs = attrs; + +out: + return memextent_mapping_result_ok(ret); +} + +error_t +memextent_create_addrspace_sparse(addrspace_create_t params) +{ + addrspace_t *addrspace = params.addrspace; + assert(addrspace != NULL); + + list_init(&addrspace->sparse_mapping_list); + + return OK; +} + +void +memextent_deactivate_addrspace_sparse(addrspace_t *addrspace) +{ + assert(addrspace != NULL); + + spinlock_acquire(&addrspace->mapping_list_lock); + + list_t *list = &addrspace->sparse_mapping_list; + + memextent_sparse_mapping_t *map = NULL; + list_foreach_container_maydelete (map, list, memextent_sparse_mapping, + mapping_list_node) { + // An object_put() call is a release operation, and if the + // refcount reaches zero it is also an acquire operation. As + // such, we should have observed all prior updates to the GPT + // despite not holding the memextent lock. Additionally, the + // mapping won't be reused until the addrspace pointer is + // cleared below, so it is also safe to clear the GPT without + // holding the lock. + gpt_clear_all(&map->gpt); + (void)list_delete_node(list, &map->mapping_list_node); + + // Use store-release to ensure the above updates are observed + // when the empty mapping is reused. This matches with the + // acquire fence in add_sparse_mapping(). + atomic_store_release(&map->addrspace, NULL); + } + + spinlock_release(&addrspace->mapping_list_lock); +} diff --git a/hyp/mem/memextent_sparse/src/memextent_tests.c b/hyp/mem/memextent_sparse/src/memextent_tests.c new file mode 100644 index 0000000..17ae138 --- /dev/null +++ b/hyp/mem/memextent_sparse/src/memextent_tests.c @@ -0,0 +1,348 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#if defined(UNIT_TESTS) + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "event_handlers.h" + +#define PHYS_MAX (1UL << GPT_PHYS_BITS) + +static addrspace_t *as1; +static addrspace_t *as2; + +static _Atomic bool tests_complete; + +static addrspace_t * +create_addrspace(vmid_t vmid) +{ + partition_t *partition = partition_get_root(); + assert(partition != NULL); + + addrspace_ptr_result_t ret; + addrspace_create_t params = { NULL }; + + ret = partition_allocate_addrspace(partition, params); + if (ret.e != OK) { + panic("Failed to create addrspace"); + } + + if (addrspace_configure(ret.r, vmid) != OK) { + panic("Failed addrspace configuration"); + } + + if (object_activate_addrspace(ret.r) != OK) { + panic("Failed addrspace activation"); + } + + return ret.r; +} + +static memextent_t * +create_memextent(memextent_t *parent, size_t offset, size_t size, bool sparse) +{ + partition_t *partition = partition_get_root(); + assert(partition != NULL); + + memextent_ptr_result_t ret; + memextent_create_t params = { .memextent_device_mem = false }; + + ret = partition_allocate_memextent(partition, params); + if (ret.e != OK) { + panic("Failed to create addrspace"); + } + + memextent_attrs_t attrs = memextent_attrs_default(); + memextent_attrs_set_memtype(&attrs, MEMEXTENT_MEMTYPE_ANY); + memextent_attrs_set_access(&attrs, PGTABLE_ACCESS_RWX); + if (sparse) { + memextent_attrs_set_type(&attrs, MEMEXTENT_TYPE_SPARSE); + } + + if (parent != NULL) { + if (memextent_configure_derive(ret.r, parent, offset, size, + attrs)) { + panic("Failed to configure derived memextent"); + } + } else { + if (memextent_configure(ret.r, offset, size, attrs) != OK) { + panic("Failed to configure memextent"); + } + } + + if (object_activate_memextent(ret.r) != OK) { + panic("Failed to activate memextent"); + } + + return ret.r; +} + +static error_t +phys_range_walk(paddr_t phys, size_t size, void *arg) +{ + phys_range_result_t *ret = (phys_range_result_t *)arg; + assert(ret != NULL); + + if ((ret->e != OK) && (size >= ret->r.size)) { + ret->r.base = phys; + ret->e = OK; + } + + return OK; +} + +static paddr_t +get_free_phys_range(size_t min_size) +{ + partition_t *partition = partition_get_root(); + assert(partition != NULL); + + phys_range_t range = { .size = min_size }; + phys_range_result_t ret = { .r = range, .e = ERROR_NOMEM }; + + error_t err = memdb_walk((uintptr_t)partition, MEMDB_TYPE_PARTITION, + phys_range_walk, &ret); + if ((err != OK) || (ret.e != OK)) { + panic("Failed to find free phys range"); + } + + return ret.r.base; +} + +static error_t +map_memextent(memextent_t *me, addrspace_t *as, vmaddr_t vbase, size_t offset, + size_t size, pgtable_vm_memtype_t memtype, + pgtable_access_t access) +{ + memextent_mapping_attrs_t map_attrs = memextent_mapping_attrs_default(); + memextent_mapping_attrs_set_memtype(&map_attrs, memtype); + memextent_mapping_attrs_set_user_access(&map_attrs, access); + memextent_mapping_attrs_set_kernel_access(&map_attrs, access); + + return memextent_map_partial(me, as, vbase, offset, size, map_attrs); +} + +static bool +lookup_addrspace(addrspace_t *as, vmaddr_t vbase, paddr_t expected_phys, + pgtable_vm_memtype_t expected_memtype, + pgtable_access_t expected_access) +{ + bool ret = false; + + paddr_t mapped_base; + size_t mapped_size; + pgtable_vm_memtype_t mapped_memtype; + pgtable_access_t mapped_vm_kernel_access; + pgtable_access_t mapped_vm_user_access; + + bool mapped = pgtable_vm_lookup(&as->vm_pgtable, vbase, &mapped_base, + &mapped_size, &mapped_memtype, + &mapped_vm_kernel_access, + &mapped_vm_user_access); + + if (mapped) { + mapped_base += vbase & (mapped_size - 1U); + ret = (expected_phys == mapped_base) && + (expected_memtype == mapped_memtype) && + (expected_access == mapped_vm_kernel_access) && + (expected_access == mapped_vm_user_access); + } + + return ret; +} + +static bool +is_owner(memextent_t *me, paddr_t phys, size_t size) +{ + return memdb_is_ownership_contiguous(phys, phys + size - 1U, + (uintptr_t)me, MEMDB_TYPE_EXTENT); +} + +void +tests_memextent_sparse_init(void) +{ + as1 = create_addrspace(33U); + as2 = create_addrspace(44U); +} + +bool +tests_memextent_sparse_start(void) +{ + error_t err; + + cpulocal_begin(); + cpu_index_t cpu = cpulocal_get_index(); + cpulocal_end(); + + if (cpu != 0U) { + goto wait; + } + + LOG(DEBUG, INFO, "Starting sparse memextent tests"); + + memextent_t *me_0_0 = create_memextent(NULL, 0U, PHYS_MAX, true); + assert(me_0_0 != NULL); + + // Test 1: Apply mapping after donate from partition. + vmaddr_t vbase = 0x80000000U; + size_t size = PGTABLE_VM_PAGE_SIZE; + paddr_t phys = get_free_phys_range(size); + pgtable_vm_memtype_t memtype = PGTABLE_VM_MEMTYPE_NORMAL_WB; + pgtable_access_t access = PGTABLE_ACCESS_RW; + + err = map_memextent(me_0_0, as1, vbase, phys, size, memtype, access); + assert(err == OK); + + bool mapped = lookup_addrspace(as1, vbase, phys, memtype, access); + assert(!mapped); + + err = memextent_donate_child(me_0_0, phys, size, false); + assert(err == OK); + + mapped = lookup_addrspace(as1, vbase, phys, memtype, access); + assert(mapped); + + err = memextent_donate_child(me_0_0, phys, size, true); + assert(err == OK); + + mapped = lookup_addrspace(as1, vbase, phys, memtype, access); + assert(!mapped); + + // Test 2: Donate between siblings. + memextent_t *me_1_0 = create_memextent(me_0_0, 0U, PHYS_MAX, true); + assert(me_1_0 != NULL); + + memextent_t *me_1_1 = create_memextent(me_0_0, 0U, PHYS_MAX, true); + assert(me_1_1 != NULL); + + size = 0x10000U; + phys = get_free_phys_range(size); + + err = memextent_donate_child(me_0_0, phys, size, false); + assert(err == OK); + + bool owner = is_owner(me_0_0, phys, size); + assert(owner); + + vmaddr_t vbase_1 = 0x60000000U; + vmaddr_t vbase_2a = 0x340404000U; + vmaddr_t vbase_2b = 0x288840000U; + + err = map_memextent(me_1_0, as1, vbase_1, phys, 0x6000U, memtype, + access); + assert(err == OK); + + err = map_memextent(me_1_0, as2, vbase_2a, phys, size, memtype, access); + assert(err == OK); + + err = map_memextent(me_1_1, as1, vbase_1 + 0x4000U, phys + 0x4000U, + 0x6000U, memtype, access); + assert(err == OK); + + err = map_memextent(me_1_1, as2, vbase_2b, phys, size, memtype, access); + assert(err == OK); + + mapped = lookup_addrspace(as1, vbase_1, phys, memtype, access); + assert(!mapped); + mapped = lookup_addrspace(as1, vbase_1 + 0x6000U, phys + 0x6000U, + memtype, access); + assert(!mapped); + mapped = lookup_addrspace(as2, vbase_2a, phys, memtype, access); + assert(!mapped); + mapped = lookup_addrspace(as2, vbase_2b, phys, memtype, access); + assert(!mapped); + + err = memextent_donate_child(me_1_0, phys, size, false); + assert(err == OK); + + owner = is_owner(me_1_0, phys, size); + assert(owner); + + mapped = lookup_addrspace(as1, vbase_1, phys, memtype, access); + assert(mapped); + mapped = lookup_addrspace(as1, vbase_1 + 0x6000U, phys + 0x6000U, + memtype, access); + assert(!mapped); + mapped = lookup_addrspace(as2, vbase_2a, phys, memtype, access); + assert(mapped); + mapped = lookup_addrspace(as2, vbase_2b, phys, memtype, access); + assert(!mapped); + + err = memextent_donate_sibling(me_1_0, me_1_1, phys, size); + assert(err == OK); + + owner = is_owner(me_1_1, phys, size); + assert(owner); + + mapped = lookup_addrspace(as1, vbase_1, phys, memtype, access); + assert(!mapped); + mapped = lookup_addrspace(as1, vbase_1 + 0x6000U, phys + 0x6000U, + memtype, access); + assert(mapped); + mapped = lookup_addrspace(as2, vbase_2a, phys, memtype, access); + assert(!mapped); + mapped = lookup_addrspace(as2, vbase_2b, phys, memtype, access); + assert(mapped); + + err = memextent_unmap_partial(me_1_0, as1, vbase_1, phys, 0x6000U); + assert(err == OK); + + err = memextent_unmap_partial(me_1_0, as2, vbase_2a, phys, size); + assert(err == OK); + + memextent_unmap_all(me_1_1); + + mapped = lookup_addrspace(as1, vbase_1, phys, memtype, access); + assert(!mapped); + mapped = lookup_addrspace(as1, vbase_1 + 0x6000U, phys + 0x6000U, + memtype, access); + assert(!mapped); + mapped = lookup_addrspace(as2, vbase_2a, phys, memtype, access); + assert(!mapped); + mapped = lookup_addrspace(as2, vbase_2b, phys, memtype, access); + assert(!mapped); + + object_put_addrspace(as1); + object_put_addrspace(as2); + + object_put_memextent(me_0_0); + object_put_memextent(me_1_0); + object_put_memextent(me_1_1); + + LOG(DEBUG, INFO, "Finished sparse memextent tests"); + + asm_event_store_and_wake(&tests_complete, true); + +wait: + while (!asm_event_load_before_wait(&tests_complete)) { + asm_event_wait(&tests_complete); + } + + return false; +} + +#else + +extern char unused; + +#endif diff --git a/hyp/mem/pgtable/armv8/pgtable.tc b/hyp/mem/pgtable/armv8/pgtable.tc index 6c501bb..7617899 100644 --- a/hyp/mem/pgtable/armv8/pgtable.tc +++ b/hyp/mem/pgtable/armv8/pgtable.tc @@ -6,8 +6,8 @@ define vmid_t public newtype uint16; #if defined(ARCH_AARCH64_USE_S2FWB) -#if !defined(ARCH_ARM_8_4_S2FWB) -#error S2FWB requires ARCH_ARM_8_4_S2FWB +#if !defined(ARCH_ARM_FEAT_S2FWB) +#error S2FWB requires ARCH_ARM_FEAT_S2FWB #endif #error S2FWB support not implemented #else @@ -46,7 +46,7 @@ define vmsa_block_entry bitfield<64> { 1:0 type uint8(const) = 0b1; 11:2 lower_attrs type vmsa_lower_attrs_t = 0; 15:12 unknown = 0; -#if defined(ARCH_ARM_8_4_TTREM) +#if defined(ARCH_ARM_FEAT_BBM) 16 nT bool = 0; #else 16 unknown = 0; @@ -82,6 +82,15 @@ define vmsa_page_and_block_attrs_entry bitfield<64> { 63:48 upper_attrs type vmsa_upper_attrs_t = 0; }; +// union type for use where the entry type is unknown +define vmsa_entry union { + base bitfield vmsa_general_entry; + page bitfield vmsa_page_entry; + block bitfield vmsa_block_entry; + table bitfield vmsa_table_entry; + attrs bitfield vmsa_page_and_block_attrs_entry; +}; + // common lower attribute structure define vmsa_common_lower_attrs bitfield<10> { 5:0 unknown; @@ -103,7 +112,7 @@ define vmsa_common_upper_attrs bitfield<16> { // page and block stage-1 upper attributes define vmsa_stg1_upper_attrs bitfield<16> { -#if defined(ARCH_ARM_8_5_BTI) +#if defined(ARCH_ARM_FEAT_BTI) 1:0 res0 uint8(const) = 0; 2 GP bool = 0; #else @@ -111,7 +120,7 @@ define vmsa_stg1_upper_attrs bitfield<16> { #endif 3 DBM bool = 0; 4 cont bool = 0; -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) 5 PXN bool = 1; 6 UXN bool = 1; #else @@ -152,7 +161,7 @@ define vmsa_stg2_upper_attrs bitfield<16> { 2:0 unknown = 0; 3 DBM bool = 0; 4 cont bool = 0; -#if defined(ARCH_ARM_8_2_TTS2UXN) +#if defined(ARCH_ARM_FEAT_XNX) 5 PXNxorUXN bool; 6 UXN bool; #else @@ -180,13 +189,19 @@ define vmsa_stg2_lower_attrs bitfield<10> { 9 unknown = 0; }; -define vmsa_tlbi_vae2_input bitfield<64> { +define vmsa_tlbi_va_input bitfield<64> { 43:0 VA type vmaddr_t lsl(12); 47:44 TTL uint8 = 0; 63:48 ASID uint16 = 0; }; -define vmsa_tlbi_ipas2e1is_input bitfield<64> { +define vmsa_tlbi_vaa_input bitfield<64> { + 43:0 VA type vmaddr_t lsl(12); + 47:44 TTL uint8 = 0; + 63:48 unknown = 0; +}; + +define vmsa_tlbi_ipa_input bitfield<64> { 35:0 IPA type vmaddr_t lsl(12); // without LPA implemented, or it's IPA 39:36 unknown = 0; @@ -197,8 +212,8 @@ define vmsa_tlbi_ipas2e1is_input bitfield<64> { 63 NS bool = 0; }; -#ifdef ARCH_ARM_8_4_TLBI -define vmsa_tlbi_vae2_range_input bitfield<64> { +#ifdef ARCH_ARM_FEAT_TLBIRANGE +define vmsa_tlbi_va_range_input bitfield<64> { 36:0 BaseADDR uint64; 38:37 TTL uint8 = 0; 43:39 NUM uint8; @@ -207,7 +222,7 @@ define vmsa_tlbi_vae2_range_input bitfield<64> { 63:48 ASID uint16 = 0; }; -define vmsa_tlbi_ipas2e1_range_input bitfield<64> { +define vmsa_tlbi_ipa_range_input bitfield<64> { 36:0 BaseADDR uint64; 38:37 TTL uint8 = 0; 43:39 NUM uint8; @@ -217,8 +232,8 @@ define vmsa_tlbi_ipas2e1_range_input bitfield<64> { 63 NS bool = 0; }; -define TLBI_RANGE_NUM_MAX constant = (1 << 5) - 1; -define TLBI_RANGE_SCALE_MAX constant = (1 << 2) - 1; +define TLBI_RANGE_NUM_MAX constant uint8 = (1 << 5) - 1; +define TLBI_RANGE_SCALE_MAX constant uint8 = (1 << 2) - 1; #endif // Gunyah extensions to table entries (software bit usage) @@ -247,23 +262,16 @@ define pgtable_stage_type enumeration(noprefix) { PGTABLE_VM_STAGE_2; }; -define pgtable_entry_types_t newtype uint32; - -define vmsa_entry_type enumeration(explicit) { - NEXT_LEVEL_TABLE = 0x1; - PAGE = 0x2; - BLOCK = 0x4; - INVALID = 0x8; - // Non-legal entry types - RESERVED = 0x10; - // Legal entry types in the wrong level (i.e. blocks in level 0 or 3, - // pages above level 3) - ERROR = 0x20; +define pgtable_entry_types bitfield<8> { + auto next_level_table bool; + auto page bool; + auto block bool; + auto invalid bool; + auto reserved bool; + auto error bool; + others unknown = 0; }; -// Alias for bitmap accepting no entry types -define VMSA_ENTRY_TYPE_NONE constant enumeration vmsa_entry_type = 0; - define pgtable_level enumeration(noprefix) { PGTABLE_LEVEL_0 = 0; PGTABLE_LEVEL_1; @@ -272,7 +280,7 @@ define pgtable_level enumeration(noprefix) { PGTABLE_LEVEL_OFFSET; }; -define PGTABLE_INVALID_LEVEL constant = 0xFFFFFFFF; +define PGTABLE_INVALID_LEVEL constant type index_t = -1; define pgtable_level_info structure { // [msb:lsb], range identifies the index in the address @@ -285,7 +293,7 @@ define pgtable_level_info structure { // useless bits block_and_page_output_address_mask type paddr_t; is_offset bool; - allowed_types type pgtable_entry_types_t; + allowed_types bitfield pgtable_entry_types; // the address range size of each entry on this level addr_size size; entry_cnt type count_t; @@ -305,6 +313,7 @@ define pgtable structure { start_level_size size; root_pgtable type paddr_t; + // FIXME: // root pointer type vmsa_level_table_t; root pointer bitfield vmsa_general_entry(atomic); @@ -319,7 +328,9 @@ extend pgtable_vm structure { }; define pgtable_hyp object { +#if defined(ARCH_ARM_FEAT_VHE) top_control structure pgtable; +#endif bottom_control structure pgtable; lock structure spinlock; }; @@ -345,7 +356,6 @@ define pgtable_translation_table_walk_event enumeration { }; define pgtable_map_modifier_args structure { - error enumeration error; orig_virtual_address type vmaddr_t; orig_size size; phys type paddr_t; @@ -359,26 +369,19 @@ define pgtable_map_modifier_args structure { // With value == -1 means there's no new page table allocated since // level 0 page table is always there. new_page_start_level type index_t; - is_failed bool; partially_mapped_size size; - try_map bool; + merge_limit size; + error enumeration error; stage enumeration pgtable_stage_type; + try_map bool; + outer_shareable bool; }; define pgtable_lookup_modifier_args structure { phys type paddr_t; size size; // must be a block/page entry - entry bitfield vmsa_general_entry; -}; - -define pgtable_remap_region structure { - is_valid bool; - virtual_address type vmaddr_t; - upper_attrs type vmsa_upper_attrs_t; - lower_attrs type vmsa_lower_attrs_t; - phys type paddr_t; - size size; + entry union vmsa_entry; }; define pgtable_unmap_modifier_args structure { @@ -386,13 +389,13 @@ define pgtable_unmap_modifier_args structure { // original virtual address, unchanged preserved_size size; stack array(maxof(enumeration pgtable_level) + 1) type paddr_t; - stage enumeration pgtable_stage_type; phys type paddr_t; size size; + stage enumeration pgtable_stage_type; + outer_shareable bool; }; define pgtable_prealloc_modifier_args structure { - error enumeration error; partition pointer object partition; // It needs to alloc new page table level during mapping, this start // index records the last level whose sub level needs to alloc page @@ -401,4 +404,5 @@ define pgtable_prealloc_modifier_args structure { // With value == -1 means there's no new page table allocated since // level 0 page table is always there. new_page_start_level type index_t; + error enumeration error; }; diff --git a/hyp/mem/pgtable/armv8/src/pgtable.c b/hyp/mem/pgtable/armv8/src/pgtable.c index 7cd43e5..d295d27 100644 --- a/hyp/mem/pgtable/armv8/src/pgtable.c +++ b/hyp/mem/pgtable/armv8/src/pgtable.c @@ -52,21 +52,29 @@ #include #endif +#ifdef HOST_TEST +#define TEST_EXPORTED +#else +#define TEST_EXPORTED static +#endif + #include #include "event_handlers.h" #include "events/pgtable.h" -#define SHIFT_4K (12) -#define SHIFT_16K (14) -#define SHIFT_64K (16) +#define SHIFT_4K (12U) +#define SHIFT_16K (14U) +#define SHIFT_64K (16U) // mask for [e, s] -#define segment_mask(e, s) (util_mask(e + 1) & (~util_mask(s))) +#define segment_mask(e, s) (util_mask((e) + 1U) & (~util_mask(s))) // Every legal entry type except next level tables static const pgtable_entry_types_t VMSA_ENTRY_TYPE_LEAF = - VMSA_ENTRY_TYPE_BLOCK | VMSA_ENTRY_TYPE_PAGE | VMSA_ENTRY_TYPE_INVALID; + pgtable_entry_types_cast(PGTABLE_ENTRY_TYPES_BLOCK_MASK | + PGTABLE_ENTRY_TYPES_PAGE_MASK | + PGTABLE_ENTRY_TYPES_INVALID_MASK); #if defined(HOST_TEST) // Definitions for host test @@ -75,17 +83,18 @@ bool pgtable_op = true; #define LOG(...) LOG_I(__VA_ARGS__, 5, 4, 3, 2, 1, 0, _unspecified_id) #define LOG_I(tclass, log_level, fmt, a0, a1, a2, a3, a4, n, ...) \ - LOG_##n(fmt, a0, a1, a2, a3, a4, __VA_ARGS__) -#define LOG_0(fmt, ...) LOG_V(fmt, 0, 0, 0, 0, 0) -#define LOG_1(fmt, a0, ...) LOG_V(fmt, a0, 0, 0, 0, 0) -#define LOG_2(fmt, a0, a1, ...) LOG_V(fmt, a0, a1, 0, 0, 0) -#define LOG_3(fmt, a0, a1, a2, ...) LOG_V(fmt, a0, a1, a2, 0, 0) -#define LOG_4(fmt, a0, a1, a2, a3, ...) LOG_V(fmt, a0, a1, a2, a3, 0) -#define LOG_5(fmt, a0, a1, a2, a3, a4, ...) LOG_V(fmt, a0, a1, a2, a3, a4) + LOG_##n((fmt), (a0), (a1), (a2), (a3), (a4), __VA_ARGS__) +#define LOG_0(fmt, ...) LOG_V((fmt), 0, 0, 0, 0, 0) +#define LOG_1(fmt, a0, ...) LOG_V((fmt), (a0), 0, 0, 0, 0) +#define LOG_2(fmt, a0, a1, ...) LOG_V((fmt), (a0), (a1), 0, 0, 0) +#define LOG_3(fmt, a0, a1, a2, ...) LOG_V((fmt), (a0), (a1), (a2), 0, 0) +#define LOG_4(fmt, a0, a1, a2, a3, ...) LOG_V((fmt), (a0), (a1), (a2), (a3), 0) +#define LOG_5(fmt, a0, a1, a2, a3, a4, ...) \ + LOG_V((fmt), (a0), (a1), (a2), (a3), (a4)) #define LOG_V(fmt, a0, a1, a2, a3, a4) \ do { \ char log[1024]; \ - snprint(log, 1024, fmt, a0, a1, a2, a3, a4); \ + snprint(log, 1024, (fmt), (a0), (a1), (a2), (a3), (a4)); \ puts(log); \ } while (0) @@ -103,11 +112,15 @@ extern bool pgtable_op; static _Thread_local bool pgtable_op; #endif -extern vmsa_general_entry_t aarch64_pt_ttbr1_level1; +extern vmsa_general_entry_t aarch64_pt_ttbr_level1; #endif -#define PGTABLE_LEVEL_NUM (PGTABLE_LEVEL__MAX + 1) +#if (CPU_PGTABLE_BBM_LEVEL > 0) && !defined(ARCH_ARM_FEAT_BBM) +#error CPU_PGTABLE_BBM_LEVEL > 0 but ARCH_ARM_FEAT_BBM not defined +#endif + +#define PGTABLE_LEVEL_NUM ((index_t)PGTABLE_LEVEL__MAX + 1U) typedef struct stack_elem { paddr_t paddr; @@ -132,56 +145,60 @@ static const pgtable_level_info_t info_4k_granules[PGTABLE_LEVEL_NUM] = { // level 0 { .msb = 47, .lsb = 39, - .table_mask = segment_mask(47, 12), + .table_mask = segment_mask(47U, 12), .block_and_page_output_address_mask = 0U, .is_offset = false, - .allowed_types = VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE, - .addr_size = 1UL << 39, - .entry_cnt = 1UL << 9, + .allowed_types = pgtable_entry_types_cast( + PGTABLE_ENTRY_TYPES_NEXT_LEVEL_TABLE_MASK), + .addr_size = util_bit(39), + .entry_cnt = (count_t)util_bit(9), .level = PGTABLE_LEVEL_0, .contiguous_entry_cnt = 0U }, // level 1 { .msb = 38, .lsb = 30, - .table_mask = segment_mask(47, 12), - .block_and_page_output_address_mask = segment_mask(47, 30), + .table_mask = segment_mask(47U, 12), + .block_and_page_output_address_mask = segment_mask(47U, 30), .is_offset = false, - .allowed_types = VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE | - VMSA_ENTRY_TYPE_BLOCK, - .addr_size = 1UL << 30, - .entry_cnt = 1UL << 9, + .allowed_types = pgtable_entry_types_cast( + PGTABLE_ENTRY_TYPES_NEXT_LEVEL_TABLE_MASK | + PGTABLE_ENTRY_TYPES_BLOCK_MASK), + .addr_size = util_bit(30), + .entry_cnt = (count_t)util_bit(9), .level = PGTABLE_LEVEL_1, .contiguous_entry_cnt = 16U }, // level 2 { .msb = 29, .lsb = 21, - .table_mask = segment_mask(47, 12), - .block_and_page_output_address_mask = segment_mask(47, 21), + .table_mask = segment_mask(47U, 12), + .block_and_page_output_address_mask = segment_mask(47U, 21), .is_offset = false, - .allowed_types = VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE | - VMSA_ENTRY_TYPE_BLOCK, - .addr_size = 1UL << 21, - .entry_cnt = 1UL << 9, + .allowed_types = pgtable_entry_types_cast( + PGTABLE_ENTRY_TYPES_NEXT_LEVEL_TABLE_MASK | + PGTABLE_ENTRY_TYPES_BLOCK_MASK), + .addr_size = util_bit(21), + .entry_cnt = (count_t)util_bit(9), .level = PGTABLE_LEVEL_2, .contiguous_entry_cnt = 16U }, // level 3 { .msb = 20, .lsb = 12, .table_mask = 0, - .block_and_page_output_address_mask = segment_mask(47, 12), + .block_and_page_output_address_mask = segment_mask(47U, 12), .is_offset = false, - .allowed_types = VMSA_ENTRY_TYPE_PAGE, - .addr_size = 1UL << 12, - .entry_cnt = 1UL << 9, - .level = PGTABLE_LEVEL_3, - .contiguous_entry_cnt = 16U }, + .allowed_types = + pgtable_entry_types_cast(PGTABLE_ENTRY_TYPES_PAGE_MASK), + .addr_size = util_bit(12), + .entry_cnt = (count_t)util_bit(9), + .level = PGTABLE_LEVEL_3, + .contiguous_entry_cnt = 16U }, // offset { .msb = 11, .lsb = 0, .table_mask = 0U, .block_and_page_output_address_mask = 0U, .is_offset = true, - .allowed_types = VMSA_ENTRY_TYPE_NONE, + .allowed_types = pgtable_entry_types_cast(0U), .addr_size = 0U, .entry_cnt = 0U, .level = PGTABLE_LEVEL_OFFSET, @@ -198,55 +215,59 @@ static const pgtable_level_info_t info_16k_granules[PGTABLE_LEVEL_NUM] = { // level 0 { .msb = 47, .lsb = 47, - .table_mask = segment_mask(47, 14), + .table_mask = segment_mask(47U, 14), .block_and_page_output_address_mask = 0U, .is_offset = false, - .allowed_types = VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE, - .addr_size = 1UL << 47, - .entry_cnt = 2U, + .allowed_types = pgtable_entry_types_cast( + PGTABLE_ENTRY_TYPES_NEXT_LEVEL_TABLE_MASK), + .addr_size = util_bit(47), + .entry_cnt = (count_t)util_bit(1), .level = PGTABLE_LEVEL_0, .contiguous_entry_cnt = 0U }, // level 1 { .msb = 46, .lsb = 36, - .table_mask = segment_mask(47, 14), - .block_and_page_output_address_mask = segment_mask(47, 36), + .table_mask = segment_mask(47U, 14), + .block_and_page_output_address_mask = segment_mask(47U, 36), .is_offset = false, - .allowed_types = VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE, - .addr_size = 1UL << 36, - .entry_cnt = 1UL << 11, + .allowed_types = pgtable_entry_types_cast( + PGTABLE_ENTRY_TYPES_NEXT_LEVEL_TABLE_MASK), + .addr_size = util_bit(36), + .entry_cnt = (count_t)util_bit(11), .level = PGTABLE_LEVEL_1, .contiguous_entry_cnt = 0U }, // level 2 { .msb = 35, .lsb = 25, - .table_mask = segment_mask(47, 14), - .block_and_page_output_address_mask = segment_mask(47, 25), + .table_mask = segment_mask(47U, 14), + .block_and_page_output_address_mask = segment_mask(47U, 25), .is_offset = false, - .allowed_types = VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE | - VMSA_ENTRY_TYPE_BLOCK, - .addr_size = 1UL << 25, - .entry_cnt = 1UL << 11, + .allowed_types = pgtable_entry_types_cast( + PGTABLE_ENTRY_TYPES_NEXT_LEVEL_TABLE_MASK | + PGTABLE_ENTRY_TYPES_BLOCK_MASK), + .addr_size = util_bit(25), + .entry_cnt = (count_t)util_bit(11), .level = PGTABLE_LEVEL_2, .contiguous_entry_cnt = 32U }, // level 3 { .msb = 24, .lsb = 14, .table_mask = 0U, - .block_and_page_output_address_mask = segment_mask(47, 14), + .block_and_page_output_address_mask = segment_mask(47U, 14), .is_offset = false, - .allowed_types = VMSA_ENTRY_TYPE_PAGE, - .addr_size = 1UL << 14, - .entry_cnt = 1UL << 11, - .level = PGTABLE_LEVEL_3, - .contiguous_entry_cnt = 128U }, + .allowed_types = + pgtable_entry_types_cast(PGTABLE_ENTRY_TYPES_PAGE_MASK), + .addr_size = util_bit(14), + .entry_cnt = (count_t)util_bit(11), + .level = PGTABLE_LEVEL_3, + .contiguous_entry_cnt = 128U }, // offset { .msb = 13, .lsb = 0, .table_mask = 0U, .block_and_page_output_address_mask = 0U, .is_offset = true, - .allowed_types = VMSA_ENTRY_TYPE_NONE, + .allowed_types = pgtable_entry_types_cast(0U), .addr_size = 0U, .entry_cnt = 0U, .level = PGTABLE_LEVEL_OFFSET, @@ -262,45 +283,48 @@ static const pgtable_level_info_t info_64k_granules[PGTABLE_LEVEL_NUM] = { // level 0 { .msb = 47, .lsb = 42, - .table_mask = segment_mask(47, 16), + .table_mask = segment_mask(47U, 16), .block_and_page_output_address_mask = 0U, .is_offset = false, // No LPA support, so no block type - .allowed_types = VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE, - .addr_size = 1UL << 42, - .entry_cnt = 1UL << 6, + .allowed_types = pgtable_entry_types_cast( + PGTABLE_ENTRY_TYPES_NEXT_LEVEL_TABLE_MASK), + .addr_size = util_bit(42), + .entry_cnt = (count_t)util_bit(6), .level = PGTABLE_LEVEL_1, .contiguous_entry_cnt = 0U }, // level 1 { .msb = 41, .lsb = 29, - .table_mask = segment_mask(47, 16), - .block_and_page_output_address_mask = segment_mask(47, 29), + .table_mask = segment_mask(47U, 16), + .block_and_page_output_address_mask = segment_mask(47U, 29), .is_offset = false, - .allowed_types = VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE | - VMSA_ENTRY_TYPE_BLOCK, - .addr_size = 1UL << 29, - .entry_cnt = 1UL << 13, + .allowed_types = pgtable_entry_types_cast( + PGTABLE_ENTRY_TYPES_NEXT_LEVEL_TABLE_MASK | + PGTABLE_ENTRY_TYPES_BLOCK_MASK), + .addr_size = util_bit(29), + .entry_cnt = (count_t)util_bit(13), .level = PGTABLE_LEVEL_2, .contiguous_entry_cnt = 32U }, // level 2 { .msb = 28, .lsb = 16, .table_mask = 0U, - .block_and_page_output_address_mask = segment_mask(47, 16), + .block_and_page_output_address_mask = segment_mask(47U, 16), .is_offset = false, - .allowed_types = VMSA_ENTRY_TYPE_PAGE, - .addr_size = 1UL << 16, - .entry_cnt = 1UL << 13, - .level = PGTABLE_LEVEL_3, - .contiguous_entry_cnt = 32U }, + .allowed_types = + pgtable_entry_types_cast(PGTABLE_ENTRY_TYPES_PAGE_MASK), + .addr_size = util_bit(16), + .entry_cnt = (count_t)util_bit(13), + .level = PGTABLE_LEVEL_3, + .contiguous_entry_cnt = 32U }, // offset { .msb = 15, .lsb = 0, .table_mask = 0U, .block_and_page_output_address_mask = 0U, .is_offset = true, - .allowed_types = VMSA_ENTRY_TYPE_NONE, + .allowed_types = pgtable_entry_types_cast(0U), .addr_size = 0U, .entry_cnt = 0U, .level = PGTABLE_LEVEL_OFFSET, @@ -319,20 +343,20 @@ void pgtable_hyp_dump(void); void -pgtable_vm_dump(pgtable_vm_t *pgtable); +pgtable_vm_dump(pgtable_vm_t *pgt); #endif // !defined(NDEBUG) #if defined(HOST_TEST) // Private type for external modifier, only used by test cases typedef pgtable_modifier_ret_t (*ext_func_t)( pgtable_t *pgt, vmaddr_t virtual_address, size_t size, index_t idx, - index_t level, vmsa_entry_type_t type, + index_t level, pgtable_entry_types_t type, stack_elem_t stack[PGTABLE_LEVEL_NUM], void *data, index_t *next_level, - vmaddr_t *next_virtual_address, size_t *next_size, paddr_t *next_table); + vmaddr_t *next_virtual_address, size_t *next_size, paddr_t next_table); typedef struct ext_modifier_args { ext_func_t func; - void *data; + void *data; } ext_modifier_args_t; void @@ -349,50 +373,48 @@ hyp_tlbi_va(vmaddr_t virtual_address) { // FIXME: before invalidate tlb, should we wait for all device/normal // memory write operations done. - vmsa_tlbi_vae2_input_t input; + vmsa_tlbi_va_input_t input; - vmsa_tlbi_vae2_input_init(&input); - vmsa_tlbi_vae2_input_set_VA(&input, virtual_address); + vmsa_tlbi_va_input_init(&input); + vmsa_tlbi_va_input_set_VA(&input, virtual_address); #ifndef HOST_TEST __asm__ volatile("tlbi VAE2IS, %[VA] ;" : "+m"(asm_ordering) - : [VA] "r"(vmsa_tlbi_vae2_input_raw(input))); + : [VA] "r"(vmsa_tlbi_va_input_raw(input))); #endif } static void -vm_tlbi_ipa(vmaddr_t virtual_address, bool issue_dvm_cmd) +vm_tlbi_ipa(vmaddr_t virtual_address, bool outer_shareable) { - vmsa_tlbi_ipas2e1is_input_t input; + vmsa_tlbi_ipa_input_t input; - vmsa_tlbi_ipas2e1is_input_init(&input); - vmsa_tlbi_ipas2e1is_input_set_IPA(&input, virtual_address); + vmsa_tlbi_ipa_input_init(&input); + vmsa_tlbi_ipa_input_set_IPA(&input, virtual_address); #ifndef HOST_TEST -#ifdef ARCH_ARM_8_4_TLBI - if (issue_dvm_cmd) { - __asm__ volatile( - "tlbi IPAS2E1OS, %[VA] ;" - : "+m"(asm_ordering) - : [VA] "r"(vmsa_tlbi_ipas2e1is_input_raw(input))); +#ifdef ARCH_ARM_FEAT_TLBIOS + if (outer_shareable) { + __asm__ volatile("tlbi IPAS2E1OS, %[VA] ;" + : "+m"(asm_ordering) + : [VA] "r"(vmsa_tlbi_ipa_input_raw(input))); } else { - __asm__ volatile( - "tlbi IPAS2E1IS, %[VA] ;" - : "+m"(asm_ordering) - : [VA] "r"(vmsa_tlbi_ipas2e1is_input_raw(input))); + __asm__ volatile("tlbi IPAS2E1IS, %[VA] ;" + : "+m"(asm_ordering) + : [VA] "r"(vmsa_tlbi_ipa_input_raw(input))); } #else - (void)issue_dvm_cmd; + (void)outer_shareable; __asm__ volatile("tlbi IPAS2E1IS, %[VA] ;" : "+m"(asm_ordering) - : [VA] "r"(vmsa_tlbi_ipas2e1is_input_raw(input))); + : [VA] "r"(vmsa_tlbi_ipa_input_raw(input))); #endif #endif } -#ifdef ARCH_ARM_8_4_TLBI +#ifdef ARCH_ARM_FEAT_TLBIRANGE static tlbi_range_tg_t hyp_tlbi_range_get_tg(count_t granule_shift) { @@ -415,120 +437,156 @@ hyp_tlbi_range_get_tg(count_t granule_shift) return tg; } +// Find a (scale, num) pair for the requested range. +// +// The range covered by the TLBI range instructions is: +// ((NUM + 1) * (2 ^ (5 * SCALE + 1)) * Translation_Granule_Size +// +// Returns false if the requested size is bigger than the maximum possible range +// size (8GB for 4K granules) after alignment. static bool hyp_tlbi_range_find_scale_num(uint64_t size, count_t granule_shift, uint8_t *scale, uint8_t *num) { - // Find a (scale, num) pair for the requested range - // The range covered by the TLBI range instructions is: - // ((NUM + 1) * (2 ^ (5 * SCALE + 1)) * Translation_Granule_Size - - uint64_t calc_num; uint8_t calc_scale; uint64_t granules = size >> granule_shift; - calc_scale = 0; - do { - count_t scale_shift = (5 * calc_scale) + 1; - uint64_t scale_mask = (1UL << scale_shift) - 1; - uint64_t aligned_granules = (granules + scale_mask) & - ~scale_mask; - calc_num = (aligned_granules >> scale_shift) - 1; + for (calc_scale = 0U; calc_scale <= TLBI_RANGE_SCALE_MAX; + calc_scale++) { + count_t scale_shift = (5U * (count_t)calc_scale) + 1U; + uint64_t aligned_granules = + util_p2align_up(granules, scale_shift); + uint64_t calc_num = (aligned_granules >> scale_shift) - 1U; if (calc_num <= TLBI_RANGE_NUM_MAX) { // Found a pair of scale, num + *scale = calc_scale; + *num = (uint8_t)calc_num; break; } - calc_scale++; - } while (calc_scale <= TLBI_RANGE_SCALE_MAX); - - *scale = calc_scale; - *num = (uint8_t)calc_num; + } return calc_scale <= TLBI_RANGE_SCALE_MAX; } static void -hyp_tlbi_va_range(vmaddr_t va_start, vmaddr_t va_end, count_t granule_shift) +hyp_tlbi_va_range(vmaddr_t va_start, size_t size, count_t granule_shift) { uint8_t num, scale; - uint64_t size = va_end - va_start; bool ret = hyp_tlbi_range_find_scale_num(size, granule_shift, &scale, &num); - assert(ret); - vmsa_tlbi_vae2_range_input_t input; - vmsa_tlbi_vae2_range_input_init(&input); - vmsa_tlbi_vae2_range_input_set_BaseADDR(&input, - va_start >> granule_shift); - vmsa_tlbi_vae2_range_input_set_NUM(&input, num); - vmsa_tlbi_vae2_range_input_set_SCALE(&input, scale); - vmsa_tlbi_vae2_range_input_set_TG(&input, - hyp_tlbi_range_get_tg(granule_shift)); + if (ret) { + vmsa_tlbi_va_range_input_t input; + vmsa_tlbi_va_range_input_init(&input); + vmsa_tlbi_va_range_input_set_BaseADDR( + &input, va_start >> granule_shift); + vmsa_tlbi_va_range_input_set_NUM(&input, num); + vmsa_tlbi_va_range_input_set_SCALE(&input, scale); + vmsa_tlbi_va_range_input_set_TG( + &input, hyp_tlbi_range_get_tg(granule_shift)); #ifndef HOST_TEST - __asm__ volatile("tlbi RVAE2IS, %[VA] ;" - : "+m"(asm_ordering) - : [VA] "r"(vmsa_tlbi_vae2_range_input_raw(input))); + __asm__ volatile( + "tlbi RVAE2IS, %[VA] ;" + : "+m"(asm_ordering) + : [VA] "r"(vmsa_tlbi_va_range_input_raw(input))); #endif + } else { + // Range is >8GB; flush the whole address space. + __asm__ volatile("tlbi ALLE2IS" : "+m"(asm_ordering)); + } } static void -hyp_tlbi_ipa_range(vmaddr_t ipa_start, vmaddr_t ipa_end, count_t granule_shift, - bool issue_dvm_cmd) +hyp_tlbi_ipa_range(vmaddr_t ipa_start, size_t size, count_t granule_shift, + bool outer_shareable) { uint8_t num, scale; - uint64_t size = ipa_end - ipa_start; bool ret = hyp_tlbi_range_find_scale_num(size, granule_shift, &scale, &num); - assert(ret); - - vmsa_tlbi_ipas2e1_range_input_t input; - vmsa_tlbi_ipas2e1_range_input_init(&input); - vmsa_tlbi_ipas2e1_range_input_set_BaseADDR(&input, - ipa_start >> granule_shift); - vmsa_tlbi_ipas2e1_range_input_set_NUM(&input, num); - vmsa_tlbi_ipas2e1_range_input_set_SCALE(&input, scale); - vmsa_tlbi_ipas2e1_range_input_set_TG( - &input, hyp_tlbi_range_get_tg(granule_shift)); + if (ret) { + vmsa_tlbi_ipa_range_input_t input; + vmsa_tlbi_ipa_range_input_init(&input); + vmsa_tlbi_ipa_range_input_set_BaseADDR( + &input, ipa_start >> granule_shift); + vmsa_tlbi_ipa_range_input_set_NUM(&input, num); + vmsa_tlbi_ipa_range_input_set_SCALE(&input, scale); + vmsa_tlbi_ipa_range_input_set_TG( + &input, hyp_tlbi_range_get_tg(granule_shift)); #ifndef HOST_TEST - if (issue_dvm_cmd) { - __asm__ volatile( - "tlbi RIPAS2E1OS, %[VA] ;" - : "+m"(asm_ordering) - : [VA] "r"(vmsa_tlbi_ipas2e1_range_input_raw(input))); - } else { +#if defined(ARCH_ARM_FEAT_TLBIOS) + if (outer_shareable) { + __asm__ volatile( + "tlbi RIPAS2E1OS, %[VA] ;" + : "+m"(asm_ordering) + : [VA] "r"( + vmsa_tlbi_ipa_range_input_raw(input))); + } else { + __asm__ volatile( + "tlbi RIPAS2E1IS, %[VA] ;" + : "+m"(asm_ordering) + : [VA] "r"( + vmsa_tlbi_ipa_range_input_raw(input))); + } +#else + (void)outer_shareable; __asm__ volatile( "tlbi RIPAS2E1IS, %[VA] ;" : "+m"(asm_ordering) - : [VA] "r"(vmsa_tlbi_ipas2e1_range_input_raw(input))); - } + : [VA] "r"(vmsa_tlbi_ipa_range_input_raw(input))); +#endif +#endif + } else { +#ifndef HOST_TEST + // Range is >8GB; flush the whole address space. +#if defined(ARCH_ARM_FEAT_TLBIOS) + if (outer_shareable) { + __asm__ volatile("tlbi VMALLS12E1OS" + : "+m"(asm_ordering)); + } else { + __asm__ volatile("tlbi VMALLS12E1IS" + : "+m"(asm_ordering)); + } +#else + __asm__ volatile("tlbi VMALLS12E1IS" : "+m"(asm_ordering)); #endif +#endif + } } #endif static void -dsb() +dsb(bool outer_shareable) { #ifndef HOST_TEST +#if defined(ARCH_ARM_FEAT_TLBIOS) + if (outer_shareable) { + __asm__ volatile("dsb osh" ::: "memory"); + } else { + __asm__ volatile("dsb ish" ::: "memory"); + } +#else + (void)outer_shareable; __asm__ volatile("dsb ish" ::: "memory"); #endif +#endif } static void -vm_tlbi_vmalle1(bool issue_dvm_cmd) +vm_tlbi_vmalle1(bool outer_shareable) { #ifndef HOST_TEST -#ifdef ARCH_ARM_8_4_TLBI - if (issue_dvm_cmd) { +#ifdef ARCH_ARM_FEAT_TLBIOS + if (outer_shareable) { __asm__ volatile("tlbi VMALLE1OS" ::: "memory"); } else { __asm__ volatile("tlbi VMALLE1IS" ::: "memory"); } #else - (void)issue_dvm_cmd; + (void)outer_shareable; __asm__ volatile("tlbi VMALLE1IS" ::: "memory"); #endif #endif @@ -545,32 +603,28 @@ addr_check(vmaddr_t virtual_address, size_t bit_count, bool is_high); #if defined(HOST_TEST) // Unit test need these helper functions -vmsa_general_entry_t +vmsa_entry_t get_entry(vmsa_level_table_t *table, index_t idx); -vmsa_entry_type_t -get_entry_type(vmsa_general_entry_t *entry, - const pgtable_level_info_t *level_info); +pgtable_entry_types_t +get_entry_type(vmsa_entry_t *entry, const pgtable_level_info_t *level_info); -error_t -get_entry_paddr(const pgtable_level_info_t *level_info, - vmsa_general_entry_t *entry, vmsa_entry_type_t type, - paddr_t *paddr); +void +get_entry_paddr(const pgtable_level_info_t *level_info, vmsa_entry_t *entry, + pgtable_entry_types_t type, paddr_t *paddr); count_t get_table_refcount(vmsa_level_table_t *table, index_t idx); #else -static vmsa_general_entry_t +static vmsa_entry_t get_entry(vmsa_level_table_t *table, index_t idx); -static vmsa_entry_type_t -get_entry_type(vmsa_general_entry_t *entry, - const pgtable_level_info_t *level_info); +static pgtable_entry_types_t +get_entry_type(vmsa_entry_t *entry, const pgtable_level_info_t *level_info); -static error_t -get_entry_paddr(const pgtable_level_info_t *level_info, - vmsa_general_entry_t *entry, vmsa_entry_type_t type, - paddr_t *paddr); +static void +get_entry_paddr(const pgtable_level_info_t *level_info, vmsa_entry_t *entry, + pgtable_entry_types_t type, paddr_t *paddr); static count_t get_table_refcount(vmsa_level_table_t *table, index_t idx); @@ -586,10 +640,10 @@ static pgtable_hyp_memtype_t map_stg1_attr_to_memtype(vmsa_lower_attrs_t attrs); static vmsa_lower_attrs_t -get_lower_attr(vmsa_general_entry_t entry); +get_lower_attr(vmsa_entry_t entry); static vmsa_upper_attrs_t -get_upper_attr(vmsa_general_entry_t entry); +get_upper_attr(vmsa_entry_t entry); static pgtable_access_t map_stg1_attr_to_access(vmsa_upper_attrs_t upper_attrs, @@ -598,8 +652,8 @@ map_stg1_attr_to_access(vmsa_upper_attrs_t upper_attrs, static void map_stg2_attr_to_access(vmsa_upper_attrs_t upper_attrs, vmsa_lower_attrs_t lower_attrs, - pgtable_access_t *kernel_access, - pgtable_access_t *user_access); + pgtable_access_t *kernel_access, + pgtable_access_t *user_access); static void map_stg2_memtype_to_attrs(pgtable_vm_memtype_t memtype, @@ -635,9 +689,9 @@ set_page_entry(vmsa_level_table_t *table, index_t idx, paddr_t addr, static void set_block_entry(vmsa_level_table_t *table, index_t idx, paddr_t addr, vmsa_upper_attrs_t upper_attrs, vmsa_lower_attrs_t lower_attrs, - bool contiguous, bool fence); + bool contiguous, bool fence, bool notlb); -#if CPU_PGTABLE_BLOCK_SPLIT_LEVEL == 1U +#if CPU_PGTABLE_BBM_LEVEL == 1U static void set_notlb_flag(vmsa_label_table_t *table, index_t idx, bool nt); #endif @@ -648,8 +702,7 @@ static bool translation_table_walk(pgtable_t *pgt, vmaddr_t virtual_address, size_t virtual_address_size, pgtable_translation_table_walk_event_t event, - pgtable_entry_types_t target_types, void *data, - bool issue_dvm_cmd); + pgtable_entry_types_t expected, void *data); static error_t alloc_level_table(partition_t *partition, size_t size, size_t alignment, @@ -657,18 +710,19 @@ alloc_level_table(partition_t *partition, size_t size, size_t alignment, static void set_pgtables(vmaddr_t virtual_address, stack_elem_t stack[PGTABLE_LEVEL_NUM], - index_t start_level, index_t cur_level, count_t initial_refcount); + index_t first_new_table_level, index_t cur_level, + count_t initial_refcount, index_t start_level); static pgtable_modifier_ret_t -map_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, index_t idx, - index_t level, vmsa_entry_type_t type, - stack_elem_t stack[PGTABLE_LEVEL_NUM], void *data, - index_t *next_level, vmaddr_t *next_virtual_address, - size_t *next_size, paddr_t *next_table, bool issue_dvm_cmd); +map_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, + vmsa_entry_t cur_entry, index_t idx, index_t cur_level, + pgtable_entry_types_t type, stack_elem_t stack[PGTABLE_LEVEL_NUM], + void *data, index_t *next_level, vmaddr_t *next_virtual_address, + size_t *next_size, paddr_t next_table); static pgtable_modifier_ret_t -lookup_modifier(pgtable_t *pgt, vmsa_general_entry_t cur_entry, index_t level, - vmsa_entry_type_t type, void *data); +lookup_modifier(pgtable_t *pgt, vmsa_entry_t cur_entry, index_t level, + pgtable_entry_types_t type, void *data); static void check_refcount(pgtable_t *pgt, partition_t *partition, vmaddr_t virtual_address, @@ -690,31 +744,36 @@ static void unmap_clear_cont_bit(vmsa_level_table_t *table, vmaddr_t virtual_address, index_t level, vmsa_page_and_block_attrs_entry_t attr_entry, - pgtable_unmap_modifier_args_t *margs, - count_t granule_shift, bool issue_dvm_cmd); + pgtable_unmap_modifier_args_t *margs, + count_t granule_shift, index_t start_level); static pgtable_modifier_ret_t unmap_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, - index_t idx, index_t level, vmsa_entry_type_t type, + index_t idx, index_t level, pgtable_entry_types_t type, stack_elem_t stack[PGTABLE_LEVEL_NUM], void *data, index_t *next_level, vmaddr_t *next_virtual_address, - size_t *next_size, paddr_t *next_table, bool only_matching, - bool issue_dvm_cmd); + size_t *next_size, bool only_matching); static pgtable_modifier_ret_t prealloc_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, - index_t level, vmsa_entry_type_t type, + index_t level, pgtable_entry_types_t type, stack_elem_t stack[PGTABLE_LEVEL_NUM], void *data, index_t *next_level, vmaddr_t *next_virtual_address, - size_t *next_size, paddr_t *next_table); + size_t *next_size); // Return entry idx, it can make sure the returned index is always in the // range static inline index_t -get_index(vmaddr_t addr, const pgtable_level_info_t *info) +get_index(vmaddr_t addr, const pgtable_level_info_t *info, bool is_first_level) { - return (index_t)((addr & segment_mask(info->msb, info->lsb)) >> - info->lsb); + index_t index; + if (is_first_level) { + index = (index_t)(addr >> info->lsb); + } else { + index = (index_t)((addr & segment_mask(info->msb, info->lsb)) >> + info->lsb); + } + return index; } #ifndef NDEBUG @@ -741,22 +800,22 @@ static inline size_t size_on_level(vmaddr_t virtual_address, size_t size, const pgtable_level_info_t *level_info) { - vmaddr_t v_s = virtual_address, v_e = virtual_address + size - 1; + vmaddr_t v_s = virtual_address, v_e = virtual_address + size - 1U; vmaddr_t l_s = 0U, l_e = 0U; - assert(!util_add_overflows(virtual_address, size - 1)); + assert(!util_add_overflows(virtual_address, size - 1U)); l_s = (virtual_address >> level_info->lsb) << level_info->lsb; // even for the level 0, it will set the bit 49, will not overflow for // 64 bit - l_e = l_s + level_info->addr_size - 1; + l_e = l_s + level_info->addr_size - 1U; - assert(!util_add_overflows(l_s, level_info->addr_size - 1)); + assert(!util_add_overflows(l_s, level_info->addr_size - 1U)); l_s = util_max(l_s, v_s); l_e = util_min(v_e, l_e); - return l_e - l_s + 1; + return l_e - l_s + 1U; } static inline vmaddr_t @@ -774,18 +833,49 @@ is_preserved_table_entry(size_t preserved_size, return preserved_size < level_info->addr_size; } +bool +pgtable_access_check(pgtable_access_t access, pgtable_access_t access_check) +{ + return (((register_t)access & (register_t)access_check) == + (register_t)access_check); +} + +bool +pgtable_access_is_equal(pgtable_access_t access, pgtable_access_t access_check) +{ + return ((register_t)access == (register_t)access_check); +} + +pgtable_access_t +pgtable_access_mask(pgtable_access_t access, pgtable_access_t access_mask) +{ + register_t combined_access = + ((register_t)access & (register_t)access_mask); + return (pgtable_access_t)combined_access; +} + +pgtable_access_t +pgtable_access_combine(pgtable_access_t access1, pgtable_access_t access2) +{ + register_t combined_access = + ((register_t)access1 | (register_t)access2); + return (pgtable_access_t)combined_access; +} + // Helper function to manipulate page table entry -vmsa_general_entry_t +TEST_EXPORTED vmsa_entry_t get_entry(vmsa_level_table_t *table, index_t idx) { partition_phys_access_enable(&table[idx]); - vmsa_general_entry_t entry = - atomic_load_explicit(&table[idx], memory_order_relaxed); + vmsa_entry_t entry = { + .base = atomic_load_explicit(&table[idx], memory_order_relaxed), + }; + partition_phys_access_disable(&table[idx]); return entry; } -bool +static bool is_high_virtual_address(vmaddr_t virtual_address) { #if ARCH_IS_64BIT @@ -798,7 +888,7 @@ is_high_virtual_address(vmaddr_t virtual_address) #endif } -bool +static bool addr_check(vmaddr_t virtual_address, size_t bit_count, bool is_high) { #if ARCH_IS_64BIT @@ -813,106 +903,93 @@ addr_check(vmaddr_t virtual_address, size_t bit_count, bool is_high) return count <= bit_count; } -vmsa_entry_type_t -get_entry_type(vmsa_general_entry_t *entry, - const pgtable_level_info_t *level_info) +TEST_EXPORTED pgtable_entry_types_t +get_entry_type(vmsa_entry_t *entry, const pgtable_level_info_t *level_info) { - vmsa_entry_type_t ret; - - if (vmsa_general_entry_get_is_valid(entry)) { - if (vmsa_general_entry_get_is_table(entry)) { - if (level_info->allowed_types & - VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE) { - ret = VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE; + pgtable_entry_types_t ret = pgtable_entry_types_default(); + vmsa_general_entry_t g = entry->base; + + if (vmsa_general_entry_get_is_valid(&g)) { + if (vmsa_general_entry_get_is_table(&g)) { + if (pgtable_entry_types_get_next_level_table( + &level_info->allowed_types)) { + pgtable_entry_types_set_next_level_table(&ret, + true); } else { - ret = VMSA_ENTRY_TYPE_PAGE; + pgtable_entry_types_set_page(&ret, true); } } else { - if (level_info->allowed_types & VMSA_ENTRY_TYPE_BLOCK) { - ret = VMSA_ENTRY_TYPE_BLOCK; + if (pgtable_entry_types_get_block( + &level_info->allowed_types)) { + pgtable_entry_types_set_block(&ret, true); } else { - ret = VMSA_ENTRY_TYPE_RESERVED; + pgtable_entry_types_set_reserved(&ret, true); } } } else { - ret = VMSA_ENTRY_TYPE_INVALID; + pgtable_entry_types_set_invalid(&ret, true); } return ret; } -error_t -get_entry_paddr(const pgtable_level_info_t *level_info, - vmsa_general_entry_t *entry, vmsa_entry_type_t type, - paddr_t *paddr) +TEST_EXPORTED void +get_entry_paddr(const pgtable_level_info_t *level_info, vmsa_entry_t *entry, + pgtable_entry_types_t type, paddr_t *paddr) { - error_t ret = OK; - vmsa_block_entry_t *blk; - vmsa_page_entry_t *pg; - vmsa_table_entry_t *tb; + vmsa_block_entry_t blk; + vmsa_page_entry_t pg; + vmsa_table_entry_t tb; *paddr = 0U; - switch (type) { - case VMSA_ENTRY_TYPE_BLOCK: - blk = (vmsa_block_entry_t *)entry; - *paddr = vmsa_block_entry_get_OutputAddress(blk) & + if (pgtable_entry_types_get_block(&type)) { + blk = entry->block; + *paddr = vmsa_block_entry_get_OutputAddress(&blk) & level_info->block_and_page_output_address_mask; - break; - case VMSA_ENTRY_TYPE_PAGE: - pg = (vmsa_page_entry_t *)entry; - *paddr = vmsa_page_entry_get_OutputAddress(pg) & + } else if (pgtable_entry_types_get_page(&type)) { + pg = entry->page; + *paddr = vmsa_page_entry_get_OutputAddress(&pg) & level_info->block_and_page_output_address_mask; - break; - - case VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE: - tb = (vmsa_table_entry_t *)entry; - *paddr = vmsa_table_entry_get_NextLevelTableAddress(tb) & + } else if (pgtable_entry_types_get_next_level_table(&type)) { + tb = entry->table; + *paddr = vmsa_table_entry_get_NextLevelTableAddress(&tb) & level_info->table_mask; - break; - - case VMSA_ENTRY_TYPE_INVALID: - case VMSA_ENTRY_TYPE_RESERVED: - case VMSA_ENTRY_TYPE_ERROR: - default: - ret = ERROR_ARGUMENT_INVALID; - break; + } else { + panic("Invalid entry get paddr"); } - - return ret; } -count_t +TEST_EXPORTED count_t get_table_refcount(vmsa_level_table_t *table, index_t idx) { - vmsa_general_entry_t g; - vmsa_table_entry_t *entry = (vmsa_table_entry_t *)&g; + vmsa_entry_t g = get_entry(table, idx); + vmsa_table_entry_t entry = g.table; - g = get_entry(table, idx); - return vmsa_table_entry_get_refcount(entry); + return vmsa_table_entry_get_refcount(&entry); } static inline void -set_table_refcount(vmsa_level_table_t *table, index_t idx, count_t count) +set_table_refcount(vmsa_level_table_t *table, index_t idx, count_t refcount) { - vmsa_general_entry_t g; - vmsa_table_entry_t *val = (vmsa_table_entry_t *)&g; + vmsa_entry_t g = get_entry(table, idx); + vmsa_table_entry_t val = g.table; - g = get_entry(table, idx); - vmsa_table_entry_set_refcount(val, count); + vmsa_table_entry_set_refcount(&val, refcount); partition_phys_access_enable(&table[idx]); - atomic_store_explicit(&table[idx], g, memory_order_relaxed); + g.table = val; + atomic_store_explicit(&table[idx], g.base, memory_order_relaxed); partition_phys_access_disable(&table[idx]); } -pgtable_vm_memtype_t +static pgtable_vm_memtype_t map_stg2_attr_to_memtype(vmsa_lower_attrs_t attrs) { vmsa_stg2_lower_attrs_t val = vmsa_stg2_lower_attrs_cast(attrs); return vmsa_stg2_lower_attrs_get_mem_attr(&val); } -pgtable_hyp_memtype_t +static pgtable_hyp_memtype_t map_stg1_attr_to_memtype(vmsa_lower_attrs_t attrs) { vmsa_stg1_lower_attrs_t val = vmsa_stg1_lower_attrs_cast(attrs); @@ -920,23 +997,21 @@ map_stg1_attr_to_memtype(vmsa_lower_attrs_t attrs) return vmsa_stg1_lower_attrs_get_attr_idx(&val); } -vmsa_lower_attrs_t -get_lower_attr(vmsa_general_entry_t entry) +static vmsa_lower_attrs_t +get_lower_attr(vmsa_entry_t entry) { - vmsa_page_and_block_attrs_entry_t *val = - (vmsa_page_and_block_attrs_entry_t *)&entry; - return vmsa_page_and_block_attrs_entry_get_lower_attrs(val); + vmsa_page_and_block_attrs_entry_t val = entry.attrs; + return vmsa_page_and_block_attrs_entry_get_lower_attrs(&val); } -vmsa_upper_attrs_t -get_upper_attr(vmsa_general_entry_t entry) +static vmsa_upper_attrs_t +get_upper_attr(vmsa_entry_t entry) { - vmsa_page_and_block_attrs_entry_t *val = - (vmsa_page_and_block_attrs_entry_t *)&entry; - return vmsa_page_and_block_attrs_entry_get_upper_attrs(val); + vmsa_page_and_block_attrs_entry_t val = entry.attrs; + return vmsa_page_and_block_attrs_entry_get_upper_attrs(&val); } -pgtable_access_t +static pgtable_access_t map_stg1_attr_to_access(vmsa_upper_attrs_t upper_attrs, vmsa_lower_attrs_t lower_attrs) { @@ -946,7 +1021,7 @@ map_stg1_attr_to_access(vmsa_upper_attrs_t upper_attrs, vmsa_stg1_ap_t ap = VMSA_STG1_AP_EL0_NONE_UPPER_READ_ONLY; pgtable_access_t ret = PGTABLE_ACCESS_NONE; -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) xn = vmsa_stg1_upper_attrs_get_PXN(&u); #else xn = vmsa_stg1_upper_attrs_get_XN(&u); @@ -974,16 +1049,19 @@ map_stg1_attr_to_access(vmsa_upper_attrs_t upper_attrs, case VMSA_STG1_AP_EL0_NONE_UPPER_READ_ONLY: ret = xn ? PGTABLE_ACCESS_R : PGTABLE_ACCESS_RX; break; + default: + // Access None + break; } return ret; } -void +static void map_stg2_attr_to_access(vmsa_upper_attrs_t upper_attrs, vmsa_lower_attrs_t lower_attrs, - pgtable_access_t *kernel_access, - pgtable_access_t *user_access) + pgtable_access_t *kernel_access, + pgtable_access_t *user_access) { // Map from S2AP to R and W access bits static const pgtable_access_t stg2_ap_map[] = { @@ -1003,20 +1081,21 @@ map_stg2_attr_to_access(vmsa_upper_attrs_t upper_attrs, vmsa_s2ap_t ap = vmsa_stg2_lower_attrs_get_S2AP(&l); pgtable_access_t rw = stg2_ap_map[ap]; -#if defined(ARCH_ARM_8_2_TTS2UXN) +#if defined(ARCH_ARM_FEAT_XNX) bool uxn = vmsa_stg2_upper_attrs_get_UXN(&u); bool pxn_xor_uxn = vmsa_stg2_upper_attrs_get_PXNxorUXN(&u); - bool pxn = pxn_xor_uxn ^ uxn; - *user_access = rw | (uxn ? 0U : PGTABLE_ACCESS_X); - *kernel_access = rw | (pxn ? 0U : PGTABLE_ACCESS_X); + bool pxn = (bool)(pxn_xor_uxn ^ uxn); + *user_access = uxn ? rw : pgtable_access_combine(rw, PGTABLE_ACCESS_X); + *kernel_access = pxn ? rw + : pgtable_access_combine(rw, PGTABLE_ACCESS_X); #else bool xn = vmsa_stg2_upper_attrs_get_XN(&u); - *user_access = rw | (xn ? 0U : PGTABLE_ACCESS_X); + *user_access = xn ? rw : pgtable_access_combine(rw, PGTABLE_ACCESS_X); *kernel_access = *user_access; #endif } -void +static void map_stg2_memtype_to_attrs(pgtable_vm_memtype_t memtype, vmsa_stg2_lower_attrs_t *lower_attrs) { @@ -1046,22 +1125,24 @@ map_stg2_memtype_to_attrs(pgtable_vm_memtype_t memtype, default: vmsa_stg2_lower_attrs_set_SH(lower_attrs, VMSA_SHAREABILITY_NON_SHAREABLE); + break; } } -void +static void map_stg1_memtype_to_attrs(pgtable_hyp_memtype_t memtype, vmsa_stg1_lower_attrs_t *lower_attrs) { vmsa_stg1_lower_attrs_set_attr_idx(lower_attrs, memtype); } -void +static void map_stg1_access_to_attrs(pgtable_access_t access, vmsa_stg1_upper_attrs_t *upper_attrs, vmsa_stg1_lower_attrs_t *lower_attrs) { - uint8_t xn, ap; + bool xn; + vmsa_stg1_ap_t ap; switch (access) { case PGTABLE_ACCESS_RX: @@ -1101,32 +1182,35 @@ map_stg1_access_to_attrs(pgtable_access_t access, } vmsa_stg1_lower_attrs_set_AP(lower_attrs, ap); -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) vmsa_stg1_upper_attrs_set_PXN(upper_attrs, xn); + // For compatibility with EL2 single stage MMU keep these equal. + vmsa_stg1_upper_attrs_set_UXN(upper_attrs, xn); #else vmsa_stg1_upper_attrs_set_XN(upper_attrs, xn); #endif -#if defined(ARCH_ARM_8_5_BTI) +#if defined(ARCH_ARM_FEAT_BTI) // Guard the executable pages only if requested from platform vmsa_stg1_upper_attrs_set_GP(upper_attrs, (!xn && platform_cpu_bti_enabled())); #endif } -void +static void map_stg2_access_to_attrs(pgtable_access_t kernel_access, pgtable_access_t user_access, vmsa_stg2_upper_attrs_t *upper_attrs, vmsa_stg2_lower_attrs_t *lower_attrs) { - bool kernel_exec = ((kernel_access & PGTABLE_ACCESS_X) != 0); - bool user_exec = ((user_access & PGTABLE_ACCESS_X) != 0); + bool kernel_exec = + pgtable_access_check(kernel_access, PGTABLE_ACCESS_X); + bool user_exec = pgtable_access_check(user_access, PGTABLE_ACCESS_X); -#if defined(ARCH_ARM_8_2_TTS2UXN) +#if defined(ARCH_ARM_FEAT_XNX) vmsa_stg2_upper_attrs_set_UXN(upper_attrs, !user_exec); vmsa_stg2_upper_attrs_set_PXNxorUXN(upper_attrs, - !kernel_exec ^ !user_exec); + (bool)(!kernel_exec ^ !user_exec)); #else vmsa_stg2_upper_attrs_set_XN(upper_attrs, !kernel_exec || !user_exec); #endif @@ -1136,8 +1220,8 @@ map_stg2_access_to_attrs(pgtable_access_t kernel_access, vmsa_s2ap_t ap; static_assert(PGTABLE_ACCESS_X == 1, "expect PGTABLE_ACCESS_X is bit 0"); - assert(((kernel_access ^ kernel_access) >> 1) == 0); - assert(kernel_access != PGTABLE_ACCESS_X); + assert(((kernel_access ^ user_access) >> 1) == 0); + assert(!pgtable_access_is_equal(kernel_access, PGTABLE_ACCESS_X)); switch (kernel_access) { case PGTABLE_ACCESS_R: @@ -1161,7 +1245,7 @@ map_stg2_access_to_attrs(pgtable_access_t kernel_access, vmsa_stg2_lower_attrs_set_S2AP(lower_attrs, ap); } -void +static void set_invalid_entry(vmsa_level_table_t *table, index_t idx) { vmsa_general_entry_t entry = { 0 }; @@ -1171,95 +1255,100 @@ set_invalid_entry(vmsa_level_table_t *table, index_t idx) partition_phys_access_disable(&table[idx]); } -void +static void set_table_entry(vmsa_level_table_t *table, index_t idx, paddr_t addr, count_t count) { - vmsa_general_entry_t g; - vmsa_table_entry_t *entry = (vmsa_table_entry_t *)&g; + vmsa_table_entry_t entry = vmsa_table_entry_default(); - vmsa_table_entry_init(entry); - vmsa_table_entry_set_NextLevelTableAddress(entry, addr); - vmsa_table_entry_set_refcount(entry, count); + vmsa_table_entry_set_NextLevelTableAddress(&entry, addr); + vmsa_table_entry_set_refcount(&entry, count); partition_phys_access_enable(&table[idx]); - atomic_store_explicit(&table[idx], g, memory_order_release); + vmsa_entry_t g = { .table = entry }; + atomic_store_explicit(&table[idx], g.base, memory_order_release); partition_phys_access_disable(&table[idx]); } -void +static void set_page_entry(vmsa_level_table_t *table, index_t idx, paddr_t addr, vmsa_upper_attrs_t upper_attrs, vmsa_lower_attrs_t lower_attrs, bool contiguous, bool fence) { - vmsa_general_entry_t g; - vmsa_page_entry_t *entry = (vmsa_page_entry_t *)&g; + vmsa_page_entry_t entry = vmsa_page_entry_default(); vmsa_common_upper_attrs_t u; - vmsa_page_entry_init(entry); - u = vmsa_common_upper_attrs_cast(upper_attrs); vmsa_common_upper_attrs_set_cont(&u, contiguous); - vmsa_page_entry_set_lower_attrs(entry, lower_attrs); + vmsa_page_entry_set_lower_attrs(&entry, lower_attrs); vmsa_page_entry_set_upper_attrs( - entry, (vmsa_upper_attrs_t)vmsa_common_upper_attrs_raw(u)); - vmsa_page_entry_set_OutputAddress(entry, addr); + &entry, (vmsa_upper_attrs_t)vmsa_common_upper_attrs_raw(u)); + vmsa_page_entry_set_OutputAddress(&entry, addr); partition_phys_access_enable(&table[idx]); + vmsa_entry_t g = { .page = entry }; if (fence) { - atomic_store_explicit(&table[idx], g, memory_order_release); + atomic_store_explicit(&table[idx], g.base, + memory_order_release); } else { - atomic_store_explicit(&table[idx], g, memory_order_relaxed); + atomic_store_explicit(&table[idx], g.base, + memory_order_relaxed); } partition_phys_access_disable(&table[idx]); } -void +static void set_block_entry(vmsa_level_table_t *table, index_t idx, paddr_t addr, vmsa_upper_attrs_t upper_attrs, vmsa_lower_attrs_t lower_attrs, - bool contiguous, bool fence) + bool contiguous, bool fence, bool notlb) { - vmsa_general_entry_t g; - vmsa_block_entry_t *entry = (vmsa_block_entry_t *)&g; + vmsa_block_entry_t entry = vmsa_block_entry_default(); vmsa_common_upper_attrs_t u; - vmsa_block_entry_init(entry); - u = vmsa_common_upper_attrs_cast(upper_attrs); vmsa_common_upper_attrs_set_cont(&u, contiguous); - vmsa_block_entry_set_lower_attrs(entry, lower_attrs); + vmsa_block_entry_set_lower_attrs(&entry, lower_attrs); vmsa_block_entry_set_upper_attrs( - entry, (vmsa_upper_attrs_t)vmsa_common_upper_attrs_raw(u)); - vmsa_block_entry_set_OutputAddress(entry, addr); + &entry, (vmsa_upper_attrs_t)vmsa_common_upper_attrs_raw(u)); + vmsa_block_entry_set_OutputAddress(&entry, addr); +#if CPU_PGTABLE_BBM_LEVEL > 0U + vmsa_block_entry_set_nT(&entry, notlb); +#else + (void)notlb; +#endif partition_phys_access_enable(&table[idx]); + vmsa_entry_t g = { .block = entry }; if (fence) { - atomic_store_explicit(&table[idx], g, memory_order_release); + atomic_store_explicit(&table[idx], g.base, + memory_order_release); } else { - atomic_store_explicit(&table[idx], g, memory_order_relaxed); + atomic_store_explicit(&table[idx], g.base, + memory_order_relaxed); } partition_phys_access_disable(&table[idx]); } -#if CPU_PGTABLE_BLOCK_SPLIT_LEVEL == 1U +#if CPU_PGTABLE_BBM_LEVEL == 1U static void set_notlb_flag(vmsa_label_table_t *table, index_t idx, bool nt) { - vmsa_general_entry_t g; - vmsa_block_entry_t *entry = (vmsa_block_entry_t *)&g; + vmsa_block_entry_t entry; partition_phys_access_enable(&table[idx]); atomic_load_explicit(&table[idx], entry, memory_order_relaxed); - assert(get_entry_type(&g) == VMSA_ENTRY_TYPE_BLOCK); - vmsa_block_entry_set_nT(g, nt); - atomic_store_explicit(&table[idx], entry, memory_order_relaxed); + vmsa_general_entry_t g = { .block = entry_cnt }; + pgtable_entry_types_t type = get_entry_type(&g); + assert(pgtable_entry_types_get_block(&type)); + vmsa_block_entry_set_nT(g.block, nt); + atomic_store_explicit(&table[idx], g.base, memory_order_relaxed); partition_phys_access_disable(&table[idx]); } -#endif // CPU_PGTABLE_BLOCK_SPLIT_LEVEL == 1U +#endif // CPU_PGTABLE_BBM_LEVEL == 1U -error_t +static error_t alloc_level_table(partition_t *partition, size_t size, size_t alignment, paddr_t *paddr, vmsa_level_table_t **table) { @@ -1267,12 +1356,13 @@ alloc_level_table(partition_t *partition, size_t size, size_t alignment, // actually only used to allocate a page alloc_ret = partition_alloc(partition, size, alignment); - if (alloc_ret.e == OK) { - memset(alloc_ret.r, 0, size); + if (compiler_expected(alloc_ret.e == OK)) { + (void)memset_s(alloc_ret.r, size, 0, size); *table = (vmsa_level_table_t *)alloc_ret.r; *paddr = partition_virt_to_phys(partition, (uintptr_t)alloc_ret.r); + assert(*paddr != PADDR_INVALID); } return alloc_ret.e; } @@ -1280,37 +1370,39 @@ alloc_level_table(partition_t *partition, size_t size, size_t alignment, // Helper function to map all sub page table/set entry count, following a FIFO // order, so the last entry to write is the one which actually hook the whole // new page table levels on the existing page table. -void +static void set_pgtables(vmaddr_t virtual_address, stack_elem_t stack[PGTABLE_LEVEL_NUM], - index_t start_level, index_t cur_level, count_t initial_refcount) + index_t first_new_table_level, index_t cur_level, + count_t initial_refcount, index_t start_level) { paddr_t lower; vmsa_level_table_t *table; const pgtable_level_info_t *level_info = NULL; index_t idx; - vmsa_general_entry_t g; - vmsa_entry_type_t type; + vmsa_entry_t g; + pgtable_entry_types_t type; count_t refcount = initial_refcount; index_t level = cur_level; - while (start_level < level) { + while (first_new_table_level < level) { lower = stack[level].paddr; - table = stack[level - 1].table; + table = stack[level - 1U].table; - assert(stack[level - 1].mapped); + assert(stack[level - 1U].mapped); - level_info = &level_conf[level - 1]; + level_info = &level_conf[level - 1U]; - idx = get_index(virtual_address, level_info); + idx = get_index(virtual_address, level_info, + ((level - 1U) == start_level)); g = get_entry(table, idx); type = get_entry_type(&g, level_info); - if (type == VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE) { + if (pgtable_entry_types_get_next_level_table(&type)) { // This should be the last level we are updating - assert(start_level == (level - 1U)); + assert(first_new_table_level == (level - 1U)); // Update the table's entry count - refcount = get_table_refcount(table, idx) + 1; + refcount = get_table_refcount(table, idx) + 1U; set_table_refcount(table, idx, refcount); } else { // Write the table entry. @@ -1324,11 +1416,36 @@ set_pgtables(vmaddr_t virtual_address, stack_elem_t stack[PGTABLE_LEVEL_NUM], } } +// Check if the existing mapping can remain unchanged. +static bool +pgtable_maybe_keep_mapping(vmsa_entry_t cur_entry, pgtable_entry_types_t type, + pgtable_map_modifier_args_t *margs, index_t level) +{ + assert(pgtable_entry_types_get_block(&type) || + pgtable_entry_types_get_page(&type)); + const pgtable_level_info_t *cur_level_info = &level_conf[level]; + + paddr_t expected_phys = margs->phys & ~util_mask(cur_level_info->lsb); + + paddr_t phys_addr; + get_entry_paddr(cur_level_info, &cur_entry, type, &phys_addr); + vmsa_upper_attrs_t upper_attrs = get_upper_attr(cur_entry); + vmsa_lower_attrs_t lower_attrs = get_lower_attr(cur_entry); + + bool keep_mapping = (phys_addr == expected_phys) && + (upper_attrs == margs->upper_attrs) && + (lower_attrs == margs->lower_attrs); + if (keep_mapping) { + margs->phys = expected_phys + cur_level_info->addr_size; + } + return keep_mapping; +} + // Check if only the page access needs to be changed and update it. static bool -pgtable_maybe_update_access(pgtable_t *pgt, +pgtable_maybe_update_access(pgtable_t *pgt, stack_elem_t stack[PGTABLE_LEVEL_NUM], index_t idx, - vmsa_entry_type_t type, + pgtable_entry_types_t type, pgtable_map_modifier_args_t *margs, index_t level, vmaddr_t virtual_address, size_t size, vmaddr_t *next_virtual_address, size_t *next_size, @@ -1344,8 +1461,9 @@ pgtable_maybe_update_access(pgtable_t *pgt, size_t addr_size = cur_level_info->addr_size; vmaddr_t entry_virtual_address = entry_start_address(virtual_address, cur_level_info); + vmaddr_t start_virtual_address = virtual_address; - if ((type == VMSA_ENTRY_TYPE_BLOCK) && + if (pgtable_entry_types_get_block(&type) && ((virtual_address != entry_virtual_address) || (size < addr_size))) { goto out; @@ -1360,11 +1478,13 @@ pgtable_maybe_update_access(pgtable_t *pgt, vmsa_level_table_t *table = stack[level].table; partition_phys_access_enable(&table[0]); while (idx != idx_stop) { - vmsa_general_entry_t cur_entry = - atomic_load_explicit(&table[idx], memory_order_relaxed); + vmsa_entry_t cur_entry = { + .base = atomic_load_explicit(&table[idx], + memory_order_relaxed), + }; vmsa_upper_attrs_t upper_attrs = get_upper_attr(cur_entry); vmsa_lower_attrs_t lower_attrs = get_lower_attr(cur_entry); -#if defined(ARCH_ARM_8_2_TTS2UXN) +#if defined(ARCH_ARM_FEAT_XNX) uint64_t xn_mask = VMSA_STG2_UPPER_ATTRS_UXN_MASK | VMSA_STG2_UPPER_ATTRS_PXNXORUXN_MASK; #else @@ -1372,6 +1492,12 @@ pgtable_maybe_update_access(pgtable_t *pgt, #endif uint64_t s2ap_mask = VMSA_STG2_LOWER_ATTRS_S2AP_MASK; + pgtable_entry_types_t cur_type = + get_entry_type(&cur_entry, cur_level_info); + if (!pgtable_entry_types_is_equal(cur_type, type)) { + goto out_access; + } + paddr_t phys_addr; get_entry_paddr(&level_conf[level], &cur_entry, type, &phys_addr); @@ -1393,38 +1519,63 @@ pgtable_maybe_update_access(pgtable_t *pgt, goto out_access; } - vmsa_page_entry_t *entry = (vmsa_page_entry_t *)&cur_entry; + vmsa_page_entry_t entry = cur_entry.page; if ((upper_attrs & xn_mask) != (margs->upper_attrs & xn_mask)) { - vmsa_page_entry_set_upper_attrs(entry, + vmsa_page_entry_set_upper_attrs(&entry, margs->upper_attrs); } if ((lower_attrs & s2ap_mask) != (margs->lower_attrs & s2ap_mask)) { - vmsa_page_entry_set_lower_attrs(entry, + vmsa_page_entry_set_lower_attrs(&entry, margs->lower_attrs); } - atomic_store_explicit(&table[idx], cur_entry, + cur_entry.page = entry; + atomic_store_explicit(&table[idx], cur_entry.base, memory_order_release); idx++; cur_phys += cur_level_info->addr_size; + virtual_address += cur_level_info->addr_size; } partition_phys_access_disable(&table[0]); ret = true; size_t updated_size = cur_phys - margs->phys; - *next_size = size - updated_size; - virtual_address += updated_size; +#if defined(ARCH_ARM_FEAT_TLBIRANGE) + if (margs->stage == PGTABLE_HYP_STAGE_1) { + dsb(false); + hyp_tlbi_va_range(start_virtual_address, updated_size, + pgt->granule_shift); + } else { + dsb(margs->outer_shareable); + hyp_tlbi_ipa_range(start_virtual_address, updated_size, + pgt->granule_shift, margs->outer_shareable); + } +#else + dsb(margs->outer_shareable); + + for (size_t offset = 0U; offset < updated_size; offset += addr_size) { + if (margs->stage == PGTABLE_HYP_STAGE_1) { + hyp_tlbi_va(start_virtual_address + offset); + } else { + vm_tlbi_ipa(start_virtual_address + offset, + margs->outer_shareable); + } + } +#endif + + *next_size = size - updated_size; + margs->phys = cur_phys; *next_virtual_address = virtual_address; // Walk back up the tree if needed if (idx == stack[level].entry_cnt) { - idx -= 1; // Last updated index - while (idx == stack[level].entry_cnt - 1) { + idx -= 1U; // Last updated index + while (idx == stack[level].entry_cnt - 1U) { if (level == pgt->start_level) { break; } else { @@ -1434,7 +1585,8 @@ pgtable_maybe_update_access(pgtable_t *pgt, cur_level_info = &level_conf[level]; // virtual_address is already stepped, use previous one // to check - idx = get_index(virtual_address, cur_level_info); + idx = get_index(virtual_address, cur_level_info, + (level == pgt->start_level)); } *next_level = level; } @@ -1451,8 +1603,7 @@ pgtable_add_table_entry(pgtable_t *pgt, pgtable_map_modifier_args_t *margs, index_t cur_level, stack_elem_t *stack, vmaddr_t virtual_address, size_t size, index_t *next_level, vmaddr_t *next_virtual_address, - size_t *next_size, paddr_t *next_table, - bool set_start_level) + size_t *next_size, bool set_start_level) { error_t ret; paddr_t new_pgtable_paddr; @@ -1473,26 +1624,28 @@ pgtable_add_table_entry(pgtable_t *pgt, pgtable_map_modifier_args_t *margs, if ((margs->new_page_start_level == PGTABLE_INVALID_LEVEL) && set_start_level) { margs->new_page_start_level = - level > pgt->start_level ? level - 1 : level; + level > pgt->start_level ? level - 1U : level; } - if (level >= (PGTABLE_LEVEL_NUM - 1)) { + if (level >= (PGTABLE_LEVEL_NUM - 1U)) { LOG(ERROR, WARN, "invalid level ({:d}).\n", level); ret = ERROR_ARGUMENT_INVALID; goto out; } // just record the new level in the stack - stack[level + 1].paddr = new_pgtable_paddr; - stack[level + 1].table = new_pgt; - stack[level + 1].mapped = true; - stack[level + 1].entry_cnt = level_conf[level + 1].entry_cnt; + stack[level + 1U] = (stack_elem_t){ + .paddr = new_pgtable_paddr, + .table = new_pgt, + .mapped = true, + .need_unmap = false, + .entry_cnt = level_conf[level + 1U].entry_cnt, + }; // guide translation_table_walk step into the new sub page table // level - *next_level = level + 1; + *next_level = level + 1U; *next_virtual_address = virtual_address; - *next_table = new_pgtable_paddr; *next_size = size; out: @@ -1505,34 +1658,31 @@ pgtable_add_table_entry(pgtable_t *pgt, pgtable_map_modifier_args_t *margs, // overlaps but is not completely covered by the range of the operation. static pgtable_modifier_ret_t pgtable_split_block(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, - index_t idx, index_t level, vmsa_entry_type_t type, + vmsa_entry_t cur_entry, index_t idx, index_t level, + pgtable_entry_types_t type, stack_elem_t stack[PGTABLE_LEVEL_NUM], pgtable_map_modifier_args_t *margs, index_t *next_level, - vmaddr_t *next_virtual_address, size_t *next_size, - paddr_t *next_table, bool issue_dvm_cmd) - + vmaddr_t *next_virtual_address, size_t *next_size) { pgtable_modifier_ret_t vret = PGTABLE_MODIFIER_RET_CONTINUE; error_t ret; - assert((level_conf[level].allowed_types & ENUM_VMSA_ENTRY_TYPE_BLOCK) != - 0U); + assert(pgtable_entry_types_get_block(&level_conf[level].allowed_types)); // Get the values of the block before it is invalidated const pgtable_level_info_t *cur_level_info = &level_conf[level]; size_t addr_size = cur_level_info->addr_size; vmaddr_t entry_virtual_address = entry_start_address(virtual_address, cur_level_info); - vmsa_general_entry_t cur_entry = get_entry(stack[level].table, idx); - vmsa_upper_attrs_t cur_upper_attrs = get_upper_attr(cur_entry); - vmsa_lower_attrs_t cur_lower_attrs = get_lower_attr(cur_entry); - paddr_t phys_addr; + vmsa_upper_attrs_t cur_upper_attrs = get_upper_attr(cur_entry); + vmsa_lower_attrs_t cur_lower_attrs = get_lower_attr(cur_entry); + paddr_t phys_addr; get_entry_paddr(cur_level_info, &cur_entry, type, &phys_addr); -#if CPU_PGTABLE_BLOCK_SPLIT_LEVEL < 2U +#if (CPU_PGTABLE_BBM_LEVEL < 2U) && !defined(PLATFORM_PGTABLE_AVOID_BBM) // We can't just replace the large entry; coherency might be broken. We // need a TLB flush. -#if CPU_PGTABLE_BLOCK_SPLIT_LEVEL < 1U +#if CPU_PGTABLE_BBM_LEVEL == 0U // The nT bit is not supported; we need a full break-before-make // sequence, with an invalid entry in the page table. This might trigger // spurious stage 2 faults on other cores or SMMUs. @@ -1541,33 +1691,31 @@ pgtable_split_block(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, // The nT bit is supported; we can set it before flushing the TLB to // ensure that the large mapping isn't cached again before we replace it // with the split mappings, but the entry itself can remain valid so - // other cores and SMMUs won't fault. + // SMMUs and other cores may not fault. set_notlb_flag(stack[level].table, idx, true); #endif - // Execute a DSB to ensure that the entry update above is complete - // before starting the TLB flush - dsb(); - // Flush the TLB entry if (margs->stage == PGTABLE_HYP_STAGE_1) { + dsb(false); hyp_tlbi_va(entry_virtual_address); } else { - vm_tlbi_ipa(entry_virtual_address, issue_dvm_cmd); - } - - if (margs->stage == PGTABLE_VM_STAGE_2) { + dsb(margs->outer_shareable); + vm_tlbi_ipa(entry_virtual_address, margs->outer_shareable); // The full stage-1 flushing below is really sub-optimal. - dsb(); - vm_tlbi_vmalle1(issue_dvm_cmd); + // FIXME: + dsb(margs->outer_shareable); + vm_tlbi_vmalle1(margs->outer_shareable); } -#else // CPU_PGTABLE_BLOCK_SPLIT_LEVEL >= 2U - (void)issue_dvm_cmd; -#endif // CPU_PGTABLE_BLOCK_SPLIT_LEVEL >= 2U +#else // (CPU_PGTABLE_BBM_LEVEL >= 2U) || PLATFORM_PGTABLE_AVOID_BBM + // Either the CPU supports page size changes without BBM, or we must + // avoid BBM to prevent unrecoverable SMMU faults. We will just replace + // the block entry with the split table entry. +#endif ret = pgtable_add_table_entry(pgt, margs, level, stack, virtual_address, size, next_level, next_virtual_address, - next_size, next_table, false); + next_size, false); if (ret != OK) { vret = PGTABLE_MODIFIER_RET_ERROR; goto out; @@ -1593,25 +1741,25 @@ pgtable_split_block(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, bool contiguous = false; #endif bool page_block_fence; - index_t start_level; + index_t new_page_start_level; if (margs->new_page_start_level != PGTABLE_INVALID_LEVEL) { - start_level = margs->new_page_start_level; + new_page_start_level = margs->new_page_start_level; page_block_fence = false; margs->new_page_start_level = PGTABLE_INVALID_LEVEL; } else { - start_level = level > pgt->start_level ? level - 1 : level; - page_block_fence = true; + new_page_start_level = level > pgt->start_level ? level - 1U + : level; + page_block_fence = true; } // Create all pages that cover the old block and hook them to // the new table entry assert(virtual_address >= entry_virtual_address); - assert(type == VMSA_ENTRY_TYPE_BLOCK); - const vmsa_entry_type_t page_or_block_type = - cur_level_info->allowed_types & - (VMSA_ENTRY_TYPE_BLOCK | VMSA_ENTRY_TYPE_PAGE); + assert(pgtable_entry_types_get_block(&type)); + const bool use_block = + pgtable_entry_types_get_block(&cur_level_info->allowed_types); paddr_t phys; idx = 0; @@ -1625,10 +1773,10 @@ pgtable_split_block(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, phys_addr += page_size; - if (page_or_block_type == VMSA_ENTRY_TYPE_BLOCK) { + if (use_block) { set_block_entry(stack[level].table, idx, phys, upper_attrs, lower_attrs, contiguous, - page_block_fence); + page_block_fence, false); } else { set_page_entry(stack[level].table, idx, phys, upper_attrs, lower_attrs, contiguous, @@ -1639,23 +1787,299 @@ pgtable_split_block(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, idx++; } -#if CPU_PGTABLE_BLOCK_SPLIT_LEVEL < 2U +#if (CPU_PGTABLE_BBM_LEVEL < 2U) && !defined(PLATFORM_PGTABLE_AVOID_BBM) + // Wait for the TLB flush before inserting the new table entry + dsb(margs->outer_shareable); +#endif + set_pgtables(entry_virtual_address, stack, new_page_start_level, level, + new_pages, pgt->start_level); + +#if (CPU_PGTABLE_BBM_LEVEL >= 2U) || defined(PLATFORM_PGTABLE_AVOID_BBM) + // Flush the old entry from the TLB now, to avoid TLB conflicts later. + if (margs->stage == PGTABLE_HYP_STAGE_1) { + dsb(false); + hyp_tlbi_va(entry_virtual_address); + } else { + dsb(margs->outer_shareable); + vm_tlbi_ipa(entry_virtual_address, margs->outer_shareable); + } +#endif + +out: + return vret; +} + +// Attempt to merge a subtree into a single block. +// +// This is called when a remap operation encounters a next-level table entry. It +// checks whether the map operation is able to merge the next-level table into a +// single large block, and replaces it with that block if possible. +// +// The criteria are as follows: +// +// - This level's entry size is less than the operation's merge limit +// - Block mappings are allowed at this level +// - The input and output addresses are equal modulo this level's entry size +// - The input address range is aligned to the next level's entry size +// - If the next-level table is allowed to contain next-level table entries, it +// does not contain any such entries +// - If try_map is true, any entries in the next-level table that are inside the +// mapped region are invalid +// - If the next-level table is not completely within the mapped region, any +// entries that are outside the mapped region are congruent, i.e. have the +// same attributes and physical addresses that they would have if the mapped +// region was extended to cover them +static pgtable_modifier_ret_t +pgtable_maybe_merge_block(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, + vmsa_entry_t cur_entry, index_t idx, index_t level, + pgtable_entry_types_t type, + stack_elem_t stack[PGTABLE_LEVEL_NUM], + pgtable_map_modifier_args_t *margs, + index_t *next_level, paddr_t next_table_paddr) +{ + assert(pgtable_entry_types_get_next_level_table(&type)); + assert(*next_level < PGTABLE_LEVEL_NUM); + + pgtable_modifier_ret_t vret = PGTABLE_MODIFIER_RET_CONTINUE; + const pgtable_level_info_t *cur_level_info = &level_conf[level]; + + if (cur_level_info->addr_size >= margs->merge_limit) { + // Block size exceeds the merge limit. There are three reasons + // we enforce this: + // + // - If a VM address space with an SMMU or other IOMMU attached + // that can't handle break-before-make or TLB conflicts + // safely, we need to disable merging entirely. + // + // - In the hypervisor address space, we need to avoid merging + // pages in a way that might cause a fault on an address that + // is touched during the hypervisor's handling of that fault, + // since that would fault recursively. + // + // - In the hypervisor address space, we need to avoid merging + // across partition ownership boundaries, because that might + // free the next-level table into the wrong partition. + goto out; + } + + if (!pgtable_entry_types_get_block(&cur_level_info->allowed_types)) { + // Block entries aren't possible at this level. This might + // happen if merge limit is set to ~0U for a VM address space. + goto out; + } + + if ((virtual_address & util_mask(cur_level_info->lsb)) != + (margs->phys & util_mask(cur_level_info->lsb))) { + // Input and output addresses misaligned for block size. + goto out; + } + + const pgtable_level_info_t *next_level_info = &level_conf[*next_level]; + size_t level_size = + size_on_level(virtual_address, size, cur_level_info); + if (((virtual_address & util_mask(next_level_info->lsb)) != 0U) || + ((level_size & util_mask(next_level_info->lsb)) != 0U)) { + // Mapping will be partial at the next level. A merge + // might be possible at the next level, but we currently + // can't extend it to this level as that requires a + // multi-level search for non-congruent entries. + goto out; + } + + count_t covered_entries = (count_t)(level_size >> next_level_info->lsb); + count_t other_entries = next_level_info->entry_cnt - covered_entries; + + count_t table_refcount = + vmsa_table_entry_get_refcount(&cur_entry.table); + + if (table_refcount < other_entries) { + // Some of the entries not covered by the map operation must be + // invalid, so a merge won't be possible. + goto out; + } + + vmaddr_t entry_virtual_address = + entry_start_address(virtual_address, cur_level_info); + paddr_t entry_phys = margs->phys & ~util_mask(cur_level_info->lsb); + + bool need_search = ((pgtable_entry_types_get_next_level_table( + &next_level_info->allowed_types)) || + (other_entries > 0U) || + (margs->try_map && (table_refcount > 0U))); + + if (need_search) { + // It's possible that the level to be merged contains entries + // that will prevent the merge. Map the level to be merged. + vmsa_level_table_t *next_table = + (vmsa_level_table_t *)partition_phys_map( + next_table_paddr, util_bit(pgt->granule_shift)); + if (next_table == NULL) { + LOG(ERROR, WARN, + "Failed to map table (pa {:#x}, level {:d}) for merge\n", + next_table_paddr, *next_level); + vret = PGTABLE_MODIFIER_RET_ERROR; + goto out; + } + + // Check for any next-level entries that will prevent merge + vmaddr_t next_level_addr = entry_virtual_address; + paddr_t expected_phys = entry_phys; + index_t next_level_idx; + for (next_level_idx = 0U; + next_level_idx < next_level_info->entry_cnt; + next_level_idx++) { + vmsa_entry_t next_level_entry = + get_entry(next_table, next_level_idx); + pgtable_entry_types_t next_level_type = get_entry_type( + &next_level_entry, next_level_info); + if (pgtable_entry_types_get_next_level_table( + &next_level_type)) { + // We don't try to handle multi-level merges. + // It's complex and typically not worth the + // effort. + break; + } + if ((margs->try_map && + (!pgtable_entry_types_get_invalid( + &next_level_type))) || + ((next_level_addr < virtual_address) || + ((next_level_addr + next_level_info->addr_size - + 1U) > (virtual_address + size - 1U)))) { + // This entry is not completely covered by the + // current map operation. It must be valid, and + // must map the same physical address as the + // current map operation would, with the same + // attributes. + if ((!pgtable_entry_types_get_block( + &next_level_type)) && + (!pgtable_entry_types_get_page( + &next_level_type))) { + // Not valid. Merging would incorrectly + // map it. + break; + } + + paddr_t phys_addr; + get_entry_paddr(next_level_info, + &next_level_entry, + next_level_type, &phys_addr); + vmsa_upper_attrs_t upper_attrs = + get_upper_attr(next_level_entry); + vmsa_lower_attrs_t lower_attrs = + get_lower_attr(next_level_entry); + if ((phys_addr != expected_phys) || + (upper_attrs != margs->upper_attrs) || + (lower_attrs != margs->lower_attrs)) { + // Inconsistent mapping; can't merge. + break; + } + } + expected_phys += next_level_info->addr_size; + next_level_addr += next_level_info->addr_size; + } + + partition_phys_unmap(next_table, next_table_paddr, + util_bit(pgt->granule_shift)); + + if (next_level_idx < next_level_info->entry_cnt) { + // We exited the next-level table check early, which + // means we found an entry that prevented merge. Nothing + // more to do. + goto out; + } + } + + assert(stack[level].mapped); + vmsa_level_table_t *cur_table = stack[level].table; + +#if (CPU_PGTABLE_BBM_LEVEL < 1U) && !defined(PLATFORM_PGTABLE_AVOID_BBM) + // The nT bit is not supported; we need a full break-before-make + // sequence, with an invalid entry in the page table. This might + // trigger spurious stage 2 faults on other cores or SMMUs. + set_invalid_entry(stack[level].table, idx); +#elif (CPU_PGTABLE_BBM_LEVEL < 2U) && !defined(PLATFORM_PGTABLE_AVOID_BBM) + // We can write the new block entry with the nT bit set, and then flush + // the old page entries. The new entry will stay out of the TLBs until + // we clear the nT bit below, so there will be no TLB conflicts, but + // there may be translation faults depending on the CPU implementation. + set_block_entry(cur_table, idx, entry_phys, margs->upper_attrs, + margs->lower_attrs, false, false, true); + +#else // (CPU_PGTABLE_BBM_LEVEL >= 2U) || PLATFORM_PGTABLE_AVOID_BBM + + // Either the CPU supports block size changes without BBM, or we must + // avoid BBM operations to prevent unrecoverable SMMU faults. + // + // We can just go ahead and write the new block entry. We still flush + // the TLB afterwards to avoid TLB conflicts. It is probably cheaper to + // do that now than to take up to 512 TLB conflict faults later, + // especially if the CPU supports range flushes. + set_block_entry(cur_table, idx, entry_phys, margs->upper_attrs, + margs->lower_attrs, false, false, false); + +#endif + + // Flush the TLB entries for the merged pages. + vmaddr_t next_level_addr = entry_virtual_address; +#ifdef ARCH_ARM_FEAT_TLBIRANGE + if (margs->stage == PGTABLE_HYP_STAGE_1) { + dsb(false); + hyp_tlbi_va_range(entry_virtual_address, + cur_level_info->addr_size, + pgt->granule_shift); + } else { + dsb(margs->outer_shareable); + hyp_tlbi_ipa_range(next_level_addr, cur_level_info->addr_size, + pgt->granule_shift, margs->outer_shareable); + } +#else + dsb((margs->stage != PGTABLE_HYP_STAGE_1) && margs->outer_shareable); + for (index_t i = 0; i < next_level_info->entry_cnt; i++) { + if (margs->stage == PGTABLE_HYP_STAGE_1) { + hyp_tlbi_va(next_level_addr); + } else { + vm_tlbi_ipa(next_level_addr, margs->outer_shareable); + } + next_level_addr += next_level_info->addr_size; + } +#endif + +#if (CPU_PGTABLE_BBM_LEVEL < 2U) && !defined(PLATFORM_PGTABLE_AVOID_BBM) + if (margs->stage != PGTABLE_HYP_STAGE_1) { + // The full stage-1 flushing below is really sub-optimal. + // FIXME: + dsb(margs->outer_shareable); + vm_tlbi_vmalle1(margs->outer_shareable); + } + // Wait for the TLB flush before inserting the new table entry - dsb(); + dsb((margs->stage != PGTABLE_HYP_STAGE_1) && margs->outer_shareable); + + set_block_entry(cur_table, idx, entry_phys, margs->upper_attrs, + margs->lower_attrs, false, false, false); #endif - set_pgtables(entry_virtual_address, stack, start_level, level, - new_pages); + + // Release the page table memory + (void)partition_free_phys(margs->partition, next_table_paddr, + util_bit(pgt->granule_shift)); + + // Ensure that translation_table_walk revisits the entry we just + // replaced, instead of traversing into the now-freed table. We don't + // update the virt or phys addresses or size, because the next call to + // map_modifier() will call pgtable_maybe_keep_mapping() to do it. + *next_level = level; + out: return vret; } static pgtable_modifier_ret_t pgtable_modify_mapping(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, - index_t idx, index_t cur_level, vmsa_entry_type_t type, + vmsa_entry_t cur_entry, index_t idx, index_t cur_level, + pgtable_entry_types_t type, stack_elem_t stack[PGTABLE_LEVEL_NUM], pgtable_map_modifier_args_t *margs, index_t *next_level, - vmaddr_t *next_virtual_address, size_t *next_size, - paddr_t *next_table, bool issue_dvm_cmd) + vmaddr_t *next_virtual_address, size_t *next_size) { pgtable_modifier_ret_t vret = PGTABLE_MODIFIER_RET_CONTINUE; index_t level = cur_level; @@ -1665,37 +2089,42 @@ pgtable_modify_mapping(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, vmaddr_t entry_virtual_address = entry_start_address(virtual_address, cur_level_info); - if ((type == VMSA_ENTRY_TYPE_BLOCK) && + if (pgtable_entry_types_get_block(&type) && ((virtual_address != entry_virtual_address) || (size < addr_size))) { // Split the block into pages - vret = pgtable_split_block(pgt, virtual_address, size, idx, - level, type, stack, margs, - next_level, next_virtual_address, - next_size, next_table, - issue_dvm_cmd); + vret = pgtable_split_block(pgt, virtual_address, size, + cur_entry, idx, level, type, stack, + margs, next_level, + next_virtual_address, next_size); } else { - // The new mapping will cover this entire range, so we need to - // unmap existing pages or blocks. The new mappings will then - // be mapped by the caller. + // The new mapping will cover this entire range, either because + // it's a single page, or because it's a block that didn't need + // to be split. We need to unmap the existing page or block. + pgtable_unmap_modifier_args_t margs2 = { 0 }; - pgtable_unmap_modifier_args_t margs2; - memset(&margs2, 0, sizeof(margs2)); margs2.partition = margs->partition; margs2.preserved_size = PGTABLE_HYP_UNMAP_PRESERVE_NONE; margs2.stage = margs->stage; - // it's a page entry - unmap_modifier(pgt, virtual_address, addr_size, idx, cur_level, - type, stack, &margs2, next_level, - next_virtual_address, next_size, next_table, - false, issue_dvm_cmd); - dsb(); + vret = unmap_modifier(pgt, virtual_address, addr_size, idx, + cur_level, type, stack, &margs2, + next_level, next_virtual_address, + next_size, false); + if (margs->stage == PGTABLE_VM_STAGE_2) { // flush entire stage 1 tlb - vm_tlbi_vmalle1(issue_dvm_cmd); - dsb(); + dsb(margs->outer_shareable); + vm_tlbi_vmalle1(margs->outer_shareable); + dsb(margs->outer_shareable); + } else { + dsb(false); } + + // Retry at the same address, so we can do the make part of the + // break-before-make sequence. + *next_virtual_address = virtual_address; + *next_size = size; } return vret; @@ -1710,102 +2139,92 @@ pgtable_modify_mapping(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, // * allocate/setup the page table level, and recursively (up to MAX_LEVEL) to // handle the mapping. After the current page table level entry setup, it will // drive @see translation_table_walk to the next entry at the same level. -pgtable_modifier_ret_t -map_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, index_t idx, - index_t cur_level, vmsa_entry_type_t type, - stack_elem_t stack[PGTABLE_LEVEL_NUM], void *data, - index_t *next_level, vmaddr_t *next_virtual_address, - size_t *next_size, paddr_t *next_table, bool issue_dvm_cmd) +static pgtable_modifier_ret_t +map_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, + vmsa_entry_t cur_entry, index_t idx, index_t cur_level, + pgtable_entry_types_t type, stack_elem_t stack[PGTABLE_LEVEL_NUM], + void *data, index_t *next_level, vmaddr_t *next_virtual_address, + size_t *next_size, paddr_t next_table) { pgtable_map_modifier_args_t *margs = (pgtable_map_modifier_args_t *)data; - pgtable_modifier_ret_t vret = PGTABLE_MODIFIER_RET_CONTINUE; - error_t ret = OK; - const pgtable_level_info_t *cur_level_info = NULL; - vmsa_level_table_t *cur_table = NULL; - uint64_t page_or_block_type = 0U; - size_t addr_size = 0U, level_size = 0U; - uint64_t allowed = 0U; - index_t start_level = 0; - bool page_block_fence = false; - index_t level = cur_level; - - // If entry different from ENTRY_INVALID case, it means that the - // specified range has already been mapped. - // If try_map is set, we will abort the mapping operation. - // If is is not set, we will first unmap and then proceed mapping the - // range anyways - if ((type == VMSA_ENTRY_TYPE_BLOCK) || (type == VMSA_ENTRY_TYPE_PAGE) || - (type == VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE)) { + pgtable_modifier_ret_t vret = PGTABLE_MODIFIER_RET_CONTINUE; + + // Attempt merging or replacement of small mappings. + if (pgtable_entry_types_get_next_level_table(&type)) { + vret = pgtable_maybe_merge_block(pgt, virtual_address, size, + cur_entry, idx, cur_level, + type, stack, margs, next_level, + next_table); + goto out; + } + + // Handle existing block or page mappings. + if (!pgtable_entry_types_get_invalid(&type)) { + // If the existing mapping is consistent with the required + // mapping, we don't need to do anything, even if try_map is + // true. + if (pgtable_maybe_keep_mapping(cur_entry, type, margs, + cur_level)) { + goto out; + } + + // If try_map is set, we will abort the mapping operation. if (margs->try_map) { margs->error = ERROR_EXISTING_MAPPING; margs->partially_mapped_size = margs->orig_size - size; vret = PGTABLE_MODIFIER_RET_ERROR; goto out; - } else { - if (type == VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE) { - margs->error = ERROR_EXISTING_MAPPING; - margs->partially_mapped_size = - margs->orig_size - size; - vret = PGTABLE_MODIFIER_RET_ERROR; - goto out; - } - - bool only_access; - only_access = pgtable_maybe_update_access( - pgt, stack, idx, type, margs, level, - virtual_address, size, next_virtual_address, - next_size, next_level); - if (only_access) { - goto out; - } - - vret = pgtable_modify_mapping( - pgt, virtual_address, size, idx, cur_level, - type, stack, margs, next_level, - next_virtual_address, next_size, next_table, - issue_dvm_cmd); - - if (vret != PGTABLE_MODIFIER_RET_STOP) { - // after modified mapping, this case indicates - // continue, the unmap modifier removes existing - // pgtable, it might free upper table level as - // well. So jump out and redo this mapping at - // the same size, same virtual address - *next_virtual_address = virtual_address; - *next_size = size; - } + } + // If this is only an access update, we can do it in place. + if (pgtable_maybe_update_access(pgt, stack, idx, type, margs, + cur_level, virtual_address, + size, next_virtual_address, + next_size, next_level)) { goto out; } + + // We need a non-trivial update using a BBM sequence and/or a + // block split. + vret = pgtable_modify_mapping(pgt, virtual_address, size, + cur_entry, idx, cur_level, type, + stack, margs, next_level, + next_virtual_address, next_size); + goto out; } assert(data != NULL); assert(pgt != NULL); // current level should be mapped + index_t level = cur_level; assert(stack[level].mapped); - cur_table = stack[level].table; + vmsa_level_table_t *cur_table = stack[level].table; - cur_level_info = &level_conf[level]; - addr_size = cur_level_info->addr_size; - allowed = cur_level_info->allowed_types; - level_size = size_on_level(virtual_address, size, cur_level_info); + const pgtable_level_info_t *cur_level_info = &level_conf[cur_level]; + size_t addr_size = cur_level_info->addr_size; + pgtable_entry_types_t allowed = cur_level_info->allowed_types; + + const bool use_block = pgtable_entry_types_get_block(&allowed); + size_t level_size = + size_on_level(virtual_address, size, cur_level_info); - // page/block type is exclusive for each other - page_or_block_type = allowed & - (VMSA_ENTRY_TYPE_BLOCK | VMSA_ENTRY_TYPE_PAGE); - if ((addr_size <= level_size) && (page_or_block_type != 0) && + if ((addr_size <= level_size) && + (use_block || pgtable_entry_types_get_page(&allowed)) && (util_is_baligned(margs->phys, addr_size))) { + index_t new_page_start_level; + bool page_block_fence; + if (margs->new_page_start_level != PGTABLE_INVALID_LEVEL) { - start_level = margs->new_page_start_level; - page_block_fence = false; + new_page_start_level = margs->new_page_start_level; + page_block_fence = false; margs->new_page_start_level = PGTABLE_INVALID_LEVEL; } else { // if current level is start level, no need to update // entry count - start_level = level > pgt->start_level ? level - 1 - : level; + new_page_start_level = + level > pgt->start_level ? level - 1U : level; page_block_fence = true; } @@ -1819,10 +2238,10 @@ map_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, index_t idx, #endif // allowed to map a block - if (page_or_block_type == VMSA_ENTRY_TYPE_BLOCK) { + if (use_block) { set_block_entry(cur_table, idx, margs->phys, margs->upper_attrs, margs->lower_attrs, - contiguous, page_block_fence); + contiguous, page_block_fence, false); } else { set_page_entry(cur_table, idx, margs->phys, margs->upper_attrs, margs->lower_attrs, @@ -1830,19 +2249,19 @@ map_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, index_t idx, } // check if need to set all page table levels - set_pgtables(virtual_address, stack, start_level, level, 1U); + set_pgtables(virtual_address, stack, new_page_start_level, + level, 1U, pgt->start_level); // update the physical address for next mapping margs->phys += addr_size; assert(!util_add_overflows(margs->phys, addr_size)); - } else if (allowed & VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE) { - ret = pgtable_add_table_entry(pgt, margs, level, stack, - virtual_address, size, next_level, - next_virtual_address, next_size, - next_table, true); + } else if (pgtable_entry_types_get_next_level_table(&allowed)) { + error_t ret = pgtable_add_table_entry( + pgt, margs, level, stack, virtual_address, size, + next_level, next_virtual_address, next_size, true); if (ret != OK) { vret = PGTABLE_MODIFIER_RET_ERROR; - goto out; + goto failed_map; } } else { LOG(ERROR, WARN, "Unexpected condition during mapping:\n"); @@ -1850,23 +2269,21 @@ map_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, index_t idx, "Mapping pa({:x}) to va({:x}), size({:d}), level({:d})", margs->phys, virtual_address, size, (register_t)level); // should not be here - vret = PGTABLE_MODIFIER_RET_ERROR; - margs->error = ERROR_ARGUMENT_INVALID; - goto out; + panic("map_modifier bad type"); } -out: +failed_map: cur_table = NULL; // free all pages if something wrong if ((vret == PGTABLE_MODIFIER_RET_ERROR) && - (margs->new_page_start_level != 0)) { + (margs->new_page_start_level != PGTABLE_INVALID_LEVEL)) { size_t pgtable_size = util_bit(pgt->granule_shift); while (margs->new_page_start_level < level) { // all new table level, no need to unmap assert(!stack[level].need_unmap); - partition_free(margs->partition, stack[level].table, - pgtable_size); + (void)partition_free(margs->partition, + stack[level].table, pgtable_size); stack[level].paddr = 0U; stack[level].table = NULL; stack[level].mapped = false; @@ -1874,6 +2291,7 @@ map_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, index_t idx, } } +out: return vret; } @@ -1883,49 +2301,33 @@ map_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, index_t idx, // entry/entries, and return to the caller. NOTE that the memory type and // memory attribute should be the same, or else, it only return the attributes // of the last entry. -pgtable_modifier_ret_t -lookup_modifier(pgtable_t *pgt, vmsa_general_entry_t cur_entry, index_t level, - vmsa_entry_type_t type, void *data) +static pgtable_modifier_ret_t +lookup_modifier(pgtable_t *pgt, vmsa_entry_t cur_entry, index_t level, + pgtable_entry_types_t type, void *data) { pgtable_lookup_modifier_args_t *margs = (pgtable_lookup_modifier_args_t *)data; const pgtable_level_info_t *cur_level_info = &level_conf[level]; - pgtable_modifier_ret_t vret = PGTABLE_MODIFIER_RET_STOP; - error_t ret = OK; - // only handle several cases, valid arguments - if ((type != VMSA_ENTRY_TYPE_PAGE) && (type != VMSA_ENTRY_TYPE_BLOCK)) { - LOG(ERROR, WARN, - "Invalid argument during lookup. Stop lookup now.\n"); - /* shouldn't be here */ - vret = PGTABLE_MODIFIER_RET_ERROR; - goto out; - } + // expected types in the walk should be page|block + assert(pgtable_entry_types_get_page(&type) || + pgtable_entry_types_get_block(&type)); assert(pgt != NULL); - ret = get_entry_paddr(cur_level_info, &cur_entry, type, &margs->phys); - if (ret != OK) { - LOG(ERROR, WARN, - "Failed to get physical address, entry type({:d}) ", type); - LOG(ERROR, WARN, "entry({:x})\n", - vmsa_general_entry_raw(cur_entry)); - vret = PGTABLE_MODIFIER_RET_ERROR; - goto out; - } + get_entry_paddr(cur_level_info, &cur_entry, type, &margs->phys); margs->entry = cur_entry; // set size & return for check margs->size = cur_level_info->addr_size; -out: - return vret; + return PGTABLE_MODIFIER_RET_STOP; } // helper to check entry count from the parent page table level, // free empty upper levels if needed -void +static void check_refcount(pgtable_t *pgt, partition_t *partition, vmaddr_t virtual_address, size_t size, index_t upper_level, stack_elem_t stack[PGTABLE_LEVEL_NUM], bool need_dec, @@ -1938,7 +2340,7 @@ check_refcount(pgtable_t *pgt, partition_t *partition, vmaddr_t virtual_address, index_t cur_idx; count_t refcount; bool is_preserved = false; - stack_elem_t *free_list[PGTABLE_LEVEL_NUM]; + stack_elem_t *free_list[PGTABLE_LEVEL_NUM]; index_t free_idx = 0; bool dec = need_dec; size_t walked_size; @@ -1948,7 +2350,8 @@ check_refcount(pgtable_t *pgt, partition_t *partition, vmaddr_t virtual_address, cur_table = stack[level].table; cur_level_info = &level_conf[level]; - cur_idx = get_index(virtual_address, cur_level_info); + cur_idx = get_index(virtual_address, cur_level_info, + (level == pgt->start_level)); refcount = get_table_refcount(cur_table, cur_idx); if (dec) { @@ -1958,7 +2361,7 @@ check_refcount(pgtable_t *pgt, partition_t *partition, vmaddr_t virtual_address, dec = false; } - if (refcount == 0) { + if (refcount == 0U) { is_preserved = is_preserved_table_entry(preserved_size, cur_level_info); @@ -2007,7 +2410,7 @@ check_refcount(pgtable_t *pgt, partition_t *partition, vmaddr_t virtual_address, walked_size = (*next_virtual_address - virtual_address); *next_size = util_max(size, walked_size) - walked_size; - free_list[free_idx] = &stack[level + 1]; + free_list[free_idx] = &stack[level + 1U]; free_idx++; // invalidate current entry set_invalid_entry(cur_table, cur_idx); @@ -2015,7 +2418,7 @@ check_refcount(pgtable_t *pgt, partition_t *partition, vmaddr_t virtual_address, dec = true; } - if ((refcount == 0) && (level > 0)) { + if ((refcount == 0U) && (level > 0U)) { // will decrease it's parent level entry count level--; } else { @@ -2027,7 +2430,7 @@ check_refcount(pgtable_t *pgt, partition_t *partition, vmaddr_t virtual_address, } // free the page table levels at one time, the free will do the fence - while (free_idx > 0) { + while (free_idx > 0U) { free_idx--; if (free_list[free_idx]->need_unmap) { @@ -2038,8 +2441,8 @@ check_refcount(pgtable_t *pgt, partition_t *partition, vmaddr_t virtual_address, free_list[free_idx]->need_unmap = false; } - partition_free_phys(partition, free_list[free_idx]->paddr, - util_bit(pgt->granule_shift)); + (void)partition_free_phys(partition, free_list[free_idx]->paddr, + util_bit(pgt->granule_shift)); free_list[free_idx]->table = NULL; free_list[free_idx]->paddr = 0U; free_list[free_idx]->mapped = false; @@ -2091,24 +2494,25 @@ static void unmap_clear_cont_bit(vmsa_level_table_t *table, vmaddr_t virtual_address, index_t level, vmsa_page_and_block_attrs_entry_t attr_entry, - pgtable_unmap_modifier_args_t *margs, - count_t granule_shift, bool issue_dvm_cmd) + pgtable_unmap_modifier_args_t *margs, + count_t granule_shift, index_t start_level) { const pgtable_level_info_t *info = &level_conf[level]; assert(info->contiguous_entry_cnt != 0U); // get index range in current table to clear cont bit - index_t cur_idx = get_index(virtual_address, info); + index_t cur_idx = + get_index(virtual_address, info, (level == start_level)); index_t idx_start = util_balign_down(cur_idx, info->contiguous_entry_cnt); - index_t idx_end = (index_t)(idx_start + info->contiguous_entry_cnt - 1); + index_t idx_end = + (index_t)(idx_start + info->contiguous_entry_cnt - 1U); // start break-before-make sequence: clear all contiguous entries for (index_t idx = idx_start; idx <= idx_end; idx++) { set_invalid_entry(table, idx); } - dsb(); // flush all contiguous entries from TLB (note that the CPU may not // implement the contiguous bit at this level, so we are required to @@ -2116,24 +2520,25 @@ unmap_clear_cont_bit(vmsa_level_table_t *table, vmaddr_t virtual_address, vmaddr_t vaddr = virtual_address & ~((util_bit(info->lsb) * info->contiguous_entry_cnt) - 1U); -#ifdef ARCH_ARM_8_4_TLBI +#ifdef ARCH_ARM_FEAT_TLBIRANGE if (margs->stage == PGTABLE_HYP_STAGE_1) { - hyp_tlbi_va_range( - vaddr, - vaddr + (info->contiguous_entry_cnt * info->addr_size), - granule_shift); + dsb(false); + hyp_tlbi_va_range(vaddr, + info->contiguous_entry_cnt * info->addr_size, + granule_shift); } else { - hyp_tlbi_ipa_range( - vaddr, - vaddr + (info->contiguous_entry_cnt * info->addr_size), - granule_shift, issue_dvm_cmd); + dsb(margs->outer_shareable); + hyp_tlbi_ipa_range(vaddr, + info->contiguous_entry_cnt * info->addr_size, + granule_shift, margs->outer_shareable); } #else + dsb(margs->outer_shareable); for (index_t i = 0; i < info->contiguous_entry_cnt; i++) { if (margs->stage == PGTABLE_HYP_STAGE_1) { hyp_tlbi_va(vaddr); } else { - vm_tlbi_ipa(vaddr, issue_dvm_cmd); + vm_tlbi_ipa(vaddr, margs->outer_shareable); } vaddr += info->addr_size; } @@ -2153,24 +2558,32 @@ unmap_clear_cont_bit(vmsa_level_table_t *table, vmaddr_t virtual_address, vmsa_page_and_block_attrs_entry_set_upper_attrs(&attr_entry, upper_attrs); - vmsa_general_entry_t entry = vmsa_general_entry_cast( - vmsa_page_and_block_attrs_entry_raw(attr_entry)); - - const vmsa_entry_type_t page_or_block_type = - info->allowed_types & - (VMSA_ENTRY_TYPE_BLOCK | VMSA_ENTRY_TYPE_PAGE); - paddr_t entry_phys; - (void)get_entry_paddr(info, &entry, page_or_block_type, &entry_phys); - entry_phys &= - ~((util_bit(info->lsb) * info->contiguous_entry_cnt) - 1U); + vmsa_entry_t entry = { + .attrs = attr_entry, + }; + const bool use_block = + pgtable_entry_types_get_block(&info->allowed_types); + paddr_t entry_phys = 0U; + pgtable_entry_types_t type = pgtable_entry_types_default(); for (index_t idx = idx_start; idx <= idx_end; idx++) { if (idx == cur_idx) { // This should be left invalid - } else if (page_or_block_type == VMSA_ENTRY_TYPE_BLOCK) { + } else if (use_block) { + pgtable_entry_types_set_block(&type, true); + get_entry_paddr(info, &entry, type, &entry_phys); + entry_phys &= ~((util_bit(info->lsb) * + info->contiguous_entry_cnt) - + 1U); + set_block_entry(table, idx, entry_phys, upper_attrs, - lower_attrs, false, false); + lower_attrs, false, false, false); } else { + pgtable_entry_types_set_page(&type, true); + get_entry_paddr(info, &entry, type, &entry_phys); + entry_phys &= ~((util_bit(info->lsb) * + info->contiguous_entry_cnt) - + 1U); set_page_entry(table, idx, entry_phys, upper_attrs, lower_attrs, false, false); } @@ -2187,20 +2600,19 @@ unmap_clear_cont_bit(vmsa_level_table_t *table, vmaddr_t virtual_address, // translation_table_walk to step onto the next entry at the same level, and // update the size as well. // * Invalidate current entry. -pgtable_modifier_ret_t +static pgtable_modifier_ret_t unmap_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, - index_t idx, index_t level, vmsa_entry_type_t type, + index_t idx, index_t level, pgtable_entry_types_t type, stack_elem_t stack[PGTABLE_LEVEL_NUM], void *data, index_t *next_level, vmaddr_t *next_virtual_address, - size_t *next_size, paddr_t *next_table, bool only_matching, - bool issue_dvm_cmd) + size_t *next_size, bool only_matching) { - const pgtable_level_info_t *cur_level_info = NULL; + const pgtable_level_info_t *cur_level_info = NULL; pgtable_unmap_modifier_args_t *margs = (pgtable_unmap_modifier_args_t *)data; pgtable_modifier_ret_t vret = PGTABLE_MODIFIER_RET_CONTINUE; - vmsa_level_table_t *cur_table = NULL; - vmsa_general_entry_t cur_entry; + vmsa_level_table_t *cur_table = NULL; + vmsa_entry_t cur_entry; bool need_dec = false; assert(pgt != NULL); @@ -2222,14 +2634,13 @@ unmap_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, // for INVALID entry. need_dec = false; - if (only_matching && ((type == VMSA_ENTRY_TYPE_BLOCK) || - (type == VMSA_ENTRY_TYPE_PAGE))) { + if (only_matching && (pgtable_entry_types_get_block(&type) || + pgtable_entry_types_get_page(&type))) { // Check if it is mapped to different phys_addr than expected // If so, do not unmap this address paddr_t phys_addr; - (void)get_entry_paddr(cur_level_info, &cur_entry, type, - &phys_addr); + get_entry_paddr(cur_level_info, &cur_entry, type, &phys_addr); if ((phys_addr < margs->phys) || (phys_addr > (margs->phys + margs->size - 1U))) { goto out; @@ -2237,7 +2648,7 @@ unmap_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, } // Split the block if necessary - if (type == VMSA_ENTRY_TYPE_BLOCK) { + if (pgtable_entry_types_get_block(&type)) { size_t addr_size = cur_level_info->addr_size; vmaddr_t entry_virtual_address = entry_start_address(virtual_address, cur_level_info); @@ -2246,7 +2657,7 @@ unmap_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, // Partial unmap; split the block into 4K pages. vmsa_page_and_block_attrs_entry_t attr_entry = vmsa_page_and_block_attrs_entry_cast( - vmsa_general_entry_raw(cur_entry)); + vmsa_general_entry_raw(cur_entry.base)); vmsa_lower_attrs_t lower_attrs; lower_attrs = vmsa_page_and_block_attrs_entry_get_lower_attrs( @@ -2255,12 +2666,11 @@ unmap_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, vmsa_page_and_block_attrs_entry_get_upper_attrs( &attr_entry); paddr_t entry_phys; - (void)get_entry_paddr(cur_level_info, &cur_entry, type, - &entry_phys); + get_entry_paddr(cur_level_info, &cur_entry, type, + &entry_phys); - pgtable_map_modifier_args_t mremap_args; - memset(&mremap_args, 0, sizeof(mremap_args)); - mremap_args.phys = entry_phys; + pgtable_map_modifier_args_t mremap_args = { 0 }; + mremap_args.phys = entry_phys; mremap_args.partition = margs->partition; mremap_args.lower_attrs = lower_attrs; mremap_args.upper_attrs = upper_attrs; @@ -2270,20 +2680,16 @@ unmap_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, mremap_args.stage = margs->stage; vret = pgtable_split_block( - pgt, entry_virtual_address, addr_size, idx, + pgt, virtual_address, size, cur_entry, idx, level, type, stack, &mremap_args, next_level, - next_virtual_address, next_size, next_table, - issue_dvm_cmd); + next_virtual_address, next_size); - // After splitting the block, restart the walk at the - // same address to descend into the new table. - *next_virtual_address = virtual_address; - *next_size = size; goto out; } } - if ((type == VMSA_ENTRY_TYPE_BLOCK) || (type == VMSA_ENTRY_TYPE_PAGE)) { + if (pgtable_entry_types_get_block(&type) || + pgtable_entry_types_get_page(&type)) { vmsa_upper_attrs_t upper_attrs = get_upper_attr(cur_entry); vmsa_common_upper_attrs_t upper_attrs_bitfield = vmsa_common_upper_attrs_cast(upper_attrs); @@ -2293,30 +2699,33 @@ unmap_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, unmap_should_clear_cont(virtual_address, size, level)) { vmsa_page_and_block_attrs_entry_t attr_entry = vmsa_page_and_block_attrs_entry_cast( - vmsa_general_entry_raw(cur_entry)); + vmsa_general_entry_raw(cur_entry.base)); unmap_clear_cont_bit(cur_table, virtual_address, level, attr_entry, margs, - pgt->granule_shift, issue_dvm_cmd); + pgt->granule_shift, + pgt->start_level); } else { set_invalid_entry(cur_table, idx); // need to decrease entry count for this table level need_dec = true; - dsb(); if (margs->stage == PGTABLE_HYP_STAGE_1) { + dsb(false); hyp_tlbi_va(virtual_address); } else { - vm_tlbi_ipa(virtual_address, issue_dvm_cmd); + dsb(margs->outer_shareable); + vm_tlbi_ipa(virtual_address, + margs->outer_shareable); } } } else { - assert(type == VMSA_ENTRY_TYPE_INVALID); + assert(pgtable_entry_types_get_invalid(&type)); } if (level != pgt->start_level) { check_refcount(pgt, margs->partition, virtual_address, size, - level - 1, stack, need_dec, + level - 1U, stack, need_dec, margs->preserved_size, next_level, next_virtual_address, next_size); } @@ -2333,12 +2742,12 @@ unmap_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, // This modifier just allocate/adds some page table level using the specified // partition. The usage of this call is to guarantee the currently used page // table level is still valid after release certain partition. -pgtable_modifier_ret_t +static pgtable_modifier_ret_t prealloc_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, - index_t level, vmsa_entry_type_t type, + index_t level, pgtable_entry_types_t type, stack_elem_t stack[PGTABLE_LEVEL_NUM], void *data, index_t *next_level, vmaddr_t *next_virtual_address, - size_t *next_size, paddr_t *next_table) + size_t *next_size) { pgtable_prealloc_modifier_args_t *margs = (pgtable_prealloc_modifier_args_t *)data; @@ -2349,7 +2758,7 @@ prealloc_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, size_t addr_size = 0U, level_size = 0U; vmsa_level_table_t *new_pgt = NULL; - assert(type == VMSA_ENTRY_TYPE_INVALID); + assert(pgtable_entry_types_get_invalid(&type)); assert(data != NULL); assert(pgt != NULL); @@ -2369,7 +2778,8 @@ prealloc_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, // set if (margs->new_page_start_level != PGTABLE_INVALID_LEVEL) { set_pgtables(virtual_address, stack, - margs->new_page_start_level, level, 0U); + margs->new_page_start_level, level, 0U, + pgt->start_level); margs->new_page_start_level = PGTABLE_INVALID_LEVEL; } @@ -2391,19 +2801,21 @@ prealloc_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, if (margs->new_page_start_level == PGTABLE_INVALID_LEVEL) { margs->new_page_start_level = - level > pgt->start_level ? level - 1 : level; + level > pgt->start_level ? level - 1U : level; } - stack[level + 1].paddr = new_pgt_paddr; - stack[level + 1].table = new_pgt; - stack[level + 1].mapped = true; - stack[level + 1].entry_cnt = level_conf[level + 1].entry_cnt; + stack[level + 1U] = (stack_elem_t){ + .paddr = new_pgt_paddr, + .table = new_pgt, + .mapped = true, + .need_unmap = false, + .entry_cnt = level_conf[level + 1U].entry_cnt, + }; // step into the next sub level, with nothing stepped *next_virtual_address = virtual_address; *next_size = size; - *next_level = level + 1; - *next_table = new_pgt_paddr; + *next_level = level + 1U; } out: @@ -2414,12 +2826,12 @@ prealloc_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, static pgtable_modifier_ret_t dump_modifier(vmaddr_t virtual_address, size_t size, stack_elem_t stack[PGTABLE_LEVEL_NUM], index_t idx, index_t level, - vmsa_entry_type_t type) + pgtable_entry_types_t type) { const pgtable_level_info_t *cur_level_info = NULL; vmsa_level_table_t *cur_table = NULL; - vmsa_general_entry_t cur_entry; - uint64_t *entry_val = (uint64_t *)&cur_entry; + vmsa_entry_t cur_entry; + uint64_t *entry_val = &cur_entry.base.bf[0]; paddr_t p; count_t refcount; vmaddr_t cur_virtual_address; @@ -2429,7 +2841,7 @@ dump_modifier(vmaddr_t virtual_address, size_t size, pgtable_modifier_ret_t vret = PGTABLE_MODIFIER_RET_CONTINUE; size_t addr_size = 0U; - if (size == 0L) { + if (size == 0U) { vret = PGTABLE_MODIFIER_RET_STOP; goto out; } @@ -2442,44 +2854,25 @@ dump_modifier(vmaddr_t virtual_address, size_t size, cur_entry = get_entry(cur_table, idx); refcount = get_table_refcount(cur_table, idx); - // No need to care if paddr is wrong - (void)get_entry_paddr(cur_level_info, &cur_entry, type, &p); + if (!pgtable_entry_types_get_invalid(&type)) { + get_entry_paddr(cur_level_info, &cur_entry, type, &p); + } else { + p = 0U; + } // FIXME: check if cur_virtual_address is right cur_virtual_address = set_index(virtual_address, cur_level_info, idx) & (~util_mask(cur_level_info->lsb)); - // FIXME: add assert for sizeof(indent) and level + assert((size_t)level < (sizeof(indent) - 1U)); indent[0] = '|'; - for (i = 0; i < level; i++) { - indent[i + 1] = '\t'; + for (i = 1; i <= level; i++) { + indent[i] = '\t'; } - indent[i + 1] = 0; + indent[i] = '\0'; - switch (type) { - case VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE: + if (pgtable_entry_types_get_next_level_table(&type)) { msg_type = "[Table]"; - break; - case VMSA_ENTRY_TYPE_BLOCK: - msg_type = "[Block]"; - break; - case VMSA_ENTRY_TYPE_PAGE: - msg_type = "[Page]"; - break; - case VMSA_ENTRY_TYPE_RESERVED: - msg_type = "[Reserved]"; - break; - case VMSA_ENTRY_TYPE_ERROR: - msg_type = "[Error]"; - break; - case VMSA_ENTRY_TYPE_INVALID: - default: - msg_type = "[Invalid]"; - break; - } - - switch (type) { - case VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE: LOG(DEBUG, INFO, "{:s}->{:s} entry[{:#x}] virtual_address({:#x})", (register_t)indent, (register_t)msg_type, *entry_val, @@ -2490,9 +2883,13 @@ dump_modifier(vmaddr_t virtual_address, size_t size, (register_t)refcount, (register_t)cur_level_info->level); LOG(DEBUG, INFO, "{:s}addr_size({:#x})", (register_t)indent, (register_t)addr_size); - break; - case VMSA_ENTRY_TYPE_BLOCK: - case VMSA_ENTRY_TYPE_PAGE: + } else if (pgtable_entry_types_get_block(&type) || + pgtable_entry_types_get_page(&type)) { + if (pgtable_entry_types_get_block(&type)) { + msg_type = "[Block]"; + } else { + msg_type = "[Page]"; + } LOG(DEBUG, INFO, "{:s}->{:s} entry[{:#x}] virtual_address({:#x})", (register_t)indent, (register_t)msg_type, @@ -2502,16 +2899,20 @@ dump_modifier(vmaddr_t virtual_address, size_t size, (register_t)cur_level_info->level); LOG(DEBUG, INFO, "{:s}addr_size({:#x})", (register_t)indent, (register_t)addr_size); - break; - case VMSA_ENTRY_TYPE_INVALID: - break; - case VMSA_ENTRY_TYPE_RESERVED: - case VMSA_ENTRY_TYPE_ERROR: - default: - LOG(DEBUG, INFO, "{:s}->{:s} virtual_address({:#x}) idx({:d})", - (register_t)indent, (register_t)msg_type, - (register_t)cur_virtual_address, (register_t)idx); - break; + } else { + if (!pgtable_entry_types_get_invalid(&type)) { + if (pgtable_entry_types_get_reserved(&type)) { + msg_type = "[Reserved]"; + } else if (pgtable_entry_types_get_error(&type)) { + msg_type = "[Error]"; + } else { + // Nothing to do + } + LOG(DEBUG, INFO, + "{:s}->{:s} virtual_address({:#x}) idx({:d})", + (register_t)indent, (register_t)msg_type, + (register_t)cur_virtual_address, (register_t)idx); + } } out: @@ -2524,13 +2925,13 @@ dump_modifier(vmaddr_t virtual_address, size_t size, #if defined(HOST_TEST) static pgtable_modifier_ret_t external_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, - index_t idx, index_t level, vmsa_entry_type_t type, + index_t idx, index_t level, pgtable_entry_types_t type, stack_elem_t stack[PGTABLE_LEVEL_NUM], void *data, index_t *next_level, vmaddr_t *next_virtual_address, - size_t *next_size, paddr_t *next_table) + size_t *next_size, paddr_t next_table) { ext_modifier_args_t *margs = (ext_modifier_args_t *)data; - void *func_data = margs->data; + void *func_data = margs->data; pgtable_modifier_ret_t ret = PGTABLE_MODIFIER_RET_STOP; if (margs->func != NULL) { @@ -2594,25 +2995,24 @@ external_modifier(pgtable_t *pgt, vmaddr_t virtual_address, size_t size, // @param data specifier an opaque data structure specific for modifier. // @return true if the finished the walking without error. Or false indicates // the failure. -bool +static bool translation_table_walk(pgtable_t *pgt, vmaddr_t virtual_address, size_t virtual_address_size, pgtable_translation_table_walk_event_t event, - pgtable_entry_types_t expected, void *data, - bool issue_dvm_cmd) -{ - paddr_t root_pa = pgt->root_pgtable; - vmsa_level_table_t *root = pgt->root; - index_t level = pgt->start_level; - index_t prev_level; - index_t prev_idx; - vmaddr_t prev_virtual_address; - size_t prev_size; - vmsa_general_entry_t prev_entry; - vmsa_entry_type_t prev_type; + pgtable_entry_types_t expected, void *data) +{ + paddr_t root_pa = pgt->root_pgtable; + vmsa_level_table_t *root = pgt->root; + index_t start_level = pgt->start_level; + index_t prev_level; + index_t prev_idx; + vmaddr_t prev_virtual_address; + size_t prev_size; + vmsa_entry_t prev_entry; + pgtable_entry_types_t prev_type; // loop control variable - index_t cur_level = level; + index_t cur_level = start_level; paddr_t cur_table_paddr = 0U; vmaddr_t cur_virtual_address = virtual_address; @@ -2620,8 +3020,8 @@ translation_table_walk(pgtable_t *pgt, vmaddr_t virtual_address, index_t cur_idx; stack_elem_t stack[PGTABLE_LEVEL_NUM]; vmsa_level_table_t *cur_table = NULL; - vmsa_general_entry_t cur_entry; - vmsa_entry_type_t cur_type; + vmsa_entry_t cur_entry; + pgtable_entry_types_t cur_type; size_t cur_size = virtual_address_size; // ret: indicates whether walking is successful or not. // done: indicates the walking got a stop sign and need to return. It @@ -2629,35 +3029,32 @@ translation_table_walk(pgtable_t *pgt, vmaddr_t virtual_address, // ignores the modifier. bool ret = false, done = false; - memset(stack, 0, sizeof(stack)); - stack[level].paddr = root_pa; - stack[level].table = root; - stack[level].mapped = true; - stack[level].entry_cnt = + stack[start_level] = (stack_elem_t){ 0U }; + stack[start_level].paddr = root_pa; + stack[start_level].table = root; + stack[start_level].mapped = true; + stack[start_level].entry_cnt = (count_t)(pgt->start_level_size / sizeof(cur_entry)); - for (cur_level = level; - cur_level < (index_t)util_array_size(level_conf);) { + while (cur_level < (index_t)util_array_size(level_conf)) { cur_level_info = &level_conf[cur_level]; - cur_idx = get_index(cur_virtual_address, cur_level_info); + cur_idx = get_index(cur_virtual_address, cur_level_info, + (cur_level == start_level)); - if (cur_level_info->is_offset) { + if (compiler_unexpected(cur_level_info->is_offset)) { // Arrived offset segment, mapping is supposed to // be finished - LOG(ERROR, WARN, - "Stepped into the leaf, shouldn't be here.\n"); - ret = true; - goto out; + panic("pgtable walk depth error"); } - if (cur_idx >= stack[cur_level].entry_cnt) { + if (compiler_unexpected(cur_idx >= + stack[cur_level].entry_cnt)) { // Index is outside the bounds of the table; address // range was not properly range-checked LOG(ERROR, WARN, "Stepped out of the table (va {:#x}, level {:d}, idx {:d})", cur_virtual_address, cur_level, cur_idx); - ret = false; - goto out; + panic("pgtable walk"); } cur_table_paddr = stack[cur_level].paddr; @@ -2667,12 +3064,11 @@ translation_table_walk(pgtable_t *pgt, vmaddr_t virtual_address, cur_table = (vmsa_level_table_t *)partition_phys_map( cur_table_paddr, stack[cur_level].entry_cnt * sizeof(cur_entry)); - if (cur_table == NULL) { + if (compiler_unexpected(cur_table == NULL)) { LOG(ERROR, WARN, "Failed to map table (pa {:#x}, level {:d}, idx {:d})\n", cur_table_paddr, cur_level, cur_idx); - ret = false; - goto out; + panic("pgtable fault"); } stack[cur_level].table = cur_table; @@ -2691,37 +3087,32 @@ translation_table_walk(pgtable_t *pgt, vmaddr_t virtual_address, prev_type = cur_type; prev_size = cur_size; - switch (cur_type) { - case VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE: + if (pgtable_entry_types_get_next_level_table(&cur_type)) { cur_level++; assert(cur_level < PGTABLE_LEVEL_NUM); - // FIXME: should we simplify the signature of this call? - if (OK != get_entry_paddr(cur_level_info, &cur_entry, - cur_type, &cur_table_paddr)) { - // something wrong - LOG(ERROR, WARN, - "Failed to get physical address: "); - LOG(ERROR, WARN, "entry({:#x})\n", - vmsa_general_entry_raw(cur_entry)); - ret = false; - goto out; - } + + get_entry_paddr( + cur_level_info, &cur_entry, + pgtable_entry_types_cast( + PGTABLE_ENTRY_TYPES_NEXT_LEVEL_TABLE_MASK), + &cur_table_paddr); + cur_level_info = &level_conf[cur_level]; + stack[cur_level] = (stack_elem_t){ 0U }; stack[cur_level].paddr = cur_table_paddr; stack[cur_level].mapped = false; stack[cur_level].table = NULL; stack[cur_level].entry_cnt = cur_level_info->entry_cnt; - break; - - case VMSA_ENTRY_TYPE_INVALID: + } else if (pgtable_entry_types_get_invalid(&cur_type) || + pgtable_entry_types_get_page(&cur_type) || + pgtable_entry_types_get_block(&cur_type)) { // for invalid entry, it must be handled by modifier. // Also by default, the next entry for the walking // is simply set to the next entry. The modifier should // guide the walking to go to the proper next entry. // Unless the modifier asks for continue the walking, // the walking process must stopped by default. - case VMSA_ENTRY_TYPE_PAGE: - case VMSA_ENTRY_TYPE_BLOCK: + // update virt address to visit next entry cur_virtual_address = step_virtual_address( cur_virtual_address, cur_level_info); @@ -2735,17 +3126,18 @@ translation_table_walk(pgtable_t *pgt, vmaddr_t virtual_address, } // If we're at the lowest level - if (cur_level_info->allowed_types == - VMSA_ENTRY_TYPE_PAGE) { - if (prev_size < cur_level_info->addr_size) { + if (pgtable_entry_types_get_page( + &cur_level_info->allowed_types)) { + if (compiler_unexpected( + prev_size < + cur_level_info->addr_size)) { // wrong, size must be at least multiple // of page - ret = false; - break; + panic("pgtable bad size"); } } - if (cur_size == 0) { + if (cur_size == 0U) { // the whole walk is done, but modifier can // still ask for loop by changing the size done = true; @@ -2755,8 +3147,9 @@ translation_table_walk(pgtable_t *pgt, vmaddr_t virtual_address, ret = true; } - while (cur_idx == stack[cur_level].entry_cnt - 1) { - if (cur_level == pgt->start_level) { + // Iterate up on the last entry in the level(s) + while (cur_idx == (stack[cur_level].entry_cnt - 1U)) { + if (cur_level == start_level) { done = true; break; } else { @@ -2767,48 +3160,46 @@ translation_table_walk(pgtable_t *pgt, vmaddr_t virtual_address, // cur_virtual_address is already stepped, use // previous one to check cur_idx = get_index(prev_virtual_address, - cur_level_info); + cur_level_info, + (cur_level == start_level)); } - break; - - case VMSA_ENTRY_TYPE_ERROR: - case VMSA_ENTRY_TYPE_RESERVED: - default: + } else { // shouldn't be here - ret = false; - goto out; + panic("pgtable corrupt entry"); } cur_table = NULL; - if ((prev_type & expected) != 0U) { + if (!pgtable_entry_types_is_empty( + pgtable_entry_types_intersection(prev_type, + expected))) { pgtable_modifier_ret_t vret; switch (event) { case PGTABLE_TRANSLATION_TABLE_WALK_EVENT_MMAP: vret = map_modifier(pgt, prev_virtual_address, - prev_size, prev_idx, - prev_level, prev_type, - stack, data, &cur_level, + prev_size, prev_entry, + prev_idx, prev_level, + prev_type, stack, data, + &cur_level, &cur_virtual_address, - &cur_size, &cur_table_paddr, - issue_dvm_cmd); + &cur_size, cur_table_paddr); break; case PGTABLE_TRANSLATION_TABLE_WALK_EVENT_UNMAP: - vret = unmap_modifier( - pgt, prev_virtual_address, prev_size, - prev_idx, prev_level, prev_type, stack, - data, &cur_level, &cur_virtual_address, - &cur_size, &cur_table_paddr, false, - issue_dvm_cmd); + vret = unmap_modifier(pgt, prev_virtual_address, + prev_size, prev_idx, + prev_level, prev_type, + stack, data, &cur_level, + &cur_virtual_address, + &cur_size, false); break; case PGTABLE_TRANSLATION_TABLE_WALK_EVENT_UNMAP_MATCH: - vret = unmap_modifier( - pgt, prev_virtual_address, prev_size, - prev_idx, prev_level, prev_type, stack, - data, &cur_level, &cur_virtual_address, - &cur_size, &cur_table_paddr, true, - issue_dvm_cmd); + vret = unmap_modifier(pgt, prev_virtual_address, + prev_size, prev_idx, + prev_level, prev_type, + stack, data, &cur_level, + &cur_virtual_address, + &cur_size, true); break; case PGTABLE_TRANSLATION_TABLE_WALK_EVENT_LOOKUP: vret = lookup_modifier(pgt, prev_entry, @@ -2820,7 +3211,7 @@ translation_table_walk(pgtable_t *pgt, vmaddr_t virtual_address, pgt, prev_virtual_address, prev_size, prev_level, prev_type, stack, data, &cur_level, &cur_virtual_address, - &cur_size, &cur_table_paddr); + &cur_size); break; #ifndef NDEBUG case PGTABLE_TRANSLATION_TABLE_WALK_EVENT_DUMP: @@ -2835,12 +3226,11 @@ translation_table_walk(pgtable_t *pgt, vmaddr_t virtual_address, pgt, prev_virtual_address, prev_size, prev_idx, prev_level, prev_type, stack, data, &cur_level, &cur_virtual_address, - &cur_size, &cur_table_paddr); + &cur_size, cur_table_paddr); break; #endif default: - vret = PGTABLE_MODIFIER_RET_ERROR; - break; + panic("pgtable bad event"); } if (vret == PGTABLE_MODIFIER_RET_STOP) { @@ -2856,8 +3246,7 @@ translation_table_walk(pgtable_t *pgt, vmaddr_t virtual_address, done = false; } else { // unknown return, just stop - done = true; - ret = false; + panic("pgtable bad vret"); } } @@ -2888,12 +3277,11 @@ translation_table_walk(pgtable_t *pgt, vmaddr_t virtual_address, } // only next table should continue the loop - if (done || (cur_size == 0L)) { + if (done || (cur_size == 0U)) { break; } } -out: - while (cur_level > pgt->start_level) { + while (cur_level > start_level) { if (stack[cur_level].mapped && stack[cur_level].need_unmap) { partition_phys_unmap(stack[cur_level].table, stack[cur_level].paddr, @@ -2909,25 +3297,25 @@ translation_table_walk(pgtable_t *pgt, vmaddr_t virtual_address, } static get_start_level_info_ret_t -get_start_level_info(const pgtable_level_info_t *infos, index_t msb) +get_start_level_info(const pgtable_level_info_t *infos, index_t msb, + bool is_stage2) { get_start_level_info_ret_t ret = { .level = 0U, .size = 0UL }; - uint8_t level = 0; - const pgtable_level_info_t *level_info = infos; - - for (level = 0; level < PGTABLE_LEVEL_NUM; level++) { - if ((msb <= level_info->msb) && (msb >= level_info->lsb)) { + uint8_t level; + count_t msb_offset = is_stage2 ? 4U : 0U; + + for (level = PGTABLE_LEVEL_NUM - 1U; level < PGTABLE_LEVEL_NUM; + level--) { + const pgtable_level_info_t *level_info = &infos[level]; + if ((msb <= (level_info->msb + msb_offset)) && + (msb >= level_info->lsb)) { + size_t entry_cnt = util_bit(msb - level_info->lsb + 1U); + ret.level = level; + ret.size = sizeof(vmsa_general_entry_t) * entry_cnt; break; } - level_info++; } - assert(level < PGTABLE_LEVEL_NUM); - - size_t entry_cnt = 1UL << (msb - level_info->lsb + 1); - - ret.level = level; - ret.size = sizeof(vmsa_general_entry_t) * entry_cnt; return ret; } @@ -2935,27 +3323,38 @@ get_start_level_info(const pgtable_level_info_t *infos, index_t msb) void pgtable_handle_boot_cold_init(void) { - error_t ret = OK; - index_t top_msb; index_t bottom_msb; const count_t page_shift = SHIFT_4K; partition_t *partition = partition_get_private(); -#if !defined(CPU_PGTABLE_BLOCK_SPLIT_LEVEL_FORCE) +#if !defined(HOST_TEST) ID_AA64MMFR2_EL1_t mmfr2 = register_ID_AA64MMFR2_EL1_read(); - assert(ID_AA64MMFR2_EL1_get_BBM(&mmfr2) == - CPU_PGTABLE_BLOCK_SPLIT_LEVEL); -#endif - -#if !defined(ARCH_ARM_8_1_VHE) -#error VHE is currently assumed + assert(ID_AA64MMFR2_EL1_get_BBM(&mmfr2) >= CPU_PGTABLE_BBM_LEVEL); #endif spinlock_init(&hyp_pgtable.lock); + hyp_pgtable.bottom_control.granule_shift = page_shift; + hyp_pgtable.bottom_control.address_bits = HYP_ASPACE_LOW_BITS; + bottom_msb = HYP_ASPACE_LOW_BITS - 1U; + + assert((HYP_ASPACE_LOW_BITS != level_conf[0].msb + 1) || + (HYP_ASPACE_LOW_BITS != level_conf[1].msb + 1) || + (HYP_ASPACE_LOW_BITS != level_conf[2].msb + 1) || + (HYP_ASPACE_LOW_BITS != level_conf[3].msb + 1)); + + get_start_level_info_ret_t bottom_info = + get_start_level_info(level_conf, bottom_msb, false); + hyp_pgtable.bottom_control.start_level = bottom_info.level; + hyp_pgtable.bottom_control.start_level_size = bottom_info.size; + +#if defined(ARCH_ARM_FEAT_VHE) + index_t top_msb; + error_t ret = OK; + // FIXME: refine with more configurable code hyp_pgtable.top_control.granule_shift = page_shift; hyp_pgtable.top_control.address_bits = HYP_ASPACE_HIGH_BITS; - top_msb = HYP_ASPACE_HIGH_BITS - 1; + top_msb = HYP_ASPACE_HIGH_BITS - 1U; // FIXME: change to static check (with constant?)?? // Might be better to use hyp_pgtable.top_control.address_bits assert((HYP_ASPACE_HIGH_BITS != level_conf[0].msb + 1) || @@ -2963,26 +3362,12 @@ pgtable_handle_boot_cold_init(void) (HYP_ASPACE_HIGH_BITS != level_conf[2].msb + 1) || (HYP_ASPACE_HIGH_BITS != level_conf[3].msb + 1)); - hyp_pgtable.bottom_control.granule_shift = page_shift; - hyp_pgtable.bottom_control.address_bits = HYP_ASPACE_LOW_BITS; - bottom_msb = HYP_ASPACE_LOW_BITS - 1; - - assert((HYP_ASPACE_LOW_BITS != level_conf[0].msb + 1) || - (HYP_ASPACE_LOW_BITS != level_conf[1].msb + 1) || - (HYP_ASPACE_LOW_BITS != level_conf[2].msb + 1) || - (HYP_ASPACE_LOW_BITS != level_conf[3].msb + 1)); - // update level info based on virtual_address bits get_start_level_info_ret_t top_info = - get_start_level_info(level_conf, top_msb); + get_start_level_info(level_conf, top_msb, false); hyp_pgtable.top_control.start_level = top_info.level; hyp_pgtable.top_control.start_level_size = top_info.size; - get_start_level_info_ret_t bottom_info = - get_start_level_info(level_conf, bottom_msb); - hyp_pgtable.bottom_control.start_level = bottom_info.level; - hyp_pgtable.bottom_control.start_level_size = bottom_info.size; - #if defined(HOST_TEST) // allocate the top page table ret = alloc_level_table(partition, top_info.size, @@ -2995,7 +3380,7 @@ pgtable_handle_boot_cold_init(void) } #else hyp_pgtable.top_control.root = - (vmsa_level_table_t *)&aarch64_pt_ttbr1_level1; + (vmsa_level_table_t *)&aarch64_pt_ttbr_level1; hyp_pgtable.top_control.root_pgtable = partition_virt_to_phys( partition, (uintptr_t)hyp_pgtable.top_control.root); #endif @@ -3011,29 +3396,37 @@ pgtable_handle_boot_cold_init(void) "Failed to allocate bottom page table level.\n"); goto out; } +#else + hyp_pgtable.bottom_control.root = + (vmsa_level_table_t *)&aarch64_pt_ttbr_level1; + hyp_pgtable.bottom_control.root_pgtable = partition_virt_to_phys( + partition, (uintptr_t)hyp_pgtable.bottom_control.root); +#endif ttbr0_phys = hyp_pgtable.bottom_control.root_pgtable; // activate the lower address space now for cold-boot pgtable_handle_boot_runtime_warm_init(); +#if defined(ARCH_ARM_FEAT_VHE) out: if (ret != OK) { panic("Failed to initialize hypervisor root page-table"); } +#endif } #if !defined(HOST_TEST) void -pgtable_handle_boot_runtime_warm_init() +pgtable_handle_boot_runtime_warm_init(void) { -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) TTBR0_EL2_t ttbr0_val = TTBR0_EL2_default(); TTBR0_EL2_set_BADDR(&ttbr0_val, ttbr0_phys); TTBR0_EL2_set_CnP(&ttbr0_val, true); TCR_EL2_E2H1_t tcr_val = register_TCR_EL2_E2H1_read(); - TCR_EL2_E2H1_set_T0SZ(&tcr_val, 64U - HYP_ASPACE_LOW_BITS); + TCR_EL2_E2H1_set_T0SZ(&tcr_val, (uint8_t)(64U - HYP_ASPACE_LOW_BITS)); TCR_EL2_E2H1_set_EPD0(&tcr_val, false); TCR_EL2_E2H1_set_ORGN0(&tcr_val, TCR_RGN_NORMAL_WRITEBACK_RA_WA); TCR_EL2_E2H1_set_IRGN0(&tcr_val, TCR_RGN_NORMAL_WRITEBACK_RA_WA); @@ -3059,12 +3452,12 @@ pgtable_hyp_destroy(partition_t *partition) // we should unmap everything virtual_address = 0x0U; - size = 1UL << hyp_pgtable.bottom_control.address_bits; + size = util_bit(hyp_pgtable.bottom_control.address_bits); pgtable_hyp_unmap(partition, virtual_address, size, PGTABLE_HYP_UNMAP_PRESERVE_NONE); virtual_address = ~util_mask(hyp_pgtable.top_control.address_bits); - size = 1UL << hyp_pgtable.top_control.address_bits; + size = util_bit(hyp_pgtable.top_control.address_bits); pgtable_hyp_unmap(partition, virtual_address, size, PGTABLE_HYP_UNMAP_PRESERVE_NONE); @@ -3086,11 +3479,11 @@ pgtable_hyp_lookup(uintptr_t virtual_address, paddr_t *mapped_base, pgtable_access_t *mapped_access) { bool walk_ret = false; - pgtable_lookup_modifier_args_t margs; - pgtable_entry_types_t entry_types; - vmsa_upper_attrs_t upper_attrs; - vmsa_lower_attrs_t lower_attrs; - pgtable_t *pgt = NULL; + pgtable_lookup_modifier_args_t margs = { 0 }; + pgtable_entry_types_t entry_types = pgtable_entry_types_default(); + vmsa_upper_attrs_t upper_attrs; + vmsa_lower_attrs_t lower_attrs; + pgtable_t *pgt = NULL; assert(mapped_base != NULL); assert(mapped_size != NULL); @@ -3099,7 +3492,12 @@ pgtable_hyp_lookup(uintptr_t virtual_address, paddr_t *mapped_base, bool is_high = is_high_virtual_address(virtual_address); if (is_high) { +#if defined(ARCH_ARM_FEAT_VHE) pgt = &hyp_pgtable.top_control; +#else + walk_ret = false; + goto out; +#endif } else { pgt = &hyp_pgtable.bottom_control; } @@ -3109,14 +3507,18 @@ pgtable_hyp_lookup(uintptr_t virtual_address, paddr_t *mapped_base, goto out; } - memset(&margs, 0, sizeof(margs)); - entry_types = VMSA_ENTRY_TYPE_BLOCK | VMSA_ENTRY_TYPE_PAGE; + if (is_high) { + virtual_address &= util_mask(pgt->address_bits); + } + + pgtable_entry_types_set_block(&entry_types, true); + pgtable_entry_types_set_page(&entry_types, true); // just try to lookup a page, but if it's a block, the modifier will // stop the walk and return success walk_ret = translation_table_walk( pgt, virtual_address, util_bit(pgt->granule_shift), PGTABLE_TRANSLATION_TABLE_WALK_EVENT_LOOKUP, entry_types, - &margs, false); + &margs); if (margs.size == 0U) { // Return error (not-mapped) if lookup found no pages. @@ -3148,8 +3550,9 @@ error_t pgtable_hyp_preallocate(partition_t *partition, uintptr_t virtual_address, size_t size) { - pgtable_prealloc_modifier_args_t margs; - pgtable_t *pgt = NULL; + pgtable_prealloc_modifier_args_t margs = { 0 }; + pgtable_t *pgt = NULL; + pgtable_entry_types_t entry_types = pgtable_entry_types_default(); assert(partition != NULL); assert((size & (size - 1)) == 0U); @@ -3157,7 +3560,12 @@ pgtable_hyp_preallocate(partition_t *partition, uintptr_t virtual_address, bool is_high = is_high_virtual_address(virtual_address); if (is_high) { +#if defined(ARCH_ARM_FEAT_VHE) pgt = &hyp_pgtable.top_control; +#else + margs.error = ERROR_ADDR_INVALID; + goto out; +#endif } else { pgt = &hyp_pgtable.bottom_control; } @@ -3168,20 +3576,26 @@ pgtable_hyp_preallocate(partition_t *partition, uintptr_t virtual_address, addr_check(virtual_address + size - 1, pgt->address_bits, is_high)); - memset(&margs, 0, sizeof(margs)); + if (is_high) { + virtual_address &= util_mask(pgt->address_bits); + } + margs.partition = partition; margs.new_page_start_level = PGTABLE_INVALID_LEVEL; margs.error = OK; + pgtable_entry_types_set_invalid(&entry_types, true); bool walk_ret = translation_table_walk( pgt, virtual_address, size, - PGTABLE_TRANSLATION_TABLE_WALK_EVENT_PREALLOC, - VMSA_ENTRY_TYPE_INVALID, &margs, false); + PGTABLE_TRANSLATION_TABLE_WALK_EVENT_PREALLOC, entry_types, + &margs); if (!walk_ret && (margs.error == OK)) { margs.error = ERROR_FAILURE; + goto out; } +out: return margs.error; } @@ -3193,25 +3607,31 @@ static error_t pgtable_do_hyp_map(partition_t *partition, uintptr_t virtual_address, size_t size, paddr_t phys, pgtable_hyp_memtype_t memtype, pgtable_access_t access, vmsa_shareability_t shareability, - bool try_map) REQUIRE_LOCK(pgtable_hyp_map_lock) + bool try_map, size_t merge_limit) + REQUIRE_LOCK(pgtable_hyp_map_lock) { - pgtable_map_modifier_args_t margs; + pgtable_map_modifier_args_t margs = { 0 }; vmsa_stg1_lower_attrs_t l; vmsa_stg1_upper_attrs_t u; - pgtable_t *pgt = NULL; + pgtable_t *pgt = NULL; - assert(pgtable_op == true); + assert(pgtable_op); assert(partition != NULL); bool is_high = is_high_virtual_address(virtual_address); if (is_high) { +#if defined(ARCH_ARM_FEAT_VHE) pgt = &hyp_pgtable.top_control; +#else + margs.error = ERROR_ADDR_INVALID; + goto out; +#endif } else { pgt = &hyp_pgtable.bottom_control; } - if (util_add_overflows(virtual_address, size - 1)) { + if (util_add_overflows(virtual_address, size - 1U)) { margs.error = ERROR_ADDR_OVERFLOW; goto out; } @@ -3232,13 +3652,16 @@ pgtable_do_hyp_map(partition_t *partition, uintptr_t virtual_address, } if (!addr_check(virtual_address, pgt->address_bits, is_high) || - !addr_check(virtual_address + size - 1, pgt->address_bits, + !addr_check(virtual_address + size - 1U, pgt->address_bits, is_high)) { margs.error = ERROR_ADDR_INVALID; goto out; } - memset(&margs, 0, sizeof(margs)); + if (is_high) { + virtual_address &= util_mask(pgt->address_bits); + } + margs.orig_virtual_address = virtual_address; margs.orig_size = size; margs.phys = phys; @@ -3255,17 +3678,19 @@ pgtable_do_hyp_map(partition_t *partition, uintptr_t virtual_address, margs.error = OK; margs.try_map = try_map; margs.stage = PGTABLE_HYP_STAGE_1; + margs.merge_limit = merge_limit; // FIXME: try to unify the level number, just use one kind of level + pgtable_entry_types_t entry_types = VMSA_ENTRY_TYPE_LEAF; + pgtable_entry_types_set_next_level_table(&entry_types, true); bool walk_ret = translation_table_walk( pgt, virtual_address, size, - PGTABLE_TRANSLATION_TABLE_WALK_EVENT_MMAP, VMSA_ENTRY_TYPE_LEAF, - &margs, false); + PGTABLE_TRANSLATION_TABLE_WALK_EVENT_MMAP, entry_types, &margs); if (!walk_ret && (margs.error == OK)) { margs.error = ERROR_FAILURE; } - if ((margs.error != OK) && (margs.partially_mapped_size != 0)) { + if ((margs.error != OK) && (margs.partially_mapped_size != 0U)) { pgtable_hyp_unmap(partition, virtual_address, margs.partially_mapped_size, PGTABLE_HYP_UNMAP_PRESERVE_ALL); @@ -3275,21 +3700,25 @@ pgtable_do_hyp_map(partition_t *partition, uintptr_t virtual_address, } error_t -pgtable_hyp_map(partition_t *partition, uintptr_t virtual_address, size_t size, - paddr_t phys, pgtable_hyp_memtype_t memtype, - pgtable_access_t access, vmsa_shareability_t shareability) +pgtable_hyp_map_merge(partition_t *partition, uintptr_t virtual_address, + size_t size, paddr_t phys, pgtable_hyp_memtype_t memtype, + pgtable_access_t access, vmsa_shareability_t shareability, + size_t merge_limit) { return pgtable_do_hyp_map(partition, virtual_address, size, phys, - memtype, access, shareability, true); + memtype, access, shareability, true, + merge_limit); } error_t -pgtable_hyp_remap(partition_t *partition, uintptr_t virtual_address, - size_t size, paddr_t phys, pgtable_hyp_memtype_t memtype, - pgtable_access_t access, vmsa_shareability_t shareability) +pgtable_hyp_remap_merge(partition_t *partition, uintptr_t virtual_address, + size_t size, paddr_t phys, + pgtable_hyp_memtype_t memtype, pgtable_access_t access, + vmsa_shareability_t shareability, size_t merge_limit) { return pgtable_do_hyp_map(partition, virtual_address, size, phys, - memtype, access, shareability, false); + memtype, access, shareability, false, + merge_limit); } // FIXME: assume the size must be multiple of single page size or available @@ -3306,17 +3735,21 @@ void pgtable_hyp_unmap(partition_t *partition, uintptr_t virtual_address, size_t size, size_t preserved_prealloc) { - pgtable_unmap_modifier_args_t margs; - pgtable_t *pgt = NULL; + pgtable_unmap_modifier_args_t margs = { 0 }; + pgtable_t *pgt = NULL; - assert(pgtable_op == true); + assert(pgtable_op); assert(partition != NULL); assert(util_is_p2_or_zero(preserved_prealloc)); bool is_high = is_high_virtual_address(virtual_address); if (is_high) { +#if defined(ARCH_ARM_FEAT_VHE) pgt = &hyp_pgtable.top_control; +#else + goto out; +#endif } else { pgt = &hyp_pgtable.bottom_control; } @@ -3330,7 +3763,10 @@ pgtable_hyp_unmap(partition_t *partition, uintptr_t virtual_address, assert(util_is_p2aligned(virtual_address, pgt->granule_shift)); assert(util_is_p2aligned(size, pgt->granule_shift)); - memset(&margs, 0, sizeof(margs)); + if (is_high) { + virtual_address &= util_mask(pgt->address_bits); + } + margs.partition = partition; margs.preserved_size = preserved_prealloc; margs.stage = PGTABLE_HYP_STAGE_1; @@ -3338,10 +3774,15 @@ pgtable_hyp_unmap(partition_t *partition, uintptr_t virtual_address, bool walk_ret = translation_table_walk( pgt, virtual_address, size, PGTABLE_TRANSLATION_TABLE_WALK_EVENT_UNMAP, - VMSA_ENTRY_TYPE_LEAF, &margs, false); + VMSA_ENTRY_TYPE_LEAF, &margs); if (!walk_ret) { panic("Error in pgtable_hyp_unmap"); } + +#if !defined(ARCH_ARM_FEAT_VHE) +out: +#endif + return; } void @@ -3353,7 +3794,7 @@ pgtable_hyp_start(void) LOCK_IMPL // ensure forward progress and because the code is not thread safe. spinlock_acquire(&hyp_pgtable.lock); #if !defined(NDEBUG) - assert(pgtable_op == false); + assert(!pgtable_op); pgtable_op = true; #endif } @@ -3361,11 +3802,9 @@ pgtable_hyp_start(void) LOCK_IMPL void pgtable_hyp_commit(void) LOCK_IMPL { -#ifndef HOST_TEST - __asm__ volatile("dsb ish" ::: "memory"); -#endif + dsb(false); #if !defined(NDEBUG) - assert(pgtable_op == true); + assert(pgtable_op); pgtable_op = false; #endif spinlock_release(&hyp_pgtable.lock); @@ -3375,36 +3814,29 @@ pgtable_hyp_commit(void) LOCK_IMPL void pgtable_hyp_dump(void) { - pgtable_entry_types_t entry_types; - vmaddr_t virtual_address = 0U; - size_t size = 0U; + pgtable_entry_types_t entry_types = + pgtable_entry_types_inverse(pgtable_entry_types_default()); + vmaddr_t virtual_address = 0U; + size_t size = 0U; LOG(DEBUG, INFO, "+---------------- page table ----------------\n"); +#if defined(ARCH_ARM_FEAT_VHE) LOG(DEBUG, INFO, "| TTBR1[{:#x}]:\n", hyp_pgtable.top_control.root_pgtable); - entry_types = VMSA_ENTRY_TYPE_BLOCK | VMSA_ENTRY_TYPE_PAGE | - VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE | - VMSA_ENTRY_TYPE_INVALID | VMSA_ENTRY_TYPE_RESERVED | - VMSA_ENTRY_TYPE_ERROR | VMSA_ENTRY_TYPE_NONE; - virtual_address = ~util_mask(hyp_pgtable.top_control.address_bits); - size = 1UL << hyp_pgtable.top_control.address_bits; + size = util_bit(hyp_pgtable.top_control.address_bits); (void)translation_table_walk(&hyp_pgtable.top_control, virtual_address, size, PGTABLE_TRANSLATION_TABLE_WALK_EVENT_DUMP, - entry_types, NULL, false); + entry_types, NULL); +#endif LOG(DEBUG, INFO, "\n"); LOG(DEBUG, INFO, "| TTBR0[{:#x}]:\n", hyp_pgtable.bottom_control.root_pgtable); - entry_types = VMSA_ENTRY_TYPE_BLOCK | VMSA_ENTRY_TYPE_PAGE | - VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE | - VMSA_ENTRY_TYPE_INVALID | VMSA_ENTRY_TYPE_RESERVED | - VMSA_ENTRY_TYPE_ERROR | VMSA_ENTRY_TYPE_NONE; - virtual_address = 0U; - size = 1UL << hyp_pgtable.bottom_control.address_bits; + size = util_bit(hyp_pgtable.bottom_control.address_bits); (void)translation_table_walk(&hyp_pgtable.bottom_control, virtual_address, size, PGTABLE_TRANSLATION_TABLE_WALK_EVENT_DUMP, - entry_types, NULL, false); + entry_types, NULL); LOG(DEBUG, INFO, "+--------------------------------------------\n\n"); } #endif @@ -3414,8 +3846,8 @@ void pgtable_hyp_ext(vmaddr_t virtual_address, size_t size, pgtable_entry_types_t entry_types, ext_func_t func, void *data) { - ext_modifier_args_t margs; - pgtable_t *pgt = NULL; + ext_modifier_args_t margs = { 0 }; + pgtable_t *pgt = NULL; bool is_high = is_high_virtual_address(virtual_address); if (is_high) { @@ -3428,14 +3860,33 @@ pgtable_hyp_ext(vmaddr_t virtual_address, size_t size, assert(addr_check(virtual_address + size - 1, pgt->address_bits, is_high)); - memset(&margs, 0, sizeof(margs)); margs.func = func; margs.data = data; + if (!util_is_p2aligned(size, pgt->granule_shift) || + !util_is_p2aligned(size, pgt->granule_shift)) { + LOG(DEBUG, INFO, "size not aligned\n"); + goto out; + } + if (!addr_check(virtual_address, pgt->address_bits, is_high)) { + LOG(DEBUG, INFO, "address out of range\n"); + goto out; + } + if (!util_is_p2aligned(virtual_address, pgt->granule_shift)) { + LOG(DEBUG, INFO, "address not aligned\n"); + goto out; + } + + if (is_high) { + virtual_address &= util_mask(pgt->address_bits); + } + (void)translation_table_walk( pgt, virtual_address, size, PGTABLE_TRANSLATION_TABLE_WALK_EVENT_EXTERNAL, entry_types, - &margs, false); + &margs); +out: + return; } #endif @@ -3445,19 +3896,16 @@ pgtable_vm_dump(pgtable_vm_t *pgt) { assert(pgt != NULL); - pgtable_entry_types_t entry_types; + pgtable_entry_types_t entry_types = + pgtable_entry_types_inverse(pgtable_entry_types_default()); size_t size = util_bit(pgt->control.address_bits); LOG(DEBUG, INFO, "+---------------- page table ----------------\n"); LOG(DEBUG, INFO, "| TTBR({:#x}):\n", pgt->control.root_pgtable); - entry_types = VMSA_ENTRY_TYPE_BLOCK | VMSA_ENTRY_TYPE_PAGE | - VMSA_ENTRY_TYPE_NEXT_LEVEL_TABLE | - VMSA_ENTRY_TYPE_INVALID | VMSA_ENTRY_TYPE_RESERVED | - VMSA_ENTRY_TYPE_ERROR | VMSA_ENTRY_TYPE_NONE; (void)translation_table_walk(&pgt->control, 0L, size, PGTABLE_TRANSLATION_TABLE_WALK_EVENT_DUMP, - entry_types, NULL, false); + entry_types, NULL); LOG(DEBUG, INFO, "+--------------------------------------------\n\n"); } #endif @@ -3467,28 +3915,27 @@ void pgtable_vm_ext(pgtable_vm_t *pgt, vmaddr_t virtual_address, size_t size, pgtable_entry_types_t entry_types, ext_func_t func, void *data) { - ext_modifier_args_t margs; + ext_modifier_args_t margs = { 0 }; assert(pgt != NULL); assert(addr_check(virtual_address, pgt->control.address_bits, false)); assert(addr_check(virtual_address + size - 1, pgt->control.address_bits, false)); - memset(&margs, 0, sizeof(margs)); margs.func = func; margs.data = data; (void)translation_table_walk( &pgt->control, virtual_address, size, PGTABLE_TRANSLATION_TABLE_WALK_EVENT_EXTERNAL, entry_types, - &margs, false); + &margs); } #endif static tcr_tg0_t vtcr_get_tg0_code(size_t granule_shift) { - tcr_tg0_t tg0 = 0U; + tcr_tg0_t tg0; switch (granule_shift) { case SHIFT_4K: @@ -3508,6 +3955,12 @@ vtcr_get_tg0_code(size_t granule_shift) } #if !defined(HOST_TEST) +// Note: the nested macros are are needed to expand the config to a number +// before it is pasted into the enum name. +#define PLATFORM_TCR_PS TCR_PS_FOR_CONFIG(PLATFORM_PHYS_ADDRESS_BITS) +#define TCR_PS_FOR_CONFIG(x) TCR_PS_FOR_SIZE(x) +#define TCR_PS_FOR_SIZE(x) TCR_PS_SIZE_##x##BITS + static void pgtable_vm_init_regs(pgtable_vm_t *vm_pgtable) { @@ -3560,13 +4013,20 @@ pgtable_vm_init_regs(pgtable_vm_t *vm_pgtable) tcr_tg0_t tg0 = vtcr_get_tg0_code(vm_pgtable->control.granule_shift); VTCR_EL2_set_TG0(&vm_pgtable->vtcr_el2, tg0); - ID_AA64MMFR0_EL1_t id_aa64mmfro = register_ID_AA64MMFR0_EL1_read(); - VTCR_EL2_set_PS(&vm_pgtable->vtcr_el2, - ID_AA64MMFR0_EL1_get_PARange(&id_aa64mmfro)); + // The output size is defined by the platform. + VTCR_EL2_set_PS(&vm_pgtable->vtcr_el2, PLATFORM_TCR_PS); + + // The platform's implemented physical address space must be no + // larger than the CPU's implemented physical address size. + ID_AA64MMFR0_EL1_t id_aa64mmfr0 = register_ID_AA64MMFR0_EL1_read(); + assert(ID_AA64MMFR0_EL1_get_PARange(&id_aa64mmfr0) >= + VTCR_EL2_get_PS(&vm_pgtable->vtcr_el2)); - // The stage-2 input address must be equal or smaller than the CPU's - // reported physical address space size. - switch (VTCR_EL2_get_PS(&vm_pgtable->vtcr_el2)) { + // The stage-2 input address size must be no larger than the CPU's + // implemented physical address size (though it may be larger than the + // platform's implemented physical address space, if that is smaller + // than the CPU's). + switch (ID_AA64MMFR0_EL1_get_PARange(&id_aa64mmfr0)) { case TCR_PS_SIZE_32BITS: assert(vm_pgtable->control.address_bits <= 32); break; @@ -3592,26 +4052,26 @@ pgtable_vm_init_regs(pgtable_vm_t *vm_pgtable) panic("bad PARange"); } -#if defined(ARCH_ARM_8_1_VMID16) +#if defined(ARCH_ARM_FEAT_VMID16) VTCR_EL2_set_VS(&vm_pgtable->vtcr_el2, true); #endif -#if defined(ARCH_ARM_8_1_TTHM) +#if defined(ARCH_ARM_FEAT_HAFDBS) VTCR_EL2_set_HA(&vm_pgtable->vtcr_el2, true); ID_AA64MMFR1_EL1_t hw_mmfr1 = register_ID_AA64MMFR1_EL1_read(); - if (ID_AA64MMFR1_EL1_get_HAFDBS(&hw_mmfr1) == 2) { + if (ID_AA64MMFR1_EL1_get_HAFDBS(&hw_mmfr1) == 2U) { VTCR_EL2_set_HD(&vm_pgtable->vtcr_el2, true); } #endif -#if defined(ARCH_ARM_8_2_TTPBHA) +#if defined(ARCH_ARM_FEAT_HPDS2) VTCR_EL2_set_HWU059(&vm_pgtable->vtcr_el2, false); VTCR_EL2_set_HWU060(&vm_pgtable->vtcr_el2, false); VTCR_EL2_set_HWU061(&vm_pgtable->vtcr_el2, false); VTCR_EL2_set_HWU062(&vm_pgtable->vtcr_el2, false); #endif -#if defined(ARCH_ARM_8_4_SEC_EL2) +#if defined(ARCH_ARM_FEAT_SEC_EL2) VTCR_EL2_set_NSW(&vm_pgtable->vtcr_el2, true); VTCR_EL2_set_NSA(&vm_pgtable->vtcr_el2, true); #endif @@ -3622,7 +4082,7 @@ pgtable_vm_init_regs(pgtable_vm_t *vm_pgtable) VTTBR_EL2_set_CnP(&vm_pgtable->vttbr_el2, true); VTTBR_EL2_set_BADDR(&vm_pgtable->vttbr_el2, vm_pgtable->control.root_pgtable); -#if defined(ARCH_ARM_8_1_VMID16) +#if defined(ARCH_ARM_FEAT_VMID16) VTTBR_EL2_set_VMID(&vm_pgtable->vttbr_el2, vm_pgtable->control.vmid); #else VTTBR_EL2_set_VMID(&vm_pgtable->vttbr_el2, @@ -3650,6 +4110,7 @@ pgtable_vm_init(partition_t *partition, pgtable_vm_t *pgtable, vmid_t vmid) goto out; } + // FIXME: // FIXME: refine with more configurable code #if PGTABLE_VM_PAGE_SIZE == 4096 pgtable->control.granule_shift = SHIFT_4K; @@ -3660,8 +4121,9 @@ pgtable_vm_init(partition_t *partition, pgtable_vm_t *pgtable, vmid_t vmid) msb = PLATFORM_VM_ADDRESS_SPACE_BITS - 1; pgtable->control.vmid = vmid; - get_start_level_info_ret_t info = get_start_level_info(level_conf, msb); - pgtable->control.start_level = info.level; + get_start_level_info_ret_t info = + get_start_level_info(level_conf, msb, true); + pgtable->control.start_level = info.level; pgtable->control.start_level_size = info.size; pgtable->issue_dvm_cmd = false; @@ -3693,15 +4155,15 @@ pgtable_vm_destroy(partition_t *partition, pgtable_vm_t *pgtable) assert(pgtable->control.root != NULL); virtual_address = 0x0U; - size = 1UL << pgtable->control.address_bits; + size = util_bit(pgtable->control.address_bits); // we should unmap everything pgtable_vm_start(pgtable); pgtable_vm_unmap(partition, pgtable, virtual_address, size); pgtable_vm_commit(pgtable); // free top level page table - partition_free(partition, pgtable->control.root, - pgtable->control.start_level_size); + (void)partition_free(partition, pgtable->control.root, + pgtable->control.start_level_size); pgtable->control.root = NULL; } @@ -3709,14 +4171,14 @@ bool pgtable_vm_lookup(pgtable_vm_t *pgtable, vmaddr_t virtual_address, paddr_t *mapped_base, size_t *mapped_size, pgtable_vm_memtype_t *mapped_memtype, - pgtable_access_t *mapped_vm_kernel_access, - pgtable_access_t *mapped_vm_user_access) + pgtable_access_t *mapped_vm_kernel_access, + pgtable_access_t *mapped_vm_user_access) { bool walk_ret; - pgtable_lookup_modifier_args_t margs; - pgtable_entry_types_t entry_types; - vmsa_upper_attrs_t upper_attrs; - vmsa_lower_attrs_t lower_attrs; + pgtable_lookup_modifier_args_t margs = { 0 }; + pgtable_entry_types_t entry_types = pgtable_entry_types_default(); + vmsa_upper_attrs_t upper_attrs; + vmsa_lower_attrs_t lower_attrs; assert(pgtable != NULL); assert(mapped_base != NULL); @@ -3732,15 +4194,16 @@ pgtable_vm_lookup(pgtable_vm_t *pgtable, vmaddr_t virtual_address, goto out; } - memset(&margs, 0, sizeof(margs)); - entry_types = VMSA_ENTRY_TYPE_BLOCK | VMSA_ENTRY_TYPE_PAGE; + pgtable_entry_types_set_block(&entry_types, true); + pgtable_entry_types_set_page(&entry_types, true); + // just try to lookup a page, but if it's a block, the modifier will // stop the walk and return success walk_ret = translation_table_walk( &pgtable->control, virtual_address, util_bit(pgtable->control.granule_shift), PGTABLE_TRANSLATION_TABLE_WALK_EVENT_LOOKUP, entry_types, - &margs, pgtable->issue_dvm_cmd); + &margs); if (margs.size == 0U) { // Return error (not-mapped) if lookup found no pages. @@ -3778,13 +4241,13 @@ error_t pgtable_vm_map(partition_t *partition, pgtable_vm_t *pgtable, vmaddr_t virtual_address, size_t size, paddr_t phys, pgtable_vm_memtype_t memtype, pgtable_access_t vm_kernel_access, - pgtable_access_t vm_user_access, bool try_map) + pgtable_access_t vm_user_access, bool try_map, bool allow_merge) { - pgtable_map_modifier_args_t margs; + pgtable_map_modifier_args_t margs = { 0 }; vmsa_stg2_lower_attrs_t l; vmsa_stg2_upper_attrs_t u; - assert(pgtable_op == true); + assert(pgtable_op); assert(pgtable != NULL); assert(partition != NULL); @@ -3795,13 +4258,14 @@ pgtable_vm_map(partition_t *partition, pgtable_vm_t *pgtable, goto fail; } - if (util_add_overflows(virtual_address, size - 1) || - !addr_check(virtual_address + size - 1, + if (util_add_overflows(virtual_address, size - 1U) || + !addr_check(virtual_address + size - 1U, pgtable->control.address_bits, false)) { margs.error = ERROR_ADDR_OVERFLOW; goto fail; } + // FIXME: // Supporting different granule sizes will need support and additional // checking to be added to memextent code. if (!util_is_p2aligned(virtual_address, @@ -3815,7 +4279,6 @@ pgtable_vm_map(partition_t *partition, pgtable_vm_t *pgtable, // FIXME: how to check phys, read tcr in init? // FIXME: no need to to check vm memtype, right? - memset(&margs, 0, sizeof(margs)); margs.orig_virtual_address = virtual_address; margs.orig_size = size; margs.phys = phys; @@ -3830,15 +4293,31 @@ pgtable_vm_map(partition_t *partition, pgtable_vm_t *pgtable, margs.error = OK; margs.try_map = try_map; margs.stage = PGTABLE_VM_STAGE_2; + margs.outer_shareable = pgtable->issue_dvm_cmd; +#if (CPU_PGTABLE_BBM_LEVEL > 0) || !defined(PLATFORM_PGTABLE_AVOID_BBM) + // We can either trigger TLB conflicts safely because they will be + // delivered to EL2, or else can use BBM. + margs.merge_limit = allow_merge ? ~(size_t)0U : 0U; +#else + // We can't use BBM, and merging without it might cause TLB conflict + // aborts in EL1. This is unsafe because: + // - the EL1 abort handler might trigger the same abort again, and + // - Linux VMs treat TLB conflict aborts as fatal errors. + (void)allow_merge; + margs.merge_limit = false; +#endif // FIXME: try to unify the level number, just use one kind of level + pgtable_entry_types_t entry_types = VMSA_ENTRY_TYPE_LEAF; + pgtable_entry_types_set_next_level_table(&entry_types, true); bool walk_ret = translation_table_walk( &pgtable->control, virtual_address, size, - PGTABLE_TRANSLATION_TABLE_WALK_EVENT_MMAP, VMSA_ENTRY_TYPE_LEAF, - &margs, pgtable->issue_dvm_cmd); + PGTABLE_TRANSLATION_TABLE_WALK_EVENT_MMAP, entry_types, &margs); - if (((margs.error != OK) || !walk_ret) && - (margs.partially_mapped_size != 0)) { + if (!walk_ret && (margs.error == OK)) { + margs.error = ERROR_FAILURE; + } + if ((margs.error != OK) && (margs.partially_mapped_size != 0U)) { pgtable_vm_unmap(partition, pgtable, virtual_address, margs.partially_mapped_size); } @@ -3851,9 +4330,9 @@ void pgtable_vm_unmap(partition_t *partition, pgtable_vm_t *pgtable, vmaddr_t virtual_address, size_t size) { - pgtable_unmap_modifier_args_t margs; + pgtable_unmap_modifier_args_t margs = { 0 }; - assert(pgtable_op == true); + assert(pgtable_op); assert(pgtable != NULL); assert(partition != NULL); @@ -3863,8 +4342,8 @@ pgtable_vm_unmap(partition_t *partition, pgtable_vm_t *pgtable, panic("Bad arguments in pgtable_vm_unmap"); } - if (util_add_overflows(virtual_address, size - 1) || - !addr_check(virtual_address + size - 1, + if (util_add_overflows(virtual_address, size - 1U) || + !addr_check(virtual_address + size - 1U, pgtable->control.address_bits, false)) { panic("Bad arguments in pgtable_vm_unmap"); } @@ -3875,16 +4354,16 @@ pgtable_vm_unmap(partition_t *partition, pgtable_vm_t *pgtable, panic("Bad arguments in pgtable_vm_unmap"); } - memset(&margs, 0, sizeof(margs)); margs.partition = partition; // no need to preserve table levels here - margs.preserved_size = PGTABLE_HYP_UNMAP_PRESERVE_NONE; - margs.stage = PGTABLE_VM_STAGE_2; + margs.preserved_size = PGTABLE_HYP_UNMAP_PRESERVE_NONE; + margs.stage = PGTABLE_VM_STAGE_2; + margs.outer_shareable = pgtable->issue_dvm_cmd; bool walk_ret = translation_table_walk( &pgtable->control, virtual_address, size, PGTABLE_TRANSLATION_TABLE_WALK_EVENT_UNMAP, - VMSA_ENTRY_TYPE_LEAF, &margs, pgtable->issue_dvm_cmd); + VMSA_ENTRY_TYPE_LEAF, &margs); if (!walk_ret) { panic("Error in pgtable_vm_unmap"); } @@ -3894,9 +4373,9 @@ void pgtable_vm_unmap_matching(partition_t *partition, pgtable_vm_t *pgtable, vmaddr_t virtual_address, paddr_t phys, size_t size) { - pgtable_unmap_modifier_args_t margs; + pgtable_unmap_modifier_args_t margs = { 0 }; - assert(pgtable_op == true); + assert(pgtable_op); assert(pgtable != NULL); assert(partition != NULL); @@ -3906,24 +4385,24 @@ pgtable_vm_unmap_matching(partition_t *partition, pgtable_vm_t *pgtable, panic("Bad arguments in pgtable_vm_unmap_matching"); } - if (util_add_overflows(virtual_address, size - 1) || - !addr_check(virtual_address + size - 1, + if (util_add_overflows(virtual_address, size - 1U) || + !addr_check(virtual_address + size - 1U, pgtable->control.address_bits, false)) { panic("Bad arguments in pgtable_vm_unmap_matching"); } - memset(&margs, 0, sizeof(margs)); margs.partition = partition; // no need to preserve table levels here - margs.preserved_size = PGTABLE_HYP_UNMAP_PRESERVE_NONE; - margs.stage = PGTABLE_VM_STAGE_2; - margs.phys = phys; - margs.size = size; + margs.preserved_size = PGTABLE_HYP_UNMAP_PRESERVE_NONE; + margs.stage = PGTABLE_VM_STAGE_2; + margs.phys = phys; + margs.size = size; + margs.outer_shareable = pgtable->issue_dvm_cmd; bool walk_ret = translation_table_walk( &pgtable->control, virtual_address, size, PGTABLE_TRANSLATION_TABLE_WALK_EVENT_UNMAP_MATCH, - VMSA_ENTRY_TYPE_LEAF, &margs, pgtable->issue_dvm_cmd); + VMSA_ENTRY_TYPE_LEAF, &margs); if (!walk_ret) { panic("Error in pgtable_vm_unmap_matching"); } @@ -3934,11 +4413,12 @@ pgtable_vm_start(pgtable_vm_t *pgtable) LOCK_IMPL { assert(pgtable != NULL); #ifndef HOST_TEST + // FIXME: // We need to to run VM pagetable code with preempt disable due to // TLB flushes. preempt_disable(); #if !defined(NDEBUG) - assert(pgtable_op == false); + assert(!pgtable_op); pgtable_op = true; #endif @@ -3952,7 +4432,9 @@ pgtable_vm_start(pgtable_vm_t *pgtable) LOCK_IMPL // original registers. if ((thread->addrspace == NULL) || (&thread->addrspace->vm_pgtable != pgtable)) { - register_VTTBR_EL2_write(pgtable->vttbr_el2); + register_VTTBR_EL2_write_ordered(pgtable->vttbr_el2, + &asm_ordering); + asm_context_sync_ordered(&asm_ordering); } #endif } @@ -3962,14 +4444,15 @@ pgtable_vm_commit(pgtable_vm_t *pgtable) LOCK_IMPL { #ifndef HOST_TEST #if !defined(NDEBUG) - assert(pgtable_op == true); + assert(pgtable_op); pgtable_op = false; #endif - __asm__ volatile("dsb ish" ::: "memory"); + dsb(pgtable->issue_dvm_cmd); // This is only needed when unmapping. Consider some flags to // track to flush requirements. - __asm__ volatile("tlbi VMALLE1IS; dsb ish" : "+m"(asm_ordering)); + vm_tlbi_vmalle1(pgtable->issue_dvm_cmd); + dsb(pgtable->issue_dvm_cmd); thread_t *thread = thread_get_self(); @@ -3978,8 +4461,8 @@ pgtable_vm_commit(pgtable_vm_t *pgtable) LOCK_IMPL // original VMID (in VTTBR_EL2) here. if ((thread->addrspace != NULL) && (&thread->addrspace->vm_pgtable != pgtable)) { - register_VTTBR_EL2_write( - thread->addrspace->vm_pgtable.vttbr_el2); + register_VTTBR_EL2_write_ordered( + thread->addrspace->vm_pgtable.vttbr_el2, &asm_ordering); } preempt_enable(); diff --git a/hyp/mem/pgtable/build.conf b/hyp/mem/pgtable/build.conf index a4de1dc..90631bd 100644 --- a/hyp/mem/pgtable/build.conf +++ b/hyp/mem/pgtable/build.conf @@ -7,6 +7,6 @@ interface pgtable arch_events armv8 pgtable.ev arch_types armv8 pgtable.tc arch_source armv8 pgtable.c -arch_configs armv8 PGTABLE_HYP_PAGE_SIZE=4096 -arch_configs armv8 PGTABLE_HYP_LARGE_PAGE_SIZE=2097152 -arch_configs armv8 PGTABLE_VM_PAGE_SIZE=4096 +arch_configs armv8 PGTABLE_HYP_PAGE_SIZE=4096U +arch_configs armv8 PGTABLE_HYP_LARGE_PAGE_SIZE=2097152U +arch_configs armv8 PGTABLE_VM_PAGE_SIZE=4096U diff --git a/hyp/mem/useraccess/aarch64/src/useraccess.c b/hyp/mem/useraccess/aarch64/src/useraccess.c index 9359e51..b668dea 100644 --- a/hyp/mem/useraccess/aarch64/src/useraccess.c +++ b/hyp/mem/useraccess/aarch64/src/useraccess.c @@ -27,7 +27,7 @@ useraccess_copy_from_to_guest_va(gvaddr_t gvaddr, void *hvaddr, size_t size, error_t ret = OK; size_t remaining = size; gvaddr_t guest_va = gvaddr; - void *hyp_buf = hvaddr; + void *hyp_buf = hvaddr; assert(hyp_buf != NULL); assert(remaining != 0U); @@ -75,10 +75,11 @@ useraccess_copy_from_to_guest_va(gvaddr_t gvaddr, void *hvaddr, size_t size, size_t mapped_size = page_size - page_offset; void *va = partition_phys_map(guest_pa, mapped_size); - MAIR_ATTR_t attr = PAR_EL1_F0_get_ATTR(&par.f0); - bool writeback = (attr | MAIR_ATTR_ALLOC_HINT_MASK) == - MAIR_ATTR_NORMAL_WB; -#if defined(ARCH_ARM_8_5_MEMTAG) + MAIR_ATTR_t attr = PAR_EL1_F0_get_ATTR(&par.f0); + bool writeback = ((index_t)attr | + (index_t)MAIR_ATTR_ALLOC_HINT_MASK) == + (index_t)MAIR_ATTR_NORMAL_WB; +#if defined(ARCH_ARM_FEAT_MTE) writeback = writeback || (attr == MAIR_ATTR_TAGGED_NORMAL_WB); #endif @@ -202,9 +203,10 @@ useraccess_copy_from_to_guest_ipa(addrspace_t *addrspace, vmaddr_t ipa, break; } - if (!force_access && (mapped_vm_kernel_access & - (from_guest ? PGTABLE_ACCESS_R - : PGTABLE_ACCESS_W)) == 0U) { + if (!force_access && + !pgtable_access_check(mapped_vm_kernel_access, + (from_guest ? PGTABLE_ACCESS_R + : PGTABLE_ACCESS_W))) { rcu_read_finish(); ret = ERROR_DENIED; break; diff --git a/hyp/misc/abort/abort.ev b/hyp/misc/abort/abort.ev index f275ff6..575ee5c 100644 --- a/hyp/misc/abort/abort.ev +++ b/hyp/misc/abort/abort.ev @@ -4,6 +4,7 @@ module abort -subscribe ipi_received[IPI_REASON_ABORT_STOP]() +subscribe ipi_received[IPI_REASON_ABORT_STOP]() noreturn subscribe scheduler_stop() + require_preempt_disabled diff --git a/hyp/misc/abort/src/abort.c b/hyp/misc/abort/src/abort.c index 8b99c40..fdbdc91 100644 --- a/hyp/misc/abort/src/abort.c +++ b/hyp/misc/abort/src/abort.c @@ -32,7 +32,7 @@ abort_handle_scheduler_stop(void) } } -noreturn bool NOINLINE +noreturn void NOINLINE abort_handle_ipi_received(void) { preempt_disable(); @@ -47,8 +47,8 @@ abort_handle_ipi_received(void) } } -noreturn void NOINLINE -abort(const char *str, abort_reason_t reason) +noreturn void NOINLINE COLD +abort(const char *str, abort_reason_t reason) LOCK_IMPL { void *from = __builtin_return_address(0); void *frame = __builtin_frame_address(0); @@ -56,7 +56,7 @@ abort(const char *str, abort_reason_t reason) // Stop all cores and disable preemption trigger_scheduler_stop_event(); -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) __asm__("xpaci %0;" : "+r"(from)); #endif TRACE_AND_LOG(ERROR, PANIC, "Abort: {:s} from PC {:#x}, FP {:#x}", diff --git a/hyp/misc/elf/src/elf_loader.c b/hyp/misc/elf/src/elf_loader.c index 5eec29b..0f8d128 100644 --- a/hyp/misc/elf/src/elf_loader.c +++ b/hyp/misc/elf/src/elf_loader.c @@ -83,6 +83,10 @@ elf_valid(void *elf_file, size_t max_size) #error unimplemented #endif + if (util_add_overflows((uintptr_t)elf_file, ehdr->e_phoff)) { + goto out; + } + Elf_Phdr *phdr = (Elf_Phdr *)((uintptr_t)elf_file + ehdr->e_phoff); Elf_Half phnum = ehdr->e_phnum; @@ -149,6 +153,7 @@ elf_load_phys(void *elf_file, size_t elf_max_size, paddr_t phys_base) Elf_Phdr *cur_phdr = &phdr[i]; Elf_Word type = cur_phdr->p_type; + // FIXME: assert(type != PT_TLS); if (type != PT_LOAD) { @@ -196,10 +201,12 @@ elf_load_phys(void *elf_file, size_t elf_max_size, paddr_t phys_base) assert(err == OK); // copy elf segment data - memcpy((char *)seg_dest, (char *)seg_base, cur_phdr->p_filesz); + (void)memcpy((char *)seg_dest, (char *)seg_base, + cur_phdr->p_filesz); // zero bss - memset((char *)(seg_dest + cur_phdr->p_filesz), 0, - cur_phdr->p_memsz - cur_phdr->p_filesz); + size_t bss_size = cur_phdr->p_memsz - cur_phdr->p_filesz; + (void)memset_s((char *)(seg_dest + cur_phdr->p_filesz), + bss_size, 0, bss_size); LOG(DEBUG, INFO, "Elf copied from {:#x} to {:#x} - size {:#x}", seg_base, seg_dest, cur_phdr->p_filesz); diff --git a/hyp/misc/gpt/build.conf b/hyp/misc/gpt/build.conf new file mode 100644 index 0000000..c8006c9 --- /dev/null +++ b/hyp/misc/gpt/build.conf @@ -0,0 +1,8 @@ +# © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +interface gpt +source gpt.c gpt_tests.c +types gpt.tc +events gpt.ev diff --git a/hyp/misc/gpt/gpt.ev b/hyp/misc/gpt/gpt.ev new file mode 100644 index 0000000..344a7e6 --- /dev/null +++ b/hyp/misc/gpt/gpt.ev @@ -0,0 +1,29 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module gpt + +subscribe gpt_values_equal[GPT_TYPE_EMPTY] + handler gpt_handle_empty_values_equal() + +subscribe gpt_walk_callback[GPT_CALLBACK_RESERVED] + handler gpt_handle_reserved_callback() noreturn + +subscribe rcu_update[RCU_UPDATE_CLASS_GPT_FREE_LEVEL] + handler gpt_handle_rcu_free_level(entry) + +#if defined(UNIT_TESTS) +subscribe tests_init + +subscribe tests_start + +subscribe gpt_value_add_offset[GPT_TYPE_TEST_A, GPT_TYPE_TEST_B, GPT_TYPE_TEST_C] + handler gpt_tests_add_offset + +subscribe gpt_values_equal[GPT_TYPE_TEST_A, GPT_TYPE_TEST_B, GPT_TYPE_TEST_C] + handler gpt_tests_values_equal(x, y) + +subscribe gpt_walk_callback[GPT_CALLBACK_TEST] + handler gpt_tests_callback(entry, base, size, arg) +#endif diff --git a/hyp/misc/gpt/gpt.tc b/hyp/misc/gpt/gpt.tc new file mode 100644 index 0000000..5339ac8 --- /dev/null +++ b/hyp/misc/gpt/gpt.tc @@ -0,0 +1,119 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +define GPT_MAX_SIZE_BITS constant type count_t = 52; +define GPT_MAX_SIZE constant size = 1 << GPT_MAX_SIZE_BITS; + +define GPT_LEVEL_LOG2_BITS constant type count_t = 2; +define GPT_LEVEL_BITS constant type count_t = 1 << GPT_LEVEL_LOG2_BITS; +define GPT_LEVEL_ENTRIES constant type count_t = 1 << GPT_LEVEL_BITS; + +define GPT_MAX_LEVELS constant type count_t = + (GPT_MAX_SIZE_BITS + GPT_LEVEL_BITS - 1) / GPT_LEVEL_BITS; + +define GPT_SHIFT_BITS constant type count_t = 6 - GPT_LEVEL_LOG2_BITS; +define GPT_TYPE_BITS constant type count_t = + 64 - GPT_MAX_SIZE_BITS - GPT_SHIFT_BITS; + +extend gpt_type enumeration { + level; +}; + +extend rcu_update_class enumeration { + gpt_free_level; +}; + +define gpt_pte_info bitfield<64> { + auto guard size; + auto shifts type count_t lsl(GPT_LEVEL_LOG2_BITS); + auto type enumeration gpt_type; +}; + +define gpt_pte structure { + info bitfield gpt_pte_info; + value union gpt_value; +}; + +define gpt_level_non_atomic structure { + entries array(GPT_LEVEL_ENTRIES) structure gpt_pte; +}; + +// FIXME: +define gpt_level_atomic structure(aligned(16)) { + entries array(GPT_LEVEL_ENTRIES) structure gpt_pte(atomic); + rcu_entry structure rcu_entry(contained); + partition pointer object partition; +}; + +define gpt_level union { + raw uintptr; + non_atomic pointer structure gpt_level_non_atomic; + atomic pointer structure gpt_level_atomic; +}; + +extend gpt_value union { + raw uint64; + level union gpt_level; +}; + +// FIXME: +define gpt_root union(aligned(16)) { + non_atomic structure gpt_pte; + atomic structure gpt_pte(atomic); +}; + +extend gpt structure { + root union gpt_root; + partition pointer object partition; + config bitfield gpt_config; + allowed_types uregister; +}; + +define gpt_read_op enumeration { + lookup; + is_contiguous; + walk; + dump_range; +}; + +define gpt_read_data structure { + entry structure gpt_entry; + base size; + size size; + cb enumeration gpt_callback; + arg union gpt_arg; +}; + +define gpt_frame_info bitfield<64> { + auto addr size; + auto shifts type count_t lsl(GPT_LEVEL_LOG2_BITS); + auto index type index_t; + auto dirty bool; +}; + +define gpt_stack_frame structure { + level union gpt_level; + info bitfield gpt_frame_info; +}; + +define gpt_stack structure { + depth type count_t; + frame array(GPT_MAX_LEVELS) structure gpt_stack_frame; +}; + +#if defined(UNIT_TESTS) +extend gpt_type enumeration { + test_a; + test_b; + test_c; +}; + +extend gpt_callback enumeration { + test; +}; + +extend gpt_arg union { + test uregister; +}; +#endif diff --git a/hyp/misc/gpt/src/gpt.c b/hyp/misc/gpt/src/gpt.c new file mode 100644 index 0000000..2c6b100 --- /dev/null +++ b/hyp/misc/gpt/src/gpt.c @@ -0,0 +1,1188 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "event_handlers.h" + +#define SIZE_T_BITS (sizeof(size_t) * (size_t)CHAR_BIT) + +static_assert(sizeof(gpt_value_t) <= sizeof(uint64_t), + "GPT value must not be larger than 64-bits!"); +static_assert(GPT_TYPE__MAX < util_bit(GPT_TYPE_BITS), + "GPT type does not fit in PTE bitfield!"); + +bool +gpt_handle_empty_values_equal(void) +{ + return true; +} + +void +gpt_handle_reserved_callback(void) +{ + panic("gpt: Reserved callback used"); +} + +static gpt_pte_t +gpt_pte_empty(void) +{ + return (gpt_pte_t){ + .info = gpt_pte_info_default(), + .value = { .raw = 0U }, + }; +} + +static size_t +get_max_size(gpt_config_t config) +{ + return util_bit(gpt_config_get_max_bits(&config)); +} + +static size_t +get_pte_addr(gpt_pte_t pte) +{ + size_t guard = gpt_pte_info_get_guard(&pte.info); + count_t shifts = gpt_pte_info_get_shifts(&pte.info); + + return guard << shifts; +} + +static size_t +get_pte_size(gpt_pte_t pte) +{ + return util_bit(gpt_pte_info_get_shifts(&pte.info)); +} + +static bool +guard_matching(gpt_pte_t pte, size_t addr) +{ + assert(gpt_pte_info_get_type(&pte.info) != GPT_TYPE_EMPTY); + + size_t guard = gpt_pte_info_get_guard(&pte.info); + count_t shifts = gpt_pte_info_get_shifts(&pte.info); + + return (addr >> shifts) == guard; +} + +static bool +entries_equal(gpt_entry_t a, gpt_entry_t b) +{ + return (a.type == b.type) && + trigger_gpt_values_equal_event(a.type, a.value, b.value); +} + +static gpt_pte_t +load_atomic_pte(_Atomic gpt_pte_t *p) +{ + return atomic_load_consume(p); +} + +static void +store_atomic_pte(_Atomic gpt_pte_t *p, gpt_pte_t pte, bool init) +{ + if (init) { + atomic_init(p, pte); + } else { + atomic_store_release(p, pte); + } +} + +static gpt_pte_t +load_root_pte(gpt_root_t *root, gpt_config_t config) +{ + gpt_pte_t pte; + + if (gpt_config_get_rcu_read(&config)) { + pte = load_atomic_pte(&root->atomic); + } else { + pte = root->non_atomic; + } + + return pte; +} + +static gpt_pte_t +load_level_pte(gpt_config_t config, gpt_level_t level, index_t i) +{ + gpt_pte_t pte; + + if (gpt_config_get_rcu_read(&config)) { + pte = load_atomic_pte(&level.atomic->entries[i]); + } else { + pte = level.non_atomic->entries[i]; + } + + return pte; +} + +static void +store_root_pte(gpt_root_t *root, gpt_config_t config, gpt_pte_t pte, bool init) +{ + if (gpt_config_get_rcu_read(&config)) { + store_atomic_pte(&root->atomic, pte, init); + } else { + root->non_atomic = pte; + } +} + +static void +store_level_pte(gpt_config_t config, gpt_level_t level, index_t i, + gpt_pte_t pte, bool init) +{ + if (gpt_config_get_rcu_read(&config)) { + store_atomic_pte(&level.atomic->entries[i], pte, init); + } else { + level.non_atomic->entries[i] = pte; + } +} + +static bool +entry_is_valid(gpt_t *gpt, gpt_entry_t entry) +{ + return (entry.type <= GPT_TYPE__MAX) && + bitmap_isset(&gpt->allowed_types, (index_t)entry.type); +} + +static bool +entry_is_valid_or_empty(gpt_t *gpt, gpt_entry_t entry) +{ + return entry_is_valid(gpt, entry) || (entry.type == GPT_TYPE_EMPTY); +} + +static bool +pte_and_entry_equal(gpt_pte_t pte, size_t curr, gpt_entry_t entry) +{ + assert(guard_matching(pte, curr)); + + size_t pte_addr = get_pte_addr(pte); + gpt_type_t pte_type = gpt_pte_info_get_type(&pte.info); + gpt_value_t pte_value = pte.value; + + trigger_gpt_value_add_offset_event(pte_type, &pte_value, + curr - pte_addr); + + gpt_entry_t other = { + .type = pte_type, + .value = pte_value, + }; + + return entries_equal(entry, other); +} + +static bool +can_replace_pte(size_t curr, size_t rem, gpt_pte_t pte) +{ + size_t pte_addr = get_pte_addr(pte); + size_t pte_size = get_pte_size(pte); + + assert(!guard_matching(pte, curr)); + + // If the remaining region completely covers the range of the PTE, it is + // safe to replace the PTE. + return (curr <= pte_addr) && ((curr + rem) >= (pte_addr + pte_size)); +} + +static bool +pte_will_conflict(size_t curr, size_t rem, gpt_entry_t old, gpt_pte_t pte) +{ + bool ret = true; + + assert(!guard_matching(pte, curr)); + + if (old.type != GPT_TYPE_EMPTY) { + // We expected the GPT to not be empty at this point. + goto out; + } + + if (gpt_pte_info_get_type(&pte.info) == GPT_TYPE_LEVEL) { + // We can only be sure of a conflict with a level if its entire + // range is going to be replaced. Otherwise, we must traverse + // the level first to be sure of a conflict. + ret = can_replace_pte(curr, rem, pte); + goto out; + } + + size_t pte_addr = get_pte_addr(pte); + + // If the range to update overlaps with the PTE, we have a conflict. + ret = (pte_addr >= curr) && (pte_addr < (curr + rem)); + +out: + return ret; +} + +static count_t +get_common_shifts(gpt_pte_t pte, size_t addr) +{ + count_t clz = compiler_clz(addr ^ get_pte_addr(pte)); + + return (count_t)SIZE_T_BITS - util_balign_down(clz, GPT_LEVEL_BITS); +} + +static index_t +get_level_index(count_t shifts, size_t addr) +{ + assert(shifts >= GPT_LEVEL_BITS); + + return (index_t)((addr >> (shifts - GPT_LEVEL_BITS)) & + util_mask(GPT_LEVEL_BITS)); +} + +static gpt_stack_frame_t * +get_curr_stack_frame(gpt_stack_t *stack) +{ + assert(stack->depth != 0U); + + index_t i = stack->depth - 1U; + assert(i < GPT_MAX_LEVELS); + + return &stack->frame[i]; +} + +static count_t +get_max_entry_shifts(gpt_stack_t *stack) +{ + count_t shifts = GPT_MAX_SIZE_BITS; + + if (stack->depth != 0U) { + gpt_stack_frame_t *frame = get_curr_stack_frame(stack); + assert(frame != NULL); + shifts = gpt_frame_info_get_shifts(&frame->info); + } + + return shifts; +} + +static count_t +get_max_possible_shifts(gpt_stack_t *stack, size_t curr, size_t rem) +{ + count_t shifts = get_max_entry_shifts(stack); + + assert(rem > 0U); + + if (curr != 0U) { + count_t align_bits = + util_balign_down(compiler_ctz(curr), GPT_LEVEL_BITS); + shifts = util_min(shifts, align_bits); + } + + if (util_bit(shifts) > rem) { + shifts = util_balign_down(compiler_ctz(rem), GPT_LEVEL_BITS); + } + + return shifts; +} + +static gpt_level_t +get_level_from_pte(gpt_pte_t pte) +{ + gpt_level_t level = pte.value.level; + assert(level.raw != 0U); + + return level; +} + +static void +go_down_level(gpt_config_t config, gpt_stack_t *stack, size_t curr, + gpt_pte_t pte) +{ + gpt_level_t level = get_level_from_pte(pte); + + assert(guard_matching(pte, curr)); + + stack->depth++; + + gpt_stack_frame_t *frame = get_curr_stack_frame(stack); + assert(frame != NULL); + + count_t shifts = gpt_pte_info_get_shifts(&pte.info); + assert(shifts >= GPT_LEVEL_BITS); + + size_t addr = get_pte_addr(pte); + assert(addr < get_max_size(config)); + + gpt_frame_info_t info = gpt_frame_info_default(); + gpt_frame_info_set_addr(&info, addr); + gpt_frame_info_set_shifts(&info, shifts - GPT_LEVEL_BITS); + + frame->level = level; + frame->info = info; +} + +static bool +check_ptes_consistent(gpt_pte_t a, gpt_pte_t b, size_t offset) +{ + bool ret = false; + + gpt_type_t type = gpt_pte_info_get_type(&a.info); + if (type != gpt_pte_info_get_type(&b.info)) { + goto out; + } + + gpt_value_t x = a.value; + gpt_value_t y = b.value; + + trigger_gpt_value_add_offset_event(type, &x, offset); + + ret = trigger_gpt_values_equal_event(type, x, y); + +out: + return ret; +} + +static void +write_pte_to_level(gpt_root_t *root, gpt_config_t config, gpt_stack_t *stack, + gpt_pte_t pte) +{ + if (stack->depth == 0U) { + store_root_pte(root, config, pte, false); + } else { + gpt_stack_frame_t *frame = get_curr_stack_frame(stack); + assert(frame != NULL); + + index_t i = gpt_frame_info_get_index(&frame->info); + assert(i < GPT_LEVEL_ENTRIES); + + store_level_pte(config, frame->level, i, pte, false); + gpt_frame_info_set_dirty(&frame->info, true); + } +} + +rcu_update_status_t +gpt_handle_rcu_free_level(rcu_entry_t *entry) +{ + gpt_level_atomic_t *level = + gpt_level_atomic_container_of_rcu_entry(entry); + assert(level != NULL); + assert(level->partition != NULL); + + (void)partition_free(level->partition, level, sizeof(*level)); + + return rcu_update_status_default(); +} + +static void +free_level(gpt_config_t config, partition_t *partition, gpt_level_t level) +{ + if (gpt_config_get_rcu_read(&config)) { + rcu_enqueue(&level.atomic->rcu_entry, + RCU_UPDATE_CLASS_GPT_FREE_LEVEL); + } else { + (void)partition_free(partition, level.non_atomic, + sizeof(gpt_level_non_atomic_t)); + } +} + +static void +try_clean(gpt_root_t *root, gpt_config_t config, partition_t *partition, + gpt_stack_t *stack, gpt_level_t level, count_t entry_shifts) +{ + count_t filled_count = 0U; + gpt_pte_t first_pte = gpt_pte_empty(); + gpt_pte_t last_filled_pte = gpt_pte_empty(); + bool can_merge = true; + + assert(partition != NULL); + + for (index_t i = 0U; i < GPT_LEVEL_ENTRIES; i++) { + gpt_pte_t curr_pte = load_level_pte(config, level, i); + + if (gpt_pte_info_get_type(&curr_pte.info) == GPT_TYPE_EMPTY) { + can_merge = false; + } else { + filled_count++; + last_filled_pte = curr_pte; + // Merging entries is only possible if they fill up + // the entire level. + if (gpt_pte_info_get_shifts(&curr_pte.info) != + entry_shifts) { + can_merge = false; + } + } + + if (can_merge) { + if (i == 0U) { + first_pte = curr_pte; + } else { + size_t offset = (size_t)i << entry_shifts; + if (!check_ptes_consistent(first_pte, curr_pte, + offset)) { + can_merge = false; + } + } + } else if (filled_count > 1U) { + break; + } else { + // We may still be able to clean, continue iterating. + } + } + + if (filled_count <= 1U) { + // Either the level is empty, or the last filled + // PTE is the only one in the level. + write_pte_to_level(root, config, stack, last_filled_pte); + free_level(config, partition, level); + } else if (can_merge) { + // All entries consistent, we can merge into one PTE. + assert(filled_count == GPT_LEVEL_ENTRIES); + count_t new_shifts = entry_shifts + GPT_LEVEL_BITS; + size_t new_guard = get_pte_addr(first_pte) >> new_shifts; + gpt_pte_info_set_guard(&first_pte.info, new_guard); + gpt_pte_info_set_shifts(&first_pte.info, new_shifts); + write_pte_to_level(root, config, stack, first_pte); + free_level(config, partition, level); + } else { + // No cases where we can free the level, do nothing. + } +} + +static void +go_up_level(gpt_root_t *root, gpt_config_t config, partition_t *partition, + gpt_stack_t *stack, bool write) +{ + assert(stack->depth > 0U); + + gpt_stack_frame_t *frame = get_curr_stack_frame(stack); + assert(frame != NULL); + + stack->depth--; + + if (write && gpt_frame_info_get_dirty(&frame->info)) { + // FIXME: Do we need a better heuristic to determine if a + // clean is required? We could maintain a count of filled + // entries in each level, but do we want this additional + // memory consumption? + count_t shifts = gpt_frame_info_get_shifts(&frame->info); + try_clean(root, config, partition, stack, frame->level, shifts); + } else { + assert(!gpt_frame_info_get_dirty(&frame->info)); + } +} + +static gpt_pte_t +get_curr_pte(gpt_root_t *root, gpt_config_t config, partition_t *partition, + gpt_stack_t *stack, size_t curr, bool write) +{ + gpt_pte_t pte; + + while (stack->depth > 0U) { + gpt_stack_frame_t *frame = get_curr_stack_frame(stack); + assert(frame != NULL); + + count_t shifts = gpt_frame_info_get_shifts(&frame->info); + size_t addr = gpt_frame_info_get_addr(&frame->info); + assert(curr >= addr); + + index_t idx = (index_t)((curr - addr) >> shifts); + if (idx < GPT_LEVEL_ENTRIES) { + gpt_frame_info_set_index(&frame->info, idx); + pte = load_level_pte(config, frame->level, idx); + goto out; + } + + go_up_level(root, config, partition, stack, write); + } + + assert(stack->depth == 0U); + + pte = load_root_pte(root, config); + +out: + return pte; +} + +static void +update_curr_pte(gpt_root_t *root, gpt_config_t config, gpt_stack_t *stack, + size_t addr, count_t shifts, gpt_type_t type, gpt_value_t value) +{ + gpt_pte_t new_pte = gpt_pte_empty(); + + if (type != GPT_TYPE_EMPTY) { + gpt_pte_info_set_guard(&new_pte.info, addr >> shifts); + gpt_pte_info_set_shifts(&new_pte.info, shifts); + gpt_pte_info_set_type(&new_pte.info, type); + new_pte.value = value; + } + + write_pte_to_level(root, config, stack, new_pte); +} + +static void +split_pte_and_fill_level(gpt_config_t config, gpt_level_t level, + gpt_pte_t old_pte, count_t shifts) +{ + size_t pte_addr = get_pte_addr(old_pte); + size_t pte_size = util_bit(shifts); + gpt_type_t type = gpt_pte_info_get_type(&old_pte.info); + gpt_value_t value = old_pte.value; + + gpt_pte_t new_pte = old_pte; + gpt_pte_info_set_shifts(&new_pte.info, shifts); + + for (index_t i = 0U; i < GPT_LEVEL_ENTRIES; i++) { + gpt_pte_info_set_guard(&new_pte.info, pte_addr >> shifts); + new_pte.value = value; + + store_level_pte(config, level, i, new_pte, true); + + pte_addr += pte_size; + + trigger_gpt_value_add_offset_event(type, &value, pte_size); + } +} + +static error_t +allocate_level(gpt_root_t *root, gpt_config_t config, partition_t *partition, + gpt_stack_t *stack, gpt_pte_t old_pte, count_t new_shifts, + bool fill) +{ + error_t err = OK; + gpt_level_t level; + size_t alloc_size; + size_t alloc_align; + + if (gpt_config_get_rcu_read(&config)) { + alloc_size = sizeof(gpt_level_atomic_t); + alloc_align = alignof(gpt_level_atomic_t); + } else { + alloc_size = sizeof(gpt_level_non_atomic_t); + alloc_align = alignof(gpt_level_non_atomic_t); + } + + void_ptr_result_t alloc_ret = + partition_alloc(partition, alloc_size, alloc_align); + if (alloc_ret.e != OK) { + err = alloc_ret.e; + goto out; + } + + if (gpt_config_get_rcu_read(&config)) { + level.atomic = (gpt_level_atomic_t *)alloc_ret.r; + *level.atomic = (gpt_level_atomic_t){ .partition = partition }; + } else { + level.non_atomic = (gpt_level_non_atomic_t *)alloc_ret.r; + *level.non_atomic = (gpt_level_non_atomic_t){ 0 }; + } + + size_t addr = get_pte_addr(old_pte); + count_t old_shifts = gpt_pte_info_get_shifts(&old_pte.info); + gpt_value_t value = { .level = level }; + + if (fill) { + assert(old_shifts == new_shifts); + split_pte_and_fill_level(config, level, old_pte, + new_shifts - GPT_LEVEL_BITS); + } else { + assert(old_shifts < new_shifts); + index_t i = get_level_index(new_shifts, addr); + store_level_pte(config, level, i, old_pte, true); + } + + update_curr_pte(root, config, stack, addr, new_shifts, GPT_TYPE_LEVEL, + value); + +out: + return err; +} + +static void +free_all_levels(gpt_config_t config, partition_t *partition, gpt_pte_t pte) +{ + gpt_level_t levels[GPT_MAX_LEVELS] = { get_level_from_pte(pte) }; + index_t level_idx[GPT_MAX_LEVELS] = { 0 }; + + count_t depth = 1U; + while (depth > 0U) { + index_t i = depth - 1U; + assert(i < GPT_MAX_LEVELS); + + gpt_level_t level = levels[i]; + assert(level.raw != 0U); + + index_t j = level_idx[i]; + if (j == GPT_LEVEL_ENTRIES) { + free_level(config, partition, level); + levels[i].raw = 0U; + level_idx[i] = 0U; + depth--; + continue; + } + + gpt_pte_t curr_pte = load_level_pte(config, level, j); + if (gpt_pte_info_get_type(&curr_pte.info) == GPT_TYPE_LEVEL) { + assert(i < (GPT_MAX_LEVELS - 1U)); + levels[i + 1U] = get_level_from_pte(curr_pte); + depth++; + } + + level_idx[i]++; + } +} + +static size_t +update_curr_pte_and_get_size(gpt_root_t *root, gpt_config_t config, + gpt_stack_t *stack, size_t curr, size_t rem, + gpt_entry_t new) +{ + count_t shifts = get_max_possible_shifts(stack, curr, rem); + + update_curr_pte(root, config, stack, curr, shifts, new.type, new.value); + + return util_bit(shifts); +} + +static size_t +get_next_pte_base(gpt_stack_t *stack, size_t curr) +{ + count_t shifts = get_max_entry_shifts(stack); + + return util_p2align_down(curr, shifts) + util_bit(shifts); +} + +static size_result_t +handle_write(gpt_root_t *root, gpt_config_t config, partition_t *partition, + gpt_stack_t *stack, size_t curr, size_t rem, gpt_entry_t old, + gpt_entry_t new, bool match) +{ + size_result_t ret = size_result_ok(0U); + gpt_pte_t pte = + get_curr_pte(root, config, partition, stack, curr, true); + gpt_type_t type = gpt_pte_info_get_type(&pte.info); + + if (type == GPT_TYPE_EMPTY) { + // Empty PTEs don't have a valid guard, which is why we can't + // perform a guard check here. + if (match && (old.type != GPT_TYPE_EMPTY)) { + // We expected a non-empty PTE at this point. + ret.e = ERROR_BUSY; + } else if (new.type == GPT_TYPE_EMPTY) { + // No need to update an already empty entry, skip to the + // next entry. + ret.r = get_next_pte_base(stack, curr) - curr; + } else { + // We can safely update the PTE. + ret.r = update_curr_pte_and_get_size( + root, config, stack, curr, rem, new); + } + } else if (!guard_matching(pte, curr)) { + // The current address isn't mapped in the GPT. + if (!match && can_replace_pte(curr, rem, pte)) { + // It is safe to overwrite this PTE. + ret.r = update_curr_pte_and_get_size( + root, config, stack, curr, rem, new); + // If the old PTE was a level, we need to ensure it + // and all levels below it are freed. + if (type == GPT_TYPE_LEVEL) { + free_all_levels(config, partition, pte); + } + } else if (match && pte_will_conflict(curr, rem, old, pte)) { + // We either expected the GPT to be filled at the + // current offset, or to be empty at the PTE's offset. + ret.e = ERROR_BUSY; + } else { + // Allocate a new common level and retry. + count_t shifts = get_common_shifts(pte, curr); + ret.e = allocate_level(root, config, partition, stack, + pte, shifts, false); + } + } else if (type == GPT_TYPE_LEVEL) { + // Guard matches for this level, traverse down it. + go_down_level(config, stack, curr, pte); + } else if (match && !pte_and_entry_equal(pte, curr, old)) { + // We aren't matching the expected old value. + ret.e = ERROR_BUSY; + } else { + // The PTE has an old value, which we may not be fully + // replacing. Determine if we need to split the PTE into a new + // level or not. + count_t old_shifts = gpt_pte_info_get_shifts(&pte.info); + count_t new_shifts = get_max_possible_shifts(stack, curr, rem); + if (old_shifts > new_shifts) { + assert(old_shifts >= GPT_LEVEL_BITS); + // Split entry into a new level and retry. + ret.e = allocate_level(root, config, partition, stack, + pte, old_shifts, true); + } else if ((old_shifts < new_shifts) && match) { + // The old PTE doesn't cover the entire region that we + // want to update, which means there is a mismatch. + ret.e = ERROR_BUSY; + } else { + // Either the shifts are matching or we don't care about + // the old entry, we can safely update the PTE. + ret.r = update_curr_pte_and_get_size( + root, config, stack, curr, rem, new); + } + } + + return ret; +} + +static error_t +do_walk_callback(gpt_read_data_t *data) +{ + error_t err = OK; + + if (data->size > 0U) { + err = trigger_gpt_walk_callback_event(data->cb, data->entry, + data->base, data->size, + data->arg); + } + + return err; +} + +static void +log_range(size_t base, size_t size, gpt_entry_t entry) +{ + if ((entry.type != GPT_TYPE_EMPTY) && (size > 0U)) { + LOG(DEBUG, INFO, "[{:#x}, {:#x}]: type {:d}, value {:#x}", base, + size, entry.type, entry.value.raw); + } +} + +static size_result_t +handle_read(gpt_root_t *root, gpt_config_t config, gpt_stack_t *stack, + size_t curr, size_t rem, gpt_read_op_t op, gpt_read_data_t *data) +{ + size_result_t ret = size_result_ok(0U); + gpt_pte_t pte = get_curr_pte(root, config, NULL, stack, curr, false); + gpt_type_t type = gpt_pte_info_get_type(&pte.info); + gpt_value_t value = { .raw = 0U }; + + size_t pte_addr = get_pte_addr(pte); + size_t pte_size = get_pte_size(pte); + size_t end_addr; + + if (type == GPT_TYPE_EMPTY) { + // The empty range ends at the next PTE. + end_addr = get_next_pte_base(stack, curr); + } else if (!guard_matching(pte, curr)) { + // The current address isn't mapped in the GPT, so we treat it + // as empty. + type = GPT_TYPE_EMPTY; + if (curr < pte_addr) { + // The range ends at the start of the PTE. + end_addr = pte_addr; + } else { + // The range ends at the next PTE. + end_addr = get_next_pte_base(stack, curr); + } + } else if (type == GPT_TYPE_LEVEL) { + go_down_level(config, stack, curr, pte); + goto out; + } else { + end_addr = util_balign_down(curr + pte_size, pte_size); + value = pte.value; + + trigger_gpt_value_add_offset_event(type, &value, + curr - pte_addr); + } + + size_t size = util_min(end_addr - curr, rem); + + gpt_entry_t curr_entry = { + .type = type, + .value = value, + }; + + gpt_entry_t cmp_entry = data->entry; + + trigger_gpt_value_add_offset_event(cmp_entry.type, &cmp_entry.value, + data->size); + + bool equal = entries_equal(curr_entry, cmp_entry); + if (equal) { + data->size += size; + } else { + if (op == GPT_READ_OP_LOOKUP) { + if (data->base == curr) { + data->entry = curr_entry; + data->size = size; + } else { + ret.e = ERROR_FAILURE; + } + } else if (op == GPT_READ_OP_IS_CONTIGUOUS) { + ret.e = ERROR_FAILURE; + } else if (op == GPT_READ_OP_WALK) { + ret.e = do_walk_callback(data); + if (curr_entry.type == cmp_entry.type) { + data->base = curr; + data->size = size; + data->entry = curr_entry; + } else { + data->base = curr + size; + data->size = 0U; + } + } else if (op == GPT_READ_OP_DUMP_RANGE) { + log_range(data->base, data->size, data->entry); + data->entry = curr_entry; + data->base = curr; + data->size = size; + } else { + panic("gpt: Invalid read operation"); + } + } + + ret.r = size; + +out: + return ret; +} + +static size_result_t +gpt_do_write(gpt_t *gpt, size_t base, size_t size, gpt_entry_t old, + gpt_entry_t new, bool match) +{ + size_result_t ret = size_result_ok(0U); + gpt_root_t *root = &gpt->root; + gpt_config_t config = gpt->config; + partition_t *partition = gpt->partition; + + gpt_stack_t stack; + stack.depth = 0U; + + gpt_entry_t x = old; + gpt_entry_t y = new; + + size_t offset = 0U; + while ((ret.e == OK) && (offset < size)) { + ret = handle_write(root, config, partition, &stack, + base + offset, size - offset, x, y, match); + if ((ret.e == OK) && (ret.r != 0U)) { + offset += ret.r; + trigger_gpt_value_add_offset_event(x.type, &x.value, + ret.r); + trigger_gpt_value_add_offset_event(y.type, &y.value, + ret.r); + } + } + + ret.r = offset; + + // Unwind the GPT stack to finish any required cleanup. + while (stack.depth > 0U) { + go_up_level(root, config, partition, &stack, true); + } + + return ret; +} + +static error_t +gpt_write(gpt_t *gpt, size_t base, size_t size, gpt_entry_t old, + gpt_entry_t new, bool match) +{ + size_result_t ret; + + assert(gpt != NULL); + + if ((size == 0U) || util_add_overflows(base, size - 1U)) { + ret = size_result_error(ERROR_ARGUMENT_INVALID); + goto out; + } + + if ((base + size - 1U) > (get_max_size(gpt->config) - 1U)) { + ret = size_result_error(ERROR_ARGUMENT_SIZE); + goto out; + } + + assert(entry_is_valid_or_empty(gpt, old)); + assert(entry_is_valid_or_empty(gpt, new)); + + ret = gpt_do_write(gpt, base, size, old, new, match); + if ((ret.e != OK) && (ret.r != 0U)) { + size_result_t revert = + gpt_do_write(gpt, base, ret.r, new, old, true); + if (revert.e != OK) { + panic("gpt: Failed to revert write!"); + } + } + +out: + return ret.e; +} + +static gpt_entry_t +gpt_entry_empty(void) +{ + return (gpt_entry_t){ + .type = GPT_TYPE_EMPTY, + .value = { .raw = 0U }, + }; +} + +static error_t +gpt_read(gpt_t *gpt, size_t base, size_t size, gpt_read_op_t op, + gpt_read_data_t *data) +{ + size_result_t ret = size_result_ok(0U); + gpt_root_t *root = &gpt->root; + gpt_config_t config = gpt->config; + + if ((size == 0U) || util_add_overflows(base, size - 1U)) { + ret.e = ERROR_ARGUMENT_INVALID; + goto out; + } + + if ((base + size - 1U) > (get_max_size(gpt->config) - 1U)) { + ret.e = ERROR_ARGUMENT_SIZE; + goto out; + } + + assert(entry_is_valid_or_empty(gpt, data->entry)); + + gpt_stack_t stack; + stack.depth = 0U; + + size_t offset = 0U; + while ((ret.e == OK) && (offset < size)) { + ret = handle_read(root, config, &stack, base + offset, + size - offset, op, data); + offset += ret.r; + } + +out: + return ret.e; +} + +error_t +gpt_init(gpt_t *gpt, partition_t *partition, gpt_config_t config, + register_t allowed_types) +{ + error_t err = OK; + + if (gpt_config_get_max_bits(&config) > GPT_MAX_SIZE_BITS) { + err = ERROR_ARGUMENT_INVALID; + goto out; + } + + if (bitmap_isset(&allowed_types, (index_t)GPT_TYPE_EMPTY) || + bitmap_isset(&allowed_types, (index_t)GPT_TYPE_LEVEL) || + ((allowed_types & ~util_mask((index_t)GPT_TYPE__MAX + 1U)) != 0U)) { + err = ERROR_ARGUMENT_INVALID; + goto out; + } + + store_root_pte(&gpt->root, config, gpt_pte_empty(), true); + + gpt->partition = object_get_partition_additional(partition); + gpt->config = config; + gpt->allowed_types = allowed_types; + +out: + return err; +} + +void +gpt_destroy(gpt_t *gpt) +{ + gpt_clear_all(gpt); + object_put_partition(gpt->partition); +} + +error_t +gpt_insert(gpt_t *gpt, size_t base, size_t size, gpt_entry_t entry, + bool expect_empty) +{ + error_t err; + + if (entry_is_valid(gpt, entry)) { + err = gpt_write(gpt, base, size, gpt_entry_empty(), entry, + expect_empty); + } else { + err = ERROR_ARGUMENT_INVALID; + } + + return err; +} + +error_t +gpt_update(gpt_t *gpt, size_t base, size_t size, gpt_entry_t old_entry, + gpt_entry_t new_entry) +{ + error_t err; + + if (entry_is_valid(gpt, old_entry) && entry_is_valid(gpt, new_entry)) { + err = gpt_write(gpt, base, size, old_entry, new_entry, true); + } else { + err = ERROR_ARGUMENT_INVALID; + } + + return err; +} + +error_t +gpt_remove(gpt_t *gpt, size_t base, size_t size, gpt_entry_t entry) +{ + error_t err; + + if (entry_is_valid(gpt, entry)) { + err = gpt_write(gpt, base, size, entry, gpt_entry_empty(), + true); + } else { + err = ERROR_ARGUMENT_INVALID; + } + + return err; +} + +error_t +gpt_clear(gpt_t *gpt, size_t base, size_t size) +{ + return gpt_write(gpt, base, size, gpt_entry_empty(), gpt_entry_empty(), + false); +} + +void +gpt_clear_all(gpt_t *gpt) +{ + error_t err = gpt_clear(gpt, 0U, get_max_size(gpt->config)); + assert(err == OK); +} + +bool +gpt_is_empty(gpt_t *gpt) +{ + gpt_pte_t pte = load_root_pte(&gpt->root, gpt->config); + + return gpt_pte_info_get_type(&pte.info) == GPT_TYPE_EMPTY; +} + +gpt_lookup_result_t +gpt_lookup(gpt_t *gpt, size_t base, size_t max_size) +{ + gpt_read_data_t read = { .base = base }; + + error_t err = gpt_read(gpt, base, max_size, GPT_READ_OP_LOOKUP, &read); + assert((err == OK) || (err == ERROR_FAILURE)); + + return (gpt_lookup_result_t){ + .entry = read.entry, + .size = read.size, + }; +} + +bool +gpt_is_contiguous(gpt_t *gpt, size_t base, size_t size, gpt_entry_t entry) +{ + bool ret; + gpt_read_data_t read = { + .entry = entry, + }; + + if (entry_is_valid(gpt, entry)) { + ret = gpt_read(gpt, base, size, GPT_READ_OP_IS_CONTIGUOUS, + &read) == OK; + } else { + ret = false; + } + + return ret; +} + +#if defined(UNIT_TESTS) +error_t +gpt_walk(gpt_t *gpt, size_t base, size_t size, gpt_type_t type, + gpt_callback_t callback, gpt_arg_t arg) +{ + error_t err; + gpt_read_data_t read = { + .entry = { .type = type }, + .cb = callback, + .arg = arg, + }; + + if ((callback < GPT_CALLBACK__MIN) || (callback > GPT_CALLBACK__MAX) || + (callback == GPT_CALLBACK_RESERVED)) { + err = ERROR_ARGUMENT_INVALID; + goto out; + } + + if (entry_is_valid(gpt, read.entry)) { + err = gpt_read(gpt, base, size, GPT_READ_OP_WALK, &read); + if (err == OK) { + err = do_walk_callback(&read); + } + } else { + err = ERROR_ARGUMENT_INVALID; + } + +out: + return err; +} +#endif + +void +gpt_dump_ranges(gpt_t *gpt) +{ + gpt_read_data_t read = { .base = 0U, .size = 0U }; + + LOG(DEBUG, INFO, "Dumping ranges of GPT {:#x}", (register_t)gpt); + + error_t err = gpt_read(gpt, 0U, get_max_size(gpt->config), + GPT_READ_OP_DUMP_RANGE, &read); + assert(err == OK); + + log_range(read.base, read.size, read.entry); +} + +void +gpt_dump_levels(gpt_t *gpt) +{ + gpt_root_t *root = &gpt->root; + gpt_config_t config = gpt->config; + + LOG(DEBUG, INFO, "Dumping levels of GPT {:#x}", (register_t)gpt); + + gpt_stack_t stack; + stack.depth = 0U; + + size_t curr = 0U; + while (curr < get_max_size(config)) { + gpt_pte_t pte = + get_curr_pte(root, config, NULL, &stack, curr, false); + count_t entry_shifts = get_max_entry_shifts(&stack); + + if (!util_is_p2aligned(curr, entry_shifts)) { + // We have already logged this PTE, go to the next + // entry. + curr = util_p2align_up(curr, entry_shifts); + continue; + } + + size_t guard = gpt_pte_info_get_guard(&pte.info); + count_t shifts = gpt_pte_info_get_shifts(&pte.info); + gpt_type_t type = gpt_pte_info_get_type(&pte.info); + gpt_value_t value = pte.value; + + LOG(DEBUG, INFO, "{:d} {:#x} {:d} {:d} {:#x}", stack.depth, + guard, shifts, type, value.raw); + + if (type == GPT_TYPE_LEVEL) { + curr = get_pte_addr(pte); + go_down_level(config, &stack, curr, pte); + } else { + curr += util_bit(entry_shifts); + } + } +} diff --git a/hyp/misc/gpt/src/gpt_tests.c b/hyp/misc/gpt/src/gpt_tests.c new file mode 100644 index 0000000..bc92978 --- /dev/null +++ b/hyp/misc/gpt/src/gpt_tests.c @@ -0,0 +1,414 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#if defined(UNIT_TESTS) + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "event_handlers.h" + +static gpt_t gpt; + +static gpt_entry_t +test_entry_init(gpt_type_t type, uint64_t value) +{ + return (gpt_entry_t){ + .type = type, + .value = { .raw = value }, + }; +} + +void +gpt_tests_add_offset(gpt_type_t type, gpt_value_t *value, size_t offset) +{ + value->raw += (type != GPT_TYPE_TEST_C) ? offset : (offset * 2); +} + +bool +gpt_tests_values_equal(gpt_value_t x, gpt_value_t y) +{ + return x.raw == y.raw; +} + +error_t +gpt_tests_callback(gpt_entry_t entry, size_t base, size_t size, gpt_arg_t arg) +{ + LOG(DEBUG, INFO, + "GPT callback: t {:d}, v {:#x}, [{:#x}, {:#x}], arg {:#x}", + entry.type, entry.value.raw, base, size, arg.test); + + return OK; +} + +void +gpt_handle_tests_init(void) +{ + partition_t *partition = partition_get_root(); + assert(partition != NULL); + + gpt_config_t config = gpt_config_default(); + gpt_config_set_max_bits(&config, GPT_MAX_SIZE_BITS); + + register_t types = 0U; + bitmap_set(&types, GPT_TYPE_TEST_A); + bitmap_set(&types, GPT_TYPE_TEST_B); + bitmap_set(&types, GPT_TYPE_TEST_C); + + error_t err = gpt_init(&gpt, partition, config, types); + assert(err == OK); +} + +bool +gpt_handle_tests_start(void) +{ + error_t err; + + preempt_disable(); + + if (cpulocal_get_index() != 0U) { + goto out; + } + + assert(gpt_is_empty(&gpt)); + + size_t base, size; + gpt_entry_t e1, e2; + + base = 0x80000000U; + size = 0x70000; + e1 = test_entry_init(GPT_TYPE_TEST_A, base); + + err = gpt_insert(&gpt, base, size, e1, true); + assert(err == OK); + + base = 0x80001000U; + size = 0x4500; + e1 = test_entry_init(GPT_TYPE_TEST_A, base); + e2 = test_entry_init(GPT_TYPE_TEST_A, 0x900000); + + err = gpt_update(&gpt, base, size, e1, e2); + assert(err == OK); + + base = 0x80020010U; + size = 0x3; + e2 = test_entry_init(GPT_TYPE_TEST_B, base); + + err = gpt_insert(&gpt, base, size, e2, false); + assert(err == OK); + + base = 0x80040400U; + size = 3; + e1 = test_entry_init(GPT_TYPE_TEST_A, 0x80040400U); + e2 = test_entry_init(GPT_TYPE_TEST_C, 0x400); + + err = gpt_update(&gpt, base, size, e1, e2); + assert(err == OK); + + base = 0x80055555; + size = 1234; + e1 = test_entry_init(GPT_TYPE_TEST_A, 0x80055555); + + err = gpt_remove(&gpt, base, size, e1); + assert(err == OK); + + gpt_dump_ranges(&gpt); + + base = 0x80000050; + size = 0x20; + e1 = test_entry_init(GPT_TYPE_TEST_A, 0x80000050); + + assert(!gpt_is_empty(&gpt)); + + bool ret = gpt_is_contiguous(&gpt, base, size, e1); + assert(ret); + + gpt_lookup_result_t lookup = gpt_lookup(&gpt, 0x8, 1U); + LOG(DEBUG, INFO, "Lookup returned: {:d} {:#x} ({:#x})", + lookup.entry.type, lookup.entry.value.raw, lookup.size); + + lookup = gpt_lookup(&gpt, 0x80040001, 2); + LOG(DEBUG, INFO, "Lookup returned: {:d} {:#x} ({:#x})", + lookup.entry.type, lookup.entry.value.raw, lookup.size); + + lookup = gpt_lookup(&gpt, 0x80050006, 0x20000); + LOG(DEBUG, INFO, "Lookup returned: {:d} {:#x} ({:#x})", + lookup.entry.type, lookup.entry.value.raw, lookup.size); + + gpt_arg_t arg; + + base = 0x80000001; + size = 0x6f000; + arg.test = 0xfeed; + + err = gpt_walk(&gpt, base, size, GPT_TYPE_TEST_A, GPT_CALLBACK_TEST, + arg); + assert(err == OK); + + base = 0x80040200U; + size = 0x800; + arg.test = 0xbeef; + + err = gpt_walk(&gpt, base, size, GPT_TYPE_TEST_C, GPT_CALLBACK_TEST, + arg); + assert(err == OK); + + base = 0x100100U; + size = 0x1; + e1 = test_entry_init(GPT_TYPE_TEST_A, base); + + err = gpt_insert(&gpt, base, size, e1, false); + assert(err == OK); + + base = 0x100300U; + size = 0x1; + e1 = test_entry_init(GPT_TYPE_TEST_A, base); + + err = gpt_insert(&gpt, base, size, e1, false); + assert(err == OK); + + base = 0x100100U; + size = 0x1; + e1 = test_entry_init(GPT_TYPE_TEST_A, base); + + err = gpt_remove(&gpt, base, size, e1); + assert(err == OK); + + gpt_dump_ranges(&gpt); + + // Partially invalid update. + base = 0x80030000U; + size = 0x50000; + e1 = test_entry_init(GPT_TYPE_TEST_A, base); + e2 = test_entry_init(GPT_TYPE_TEST_B, base); + + err = gpt_update(&gpt, base, size, e1, e2); + assert(err != OK); + + size = 0x10; + + err = gpt_update(&gpt, base, size, e1, e2); + assert(err == OK); + + gpt_dump_ranges(&gpt); + + // Attempt to insert invalid type. + e1 = test_entry_init(GPT_TYPE_LEVEL, 0x213123123123); + + err = gpt_insert(&gpt, 0x919100123f23, 0x1012301230, e1, false); + assert(err != OK); + + base = 0x70000000U; + size = 0x20000000U; + e1 = test_entry_init(GPT_TYPE_TEST_B, 0x50000000U); + + err = gpt_insert(&gpt, base, size, e1, false); + assert(err == OK); + + gpt_dump_levels(&gpt); + + err = gpt_clear(&gpt, 0U, 0x100000000U); + assert(err == OK); + + assert(gpt_is_empty(&gpt)); + + base = 0U; + size = 1U; + e1 = test_entry_init(GPT_TYPE_TEST_A, base); + + err = gpt_insert(&gpt, base, size, e1, false); + assert(err == OK); + + for (index_t i = 0U; i < GPT_MAX_SIZE_BITS; i += GPT_LEVEL_BITS) { + base = util_bit(i); + size = 1U; + e1 = test_entry_init(GPT_TYPE_TEST_A, base); + + err = gpt_insert(&gpt, base, size, e1, false); + assert(err == OK); + } + + gpt_dump_ranges(&gpt); + + for (index_t i = 0U; i < GPT_MAX_SIZE_BITS; i += GPT_LEVEL_BITS) { + base = util_bit(i); + size = 1U; + e1 = test_entry_init(GPT_TYPE_TEST_A, base); + + err = gpt_remove(&gpt, base, size, e1); + assert(err == OK); + } + + gpt_clear_all(&gpt); + + for (index_t i = 0U; i < GPT_LEVEL_ENTRIES; i++) { + base = 0xffff00000000 + (i << 8); + size = 1U << 8; + e1 = test_entry_init(GPT_TYPE_TEST_A, base); + + err = gpt_insert(&gpt, base, size, e1, true); + assert(err == OK); + } + + gpt_dump_levels(&gpt); + + e1 = test_entry_init(GPT_TYPE_TEST_A, 1U); + + err = gpt_insert(&gpt, 0x1000, 1U, e1, false); + assert(err == OK); + + err = gpt_insert(&gpt, 0x1002, 1U, e1, false); + assert(err == OK); + + err = gpt_insert(&gpt, 0x1002, 1U, e1, false); + assert(err == OK); + + e2 = test_entry_init(GPT_TYPE_TEST_B, 1U); + + err = gpt_insert(&gpt, 0x1010, 1U, e2, false); + assert(err == OK); + + gpt_dump_levels(&gpt); + + e1 = test_entry_init(GPT_TYPE_TEST_C, 2U); + + err = gpt_insert(&gpt, 0U, 0x2000, e1, true); + assert(err != OK); + + gpt_clear(&gpt, 1U, 0x10000U); + + gpt_dump_levels(&gpt); + + gpt_clear_all(&gpt); + + e1 = test_entry_init(GPT_TYPE_TEST_A, 0xdeadbeefbeadfeedU); + + // Base and size cause overflow + err = gpt_insert(&gpt, 0xffffffffffffff00, 0x0, e1, true); + assert(err == ERROR_ARGUMENT_INVALID); + + err = gpt_insert(&gpt, 0xffffffffffffff00, 0x100, e1, true); + assert(err == ERROR_ARGUMENT_SIZE); + + err = gpt_insert(&gpt, 0xffffffffffffffff, 0x33333333, e1, true); + assert(err == ERROR_ARGUMENT_INVALID); + + // Larger than max size supported + err = gpt_insert(&gpt, util_bit(GPT_MAX_SIZE_BITS), 0x1, e1, true); + assert(err == ERROR_ARGUMENT_SIZE); + + err = gpt_insert(&gpt, util_bit(GPT_MAX_SIZE_BITS + 1), 0x33333333, e1, + true); + assert(err == ERROR_ARGUMENT_SIZE); + + err = gpt_insert(&gpt, util_bit(GPT_MAX_SIZE_BITS) - 1, 0x1, e1, true); + assert(err == OK); + + err = gpt_insert(&gpt, util_bit(GPT_MAX_SIZE_BITS) - 1, 0x1, e1, true); + assert(err == ERROR_BUSY); + + err = gpt_clear(&gpt, util_bit(GPT_MAX_SIZE_BITS) - 1, 0x1U); + assert(err == OK); + + err = gpt_insert(&gpt, util_bit(GPT_MAX_SIZE_BITS) - 2, 0x2, e1, true); + assert(err == OK); + + err = gpt_insert(&gpt, util_bit(GPT_MAX_SIZE_BITS) - 1, 0x1, e1, true); + assert(err == ERROR_BUSY); + + err = gpt_walk(&gpt, 0, util_bit(GPT_MAX_SIZE_BITS), GPT_TYPE_TEST_A, + GPT_CALLBACK_TEST, arg); + assert(err == OK); + + err = gpt_clear(&gpt, util_bit(GPT_MAX_SIZE_BITS) - 1, 0x1U); + assert(err == OK); + + err = gpt_walk(&gpt, 0, util_bit(GPT_MAX_SIZE_BITS), GPT_TYPE_TEST_A, + GPT_CALLBACK_TEST, arg); + assert(err == OK); + + err = gpt_insert(&gpt, util_bit(GPT_MAX_SIZE_BITS) - 2, 0x1, e1, true); + assert(err == ERROR_BUSY); + + e2 = e1; + e2.value.raw += 1; + err = gpt_insert(&gpt, util_bit(GPT_MAX_SIZE_BITS) - 1, 0x1, e2, true); + assert(err == OK); + + err = gpt_walk(&gpt, 0, util_bit(GPT_MAX_SIZE_BITS), GPT_TYPE_TEST_A, + GPT_CALLBACK_TEST, arg); + assert(err == OK); + + err = gpt_insert(&gpt, 0x4000000000000, 0x33333333, e1, true); + assert(err == OK); + + e1 = test_entry_init(GPT_TYPE_TEST_A, 0x3333444455550000U); + + err = gpt_insert(&gpt, 0x234000000000, 0x679823213, e1, true); + assert(err == OK); + + // Attempt to insert range with overlap at end + err = gpt_insert(&gpt, 0x233cdf123000, 0x82681e3f2, e1, true); + assert(err != OK); + + e1 = test_entry_init(GPT_TYPE_TEST_A, 0x3333444455556666U); + e2 = test_entry_init(GPT_TYPE_TEST_B, 0x777788889999); + + // Attempt to update range that doesn't match at the end + err = gpt_update(&gpt, 0x234000006666, 0x73f8c532ab, e1, e2); + assert(err != OK); + + err = gpt_update(&gpt, 0x234000006666, 0x123e34, e1, e2); + assert(err == OK); + + gpt_dump_ranges(&gpt); + + base = 0x7ab2348e293; + size = 0x123809193; + + e1 = test_entry_init(GPT_TYPE_TEST_A, 0x183741ea175); + + err = gpt_insert(&gpt, 0U, base, e1, false); + assert(err == OK); + + e1.value.raw += base + size; + + err = gpt_insert(&gpt, base + size, GPT_MAX_SIZE - base - size, e1, + false); + assert(err == OK); + + e1.value.raw -= size; + + err = gpt_insert(&gpt, base, size, e1, false); + assert(err == OK); + + e1.value.raw -= base; + + // GPT should now have a single contiguous entry + assert(gpt_is_contiguous(&gpt, 0U, GPT_MAX_SIZE, e1)); + + gpt_dump_ranges(&gpt); + + gpt_destroy(&gpt); +out: + preempt_enable(); + + return false; +} + +#else + +extern char unused; + +#endif diff --git a/hyp/misc/gpt/tests/Makefile b/hyp/misc/gpt/tests/Makefile new file mode 100644 index 0000000..32afa66 --- /dev/null +++ b/hyp/misc/gpt/tests/Makefile @@ -0,0 +1,59 @@ +# © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +CC=clang + +INCLUDE=-I../../../interfaces/gpt/include +INCLUDE+=-I../../../../build/qemu/unittests-qemu/debug/include +INCLUDE+=-I../../../../build/qemu/unittests-qemu/debug/events/include +INCLUDE+=-I../../../../build/qemu/unittests-qemu/debug/events/gpt/include +INCLUDE+=-I../../../../build/qemu/unittests-qemu/debug/objects/include +INCLUDE+=-I../../../interfaces/cpulocal/include +INCLUDE+=-I../../../interfaces/log/include +INCLUDE+=-I../../../interfaces/object/include +INCLUDE+=-I../../../interfaces/partition/include +INCLUDE+=-I../../../interfaces/preempt/include +INCLUDE+=-I../../../interfaces/rcu/include +INCLUDE+=-I../../../interfaces/trace/include +INCLUDE+=-I../../../interfaces/util/include +INCLUDE+=-I../../../misc/log_standard/include +INCLUDE+=-I../../../arch/armv8/include +INCLUDE+=-imacros ../../../interfaces/util/include/attributes.h +INCLUDE+=-imacros ../../../core/spinlock_ticket/include/spinlock_attrs.h +INCLUDE+=-imacros ../../../core/preempt/include/preempt_attrs.h +INCLUDE+=-imacros ../../../interfaces/rcu/include/rcu_attrs.h + +DEF=-DHYP_STANDALONE_TEST +DEF+=-DUNIT_TESTS + +CFLAGS+=-m64 -mcx16 -std=gnu18 -O1 -g +CFLAGS+=-Wall -Werror -Wno-gcc-compat -Wno-gnu-alignof-expression +CFLAGS+=-ffunction-sections -fdata-sections +CFLAGS+=-fsanitize=address -fno-omit-frame-pointer +#CFLAGS+=--coverage +CFLAGS+=$(INCLUDE) +CFLAGS+=$(DEF) + +LDFLAGS+=-Wl,--gc-sections + +SRC=../src/gpt.c ../src/gpt_tests.c host_tests.c +SRC+=../../../core/util/src/bitmap.c +SRC+=../../../misc/log_standard/src/string_util.c +SRC+=../../../../build/qemu/unittests-qemu/debug/hyp/core/base/accessors.c +SRC+=../../../../build/qemu/unittests-qemu/debug/hyp/core/base/hypresult.c + +OBJ=$(patsubst %.c,%.o,$(SRC)) + +TARGET=gpt_tests + +%.o: %.c + $(CC) $(CFLAGS) -c $< -o $@ + +$(TARGET): $(OBJ) + $(CC) $(CFLAGS) $(LDFLAGS) $(OBJ) -o $@ + +default: $(TARGET) + +clean: + rm -f $(TARGET) $(OBJ) diff --git a/hyp/misc/gpt/tests/host_tests.c b/hyp/misc/gpt/tests/host_tests.c new file mode 100644 index 0000000..5d48124 --- /dev/null +++ b/hyp/misc/gpt/tests/host_tests.c @@ -0,0 +1,209 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include + +#define timer_t hyp_timer_t +#include +#undef timer_t + +#define register_t std_register_t +#include +#include +#undef register_t + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "event_handlers.h" +#include "string_util.h" +#include "trace_helpers.h" + +gpt_t gpt; +partition_t host_partition; +trace_control_t hyp_trace; + +void +assert_failed(const char *file, int line, const char *func, const char *err) +{ + printf("Assert failed in %s at %s:%d: %s\n", func, file, line, err); + exit(-1); +} + +void +panic(const char *str) +{ + printf("Panic: %s\n", str); + exit(-1); +} + +static void +trace_and_log_init(void) +{ + register_t flags = 0U; + + TRACE_SET_CLASS(flags, ERROR); + TRACE_SET_CLASS(flags, TRACE_BUFFER); + TRACE_SET_CLASS(flags, DEBUG); + + atomic_init(&hyp_trace.enabled_class_flags, flags); +} + +void +trigger_trace_log_event(trace_id_t id, trace_action_t action, const char *arg0, + register_t arg1, register_t arg2, register_t arg3, + register_t arg4, register_t arg5) +{ + char log[1024]; + (void)snprint(log, util_array_size(log), arg0, arg1, arg2, arg3, arg4, + arg5); + puts(log); +} + +partition_t * +object_get_partition_additional(partition_t *partition) +{ + assert(partition != NULL); + + return partition; +} + +void +object_put_partition(partition_t *partition) +{ + assert(partition != NULL); +} + +partition_t * +partition_get_root(void) +{ + return &host_partition; +} + +void_ptr_result_t +partition_alloc(partition_t *partition, size_t bytes, size_t min_alignment) +{ + assert(partition != NULL); + assert(bytes > 0U); + + void *mem = aligned_alloc(min_alignment, bytes); + + return (mem != NULL) ? void_ptr_result_ok(mem) + : void_ptr_result_error(ERROR_NOMEM); +} + +error_t +partition_free(partition_t *partition, void *mem, size_t bytes) +{ + assert(partition != NULL); + assert(bytes > 0U); + + free(mem); + + return OK; +} + +void +preempt_disable(void) +{ +} + +void +preempt_enable(void) +{ +} + +void +rcu_read_start(void) +{ +} + +void +rcu_read_finish(void) +{ +} + +void +rcu_enqueue(rcu_entry_t *rcu_entry, rcu_update_class_t rcu_update_class) +{ + assert(rcu_update_class == RCU_UPDATE_CLASS_GPT_FREE_LEVEL); + + (void)gpt_handle_rcu_free_level(rcu_entry); +} + +cpu_index_t +cpulocal_check_index(cpu_index_t cpu) +{ + return cpu; +} + +cpu_index_t +cpulocal_get_index_unsafe(void) +{ + return 0U; +} + +void +trigger_gpt_value_add_offset_event(gpt_type_t type, gpt_value_t *value, + size_t offset) +{ + if ((type == GPT_TYPE_TEST_A) || (type == GPT_TYPE_TEST_B) || + (type == GPT_TYPE_TEST_C)) { + gpt_tests_add_offset(type, value, offset); + } else { + // Nothing to do + } +} + +bool +trigger_gpt_values_equal_event(gpt_type_t type, gpt_value_t x, gpt_value_t y) +{ + bool ret; + + if ((type == GPT_TYPE_TEST_A) || (type == GPT_TYPE_TEST_B) || + (type == GPT_TYPE_TEST_C)) { + ret = gpt_tests_values_equal(x, y); + } else if (GPT_TYPE_EMPTY) { + ret = gpt_handle_empty_values_equal(); + } else { + ret = false; + } + + return ret; +} + +error_t +trigger_gpt_walk_callback_event(gpt_callback_t callback, gpt_entry_t entry, + size_t base, size_t size, gpt_arg_t arg) +{ + error_t ret; + + if (callback == GPT_CALLBACK_RESERVED) { + gpt_handle_reserved_callback(); + } else if (callback == GPT_CALLBACK_TEST) { + ret = gpt_tests_callback(entry, base, size, arg); + } else { + ret = ERROR_ARGUMENT_INVALID; + } + + return ret; +} + +int +main(void) +{ + trace_and_log_init(); + + gpt_handle_tests_init(); + + gpt_handle_tests_start(); + + return 0; +} diff --git a/hyp/misc/log_standard/log.tc b/hyp/misc/log_standard/log.tc index 31cf8de..1c77661 100644 --- a/hyp/misc/log_standard/log.tc +++ b/hyp/misc/log_standard/log.tc @@ -15,8 +15,8 @@ define log_control object { }; extend trace_class enumeration { - LOG_BUFFER; - LOG_TRACE_BUFFER; // Put the trace messages in the log buffer + LOG_BUFFER = 5; + LOG_TRACE_BUFFER = 6; // Put the trace messages in the log buffer }; extend trace_action enumeration { diff --git a/hyp/misc/log_standard/src/log.c b/hyp/misc/log_standard/src/log.c index 298c756..daedb83 100644 --- a/hyp/misc/log_standard/src/log.c +++ b/hyp/misc/log_standard/src/log.c @@ -8,8 +8,11 @@ #include #include +#include #include +#include +#include #include #include @@ -17,8 +20,7 @@ #include #include - -#include +#include #include "event_handlers.h" #include "string_util.h" @@ -26,11 +28,15 @@ // FIXME: the log temporary buffer is placed on the stack // Note: thread_stack_size_default = 4096 -#define LOG_TEMP_BUFFER_SIZE 256 +#define LOG_TIMESTAMP_BUFFER_SIZE 24U +#define LOG_ENTRY_BUFFER_SIZE 256U extern char hyp_log_buffer[]; char hyp_log_buffer[LOG_BUFFER_SIZE]; +static_assert(LOG_BUFFER_SIZE > LOG_ENTRY_BUFFER_SIZE, + "LOG_BUFFER_SIZE too small"); + // Global visibility - for debug extern log_control_t hyp_log; @@ -40,11 +46,13 @@ log_control_t hyp_log = { .log_magic = LOG_MAGIC, .log_buffer = hyp_log_buffer }; void -log_init() +log_init(void) { register_t flags = 0U; TRACE_SET_CLASS(flags, LOG_BUFFER); trace_set_class_flags(flags); + + assert_debug(hyp_log.buffer_size == (index_t)LOG_BUFFER_SIZE); } void @@ -52,10 +60,10 @@ log_standard_handle_trace_log(trace_id_t id, trace_action_t action, const char *fmt, register_t arg0, register_t arg1, register_t arg2, register_t arg3, register_t arg4) { - index_t idx, prev_idx; + index_t next_idx, prev_idx, orig_idx; size_result_t ret; - size_t size; - char temp_buf[LOG_TEMP_BUFFER_SIZE]; + size_t entry_size, timestamp_size; + char entry_buf[LOG_ENTRY_BUFFER_SIZE]; // Add the data to the log buffer only if: // - The requested action is logging, and logging is enabled, or @@ -75,53 +83,98 @@ log_standard_handle_trace_log(trace_id_t id, trace_action_t action, goto out; } - ret = snprint(temp_buf, LOG_TEMP_BUFFER_SIZE, fmt, arg0, arg1, arg2, - arg3, arg4); + // Add the time-stamp and core number + // We could use platform_convert_ticks_to_ns() here, but the resulting + // values will be too big and unwieldy to use. Since log formatting is a + // slow process anyway, might as well add some nice time-stamps. + ticks_t now = platform_timer_get_current_ticks(); + nanoseconds_t ns = platform_convert_ticks_to_ns(now); + + microseconds_t usec = ns / (microseconds_t)1000; + uint64_t sec = usec / TIMER_MICROSECS_IN_SECOND; + + ret = snprint(entry_buf, LOG_TIMESTAMP_BUFFER_SIZE, + "{:d} {:4d}.{:06d} ", cpulocal_get_index_unsafe(), sec, + usec % TIMER_MICROSECS_IN_SECOND, 0, 0); + if (ret.e == ERROR_STRING_TRUNCATED) { + // The truncated string will have a NULL terminator, remove it + timestamp_size = LOG_TIMESTAMP_BUFFER_SIZE - 1U; + } else if (ret.e != OK) { + // Should not happen + goto out; + } else { + // Do not count the NULL character since we will write over it + // shortly. + timestamp_size = ret.r; + } + + // Add the log message after the time-stamp + ret = snprint(entry_buf + timestamp_size, + LOG_ENTRY_BUFFER_SIZE - timestamp_size, fmt, arg0, arg1, + arg2, arg3, arg4); if (ret.e == ERROR_STRING_TRUNCATED) { - size = LOG_TEMP_BUFFER_SIZE; + entry_size = LOG_ENTRY_BUFFER_SIZE; } else if ((ret.e == ERROR_STRING_MISSING_ARGUMENT) || (ret.r == 0U)) { goto out; } else { - size = ret.r + 1; + entry_size = timestamp_size + ret.r + 1U; + assert(entry_size <= LOG_ENTRY_BUFFER_SIZE); } - trigger_log_message_event(id, temp_buf); + const index_t buffer_size = (index_t)LOG_BUFFER_SIZE; - /* Atomically update the index first */ - prev_idx = atomic_fetch_add_explicit(&hyp_log.head, size, + // Inform the subscribers of the entry without the time-stamp + trigger_log_message_event(id, entry_buf + timestamp_size); + + // Atomically update the index first + orig_idx = atomic_fetch_add_explicit(&hyp_log.head, entry_size, memory_order_relaxed); - idx = prev_idx + (index_t)size; - while (compiler_unexpected(idx >= hyp_log.buffer_size)) { - index_t old_idx = idx; + prev_idx = orig_idx; + while (compiler_unexpected(prev_idx >= buffer_size)) { + prev_idx -= buffer_size; + } - idx -= hyp_log.buffer_size; + next_idx = orig_idx + (index_t)entry_size; + // If we wrap, something is really wrong + assert(next_idx > orig_idx); - (void)atomic_compare_exchange_strong_explicit( - &hyp_log.head, &old_idx, idx, memory_order_relaxed, - memory_order_relaxed); + if (compiler_unexpected(next_idx >= buffer_size)) { + index_t old_idx = next_idx; - if (compiler_unexpected(prev_idx >= hyp_log.buffer_size)) { - prev_idx -= hyp_log.buffer_size; + while (next_idx >= buffer_size) { + next_idx -= buffer_size; } + + // Try to reduce the index in the shared variable so it is no + // longer overflowed. We don't care if it fails because that + // means somebody else has done it concurrently. + (void)atomic_compare_exchange_strong_explicit( + &hyp_log.head, &old_idx, next_idx, memory_order_relaxed, + memory_order_relaxed); } prefetch_store_stream(&hyp_log.log_buffer[prev_idx]); - /* check for log wrap around, and split the string if required */ - if (compiler_expected((prev_idx < idx))) { - memcpy(&hyp_log.log_buffer[prev_idx], temp_buf, size); - CACHE_CLEAN_RANGE(&hyp_log.log_buffer[prev_idx], size); + size_t buf_remaining = (size_t)buffer_size - (size_t)prev_idx; + + // Copy the whole entry if it fits + if (compiler_expected(buf_remaining >= entry_size)) { + (void)memcpy(&hyp_log.log_buffer[prev_idx], entry_buf, + entry_size); + CACHE_CLEAN_RANGE(&hyp_log.log_buffer[prev_idx], entry_size); } else { - size_t first_part = hyp_log.buffer_size - (size_t)prev_idx; - memcpy(&hyp_log.log_buffer[prev_idx], temp_buf, first_part); + // Otherwise copy the first bit of entry to the tail of the + // buffer and wrap to the start for the remainder. + size_t first_part = buf_remaining; + (void)memcpy(&hyp_log.log_buffer[prev_idx], entry_buf, + first_part); CACHE_CLEAN_RANGE(&hyp_log.log_buffer[prev_idx], first_part); - size_t remaining = size - first_part; - if (remaining > 0) { - memcpy(&hyp_log.log_buffer[0], temp_buf + first_part, - remaining); - CACHE_CLEAN_RANGE(&hyp_log.log_buffer[0], remaining); - } + size_t second_part = entry_size - first_part; + + (void)memcpy(&hyp_log.log_buffer[0], entry_buf + first_part, + second_part); + CACHE_CLEAN_RANGE(&hyp_log.log_buffer[0], second_part); } out: // Nothing to do diff --git a/hyp/misc/log_standard/src/string_util.c b/hyp/misc/log_standard/src/string_util.c index 3cdb6f6..7e98d46 100644 --- a/hyp/misc/log_standard/src/string_util.c +++ b/hyp/misc/log_standard/src/string_util.c @@ -14,8 +14,8 @@ // TODO: Add more failure test cases -#define MAX_TERMINATOR 16 -#define MAX_ARG_CNT 5 +#define MAX_TERMINATOR 16U +#define MAX_ARG_CNT 5U // The string formatter processing states, the order of these needs to be // preserved. @@ -46,7 +46,7 @@ typedef enum align { ALIGN_CENTER } align_t; -typedef enum sign { +typedef enum sign_e { // default SIGN_NEG = 0, SIGN_BOTH, @@ -143,7 +143,7 @@ atodec(const char *c, size_t len) while (len != 0U) { v *= 10U; // TODO: Check if *c is in '0' and '9' for debug - v += *c - '0'; + v += (uint64_t)(*c) - (uint64_t)('0'); c++; len--; } @@ -173,7 +173,7 @@ itoa(char *buf, size_t *size, uint64_t val, uint8_t base, fmt_info_t *info, { const char digit[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; - char *pos = buf, padding_char = ' ', *tail = NULL; + char *pos = buf, padding_char = ' ', *tail = NULL; size_t padding_cnt = 0U, content_cnt = 0U, padding_ret = 0U; size_t padding_left_cnt = 0U, padding_right_cnt = 0U; size_t padding_after_sign = 0U, padding_after_prefix = 0U; @@ -202,14 +202,14 @@ itoa(char *buf, size_t *size, uint64_t val, uint8_t base, fmt_info_t *info, padding_cnt = util_max(info->min_width, content_cnt) - content_cnt; - if ((padding_cnt > 0U) && info->alternate_form && (base != 10)) { + if ((padding_cnt > 0U) && info->alternate_form && (base != 10U)) { // Remove two chars of prefix - padding_cnt -= 2; + padding_cnt -= 2U; } if ((padding_cnt > 0U) && ((info->sign == SIGN_BOTH) || (info->sign == SIGN_POS_LEADING) || ((info->sign == SIGN_NEG) && !positive))) { - padding_cnt -= 1; + padding_cnt -= 1U; } // Padding left with white space by default padding_left_cnt = padding_cnt; @@ -248,8 +248,10 @@ itoa(char *buf, size_t *size, uint64_t val, uint8_t base, fmt_info_t *info, } else if (info->alignment == ALIGN_CENTER) { padding_after_prefix = 0U; padding_after_sign = 0U; - padding_left_cnt = padding_cnt / 2; + padding_left_cnt = padding_cnt / 2U; padding_right_cnt = padding_cnt - padding_left_cnt; + } else { + // Nothing to do } padding_ret = insert_padding(pos, remaining, padding_char, @@ -279,6 +281,7 @@ itoa(char *buf, size_t *size, uint64_t val, uint8_t base, fmt_info_t *info, remaining--; break; default: + // Unusual base. Nothing to do break; } @@ -296,6 +299,7 @@ itoa(char *buf, size_t *size, uint64_t val, uint8_t base, fmt_info_t *info, remaining--; break; default: + // Unusual base. Nothing to do break; } @@ -390,7 +394,7 @@ sitoa(char *buf, size_t *size, int64_t val, uint8_t base, fmt_info_t *info) { bool positive = true; - if (val < 0U) { + if (val < 0) { positive = false; val = -val; } else { @@ -434,13 +438,15 @@ stringtoa(char *buf, size_t *size, char *val_str, fmt_info_t *info) padding_left_cnt = 0U; padding_right_cnt = padding_cnt; } else if (info->alignment == ALIGN_CENTER) { - padding_left_cnt = padding_cnt / 2; + padding_left_cnt = padding_cnt / 2U; padding_right_cnt = padding_cnt - padding_left_cnt; + } else { + // Nothing to do. } p = util_min(padding_left_cnt, remaining); if (p > 0U) { - memset(pos, padding_char, p); + (void)insert_padding(pos, p, padding_char, p); remaining -= p; pos += p; } @@ -451,7 +457,7 @@ stringtoa(char *buf, size_t *size, char *val_str, fmt_info_t *info) p = util_min(slen, remaining); if (p > 0U) { - memcpy(pos, val_str, p); + (void)memcpy(pos, val_str, p); remaining -= p; pos += p; } @@ -462,7 +468,7 @@ stringtoa(char *buf, size_t *size, char *val_str, fmt_info_t *info) p = util_min(padding_right_cnt, remaining); if (p > 0U) { - memset(pos, padding_char, p); + (void)insert_padding(pos, p, padding_char, p); remaining -= p; } if ((remaining + p) < padding_right_cnt) { @@ -597,14 +603,16 @@ check_minwidth(const char *fmt, fmt_info_t *info) if (in_range(fmt[1], '0', '9')) { ret = RET_TOKEN_NEXT_CHAR; } else { - info->min_width = atodec( - info->minwidth_start, - (size_t)(fmt - info->minwidth_start + 1)); + ptrdiff_t len = fmt - info->minwidth_start; + assert(len >= 0); + info->min_width = atodec(info->minwidth_start, + ((size_t)len + 1U)); info->minwidth_start = NULL; ret = RET_TOKEN_FOUND; } } else { - ret = RET_TOKEN_NEXT_STAGE; + ret = RET_TOKEN_NEXT_STAGE; + info->minwidth_start = NULL; } return ret; @@ -613,7 +621,7 @@ check_minwidth(const char *fmt, fmt_info_t *info) static inline ret_token_t check_precise(const char *fmt, fmt_info_t *info) { - ret_token_t ret = false; + ret_token_t ret; if ((*fmt == '.') && in_range(fmt[1], '0', '9')) { info->precise_start = NULL; @@ -629,14 +637,16 @@ check_precise(const char *fmt, fmt_info_t *info) if (in_range(fmt[1], '0', '9')) { ret = RET_TOKEN_NEXT_CHAR; } else { + ptrdiff_t len = fmt - info->precise_start; + assert(len >= 0); info->precise = - atodec(info->precise_start, - (size_t)(fmt - info->precise_start + 1)); + atodec(info->precise_start, ((size_t)len + 1U)); info->precise_start = NULL; ret = RET_TOKEN_FOUND; } } else { - ret = RET_TOKEN_NEXT_STAGE; + ret = RET_TOKEN_NEXT_STAGE; + info->precise_start = NULL; } return ret; @@ -715,6 +725,7 @@ check_token(count_t stage, const char *fmt, fmt_info_t *info) ret = check_end(fmt, info); break; default: + // Nothing to do break; } @@ -747,7 +758,7 @@ get_next_fmt(const char *fmt, fmt_info_t *info, size_t *consumed_len, break; case RET_TOKEN_STOP: - *consumed_len = idx + 1; + *consumed_len = (size_t)idx + 1U; return OK; case RET_TOKEN_FOUND: @@ -769,7 +780,7 @@ get_next_fmt(const char *fmt, fmt_info_t *info, size_t *consumed_len, // Nothing found if (stage == STAGE_START) { *literal_len = idx; - *consumed_len = idx + 1; + *consumed_len = (size_t)idx + 1U; } *end = true; } @@ -819,9 +830,9 @@ snprint(char *str, size_t size, const char *format, register_t arg0, { const char *fmt = format; // Current buffer pointer, increasing while filling strings into - char *buf = str; + char *buf = str; error_t ret = OK; - size_t remaining = size - 1; // space for terminating null + size_t remaining = size - 1U; // space for terminating null register_t args[MAX_ARG_CNT] = { arg0, arg1, arg2, arg3, arg4 }; index_t arg_idx = 0U; bool end = false; @@ -843,7 +854,7 @@ snprint(char *str, size_t size, const char *format, register_t arg0, // Copy literal characters to output buffer p = util_min(literal_len, remaining); if (p > 0U) { - memcpy(buf, fmt, p); + (void)memcpy(buf, fmt, p); } fmt += consumed_len; @@ -884,8 +895,12 @@ snprint(char *str, size_t size, const char *format, register_t arg0, // Add the terminator *buf = '\0'; - if (ret != ERROR_STRING_TRUNCATED) { - size = (size - 1) - remaining; + + size_t written; + if (ret == ERROR_STRING_TRUNCATED) { + written = size; + } else { + written = (size - 1U) - remaining; } - return (size_result_t){ .e = ret, .r = size }; + return (size_result_t){ .e = ret, .r = written }; } diff --git a/hyp/misc/prng_hw/src/prng_api.c b/hyp/misc/prng_hw/src/prng_api.c index ad2824a..5f9ea1c 100644 --- a/hyp/misc/prng_hw/src/prng_api.c +++ b/hyp/misc/prng_hw/src/prng_api.c @@ -17,7 +17,7 @@ hypercall_prng_get_entropy(count_t num_bytes) { hypercall_prng_get_entropy_result_t ret = { 0 }; - if ((num_bytes == 0) || (num_bytes > sizeof(uint32_t) * 4)) { + if ((num_bytes == 0U) || (num_bytes > (sizeof(uint32_t) * 4U))) { ret.error = ERROR_ARGUMENT_SIZE; goto out; } @@ -36,7 +36,7 @@ hypercall_prng_get_entropy(count_t num_bytes) ticks_t last_read = thread->prng_last_read & ~util_mask(2); assert(now >= last_read); - count_t read_count = thread->prng_last_read & util_mask(2); + count_t read_count = (count_t)(thread->prng_last_read & util_mask(2)); // Read rate-limit window is 33ms per thread to reduce DoS. if ((now - last_read) < platform_convert_ns_to_ticks(33000000U)) { @@ -58,7 +58,7 @@ hypercall_prng_get_entropy(count_t num_bytes) goto out; } } - if (num_bytes >= (2 * sizeof(uint32_t))) { + if (num_bytes >= (2U * sizeof(uint32_t))) { error_t err = platform_get_random32(&ret.data1); if (err != OK) { @@ -66,7 +66,7 @@ hypercall_prng_get_entropy(count_t num_bytes) goto out; } } - if (num_bytes >= (3 * sizeof(uint32_t))) { + if (num_bytes >= (3U * sizeof(uint32_t))) { error_t err = platform_get_random32(&ret.data2); if (err != OK) { @@ -74,7 +74,7 @@ hypercall_prng_get_entropy(count_t num_bytes) goto out; } } - if (num_bytes >= (4 * sizeof(uint32_t))) { + if (num_bytes >= (4U * sizeof(uint32_t))) { error_t err = platform_get_random32(&ret.data3); if (err != OK) { diff --git a/hyp/misc/prng_simple/src/chacha20.c b/hyp/misc/prng_simple/src/chacha20.c index 42e480b..22ca75a 100644 --- a/hyp/misc/prng_simple/src/chacha20.c +++ b/hyp/misc/prng_simple/src/chacha20.c @@ -56,15 +56,15 @@ chacha20_block(const uint32_t (*key)[8], uint32_t counter, count_t i; // Setup output with input - for (i = 0U; i < 4; i++) { + for (i = 0U; i < 4U; i++) { (*out)[i] = chacha20_const[i]; } - for (i = 0U; i < 8; i++) { - (*out)[i + 4] = (*key)[i]; + for (i = 0U; i < 8U; i++) { + (*out)[i + 4U] = (*key)[i]; } (*out)[12] = counter; - for (i = 0U; i < 3; i++) { - (*out)[i + 13] = (*nonce)[i]; + for (i = 0U; i < 3U; i++) { + (*out)[i + 13U] = (*nonce)[i]; } // Run 20 rounds (10 column interleaved with 10 diagonal rounds) @@ -73,14 +73,14 @@ chacha20_block(const uint32_t (*key)[8], uint32_t counter, } // Add the original state to the result - for (i = 0U; i < 4; i++) { + for (i = 0U; i < 4U; i++) { (*out)[i] += chacha20_const[i]; } - for (i = 0U; i < 8; i++) { - (*out)[4 + i] += (*key)[i]; + for (i = 0U; i < 8U; i++) { + (*out)[4U + i] += (*key)[i]; } (*out)[12] += counter; - for (i = 0U; i < 3; i++) { - (*out)[13 + i] += (*nonce)[i]; + for (i = 0U; i < 3U; i++) { + (*out)[13U + i] += (*nonce)[i]; } } diff --git a/hyp/misc/prng_simple/src/prng_simple.c b/hyp/misc/prng_simple/src/prng_simple.c index 1830b19..f0d27f0 100644 --- a/hyp/misc/prng_simple/src/prng_simple.c +++ b/hyp/misc/prng_simple/src/prng_simple.c @@ -49,8 +49,8 @@ #include "event_handlers.h" #define WORD_BITS 32U -#define BLOCK_WORDS (512 / WORD_BITS) -#define KEY_WORDS (256 / WORD_BITS) +#define BLOCK_WORDS (512U / WORD_BITS) +#define KEY_WORDS (256U / WORD_BITS) #define BUFFER_KEY_OFFSET 0U #define BUFFER_DATA_OFFSET KEY_WORDS // first bytes are reserved for the key #define BUFFER_BLOCKS 4U @@ -96,16 +96,18 @@ prng_simple_handle_boot_runtime_first_init(void) } prng_data = (prng_data_t *)ret.r; + assert(prng_data != NULL); - memset_s(prng_data, sizeof(*prng_data), 0, sizeof(*prng_data)); + (void)memset_s(prng_data, sizeof(*prng_data), 0, sizeof(*prng_data)); prng_data->pool_index = BUFFER_WORDS; // Buffer is Empty - memscpy(&prng_data->key, sizeof(prng_data->key), hypervisor_prng_seed, - sizeof(prng_data->key)); + (void)memscpy(&prng_data->key, sizeof(prng_data->key), + hypervisor_prng_seed, sizeof(prng_data->key)); // Ensure no stale copies remain in ram - memset_s(hypervisor_prng_seed, sizeof(hypervisor_prng_seed), 0, - sizeof(hypervisor_prng_seed)); + assert(hypervisor_prng_seed != NULL); + (void)memset_s(hypervisor_prng_seed, sizeof(hypervisor_prng_seed), 0, + sizeof(hypervisor_prng_seed)); CACHE_CLEAN_INVALIDATE_OBJECT(hypervisor_prng_seed); prng_data->key_timestamp = platform_timer_get_current_ticks(); @@ -123,12 +125,12 @@ prng_simple_handle_boot_runtime_first_init(void) prng_data->nonce[2] = serial[2]; // Add in some chip specific noise - prng_data->nonce[1] ^= hypervisor_prng_nonce & 0xffffffffU; - prng_data->nonce[2] ^= hypervisor_prng_nonce >> 32; + prng_data->nonce[1] ^= (uint32_t)(hypervisor_prng_nonce & 0xffffffffU); + prng_data->nonce[2] ^= (uint32_t)(hypervisor_prng_nonce >> 32); // Ensure no stale copies remain in ram - memset_s(&hypervisor_prng_nonce, sizeof(hypervisor_prng_nonce), 0, - sizeof(hypervisor_prng_nonce)); + (void)memset_s(&hypervisor_prng_nonce, sizeof(hypervisor_prng_nonce), 0, + sizeof(hypervisor_prng_nonce)); CACHE_CLEAN_INVALIDATE_OBJECT(hypervisor_prng_nonce); prng_initialized = true; @@ -138,6 +140,7 @@ prng_simple_handle_boot_runtime_first_init(void) void prng_simple_handle_boot_hypervisor_start(void) { + // FIXME: // Post boot prng_data protection // * allocate an unmapped 4K page for the prng_data // * Aarch64 PAN implementation: @@ -167,7 +170,7 @@ add_platform_entropy(void) REQUIRE_SPINLOCK(prng_lock) prng_data->key[7] ^= new.word[7]; // Ensure no stale copy remains on the stack - memset_s(&new, sizeof(new), 0, sizeof(new)); + (void)memset_s(&new, sizeof(new), 0, sizeof(new)); CACHE_CLEAN_INVALIDATE_OBJECT(new); success = true; @@ -218,15 +221,15 @@ prng_update(void) REQUIRE_SPINLOCK(prng_lock) } // Fast key update from block 0 - memscpy(prng_data->key, sizeof(prng_data->key), - &prng_data->entropy_pool[0], - sizeof(prng_data->entropy_pool[0])); + (void)memscpy(prng_data->key, sizeof(prng_data->key), + &prng_data->entropy_pool[0], + sizeof(prng_data->entropy_pool[0])); // Ensure no stale copies remain in ram CACHE_CLEAN_FIXED_RANGE(prng_data->key, 32U); // Clear the used bytes just in case - memset_s(&prng_data->entropy_pool[0], - sizeof(prng_data->entropy_pool[0]), 0, - BUFFER_DATA_OFFSET * sizeof(uint32_t)); + (void)memset_s(&prng_data->entropy_pool[0], + sizeof(prng_data->entropy_pool[0]), 0, + BUFFER_DATA_OFFSET * sizeof(uint32_t)); // Ensure no stale copies remain in ram CACHE_CLEAN_FIXED_RANGE(&prng_data->entropy_pool[0], BUFFER_DATA_OFFSET * sizeof(uint32_t)); @@ -245,12 +248,12 @@ prng_get64(void) count_t index = prng_data->pool_index; - if (index > (BUFFER_WORDS - (64 / WORD_BITS))) { + if (index > (BUFFER_WORDS - (64U / WORD_BITS))) { // Not enough buffered randomness, get more prng_update(); index = prng_data->pool_index; } - prng_data->pool_index += (64 / WORD_BITS); + prng_data->pool_index += (64U / WORD_BITS); index_t block = index / BLOCK_WORDS; index_t word = index % BLOCK_WORDS; @@ -260,13 +263,14 @@ prng_get64(void) ret.r |= (uint64_t)data[1] << 32; ret.e = OK; + // Pointer difference in bytes + ptrdiff_t len = (char *)data - (char *)prng_data->entropy_pool[0]; + + assert(len >= 0); // Clear used bits - memset_s(data, - sizeof(prng_data->entropy_pool) - - ((uintptr_t)data - - (uintptr_t)&prng_data->entropy_pool[0]), - 0, sizeof(ret.r)); + (void)memset_s(data, sizeof(prng_data->entropy_pool) - (size_t)len, 0, + sizeof(ret.r)); // Ensure used bits are cleared from caches CACHE_CLEAN_FIXED_RANGE(data, sizeof(ret.r)); diff --git a/hyp/misc/qcbor/build.conf b/hyp/misc/qcbor/build.conf new file mode 100644 index 0000000..65e5c2d --- /dev/null +++ b/hyp/misc/qcbor/build.conf @@ -0,0 +1,6 @@ +# © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +interface qcbor +source qcbor_encode.c UsefulBuf.c diff --git a/hyp/misc/qcbor/src/UsefulBuf.c b/hyp/misc/qcbor/src/UsefulBuf.c new file mode 100644 index 0000000..5ccf247 --- /dev/null +++ b/hyp/misc/qcbor/src/UsefulBuf.c @@ -0,0 +1,447 @@ +/*============================================================================== + Copyright (c) 2016-2018, The Linux Foundation. + Copyright (c) 2018-2022, Laurence Lundblade. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of The Linux Foundation nor the names of its + contributors, nor the name "Laurence Lundblade" may be used to + endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + =============================================================================*/ + +/*============================================================================= + FILE: UsefulBuf.c + + DESCRIPTION: General purpose input and output buffers + + EDIT HISTORY FOR FILE: + + This section contains comments describing changes made to the module. + Notice that changes are listed in reverse chronological order. + + when who what, where, why + -------- ---- --------------------------------------------------- + 19/12/2022 llundblade Don't pass NULL to memmove when adding empty data. + 4/11/2022 llundblade Add GetOutPlace and Advance to UsefulOutBuf + 3/6/2021 mcr/llundblade Fix warnings related to --Wcast-qual + 01/28/2020 llundblade Refine integer signedness to quiet static analysis. + 01/08/2020 llundblade Documentation corrections & improved code formatting. + 11/08/2019 llundblade Re check pointer math and update comments + 3/6/2019 llundblade Add UsefulBuf_IsValue() + 09/07/17 llundbla Fix critical bug in UsefulBuf_Find() -- a read off + the end of memory when the bytes to find is longer + than the bytes to search. + 06/27/17 llundbla Fix UsefulBuf_Compare() bug. Only affected comparison + for < or > for unequal length buffers. Added + UsefulBuf_Set() function. + 05/30/17 llundbla Functions for NULL UsefulBufs and const / unconst + 11/13/16 llundbla Initial Version. + + ============================================================================*/ + +#define ENABLE_DECODE_ROUTINES +#include "qcbor/UsefulBuf.h" + +// used to catch use of uninitialized or corrupted UsefulOutBuf +#define USEFUL_OUT_BUF_MAGIC (0x0B0F) + +int +memcmp(const void *s1, const void *s2, size_t n); + +/* + Public function -- see UsefulBuf.h + */ +UsefulBufC +UsefulBuf_CopyOffset(UsefulBuf Dest, size_t uOffset, const UsefulBufC Src) +{ + // Do this with subtraction so it doesn't give erroneous + // result if uOffset + Src.len overflows + if (uOffset > Dest.len || Src.len > Dest.len - uOffset) { // uOffset + + // Src.len > + // Dest.len + return NULLUsefulBufC; + } + + memcpy((uint8_t *)Dest.ptr + uOffset, Src.ptr, Src.len); + + return (UsefulBufC){ Dest.ptr, Src.len + uOffset }; +} + +/* + Public function -- see UsefulBuf.h + */ +int +UsefulBuf_Compare(const UsefulBufC UB1, const UsefulBufC UB2) +{ + // use the comparisons rather than subtracting lengths to + // return an int instead of a size_t + if (UB1.len < UB2.len) { + return -1; + } else if (UB1.len > UB2.len) { + return 1; + } // else UB1.len == UB2.len + + return memcmp(UB1.ptr, UB2.ptr, UB1.len); +} + +/* + Public function -- see UsefulBuf.h + */ +size_t +UsefulBuf_IsValue(const UsefulBufC UB, uint8_t uValue) +{ + if (UsefulBuf_IsNULLOrEmptyC(UB)) { + /* Not a match */ + return 0; + } + + const uint8_t *const pEnd = (const uint8_t *)UB.ptr + UB.len; + for (const uint8_t *p = UB.ptr; p < pEnd; p++) { + if (*p != uValue) { + /* Byte didn't match */ + /* Cast from signed to unsigned . Safe because the loop + * increments.*/ + return (size_t)(p - (const uint8_t *)UB.ptr); + } + } + + /* Success. All bytes matched */ + return SIZE_MAX; +} + +/* + Public function -- see UsefulBuf.h + */ +size_t +UsefulBuf_FindBytes(UsefulBufC BytesToSearch, UsefulBufC BytesToFind) +{ + if (BytesToSearch.len < BytesToFind.len) { + return SIZE_MAX; + } + + for (size_t uPos = 0; uPos <= BytesToSearch.len - BytesToFind.len; + uPos++) { + if (!UsefulBuf_Compare( + (UsefulBufC){ ((const uint8_t *)BytesToSearch.ptr) + + uPos, + BytesToFind.len }, + BytesToFind)) { + return uPos; + } + } + + return SIZE_MAX; +} + +/* + Public function -- see UsefulBuf.h + + Code Reviewers: THIS FUNCTION DOES POINTER MATH + */ +void +UsefulOutBuf_Init(UsefulOutBuf *pMe, UsefulBuf Storage) +{ + pMe->magic = USEFUL_OUT_BUF_MAGIC; + UsefulOutBuf_Reset(pMe); + pMe->UB = Storage; + +#if 0 + // This check is off by default. + + // The following check fails on ThreadX + + // Sanity check on the pointer and size to be sure we are not + // passed a buffer that goes off the end of the address space. + // Given this test, we know that all unsigned lengths less than + // me->size are valid and won't wrap in any pointer additions + // based off of pStorage in the rest of this code. + const uintptr_t ptrM = UINTPTR_MAX - Storage.len; + if(Storage.ptr && (uintptr_t)Storage.ptr > ptrM) // Check #0 + me->err = 1; +#endif +} + +/* + Public function -- see UsefulBuf.h + + The core of UsefulOutBuf -- put some bytes in the buffer without writing off + the end of it. + + Code Reviewers: THIS FUNCTION DOES POINTER MATH + + This function inserts the source buffer, NewData, into the destination + buffer, me->UB.ptr. + + Destination is represented as: + me->UB.ptr -- start of the buffer + me->UB.len -- size of the buffer UB.ptr + me->data_len -- length of value data in UB + + Source is data: + NewData.ptr -- start of source buffer + NewData.len -- length of source buffer + + Insertion point: + uInsertionPos. + + Steps: + + 0. Corruption checks on UsefulOutBuf + + 1. Figure out if the new data will fit or not + + 2. Is insertion position in the range of valid data? + + 3. If insertion point is not at the end, slide data to the right of the + insertion point to the right + + 4. Put the new data in at the insertion position. + + */ +void +UsefulOutBuf_InsertUsefulBuf(UsefulOutBuf *pMe, UsefulBufC NewData, + size_t uInsertionPos) +{ + if (pMe->err) { + // Already in error state. + return; + } + + /* 0. Sanity check the UsefulOutBuf structure */ + // A "counter measure". If magic number is not the right number it + // probably means me was not initialized or it was corrupted. Attackers + // can defeat this, but it is a hurdle and does good with very + // little code. + if (pMe->magic != USEFUL_OUT_BUF_MAGIC) { + pMe->err = 1; + return; // Magic number is wrong due to uninitalization or + // corrption + } + + // Make sure valid data is less than buffer size. This would only occur + // if there was corruption of me, but it is also part of the checks to + // be sure there is no pointer arithmatic under/overflow. + if (pMe->data_len > pMe->UB.len) { // Check #1 + pMe->err = 1; + // Offset of valid data is off the end of the UsefulOutBuf due + // to uninitialization or corruption + return; + } + + /* 1. Will it fit? */ + // WillItFit() is the same as: NewData.len <= (me->UB.len - + // me->data_len) Check #1 makes sure subtraction in RoomLeft will not + // wrap around + if (!UsefulOutBuf_WillItFit(pMe, NewData.len)) { // Check #2 + // The new data will not fit into the the buffer. + pMe->err = 1; + return; + } + + /* 2. Check the Insertion Position */ + // This, with Check #1, also confirms that uInsertionPos <= me->data_len + // and that uInsertionPos + pMe->UB.ptr will not wrap around the end of + // the address space. + if (uInsertionPos > pMe->data_len) { // Check #3 + // Off the end of the valid data in the buffer. + pMe->err = 1; + return; + } + + /* 3. Slide existing data to the right */ + if (!UsefulOutBuf_IsBufferNULL(pMe)) { + uint8_t *pSourceOfMove = + ((uint8_t *)pMe->UB.ptr) + uInsertionPos; // PtrMath #1 + size_t uNumBytesToMove = + pMe->data_len - uInsertionPos; // PtrMath + // #2 + uint8_t *pDestinationOfMove = + pSourceOfMove + NewData.len; // PtrMath + // #3 + + // To know memmove won't go off end of destination, see PtrMath + // #4 Use memove because it handles overlapping buffers + memmove(pDestinationOfMove, pSourceOfMove, uNumBytesToMove); + + /* 4. Put the new data in */ + uint8_t *pInsertionPoint = pSourceOfMove; + // To know memmove won't go off end of destination, see PtrMath + // #5 + if (NewData.ptr != NULL) { + memmove(pInsertionPoint, NewData.ptr, NewData.len); + } + } + + pMe->data_len += NewData.len; +} + +/* + Rationale that describes why the above pointer math is safe + + PtrMath #1 will never wrap around over because + Check #0 in UsefulOutBuf_Init makes sure me->UB.ptr + me->UB.len doesn't + wrap Check #1 makes sure me->data_len is less than me->UB.len Check #3 makes + sure uInsertionPos is less than me->data_len + + PtrMath #2 will never wrap around under because + Check #3 makes sure uInsertionPos is less than me->data_len + + PtrMath #3 will never wrap around over because + PtrMath #1 is checked resulting in pSourceOfMove being between me->UB.ptr + and me->UB.ptr + me->data_len Check #2 that NewData.len will fit in the unused + space left in me->UB + + PtrMath #4 will never wrap under because + Calculation for extent or memmove is uRoomInDestination = me->UB.len - + (uInsertionPos + NewData.len) Check #3 makes sure uInsertionPos is less than + me->data_len Check #3 allows Check #2 to be refactored as NewData.Len > + (me->size - uInsertionPos) This algebraically rearranges to me->size > + uInsertionPos + NewData.len + + PtrMath #5 will never wrap under because + Calculation for extent of memove is uRoomInDestination = me->UB.len - + uInsertionPos; Check #1 makes sure me->data_len is less than me->size Check #3 + makes sure uInsertionPos is less than me->data_len + */ + +/* + * Public function for advancing data length. See qcbor/UsefulBuf.h + */ +void +UsefulOutBuf_Advance(UsefulOutBuf *pMe, size_t uAmount) +{ + /* This function is a trimmed down version of + * UsefulOutBuf_InsertUsefulBuf(). This could be combined with the + * code in UsefulOutBuf_InsertUsefulBuf(), but that would make + * UsefulOutBuf_InsertUsefulBuf() bigger and this will be very + * rarely used. + */ + + if (pMe->err) { + /* Already in error state. */ + return; + } + + /* 0. Sanity check the UsefulOutBuf structure + * + * A "counter measure". If magic number is not the right number it + * probably means me was not initialized or it was + * corrupted. Attackers can defeat this, but it is a hurdle and + * does good with very little code. + */ + if (pMe->magic != USEFUL_OUT_BUF_MAGIC) { + pMe->err = 1; + return; /* Magic number is wrong due to uninitalization or + corrption */ + } + + /* Make sure valid data is less than buffer size. This would only + * occur if there was corruption of me, but it is also part of the + * checks to be sure there is no pointer arithmatic + * under/overflow. + */ + if (pMe->data_len > pMe->UB.len) { // Check #1 + pMe->err = 1; + /* Offset of valid data is off the end of the UsefulOutBuf due + * to uninitialization or corruption. + */ + return; + } + + /* 1. Will it fit? + * + * WillItFit() is the same as: NewData.len <= (me->UB.len - + * me->data_len) Check #1 makes sure subtraction in RoomLeft will + * not wrap around + */ + if (!UsefulOutBuf_WillItFit(pMe, uAmount)) { /* Check #2 */ + /* The new data will not fit into the the buffer. */ + pMe->err = 1; + return; + } + + pMe->data_len += uAmount; +} + +/* + Public function -- see UsefulBuf.h + */ +UsefulBufC +UsefulOutBuf_OutUBuf(UsefulOutBuf *pMe) +{ + if (pMe->err) { + return NULLUsefulBufC; + } + + if (pMe->magic != USEFUL_OUT_BUF_MAGIC) { + pMe->err = 1; + return NULLUsefulBufC; + } + + return (UsefulBufC){ pMe->UB.ptr, pMe->data_len }; +} + +/* + Public function -- see UsefulBuf.h + + Copy out the data accumulated in to the output buffer. + */ +UsefulBufC +UsefulOutBuf_CopyOut(UsefulOutBuf *pMe, UsefulBuf pDest) +{ + const UsefulBufC Tmp = UsefulOutBuf_OutUBuf(pMe); + if (UsefulBuf_IsNULLC(Tmp)) { + return NULLUsefulBufC; + } + return UsefulBuf_Copy(pDest, Tmp); +} + +/* + Public function -- see UsefulBuf.h + + The core of UsefulInputBuf -- consume bytes without going off end of buffer. + + Code Reviewers: THIS FUNCTION DOES POINTER MATH + */ +const void * +UsefulInputBuf_GetBytes(UsefulInputBuf *pMe, size_t uAmount) +{ + // Already in error state. Do nothing. + if (pMe->err) { + return NULL; + } + + if (!UsefulInputBuf_BytesAvailable(pMe, uAmount)) { + // Number of bytes asked for at current position are more than + // available + pMe->err = 1; + return NULL; + } + + // This is going to succeed + const void *const result = ((const uint8_t *)pMe->UB.ptr) + pMe->cursor; + // Will not overflow because of check using + // UsefulInputBuf_BytesAvailable() + pMe->cursor += uAmount; + return result; +} diff --git a/hyp/misc/qcbor/src/qcbor_encode.c b/hyp/misc/qcbor/src/qcbor_encode.c new file mode 100644 index 0000000..22521aa --- /dev/null +++ b/hyp/misc/qcbor/src/qcbor_encode.c @@ -0,0 +1,1076 @@ +/*============================================================================== + Copyright (c) 2016-2018, The Linux Foundation. + Copyright (c) 2018-2022, Laurence Lundblade. + Copyright (c) 2021, Arm Limited. + All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of The Linux Foundation nor the names of its + contributors, nor the name "Laurence Lundblade" may be used to + endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + =============================================================================*/ + +#include "qcbor/qcbor_encode.h" + +/** + * The entire implementation of the QCBOR encoder. + */ + +/* + * == Nesting Tracking == + * + * The following functions and data type QCBORTrackNesting implement + * the nesting management for encoding. + * + * CBOR's two nesting types, arrays and maps, are tracked here. There + * is a limit of QCBOR_MAX_ARRAY_NESTING to the number of arrays and + * maps that can be nested in one encoding so the encoding context + * stays small enough to fit on the stack. + * + * When an array/map is opened, pCurrentNesting points to the element + * in pArrays that records the type, start position and accumulates a + * count of the number of items added. When closed the start position + * is used to go back and fill in the type and number of items in the + * array/map. + * + * Encoded output can be a CBOR Sequence (RFC 8742) in which case + * there is no top-level array or map. It starts out with a string, + * integer or other non-aggregate type. It may have an array or map + * other than at the start, in which case that nesting is tracked + * here. + * + * QCBOR has a special feature to allow constructing byte string + * wrapped CBOR directly into the output buffer, so no extra buffer is + * needed for byte string wrapping. This is implemented as nesting + * with the type CBOR_MAJOR_TYPE_BYTE_STRING and is tracked here. Byte + * string wrapped CBOR is used by COSE for data that is to be hashed. + */ +static inline void +Nesting_Init(QCBORTrackNesting *pNesting) +{ + /* Assumes pNesting has been zeroed. */ + pNesting->pCurrentNesting = &pNesting->pArrays[0]; + /* Implied CBOR array at the top nesting level. This is never + * returned, but makes the item count work correctly. + */ + pNesting->pCurrentNesting->uMajorType = CBOR_MAJOR_TYPE_ARRAY; +} + +static inline uint8_t +Nesting_Increase(QCBORTrackNesting *pNesting, uint8_t uMajorType, uint32_t uPos) +{ + if (pNesting->pCurrentNesting == + &pNesting->pArrays[QCBOR_MAX_ARRAY_NESTING]) { + return QCBOR_ERR_ARRAY_NESTING_TOO_DEEP; + } else { + pNesting->pCurrentNesting++; + pNesting->pCurrentNesting->uCount = 0; + pNesting->pCurrentNesting->uStart = uPos; + pNesting->pCurrentNesting->uMajorType = uMajorType; + return QCBOR_SUCCESS; + } +} + +static inline void +Nesting_Decrease(QCBORTrackNesting *pNesting) +{ + if (pNesting->pCurrentNesting > &pNesting->pArrays[0]) { + pNesting->pCurrentNesting--; + } +} + +static inline uint8_t +Nesting_Increment(QCBORTrackNesting *pNesting) +{ +#ifndef QCBOR_DISABLE_ENCODE_USAGE_GUARDS + if (1 >= QCBOR_MAX_ITEMS_IN_ARRAY - pNesting->pCurrentNesting->uCount) { + return QCBOR_ERR_ARRAY_TOO_LONG; + } +#endif /* QCBOR_DISABLE_ENCODE_USAGE_GUARDS */ + + pNesting->pCurrentNesting->uCount++; + + return QCBOR_SUCCESS; +} + +static inline void +Nesting_Decrement(QCBORTrackNesting *pNesting) +{ + /* No error check for going below 0 here needed because this + * is only used by QCBOREncode_CancelBstrWrap() and it checks + * the nesting level before calling this. */ + pNesting->pCurrentNesting->uCount--; +} + +static inline uint16_t +Nesting_GetCount(QCBORTrackNesting *pNesting) +{ + /* The nesting count recorded is always the actual number of + * individual data items in the array or map. For arrays CBOR uses + * the actual item count. For maps, CBOR uses the number of pairs. + * This function returns the number needed for the CBOR encoding, + * so it divides the number of items by two for maps to get the + * number of pairs. + */ + if (pNesting->pCurrentNesting->uMajorType == CBOR_MAJOR_TYPE_MAP) { + /* Cast back to uint16_t after integer promotion from bit shift + */ + return (uint16_t)(pNesting->pCurrentNesting->uCount >> 1); + } else { + return pNesting->pCurrentNesting->uCount; + } +} + +static inline uint32_t +Nesting_GetStartPos(QCBORTrackNesting *pNesting) +{ + return pNesting->pCurrentNesting->uStart; +} + +#ifndef QCBOR_DISABLE_ENCODE_USAGE_GUARDS +static inline uint8_t +Nesting_GetMajorType(QCBORTrackNesting *pNesting) +{ + return pNesting->pCurrentNesting->uMajorType; +} + +static inline bool +Nesting_IsInNest(QCBORTrackNesting *pNesting) +{ + return pNesting->pCurrentNesting == &pNesting->pArrays[0] ? false + : true; +} +#endif /* QCBOR_DISABLE_ENCODE_USAGE_GUARDS */ + +/* + * == Major CBOR Types == + * + * Encoding of the major CBOR types is by these functions: + * + * CBOR Major Type Public Function + * 0 QCBOREncode_AddUInt64() + * 0, 1 QCBOREncode_AddUInt64(), QCBOREncode_AddInt64() + * 2, 3 QCBOREncode_AddBuffer() + * 4, 5 QCBOREncode_OpenMapOrArray(), QCBOREncode_CloseMapOrArray(), + * QCBOREncode_OpenMapOrArrayIndefiniteLength(), + * QCBOREncode_CloseMapOrArrayIndefiniteLength() + * 6 QCBOREncode_AddTag() + * 7 QCBOREncode_AddDouble(), QCBOREncode_AddFloat(), + * QCBOREncode_AddDoubleNoPreferred(), + * QCBOREncode_AddFloatNoPreferred(), QCBOREncode_AddType7() + * + * Additionally, encoding of decimal fractions and bigfloats is by + * QCBOREncode_AddExponentAndMantissa() and byte strings that wrap + * encoded CBOR are handled by QCBOREncode_OpenMapOrArray() and + * QCBOREncode_CloseBstrWrap2(). + * + * + * == Error Tracking Plan == + * + * Errors are tracked internally and not returned until + * QCBOREncode_Finish() or QCBOREncode_GetErrorState() is called. The + * CBOR errors are in me->uError. UsefulOutBuf also tracks whether + * the buffer is full or not in its context. Once either of these + * errors is set they are never cleared. Only QCBOREncode_Init() + * resets them. Or said another way, they must never be cleared or + * we'll tell the caller all is good when it is not. + * + * Only one error code is reported by QCBOREncode_Finish() even if + * there are multiple errors. The last one set wins. The caller might + * have to fix one error to reveal the next one they have to fix. + * This is OK. + * + * The buffer full error tracked by UsefulBuf is only pulled out of + * UsefulBuf in QCBOREncode_Finish() so it is the one that usually + * wins. UsefulBuf will never go off the end of the buffer even if it + * is called again and again when full. + * + * QCBOR_DISABLE_ENCODE_USAGE_GUARDS disables about half of the error + * checks here to reduce code size by about 150 bytes leaving only the + * checks for size to avoid buffer overflow. If the calling code is + * completely correct, checks are completely unnecessary. For + * example, there is no need to check that all the opens are matched + * by a close. + * + * QCBOR_DISABLE_ENCODE_USAGE_GUARDS also disables the check for more + * than QCBOR_MAX_ITEMS_IN_ARRAY in an array. Since + * QCBOR_MAX_ITEMS_IN_ARRAY is very large (65,535) it is very unlikely + * to be reached. If it is reached, the count will wrap around to zero + * and CBOR that is not well formed will be produced, but there will + * be no buffers overrun and new security issues in the code. + * + * The 8 errors returned here fall into three categories: + * + * Sizes + * QCBOR_ERR_BUFFER_TOO_LARGE -- Encoded output exceeded UINT32_MAX + * QCBOR_ERR_BUFFER_TOO_SMALL -- Output buffer too small + * QCBOR_ERR_ARRAY_NESTING_TOO_DEEP -- Nesting > QCBOR_MAX_ARRAY_NESTING1 + * QCBOR_ERR_ARRAY_TOO_LONG -- Too many items added to an array/map + * [1] + * + * Nesting constructed incorrectly + * QCBOR_ERR_TOO_MANY_CLOSES -- More close calls than opens [1] + * QCBOR_ERR_CLOSE_MISMATCH -- Type of close does not match open [1] + * QCBOR_ERR_ARRAY_OR_MAP_STILL_OPEN -- Finish called without enough closes + * [1] + * + * Would generate not-well-formed CBOR + * QCBOR_ERR_ENCODE_UNSUPPORTED -- Simple type between 24 and 31 [1] + * + * [1] indicated disabled by QCBOR_DISABLE_ENCODE_USAGE_GUARDS + */ + +/* + Public function for initialization. See qcbor/qcbor_encode.h + */ +void +QCBOREncode_Init(QCBOREncodeContext *me, UsefulBuf Storage) +{ + memset(me, 0, sizeof(QCBOREncodeContext)); + UsefulOutBuf_Init(&(me->OutBuf), Storage); + Nesting_Init(&(me->nesting)); +} + +/* + * Public function to encode a CBOR head. See qcbor/qcbor_encode.h + */ +UsefulBufC +QCBOREncode_EncodeHead(UsefulBuf buffer, uint8_t uMajorType, uint8_t uMinLen, + uint64_t uArgument) +{ + /* + * == Description of the CBOR Head == + * + * The head of a CBOR data item + * +---+-----+ +--------+ +--------+ +--------+ +--------+ + * |M T| A R G U M E N T . . . | + * +---+-----+ +--------+ +--------+ +--------+ ... +--------+ + * + * Every CBOR data item has a "head". It is made up of the "major + * type" and the "argument". + * + * The major type indicates whether the data item is an integer, + * string, array or such. It is encoded in 3 bits giving it a range + * from 0 to 7. 0 indicates the major type is a positive integer, + * 1 a negative integer, 2 a byte string and so on. + * + * These 3 bits are the first part of the "initial byte" in a data + * item. Every data item has an initial byte, and some only have + * the initial byte. + * + * The argument is essentially a number between 0 and UINT64_MAX + * (18446744073709551615). This number is interpreted to mean + * different things for the different major types. For major type + * 0, a positive integer, it is value of the data item. For major + * type 2, a byte string, it is the length in bytes of the byte + * string. For major type 4, an array, it is the number of data + * items in the array. + * + * Special encoding is used so that the argument values less than + * 24 can be encoded very compactly in the same byte as the major + * type is encoded. When the lower 5 bits of the initial byte have + * a value less than 24, then that is the value of the argument. + * + * If the lower 5 bits of the initial byte are less than 24, then + * they are the value of the argument. This allows integer values 0 + * - 23 to be CBOR encoded in just one byte. + * + * When the value of lower 5 bits are 24, 25, 26, or 27 the + * argument is encoded in 1, 2, 4 or 8 bytes following the initial + * byte in network byte order (bit endian). The cases when it is + * 28, 29 and 30 are reserved for future use. The value 31 is a + * special indicator for indefinite length strings, arrays and + * maps. + * + * The lower 5 bits are called the "additional information." + * + * Thus the CBOR head may be 1, 2, 3, 5 or 9 bytes long. + * + * It is legal in CBOR to encode the argument using any of these + * lengths even if it could be encoded in a shorter length. For + * example it is legal to encode a data item representing the + * positive integer 0 in 9 bytes even though it could be encoded in + * only 0. This is legal to allow for for very simple code or even + * hardware-only implementations that just output a register + * directly. + * + * CBOR defines preferred encoding as the encoding of the argument + * in the smallest number of bytes needed to encode it. + * + * This function takes the major type and argument as inputs and + * outputs the encoded CBOR head for them. It does conversion to + * network byte order. It implements CBOR preferred encoding, + * outputting the shortest representation of the argument. + * + * == Endian Conversion == + * + * This code does endian conversion without hton() or knowing the + * endianness of the machine by using masks and shifts. This avoids + * the dependency on hton() and the mess of figuring out how to + * find the machine's endianness. + * + * This is a good efficient implementation on little-endian + * machines. A faster and smaller implementation is possible on + * big-endian machines because CBOR/network byte order is + * big-endian. However big-endian machines are uncommon. + * + * On x86, this is about 150 bytes instead of 500 bytes for the + * original, more formal unoptimized code. + * + * This also does the CBOR preferred shortest encoding for integers + * and is called to do endian conversion for floats. + * + * It works backwards from the least significant byte to the most + * significant byte. + * + * == Floating Point == + * + * When the major type is 7 and the 5 lower bits have the values + * 25, 26 or 27, the argument is a floating-point number that is + * half, single or double-precision. Note that it is not the + * conversion from a floating-point value to an integer value like + * converting 0x00 to 0.00, it is the interpretation of the bits in + * the argument as an IEEE 754 float-point number. + * + * Floating-point numbers must be converted to network byte + * order. That is accomplished here by exactly the same code that + * converts integer arguments to network byte order. + * + * There is preferred encoding for floating-point numbers in CBOR, + * but it is very different than for integers and it is not + * implemented here. Half-precision is preferred to + * single-precision which is preferred to double-precision only if + * the conversion can be performed without loss of precision. Zero + * and infinity can always be converted to half-precision, without + * loss but 3.141592653589 cannot. + * + * The way this function knows to not do preferred encoding on the + * argument passed here when it is a floating point number is the + * uMinLen parameter. It should be 2, 4 or 8 for half, single and + * double precision floating point values. This prevents and the + * incorrect removal of leading zeros when encoding arguments that + * are floating-point numbers. + * + * == Use of Type int and Static Analyzers == + * + * The type int is used here for several variables because of the + * way integer promotion works in C for variables that are uint8_t + * or uint16_t. The basic rule is that they will always be promoted + * to int if they will fit. These integer variables here need only + * hold values less than 255 so they will always fit into an int. + * + * Most of values stored are never negative, so one might think + * that unsigned int would be more correct than int. However the C + * integer promotion rules only promote to unsigned int if the + * result won't fit into an int even if the promotion is for an + * unsigned variable like uint8_t. + * + * By declaring these int, there are few implicit conversions and + * fewer casts needed. Code size is reduced a little. It makes + * static analyzers happier. + * + * Note also that declaring these uint8_t won't stop integer wrap + * around if the code is wrong. It won't make the code more + * correct. + * + * https://stackoverflow.com/questions/46073295/implicit-type-promotion-rules + * https://stackoverflow.com/questions/589575/what-does-the-c-standard-state-the-size-of-int-long-type-to-be + * + * Code Reviewers: THIS FUNCTION DOES POINTER MATH + */ + + /* The buffer must have room for the largest CBOR HEAD + one + * extra. The one extra is needed for this code to work as it does + * a pre-decrement. + */ + if (buffer.len < QCBOR_HEAD_BUFFER_SIZE) { + return NULLUsefulBufC; + } + + /* Pointer to last valid byte in the buffer */ + uint8_t *const pBufferEnd = + &((uint8_t *)buffer.ptr)[QCBOR_HEAD_BUFFER_SIZE - 1]; + + /* Point to the last byte and work backwards */ + uint8_t *pByte = pBufferEnd; + /* The 5 bits in the initial byte that are not the major type */ + int nAdditionalInfo; + + if (uMajorType > QCBOR_INDEFINITE_LEN_TYPE_MODIFIER) { + /* Special case for start & end of indefinite length */ + uMajorType = uMajorType - QCBOR_INDEFINITE_LEN_TYPE_MODIFIER; + /* This takes advantage of design of CBOR where additional info + * is 31 for both opening and closing indefinite length + * maps and arrays. + */ +#if CBOR_SIMPLE_BREAK != LEN_IS_INDEFINITE +#error additional info for opening array not the same as for closing +#endif + nAdditionalInfo = CBOR_SIMPLE_BREAK; + + } else if (uArgument < CBOR_TWENTY_FOUR && uMinLen == 0) { + /* Simple case where argument is < 24 */ + nAdditionalInfo = (int)uArgument; + + } else { + /* This encodes the argument in 1,2,4 or 8 bytes. The outer loop + * runs once for 1 byte and 4 times for 8 bytes. The inner loop + * runs 1, 2 or 4 times depending on outer loop counter. This + * works backwards shifting 8 bits off the argument being + * encoded at a time until all bits from uArgument have been + * encoded and the minimum encoding size is reached. Minimum + * encoding size is for floating-point numbers that have some + * zero-value bytes that must be output. + */ + static const uint8_t aIterate[] = { 1, 1, 2, 4 }; + + /* uMinLen passed in is unsigned, but goes negative in the loop + * so it must be converted to a signed value. + */ + int nMinLen = (int)uMinLen; + int i; + for (i = 0; uArgument || nMinLen > 0; i++) { + const int nIterations = (int)aIterate[i]; + for (int j = 0; j < nIterations; j++) { + *--pByte = (uint8_t)(uArgument & 0xff); + uArgument = uArgument >> 8; + } + nMinLen -= nIterations; + } + + nAdditionalInfo = LEN_IS_ONE_BYTE - 1 + i; + } + + /* This expression integer-promotes to type int. The code above in + * function guarantees that nAdditionalInfo will never be larger + * than 0x1f. The caller may pass in a too-large uMajor type. The + * conversion to unint8_t will cause an integer wrap around and + * incorrect CBOR will be generated, but no security issue will + * occur. + */ + const int nInitialByte = (uMajorType << 5) + nAdditionalInfo; + *--pByte = (uint8_t)nInitialByte; + +#ifdef EXTRA_ENCODE_HEAD_CHECK + /* This is a sanity check that can be turned on to verify the + * pointer math in this function is not going wrong. Turn it on and + * run the whole test suite to perform the check. + */ + if (pBufferEnd - pByte > 9 || pBufferEnd - pByte < 1 || + pByte < (uint8_t *)buffer.ptr) { + return NULLUsefulBufC; + } +#endif /* EXTRA_ENCODE_HEAD_CHECK */ + + /* Length will not go negative because the loops run for at most 8 + * decrements of pByte, only one other decrement is made, and the array + * is sized for this. + */ + return (UsefulBufC){ pByte, (size_t)(pBufferEnd - pByte) }; +} + +/** + * @brief Append the CBOR head, the major type and argument + * + * @param me Encoder context. + * @param uMajorType Major type to insert. + * @param uArgument The argument (an integer value or a length). + * @param uMinLen The minimum number of bytes for encoding the CBOR + * argument. + * + * This formats the CBOR "head" and appends it to the output. + */ +static void +AppendCBORHead(QCBOREncodeContext *me, uint8_t uMajorType, uint64_t uArgument, + uint8_t uMinLen) +{ + /* A stack buffer large enough for a CBOR head */ + UsefulBuf_MAKE_STACK_UB(pBufferForEncodedHead, QCBOR_HEAD_BUFFER_SIZE); + + UsefulBufC EncodedHead = QCBOREncode_EncodeHead( + pBufferForEncodedHead, uMajorType, uMinLen, uArgument); + + /* No check for EncodedHead == NULLUsefulBufC is performed here to + * save object code. It is very clear that pBufferForEncodedHead is + * the correct size. If EncodedHead == NULLUsefulBufC then + * UsefulOutBuf_AppendUsefulBuf() will do nothing so there is no + * security hole introduced. + */ + + UsefulOutBuf_AppendUsefulBuf(&(me->OutBuf), EncodedHead); +} + +/** + * @brief Check for errors when decreasing nesting. + * + * @param pMe QCBOR encoding context. + * @param uMajorType The major type of the nesting. + * + * Check that there is no previous error, that there is actually some + * nesting and that the major type of the opening of the nesting + * matches the major type of the nesting being closed. + * + * This is called when closing maps, arrays, byte string wrapping and + * open/close of byte strings. + */ +static bool +CheckDecreaseNesting(QCBOREncodeContext *pMe, uint8_t uMajorType) +{ +#ifndef QCBOR_DISABLE_ENCODE_USAGE_GUARDS + if (pMe->uError != QCBOR_SUCCESS) { + return true; + } + + if (!Nesting_IsInNest(&(pMe->nesting))) { + pMe->uError = QCBOR_ERR_TOO_MANY_CLOSES; + return true; + } + + if (Nesting_GetMajorType(&(pMe->nesting)) != uMajorType) { + pMe->uError = QCBOR_ERR_CLOSE_MISMATCH; + return true; + } + +#else + /* None of these checks are performed if the encode guards are + * turned off as they all relate to correct calling. + * + * Turning off all these checks does not turn off any checking for + * buffer overflows or pointer issues. + */ + + (void)uMajorType; + (void)pMe; +#endif + + return false; +} + +/** + * @brief Insert the CBOR head for a map, array or wrapped bstr + * + * @param me QCBOR encoding context. + * @param uMajorType One of CBOR_MAJOR_TYPE_XXXX. + * @param uLen The length of the data item. + * + * When an array, map or bstr was opened, nothing was done but note + * the position. This function goes back to that position and inserts + * the CBOR Head with the major type and length. + */ +static void +InsertCBORHead(QCBOREncodeContext *me, uint8_t uMajorType, size_t uLen) +{ + if (CheckDecreaseNesting(me, uMajorType)) { + return; + } + + if (uMajorType == CBOR_MAJOR_NONE_TYPE_OPEN_BSTR) { + uMajorType = CBOR_MAJOR_TYPE_BYTE_STRING; + } + + /* A stack buffer large enough for a CBOR head (9 bytes) */ + UsefulBuf_MAKE_STACK_UB(pBufferForEncodedHead, QCBOR_HEAD_BUFFER_SIZE); + + UsefulBufC EncodedHead = QCBOREncode_EncodeHead(pBufferForEncodedHead, + uMajorType, 0, uLen); + + /* No check for EncodedHead == NULLUsefulBufC is performed here to + * save object code. It is very clear that pBufferForEncodedHead is + * the correct size. If EncodedHead == NULLUsefulBufC then + * UsefulOutBuf_InsertUsefulBuf() will do nothing so there is no + * security hole introduced. + */ + UsefulOutBuf_InsertUsefulBuf(&(me->OutBuf), EncodedHead, + Nesting_GetStartPos(&(me->nesting))); + + Nesting_Decrease(&(me->nesting)); +} + +/** + * @brief Increment item counter for maps and arrays. + * + * @param pMe QCBOR encoding context. + * + * This is mostly a separate function to make code more readable and + * to have fewer occurrences of #ifndef QCBOR_DISABLE_ENCODE_USAGE_GUARDS + */ +static inline void +IncrementMapOrArrayCount(QCBOREncodeContext *pMe) +{ +#ifndef QCBOR_DISABLE_ENCODE_USAGE_GUARDS + if (pMe->uError == QCBOR_SUCCESS) { + pMe->uError = Nesting_Increment(&(pMe->nesting)); + } +#else + (void)Nesting_Increment(&(pMe->nesting)); +#endif /* QCBOR_DISABLE_ENCODE_USAGE_GUARDS */ +} + +/* + * Public functions for adding unsigned integers. See qcbor/qcbor_encode.h + */ +void +QCBOREncode_AddUInt64(QCBOREncodeContext *me, uint64_t uValue) +{ + AppendCBORHead(me, CBOR_MAJOR_TYPE_POSITIVE_INT, uValue, 0); + + IncrementMapOrArrayCount(me); +} + +/* + * Public functions for adding signed integers. See qcbor/qcbor_encode.h + */ +void +QCBOREncode_AddInt64(QCBOREncodeContext *me, int64_t nNum) +{ + uint8_t uMajorType; + uint64_t uValue; + + if (nNum < 0) { + /* In CBOR -1 encodes as 0x00 with major type negative int. + * First add one as a signed integer because that will not + * overflow. Then change the sign as needed for encoding. (The + * opposite order, changing the sign and subtracting, can cause + * an overflow when encoding INT64_MIN. */ + int64_t nTmp = nNum + 1; + uValue = (uint64_t)-nTmp; + uMajorType = CBOR_MAJOR_TYPE_NEGATIVE_INT; + } else { + uValue = (uint64_t)nNum; + uMajorType = CBOR_MAJOR_TYPE_POSITIVE_INT; + } + AppendCBORHead(me, uMajorType, uValue, 0); + + IncrementMapOrArrayCount(me); +} + +/* + * Semi-private function. It is exposed to user of the interface, but + * one of its inline wrappers will usually be called instead of this. + * + * See qcbor/qcbor_encode.h + * + * This does the work of adding actual strings bytes to the CBOR + * output (as opposed to adding numbers and opening / closing + * aggregate types). + + * There are four use cases: + * CBOR_MAJOR_TYPE_BYTE_STRING -- Byte strings + * CBOR_MAJOR_TYPE_TEXT_STRING -- Text strings + * CBOR_MAJOR_NONE_TYPE_RAW -- Already-encoded CBOR + * CBOR_MAJOR_NONE_TYPE_BSTR_LEN_ONLY -- Special case + * + * The first two add the head plus the actual bytes. The third just + * adds the bytes as the heas is presumed to be in the bytes. The + * fourth just adds the head for the very special case of + * QCBOREncode_AddBytesLenOnly(). + */ +void +QCBOREncode_AddBuffer(QCBOREncodeContext *me, uint8_t uMajorType, + UsefulBufC Bytes) +{ + /* If it is not Raw CBOR, add the type and the length */ + if (uMajorType != CBOR_MAJOR_NONE_TYPE_RAW) { + uint8_t uRealMajorType = uMajorType; + if (uRealMajorType == CBOR_MAJOR_NONE_TYPE_BSTR_LEN_ONLY) { + uRealMajorType = CBOR_MAJOR_TYPE_BYTE_STRING; + } + AppendCBORHead(me, uRealMajorType, Bytes.len, 0); + } + + if (uMajorType != CBOR_MAJOR_NONE_TYPE_BSTR_LEN_ONLY) { + /* Actually add the bytes */ + UsefulOutBuf_AppendUsefulBuf(&(me->OutBuf), Bytes); + } + + IncrementMapOrArrayCount(me); +} + +/* + * Public functions for adding a tag. See qcbor/qcbor_encode.h + */ +void +QCBOREncode_AddTag(QCBOREncodeContext *me, uint64_t uTag) +{ + AppendCBORHead(me, CBOR_MAJOR_TYPE_TAG, uTag, 0); +} + +/* + * Semi-private function. It is exposed to user of the interface, but + * one of its inline wrappers will usually be called instead of this. + * + * See header qcbor/qcbor_encode.h + */ +void +QCBOREncode_AddType7(QCBOREncodeContext *me, uint8_t uMinLen, uint64_t uNum) +{ +#ifndef QCBOR_DISABLE_ENCODE_USAGE_GUARDS + if (me->uError == QCBOR_SUCCESS) { + if (uNum >= CBOR_SIMPLEV_RESERVED_START && + uNum <= CBOR_SIMPLEV_RESERVED_END) { + me->uError = QCBOR_ERR_ENCODE_UNSUPPORTED; + return; + } + } +#endif /* QCBOR_DISABLE_ENCODE_USAGE_GUARDS */ + + /* AppendCBORHead() does endian swapping for the float / double */ + AppendCBORHead(me, CBOR_MAJOR_TYPE_SIMPLE, uNum, uMinLen); + + IncrementMapOrArrayCount(me); +} + +#ifndef USEFULBUF_DISABLE_ALL_FLOAT +/* + * Public functions for adding a double. See qcbor/qcbor_encode.h + */ +void +QCBOREncode_AddDoubleNoPreferred(QCBOREncodeContext *me, double dNum) +{ + QCBOREncode_AddType7(me, sizeof(uint64_t), + UsefulBufUtil_CopyDoubleToUint64(dNum)); +} + +/* + * Public functions for adding a double. See qcbor/qcbor_encode.h + */ +void +QCBOREncode_AddDouble(QCBOREncodeContext *me, double dNum) +{ +#ifndef QCBOR_DISABLE_PREFERRED_FLOAT + const IEEE754_union uNum = IEEE754_DoubleToSmallest(dNum); + + QCBOREncode_AddType7(me, uNum.uSize, uNum.uValue); +#else /* QCBOR_DISABLE_PREFERRED_FLOAT */ + QCBOREncode_AddDoubleNoPreferred(me, dNum); +#endif /* QCBOR_DISABLE_PREFERRED_FLOAT */ +} + +/* + * Public functions for adding a float. See qcbor/qcbor_encode.h + */ +void +QCBOREncode_AddFloatNoPreferred(QCBOREncodeContext *me, float fNum) +{ + QCBOREncode_AddType7(me, sizeof(uint32_t), + UsefulBufUtil_CopyFloatToUint32(fNum)); +} + +/* + * Public functions for adding a float. See qcbor/qcbor_encode.h + */ +void +QCBOREncode_AddFloat(QCBOREncodeContext *me, float fNum) +{ +#ifndef QCBOR_DISABLE_PREFERRED_FLOAT + const IEEE754_union uNum = IEEE754_FloatToSmallest(fNum); + + QCBOREncode_AddType7(me, uNum.uSize, uNum.uValue); +#else /* QCBOR_DISABLE_PREFERRED_FLOAT */ + QCBOREncode_AddFloatNoPreferred(me, fNum); +#endif /* QCBOR_DISABLE_PREFERRED_FLOAT */ +} +#endif /* USEFULBUF_DISABLE_ALL_FLOAT */ + +#ifndef QCBOR_DISABLE_EXP_AND_MANTISSA +/* + * Semi-public function. It is exposed to the user of the interface, + * but one of the inline wrappers will usually be called rather than + * this. + * + * See qcbor/qcbor_encode.h + * + * Improvement: create another version of this that only takes a big + * number mantissa and converts the output to a type 0 or 1 integer + * when mantissa is small enough. + */ +void +QCBOREncode_AddExponentAndMantissa(QCBOREncodeContext *pMe, uint64_t uTag, + UsefulBufC BigNumMantissa, + bool bBigNumIsNegative, int64_t nMantissa, + int64_t nExponent) +{ + /* This is for encoding either a big float or a decimal fraction, + * both of which are an array of two items, an exponent and a + * mantissa. The difference between the two is that the exponent + * is base-2 for big floats and base-10 for decimal fractions, but + * that has no effect on the code here. + */ + if (uTag != CBOR_TAG_INVALID64) { + QCBOREncode_AddTag(pMe, uTag); + } + QCBOREncode_OpenArray(pMe); + QCBOREncode_AddInt64(pMe, nExponent); + if (!UsefulBuf_IsNULLC(BigNumMantissa)) { + if (bBigNumIsNegative) { + QCBOREncode_AddNegativeBignum(pMe, BigNumMantissa); + } else { + QCBOREncode_AddPositiveBignum(pMe, BigNumMantissa); + } + } else { + QCBOREncode_AddInt64(pMe, nMantissa); + } + QCBOREncode_CloseArray(pMe); +} +#endif /* QCBOR_DISABLE_EXP_AND_MANTISSA */ + +/* + * Semi-public function. It is exposed to the user of the interface, + * but one of the inline wrappers will usually be called rather than + * this. + * + * See qcbor/qcbor_encode.h + */ +void +QCBOREncode_OpenMapOrArray(QCBOREncodeContext *me, uint8_t uMajorType) +{ + /* Add one item to the nesting level we are in for the new map or array + */ + IncrementMapOrArrayCount(me); + + /* The offset where the length of an array or map will get written + * is stored in a uint32_t, not a size_t to keep stack usage + * smaller. This checks to be sure there is no wrap around when + * recording the offset. Note that on 64-bit machines CBOR larger + * than 4GB can be encoded as long as no array/map offsets occur + * past the 4GB mark, but the public interface says that the + * maximum is 4GB to keep the discussion simpler. + */ + size_t uEndPosition = UsefulOutBuf_GetEndPosition(&(me->OutBuf)); + + /* QCBOR_MAX_ARRAY_OFFSET is slightly less than UINT32_MAX so this + * code can run on a 32-bit machine and tests can pass on a 32-bit + * machine. If it was exactly UINT32_MAX, then this code would not + * compile or run on a 32-bit machine and an #ifdef or some machine + * size detection would be needed reducing portability. + */ + if (uEndPosition >= QCBOR_MAX_ARRAY_OFFSET) { + me->uError = QCBOR_ERR_BUFFER_TOO_LARGE; + + } else { + /* Increase nesting level because this is a map or array. Cast + * from size_t to uin32_t is safe because of check above. + */ + me->uError = Nesting_Increase(&(me->nesting), uMajorType, + (uint32_t)uEndPosition); + } +} + +/* + * Semi-public function. It is exposed to the user of the interface, + * but one of the inline wrappers will usually be called rather than + * this. + * + * See qcbor/qcbor_encode.h + */ +void +QCBOREncode_OpenMapOrArrayIndefiniteLength(QCBOREncodeContext *me, + uint8_t uMajorType) +{ + /* Insert the indefinite length marker (0x9f for arrays, 0xbf for maps) + */ + AppendCBORHead(me, uMajorType, 0, 0); + + /* Call the definite-length opener just to do the bookkeeping for + * nesting. It will record the position of the opening item in the + * encoded output but this is not used when closing this open. + */ + QCBOREncode_OpenMapOrArray(me, uMajorType); +} + +/* + * Public functions for closing arrays and maps. See qcbor/qcbor_encode.h + */ +void +QCBOREncode_CloseMapOrArray(QCBOREncodeContext *me, uint8_t uMajorType) +{ + InsertCBORHead(me, uMajorType, Nesting_GetCount(&(me->nesting))); +} + +/* + * Public functions for closing bstr wrapping. See qcbor/qcbor_encode.h + */ +void +QCBOREncode_CloseBstrWrap2(QCBOREncodeContext *me, bool bIncludeCBORHead, + UsefulBufC *pWrappedCBOR) +{ + const size_t uInsertPosition = Nesting_GetStartPos(&(me->nesting)); + const size_t uEndPosition = UsefulOutBuf_GetEndPosition(&(me->OutBuf)); + + /* This subtraction can't go negative because the UsefulOutBuf + * always only grows and never shrinks. UsefulOutBut itself also + * has defenses such that it won't write where it should not even + * if given incorrect input lengths. + */ + const size_t uBstrLen = uEndPosition - uInsertPosition; + + /* Actually insert */ + InsertCBORHead(me, CBOR_MAJOR_TYPE_BYTE_STRING, uBstrLen); + + if (pWrappedCBOR) { + /* Return pointer and length to the enclosed encoded CBOR. The + * intended use is for it to be hashed (e.g., SHA-256) in a COSE + * implementation. This must be used right away, as the pointer + * and length go invalid on any subsequent calls to this + * function because there might be calls to + * InsertEncodedTypeAndNumber() that slides data to the right. + */ + size_t uStartOfNew = uInsertPosition; + if (!bIncludeCBORHead) { + /* Skip over the CBOR head to just get the inserted bstr + */ + const size_t uNewEndPosition = + UsefulOutBuf_GetEndPosition(&(me->OutBuf)); + uStartOfNew += uNewEndPosition - uEndPosition; + } + const UsefulBufC PartialResult = + UsefulOutBuf_OutUBuf(&(me->OutBuf)); + *pWrappedCBOR = UsefulBuf_Tail(PartialResult, uStartOfNew); + } +} + +/* + * Public function for canceling a bstr wrap. See qcbor/qcbor_encode.h + */ +void +QCBOREncode_CancelBstrWrap(QCBOREncodeContext *pMe) +{ + if (CheckDecreaseNesting(pMe, CBOR_MAJOR_TYPE_BYTE_STRING)) { + return; + } + +#ifndef QCBOR_DISABLE_ENCODE_USAGE_GUARDS + const size_t uCurrent = UsefulOutBuf_GetEndPosition(&(pMe->OutBuf)); + if (pMe->nesting.pCurrentNesting->uStart != uCurrent) { + pMe->uError = QCBOR_ERR_CANNOT_CANCEL; + return; + } + /* QCBOREncode_CancelBstrWrap() can't correctly undo + * QCBOREncode_BstrWrapInMap() or QCBOREncode_BstrWrapInMapN(). It + * can't undo the labels they add. It also doesn't catch the error + * of using it this way. QCBOREncode_CancelBstrWrap() is used + * infrequently and the the result is incorrect CBOR, not a + * security hole, so no extra code or state is added to handle this + * condition. + */ +#endif /* QCBOR_DISABLE_ENCODE_USAGE_GUARDS */ + + Nesting_Decrease(&(pMe->nesting)); + Nesting_Decrement(&(pMe->nesting)); +} + +/* + * Public function for opening a byte string. See qcbor/qcbor_encode.h + */ +void +QCBOREncode_OpenBytes(QCBOREncodeContext *pMe, UsefulBuf *pPlace) +{ + *pPlace = UsefulOutBuf_GetOutPlace(&(pMe->OutBuf)); +#ifndef QCBOR_DISABLE_ENCODE_USAGE_GUARDS + // TODO: is this right? + uint8_t uMajorType = Nesting_GetMajorType(&(pMe->nesting)); + if (uMajorType == CBOR_MAJOR_NONE_TYPE_OPEN_BSTR) { + pMe->uError = QCBOR_ERR_OPEN_BYTE_STRING; + return; + } +#endif /* QCBOR_DISABLE_ENCODE_USAGE_GUARDS */ + + QCBOREncode_OpenMapOrArray(pMe, CBOR_MAJOR_NONE_TYPE_OPEN_BSTR); +} + +/* + * Public function for closing a byte string. See qcbor/qcbor_encode.h + */ +void +QCBOREncode_CloseBytes(QCBOREncodeContext *pMe, const size_t uAmount) +{ + UsefulOutBuf_Advance(&(pMe->OutBuf), uAmount); + if (UsefulOutBuf_GetError(&(pMe->OutBuf))) { + /* Advance too far. Normal off-end error handling in effect + * here. */ + return; + } + + InsertCBORHead(pMe, CBOR_MAJOR_NONE_TYPE_OPEN_BSTR, uAmount); +} + +/* + * Public function for closing arrays and maps. See qcbor/qcbor_encode.h + */ +void +QCBOREncode_CloseMapOrArrayIndefiniteLength(QCBOREncodeContext *pMe, + uint8_t uMajorType) +{ + if (CheckDecreaseNesting(pMe, uMajorType)) { + return; + } + + /* Append the break marker (0xff for both arrays and maps) */ + AppendCBORHead(pMe, CBOR_MAJOR_NONE_TYPE_SIMPLE_BREAK, + CBOR_SIMPLE_BREAK, 0); + Nesting_Decrease(&(pMe->nesting)); +} + +/* + * Public function to finish and get the encoded result. See + * qcbor/qcbor_encode.h + */ +QCBORError +QCBOREncode_Finish(QCBOREncodeContext *me, UsefulBufC *pEncodedCBOR) +{ + QCBORError uReturn = QCBOREncode_GetErrorState(me); + + if (uReturn != QCBOR_SUCCESS) { + goto Done; + } + +#ifndef QCBOR_DISABLE_ENCODE_USAGE_GUARDS + if (Nesting_IsInNest(&(me->nesting))) { + uReturn = QCBOR_ERR_ARRAY_OR_MAP_STILL_OPEN; + goto Done; + } +#endif /* QCBOR_DISABLE_ENCODE_USAGE_GUARDS */ + + *pEncodedCBOR = UsefulOutBuf_OutUBuf(&(me->OutBuf)); + +Done: + return uReturn; +} + +/* + * Public functions to get size of the encoded result. See qcbor/qcbor_encode.h + */ +QCBORError +QCBOREncode_FinishGetSize(QCBOREncodeContext *me, size_t *puEncodedLen) +{ + UsefulBufC Enc; + + QCBORError nReturn = QCBOREncode_Finish(me, &Enc); + + if (nReturn == QCBOR_SUCCESS) { + *puEncodedLen = Enc.len; + } + + return nReturn; +} diff --git a/hyp/misc/smc_trace/build.conf b/hyp/misc/smc_trace/build.conf index bdff37f..740c9ac 100644 --- a/hyp/misc/smc_trace/build.conf +++ b/hyp/misc/smc_trace/build.conf @@ -6,4 +6,4 @@ interface smc_trace source smc_trace.c types smc_trace.tc # 32KB SMC trace size -configs HYP_SMC_LOG_NUM=255 +configs HYP_SMC_LOG_NUM=255U diff --git a/hyp/misc/smc_trace/smc_trace.tc b/hyp/misc/smc_trace/smc_trace.tc index 0076556..e4d0e58 100644 --- a/hyp/misc/smc_trace/smc_trace.tc +++ b/hyp/misc/smc_trace/smc_trace.tc @@ -10,10 +10,10 @@ define smc_trace_entry structure { pcpu uint8; vcpu uint8; vmid type vmid_t; - res0 uint8(const); - regs uint8; // number of used x registers below x array(14) type register_t; timestamp uint64; + regs uint8; // The number of X registers used above + res0 uint8(const); }; define smc_trace structure(aligned(64)) { diff --git a/hyp/misc/smc_trace/src/smc_trace.c b/hyp/misc/smc_trace/src/smc_trace.c index d012df8..b708d35 100644 --- a/hyp/misc/smc_trace/src/smc_trace.c +++ b/hyp/misc/smc_trace/src/smc_trace.c @@ -9,6 +9,7 @@ #include +#include #include #include #include @@ -17,12 +18,11 @@ #include #include +#include #include -#include - extern smc_trace_t *hyp_smc_trace; -smc_trace_t *hyp_smc_trace; +smc_trace_t *hyp_smc_trace; void smc_trace_init(partition_t *partition) @@ -32,13 +32,14 @@ smc_trace_init(partition_t *partition) } void_ptr_result_t alloc_ret = partition_alloc( - partition, sizeof(smc_trace_t), alignof(smc_trace_t)); + partition, sizeof(*hyp_smc_trace), alignof(*hyp_smc_trace)); if (alloc_ret.e != OK) { panic("Error allocating smc trace buffer"); } hyp_smc_trace = (smc_trace_t *)alloc_ret.r; - memset(hyp_smc_trace, 0, sizeof(*hyp_smc_trace)); + (void)memset_s(hyp_smc_trace, sizeof(*hyp_smc_trace), 0, + sizeof(*hyp_smc_trace)); } void @@ -52,14 +53,14 @@ smc_trace_log(smc_trace_id_t id, register_t (*registers)[SMC_TRACE_REG_MAX], assert(num_registers <= SMC_TRACE_REG_MAX); uint64_t timestamp = arch_get_timestamp(); - cpu_index_t pcpu = cpulocal_get_index(); + cpu_index_t pcpu = cpulocal_get_index_unsafe(); cpu_index_t vcpu = 0U; vmid_t vmid = 0U; #if defined(INTERFACE_VCPU) thread_t *current = thread_get_self(); - if (current->kind == THREAD_KIND_VCPU) { + if (compiler_expected(current->kind == THREAD_KIND_VCPU)) { assert(current->addrspace != NULL); vmid = current->addrspace->vmid; vcpu = current->psci_index; @@ -69,10 +70,10 @@ smc_trace_log(smc_trace_id_t id, register_t (*registers)[SMC_TRACE_REG_MAX], index_t cur_idx = atomic_fetch_add_explicit(&hyp_smc_trace->next_idx, 1, memory_order_consume); if (cur_idx >= HYP_SMC_LOG_NUM) { - index_t next_idx = cur_idx + 1; + index_t next_idx = cur_idx + 1U; cur_idx -= HYP_SMC_LOG_NUM; (void)atomic_compare_exchange_strong_explicit( - &hyp_smc_trace->next_idx, &next_idx, cur_idx + 1, + &hyp_smc_trace->next_idx, &next_idx, cur_idx + 1U, memory_order_relaxed, memory_order_relaxed); } assert(cur_idx < HYP_SMC_LOG_NUM); @@ -84,7 +85,7 @@ smc_trace_log(smc_trace_id_t id, register_t (*registers)[SMC_TRACE_REG_MAX], prefetch_store_stream(entry); - entry->id = (uint8_t)id; + entry->id = id; entry->pcpu = (uint8_t)pcpu; entry->vcpu = (uint8_t)vcpu; entry->vmid = vmid; diff --git a/hyp/misc/spectre_arm/build.conf b/hyp/misc/spectre_arm/build.conf new file mode 100644 index 0000000..becf7aa --- /dev/null +++ b/hyp/misc/spectre_arm/build.conf @@ -0,0 +1,30 @@ +# © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +configs SPECTRE_CORTEX_A53_NO_SPECULATION=1 +configs SPECTRE_CORTEX_A55_NO_SPECULATION=1 +configs SPECTRE_QEMU_NO_SPECULATION=1 +configs SPECTRE_CORTEX_A57_BHB_LOOP_FLUSH=8 +configs SPECTRE_CORTEX_A72_BHB_LOOP_FLUSH=8 +configs SPECTRE_CORTEX_A73_BPIALL=1 +configs SPECTRE_CORTEX_A75_BPIALL=1 +configs SPECTRE_CORTEX_A76_BHB_LOOP_FLUSH=24 +configs SPECTRE_CORTEX_A76AE_BHB_LOOP_FLUSH=24 +configs SPECTRE_CORTEX_A77_BHB_LOOP_FLUSH=24 +configs SPECTRE_CORTEX_A78_BHB_LOOP_FLUSH=32 +configs SPECTRE_CORTEX_A78AE_BHB_LOOP_FLUSH=32 +configs SPECTRE_CORTEX_A78C_BHB_LOOP_FLUSH=32 +configs SPECTRE_CORTEX_X1_BHB_LOOP_FLUSH=32 +configs SPECTRE_CORTEX_X2_BHB_LOOP_FLUSH=32 +configs SPECTRE_CORTEX_A510_NO_SPECULATION=1 +configs SPECTRE_CORTEX_A710_BHB_LOOP_FLUSH=32 +configs SPECTRE_CORTEX_A715_BHB_LOOP_FLUSH=38 +configs SPECTRE_NEOVERSE_N1_BHB_LOOP_FLUSH=24 +configs SPECTRE_NEOVERSE_N2_BHB_LOOP_FLUSH=32 +configs SPECTRE_NEOVERSE_V1_BHB_LOOP_FLUSH=32 +configs SPECTRE_CORTEX_X3_BHB_LOOP_FLUSH=132 + +# Unknown cores, we expect a HW workaround is present. We check the ID +# registers and panic if this isn't true. +configs SPECTRE_UNKNOWN_BHB_HW_WORKAROUND=1 diff --git a/hyp/misc/trace_standard/src/trace.c b/hyp/misc/trace_standard/src/trace.c index 18dec96..3475308 100644 --- a/hyp/misc/trace_standard/src/trace.c +++ b/hyp/misc/trace_standard/src/trace.c @@ -56,14 +56,15 @@ trace_init_common(partition_t *partition, void *base, size_t size, { count_t global_entries, local_entries; - assert(size != 0); + assert(size != 0U); assert(base != NULL); - assert(buffer_count != 0); + assert(buffer_count != 0U); - if (buffer_count == 1) { + if (buffer_count == 1U) { // Allocate all the area to the global buffer - global_entries = (count_t)(size / TRACE_BUFFER_ENTRY_SIZE); - local_entries = 0; + global_entries = + (count_t)(size / (size_t)TRACE_BUFFER_ENTRY_SIZE); + local_entries = 0; } else { // Ensure the count is one global buffer + one per each CPU assert(buffer_count == TRACE_BUFFER_NUM); @@ -71,8 +72,10 @@ trace_init_common(partition_t *partition, void *base, size_t size, // to the size reserved for each local buffer assert(size >= (PER_CPU_TRACE_ENTRIES * TRACE_BUFFER_ENTRY_SIZE * TRACE_BUFFER_NUM)); - global_entries = (count_t)(size / TRACE_BUFFER_ENTRY_SIZE) - - (PER_CPU_TRACE_ENTRIES * PLATFORM_MAX_CORES); + global_entries = + (count_t)((size / (size_t)TRACE_BUFFER_ENTRY_SIZE) - + ((size_t)PER_CPU_TRACE_ENTRIES * + PLATFORM_MAX_CORES)); local_entries = PER_CPU_TRACE_ENTRIES; } @@ -83,15 +86,15 @@ trace_init_common(partition_t *partition, void *base, size_t size, count_t entries; trace_buffer_header_t *ptr = (trace_buffer_header_t *)base; for (count_t i = 0U; i < buffer_count; i++) { - if (i == 0) { + if (i == 0U) { entries = global_entries; } else { entries = local_entries; } trace_buffer_header_t *tb = ptr; ptr += entries; - memset(tb, 0, sizeof(*tb)); + *tb = (trace_buffer_header_t){ 0U }; tb->buf_magic = TRACE_MAGIC_BUFFER; tb->entries = entries - 1U; tb->not_wrapped = true; @@ -103,7 +106,7 @@ trace_init_common(partition_t *partition, void *base, size_t size, hyp_trace.num_bufs = buffer_count; // Total size of the trace buffer, in units of 64 bytes. - hyp_trace.area_size_64 = (uint32_t)(size / 64); + hyp_trace.area_size_64 = (uint32_t)(size / 64U); } void @@ -112,7 +115,8 @@ trace_boot_init(void) register_t flags = 0U; hyp_trace.flags = trace_control_flags_default(); - trace_control_flags_set_format(&hyp_trace.flags, TRACE_FORMAT); + trace_control_flags_set_format(&hyp_trace.flags, + (trace_format_t)TRACE_FORMAT); // Default to enable trace buffer and error traces TRACE_SET_CLASS(flags, ERROR); @@ -128,32 +132,30 @@ trace_boot_init(void) atomic_init(&hyp_trace.enabled_class_flags, flags); // Setup internal flags that cannot be changed by hypercalls - trace_public_class_flags = ~(0U); + trace_public_class_flags = ~(register_t)0; + TRACE_CLEAR_CLASS(trace_public_class_flags, ERROR); TRACE_CLEAR_CLASS(trace_public_class_flags, LOG_BUFFER); TRACE_CLEAR_CLASS(trace_public_class_flags, LOG_TRACE_BUFFER); - trace_init_common(partition_get_private(), &trace_boot_buffer, - TRACE_BOOT_ENTRIES * TRACE_BUFFER_ENTRY_SIZE, 1U, - &trace_buffer_global); + trace_init_common( + partition_get_private(), &trace_boot_buffer, + ((size_t)TRACE_BOOT_ENTRIES * (size_t)TRACE_BUFFER_ENTRY_SIZE), + 1U, &trace_buffer_global); } -void -trace_init(partition_t *partition, size_t size) +static void +trace_buffer_init(partition_t *partition, void *base, size_t size) + REQUIRE_PREEMPT_DISABLED { assert(size != 0); - - void_ptr_result_t alloc_ret = partition_alloc( - partition, size, alignof(trace_buffer_header_t)); - if (alloc_ret.e != OK) { - panic("Error allocating trace buffer"); - } + assert(base != NULL); trace_buffer_header_t *tbs[TRACE_BUFFER_NUM]; - trace_init_common(partition, alloc_ret.r, size, TRACE_BUFFER_NUM, tbs); + trace_init_common(partition, base, size, TRACE_BUFFER_NUM, tbs); // The global buffer will be the first, followed by the local buffers trace_buffer_global = tbs[0]; for (cpu_index_t i = 0U; i < PLATFORM_MAX_CORES; i++) { - bitmap_set(tbs[i + 1]->cpu_mask, i); + bitmap_set(tbs[i + 1U]->cpu_mask, i); // The global buffer is first, hence the increment by 1 CPULOCAL_BY_INDEX(trace_buffer, i) = tbs[i + 1]; } @@ -169,15 +171,12 @@ trace_init(partition_t *partition, size_t size) index_t head = atomic_load_explicit(&tb->head, memory_order_relaxed); size_t cpy_size = head * sizeof(trace_buffer_entry_t); - if (cpy_size) { - trace_buffer_entry_t *src_buf = - (trace_buffer_entry_t *)((uintptr_t)tb + - TRACE_BUFFER_HEADER_SIZE); - trace_buffer_entry_t *dst_buf = - (trace_buffer_entry_t *)((uintptr_t)trace_buffer + - TRACE_BUFFER_HEADER_SIZE); + if (cpy_size != 0U) { + // The log entries follow on immediately after the header + char *src_buf = (char *)(tb + 1); + char *dst_buf = (char *)(trace_buffer + 1); - memcpy(dst_buf, src_buf, cpy_size); + (void)memcpy(dst_buf, src_buf, cpy_size); CACHE_CLEAN_INVALIDATE_RANGE(dst_buf, cpy_size); } @@ -185,6 +184,33 @@ trace_init(partition_t *partition, size_t size) atomic_store_release(&trace_buffer->head, head); } +#if defined(PLATFORM_TRACE_STANDALONE_REGION) +void +trace_single_region_init(partition_t *partition, paddr_t base, size_t size) +{ + assert(size != 0); + assert(base != 0); + + // Call to initiaize the trace buffer + trace_buffer_init(partition, (void *)base, size); +} +#else +void +trace_init(partition_t *partition, size_t size) +{ + assert(size != 0); + + void_ptr_result_t alloc_ret = partition_alloc( + partition, size, alignof(trace_buffer_header_t)); + if (alloc_ret.e != OK) { + panic("Error allocating trace buffer"); + } + + // Call to initiaize the trace buffer + trace_buffer_init(partition, alloc_ret.r, size); +} +#endif + // Log a trace with specified trace class. // // id: ID of this trace event. @@ -222,7 +248,7 @@ trace_standard_handle_trace_log(trace_id_t id, trace_action_t action, goto out; } - cpu_id = cpulocal_get_index(); + cpu_id = cpulocal_get_index_unsafe(); timestamp = arch_get_timestamp(); trace_info_init(&trace_info); @@ -252,19 +278,19 @@ trace_standard_handle_trace_log(trace_id_t id, trace_action_t action, // Atomically grab the next entry in the buffer head = atomic_fetch_add_explicit(&tb->head, 1, memory_order_consume); if (compiler_unexpected(head >= entries)) { - index_t new_head = head + 1; + index_t new_head = head + 1U; tb->not_wrapped = false; head -= entries; (void)atomic_compare_exchange_strong_explicit( - &tb->head, &new_head, head + 1, memory_order_relaxed, + &tb->head, &new_head, head + 1U, memory_order_relaxed, memory_order_relaxed); } trace_buffer_entry_t *buffers = (trace_buffer_entry_t *)((uintptr_t)tb + - TRACE_BUFFER_HEADER_SIZE); + (uintptr_t)TRACE_BUFFER_HEADER_SIZE); #if defined(ARCH_ARM) && defined(ARCH_IS_64BIT) && ARCH_IS_64BIT // Store using non-temporal store instructions. Also, if the entry @@ -312,15 +338,15 @@ trace_standard_handle_trace_log(trace_id_t id, trace_action_t action, void trace_set_class_flags(register_t flags) { - atomic_fetch_or_explicit(&hyp_trace.enabled_class_flags, flags, - memory_order_relaxed); + (void)atomic_fetch_or_explicit(&hyp_trace.enabled_class_flags, flags, + memory_order_relaxed); } void trace_clear_class_flags(register_t flags) { - atomic_fetch_and_explicit(&hyp_trace.enabled_class_flags, ~flags, - memory_order_relaxed); + (void)atomic_fetch_and_explicit(&hyp_trace.enabled_class_flags, ~flags, + memory_order_relaxed); } void @@ -339,7 +365,7 @@ trace_update_class_flags(register_t set_flags, register_t clear_flags) } register_t -trace_get_class_flags() +trace_get_class_flags(void) { return atomic_load_explicit(&hyp_trace.enabled_class_flags, memory_order_relaxed); diff --git a/hyp/misc/trace_standard/test/basic_test.c b/hyp/misc/trace_standard/test/basic_test.c index e4de1e8..d83f3ab 100644 --- a/hyp/misc/trace_standard/test/basic_test.c +++ b/hyp/misc/trace_standard/test/basic_test.c @@ -38,8 +38,9 @@ cpulocal_get_index(void) int i = 0; for (i = 0; i < THREAD_CNT; ++i) { - if (pthread_equal(tid[i], id)) + if (pthread_equal(tid[i], id)) { break; + } } return i; @@ -83,7 +84,7 @@ void * thread_run(void *val) { int i = 0; - int *tid = (int *)val; + int *tid = (int *)val; trace_class_t class_map = 0L; trace_id_t id = 0; diff --git a/hyp/misc/trace_standard/trace.tc b/hyp/misc/trace_standard/trace.tc index 0dbe572..592e804 100644 --- a/hyp/misc/trace_standard/trace.tc +++ b/hyp/misc/trace_standard/trace.tc @@ -13,8 +13,8 @@ define TRACE_VERSION constant = 0x0100; #include extend trace_class enumeration { - TRACE_BUFFER; - TRACE_LOG_BUFFER; // Put the log messages in the trace buffer + TRACE_BUFFER = 3; + TRACE_LOG_BUFFER = 4; // Put the log messages in the trace buffer }; define trace_format enumeration { diff --git a/hyp/misc/vet/build.conf b/hyp/misc/vet/build.conf new file mode 100644 index 0000000..5ace749 --- /dev/null +++ b/hyp/misc/vet/build.conf @@ -0,0 +1,8 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +interface vet +types vet.tc +events vet.ev +source vet.c diff --git a/hyp/misc/vet/src/vet.c b/hyp/misc/vet/src/vet.c new file mode 100644 index 0000000..b40919b --- /dev/null +++ b/hyp/misc/vet/src/vet.c @@ -0,0 +1,162 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "event_handlers.h" + +asm_ordering_dummy_t vet_ordering; + +static bool trace_disabled = false; + +void +vet_handle_boot_cold_init(void) +{ + platform_cpu_features_t features = platform_get_cpu_features(); + + trace_disabled = platform_cpu_features_get_trace_disable(&features); + if (trace_disabled) { + LOG(ERROR, INFO, "trace disabled"); + } +} + +error_t +vet_handle_thread_context_switch_pre(void) +{ + thread_t *vcpu = thread_get_self(); + assert(vcpu != NULL); + + if (vcpu_option_flags_get_trace_allowed(&vcpu->vcpu_options)) { + vet_update_trace_unit_status(vcpu); + + if (vcpu->vet_trace_unit_enabled) { + vet_flush_trace(vcpu); + vet_disable_trace(); + vet_save_trace_thread_context(vcpu); + } + + vet_update_trace_buffer_status(vcpu); + + if (vcpu->vet_trace_buffer_enabled) { + vet_flush_buffer(vcpu); + vet_disable_buffer(); + vet_save_buffer_thread_context(vcpu); + } + } + + return OK; +} + +void +vet_handle_thread_load_state(void) +{ + thread_t *vcpu = thread_get_self(); + assert(vcpu != NULL); + + if (vcpu_option_flags_get_trace_allowed(&vcpu->vcpu_options)) { + if (vcpu->vet_trace_buffer_enabled) { + vet_restore_buffer_thread_context(vcpu); + vet_enable_buffer(); + } + + if (vcpu->vet_trace_unit_enabled) { + vet_restore_trace_thread_context(vcpu); + vet_enable_trace(); + } + } +} + +bool +vet_handle_vcpu_activate_thread(thread_t *thread, vcpu_option_flags_t options) +{ + bool ret; + + assert(thread->kind == THREAD_KIND_VCPU); + + bool hlos = vcpu_option_flags_get_hlos_vm(&options); + bool trace_allowed = vcpu_option_flags_get_trace_allowed(&options); + + // TODO: currently we always give HLOS trace access. + if (trace_allowed && trace_disabled) { + // Not permitted + ret = false; + } else if (hlos && !trace_disabled) { + // Give HLOS threads trace access + vcpu_option_flags_set_trace_allowed(&thread->vcpu_options, + true); + ret = true; + } else if (!hlos && trace_allowed) { + // Not supported + ret = false; + } else { + ret = true; + } + + return ret; +} + +error_t +vet_handle_power_cpu_suspend(bool may_poweroff) +{ + assert_cpulocal_safe(); + rcu_read_start(); + + thread_t *vcpu = scheduler_get_primary_vcpu(cpulocal_get_index()); + + if (may_poweroff && (vcpu != NULL) && vcpu->vet_trace_buffer_enabled) { + vet_save_buffer_power_context(); + } + + if ((vcpu != NULL) && vcpu->vet_trace_unit_enabled) { + vet_save_trace_power_context(may_poweroff); + } + + rcu_read_finish(); + return OK; +} + +void +vet_unwind_power_cpu_suspend(void) +{ + assert_cpulocal_safe(); + rcu_read_start(); + + thread_t *vcpu = scheduler_get_primary_vcpu(cpulocal_get_index()); + + if ((vcpu != NULL) && vcpu->vet_trace_unit_enabled) { + vet_restore_trace_power_context(false); + } + + rcu_read_finish(); +} + +void +vet_handle_power_cpu_resume(bool was_poweroff) +{ + assert_cpulocal_safe(); + rcu_read_start(); + + thread_t *vcpu = scheduler_get_primary_vcpu(cpulocal_get_index()); + + if (was_poweroff && (vcpu != NULL) && vcpu->vet_trace_buffer_enabled) { + vet_restore_buffer_power_context(); + } + + if ((vcpu != NULL) && vcpu->vet_trace_unit_enabled) { + vet_restore_trace_power_context(was_poweroff); + } + + rcu_read_finish(); +} diff --git a/hyp/misc/vet/vet.ev b/hyp/misc/vet/vet.ev new file mode 100644 index 0000000..d87eed0 --- /dev/null +++ b/hyp/misc/vet/vet.ev @@ -0,0 +1,22 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module vet + +subscribe boot_cold_init() + +subscribe thread_context_switch_pre() + require_preempt_disabled + +subscribe thread_load_state() + require_preempt_disabled + +subscribe vcpu_activate_thread + +subscribe power_cpu_suspend(may_poweroff) + unwinder() + require_preempt_disabled + +subscribe power_cpu_resume(was_poweroff) + require_preempt_disabled diff --git a/hyp/misc/vet/vet.tc b/hyp/misc/vet/vet.tc new file mode 100644 index 0000000..4f81a37 --- /dev/null +++ b/hyp/misc/vet/vet.tc @@ -0,0 +1,10 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +extend thread object module vet { + // Indicate if the trace unit/buffer is enabled. If it's not enabled, + // we can skip the thread/power context save/restore. + trace_unit_enabled bool; + trace_buffer_enabled bool; +}; diff --git a/hyp/platform/arm_arch_timer/aarch64/src/platform_timer.c b/hyp/platform/arm_arch_timer/aarch64/src/platform_timer.c index 92aec06..d767db9 100644 --- a/hyp/platform/arm_arch_timer/aarch64/src/platform_timer.c +++ b/hyp/platform/arm_arch_timer/aarch64/src/platform_timer.c @@ -30,7 +30,7 @@ static hwirq_t *hyp_timer_hwirq; #endif static void -platform_timer_enable_and_unmask() +platform_timer_enable_and_unmask(void) { CNT_CTL_t cnthp_ctl; @@ -41,7 +41,7 @@ platform_timer_enable_and_unmask() } void -platform_timer_cancel_timeout() +platform_timer_cancel_timeout(void) { CNT_CTL_t cnthp_ctl; @@ -53,13 +53,13 @@ platform_timer_cancel_timeout() } uint32_t -platform_timer_get_frequency() +platform_timer_get_frequency(void) { return PLATFORM_ARCH_TIMER_FREQ; } uint64_t -platform_timer_get_current_ticks() +platform_timer_get_current_ticks(void) { // This register read below is allowed to occur speculatively at any // time after the most recent context sync event. If caller the wants @@ -72,7 +72,7 @@ platform_timer_get_current_ticks() } uint64_t -platform_timer_get_timeout() +platform_timer_get_timeout(void) { CNT_CVAL_t cnthp_cval = register_CNTHP_CVAL_EL2_read_volatile_ordered(&asm_ordering); @@ -94,23 +94,19 @@ platform_timer_set_timeout(ticks_t timeout) ticks_t platform_convert_ns_to_ticks(nanoseconds_t ns) { - __uint128_t ticks = ((__uint128_t)ns * PLATFORM_TIMER_FREQ_SCALE) >> 64; - ticks = ticks << PLATFORM_TIMER_NS_SHIFT; - - return (ticks_t)ticks; + return (ticks_t)((ns * PLATFORM_TIMER_NS_TO_FREQ_MULT) / + PLATFORM_TIMER_FREQ_TO_NS_MULT); } nanoseconds_t platform_convert_ticks_to_ns(ticks_t ticks) { - __uint128_t ns = ((__uint128_t)ticks * PLATFORM_TIMER_NS_SCALE) >> 64; - ns = ns << PLATFORM_TIMER_FREQ_SHIFT; - - return (nanoseconds_t)ns; + return (nanoseconds_t)((ticks * PLATFORM_TIMER_FREQ_TO_NS_MULT) / + PLATFORM_TIMER_NS_TO_FREQ_MULT); } void -platform_timer_handle_boot_cpu_cold_init() +platform_timer_handle_boot_cpu_cold_init(void) { CNTFRQ_EL0_t cntfrq = register_CNTFRQ_EL0_read(); assert(CNTFRQ_EL0_get_ClockFrequency(&cntfrq) == @@ -125,7 +121,7 @@ platform_timer_handle_boot_cpu_cold_init() #if !defined(IRQ_NULL) void -platform_timer_handle_boot_hypervisor_start() +platform_timer_handle_boot_hypervisor_start(void) { // Create the hyp arch timer IRQ hwirq_create_t params = { @@ -166,7 +162,8 @@ platform_timer_ndelay(nanoseconds_t duration) // NOTE: assume we don't have overflow case since it covers huge range. // And assumes the timer is always enabled/configured correctly. - while (platform_timer_get_current_ticks() < target_ticks) - ; + while (platform_timer_get_current_ticks() < target_ticks) { + // Wait for the delay period + } } #endif diff --git a/hyp/platform/arm_arch_timer/templates/platform_timer_consts.h.tmpl b/hyp/platform/arm_arch_timer/templates/platform_timer_consts.h.tmpl index 7ce2e93..1cdf02a 100644 --- a/hyp/platform/arm_arch_timer/templates/platform_timer_consts.h.tmpl +++ b/hyp/platform/arm_arch_timer/templates/platform_timer_consts.h.tmpl @@ -4,23 +4,17 @@ // // SPDX-License-Identifier: BSD-3-Clause +#import math + #set $ns_in_s = 1000000000 -#set freq_shift = 0 -#set ns_shift = 0 -#set $freq = $PLATFORM_ARCH_TIMER_FREQ -#while freq / ($ns_in_s << $ns_shift) > 1 -#set ns_shift += 1 -#end while -#while $ns_in_s / ($freq << $freq_shift) > 1 -#set freq_shift += 1 -#end while +#set $gcd = math.gcd(ns_in_s, $PLATFORM_ARCH_TIMER_FREQ) -\#define NS_IN_S (uint64_t)${ns_in_s}U +#set $ns_to_freq_mult = $PLATFORM_ARCH_TIMER_FREQ//$gcd +#set $freq_to_ns_mult = $ns_in_s//$gcd -\#define PLATFORM_TIMER_FREQ_SHIFT $freq_shift -\#define PLATFORM_TIMER_NS_SHIFT $ns_shift +## Ensure that we can have at least 20 yrs uptime without timer overflow +#set $secs_per_year = 60*60*24*365.25 +#assert ($PLATFORM_ARCH_TIMER_FREQ * $freq_to_ns_mult * $secs_per_year * 20) < (1 << 64) -\#define PLATFORM_TIMER_FREQ_SCALE \ - (uint64_t)(((__uint128_t)$freq << 64) / ($ns_in_s << $ns_shift)) -\#define PLATFORM_TIMER_NS_SCALE \ - (uint64_t)(((__uint128_t)$ns_in_s << 64) / ($freq << $freq_shift)) +\#define PLATFORM_TIMER_NS_TO_FREQ_MULT (uint64_t)$ns_to_freq_mult +\#define PLATFORM_TIMER_FREQ_TO_NS_MULT (uint64_t)$freq_to_ns_mult diff --git a/hyp/platform/arm_arch_timer_lp/build.conf b/hyp/platform/arm_arch_timer_lp/build.conf new file mode 100644 index 0000000..437bd74 --- /dev/null +++ b/hyp/platform/arm_arch_timer_lp/build.conf @@ -0,0 +1,9 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +interface platform +base_module hyp/platform/gicv3 +types platform_timer_lp.tc platform_timer_lp-regs.tc +events platform_timer_lp.ev +source platform_timer_lp.c diff --git a/hyp/platform/arm_arch_timer_lp/platform_timer_lp-regs.tc b/hyp/platform/arm_arch_timer_lp/platform_timer_lp-regs.tc new file mode 100644 index 0000000..08a8f8e --- /dev/null +++ b/hyp/platform/arm_arch_timer_lp/platform_timer_lp-regs.tc @@ -0,0 +1,28 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +// Lower-power timer registers and bitfields definitions + +define CNTEL0ACR bitfield<32> { + 0 EL0PCTEN bool; + 1 EL0VCTEN bool; + 7:2 unknown=0; + 8 EL0VTEN bool; + 9 EL0PTEN bool; + 31:10 unknown=0; +}; + +define CNTFRQ bitfield<32> { + 31:0 ClockFrequency uint32; +}; + +#define CNTx_CTL(x) \ +define CNT##x##_CTL bitfield<32> { \ + 0 ENABLE bool; \ + 1 IMASK bool; \ + 2 ISTATUS bool; \ + 31:3 unknown=0; \ +}; +CNTx_CTL(P) +CNTx_CTL(V) diff --git a/hyp/platform/arm_arch_timer_lp/platform_timer_lp.ev b/hyp/platform/arm_arch_timer_lp/platform_timer_lp.ev new file mode 100644 index 0000000..b2c0c83 --- /dev/null +++ b/hyp/platform/arm_arch_timer_lp/platform_timer_lp.ev @@ -0,0 +1,23 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module arm_arch_timer_lp + +subscribe boot_cold_init + handler platform_timer_lp_handle_boot_cold_init() + +subscribe boot_hypervisor_start + handler platform_timer_lp_handle_boot_hypervisor_start + +subscribe irq_received[HWIRQ_ACTION_HYP_TIMER_LP] + handler platform_timer_lp_handle_irq_received() + require_preempt_disabled + +#if defined(MODULE_VM_ROOTVM) +subscribe rootvm_init + handler platform_timer_lp_handle_rootvm_init(root_cspace, hyp_env) + // We need to run after the creation of the device memeextent and when + // when VIRQs can are ready to be bound + priority -11 +#endif diff --git a/hyp/platform/arm_arch_timer_lp/platform_timer_lp.tc b/hyp/platform/arm_arch_timer_lp/platform_timer_lp.tc new file mode 100644 index 0000000..da2bd72 --- /dev/null +++ b/hyp/platform/arm_arch_timer_lp/platform_timer_lp.tc @@ -0,0 +1,21 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +define cntbase structure(aligned(PGTABLE_HYP_PAGE_SIZE)) { + pct @ 0x000 uint64(atomic); + vct @ 0x008 uint64(atomic); + frq @ 0x010 uint32(atomic); + el0acr @ 0x014 bitfield CNTEL0ACR(atomic); + voff @ 0x018 uint64(atomic); + p_cval @ 0x020 uint64(atomic); + p_tval @ 0x028 uint32(atomic); + p_ctl @ 0x02c bitfield CNTP_CTL(atomic); + v_cval @ 0x030 uint64(atomic); + v_tval @ 0x038 uint32(atomic); + v_ctl @ 0x03c bitfield CNTV_CTL(atomic); +}; + +extend hwirq_action enumeration { + hyp_timer_lp; +}; diff --git a/hyp/platform/arm_arch_timer_lp/src/platform_timer_lp.c b/hyp/platform/arm_arch_timer_lp/src/platform_timer_lp.c new file mode 100644 index 0000000..32a5886 --- /dev/null +++ b/hyp/platform/arm_arch_timer_lp/src/platform_timer_lp.c @@ -0,0 +1,217 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "event_handlers.h" +#include "gicv3.h" + +static cntbase_t *hyp_timer_cnt; +static size_t virt_hyp_timer_size; + +static GICD_IROUTER_t current_route; + +static void +platform_timer_lp_enable_and_unmask(void) +{ + CNTP_CTL_t cntp_ctl; + CNTP_CTL_init(&cntp_ctl); + CNTP_CTL_set_ENABLE(&cntp_ctl, true); + CNTP_CTL_set_IMASK(&cntp_ctl, false); + + atomic_store_relaxed(&hyp_timer_cnt->p_ctl, cntp_ctl); +} + +void +platform_timer_lp_set_timeout(ticks_t timeout) +{ + assert_preempt_disabled(); + + atomic_store_relaxed(&hyp_timer_cnt->p_cval, timeout); + platform_timer_lp_enable_and_unmask(); +} + +uint64_t +platform_timer_lp_get_timeout(void) +{ + return atomic_load_relaxed(&hyp_timer_cnt->p_cval); +} + +void +platform_timer_lp_cancel_timeout(void) +{ + CNTP_CTL_t cntp_ctl; + CNTP_CTL_init(&cntp_ctl); + CNTP_CTL_set_ENABLE(&cntp_ctl, false); + CNTP_CTL_set_IMASK(&cntp_ctl, true); + + atomic_store_relaxed(&hyp_timer_cnt->p_ctl, cntp_ctl); +} + +uint32_t +platform_timer_lp_get_frequency(void) +{ + return atomic_load_relaxed(&hyp_timer_cnt->frq); +} + +uint64_t +platform_timer_lp_get_current_ticks(void) +{ + return atomic_load_relaxed(&hyp_timer_cnt->pct); +} + +void +platform_timer_lp_visibility(bool visible) +{ + CNTEL0ACR_t acr = CNTEL0ACR_default(); + + CNTEL0ACR_set_EL0VCTEN(&acr, visible); + CNTEL0ACR_set_EL0VTEN(&acr, visible); + + atomic_store_relaxed(&hyp_timer_cnt->el0acr, acr); +} + +void +platform_timer_lp_handle_boot_cold_init(void) +{ + size_t timer_size = PGTABLE_HYP_PAGE_SIZE; + + // Allocate hyp_timer_cnt + + virt_range_result_t range = hyp_aspace_allocate(timer_size); + if (range.e != OK) { + panic("timer_lp: Allocation failed."); + } + + hyp_timer_cnt = (cntbase_t *)range.r.base; + virt_hyp_timer_size = range.r.size; + + // Map the low power timer + + pgtable_hyp_start(); + + error_t err = pgtable_hyp_map( + partition_get_private(), (uintptr_t)hyp_timer_cnt, timer_size, + PLATFORM_HYP_ARCH_TIMER_LP_BASE, PGTABLE_HYP_MEMTYPE_DEVICE, + PGTABLE_ACCESS_RW, VMSA_SHAREABILITY_NON_SHAREABLE); + if (err != OK) { + panic("timer_lp: Mapping failed."); + } + + pgtable_hyp_commit(); + + assert(platform_timer_lp_get_frequency() == + PLATFORM_ARCH_TIMER_LP_FREQ); + + // In this code we are assuming that the generic timer is in sync with + // the low power timer and therefore, we do not need to convert + // timeouts. + static_assert(PLATFORM_ARCH_TIMER_LP_FREQ == PLATFORM_ARCH_TIMER_FREQ, + "Arch timer and lp timer must have the same frequency"); + + platform_timer_lp_visibility(true); + + // FIXME: + // Unmap/remap the privileged timer frame in the HLOS S2 address space +} + +void +platform_timer_lp_handle_boot_hypervisor_start(void) +{ + // Create the low power timer IRQ + hwirq_create_t params = { + .irq = PLATFORM_HYP_ARCH_TIMER_LP_IRQ, + .action = HWIRQ_ACTION_HYP_TIMER_LP, + }; + + hwirq_ptr_result_t ret = + partition_allocate_hwirq(partition_get_private(), params); + + if (ret.e != OK) { + panic("Failed to create low power timer IRQ"); + } + + if (object_activate_hwirq(ret.r) != OK) { + panic("Failed to activate low power timer IRQ"); + } + + irq_enable(ret.r); +} + +bool +platform_timer_lp_handle_irq_received(void) +{ + trigger_platform_timer_lp_expiry_event(); + + return true; +} + +void +platform_timer_lp_set_timeout_and_route(ticks_t timeout, cpu_index_t cpu_index) +{ + MPIDR_EL1_t mpidr = platform_cpu_index_to_mpidr(cpu_index); + GICD_IROUTER_t phys_route = GICD_IROUTER_default(); + GICD_IROUTER_set_IRM(&phys_route, false); + GICD_IROUTER_set_Aff0(&phys_route, MPIDR_EL1_get_Aff0(&mpidr)); + GICD_IROUTER_set_Aff1(&phys_route, MPIDR_EL1_get_Aff1(&mpidr)); + GICD_IROUTER_set_Aff2(&phys_route, MPIDR_EL1_get_Aff2(&mpidr)); + GICD_IROUTER_set_Aff3(&phys_route, MPIDR_EL1_get_Aff3(&mpidr)); + + if (!GICD_IROUTER_is_equal(current_route, phys_route)) { + if (gicv3_spi_set_route(PLATFORM_HYP_ARCH_TIMER_LP_IRQ, + phys_route) != OK) { + panic("LPTimer: Failed to set the IRQ route!"); + } + current_route = phys_route; + } + + platform_timer_lp_set_timeout(timeout); +} + +#if defined(MODULE_VM_ROOTVM) +void +platform_timer_lp_handle_rootvm_init(cspace_t *cspace, hyp_env_data_t *hyp_env) +{ + memextent_ptr_result_t m = cspace_lookup_memextent( + cspace, hyp_env->device_me_capid, CAP_RIGHTS_MEMEXTENT_DERIVE); + if (compiler_unexpected(m.e != OK)) { + panic("Failed to find device memextent."); + } + + memextent_t *parent = m.r; + + memextent_ptr_result_t me_ret = memextent_derive( + parent, PLATFORM_HYP_ARCH_TIMER_LP_BASE, (size_t)1U << 12, + MEMEXTENT_MEMTYPE_DEVICE, PGTABLE_ACCESS_RW); + if (me_ret.e != OK) { + panic("Failed creation of low power timer memextent"); + } + + object_put_memextent(parent); +} +#endif diff --git a/hyp/platform/arm_dsu/aarch64/src/platform_dsu.c b/hyp/platform/arm_dsu/aarch64/src/platform_dsu.c new file mode 100644 index 0000000..3e87d92 --- /dev/null +++ b/hyp/platform/arm_dsu/aarch64/src/platform_dsu.c @@ -0,0 +1,109 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#if defined(INTERFACE_VCPU) +#include +#endif + +#include +#include + +#include "event_handlers.h" + +#if defined(INTERFACE_VCPU) + +// The module to trap and emulate the read accesses to the cluster register. We +// currently only emulate IMP_CLUSTERIDR_EL1 and treat the rest as RAZ. + +// Before reading any cluster registers we need to apply the DSU SCLK +// gating erratum (2,313,941) workaround, which is executing a dummy +// cache maintenance operation instruction immediately prior to +// accessing the register. +static inline void +platform_dsu_apply_sclk_gating_erratum_workaround(void) REQUIRE_PREEMPT_DISABLED +{ + register_t dummy = 0; + + assert_preempt_disabled(); + __asm__ volatile("DC CIVAC, %[VA]; " + : "+m"(asm_ordering) + : [VA] "r"(&dummy)); +} + +static inline IMP_CLUSTERIDR_EL1_t +register_CLUSTERIDR_EL1_read(void) +{ + IMP_CLUSTERIDR_EL1_t val; + + preempt_disable(); + platform_dsu_apply_sclk_gating_erratum_workaround(); + val = register_IMP_CLUSTERIDR_EL1_read_ordered(&asm_ordering); + preempt_enable(); + + return val; +} + +vcpu_trap_result_t +arm_dsu_handle_vcpu_trap_sysreg_read(ESR_EL2_ISS_MSR_MRS_t iss) +{ + register_t val = 0ULL; // Default action is RAZ + vcpu_trap_result_t ret = VCPU_TRAP_RESULT_EMULATED; + thread_t *thread = thread_get_self(); + + // Assert this is a read + assert(ESR_EL2_ISS_MSR_MRS_get_Direction(&iss)); + + uint8_t reg_num = ESR_EL2_ISS_MSR_MRS_get_Rt(&iss); + + // Remove the fields that are not used in the comparison + ESR_EL2_ISS_MSR_MRS_t temp_iss = iss; + ESR_EL2_ISS_MSR_MRS_set_Rt(&temp_iss, 0U); + ESR_EL2_ISS_MSR_MRS_set_Direction(&temp_iss, false); + + switch (ESR_EL2_ISS_MSR_MRS_raw(temp_iss)) { + case ISS_MRS_MSR_IMP_CLUSTERIDR_EL1: { + IMP_CLUSTERIDR_EL1_t clusteridr = + register_CLUSTERIDR_EL1_read(); + val = IMP_CLUSTERIDR_EL1_raw(clusteridr); + break; + } + default: { + uint8_t crn, crm; + + crn = ESR_EL2_ISS_MSR_MRS_get_CRn(&iss); + crm = ESR_EL2_ISS_MSR_MRS_get_CRm(&iss); + + if ((crn == 15U) && ((crm == 3U) || (crm == 4U))) { + TRACE_AND_LOG(DEBUG, WARN, + "Emulated RAZ for cluster register: ISS " + "{:#x}", + ESR_EL2_ISS_MSR_MRS_raw(iss)); + } else { + ret = VCPU_TRAP_RESULT_UNHANDLED; + } + break; + } + } + + // Update the thread's register + if (ret == VCPU_TRAP_RESULT_EMULATED) { + vcpu_gpr_write(thread, reg_num, val); + } + + return ret; +} + +#endif diff --git a/hyp/platform/arm_dsu/build.conf b/hyp/platform/arm_dsu/build.conf new file mode 100644 index 0000000..fa1cb36 --- /dev/null +++ b/hyp/platform/arm_dsu/build.conf @@ -0,0 +1,8 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +interface platform +types platform_dsu.tc +events platform_dsu.ev +arch_source aarch64 platform_dsu.c diff --git a/hyp/platform/arm_dsu/platform_dsu.ev b/hyp/platform/arm_dsu/platform_dsu.ev new file mode 100644 index 0000000..c9e3a71 --- /dev/null +++ b/hyp/platform/arm_dsu/platform_dsu.ev @@ -0,0 +1,11 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module arm_dsu + +#if defined(INTERFACE_VCPU) + +subscribe vcpu_trap_sysreg_read + +#endif diff --git a/hyp/platform/arm_dsu/platform_dsu.tc b/hyp/platform/arm_dsu/platform_dsu.tc new file mode 100644 index 0000000..6ac8743 --- /dev/null +++ b/hyp/platform/arm_dsu/platform_dsu.tc @@ -0,0 +1,13 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#if defined(INTERFACE_VCPU) + +define IMP_CLUSTERIDR_EL1 bitfield<64> { + 3:0 Revision uint8; + 7:4 Variant uint8; + 63:8 unknown=0; +}; + +#endif diff --git a/hyp/platform/arm_fgt/aarch64/arm_fgt_aarch64.ev b/hyp/platform/arm_fgt/aarch64/arm_fgt_aarch64.ev new file mode 100644 index 0000000..79dfe50 --- /dev/null +++ b/hyp/platform/arm_fgt/aarch64/arm_fgt_aarch64.ev @@ -0,0 +1,9 @@ +// © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module arm_fgt + +#if defined(INTERFACE_VCPU) && defined(ARCH_ARM_FEAT_FGT) && ARCH_ARM_FEAT_FGT +subscribe thread_load_state() +#endif diff --git a/hyp/platform/arm_fgt/aarch64/arm_fgt_aarch64.tc b/hyp/platform/arm_fgt/aarch64/arm_fgt_aarch64.tc new file mode 100644 index 0000000..7b81b56 --- /dev/null +++ b/hyp/platform/arm_fgt/aarch64/arm_fgt_aarch64.tc @@ -0,0 +1,9 @@ +// © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#if defined(INTERFACE_VCPU) && defined(ARCH_ARM_FEAT_FGT) && ARCH_ARM_FEAT_FGT +extend vcpu_el2_registers structure { + hfgwtr_el2 bitfield HFGWTR_EL2; +}; +#endif diff --git a/hyp/platform/arm_fgt/aarch64/src/arm_fgt.c b/hyp/platform/arm_fgt/aarch64/src/arm_fgt.c new file mode 100644 index 0000000..fadb7fb --- /dev/null +++ b/hyp/platform/arm_fgt/aarch64/src/arm_fgt.c @@ -0,0 +1,58 @@ +// © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#if defined(ARCH_ARM_FEAT_FGT) && ARCH_ARM_FEAT_FGT + +#include + +#include +#include +#include +#include + +#include "event_handlers.h" + +bool +arm_fgt_is_allowed(void) +{ +#if defined(PLATFORM_FGT_OPTIONAL) + const global_options_t *global_options = globals_get_options(); + return compiler_expected(global_options_get_fgt(global_options)); +#else + return true; +#endif +} + +void +arm_fgt_handle_boot_cold_init(void) +{ + global_options_t options = global_options_default(); + global_options_set_fgt(&options, true); + globals_set_options(options); + +#if defined(PLATFORM_FGT_OPTIONAL) + // TZ might be restricting access to FGT, check first + platform_cpu_features_t features = platform_get_cpu_features(); + if (platform_cpu_features_get_fgt_disable(&features)) { + globals_clear_options(options); + } +#endif +} + +#if defined(INTERFACE_VCPU) +void +arm_fgt_handle_thread_load_state(void) +{ + thread_t *thread = thread_get_self(); + if (compiler_expected((thread->kind == THREAD_KIND_VCPU) && + arm_fgt_is_allowed())) { + register_HFGWTR_EL2_write(thread->vcpu_regs_el2.hfgwtr_el2); + } +} +#endif + +#endif diff --git a/hyp/platform/arm_fgt/arm_fgt.ev b/hyp/platform/arm_fgt/arm_fgt.ev new file mode 100644 index 0000000..dbcedd3 --- /dev/null +++ b/hyp/platform/arm_fgt/arm_fgt.ev @@ -0,0 +1,9 @@ +// © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module arm_fgt + +#if defined(ARCH_ARM_FEAT_FGT) && ARCH_ARM_FEAT_FGT +subscribe boot_cold_init() +#endif diff --git a/hyp/platform/arm_fgt/build.conf b/hyp/platform/arm_fgt/build.conf new file mode 100644 index 0000000..4c99c12 --- /dev/null +++ b/hyp/platform/arm_fgt/build.conf @@ -0,0 +1,10 @@ +# © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +interface arm_fgt + +events arm_fgt.ev +arch_types aarch64 arm_fgt_aarch64.tc +arch_events aarch64 arm_fgt_aarch64.ev +arch_source aarch64 arm_fgt.c diff --git a/hyp/platform/arm_generic/aarch64/src/cpu.c b/hyp/platform/arm_generic/aarch64/src/cpu.c new file mode 100644 index 0000000..b1d38d4 --- /dev/null +++ b/hyp/platform/arm_generic/aarch64/src/cpu.c @@ -0,0 +1,131 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include + +#include +#include +#include + +platform_mpidr_mapping_t +platform_cpu_get_mpidr_mapping(void) +{ + MPIDR_EL1_t real_mpidr = register_MPIDR_EL1_read(); + + return (platform_mpidr_mapping_t){ + .aff_shift = { PLATFORM_MPIDR_AFF0_SHIFT, + PLATFORM_MPIDR_AFF1_SHIFT, + PLATFORM_MPIDR_AFF2_SHIFT, + PLATFORM_MPIDR_AFF3_SHIFT }, + .aff_mask = { PLATFORM_MPIDR_AFF0_MASK, + PLATFORM_MPIDR_AFF1_MASK, + PLATFORM_MPIDR_AFF2_MASK, + PLATFORM_MPIDR_AFF3_MASK }, + .multi_thread = MPIDR_EL1_get_MT(&real_mpidr), + .uniprocessor = MPIDR_EL1_get_U(&real_mpidr), + }; +} + +MPIDR_EL1_t +platform_cpu_map_index_to_mpidr(const platform_mpidr_mapping_t *mapping, + index_t index) +{ + MPIDR_EL1_t mpidr = MPIDR_EL1_default(); + + assert(mapping->aff_shift[0] < 32U); + assert(mapping->aff_shift[1] < 32U); + assert(mapping->aff_shift[2] < 32U); + assert(mapping->aff_shift[3] < 32U); + + MPIDR_EL1_set_Aff0(&mpidr, (uint8_t)((index >> mapping->aff_shift[0]) & + mapping->aff_mask[0])); + MPIDR_EL1_set_Aff1(&mpidr, (uint8_t)((index >> mapping->aff_shift[1]) & + mapping->aff_mask[1])); + MPIDR_EL1_set_Aff2(&mpidr, (uint8_t)((index >> mapping->aff_shift[2]) & + mapping->aff_mask[2])); + MPIDR_EL1_set_Aff3(&mpidr, (uint8_t)((index >> mapping->aff_shift[3]) & + mapping->aff_mask[3])); + MPIDR_EL1_set_MT(&mpidr, mapping->multi_thread); + MPIDR_EL1_set_U(&mpidr, mapping->uniprocessor); + + return mpidr; +} + +index_t +platform_cpu_map_mpidr_to_index(const platform_mpidr_mapping_t *mapping, + MPIDR_EL1_t mpidr) +{ + index_t index = 0U; + + assert(mapping->aff_shift[0] < 32U); + assert(mapping->aff_shift[1] < 32U); + assert(mapping->aff_shift[2] < 32U); + assert(mapping->aff_shift[3] < 32U); + + index |= ((index_t)MPIDR_EL1_get_Aff0(&mpidr) & + (index_t)mapping->aff_mask[0]) + << mapping->aff_shift[0]; + index |= ((index_t)MPIDR_EL1_get_Aff1(&mpidr) & + (index_t)mapping->aff_mask[1]) + << mapping->aff_shift[1]; + index |= ((index_t)MPIDR_EL1_get_Aff2(&mpidr) & + (index_t)mapping->aff_mask[2]) + << mapping->aff_shift[2]; + index |= ((index_t)MPIDR_EL1_get_Aff3(&mpidr) & + (index_t)mapping->aff_mask[3]) + << mapping->aff_shift[3]; + + return index; +} + +bool +platform_cpu_map_mpidr_valid(const platform_mpidr_mapping_t *mapping, + MPIDR_EL1_t mpidr) +{ + bool valid = true; + + assert(mapping->aff_shift[0] < 32U); + assert(mapping->aff_shift[1] < 32U); + assert(mapping->aff_shift[2] < 32U); + assert(mapping->aff_shift[3] < 32U); + + if ((MPIDR_EL1_get_Aff0(&mpidr) & ~mapping->aff_mask[0]) != 0U) { + valid = false; + } + if ((MPIDR_EL1_get_Aff1(&mpidr) & ~mapping->aff_mask[1]) != 0U) { + valid = false; + } + if ((MPIDR_EL1_get_Aff2(&mpidr) & ~mapping->aff_mask[2]) != 0U) { + valid = false; + } + if ((MPIDR_EL1_get_Aff3(&mpidr) & ~mapping->aff_mask[3]) != 0U) { + valid = false; + } + + return valid; +} + +MPIDR_EL1_t +platform_cpu_index_to_mpidr(index_t index) +{ + platform_mpidr_mapping_t mapping = platform_cpu_get_mpidr_mapping(); + return platform_cpu_map_index_to_mpidr(&mapping, index); +} + +index_t +platform_cpu_mpidr_to_index(MPIDR_EL1_t mpidr) +{ + platform_mpidr_mapping_t mapping = platform_cpu_get_mpidr_mapping(); + return platform_cpu_map_mpidr_to_index(&mapping, mpidr); +} + +bool +platform_cpu_mpidr_valid(MPIDR_EL1_t mpidr) +{ + platform_mpidr_mapping_t mapping = platform_cpu_get_mpidr_mapping(); + return platform_cpu_map_mpidr_valid(&mapping, mpidr); +} diff --git a/hyp/platform/arm_generic/build.conf b/hyp/platform/arm_generic/build.conf new file mode 100644 index 0000000..cbd21ba --- /dev/null +++ b/hyp/platform/arm_generic/build.conf @@ -0,0 +1,7 @@ +# © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +interface platform + +arch_source aarch64 cpu.c diff --git a/hyp/platform/arm_pmu/aarch64/src/platform_pmu.c b/hyp/platform/arm_pmu/aarch64/src/platform_pmu.c index 517eb69..acaec56 100644 --- a/hyp/platform/arm_pmu/aarch64/src/platform_pmu.c +++ b/hyp/platform/arm_pmu/aarch64/src/platform_pmu.c @@ -30,10 +30,10 @@ static hwirq_t *pmu_hwirq; CPULOCAL_DECLARE_STATIC(bool, pmu_irq_active); void -platform_pmu_handle_boot_cpu_cold_init() +platform_pmu_handle_boot_cpu_cold_init(void) { // Disable all the interrupts at the startup - sysreg64_write(PMINTENCLR_EL1, ~0U); + sysreg64_write(PMINTENCLR_EL1, ~0UL); CPULOCAL(pmu_irq_active) = false; if (pmu_hwirq != NULL) { @@ -42,7 +42,7 @@ platform_pmu_handle_boot_cpu_cold_init() } void -platform_pmu_handle_boot_hypervisor_start() +platform_pmu_handle_boot_hypervisor_start(void) { // Create the PMU IRQ hwirq_create_t params = { @@ -67,18 +67,18 @@ platform_pmu_handle_boot_hypervisor_start() } bool -platform_pmu_is_hw_irq_pending() +platform_pmu_is_hw_irq_pending(void) { uint64_t pmovsset, pmintenset; sysreg64_read_ordered(PMINTENSET_EL1, pmintenset, asm_ordering); sysreg64_read_ordered(PMOVSSET_EL0, pmovsset, asm_ordering); - return ((pmovsset & pmintenset) != 0); + return (pmovsset & pmintenset) != 0U; } void -platform_pmu_hw_irq_deactivate() +platform_pmu_hw_irq_deactivate(void) { if (CPULOCAL(pmu_irq_active)) { CPULOCAL(pmu_irq_active) = false; @@ -87,7 +87,7 @@ platform_pmu_hw_irq_deactivate() } error_t -arm_pmu_handle_power_cpu_suspend() +arm_pmu_handle_power_cpu_suspend(void) { platform_pmu_hw_irq_deactivate(); diff --git a/hyp/platform/arm_pmu/platform_pmu.ev b/hyp/platform/arm_pmu/platform_pmu.ev index f7ae64f..629a3f7 100644 --- a/hyp/platform/arm_pmu/platform_pmu.ev +++ b/hyp/platform/arm_pmu/platform_pmu.ev @@ -14,5 +14,7 @@ subscribe boot_hypervisor_start subscribe irq_received[HWIRQ_ACTION_PMU] handler platform_pmu_handle_irq_received() + require_preempt_disabled subscribe power_cpu_suspend() + require_preempt_disabled diff --git a/hyp/platform/arm_rng/aarch64/src/rng.c b/hyp/platform/arm_rng/aarch64/src/rng.c index b253665..ea9b89a 100644 --- a/hyp/platform/arm_rng/aarch64/src/rng.c +++ b/hyp/platform/arm_rng/aarch64/src/rng.c @@ -16,8 +16,8 @@ #include -#if !defined(ARCH_ARM_8_5_RNG) || !ARCH_ARM_8_5_RNG -#error ARCH_ARM_8_5_RNG not set +#if !defined(ARCH_ARM_FEAT_RNG) || !ARCH_ARM_FEAT_RNG +#error ARCH_ARM_FEAT_RNG not set #endif // We use a per-cpu counter in case the implementation is not shared, and we @@ -50,7 +50,8 @@ platform_get_entropy(platform_prng_data256_t *data) } while ((i < 4U) && (retries != 0U)); if (i == 4U) { - memscpy(data, sizeof(*data), &prng_data, sizeof(prng_data)); + (void)memscpy(data, sizeof(*data), &prng_data, + sizeof(prng_data)); ret = OK; // Issue a reseed read, ignoring the result. @@ -101,8 +102,7 @@ platform_get_random32(uint32_t *data) error_t platform_get_rng_uuid(uint32_t data[4]) { - // uuidgen -s -n @url -N "qualcomm.com/gunyah/trng/rndr" - // UUID: 16397d4e-a2ea-5fe2-92a1-433d45546e21 + // Gunyah generic RNDR - ARM TRNG interface UUID data[0] = 0x45546e21U; data[1] = 0x92a1433dU; data[2] = 0xa2ea5fe2U; diff --git a/hyp/platform/arm_smccc/aarch64/src/smccc_call.c b/hyp/platform/arm_smccc/aarch64/src/smccc_call.c index 07d48aa..c6b9d24 100644 --- a/hyp/platform/arm_smccc/aarch64/src/smccc_call.c +++ b/hyp/platform/arm_smccc/aarch64/src/smccc_call.c @@ -29,8 +29,9 @@ smccc_1_1_do_call(smccc_function_id_t fn_id, uint64_t (*args)[6], register_t trace_regs[SMC_TRACE_REG_MAX]; trace_regs[0] = smccc_function_id_raw(fn_id); - memscpy(&trace_regs[1], sizeof(trace_regs) - sizeof(trace_regs[0]), - args, sizeof(*args)); + (void)memscpy(&trace_regs[1], + sizeof(trace_regs) - sizeof(trace_regs[0]), *args, + sizeof(*args)); trace_regs[7] = client_id; smc_trace_log(SMC_TRACE_ID_EL2_64CAL, &trace_regs, 8U); @@ -73,7 +74,7 @@ smccc_1_1_do_call(smccc_function_id_t fn_id, uint64_t (*args)[6], } #if defined(INTERFACE_SMC_TRACE) - memscpy(&trace_regs[0], sizeof(trace_regs), ret, sizeof(*ret)); + (void)memscpy(&trace_regs[0], sizeof(trace_regs), *ret, sizeof(*ret)); trace_regs[4] = 0U; trace_regs[5] = 0U; trace_regs[6] = x6; @@ -90,11 +91,12 @@ smccc_1_1_call(smccc_function_id_t fn_id, uint64_t (*args)[6], assert(ret != NULL); #if defined(INTERFACE_VCPU) - bool is_vcpu = thread_get_self()->kind == THREAD_KIND_VCPU; + bool is_vcpu = (client_id != CLIENT_ID_HYP); bool is_fast = smccc_function_id_get_is_fast(&fn_id); if (is_vcpu && !is_fast) { preempt_disable(); + assert(thread_get_self()->kind == THREAD_KIND_VCPU); bool pending_wakeup = vcpu_block_start(); if (pending_wakeup) { // Assert a local IPI. This notifies secure world of the diff --git a/hyp/platform/arm_trng_fi/src/arm_trng.c b/hyp/platform/arm_trng_fi/src/arm_trng.c index bd7d4aa..2e07902 100644 --- a/hyp/platform/arm_trng_fi/src/arm_trng.c +++ b/hyp/platform/arm_trng_fi/src/arm_trng.c @@ -16,7 +16,7 @@ #include "event_handlers.h" -// FIXME: ABI checks disabled due to spec issue. +// FIXME: ABI checks disabled since Linux driver is non-compliant. #define LINUX_TRNG_WORKAROUND static void NOINLINE @@ -32,12 +32,12 @@ arm_trng_fi_read(vcpu_gpr_t *regs, uint64_t bits, bool smc64) } else { uint32_t data[192 / 32] = { 0 }; count_t remain = (count_t)bits; - int32_t i; + int32_t i = (int32_t)util_array_size(data) - 1; assert(bits <= 192); // Read N-bits of entropy - for (i = util_array_size(data) - 1; remain != 0U; i--) { + while (remain != 0U) { assert(i >= 0); error_t err = platform_get_random32(&data[i]); @@ -46,12 +46,12 @@ arm_trng_fi_read(vcpu_gpr_t *regs, uint64_t bits, bool smc64) } if (remain < 32U) { // Mask any unrequested bits - uint32_t mask = (1U << remain) - 1U; - data[i] &= mask; + data[i] &= (uint32_t)util_mask(remain); remain = 0U; } else { remain -= 32U; } + i--; } if (remain != 0U) { regs->x[0] = (uint64_t)ARM_TRNG_RET_NO_ENTROPY; @@ -68,7 +68,7 @@ arm_trng_fi_read(vcpu_gpr_t *regs, uint64_t bits, bool smc64) regs->x[1] = data[3]; } // Erase entropy from the stack - memset_s(data, sizeof(data), 0U, sizeof(data)); + (void)memset_s(data, sizeof(data), 0, sizeof(data)); CACHE_CLEAN_INVALIDATE_OBJECT(data); regs->x[0] = (uint64_t)ARM_TRNG_RET_SUCCESS; @@ -109,7 +109,7 @@ static bool arm_trng_fi_handle_call(void) { bool handled = false; - thread_t *current = thread_get_self(); + thread_t *current = thread_get_self(); smccc_function_id_t function_id = smccc_function_id_cast((uint32_t)current->vcpu_regs_gpr.x[0]); smccc_interface_id_t interface = @@ -121,12 +121,12 @@ arm_trng_fi_handle_call(void) (!smccc_function_id_get_is_fast(&function_id)))) { goto out; } - if ((function < ARM_TRNG_FUNCTION__MIN) || - (function > ARM_TRNG_FUNCTION__MAX)) { + if ((function < (smccc_function_t)ARM_TRNG_FUNCTION__MIN) || + (function > (smccc_function_t)ARM_TRNG_FUNCTION__MAX)) { goto out; } - arm_trng_function_t f = function; + arm_trng_function_t f = (arm_trng_function_t)function; bool is_smc64 = smccc_function_id_get_is_smc64(&function_id); // Setup the default return @@ -156,6 +156,7 @@ arm_trng_fi_handle_call(void) case ARM_TRNG_FUNCTION_TRNG_GET_UUID: case ARM_TRNG_FUNCTION_LAST_ID: default: + // Unimplemented break; } } else { @@ -191,7 +192,9 @@ arm_trng_fi_handle_call(void) (smccc_function_id_get_res0(&fid) != 0U)) { break; } - switch (smccc_function_id_get_function(&fid)) { + smccc_function_t fn = + smccc_function_id_get_function(&fid); + switch ((arm_trng_function_t)fn) { case ARM_TRNG_FUNCTION_TRNG_VERSION: case ARM_TRNG_FUNCTION_TRNG_FEATURES: case ARM_TRNG_FUNCTION_TRNG_GET_UUID: @@ -204,6 +207,10 @@ arm_trng_fi_handle_call(void) current->vcpu_regs_gpr.x[0] = (uint64_t)ARM_TRNG_RET_SUCCESS; break; + case ARM_TRNG_FUNCTION_LAST_ID: + default: + // Nothing to do + break; } break; } @@ -242,6 +249,7 @@ arm_trng_fi_handle_call(void) break; case ARM_TRNG_FUNCTION_LAST_ID: default: + // Nothing to do break; } } diff --git a/hyp/platform/ete/aarch64/ete.tc b/hyp/platform/ete/aarch64/ete.tc new file mode 100644 index 0000000..ae88775 --- /dev/null +++ b/hyp/platform/ete/aarch64/ete.tc @@ -0,0 +1,145 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + + +define ETE_TRCLAR_UNLOCK constant = 0xC5ACCE55; +define ETE_TRCLAR_LOCK constant = 0x0; + +define TRCSTATR bitfield<64> { + 0 idle bool; + 1 pmstable bool; + others unknown = 0; +}; + +define TRCPRGCTLR bitfield<64> { + 0 EN bool; + others unknown = 0; +}; + +define TRCIDR5 bitfield<64> { + 11:9 NUMEXTINSEL uint8; + 27:25 NUMSEQSTATE uint8; + 30:28 NUMCNTR uint8; + others unknown = 0; +}; + +define TRCIDR4 bitfield<64> { + 3:0 NUMACPAIRS uint8; + 7:4 NUMDVC uint8; + 8 SUPPDAC uint8; + 15:12 NUMPC uint8; + 19:16 NUMRSPAIR uint8; + 23:20 NUMSSCC uint8; + 27:24 NUMCIDC uint8; + 31:28 NUMVMIDC uint8; + others unknown = 0; +}; + +define TRCIDR3 bitfield<64> { + 26 STALLCTL bool; + others unknown = 0; +}; + +define TRCIDR2 bitfield<64> { + 9:5 CIDSIZE uint8; + 14:10 VMIDSIZE uint8; + others unknown = 0; +}; + +define TRCIDR0 bitfield<64> { + 14 QFILT uint8; + others unknown = 0; +}; + +define ete_context structure { + TRCPRGCTLR uint64; + TRCCONFIGR uint64; + TRCAUXCTLR uint64; + TRCEVENTCTL0R uint64; + TRCEVENTCTL1R uint64; + TRCRSR uint64; + + TRCSTALLCTLR uint64; + + TRCTSCTLR uint64; + TRCSYNCPR uint64; + TRCCCCTLR uint64; + TRCBBCTLR uint64; + TRCTRACEIDR uint64; + TRCQCTLR uint64; + TRCVICTLR uint64; + TRCVIIECTLR uint64; + TRCVISSCTLR uint64; + +#if TRCIDR4_NUMPC > 0 + TRCVIPCSSCTLR uint64; +#endif + +#if TRCIDR5_NUMSEQSTATE > 0 + TRCSEQEVR array(3) uint64; +#endif + + TRCSEQRSTEVR uint64; + TRCSEQSTR uint64; + + TRCEXTINSELR array(TRCIDR5_NUMEXTINSEL) uint64; + + TRCCNTRLDVR array(TRCIDR5_NUMCNTR) uint64; + + TRCCNTCTLR array(TRCIDR5_NUMCNTR) uint64; + + TRCCNTVR array(TRCIDR5_NUMCNTR) uint64; + + // following are optional + // TRCIMSPEC1 uint64; + // TRCIMSPEC2 uint64; + // TRCIMSPEC3 uint64; + // TRCIMSPEC4 uint64; + // TRCIMSPEC5 uint64; + // TRCIMSPEC6 uint64; + // TRCIMSPEC7 uint64; + + // special case + // ((TRCIDR4.NUMRSPAIR + 1) * 2) > n + // range 2 - 31 + TRCRSCTLR array(TRCRSCTLR_CNT) uint64; + + TRCSSCCR array(TRCIDR4_NUMSSCC) uint64; + + TRCSSCSR array(TRCIDR4_NUMSSCC) uint64; + +#if TRCIDR4_NUMPC > 0 + // special case + // TRCSSCSR.PC == 0b1 + TRCSSPCICR array(TRCIDR4_NUMPC) uint64; +#endif + + + // TRCIDR4.NUMACPAIRS * 2 > n + TRCACVR array(TRCACVR_CNT) uint64; + + // TRCIDR4.NUMACPAIRS * 2 > n + TRCACATR array(TRCACATR_CNT) uint64; + + TRCCIDCVR array(TRCIDR4_NUMCIDC) uint64; + + TRCVMIDCVR array(TRCIDR4_NUMVMIDC) uint64; + +#if (TRCIDR4_NUMCIDC > 0x0) && (TRCIDR2_CIDSIZE > 0) + TRCCIDCCTLR0 uint64; +#endif + +#if (TRCIDR4_NUMCIDC > 0x4) && (TRCIDR2_CIDSIZE > 0) + TRCCIDCCTLR1 uint64; +#endif + +#if (TRCIDR4_NUMVMIDC > 0x0) && (TRCIDR2_VMIDSIZE > 0) + TRCVMIDCCTLR0 uint64; +#endif + +#if (TRCIDR4_NUMVMIDC > 0x4) && (TRCIDR2_VMIDSIZE > 0) + TRCVMIDCCTLR1 uint64; +#endif + TRFCR_EL1 bitfield TRFCR_EL1; +}; diff --git a/hyp/platform/ete/build.conf b/hyp/platform/ete/build.conf new file mode 100644 index 0000000..6d5db9c --- /dev/null +++ b/hyp/platform/ete/build.conf @@ -0,0 +1,9 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +arch_types aarch64 ete.tc +local_include +template typed ete_save_restore.h +source ete.c +events ete.ev diff --git a/hyp/platform/ete/ete.ev b/hyp/platform/ete/ete.ev new file mode 100644 index 0000000..e87db57 --- /dev/null +++ b/hyp/platform/ete/ete.ev @@ -0,0 +1,7 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module ete + +subscribe boot_cpu_cold_init() diff --git a/hyp/platform/ete/include/ete.h b/hyp/platform/ete/include/ete.h new file mode 100644 index 0000000..186ea7f --- /dev/null +++ b/hyp/platform/ete/include/ete.h @@ -0,0 +1,9 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +void +ete_save_context_percpu(cpu_index_t cpu, bool may_poweroff); + +void +ete_restore_context_percpu(cpu_index_t cpu, bool was_poweroff); diff --git a/hyp/platform/ete/src/ete.c b/hyp/platform/ete/src/ete.c new file mode 100644 index 0000000..517770e --- /dev/null +++ b/hyp/platform/ete/src/ete.c @@ -0,0 +1,139 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "ete.h" +#include "ete_save_restore.h" +#include "event_handlers.h" + +CPULOCAL_DECLARE_STATIC(ete_context_t, ete_contexts); +CPULOCAL_DECLARE_STATIC(uint64_t, ete_claim_tag); + +void +ete_handle_boot_cpu_cold_init(void) +{ + TRCIDR2_t trcidr2 = register_TRCIDR2_read(); + TRCIDR4_t trcidr4 = register_TRCIDR4_read(); + TRCIDR5_t trcidr5 = register_TRCIDR5_read(); + + (void)trcidr2; + assert(TRCIDR2_get_CIDSIZE(&trcidr2) == TRCIDR2_CIDSIZE); + assert(TRCIDR2_get_VMIDSIZE(&trcidr2) == TRCIDR2_VMIDSIZE); + + (void)trcidr4; + assert(TRCIDR4_get_NUMPC(&trcidr4) == TRCIDR4_NUMPC); + + assert(TRCIDR4_get_NUMRSPAIR(&trcidr4) == TRCIDR4_NUMRSPAIR); + assert(TRCIDR4_get_NUMACPAIRS(&trcidr4) == TRCIDR4_NUMACPAIRS); + + assert(TRCIDR4_get_NUMSSCC(&trcidr4) == TRCIDR4_NUMSSCC); + assert(TRCIDR4_get_NUMCIDC(&trcidr4) == TRCIDR4_NUMCIDC); + assert(TRCIDR4_get_NUMVMIDC(&trcidr4) == TRCIDR4_NUMVMIDC); + + (void)trcidr5; + assert(TRCIDR5_get_NUMSEQSTATE(&trcidr5) == TRCIDR5_NUMSEQSTATE); + assert(TRCIDR5_get_NUMEXTINSEL(&trcidr5) == TRCIDR5_NUMEXTINSEL); + assert(TRCIDR5_get_NUMCNTR(&trcidr5) == TRCIDR5_NUMCNTR); +} + +static void +ete_wait_stable(bool wait_pmstable, bool wait_idle) +{ + bool pmstable = false; + bool idle = false; + + // wait 100us + ticks_t start = platform_timer_get_current_ticks(); + ticks_t timeout = start + platform_convert_ns_to_ticks(100U * 1000U); + while (1) { + TRCSTATR_t trcstatr = + register_TRCSTATR_read_ordered(&vet_ordering); + + // should be a volatile read + pmstable = TRCSTATR_get_pmstable(&trcstatr); + idle = TRCSTATR_get_idle(&trcstatr); + + // possible exit conditions: + // * when pmstable and idle are true + // * when idle and not waiting for pmstable + // * when pmstable and not waiting for idle + if ((pmstable && idle) || (idle && (!wait_pmstable)) || + (pmstable && (!wait_idle))) { + break; + } + + if (platform_timer_get_current_ticks() > timeout) { + TRACE_AND_LOG(ERROR, INFO, + "ETE: programmers model is not stable"); + break; + } + } +} + +void +ete_save_context_percpu(cpu_index_t cpu, bool may_poweroff) +{ + // Synchronise the trace unit. EL2 trace is always prohibited, so + // we don't need to prohibit trace first. + __asm__ volatile("tsb csync" : "+m"(vet_ordering)); + + ete_context_t *ctx = &CPULOCAL_BY_INDEX(ete_contexts, cpu); + + // Save and clear TRCPRGCTLR + ctx->TRCPRGCTLR = register_TRCPRGCTLR_read_ordered(&vet_ordering); + register_TRCPRGCTLR_write_ordered(0U, &vet_ordering); + asm_context_sync_ordered(&vet_ordering); + + if (may_poweroff) { + // Wait until the programming interface is stable + ete_wait_stable(true, false); + + // Save the remaining registers + ete_save_registers(ctx, &vet_ordering); + CPULOCAL_BY_INDEX(ete_claim_tag, cpu) = + register_TRCCLAIMCLR_read_ordered(&vet_ordering); + + // Wait until all writes to the trace buffer are complete + ete_wait_stable(false, true); + } else { + // Wait until all writes to the trace buffer are complete + ete_wait_stable(true, true); + } +} + +void +ete_restore_context_percpu(cpu_index_t cpu, bool was_poweroff) +{ + ete_context_t *ctx = &CPULOCAL_BY_INDEX(ete_contexts, cpu); + + if (was_poweroff) { + // Restore all of the registers other than TRCPRGCTLR + ete_restore_registers(ctx, &vet_ordering); + asm_context_sync_ordered(&vet_ordering); + } + + register_TRCPRGCTLR_write_ordered(ctx->TRCPRGCTLR, &vet_ordering); + asm_context_sync_ordered(&vet_ordering); + + register_TRCCLAIMSET_write_ordered( + CPULOCAL_BY_INDEX(ete_claim_tag, cpu), &vet_ordering); +} diff --git a/hyp/platform/ete/templates/ete_save_restore.h.tmpl b/hyp/platform/ete/templates/ete_save_restore.h.tmpl new file mode 100644 index 0000000..63b938c --- /dev/null +++ b/hyp/platform/ete/templates/ete_save_restore.h.tmpl @@ -0,0 +1,160 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +static void +ete_save_registers(ete_context_t *context, + asm_ordering_dummy_t *ordering_var) +{ +#set ete_contexts = [c for c in $definitions \ + if c.category == 'structure' and c.type_name == 'ete_context'] +#assert len($ete_contexts) == 1 +#set ete_context = $ete_contexts[0] + +#for d in $ete_context.declarations + #if d.member_name == "TRCRSCTLR" + #continue + #end if + #set is_array = False + #set array_size = 0 + + #if $d.complex_type + #set current_type = $d.compound_type + + #while $current_type is not None + #if $current_type.is_array + #set is_array = True + #set array_size = $current_type.length + #break + #end if + + #set $current_type = $current_type.base_type + + #if $current_type is None + #break + #end if + #end while + #end if + + #if $is_array + #set idx = 0 + #set reg_offset = 0 + + #if d.member_name == "TRCRSCTLR" + #set reg_offset = 2 + #end if + + #while $idx < $array_size + #set reg_idx = $idx + $reg_offset + + #if d.member_name == "TRCSSPCICR" + uint64_t trcsscsr${reg_idx} = register_TRCSSCSR${reg_idx}_read_ordered(ordering_var); + if (trcsscsr${reg_idx} & 0x8) { + #end if + + context->${d.member_name}[${idx}] = register_${d.member_name}${reg_idx}_read_ordered(ordering_var); + + #if d.member_name == "TRCSSPCICR" + } + #end if + + #set idx = $idx + 1 + #end while + #else + #if d.member_name == "TRCSTALLCTLR" + TRCIDR3_t trcidr3 = register_TRCIDR3_read(); + if (TRCIDR3_get_STALLCTL(&trcidr3)) { + #else if d.member_name == "TRCQCTLR" + TRCIDR0_t trcidr0 = register_TRCIDR0_read(); + if (TRCIDR0_get_QFILT(&trcidr0)) { + #end if + + context->$d.member_name = register_${d.member_name}_read_ordered(ordering_var); + + #if d.member_name == "TRCSTALLCTLR" + } + #else if d.member_name == "TRCQCTLR" + } + #end if + #end if +#end for +} + +static void +ete_restore_registers(ete_context_t *context, + asm_ordering_dummy_t *ordering_var) +{ +#set ete_contexts = [c for c in $definitions \ + if c.category == 'structure' and c.type_name == 'ete_context'] +#assert len($ete_contexts) == 1 +#set ete_context = $ete_contexts[0] + +#for d in $ete_context.declarations + #if d.member_name == "TRCRSCTLR" + #continue + #end if + #set is_array = False + #set array_size = 0 + + #if $d.complex_type + #set current_type = $d.compound_type + + #while $current_type is not None + #if $current_type.is_array + #set is_array = True + #set array_size = $current_type.length + #break + #end if + + #set $current_type = $current_type.base_type + + #if $current_type is None + #break + #end if + #end while + #end if + + #if $is_array + #set idx = 0 + #set reg_offset = 0 + + #if d.member_name == "TRCRSCTLR" + #set reg_offset = 2 + #end if + + #while $idx < $array_size + #set reg_idx = $idx + $reg_offset + + #if d.member_name == "TRCSSPCICR" + uint64_t trcsscsr${reg_idx} = register_TRCSSCSR${reg_idx}_read_ordered(ordering_var); + if (trcsscsr${reg_idx} & 0x8) { + #end if + + register_${d.member_name}${reg_idx}_write_ordered(context->${d.member_name}[${idx}], ordering_var); + + #if d.member_name == "TRCSSPCICR" + } + #end if + + #set idx = $idx + 1 + #end while + #else + #if d.member_name == "TRCSTALLCTLR" + TRCIDR3_t trcidr3 = register_TRCIDR3_read(); + if (TRCIDR3_get_STALLCTL(&trcidr3)) { + #else if d.member_name == "TRCQCTLR" + TRCIDR0_t trcidr0 = register_TRCIDR0_read(); + if (TRCIDR0_get_QFILT(&trcidr0)) { + #end if + + register_${d.member_name}_write_ordered(context->$d.member_name, ordering_var); + + #if d.member_name == "TRCSTALLCTLR" + } + #else if d.member_name == "TRCQCTLR" + } + #end if + + #end if +#end for +} diff --git a/hyp/platform/etm/aarch64/etm-regs.tc b/hyp/platform/etm/aarch64/etm-regs.tc new file mode 100644 index 0000000..20544ef --- /dev/null +++ b/hyp/platform/etm/aarch64/etm-regs.tc @@ -0,0 +1,48 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +define ETM_TRCVI_CTLR_EXLEVEL_S bitfield<4> { + 0 el0 bool = 0x1; + 1 el1 bool = 0x1; + 2 res0 bool; + 3 el3 bool = 0x1; +}; + +define ETM_TRCVI_CTLR_EXLEVEL_NS bitfield<4> { + 0 el0 bool = 0x1; + 1 el1 bool = 0x1; + 2 el2 bool = 0x1; + 3 res0 bool; +}; + +define ETM_TRCVI_CTLR bitfield<32> { + 7:0 event uint8; + 8 res0 bool; + 9 ssstatus bool; + 10 trcreset bool; + 11 trcerr bool; + 15:12 res1 uint8; + 19:16 exlevel_s uint8 = 0xf; + 23:20 exlevel_ns uint8 = 0xf; + 31:24 unknown=0; +}; + +define ETM_EVENT bitfield<8> { + 4:0 sel uint8; + 6:5 res0 uint8; + 7 is_single_resource bool; +}; + + +define ETM_TRCSTATR bitfield<32> { + 0 idle bool; + 1 pmstable bool; + 31:2 res0 uint32; +}; + +define ETM_TRCOSLSR bitfield<32> { + 3,0 oslm uint8; + 1 oslk bool; + 2 ntt bool; +}; diff --git a/hyp/platform/etm/build.conf b/hyp/platform/etm/build.conf new file mode 100644 index 0000000..5e5d555 --- /dev/null +++ b/hyp/platform/etm/build.conf @@ -0,0 +1,9 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +types etm.tc +arch_types aarch64 etm-regs.tc +local_include +events etm.ev +source etm.c diff --git a/hyp/platform/etm/etm.ev b/hyp/platform/etm/etm.ev new file mode 100644 index 0000000..de336e6 --- /dev/null +++ b/hyp/platform/etm/etm.ev @@ -0,0 +1,22 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module etm + +subscribe boot_hypervisor_start + // Run after soc module init so we can query security state + priority -1 + +subscribe power_cpu_online + require_preempt_disabled + +subscribe power_cpu_suspend(may_poweroff) + unwinder(may_poweroff) + require_preempt_disabled + +subscribe power_cpu_resume(was_poweroff) + require_preempt_disabled + +subscribe power_cpu_offline + require_preempt_disabled diff --git a/hyp/platform/etm/etm.tc b/hyp/platform/etm/etm.tc new file mode 100644 index 0000000..0482859 --- /dev/null +++ b/hyp/platform/etm/etm.tc @@ -0,0 +1,80 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +// register offset + +define ETM_TRCLAR_UNLOCK constant = 0xC5ACCE55; +define ETM_TRCLAR_LOCK constant = 0x0; + +define ETM_TRCOSLAR_UNLOCK constant = 0x0; +define ETM_TRCOSLAR_LOCK constant = 0x1; + +define ETM_TRCPRGCTLR_ENABLE constant = 0x1; + +define etm_trcdv structure(aligned(16)) { + value uint64; +}; + +define etm structure(aligned(PGTABLE_HYP_PAGE_SIZE)) { + // main control & configuration regsters + trcprgctlr @ 0x4 uint32(atomic); + trcprocselr @ 0x8 uint32(atomic); + trcstatr @ 0xC bitfield ETM_TRCSTATR(atomic); + trcconfigr @ 0x10 uint32(atomic); + trcauxctlr @ 0x18 uint32(atomic); + trceventctl0r @ 0x20 uint32(atomic); + trceventctl1r @ 0x24 uint32(atomic); + trcstallctlr @ 0x2c uint32(atomic); + trctsctlr @ 0x30 uint32(atomic); + trcsyncpr @ 0x34 uint32(atomic); + trcccctlr @ 0x38 uint32(atomic); + trcbbctlr @ 0x3c uint32(atomic); + trctraceidr @ 0x40 uint32(atomic); + trcqctlr @ 0x44 uint32(atomic); + + // filtering control registers + trcvictlr @ 0x80 uint32(atomic); + trcviiectlr @ 0x84 uint32(atomic); + trcvissctlr @ 0x88 uint32(atomic); + trcvipcssctlr @ 0x8c uint32(atomic); + trcvdctlr @ 0xa0 uint32(atomic); + trcvdsacctlr @ 0xa4 uint32(atomic); + trcvdarcctlr @ 0xa8 uint32(atomic); + + // derived resources registers + trcseqevr @ 0x100 array(3) uint32(atomic); + trcseqrstevr @ 0x118 uint32(atomic); + trcseqstr @ 0x11c uint32(atomic); + trcextinselr @ 0x120 uint32(atomic); + trccntrldvr @ 0x140 array(4) uint32(atomic); + trccntctlr @ 0x150 array(4) uint32(atomic); + trccntvr @ 0x160 array(4) uint32(atomic); + + // resource selection registers (note: elements 0 and 1 are reserved) + trcrsctlr2 @ 0x208 array(30) uint32(atomic); + + // single shot comparator registers + trcssccr @ 0x280 array(8) uint32(atomic); + trcsscsr @ 0x2a0 array(8) uint32(atomic); + trcsspcicr @ 0x2c0 array(8) uint32(atomic); + + // OS lock registers + trcoslar @ 0x0300 uint32(atomic); + trcoslsr @ 0x0304 bitfield ETM_TRCOSLSR(atomic); + + // comparator registers + trcacvr @ 0x400 array(16) uint64(atomic); + trcacatr @ 0x480 array(16) uint64(atomic); + trcdvcvr @ 0x500 array(8) structure etm_trcdv; + trcdvcmr @ 0x580 array(8) structure etm_trcdv; + trccidcvr @ 0x600 array(8) uint64(atomic); + trcvmidcvr @ 0x640 array(8) uint64(atomic); + trccidcctlr @ 0x680 array(2) uint32(atomic); + trcvmidcctlr @ 0x688 array(2) uint32(atomic); + + // Software lock and claim tag registers + trcclaimset @ 0xfa0 uint32(atomic); + trcclaimclr @ 0xfa4 uint32(atomic); + trclar @ 0x0FB0 uint32(atomic); +}; diff --git a/hyp/platform/etm/include/etm.h b/hyp/platform/etm/include/etm.h new file mode 100644 index 0000000..7864ef8 --- /dev/null +++ b/hyp/platform/etm/include/etm.h @@ -0,0 +1,16 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +void +etm_set_reg(cpu_index_t cpu, size_t offset, register_t val, size_t access_size); + +void +etm_get_reg(cpu_index_t cpu, size_t offset, register_t *val, + size_t access_size); + +void +etm_save_context_percpu(cpu_index_t cpu); + +void +etm_restore_context_percpu(cpu_index_t cpu); diff --git a/hyp/platform/etm/src/etm.c b/hyp/platform/etm/src/etm.c new file mode 100644 index 0000000..46c7db3 --- /dev/null +++ b/hyp/platform/etm/src/etm.c @@ -0,0 +1,500 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "etm.h" +#include "event_handlers.h" + +#if defined(PLATFORM_ETM_REG_WRITE_WORKAROUND) +// this work around is for context save, since we are writing lots of registers +// back to back, it could block other master on NOC. +#define CTX_WRITE_WORKAROUND platform_timer_ndelay(20000) +#else +#define CTX_WRITE_WORKAROUND +#endif + +#if !defined(ETM_USE_SOFTWARE_LOCK) +// Using or implementing TRCLAR is deprecated. Linux doesn't use it. +#define ETM_USE_SOFTWARE_LOCK 0 +#endif + +typedef struct { + size_t reg_offset; + size_t access_size; + size_t count; + size_t stride; +} context_register_info_t; + +static etm_t *mapped_etms[PLATFORM_MAX_CORES]; + +static register_t *etm_contexts[PLATFORM_MAX_CORES]; + +static uint32_t etm_claim_tag[PLATFORM_MAX_CORES]; + +static uint32_t etm_cprgctlr[PLATFORM_MAX_CORES]; + +#define ETM_REGISTER(name) \ + { \ + .reg_offset = offsetof(etm_t, name), \ + .access_size = util_sizeof_member(etm_t, name), .count = 1, \ + .stride = 0 \ + } + +#define ETM_REGISTER_ARRAY(name) \ + { \ + .reg_offset = offsetof(etm_t, name), \ + .access_size = util_sizeof_member(etm_t, name[0]), \ + .count = util_sizeof_member(etm_t, name) / \ + util_sizeof_member(etm_t, name[0]), \ + .stride = util_sizeof_member(etm_t, name[0]) \ + } + +#define ETM_REGISTER_SPARSE_ARRAY(name) \ + { \ + .reg_offset = offsetof(etm_t, name), \ + .access_size = util_sizeof_member(etm_t, name[0].value), \ + .count = util_sizeof_member(etm_t, name) / \ + util_sizeof_member(etm_t, name[0]), \ + .stride = util_sizeof_member(etm_t, name[0]) \ + } + +// NOTE: registers are saved in the context memory region based on their +// index in context_register_list. Make sure the alignment is correct +static const context_register_info_t context_register_list[] = { + // main control & configuration regsters + ETM_REGISTER(trcprocselr), + ETM_REGISTER(trcconfigr), + ETM_REGISTER(trcauxctlr), + ETM_REGISTER(trceventctl0r), + ETM_REGISTER(trceventctl1r), + ETM_REGISTER(trcstallctlr), + ETM_REGISTER(trctsctlr), + ETM_REGISTER(trcsyncpr), + ETM_REGISTER(trcccctlr), + ETM_REGISTER(trcbbctlr), + ETM_REGISTER(trctraceidr), + ETM_REGISTER(trcqctlr), + + // filtering control registers + ETM_REGISTER(trcvictlr), + ETM_REGISTER(trcviiectlr), + ETM_REGISTER(trcvissctlr), + ETM_REGISTER(trcvipcssctlr), + ETM_REGISTER(trcvdctlr), + ETM_REGISTER(trcvdsacctlr), + ETM_REGISTER(trcvdarcctlr), + + // derived resources registers + ETM_REGISTER_ARRAY(trcseqevr), + ETM_REGISTER(trcseqrstevr), + ETM_REGISTER(trcseqstr), + ETM_REGISTER(trcextinselr), + ETM_REGISTER_ARRAY(trccntrldvr), + ETM_REGISTER_ARRAY(trccntctlr), + ETM_REGISTER_ARRAY(trccntvr), + + // resource selection registers + ETM_REGISTER_ARRAY(trcrsctlr2), + + // comparator registers + ETM_REGISTER_ARRAY(trcacvr), + ETM_REGISTER_ARRAY(trcacatr), + + ETM_REGISTER_SPARSE_ARRAY(trcdvcvr), + ETM_REGISTER_SPARSE_ARRAY(trcdvcmr), + + ETM_REGISTER_ARRAY(trccidcvr), + ETM_REGISTER_ARRAY(trccidcctlr), + + ETM_REGISTER_ARRAY(trcvmidcvr), + ETM_REGISTER_ARRAY(trcvmidcctlr), + + // single shot comparator registers + ETM_REGISTER_ARRAY(trcssccr), + ETM_REGISTER_ARRAY(trcsscsr), + ETM_REGISTER_ARRAY(trcsspcicr), +}; + +static size_t +etm_get_context_size_percpu(void) +{ + size_t ret = 0UL; + + // can be optimized by read last entry offset + size, but need to + // restrict the timing to call this context + for (index_t i = 0; i < util_array_size(context_register_list); i++) { + const context_register_info_t *info = &context_register_list[i]; + ret += sizeof(uint64_t) * info->count; + } + return ret; +} + +void +etm_handle_boot_hypervisor_start(void) +{ + if (compiler_expected(platform_security_state_debug_disabled())) { + goto out; + } + + partition_t *hyp_partition = partition_get_private(); + + // FIXME: remove when read from device tree + paddr_t etm_base = PLATFORM_ETM_BASE; + size_t etm_stride = PLATFORM_ETM_STRIDE; + + for (cpu_index_t i = 0; cpulocal_index_valid(i); i++) { + virt_range_result_t range = + hyp_aspace_allocate(PLATFORM_ETM_SIZE_PERCPU); + if (range.e != OK) { + panic("ETM: Address allocation failed."); + } + + paddr_t cur_base = etm_base + i * etm_stride; + + pgtable_hyp_start(); + + error_t ret = pgtable_hyp_map( + hyp_partition, range.r.base, PLATFORM_ETM_SIZE_PERCPU, + cur_base, PGTABLE_HYP_MEMTYPE_NOSPEC_NOCOMBINE, + PGTABLE_ACCESS_RW, VMSA_SHAREABILITY_NON_SHAREABLE); + if (ret != OK) { + panic("ETM: Mapping of etm register failed."); + } + mapped_etms[i] = (etm_t *)range.r.base; + + pgtable_hyp_commit(); + } + + // allocate contexts + size_t context_size = etm_get_context_size_percpu(); + for (cpu_index_t i = 0; cpulocal_index_valid(i); i++) { + void_ptr_result_t alloc_r = partition_alloc( + hyp_partition, context_size, alignof(uint64_t)); + if (alloc_r.e != OK) { + panic("failed to allocate ETM context memory"); + } + + etm_contexts[i] = (register_t *)alloc_r.r; + (void)memset_s(etm_contexts[i], context_size, 0, context_size); + } + +out: + return; +} + +void +etm_set_reg(cpu_index_t cpu, size_t offset, register_t val, size_t access_size) +{ + (void)access_size; + + assert(cpulocal_index_valid(cpu)); + + assert(offset < (sizeof(*mapped_etms[cpu]) - access_size)); + uintptr_t base = (uintptr_t)mapped_etms[cpu]; + + if (access_size == sizeof(uint32_t)) { + _Atomic uint32_t *reg = (_Atomic uint32_t *)(base + offset); + atomic_store_relaxed(reg, val); + } else if (access_size == sizeof(uint64_t)) { + _Atomic uint64_t *reg = (_Atomic uint64_t *)(base + offset); + atomic_store_relaxed(reg, val); + } else { + panic("ETM: invalid access size"); + } +} + +void +etm_get_reg(cpu_index_t cpu, size_t offset, register_t *val, size_t access_size) +{ + (void)access_size; + + assert(cpulocal_index_valid(cpu)); + + uintptr_t base = (uintptr_t)mapped_etms[cpu]; + + if (access_size == sizeof(uint32_t)) { + // Regards the ETM v4 doc: HW implementation supports 32-bit + // accesses to access 32-bit registers or either half of a + // 64-bit register. + _Atomic uint32_t *reg = (_Atomic uint32_t *)(base + offset); + + *val = atomic_load_relaxed(reg); + } else if (access_size == sizeof(uint64_t)) { + _Atomic uint64_t *reg = (_Atomic uint64_t *)(base + offset); + + *val = atomic_load_relaxed(reg); + } else { + panic("ETM: invalid access size"); + } +} + +static void +etm_unlock_percpu(cpu_index_t cpu) +{ +#if ETM_USE_SOFTWARE_LOCK + atomic_store_relaxed(&mapped_etms[cpu]->trclar, ETM_TRCLAR_UNLOCK); + CTX_WRITE_WORKAROUND; +#else + (void)cpu; +#endif +} + +#if ETM_USE_SOFTWARE_LOCK +static void +etm_lock_percpu(cpu_index_t cpu) +{ + atomic_store_relaxed(&mapped_etms[cpu]->trclar, ETM_TRCLAR_LOCK); + CTX_WRITE_WORKAROUND; +} +#endif + +static void +etm_os_unlock_percpu(cpu_index_t cpu) +{ + atomic_store_relaxed(&mapped_etms[cpu]->trcoslar, ETM_TRCOSLAR_UNLOCK); + + // Note: no write delay workaround for this register, to avoid delaying + // resume when the ETM is not being used. It is always written last + // in the sequence anyway, so a delay after it is useless. +} + +static void +etm_os_lock_percpu(cpu_index_t cpu) +{ + atomic_store_relaxed(&mapped_etms[cpu]->trcoslar, ETM_TRCOSLAR_LOCK); + + // Note: no write delay workaround for this register, to avoid delaying + // suspend when the ETM is not being used. The suspend sequence should + // start with a conditional CTX_WRITE_WORKAROUND as a substitute. +} + +static index_t +etm_save_context_registers(cpu_index_t cpu, const context_register_info_t *info, + index_t context_register_index) +{ + register_t *context = etm_contexts[cpu]; + + index_t cur_register_index = context_register_index; + + for (index_t i = 0; i < info->count; i++, cur_register_index++) { + size_t reg_offset = info->reg_offset + i * info->stride; + + register_t *reg = context + cur_register_index; + + etm_get_reg(cpu, reg_offset, reg, info->access_size); + } + + return cur_register_index; +} + +void +etm_save_context_percpu(cpu_index_t cpu) +{ + // dsb isb + __asm__ volatile("dsb ish; isb" ::: "memory"); + + // Delay after taking the OS lock in the caller + CTX_WRITE_WORKAROUND; + + // pull trcstatr.pmstable until it's stable + // wait up to 100us + ticks_t start = platform_timer_get_current_ticks(); + ticks_t timeout = start + platform_convert_ns_to_ticks(100U * 1000U); + do { + ETM_TRCSTATR_t trcstatr = + atomic_load_relaxed(&mapped_etms[cpu]->trcstatr); + if (ETM_TRCSTATR_get_pmstable(&trcstatr)) { + break; + } + + if (platform_timer_get_current_ticks() > timeout) { + TRACE_AND_LOG(ERROR, INFO, + "ETM: programmers model is not stable"); + break; + } + } while (1); + + etm_cprgctlr[cpu] = atomic_load_relaxed(&mapped_etms[cpu]->trcprgctlr); + if ((etm_cprgctlr[cpu] & ETM_TRCPRGCTLR_ENABLE) != 0U) { + // save all context registers + index_t context_register_index = 0U; + for (index_t i = 0; i < util_array_size(context_register_list); + i++) { + const context_register_info_t *info = + &context_register_list[i]; + + context_register_index = etm_save_context_registers( + cpu, info, context_register_index); + } + + etm_claim_tag[cpu] = + atomic_load_relaxed(&mapped_etms[cpu]->trcclaimclr); + + // poll until trcstatr.idle + count_t idle_count = 100U; + bool idle = false; + while (idle_count > 0U) { + // should be a volatile read + ETM_TRCSTATR_t trcstatr = atomic_load_relaxed( + &mapped_etms[cpu]->trcstatr); + idle = ETM_TRCSTATR_get_idle(&trcstatr); + + if (idle) { + break; + } + + idle_count--; + platform_timer_ndelay(1000); + } + + if ((!idle) && (idle_count == 0)) { + LOG(ERROR, WARN, + "ETM: waiting idle timeout for context save"); + } + } + return; +} + +static index_t +etm_restore_context_registers(cpu_index_t cpu, + const context_register_info_t *info, + index_t context_register_index) +{ + register_t *context = etm_contexts[cpu]; + + index_t cur_register_index = context_register_index; + + for (index_t i = 0; i < info->count; i++, cur_register_index++) { + size_t reg_offset = info->reg_offset + i * info->stride; + + register_t *reg = context + cur_register_index; + + etm_set_reg(cpu, reg_offset, *reg, info->access_size); + CTX_WRITE_WORKAROUND; + } + + return cur_register_index; +} + +void +etm_restore_context_percpu(cpu_index_t cpu) +{ + if (((etm_cprgctlr[cpu]) & ETM_TRCPRGCTLR_ENABLE) != 0U) { + // restore all context registers + index_t context_register_index = 0U; + for (index_t i = 0; i < util_array_size(context_register_list); + i++) { + const context_register_info_t *info = + &context_register_list[i]; + + context_register_index = etm_restore_context_registers( + cpu, info, context_register_index); + } + + // set claim tag + atomic_store_relaxed(&mapped_etms[cpu]->trcclaimset, + etm_claim_tag[cpu]); + CTX_WRITE_WORKAROUND; + + atomic_store_relaxed(&mapped_etms[cpu]->trcprgctlr, + ETM_TRCPRGCTLR_ENABLE); + } + + return; +} + +void +etm_handle_power_cpu_online(void) +{ + if (compiler_unexpected(!platform_security_state_debug_disabled())) { + cpu_index_t cpu = cpulocal_get_index(); + etm_unlock_percpu(cpu); + etm_os_unlock_percpu(cpu); + } +} + +void +etm_handle_power_cpu_offline(void) +{ + (void)etm_handle_power_cpu_suspend(true); +} + +error_t +etm_handle_power_cpu_suspend(bool may_poweroff) +{ + if (may_poweroff && + compiler_unexpected(!platform_security_state_debug_disabled())) { + cpu_index_t cpu = cpulocal_get_index(); + + etm_unlock_percpu(cpu); + etm_os_lock_percpu(cpu); + + etm_save_context_percpu(cpu); + } + + return OK; +} + +void +etm_unwind_power_cpu_suspend(bool may_poweroff) +{ + if (may_poweroff && + compiler_unexpected(!platform_security_state_debug_disabled())) { + cpu_index_t cpu = cpulocal_get_index(); + etm_os_unlock_percpu(cpu); + +#if ETM_USE_SOFTWARE_LOCK +#error Restore software lock from before suspend (don't lock unconditionally) +#endif + } +} + +void +etm_handle_power_cpu_resume(bool was_poweroff) +{ + if (compiler_expected(platform_security_state_debug_disabled())) { + goto out; + } + + cpu_index_t cpu = cpulocal_get_index(); + + if (was_poweroff) { + etm_unlock_percpu(cpu); + + // check lock os los with trcoslsr.oslk == 1 + ETM_TRCOSLSR_t trcoslsr = + atomic_load_relaxed(&mapped_etms[cpu]->trcoslsr); + if (!ETM_TRCOSLSR_get_oslk(&trcoslsr)) { + LOG(ERROR, WARN, "etm: os is not locked"); + etm_os_lock_percpu(cpu); + } + + etm_restore_context_percpu(cpu); + } + + etm_os_unlock_percpu(cpu); + +#if ETM_USE_SOFTWARE_LOCK +#error Restore software lock from before suspend (don't lock unconditionally) +#endif + +out: + return; +} diff --git a/hyp/platform/gicv3/aarch64/gicv3-regs.tc b/hyp/platform/gicv3/aarch64/gicv3-regs.tc index 705fa86..7f7ba3c 100644 --- a/hyp/platform/gicv3/aarch64/gicv3-regs.tc +++ b/hyp/platform/gicv3/aarch64/gicv3-regs.tc @@ -97,7 +97,12 @@ define ICH_HCR_EL2 bitfield<64> { 5 VGrp0DIE bool; 6 VGrp1EIE bool; 7 VGrp1DIE bool; +#if GICV3_HAS_VLPI_V4_1 + // Note: despite the name, this is negated (1 means not counted) + 8 vSGIEOICount bool; +#else 8 unknown=0; +#endif 9 unknown=0; 10 TC bool; 11 TALL0 bool; diff --git a/hyp/platform/gicv3/build.conf b/hyp/platform/gicv3/build.conf index 4c5605d..9124bd2 100644 --- a/hyp/platform/gicv3/build.conf +++ b/hyp/platform/gicv3/build.conf @@ -5,7 +5,7 @@ local_include types gicv3.tc arch_types aarch64 gicv3-regs.tc -configs IRQ_SPARSE_IDS=0 IRQ_HAS_MSI=0 +configs IRQ_SPARSE_IDS=GICV3_HAS_LPI IRQ_HAS_MSI=GICV3_HAS_LPI configs PLATFORM_MSI_CONTROLLER_COUNT=PLATFORM_GITS_COUNT arch_configs aarch64 AARCH64_ICC_REGS=1 template simple gich_lrs.h.tmpl diff --git a/hyp/platform/gicv3/gicv3.ev b/hyp/platform/gicv3/gicv3.ev index e4e4f1c..4430f78 100644 --- a/hyp/platform/gicv3/gicv3.ev +++ b/hyp/platform/gicv3/gicv3.ev @@ -13,6 +13,7 @@ subscribe boot_cpu_cold_init priority 10 subscribe boot_cpu_warm_init + require_preempt_disabled subscribe power_cpu_suspend() unwinder gicv3_handle_power_cpu_resume() @@ -32,3 +33,28 @@ subscribe vcpu_poweroff(current) require_scheduler_lock(current) #endif // INTERFACE_VCPU && GICV3_HAS_1N + +#if defined(GICV3_ENABLE_VPE) && GICV3_ENABLE_VPE +subscribe scheduler_affinity_changed(thread, prev_cpu, next_cpu) + require_scheduler_lock(thread) + +#if GICV3_HAS_VLPI_V4_1 +subscribe object_create_thread + +subscribe irq_received[HWIRQ_ACTION_GICV3_ITS_DOORBELL] + handler gicv3_vpe_handle_irq_received_doorbell(hwirq) +#endif +#endif + +#if GICV3_HAS_ITS +subscribe abort_kernel() +#endif + +subscribe boot_hypervisor_handover + require_preempt_disabled + +subscribe power_cpu_online() + require_preempt_disabled + +subscribe power_cpu_offline() + require_preempt_disabled diff --git a/hyp/platform/gicv3/gicv3.tc b/hyp/platform/gicv3/gicv3.tc index f330210..75759ca 100644 --- a/hyp/platform/gicv3/gicv3.tc +++ b/hyp/platform/gicv3/gicv3.tc @@ -13,6 +13,7 @@ define irq_t newtype uint32; // We always allocate one fixed collection per physical CPU. They're // apparently intended for group rerouting of IRQs but that is not terribly // useful in practice. +// FIXME: //define gic_its_ic_id_t newtype type cpu_index_t; define gic_its_ic_id_t newtype uint16; @@ -33,6 +34,10 @@ define gic_its_rdbase union { pta0 type gic_its_rdbase_pta0_t; }; +#if GICV3_HAS_VLPI +define gic_its_vpe_id_t newtype uint16; +#endif + #endif // GICV3_HAS_ITS define GIC_SGI_BASE constant type irq_t = 0; @@ -152,6 +157,17 @@ define gicr_rd_base structure(aligned(PGTABLE_HYP_PAGE_SIZE)) { syncr @ 0x00C0 bitfield GICR_SYNCR(atomic); }; +#if GICV3_HAS_VLPI +define gicr_vlpi_base structure(aligned(PGTABLE_HYP_PAGE_SIZE)) { + vpropbaser @ 0x0070 bitfield GICR_VPROPBASER(atomic); + vpendbaser @ 0x0078 bitfield GICR_VPENDBASER(atomic); +#if GICV3_HAS_VLPI_V4_1 + vsgir @ 0x0080 bitfield GICR_VSGIR(atomic); + vsgipendr @ 0x0088 bitfield GICR_VSGIPENDR(atomic); +#endif +}; +#endif + define gicr_sgi_base structure(aligned(PGTABLE_HYP_PAGE_SIZE)) { reserved @ 0x0000 uint32(atomic); igroupr0 @ 0x0080 uint32(atomic); @@ -181,12 +197,45 @@ define gicr structure(aligned(65536)) { rd @ 0 structure gicr_rd_base; PIDR2 @ 0xFFE8 uint32(atomic); sgi @ 0x10000 structure gicr_sgi_base; +#if GICV3_HAS_VLPI + vlpi @ 0x20000 structure gicr_vlpi_base; +#endif }; +define gicr_cpu structure { + icc_sgi1r bitfield ICC_SGIR_EL1; + gicr pointer structure gicr; +#if GICV3_HAS_LPI + lpi_pending_bitmap pointer type register_t; + lpi_pendbase bitfield GICR_PENDBASER; +#if defined(GICV3_ENABLE_VPE) && GICV3_ENABLE_VPE && GICV3_HAS_VLPI_V4_1 + vsgi_query_lock structure spinlock; +#endif +#endif + + // False if the CPU is powering down + // Protect this with the SPI lock when we enable, route or migrate + // a SPI + online bool; +}; + +#if GICV3_HAS_LPI +define gic_lpi_prop bitfield<8> { + 0 enable bool; + 1 res1 bool(const)=1; + 7:2 priority uint8 lsl(2); +}; +#endif + define GICR_PAGE_MASK constant = ((1 << 16) - 1); +#if GICV3_HAS_VLPI +// GIC Redistributor stride (four 64k pages per core) +define GICR_STRIDE_SHIFT constant uint8 = 16 + 2; +#else // GIC Redistributor stride (two 64k pages per core) -define GICR_STRIDE_SHIFT constant = 16 + 1; +define GICR_STRIDE_SHIFT constant uint8 = 16 + 1; +#endif define gicv3_irq_type enumeration { sgi; @@ -196,6 +245,9 @@ define gicv3_irq_type enumeration { #if GICV3_EXT_IRQS ppi_ext; spi_ext; +#endif +#if GICV3_HAS_LPI + lpi; #endif reserved; }; @@ -336,12 +388,24 @@ define GICR_IIDR bitfield<32> { define GICR_INVALLR bitfield<64> { 31:0 unknown=0; +#if GICV3_HAS_VLPI_V4_1 + 47:32 vPEID type gic_its_vpe_id_t; + 62:48 unknown=0; + 63 V bool; +#else 63:32 unknown=0; +#endif }; define GICR_INVLPIR bitfield<64> { 31:0 pINTID type irq_t; +#if GICV3_HAS_VLPI_V4_1 + 47:32 vPEID type gic_its_vpe_id_t; + 62:48 unknown=0; + 63 V bool; +#else 63:32 unknown=0; +#endif }; define GICR_PENDBASER bitfield<64> { @@ -406,6 +470,46 @@ define GICR_TYPER bitfield<64> { 63:56 Aff3 uint8; }; +#if GICV3_HAS_VLPI_V4_1 + +define GICR_VPENDBASER bitfield<64> { + 15:0 vPEID type gic_its_vpe_id_t; + 57:16 unknown=0; + 58 vGrp1En bool; + 59 vGrp0En bool; + 60 Dirty bool; + 61 PendingLast bool; + 62 Doorbell bool; + 63 Valid bool; +}; + +define GICR_VPROPBASER bitfield<64> { + 6:0 Size type count_t; + 9:7 InnerCache uint8; + 11:10 Shareability uint8; + 51:12 Physical_Address type paddr_t lsl(12); + 52 Z bool; + 54:53 Page_Size enumeration GITS_BASER_Page_Size; + 55 Indirect bool; + 58:56 OuterCache uint8; + 61:59 Entry_Size size; + 62 unknown=0; + 63 Valid bool; +}; + +define GICR_VSGIR bitfield<32> { + 15:0 vPEID type gic_its_vpe_id_t; + 31:16 unknown=0; +}; + +define GICR_VSGIPENDR bitfield<32> { + 15:0 Pending uint32; + 30:16 unknown=0; + 31 Busy bool; +}; + +#else + define GICR_VPENDBASER bitfield<64> { 6:0 unknown=0; 9:7 InnerCache uint8; @@ -432,6 +536,8 @@ define GICR_VPROPBASER bitfield<64> { 63:59 unknown=0; }; +#endif + define GICR_WAKER bitfield<32> { 0 IMPDEF bool; 1 ProcessorSleep bool; @@ -525,6 +631,9 @@ define gits_ctl_base structure(aligned(PGTABLE_HYP_PAGE_SIZE)) { typer @ 0x8 bitfield GITS_TYPER(atomic); mpamidr @ 0x10 bitfield GITS_MPAMIDR(atomic); partidr @ 0x14 bitfield GITS_PARTIDR(atomic); +#if GICV3_HAS_VLPI_V4_1 + mpidr @ 0x18 bitfield GITS_MPIDR(atomic); +#endif cbaser @ 0x80 bitfield GITS_CBASER(atomic); cwriter @ 0x88 bitfield GITS_CWRITER(atomic); creadr @ 0x90 bitfield GITS_CREADR(atomic); @@ -535,14 +644,28 @@ define gits_xlate_base structure(aligned(PGTABLE_HYP_PAGE_SIZE)) { translater @ 0x40 bitfield GITS_TRANSLATER(atomic); }; +#if GICV3_HAS_VLPI_V4_1 +define gits_vsgi_base structure(aligned(PGTABLE_HYP_PAGE_SIZE)) { + sgir @ 0x20 bitfield GITS_SGIR(atomic); +}; +#endif + define gits structure(aligned(65536)) { ctl @ 0 structure gits_ctl_base; PIDR2 @ 0xFFE8 uint32(atomic); xlate @ 0x10000 structure gits_xlate_base; +#if GICV3_HAS_VLPI_V4_1 + vsgi @ 0x20000 structure gits_vsgi_base; +#endif }; +#if GICV3_HAS_VLPI_V4_1 +// GIC ITS stride (three 64k pages per ITS, rounded up to four) +define GITS_STRIDE_SHIFT constant = 16 + 2; +#else // GIC ITS stride (two 64k pages per ITS) define GITS_STRIDE_SHIFT constant = 16 + 1; +#endif // ITS memory-mapped registers @@ -555,6 +678,9 @@ define GITS_BASER_Page_Size enumeration(explicit) { define GITS_BASER_Type enumeration(explicit) { Unimplemented = 0b000; Devices = 0b001; +#if GICV3_HAS_VLPI_V4_1 + vPEs = 0b010; +#endif Collections = 0b100; }; @@ -635,12 +761,30 @@ define GITS_MPAMIDR bitfield<32> { 31:24 unknown=0; }; +#if GICV3_HAS_VLPI_V4_1 +define GITS_MPIDR bitfield<32> { + 7:0 unknown=0; + 15:8 Aff1 uint8; + 23:16 Aff2 uint8; + 31:24 Aff3 uint8; +}; +#endif // GICV3_HAS_VLPI_V4_1 + define GITS_PARTIDR bitfield<32> { 15:0 PARTID uint16; 23:16 PMG uint8; 31:24 unknown=0; }; +#if GICV3_HAS_VLPI_V4_1 +define GITS_SGIR bitfield<64> { + 3:0 vINTID type virq_t; + 31:4 unknown=0; + 47:32 vPEID type gic_its_vpe_id_t; + 63:48 unknown=0; +}; +#endif // GICV3_HAS_VLPI_V4_1 + define GITS_TRANSLATER bitfield<32> { 31:0 event_id type platform_msi_event_id_t; }; @@ -678,12 +822,16 @@ define GITS_TYPER bitfield<64> { // ITS commands // Note: these are duplicated in the type definitions below. +// FIXME: define gic_its_cmd_id enumeration { clear = 0x4; discard = 0xf; int = 0x3; inv = 0xc; invall = 0xd; +#if GICV3_HAS_VLPI_V4_1 + invdb = 0x2e; +#endif mapc = 0x9; mapd = 0x8; mapi = 0xb; @@ -691,6 +839,17 @@ define gic_its_cmd_id enumeration { movall = 0xe; movi = 0x1; sync = 0x5; +#if GICV3_HAS_VLPI + vinvall = 0x2d; + vmapi = 0x2b; + vmapp = 0x29; + vmapti = 0x2a; + vmovi = 0x21; + vmovp = 0x22; +#if GICV3_HAS_VLPI_V4_1 + vsgi = 0x23; +#endif +#endif }; define gic_its_cmd_base bitfield<256> { @@ -737,6 +896,17 @@ define gic_its_cmd_invall bitfield<256> { 255:144 unknown=0; }; +#if GICV3_HAS_VLPI_V4_1 + +define gic_its_cmd_invdb bitfield<256> { + 7:0 cmd enumeration gic_its_cmd_id(const)=0x2e; + 95:8 unknown=0; + 111:96 vpe_id type gic_its_vpe_id_t; + 255:112 unknown=0; +}; + +#endif // GICV3_HAS_VLPI_V4_1 + define gic_its_cmd_mapc bitfield<256> { 7:0 cmd enumeration gic_its_cmd_id(const)=0x9; 127:8 unknown=0; @@ -805,6 +975,151 @@ define gic_its_cmd_sync bitfield<256> { 255:180 unknown=0; }; +#if GICV3_HAS_VLPI + +define gic_its_cmd_vinvall bitfield<256> { + 7:0 cmd enumeration gic_its_cmd_id(const)=0x2d; + 95:8 unknown=0; + 111:96 vpe_id type gic_its_vpe_id_t; + 255:112 unknown=0; +}; + +define gic_its_cmd_vmapi bitfield<256> { + 7:0 cmd enumeration gic_its_cmd_id(const)=0x2b; + 31:8 unknown=0; + 63:32 device_id type platform_msi_device_id_t; + 95:64 event_id type platform_msi_event_id_t; + 111:96 vpe_id type gic_its_vpe_id_t; + 159:112 unknown=0; + 191:160 db_lpi type irq_t; + 255:192 unknown=0; +}; + +#if !GICV3_HAS_VLPI_V4_1 + +define gic_its_cmd_vmapp bitfield<256> { + 7:0 cmd enumeration gic_its_cmd_id(const)=0x29; + 95:8 unknown=0; + 111:96 vpe_id type gic_its_vpe_id_t; + 143:112 unknown=0; + 179:144 rdbase uregister; // union gic_its_rdbase; + 190:180 unknown=0; + 191 valid bool; + // Note: the VPT_size field in the GICv4.1 VMAPP command is _not_ + // offset by 1, so we deviate from the naming in the spec to ensure + // that this field is used correctly. + 196:192 vpt_size_minus_one type count_t; + 207:197 unknown=0; + 243:208 vpt_addr type paddr_t lsl(16); + 255:244 unknown=0; +}; + +#else // GICV3_HAS_VLPI_V4_1 + +define gic_its_cmd_vmapp bitfield<256> { + 7:0 cmd enumeration gic_its_cmd_id(const)=0x29; + 8 alloc bool; + 9 ptz bool; + 15:10 unknown=0; + 51:16 vconf_addr type paddr_t lsl(16); + 63:52 unknown=0; + 95:64 db_lpi type irq_t; + 111:96 vpe_id type gic_its_vpe_id_t; + 143:112 unknown=0; + 179:144 rdbase uregister; // union gic_its_rdbase; + 190:180 unknown=0; + 191 valid bool; + 199:192 vpt_size type count_t; + 207:200 unknown=0; + 243:208 vpt_addr type paddr_t lsl(16); + 255:244 unknown=0; +}; + +#endif // GICV3_HAS_VLPI_V4_1 + +define gic_its_cmd_vmapti bitfield<256> { + 7:0 cmd enumeration gic_its_cmd_id(const)=0x2a; + 31:8 unknown=0; + 63:32 device_id type platform_msi_device_id_t; + 95:64 event_id type platform_msi_event_id_t; + 111:96 vpe_id type gic_its_vpe_id_t; + 127:112 unknown=0; + 159:128 vlpi type virq_t; + 191:160 db_lpi type irq_t; + 255:192 unknown=0; +}; + +define gic_its_cmd_vmovi bitfield<256> { + 7:0 cmd enumeration gic_its_cmd_id(const)=0x21; + 31:8 unknown=0; + 63:32 device_id type platform_msi_device_id_t; + 95:64 event_id type platform_msi_event_id_t; + 111:96 vpe_id type gic_its_vpe_id_t; + 127:112 unknown=0; + 128 db bool; + 159:129 unknown=0; + 191:160 db_lpi type irq_t; + 255:192 unknown=0; +}; + +#if !GICV3_HAS_VLPI_V4_1 + +define gic_its_cmd_vmovp bitfield<256> { + 7:0 cmd enumeration gic_its_cmd_id(const)=0x22; + 31:8 unknown=0; + 47:32 seqnum uint16; + 63:48 unknown=0; + 79:64 itslist uint16; + 95:80 unknown=0; + 111:96 vpe_id type gic_its_vpe_id_t; + 143:112 unknown=0; + 179:144 rdbase uregister; // union gic_its_rdbase; + 255:180 unknown=0; +}; + +#else // GICV3_HAS_VLPI_V4_1 + +define gic_its_cmd_vmovp bitfield<256> { + 7:0 cmd enumeration gic_its_cmd_id(const)=0x22; + 31:8 unknown=0; + 47:32 seqnum uint16; + 63:48 unknown=0; + 79:64 itslist uint16; + 95:80 unknown=0; + 111:96 vpe_id type gic_its_vpe_id_t; + 143:112 unknown=0; + 179:144 rdbase uregister; // union gic_its_rdbase; + 190:180 unknown=0; + 191 db bool; + 223:192 db_lpi type irq_t; + 255:224 unknown=0; +}; + +define gic_its_cmd_vsgi bitfield<256> { + 7:0 cmd enumeration gic_its_cmd_id(const)=0x23; + 8 enable bool; + 9 clear bool; + 10 group1 bool; + 19:11 unknown=0; + 23:20 priority uint8 lsl(4); + 31:24 unknown=0; + 35:32 sgi type irq_t; + 95:36 unknown=0; + 111:96 vpe_id type gic_its_vpe_id_t; + 255:112 unknown=0; +}; + +#endif // GICV3_HAS_VLPI_V4_1 + +define gic_its_cmd_vsync bitfield<256> { + 7:0 cmd enumeration gic_its_cmd_id(const)=0x25; + 95:8 unknown=0; + 111:96 vpe_id type gic_its_vpe_id_t; + 255:112 unknown=0; +}; + +#endif // GICV3_HAS_VLPI + define gic_its_cmd union(aligned(32)) { clear bitfield gic_its_cmd_clear; discard bitfield gic_its_cmd_discard; @@ -812,6 +1127,9 @@ define gic_its_cmd union(aligned(32)) { int_ bitfield gic_its_cmd_int; inv bitfield gic_its_cmd_inv; invall bitfield gic_its_cmd_invall; +#if GICV3_HAS_VLPI_V4_1 + invdb bitfield gic_its_cmd_invdb; +#endif mapc bitfield gic_its_cmd_mapc; mapd bitfield gic_its_cmd_mapd; mapi bitfield gic_its_cmd_mapi; @@ -819,6 +1137,18 @@ define gic_its_cmd union(aligned(32)) { movall bitfield gic_its_cmd_movall; movi bitfield gic_its_cmd_movi; sync bitfield gic_its_cmd_sync; +#if GICV3_HAS_VLPI + vinvall bitfield gic_its_cmd_vinvall; + vmapi bitfield gic_its_cmd_vmapi; + vmapp bitfield gic_its_cmd_vmapp; + vmapti bitfield gic_its_cmd_vmapti; + vmovi bitfield gic_its_cmd_vmovi; + vmovp bitfield gic_its_cmd_vmovp; +#if GICV3_HAS_VLPI_V4_1 + vsgi bitfield gic_its_cmd_vsgi; +#endif + vsync bitfield gic_its_cmd_vsync; +#endif // GICV3_HAS_VLPI // Used to read the command ID when parsing commands base bitfield gic_its_cmd_base; @@ -840,8 +1170,6 @@ define gicv3_its_driver_state structure { // Level 1 tables for indirect bases. NULL if indirection is not used. indirect_device_table structure gicv3_its_indirect_table; - // Lock protecting the command queue - cmd_queue_lock structure spinlock; // Virtual and physical pointers to this ITS's command queue cmd_queue pointer array(GICV3_ITS_QUEUE_LEN) union gic_its_cmd; cmd_queue_phys type paddr_t; @@ -849,6 +1177,8 @@ define gicv3_its_driver_state structure { cmd_queue_head type count_t; // Next sequence number to be processed by the ITS cmd_queue_cached_tail type count_t; + // Lock protecting the command queue + cmd_queue_lock structure spinlock; // True if commands need to be cache-flushed before submission cmd_queue_flush bool; @@ -872,4 +1202,48 @@ define gicv3_its_indirect_table structure { entries_l2 type count_t; }; +#if GICV3_HAS_VLPI + +// Per-VCPU state +extend thread object module gicv3_its { + // Globally unique VPE ID, allocated by the ITS driver + vpe_id type gic_its_vpe_id_t; + + // Physical CPU to which this VPE was last mapped in the ITS + mapped_cpu type cpu_index_t; + +#if GICV3_HAS_VLPI_V4_1 + // Doorbell LPI, allocated by the ITS driver + doorbell pointer object hwirq; + + // True if the GICR may not have finished reading the pending tables + // yet after waking this VCPU with a doorbell, so there may be a + // pending vSGI or vLPI that is not yet delivered. + // + // In this case it is not safe to allow the VCPU to enter a low-power + // state, because doing so might trap the VCPU in a loop where a + // pending vSGI or vLPI repeatedly wakes it but it never stays awake + // long enough to actually receive the interrupt. + need_wakeup_check bool; +#else // !GICV3_HAS_VLPI_V4_1 + // LPI configuration and pending table physical addresses, provided by + // the VGITS. For GICv4.1 these are passed to the hardware with VMAPP + // and don't need to be stored in software (though the virtual GICR + // will store the corresponding virtual addresses). + config_table type paddr_t; + pending_table type paddr_t; +#endif // !GICV3_HAS_VLPI_V4_1 +}; + +extend hwirq_action enumeration { + gicv3_its_doorbell; +}; + +extend hwirq object module gicv3_its { + // RCU-protected pointer to the VCPU woken by this doorbell + vcpu pointer(atomic) object thread; +}; + +#endif // GICV3_HAS_VLPI + #endif // GICV3_HAS_ITS diff --git a/hyp/platform/gicv3/include/gicv3.h b/hyp/platform/gicv3/include/gicv3.h index a4d1095..733e404 100644 --- a/hyp/platform/gicv3/include/gicv3.h +++ b/hyp/platform/gicv3/include/gicv3.h @@ -76,3 +76,146 @@ gicv3_ipi_one(ipi_reason_t ipi, cpu_index_t cpu); void gicv3_ipi_clear(ipi_reason_t ipi); + +#if GICV3_HAS_LPI + +// LPI configuration cache invalidation. +// +// If the virtual GICR internally caches VLPI configuration (rather than mapping +// a guest-accessible address to the ITS directly), it must have already updated +// the cache before calling any of these functions. +// +// There are five variants: LPI by (device, event) pair, LPI by IRQ number, VLPI +// by VIRQ number, all LPIs by physical CPU ID, or all VLPIs by VCPU. +// +// The first variant queues an INV command on the relevant ITS. It is only +// implemented if there is at least one ITS, and is therefore declared in +// gicv3_its.h rather than here. +// +// The second and third variants are only implemented on GICv4.1, or (for the +// second variant) on GICv3 with no ITS. On GICv4.0 or GICv3 with an ITS, the +// caller must instead either find or synthesise a (device, event) pair that is +// mapped to the given LPI or VLPI, and then call the first variant. +// +// The fourth variant uses the GICR if possible (GICv4.1 or GICv3 with no ITS) +// and queues an INVALL command on the ITS otherwise. +// +// The fifth variant is only available on GICv4.1; the caller must otherwise +// scan the virtual IC and call the first variant for every (device, event) pair +// mapped to it. +// +// These operations are not guaranteed to complete immediately. The first +// variant returns a sequence number which can be used to poll or wait using the +// functions above. The remaining variants have corresponding functions to poll +// completion of all preceding calls for a specified PCPU or VCPU; note that +// they may spuriously show non-completion because all VCPUs affine to a PCPU +// share the completion state of that PCPU. +#if !GICV3_HAS_ITS || GICV3_HAS_VLPI_V4_1 +void +gicv3_lpi_inv_by_id(cpu_index_t cpu, irq_t lpi); +#endif + +#if GICV3_HAS_VLPI_V4_1 +void +gicv3_vlpi_inv_by_id(thread_t *vcpu, virq_t vlpi); +#endif + +void +gicv3_lpi_inv_all(cpu_index_t cpu); + +#if GICV3_HAS_VLPI_V4_1 +void +gicv3_vlpi_inv_all(thread_t *vcpu); +#endif + +bool +gicv3_lpi_inv_pending(cpu_index_t cpu); + +#if GICV3_HAS_VLPI_V4_1 +bool +gicv3_vlpi_inv_pending(thread_t *vcpu); +#endif + +#if defined(GICV3_ENABLE_VPE) && GICV3_ENABLE_VPE + +// Virtual PE scheduling. +// +// These functions must be called to inform the GICR when the current VCPU +// has been mapped to a vPE ID with gicv3_its_vpe_map() and is not currently +// blocked in EL2 or EL3 nor set to sleep in its virtual GICR_WAKER. +// +// Points at which these functions must be called include context switching, +// entering or leaving the WFI fastpath, entering or leaving an interruptible +// call to EL3, or changing GICR_WAKER.ProcessorSleep or GICR_CTLR.EnableLPIs +// on the current VCPU. +// +// The _schedule function takes boolean arguments indicating whether direct vSGI +// delivery and the default doorbell should be enabled for each of the two +// interrupt groups. If these values must be changed for the running VCPU, e.g. +// due to a GICD_CTLR write, the VCPU must be descheduled and then scheduled +// with the new values. Note that these values have no effect on LPIs with +// individual doorbells, and therefore do nothing for GICv4.0. +// +// This function must call gicv3_vpe_sync_deschedule() to wait for the most +// recent deschedule to complete, so it should be called as late as possible. +void +gicv3_vpe_schedule(bool enable_group0, bool enable_group1) + REQUIRE_PREEMPT_DISABLED; + +// The _deschedule function takes a boolean argument indicating whether the +// previously scheduled VCPU is waiting for interrupts, and therefore requires a +// doorbell IRQ to wake it. It returns a boolean value which is true if a +// doorbell was requested but at least one VLPI or VSGI was already pending, in +// which case the VCPU should to be woken and rescheduled immediately. +// +// This function may not take effect immediately, as the GICR may take some +// time to scan its pending VLPI tables and synchronise with the ITSs to fully +// deschedule the vPE, and this function only waits for that synchronisation to +// complete if enable_doorbell is true. Subsequent calls to _schedule must call +// gicv3_vpe_sync_deschedule() to wait until it has taken effect. Therefore this +// function should be called as early as possible once it is known that a VCPU +// must be descheduled. +bool +gicv3_vpe_deschedule(bool enable_doorbell) REQUIRE_PREEMPT_DISABLED; + +// Check whether a VCPU can safely block waiting for interrupts. +// +// Returns true if the current VCPU was previously woken by a pending vLPI or +// vSGI, a gicv3_vpe_schedule() call has been made for the current VCPU, and the +// GICR is not yet known to have finished scheduling the VCPU. +// +// This is used to prevent the VCPU entering a loop where it is woken by a +// doorbell or the PendingLast bit due to a pending vLPI or vSGI, but then +// blocks agoin before the GICR delivers the interrupt. +// +// The VGIC must ensure that this is called at some point during any VCPU idle +// loop or suspend / resume path such that the VCPU does not block while it +// returns true, and will observe the pending interrupt after it returns false. +// +// If the retry_trap argument is true, the result will indicate the state of the +// GICR before this function was called (i.e. when the trap that triggered it +// occurred). Otherwise, it will indicate the state of the GICR after the +// function was called. +bool +gicv3_vpe_check_wakeup(bool retry_trap); + +// Poll until any pending vPE deschedule is complete on the specified CPU. +// +// If the maybe_scheduled boolean is false, this function asserts that there +// is no currently scheduled vPE. If it is true, the function has no effect if +// there is a currently scheduled vPE. This is called by gicv3_vpe_schedule(), +// but may also be called elsewhere when it is necessary to guarantee that the +// GICR has completely descheduled a VCPU. +void +gicv3_vpe_sync_deschedule(cpu_index_t cpu, bool maybe_scheduled) + REQUIRE_PREEMPT_DISABLED; + +#if GICV3_HAS_VLPI_V4_1 +// Ask the GICR for a specific VCPU's pending vSGI state. +uint32_result_t +gicv3_vpe_vsgi_query(thread_t *vcpu); +#endif + +#endif // GICV3_ENABLE_VPE + +#endif // GICV3_HAS_LPI diff --git a/hyp/platform/gicv3/include/gicv3_config.h b/hyp/platform/gicv3/include/gicv3_config.h index bf03bc4..788a7de 100644 --- a/hyp/platform/gicv3/include/gicv3_config.h +++ b/hyp/platform/gicv3/include/gicv3_config.h @@ -12,4 +12,14 @@ // Command queue length #define GICV3_ITS_QUEUE_LEN 128U +#if GICV3_HAS_VLPI + +// Default vPE ID range. We don't support sharing these, so this limits the +// number of VCPUs that may be attached to at least one VGITS. +#if !defined(GICV3_ITS_VPES) +#define GICV3_ITS_VPES 64U +#endif + +#endif // GICV3_HAS_ITS + #endif // GICV3_HAS_ITS diff --git a/hyp/platform/gicv3/src/gicv3.c b/hyp/platform/gicv3/src/gicv3.c index 00a372f..e622b3d 100644 --- a/hyp/platform/gicv3/src/gicv3.c +++ b/hyp/platform/gicv3/src/gicv3.c @@ -27,6 +27,9 @@ #include #include #include +#if defined(GICV3_ENABLE_VPE) && GICV3_ENABLE_VPE +#include +#endif #include @@ -42,22 +45,67 @@ #define GICV3_DEBUG 0 #endif +static_assert(!GICV3_HAS_ITS || GICV3_HAS_LPI, + "An ITS cannot be present without LPI support"); +#if defined(GICV3_ENABLE_VPE) +static_assert(!GICV3_ENABLE_VPE || GICV3_HAS_VLPI, + "VPE support cannot be enabled unless VLPIs are implemented"); +#endif +static_assert(!GICV3_HAS_VLPI || GICV3_HAS_ITS, + "VLPIs (GICv4) cannot be implemented without an ITS"); +static_assert(!GICV3_HAS_VLPI_V4_1 || GICV3_HAS_VLPI, + "VPEs (GICv4.1) cannot be implemented without VLPIs (GICv4.0)"); +static_assert(!GICV3_HAS_LPI || !GICV3_HAS_ITS || GICV3_HAS_VLPI_V4_1, + "LPIs only supported if ITS is absent or GICv4.1 is implemented"); + #define GICD_ENABLE_GET_N(x) ((x) >> 5) -#define GIC_ENABLE_BIT(x) (uint32_t)(1UL << ((x)&31UL)) +#define GIC_ENABLE_BIT(x) (uint32_t)(util_bit((x)&31UL)) static gicd_t *gicd; -static gicr_t *mapped_gicrs[PLATFORM_MAX_CORES]; -static paddr_t phys_gicrs[PLATFORM_MAX_CORES]; +static gicr_t *mapped_gicrs[PLATFORM_GICR_COUNT]; +#if GICV3_HAS_ITS +static gits_t *mapped_gitss[PLATFORM_GITS_COUNT]; +#endif + +// It is sometimes desirable to be able to inspect the state of GIC in the +// debugger from the secure world point of view. Expose the physical addresses +// of GICD, GICR and GITS so the debuggers can find them. +extern paddr_t platform_gicd_base; +paddr_t platform_gicd_base = PLATFORM_GIC_BASE; +extern paddr_t platform_gicrs_bases[PLATFORM_GICR_COUNT]; +paddr_t platform_gicrs_bases[PLATFORM_GICR_COUNT]; +#if GICV3_HAS_ITS +extern paddr_t platform_gits_base; +paddr_t platform_gits_base = PLATFORM_GITS_BASE; +#endif // Several of the IRQ configuration registers can only be updated using writes // that affect more than one IRQ at a time, notably GICD_ICFGR and GICD_ICLAR. // This lock is used to make the read-modify-write sequences atomic. static spinlock_t bitmap_update_lock; -typedef struct gicr_cpu { - ICC_SGIR_EL1_t icc_sgi1r; - gicr_t *gicr; -} gicr_cpu_t; +// Guard between SPI set route and CPU poweroff SPI migration +static spinlock_t spi_route_lock; + +#if GICV3_HAS_LPI +#if GICV3_HAS_VLPI_V4_1 +// Currently we only need one LPI per VCPU with a VGIC attachment, to be used +// as a GICv4.1 default scheduling doorbell. We therefore allocate the minimum +// nonzero number of LPIs. +#define GIC_LPI_NUM 8192U +#else +#error define GIC_LPI_NUM +#endif +static_assert(util_is_p2(GIC_LPI_BASE + GIC_LPI_NUM) && (GIC_LPI_NUM > 0), + "Hard-coded max LPI count must be 8192 less than a power of two"); + +#define GIC_LPI_PROP_ALIGNMENT ((size_t)util_bit(GICR_PROPBASER_PA_SHIFT)) +static gic_lpi_prop_t alignas(GIC_LPI_PROP_ALIGNMENT) + gic_lpi_prop_table[GIC_LPI_NUM]; + +static GICR_PROPBASER_t gic_lpi_propbase; +#endif // GICV3_HAS_LPI + CPULOCAL_DECLARE_STATIC(gicr_cpu_t, gicr_cpu); static GICD_CTLR_NS_t @@ -96,31 +144,66 @@ gicr_wait_for_write(gicr_t *gicr) atomic_device_fence(memory_order_acquire); } +#if GICV3_HAS_LPI && (!GICV3_HAS_ITS || GICV3_HAS_VLPI_V4_1) +static void +gicr_wait_for_sync(gicr_t *gicr) +{ + // Order the write we're waiting for before the loads in the poll + atomic_device_fence(memory_order_seq_cst); + + GICR_SYNCR_t syncr = atomic_load_relaxed(&gicr->rd.syncr); + + while (GICR_SYNCR_get_Busy(&syncr) != 0U) { + asm_yield(); + syncr = atomic_load_relaxed(&gicr->rd.syncr); + } + + // Order the successful load in the poll before anything afterwards + atomic_device_fence(memory_order_acquire); +} +#endif + static count_t gicv3_spi_max_cache; #if GICV3_EXT_IRQS static count_t gicv3_spi_ext_max_cache; static count_t gicv3_ppi_ext_max_cache; #endif +#if GICV3_HAS_LPI +static count_t gicv3_lpi_max_cache; +#endif + +static_assert(PLATFORM_GICR_COUNT >= PLATFORM_MAX_CORES, + "There must be enough GICRs for all possible CPUs"); + +static error_t +gicv3_spi_set_route_internal(irq_t irq, GICD_IROUTER_t route); static void gicr_set_percpu(cpu_index_t cpu) { - psci_mpidr_t mpidr = platform_cpu_index_to_mpidr(cpu); - uint8_t aff0 = psci_mpidr_get_Aff0(&mpidr); - uint8_t aff1 = psci_mpidr_get_Aff1(&mpidr); - uint8_t aff2 = psci_mpidr_get_Aff2(&mpidr); - uint8_t aff3 = psci_mpidr_get_Aff3(&mpidr); - - size_t gicr_stride = 1U << GICR_STRIDE_SHIFT; // 64k for v3, 128k for v4 + MPIDR_EL1_t mpidr = platform_cpu_index_to_mpidr(cpu); + uint8_t aff0 = MPIDR_EL1_get_Aff0(&mpidr); + uint8_t aff1 = MPIDR_EL1_get_Aff1(&mpidr); + uint8_t aff2 = MPIDR_EL1_get_Aff2(&mpidr); + uint8_t aff3 = MPIDR_EL1_get_Aff3(&mpidr); + + size_t gicr_stride = util_bit(GICR_STRIDE_SHIFT); // 64k for v3, 128k + // for v4 GICR_TYPER_t gicr_typer; - gicr_t *gicr = NULL; + gicr_t *gicr = NULL; +#if GICV3_HAS_ITS + paddr_t gicr_phys = 0U; +#endif // Search for the redistributor that matches this affinity value. We // assume that the stride that separates all redistributors is the same. - for (cpu_index_t i = 0U; cpulocal_index_valid(i); i++) { + for (index_t i = 0U; i < PLATFORM_GICR_COUNT; i++) { gicr = mapped_gicrs[i]; assert(gicr != NULL); +#if GICV3_HAS_ITS + gicr_phys = platform_gicrs_bases[i]; +#endif gicr_typer = atomic_load_relaxed(&gicr->rd.typer); if ((GICR_TYPER_get_Aff0(&gicr_typer) == aff0) && @@ -144,6 +227,12 @@ gicr_set_percpu(cpu_index_t cpu) } CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr = gicr; + +#if GICV3_HAS_ITS + // Inform the ITS driver of this CPU's redistributor + gicv3_its_init_cpu(cpu, gicr, gicr_phys, + GICR_TYPER_get_Processor_Num(&gicr_typer)); +#endif // GICV3_HAS_ITS } count_t @@ -161,6 +250,12 @@ gicv3_irq_max(void) } #endif +#if GICV3_HAS_LPI + if (gicv3_lpi_max_cache != 0U) { + result = gicv3_lpi_max_cache; + } +#endif + return result; } @@ -181,6 +276,12 @@ gicv3_get_irq_type(irq_t irq) } else if ((irq >= GIC_SPECIAL_INTIDS_BASE) && (irq < (GIC_SPECIAL_INTIDS_BASE + GIC_SPECIAL_INTIDS_NUM))) { type = GICV3_IRQ_TYPE_SPECIAL; +#if GICV3_HAS_LPI + } else if ((irq >= GIC_LPI_BASE) && + (irq < (GIC_LPI_BASE + GIC_LPI_NUM)) && + (irq <= gicv3_lpi_max_cache)) { + type = GICV3_IRQ_TYPE_LPI; +#endif #if GICV3_EXT_IRQS } else if ((irq >= GIC_PPI_EXT_BASE) && (irq < (GIC_PPI_EXT_BASE + GIC_PPI_EXT_NUM)) && @@ -208,6 +309,14 @@ gicv3_irq_is_percpu(irq_t irq) case GICV3_IRQ_TYPE_PPI: #if GICV3_EXT_IRQS case GICV3_IRQ_TYPE_PPI_EXT: +#endif +#if GICV3_HAS_LPI + case GICV3_IRQ_TYPE_LPI: + // LPIs are treated as percpu because we need to know which GICR + // to operate on. Note that we don't support platforms with an + // ITS unless they also have GICv4.1. If we did, we would have + // to reverse translate the LPI number to an event and device + // ID, and use them to queue an ITS command. #endif ret = true; break; @@ -230,8 +339,11 @@ is_irq_reserved(irq_t irq) { uint8_t ipriority; + // Assume that all CPUs have the same set of reserved SGIs / PPIs, + // so it doesn't matter which GICR we check. assert(irq <= gicv3_irq_max()); - gicr_t *gicr = CPULOCAL(gicr_cpu).gicr; + cpu_index_t cpu = cpulocal_check_index(cpulocal_get_index_unsafe()); + gicr_t *gicr = CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr; switch (gicv3_get_irq_type(irq)) { case GICV3_IRQ_TYPE_SGI: @@ -241,6 +353,12 @@ is_irq_reserved(irq_t irq) case GICV3_IRQ_TYPE_SPI: ipriority = atomic_load_relaxed(&gicd->ipriorityr[irq]); break; +#if GICV3_HAS_LPI + case GICV3_IRQ_TYPE_LPI: + // All LPIs are in group 1 and are not reserved. + ipriority = GIC_PRIORITY_DEFAULT; + break; +#endif #if GICV3_EXT_IRQS case GICV3_IRQ_TYPE_PPI_EXT: ipriority = atomic_load_relaxed( @@ -279,34 +397,227 @@ gicv3_irq_check(irq_t irq) return ret; } -// In boot_cold we map the distributor and all the redistributors, based on -// their base addresses and sizes read from the device tree. We then initialize -// the distributor. -void -gicv3_handle_boot_cold_init(cpu_index_t cpu) +static bool +gicv3_spi_is_enabled(irq_t irq) { - partition_t *hyp_partition = partition_get_private(); + uint32_t isenabler = + atomic_load_relaxed(&gicd->isenabler[GICD_ENABLE_GET_N(irq)]); + bool enabled = (isenabler & GIC_ENABLE_BIT(irq)) != 0U; - // FIXME: remove when read from device tree - paddr_t gicd_base = PLATFORM_GICD_BASE; - size_t gicd_size = 0x10000U; // GICD is always 64K - paddr_t gicr_base = PLATFORM_GICR_BASE; - size_t gicr_stride = (size_t)1U << GICR_STRIDE_SHIFT; - size_t gicr_size = PLATFORM_MAX_CORES << GICR_STRIDE_SHIFT; - size_t gits_size = 0U; +#if GICV3_EXT_IRQS +#error Check the extended IRQs +#endif - virt_range_result_t range = hyp_aspace_allocate( - util_balign_up(gicd_size, gicr_size) + gicr_size + gits_size); - if (range.e != OK) { - panic("gicv3: Address allocation failed."); + return enabled; +} + +static bool +gicv3_spi_get_route_cpu_affinity(const GICD_IROUTER_t *route, cpu_index_t *cpu) +{ + bool found = false; + + MPIDR_EL1_t mpidr = MPIDR_EL1_default(); + MPIDR_EL1_set_Aff0(&mpidr, GICD_IROUTER_get_Aff0(route)); + MPIDR_EL1_set_Aff1(&mpidr, GICD_IROUTER_get_Aff1(route)); + MPIDR_EL1_set_Aff2(&mpidr, GICD_IROUTER_get_Aff2(route)); + MPIDR_EL1_set_Aff3(&mpidr, GICD_IROUTER_get_Aff3(route)); + + platform_mpidr_mapping_t mapping = platform_cpu_get_mpidr_mapping(); + + if (platform_cpu_map_mpidr_valid(&mapping, mpidr)) { + *cpu = (cpu_index_t)platform_cpu_map_mpidr_to_index(&mapping, + mpidr); + found = true; + } + + return found; +} + +static void +gicv3_spi_set_route_cpu_affinity(GICD_IROUTER_t *route, cpu_index_t cpu) +{ + assert(cpu < PLATFORM_MAX_CORES); + + MPIDR_EL1_t mpidr = platform_cpu_index_to_mpidr(cpu); + GICD_IROUTER_set_IRM(route, 0); + GICD_IROUTER_set_Aff0(route, MPIDR_EL1_get_Aff0(&mpidr)); + GICD_IROUTER_set_Aff1(route, MPIDR_EL1_get_Aff1(&mpidr)); + GICD_IROUTER_set_Aff2(route, MPIDR_EL1_get_Aff2(&mpidr)); + GICD_IROUTER_set_Aff3(route, MPIDR_EL1_get_Aff3(&mpidr)); +} + +#if GICV3_HAS_LPI || GICV3_HAS_ITS +static void +gicv3_boot_cold_init_lpis(GICD_TYPER_t typer, partition_t *hyp_partition, + GICR_TYPER_t gicr_typer) +{ + // Check that LPIs are actually supported, and determine the maximum + // LPI number. + bool have_lpis = GICD_TYPER_get_LPIS(&typer); + count_t idbits = GICD_TYPER_get_IDbits(&typer); + count_t lpibits = GICD_TYPER_get_num_LPIs(&typer); + if (!have_lpis || (idbits < 13U)) { + gicv3_lpi_max_cache = 0U; + } else if (lpibits == 0U) { + gicv3_lpi_max_cache = (count_t)util_mask(idbits + 1U); + } else { + gicv3_lpi_max_cache = + GIC_LPI_BASE + (count_t)util_mask(lpibits + 1U); + } + + // Limit configured LPIs to the hard-coded maximum if the hardware + // supports more than we need. + gicv3_lpi_max_cache = + util_min(gicv3_lpi_max_cache, GIC_LPI_BASE + GIC_LPI_NUM - 1U); + + if (gicv3_lpi_max_cache > 0U) { + // Set all LPI property entries to disabled and default + // priority. + gic_lpi_prop_t lpi_prop = gic_lpi_prop_default(); + gic_lpi_prop_set_priority(&lpi_prop, GIC_PRIORITY_DEFAULT); + for (count_t i = 0U; i < GIC_LPI_NUM; i++) { + gic_lpi_prop_table[i] = lpi_prop; + } + +#if defined(GICV3_CACHE_INCOHERENT) && GICV3_CACHE_INCOHERENT + CACHE_CLEAN_OBJECT(gic_lpi_prop_table); +#endif + + // Calculate the GICR_PROPBASE value that points to the config + // table + gic_lpi_propbase = GICR_PROPBASER_default(); + // IDbits: the number of bits required to represent all + // configured LPIs, minus one + GICR_PROPBASER_set_IDbits( + &gic_lpi_propbase, + 32U - compiler_clz((uint32_t)gicv3_lpi_max_cache) - 1U); + // InnerCache == 7: Inner write back, read + write alloc + GICR_PROPBASER_set_InnerCache(&gic_lpi_propbase, 7U); + // Shareability == 1: Inner shareable + GICR_PROPBASER_set_Shareability(&gic_lpi_propbase, 1U); + // PA: physical address of the LPI property table + GICR_PROPBASER_set_PA( + &gic_lpi_propbase, + partition_virt_to_phys(hyp_partition, + (uintptr_t)&gic_lpi_prop_table)); + // OuterCache == 0: Inner and outer attributes are the same + GICR_PROPBASER_set_OuterCache(&gic_lpi_propbase, 0U); + + // Allocate LPI pending bitmap and calculate GICR_PENDBASE per + // CPU + for (cpu_index_t i = 0U; i < PLATFORM_MAX_CORES; i++) { + // One bit per IRQ number (including the first 8192 + // which are not LPIs); must be 64k-aligned and + // zero-initialised. We allocate these from the heap + // rather than BSS in the hope of not wasting the space + // between them (~0.5MiB total) + size_t lpi_bitmap_sz = (size_t)gicv3_lpi_max_cache / 8U; + void_ptr_result_t alloc_r = partition_alloc( + hyp_partition, lpi_bitmap_sz, + (size_t)util_bit(GICR_PENDBASER_PA_SHIFT)); + if (alloc_r.e != OK) { + panic("Unable to allocate physical LPI pending bitmap"); + } + (void)memset_s(alloc_r.r, lpi_bitmap_sz, 0, + lpi_bitmap_sz); + CPULOCAL_BY_INDEX(gicr_cpu, i).lpi_pending_bitmap = + (register_t *)alloc_r.r; + + GICR_PENDBASER_t pendbase = GICR_PENDBASER_default(); + // InnerCache == 7: Inner write back, read + write alloc + GICR_PENDBASER_set_InnerCache(&pendbase, 7U); + // Shareability == 1: Inner shareable + GICR_PENDBASER_set_Shareability(&pendbase, 1U); + // PA: 64k-aligned physical address of the LPI pending + // bitmap + paddr_t lpi_bitmap_phys = partition_virt_to_phys( + hyp_partition, (uintptr_t)alloc_r.r); + assert(util_is_p2aligned(lpi_bitmap_phys, + GICR_PENDBASER_PA_SHIFT)); + GICR_PENDBASER_set_PA(&pendbase, lpi_bitmap_phys); + // OuterCache == 0: Inner and outer attributes are the + // same + GICR_PENDBASER_set_OuterCache(&pendbase, 0U); + // PTZ: table has been zeroed + GICR_PENDBASER_set_PTZ(&pendbase, true); + CPULOCAL_BY_INDEX(gicr_cpu, i).lpi_pendbase = pendbase; + +#if defined(GICV3_CACHE_INCOHERENT) && GICV3_CACHE_INCOHERENT + CACHE_CLEAN_RANGE(lpi_pending_bitmap, lpi_bitmap_sz); +#endif + +#if defined(GICV3_ENABLE_VPE) && GICV3_ENABLE_VPE && GICV3_HAS_VLPI_V4_1 + spinlock_init( + &CPULOCAL_BY_INDEX(gicr_cpu, i).vsgi_query_lock); +#endif + } + } + +#if GICV3_HAS_ITS + // Initialise the ITSs + gicv3_its_init(&mapped_gitss, + (count_t)4U - (count_t)GICR_TYPER_get_CommonLPIAff( + &gicr_typer)); +#endif // GICV3_HAS_ITS + +#if GICV3_HAS_VLPI_V4_1 + // Check the supported vPE range + GICD_TYPER2_t typer2 = atomic_load_relaxed(&gicd->typer2); + count_t vpe_bits = GICD_TYPER2_get_VIL(&typer2) + ? 16U + : (GICD_TYPER2_get_VID(&typer2) + 1U); + assert(GICV3_ITS_VPES < ((count_t)util_bit(vpe_bits))); +#endif // GICV3_HAS_VLPI_V4_1 +} +#endif + +#if GICV3_HAS_ITS +static void +gicv3_map_its(size_t gicr_size, size_t gits_size, partition_t *hyp_partition, + size_t gits_stride) +{ + // Map the ITS registers and calculate their virtual addresses. + mapped_gitss[0] = (gits_t *)util_p2align_up( + (uintptr_t)mapped_gicrs[0] + gicr_size, GITS_STRIDE_SHIFT); + + // Note: we are assuming here that the ITSs are physically contiguous, + // which is a reasonable configuration but is not actually required by + // the spec. This is not yet a significant issue because we have no + // platform with multiple ITSs. + pgtable_hyp_start(); + error_t ret = pgtable_hyp_map(hyp_partition, (uintptr_t)mapped_gitss[0], + gits_size, platform_gits_base, + PGTABLE_HYP_MEMTYPE_NOSPEC_NOCOMBINE, + PGTABLE_ACCESS_RW, + VMSA_SHAREABILITY_NON_SHAREABLE); + if (ret != OK) { + panic("gicv3: Mapping of ITSs failed."); + } + pgtable_hyp_commit(); +#if (PLATFORM_GITS_COUNT > 1U) + for (cpu_index_t i = 1U; i < PLATFORM_GITS_COUNT; i++) { + mapped_gitss[i] = (gits_t *)((uintptr_t)mapped_gitss[i - 1U] + + gits_stride); } +#else + (void)gits_stride; +#endif +} +#endif + +static void +gicv3_map_gicd_and_gicrs(size_t gicr_size, partition_t *hyp_partition, + size_t gicd_size, virt_range_result_t range) +{ + paddr_t gicr_base = PLATFORM_GICR_BASE; + size_t gicr_stride = (size_t)util_bit(GICR_STRIDE_SHIFT); pgtable_hyp_start(); // Map the distributor gicd = (gicd_t *)range.r.base; error_t ret = pgtable_hyp_map(hyp_partition, (uintptr_t)gicd, gicd_size, - gicd_base, + platform_gicd_base, PGTABLE_HYP_MEMTYPE_NOSPEC_NOCOMBINE, PGTABLE_ACCESS_RW, VMSA_SHAREABILITY_NON_SHAREABLE); @@ -315,11 +626,12 @@ gicv3_handle_boot_cold_init(cpu_index_t cpu) } // Map the redistributors and calculate their addresses - phys_gicrs[0] = gicr_base; + platform_gicrs_bases[0] = gicr_base; mapped_gicrs[0] = - (gicr_t *)(range.r.base + util_balign_up(gicd_size, gicr_size)); + (gicr_t *)(range.r.base + + util_p2align_up(gicd_size, GICR_STRIDE_SHIFT)); ret = pgtable_hyp_map(hyp_partition, (uintptr_t)mapped_gicrs[0], - gicr_size, phys_gicrs[0], + gicr_size, platform_gicrs_bases[0], PGTABLE_HYP_MEMTYPE_NOSPEC_NOCOMBINE, PGTABLE_ACCESS_RW, VMSA_SHAREABILITY_NON_SHAREABLE); @@ -329,10 +641,11 @@ gicv3_handle_boot_cold_init(cpu_index_t cpu) pgtable_hyp_commit(); - for (cpu_index_t i = 1; cpulocal_index_valid(i); i++) { + for (index_t i = 1; i < PLATFORM_GICR_COUNT; i++) { mapped_gicrs[i] = (gicr_t *)((uintptr_t)mapped_gicrs[i - 1U] + gicr_stride); - phys_gicrs[i] = phys_gicrs[i - 1U] + gicr_stride; + platform_gicrs_bases[i] = + platform_gicrs_bases[i - 1U] + gicr_stride; // Ensure that DPG1NS is set, so the GICD does not try to route // 1-of-N interrupts to this GICR before we have set it up (when @@ -344,6 +657,43 @@ gicv3_handle_boot_cold_init(cpu_index_t cpu) GICR_CTLR_set_DPG1NS(&gicr_ctlr, true); atomic_store_relaxed(&mapped_gicrs[i]->rd.ctlr, gicr_ctlr); } +} + +// In boot_cold we map the distributor and all the redistributors, based on +// their base addresses and sizes read from the device tree. We then initialize +// the distributor. +void +gicv3_handle_boot_cold_init(cpu_index_t cpu) +{ + partition_t *hyp_partition = partition_get_private(); + + // FIXME: remove when read from device tree + size_t gicr_size = PLATFORM_GICR_COUNT * util_bit(GICR_STRIDE_SHIFT); + size_t gicd_size = 0x10000U; // GICD is always 64K + + static_assert(PLATFORM_GICR_SIZE == PLATFORM_GICR_COUNT + << GICR_STRIDE_SHIFT, + "bad PLATFORM_GICR_SIZE"); + +#if GICV3_HAS_ITS + size_t gits_stride = (size_t)util_bit(GITS_STRIDE_SHIFT); + size_t gits_size = (size_t)PLATFORM_GITS_COUNT << GITS_STRIDE_SHIFT; +#else + size_t gits_size = 0U; +#endif + + virt_range_result_t range = hyp_aspace_allocate( + util_p2align_up(gicd_size, GICR_STRIDE_SHIFT) + gicr_size + + gits_size); + if (range.e != OK) { + panic("gicv3: Address allocation failed."); + } + + gicv3_map_gicd_and_gicrs(gicr_size, hyp_partition, gicd_size, range); + +#if GICV3_HAS_ITS + gicv3_map_its(gicr_size, gits_size, hyp_partition, gits_stride); +#endif // Disable the distributor atomic_store_relaxed(&gicd->ctlr, @@ -375,7 +725,7 @@ gicv3_handle_boot_cold_init(cpu_index_t cpu) #if GICV3_EXT_IRQS || GICV3_HAS_ITS // Pick an arbitrary GICR to probe extended PPI and common LPI affinity // (we assume that these are the same across all GICRs) - gicr_t *gicr = mapped_gicrs[0]; + gicr_t *gicr = mapped_gicrs[0]; GICR_TYPER_t gicr_typer = atomic_load_relaxed(&gicr->rd.typer); #endif @@ -430,12 +780,16 @@ gicv3_handle_boot_cold_init(cpu_index_t cpu) } #endif +#if GICV3_HAS_LPI + gicv3_boot_cold_init_lpis(typer, hyp_partition, gicr_typer); +#endif // GICV3_HAS_LPI + // Route all SPIs to the boot CPU by default. - psci_mpidr_t mpidr = platform_cpu_index_to_mpidr(cpu); - uint8_t aff0 = psci_mpidr_get_Aff0(&mpidr); - uint8_t aff1 = psci_mpidr_get_Aff1(&mpidr); - uint8_t aff2 = psci_mpidr_get_Aff2(&mpidr); - uint8_t aff3 = psci_mpidr_get_Aff3(&mpidr); + MPIDR_EL1_t mpidr = platform_cpu_index_to_mpidr(cpu); + uint8_t aff0 = MPIDR_EL1_get_Aff0(&mpidr); + uint8_t aff1 = MPIDR_EL1_get_Aff1(&mpidr); + uint8_t aff2 = MPIDR_EL1_get_Aff2(&mpidr); + uint8_t aff3 = MPIDR_EL1_get_Aff3(&mpidr); GICD_IROUTER_t irouter = GICD_IROUTER_default(); GICD_IROUTER_set_IRM(&irouter, false); GICD_IROUTER_set_Aff0(&irouter, aff0); @@ -474,14 +828,14 @@ gicv3_handle_boot_cold_init(cpu_index_t cpu) for (cpu_index_t i = 0U; i < PLATFORM_MAX_CORES; i++) { mpidr = platform_cpu_index_to_mpidr(i); - aff0 = psci_mpidr_get_Aff0(&mpidr); - aff1 = psci_mpidr_get_Aff1(&mpidr); - aff2 = psci_mpidr_get_Aff2(&mpidr); - aff3 = psci_mpidr_get_Aff3(&mpidr); + aff0 = MPIDR_EL1_get_Aff0(&mpidr); + aff1 = MPIDR_EL1_get_Aff1(&mpidr); + aff2 = MPIDR_EL1_get_Aff2(&mpidr); + aff3 = MPIDR_EL1_get_Aff3(&mpidr); ICC_SGIR_EL1_t icc_sgi1r = ICC_SGIR_EL1_default(); ICC_SGIR_EL1_set_TargetList(&icc_sgi1r, - (uint16_t)(1U << (aff0 % 16U))); + (uint16_t)util_bit(aff0 % 16U)); ICC_SGIR_EL1_set_RS(&icc_sgi1r, aff0 / 16U); ICC_SGIR_EL1_set_Aff1(&icc_sgi1r, aff1); ICC_SGIR_EL1_set_Aff2(&icc_sgi1r, aff2); @@ -492,7 +846,15 @@ gicv3_handle_boot_cold_init(cpu_index_t cpu) // Set up gicr for each CPU for (cpu_index_t i = 0U; i < PLATFORM_MAX_CORES; i++) { - gicr_set_percpu(i); + if (platform_cpu_exists(i)) { + gicr_set_percpu(i); + + // Set online state for interrupt migration at poweroff + // Set boot cpu as online. Secondary CPUs will be set + // by 'power_cpu_online' handler + gicr_cpu_t *gc = &CPULOCAL_BY_INDEX(gicr_cpu, i); + gc->online = (i == cpu); + } } #if GICV3_HAS_GICD_ICLAR @@ -508,6 +870,7 @@ gicv3_handle_boot_cold_init(cpu_index_t cpu) #endif spinlock_init(&bitmap_update_lock); + spinlock_init(&spi_route_lock); } // In the boot_cpu_cold we search for the redistributor that corresponds to the @@ -575,6 +938,54 @@ gicv3_handle_boot_cpu_cold_init(cpu_index_t cpu) GICR_CTLR_set_DPG1NS(&ctlr, false); atomic_store_relaxed(&gicr->rd.ctlr, ctlr); +#if GICV3_HAS_LPI + GICR_TYPER_t typer = atomic_load_relaxed(&gicr->rd.typer); + + assert(GICR_TYPER_get_PLPIS(&typer)); + +#if !GICV3_HAS_ITS + // Direct LPIs must be supported if there is no ITS; otherwise there + // would be no way to generate LPIs. + // + // Note that three of the five registers covered by this feature, + // INVLPIR, INVALLR and SYNCR, are also mandatory in GICv4.1; however, + // SETLPIR and CLRLPIR are not, so this bit might not be set by a + // GICv4.1 implementation. In GICv4.1, a separate ID bit is added to + // indicate support for those three registers; see below. + assert(GICR_TYPER_get_DirectLPI(&typer)); +#endif // !GICV3_HAS_ITS + +#if GICV3_HAS_VLPI_V4_1 + // GICv4.1 requires the GICR to use the vPE-format VPENDBASER, and to + // support the invalidate registers (a subset of DirectLPI), VSGI + // delivery, and polling for completion of vPE scheduling. + assert(GICR_TYPER_get_RVPEID(&typer) && GICR_TYPER_get_VSGI(&typer) && + GICR_TYPER_get_Dirty(&typer) && GICR_CTLR_get_IR(&ctlr)); +#elif GICV3_HAS_VLPI + // GICv4.0 requires the GICR not to use the vPE-format VPENDBASER. + assert(!GICR_TYPER_get_RVPEID(&typer)); +#endif + + if (gicv3_lpi_max_cache > 0U) { + // Set the base registers + atomic_store_relaxed(&gicr->rd.propbaser, gic_lpi_propbase); + GICR_PENDBASER_t *pendbase = + &CPULOCAL_BY_INDEX(gicr_cpu, cpu).lpi_pendbase; + atomic_store_relaxed(&gicr->rd.pendbaser, *pendbase); + + // Read back pendbaser to assert that shareability is nonzero + // (i.e. accesses to the table are cache-coherent) and also to + // clear PTZ in case we need to rewrite it + *pendbase = atomic_load_relaxed(&gicr->rd.pendbaser); + assert(GICR_PENDBASER_get_Shareability(pendbase) != 0U); + + // Enable LPIs (note: this may be permanent until reset) + GICR_CTLR_set_Enable_LPIs(&ctlr, true); + atomic_store_release(&gicr->rd.ctlr, ctlr); + } + +#endif // GICV3_HAS_LPI + #if PLATFORM_IPI_LINES > ENUM_IPI_REASON_MAX_VALUE // Enable the SGIs used by IPIs atomic_store_release(&gicr->sgi.isenabler0, @@ -589,7 +1000,15 @@ gicv3_handle_boot_cpu_cold_init(cpu_index_t cpu) void gicv3_handle_boot_cpu_warm_init(void) { - struct asm_ordering_dummy gic_init_order; + asm_ordering_dummy_t gic_init_order; + +#if GICV3_HAS_LPI + GICR_CTLR_t ctlr = + atomic_load_relaxed(&CPULOCAL(gicr_cpu).gicr->rd.ctlr); + + // LPIs should have already been enabled + assert(GICR_CTLR_get_Enable_LPIs(&ctlr) == (gicv3_lpi_max_cache != 0U)); +#endif // Enable system register access and disable FIQ and IRQ bypass ICC_SRE_EL2_t icc_sre = ICC_SRE_EL2_default(); @@ -662,6 +1081,11 @@ gicv3_handle_power_cpu_suspend(void) atomic_store_relaxed(&gicr->rd.waker, waker); #endif +#if defined(GICV3_ENABLE_VPE) && GICV3_ENABLE_VPE + // Ensure that the GICR has finished descheduling the last vPE. + gicv3_vpe_sync_deschedule(cpulocal_get_index(), false); +#endif + #if GICV3_HAS_SECURITY_DISABLED // Wait for gicr to be off GICR_WAKER_t waker_read; @@ -676,7 +1100,7 @@ gicv3_handle_power_cpu_suspend(void) void gicv3_handle_power_cpu_resume(void) { - struct asm_ordering_dummy gic_enable_order; + asm_ordering_dummy_t gic_enable_order; // Enable group 1 interrupts ICC_IGRPEN_EL1_t icc_grpen1 = ICC_IGRPEN_EL1_default(); @@ -719,17 +1143,56 @@ gicv3_irq_enable(irq_t irq) switch (irq_type) { case GICV3_IRQ_TYPE_SPI: + // Ensure the route is still valid if we enable the irq for the + // first time + + // Take the SPI lock so we can fetch the current route + // If it has a specific target CPU (not 1:N) ensure it is online + spinlock_acquire(&spi_route_lock); + + if (!gicv3_spi_is_enabled(irq)) { + GICD_IROUTER_t route = + atomic_load_relaxed(&gicd->irouter[irq]); + if (!GICD_IROUTER_get_IRM(&route)) { + cpu_index_t target; + gicr_cpu_t *gc = NULL; + bool valid = gicv3_spi_get_route_cpu_affinity( + &route, &target); + if (valid) { + gc = &CPULOCAL_BY_INDEX(gicr_cpu, + target); + } + // Set affinity to this CPU if needed + if (!valid || !gc->online) { + gc = &CPULOCAL(gicr_cpu); + assert(gc->online); + gicv3_spi_set_route_cpu_affinity( + &route, cpulocal_get_index()); + (void)gicv3_spi_set_route_internal( + irq, route); + } + } + } + atomic_store_release(&gicd->isenabler[GICD_ENABLE_GET_N(irq)], GIC_ENABLE_BIT(irq)); + + spinlock_release(&spi_route_lock); + break; #if GICV3_EXT_IRQS case GICV3_IRQ_TYPE_SPI_EXT: // Extended SPI +#error TODO check route is valid + atomic_store_release(&gicd->isenabler_e[GICD_ENABLE_GET_N( irq - GIC_SPI_EXT_BASE)], GIC_ENABLE_BIT(irq - GIC_SPI_EXT_BASE)); break; +#endif +#if GICV3_HAS_LPI + case GICV3_IRQ_TYPE_LPI: #endif case GICV3_IRQ_TYPE_SGI: case GICV3_IRQ_TYPE_PPI: @@ -748,7 +1211,7 @@ gicv3_irq_enable_percpu(irq_t irq, cpu_index_t cpu) { assert(irq <= gicv3_irq_max()); - gicr_t *gicr = CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr; + gicr_t *gicr = CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr; gicv3_irq_type_t irq_type = gicv3_get_irq_type(irq); switch (irq_type) { @@ -767,6 +1230,13 @@ gicv3_irq_enable_percpu(irq_t irq, cpu_index_t cpu) GIC_ENABLE_BIT(irq - GIC_PPI_EXT_BASE)); break; } +#endif +#if GICV3_HAS_LPI + case GICV3_IRQ_TYPE_LPI: + gic_lpi_prop_set_enable(&gic_lpi_prop_table[irq - GIC_LPI_BASE], + true); + gicv3_lpi_inv_by_id(cpu, irq); + break; #endif case GICV3_IRQ_TYPE_SPI: #if GICV3_EXT_IRQS @@ -797,7 +1267,7 @@ gicv3_irq_disable(irq_t irq) case GICV3_IRQ_TYPE_SPI: atomic_store_relaxed(&gicd->icenabler[GICD_ENABLE_GET_N(irq)], GIC_ENABLE_BIT(irq)); - gicd_wait_for_write(); + (void)gicd_wait_for_write(); break; #if GICV3_EXT_IRQS @@ -806,9 +1276,12 @@ gicv3_irq_disable(irq_t irq) atomic_store_relaxed(&gicd->icenabler_e[GICD_ENABLE_GET_N( irq - GIC_SPI_EXT_BASE)], GIC_ENABLE_BIT(irq - GIC_SPI_EXT_BASE)); - gicd_wait_for_write(); + (void)gicd_wait_for_write(); break; } +#endif +#if GICV3_HAS_LPI + case GICV3_IRQ_TYPE_LPI: #endif case GICV3_IRQ_TYPE_SGI: case GICV3_IRQ_TYPE_PPI: @@ -847,6 +1320,9 @@ gicv3_irq_cancel_nowait(irq_t irq) // As above, there is no way to guarantee completion. break; } +#endif +#if GICV3_HAS_LPI + case GICV3_IRQ_TYPE_LPI: #endif case GICV3_IRQ_TYPE_SGI: case GICV3_IRQ_TYPE_PPI: @@ -865,7 +1341,7 @@ gicv3_irq_disable_percpu_nowait(irq_t irq, cpu_index_t cpu) { assert(irq <= gicv3_irq_max()); - gicr_t *gicr = CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr; + gicr_t *gicr = CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr; gicv3_irq_type_t irq_type = gicv3_get_irq_type(irq); switch (irq_type) { @@ -883,6 +1359,13 @@ gicv3_irq_disable_percpu_nowait(irq_t irq, cpu_index_t cpu) GIC_ENABLE_BIT(irq - GIC_PPI_EXT_BASE)); break; } +#endif +#if GICV3_HAS_LPI + case GICV3_IRQ_TYPE_LPI: + gic_lpi_prop_set_enable(&gic_lpi_prop_table[irq - GIC_LPI_BASE], + false); + gicv3_lpi_inv_by_id(cpu, irq); + break; #endif case GICV3_IRQ_TYPE_SPI: #if GICV3_EXT_IRQS @@ -899,7 +1382,15 @@ void gicv3_irq_disable_percpu(irq_t irq, cpu_index_t cpu) { gicv3_irq_disable_percpu_nowait(irq, cpu); +#if GICV3_HAS_LPI + if (gicv3_get_irq_type(irq) == GICV3_IRQ_TYPE_LPI) { + gicr_wait_for_sync(CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr); + } else { + gicr_wait_for_write(CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr); + } +#else gicr_wait_for_write(CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr); +#endif } void @@ -921,7 +1412,7 @@ gicv3_irq_set_trigger_percpu(irq_t irq, irq_trigger_t trigger, cpu_index_t cpu) { irq_trigger_result_t ret; - gicr_t *gicr = CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr; + gicr_t *gicr = CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr; gicv3_irq_type_t irq_type = gicv3_get_irq_type(irq); // We do not support this behavior for now @@ -1015,6 +1506,9 @@ gicv3_irq_set_trigger_percpu(irq_t irq, irq_trigger_t trigger, cpu_index_t cpu) case GICV3_IRQ_TYPE_SPI: #if GICV3_EXT_IRQS case GICV3_IRQ_TYPE_SPI_EXT: +#endif +#if GICV3_HAS_LPI + case GICV3_IRQ_TYPE_LPI: #endif case GICV3_IRQ_TYPE_SPECIAL: case GICV3_IRQ_TYPE_RESERVED: @@ -1207,6 +1701,12 @@ gicv3_irq_set_trigger(irq_t irq, irq_trigger_t trigger) break; } +#endif +#if GICV3_HAS_LPI + case GICV3_IRQ_TYPE_LPI: + // LPIs are always message-signalled + ret = irq_trigger_result_ok(IRQ_TRIGGER_MESSAGE); + break; #endif case GICV3_IRQ_TYPE_SPECIAL: case GICV3_IRQ_TYPE_RESERVED: @@ -1221,11 +1721,14 @@ gicv3_irq_set_trigger(irq_t irq, irq_trigger_t trigger) return ret; } -error_t -gicv3_spi_set_route(irq_t irq, GICD_IROUTER_t route) +static error_t +gicv3_spi_set_route_internal(irq_t irq, GICD_IROUTER_t route) + REQUIRE_SPINLOCK(spi_route_lock) { error_t ret; + assert_preempt_disabled(); + switch (gicv3_get_irq_type(irq)) { case GICV3_IRQ_TYPE_SPI: atomic_store_relaxed(&gicd->irouter[irq - GIC_SPI_BASE], route); @@ -1242,6 +1745,9 @@ gicv3_spi_set_route(irq_t irq, GICD_IROUTER_t route) case GICV3_IRQ_TYPE_PPI: #if GICV3_EXT_IRQS case GICV3_IRQ_TYPE_PPI_EXT: +#endif +#if GICV3_HAS_LPI + case GICV3_IRQ_TYPE_LPI: #endif case GICV3_IRQ_TYPE_SPECIAL: case GICV3_IRQ_TYPE_RESERVED: @@ -1253,6 +1759,45 @@ gicv3_spi_set_route(irq_t irq, GICD_IROUTER_t route) return ret; } +error_t +gicv3_spi_set_route(irq_t irq, GICD_IROUTER_t route) +{ + error_t ret = ERROR_ARGUMENT_INVALID; + + spinlock_acquire(&spi_route_lock); + + // If the SPI is enabled and routes to a specific CPU we need to check + // it is online. The route is also checked when the SPI is enabled. + // If using 1:N routing, the GIC decides which CPU should get it. + if (!GICD_IROUTER_get_IRM(&route)) { + cpu_index_t cpu; + + // Determine the target CPU for the route + if (!gicv3_spi_get_route_cpu_affinity(&route, &cpu)) { + goto out; + } + + // If the interrupt is enabled check the target CPU is online + if (gicv3_spi_is_enabled(irq)) { + gicr_cpu_t *gc = &CPULOCAL_BY_INDEX(gicr_cpu, cpu); + + // If the target CPU is offline adjust the route + if (!gc->online) { + gc = &CPULOCAL(gicr_cpu); + assert(gc->online); + gicv3_spi_set_route_cpu_affinity( + &route, cpulocal_get_index()); + } + } + } + + ret = gicv3_spi_set_route_internal(irq, route); + +out: + spinlock_release(&spi_route_lock); + return ret; +} + #if GICV3_HAS_GICD_ICLAR error_t gicv3_spi_set_classes(irq_t irq, bool class0, bool class1) @@ -1318,6 +1863,9 @@ gicv3_spi_set_classes(irq_t irq, bool class0, bool class1) case GICV3_IRQ_TYPE_PPI: #if GICV3_EXT_IRQS case GICV3_IRQ_TYPE_PPI_EXT: +#endif +#if GICV3_HAS_LPI + case GICV3_IRQ_TYPE_LPI: #endif case GICV3_IRQ_TYPE_SPECIAL: case GICV3_IRQ_TYPE_RESERVED: @@ -1360,7 +1908,7 @@ gicv3_irq_acknowledge(void) assert(intid <= ENUM_IPI_REASON_MAX_VALUE); trigger_platform_ipi_event((ipi_reason_t)intid); #else - trigger_platform_ipi_event(); + (void)trigger_platform_ipi_event(); #endif gicv3_irq_deactivate(intid); ret.e = ERROR_RETRY; @@ -1394,6 +1942,13 @@ gicv3_irq_deactivate(irq_t irq) { assert(irq <= gicv3_irq_max()); +#if GICV3_HAS_LPI + if (irq >= GIC_LPI_BASE) { + // Deactivation is meaningless for LPIs (apart from the priority + // drop which is done separately), so skip the barriers and + // register write + } else +#endif { ICC_DIR_EL1_t dir = ICC_DIR_EL1_default(); @@ -1435,6 +1990,9 @@ gicv3_irq_deactivate_percpu(irq_t irq, cpu_index_t cpu) case GICV3_IRQ_TYPE_SPI: #if GICV3_EXT_IRQS case GICV3_IRQ_TYPE_SPI_EXT: +#endif +#if GICV3_HAS_LPI + case GICV3_IRQ_TYPE_LPI: #endif case GICV3_IRQ_TYPE_SPECIAL: case GICV3_IRQ_TYPE_RESERVED: @@ -1518,7 +2076,7 @@ platform_ipi_one(cpu_index_t cpu) #if defined(INTERFACE_VCPU) && INTERFACE_VCPU && GICV3_HAS_1N -void +error_t gicv3_handle_vcpu_poweron(thread_t *vcpu) { if (vcpu_option_flags_get_hlos_vm(&vcpu->vcpu_options)) { @@ -1529,11 +2087,13 @@ gicv3_handle_vcpu_poweron(thread_t *vcpu) // No locking, because we assume that the hlos_vm flag is only // set on one VCPU per physical CPU. Also we are assuming here // that DPGs are implemented. - gicr_t *gicr = CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr; + gicr_t *gicr = CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr; GICR_CTLR_t gicr_ctlr = atomic_load_relaxed(&gicr->rd.ctlr); GICR_CTLR_set_DPG1NS(&gicr_ctlr, false); atomic_store_relaxed(&gicr->rd.ctlr, gicr_ctlr); } + + return OK; } error_t @@ -1547,7 +2107,7 @@ gicv3_handle_vcpu_poweroff(thread_t *vcpu) // No locking, because we assume that the hlos_vm flag is only // set on one VCPU per physical CPU. Also we are assuming here // that DPGs are implemented. - gicr_t *gicr = CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr; + gicr_t *gicr = CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr; GICR_CTLR_t gicr_ctlr = atomic_load_relaxed(&gicr->rd.ctlr); GICR_CTLR_set_DPG1NS(&gicr_ctlr, true); atomic_store_relaxed(&gicr->rd.ctlr, gicr_ctlr); @@ -1557,3 +2117,528 @@ gicv3_handle_vcpu_poweroff(thread_t *vcpu) } #endif // INTERFACE_VCPU && GICV3_HAS_1N + +#if GICV3_HAS_LPI +const irq_t platform_irq_msi_base = GIC_LPI_BASE; + +irq_t +platform_irq_msi_max(void) +{ + return gicv3_lpi_max_cache; +} + +#if !GICV3_HAS_ITS || GICV3_HAS_VLPI_V4_1 +void +gicv3_lpi_inv_by_id(cpu_index_t cpu, irq_t lpi) +{ + gicr_t *gicr = CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr; + + GICR_INVLPIR_t invlpir = GICR_INVLPIR_default(); + GICR_INVLPIR_set_pINTID(&invlpir, lpi); + atomic_store_release(&gicr->rd.invlpir, invlpir); +} + +// This is an ITS function for GICv3 with ITS or GICv4.0, and a GICR function +// for GICv3 without ITS or GICv4.1. See gicv3_its.c for the ITS version. +void +gicv3_lpi_inv_all(cpu_index_t cpu) +{ + gicr_t *gicr = CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr; + + GICR_INVALLR_t invallr = GICR_INVALLR_default(); + atomic_store_release(&gicr->rd.invallr, invallr); +} + +// This is an ITS function for GICv3 with ITS or GICv4.0, and a GICR function +// for GICv3 without ITS or GICv4.1. See gicv3_its.c for the ITS version. +bool +gicv3_lpi_inv_pending(cpu_index_t cpu) +{ + gicr_t *gicr = CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr; + GICR_SYNCR_t syncr = atomic_load_acquire(&gicr->rd.syncr); + return GICR_SYNCR_get_Busy(&syncr); +} +#endif // !GICV3_HAS_ITS || GICV3_HAS_VLPI_V4_1 + +#if defined(GICV3_ENABLE_VPE) && GICV3_ENABLE_VPE + +#if GICV3_HAS_VLPI_V4_1 +void +gicv3_vlpi_inv_by_id(thread_t *vcpu, virq_t vlpi) +{ + scheduler_lock(vcpu); + if ((vcpu->gicv3_its_doorbell != NULL) && + (vcpu->scheduler_affinity < PLATFORM_MAX_CORES)) { + gicr_t *gicr = + CPULOCAL_BY_INDEX(gicr_cpu, vcpu->scheduler_affinity) + .gicr; + + GICR_INVLPIR_t invlpir = GICR_INVLPIR_default(); + GICR_INVLPIR_set_V(&invlpir, true); + GICR_INVLPIR_set_vPEID(&invlpir, vcpu->gicv3_its_vpe_id); + GICR_INVLPIR_set_pINTID(&invlpir, vlpi); + atomic_store_release(&gicr->rd.invlpir, invlpir); + } + scheduler_unlock(vcpu); +} + +void +gicv3_vlpi_inv_all(thread_t *vcpu) +{ + scheduler_lock(vcpu); + if ((vcpu->gicv3_its_doorbell != NULL) && + (vcpu->scheduler_affinity < PLATFORM_MAX_CORES)) { + gicr_t *gicr = + CPULOCAL_BY_INDEX(gicr_cpu, vcpu->scheduler_affinity) + .gicr; + + GICR_INVALLR_t invallr = GICR_INVALLR_default(); + GICR_INVALLR_set_V(&invallr, true); + GICR_INVALLR_set_vPEID(&invallr, vcpu->gicv3_its_vpe_id); + atomic_store_release(&gicr->rd.invallr, invallr); + } + scheduler_unlock(vcpu); +} + +bool +gicv3_vlpi_inv_pending(thread_t *vcpu) +{ + bool busy = false; + + scheduler_lock(vcpu); + if ((vcpu->gicv3_its_doorbell != NULL) && + (vcpu->scheduler_affinity < PLATFORM_MAX_CORES)) { + gicr_t *gicr = + CPULOCAL_BY_INDEX(gicr_cpu, vcpu->scheduler_affinity) + .gicr; + GICR_SYNCR_t syncr = atomic_load_acquire(&gicr->rd.syncr); + busy = GICR_SYNCR_get_Busy(&syncr); + } + scheduler_unlock(vcpu); + + return busy; +} + +void +gicv3_vpe_schedule(bool enable_group0, bool enable_group1) +{ + // Current thread must be a mapped VCPU + thread_t *current = thread_get_self(); + assert(current != NULL); + assert_preempt_disabled(); + + if (cpulocal_index_valid(current->gicv3_its_mapped_cpu)) { + assert(cpulocal_get_index() == current->gicv3_its_mapped_cpu); + gicr_t *gicr = CPULOCAL(gicr_cpu).gicr; + assert(gicr != NULL); + + // Wait until the GICR has finished descheduling any previous + // vPE. Note that we only wait in deschedule if we're enabling + // the doorbell. + gicv3_vpe_sync_deschedule(cpulocal_get_index(), false); + + // Write the new valid VPENDBASER + GICR_VPENDBASER_t vpendbaser = GICR_VPENDBASER_default(); + GICR_VPENDBASER_set_vPEID(&vpendbaser, + current->gicv3_its_vpe_id); + GICR_VPENDBASER_set_vGrp1En(&vpendbaser, enable_group1); + GICR_VPENDBASER_set_vGrp0En(&vpendbaser, enable_group0); + // Note: PendingLast is RES1 when setting Valid + GICR_VPENDBASER_set_PendingLast(&vpendbaser, true); + GICR_VPENDBASER_set_Valid(&vpendbaser, true); + atomic_store_relaxed(&gicr->vlpi.vpendbaser, vpendbaser); + + TRACE(DEBUG, INFO, + "gicv3_vpe_schedule: {:#x} -> vpendbase {:#x}", + (uintptr_t)current, GICR_VPENDBASER_raw(vpendbaser)); + } +} + +bool +gicv3_vpe_deschedule(bool enable_doorbell) +{ + bool wakeup = false; + + // Current thread must be a mapped VCPU + thread_t *current = thread_get_self(); + assert(current != NULL); + assert_preempt_disabled(); + + if (cpulocal_index_valid(current->gicv3_its_mapped_cpu)) { + assert(cpulocal_get_index() == current->gicv3_its_mapped_cpu); + gicr_t *gicr = CPULOCAL(gicr_cpu).gicr; + assert(gicr != NULL); + + // Wait until the pending table has been parsed after any + // previous vPE scheduling operation, which could have been very + // recent. The GICR appears to reject deschedule operations if + // this is not done. + GICR_VPENDBASER_t vpendbaser; + do { + vpendbaser = + atomic_load_relaxed(&gicr->vlpi.vpendbaser); + assert(GICR_VPENDBASER_get_Valid(&vpendbaser)); + } while (GICR_VPENDBASER_get_Dirty(&vpendbaser)); + + // Write an invalid VPENDBASER. + GICR_VPENDBASER_set_Valid(&vpendbaser, false); + GICR_VPENDBASER_set_Doorbell(&vpendbaser, enable_doorbell); + // If we are not enabling the doorbell, write PendingLast as 1 + // to tell the GICR that we don't care, so the GICR can avoid + // wasting time calculating it. + GICR_VPENDBASER_set_PendingLast(&vpendbaser, !enable_doorbell); + atomic_store_relaxed(&gicr->vlpi.vpendbaser, vpendbaser); + + TRACE(DEBUG, INFO, + "gicv3_vpe_deschedule: {:#x} -> vpendbase {:#x}", + (uintptr_t)current, GICR_VPENDBASER_raw(vpendbaser)); + + if (enable_doorbell) { + // Read back VPENDBASER to get PendingLast which + // indicates that a VLPI or VSGI is already pending + // (which will suppress the doorbell LPI, so we must + // wake the thread now). + // + // This could be deferred to reduce the time spent in + // the loop, but it must be done before the new thread's + // ICH_MDCR_EL2 is loaded, so PendingLast calculation + // uses this thread's group enable bits. + do { + vpendbaser = atomic_load_relaxed( + &gicr->vlpi.vpendbaser); + assert(!GICR_VPENDBASER_get_Valid(&vpendbaser)); + } while (GICR_VPENDBASER_get_Dirty(&vpendbaser)); + wakeup = GICR_VPENDBASER_get_PendingLast(&vpendbaser); + + if (wakeup) { + current->gicv3_its_need_wakeup_check = true; + } + } + } + + return wakeup; +} + +bool +gicv3_vpe_check_wakeup(bool retry_trap) +{ + thread_t *current = thread_get_self(); + bool might_wake = false; + assert(current->kind == THREAD_KIND_VCPU); + + cpulocal_begin(); + + if (cpulocal_index_valid(current->gicv3_its_mapped_cpu) && + current->gicv3_its_need_wakeup_check) { + // The VCPU is scheduled in the GICR. Check whether the + // scheduling operation has finished; if so, we won't need to + // exit next time we get here. + assert(cpulocal_get_index() == current->gicv3_its_mapped_cpu); + gicr_t *gicr = CPULOCAL(gicr_cpu).gicr; + assert(gicr != NULL); + GICR_VPENDBASER_t vpendbaser = + atomic_load_relaxed(&gicr->vlpi.vpendbaser); + TRACE(DEBUG, INFO, + "gicv3_vpe_check_wakeup: may wakeup, vpendbase {:#x}, retry {:d}", + GICR_VPENDBASER_raw(vpendbaser), (register_t)retry_trap); + assert(GICR_VPENDBASER_get_Valid(&vpendbaser)); + if (!GICR_VPENDBASER_get_Dirty(&vpendbaser)) { + __asm__ volatile("dsb ish" ::: "memory"); + current->gicv3_its_need_wakeup_check = false; + } + + if (retry_trap || GICR_VPENDBASER_get_Dirty(&vpendbaser)) { + might_wake = true; + } + } else { + TRACE(DEBUG, INFO, "gicv3_vpe_check_wakeup: no wakeup"); + } + + cpulocal_end(); + + return might_wake; +} + +void +gicv3_vpe_sync_deschedule(cpu_index_t cpu, bool maybe_scheduled) +{ + assert_cpulocal_safe(); + gicr_t *gicr = CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr; + assert(gicr != NULL); + + GICR_VPENDBASER_t vpendbaser; + do { + vpendbaser = atomic_load_relaxed(&gicr->vlpi.vpendbaser); + TRACE(DEBUG, INFO, "gicv3_vpe_sync_deschedule: vpendbase {:#x}", + GICR_VPENDBASER_raw(vpendbaser)); + assert(maybe_scheduled || + !GICR_VPENDBASER_get_Valid(&vpendbaser)); + } while (!GICR_VPENDBASER_get_Valid(&vpendbaser) && + GICR_VPENDBASER_get_Dirty(&vpendbaser)); +} + +bool +gicv3_vpe_handle_irq_received_doorbell(hwirq_t *hwirq) +{ + thread_t *vcpu = atomic_load_consume(&hwirq->gicv3_its_vcpu); + + scheduler_lock(vcpu); + vcpu->gicv3_its_need_wakeup_check = true; + vcpu_wakeup(vcpu); + scheduler_unlock(vcpu); + + return true; +} + +uint32_result_t +gicv3_vpe_vsgi_query(thread_t *vcpu) +{ + uint32_result_t ret; + cpu_index_t cpu = vcpu->gicv3_its_mapped_cpu; + + if (cpulocal_index_valid(cpu)) { + gicr_t *gicr = CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr; + spinlock_t *lock = + &CPULOCAL_BY_INDEX(gicr_cpu, cpu).vsgi_query_lock; + + spinlock_acquire(lock); + GICR_VSGIPENDR_t pend; +#if !defined(NDEBUG) + pend = atomic_load_acquire(&gicr->vlpi.vsgipendr); + assert(!GICR_VSGIPENDR_get_Busy(&pend)); +#endif + GICR_VSGIR_t vsgir = GICR_VSGIR_default(); + GICR_VSGIR_set_vPEID(&vsgir, vcpu->gicv3_its_vpe_id); + atomic_store_release(&gicr->vlpi.vsgir, vsgir); + do { + pend = atomic_load_relaxed(&gicr->vlpi.vsgipendr); + } while (GICR_VSGIPENDR_get_Busy(&pend)); + spinlock_release(lock); + + ret = uint32_result_ok(GICR_VSGIPENDR_get_Pending(&pend)); + } else { + ret = uint32_result_error(ERROR_IDLE); + } + + return ret; +} + +#elif GICV3_HAS_VLPI // && !GICV3_HAS_VLPI_V4_1 +#error VPE scheduling is not implemented for GICv4.0 +#endif // GICV3_HAS_VLPI && !GICV3_HAS_VLPI_V4_1 + +#endif // defined(GICV3_ENABLE_VPE) && GICV3_ENABLE_VPE + +static void +gicv3_lpi_disable_all(void) REQUIRE_PREEMPT_DISABLED +{ +#if defined(GICV3_ENABLE_VPE) && GICV3_ENABLE_VPE + // Deschedule the current vPE, if any, and discard wakeups. + (void)gicv3_vpe_deschedule(false); +#endif + +#if defined(GICV3_ENABLE_VPE) && GICV3_ENABLE_VPE + for (cpu_index_t cpu = 0U; cpu < PLATFORM_MAX_CORES; cpu++) { + if (CPULOCAL_BY_INDEX(gicr_cpu, cpu).gicr == NULL) { + // No GICR for this CPU index, skip it + continue; + } + + // Ensure that the GICR has finished descheduling the last vPE + gicv3_vpe_sync_deschedule(cpu, false); + } +#endif + + for (index_t i = 0U; i < PLATFORM_GICR_COUNT; i++) { + gicr_t *gicr = mapped_gicrs[i]; + + // Disable LPIs and clear the vPE table base + GICR_CTLR_t gicr_ctlr = atomic_load_relaxed(&gicr->rd.ctlr); + GICR_CTLR_set_Enable_LPIs(&gicr_ctlr, false); + atomic_store_relaxed(&gicr->rd.ctlr, gicr_ctlr); +#if defined(GICV3_ENABLE_VPE) && GICV3_ENABLE_VPE + atomic_store_relaxed(&gicr->vlpi.vpropbaser, + GICR_VPROPBASER_default()); +#endif + + // Wait for the GICR to finish disabling LPIs before clearing + // the PROPBASE and PENDBASE registers + gicr_wait_for_write(gicr); + + atomic_store_relaxed(&gicr->rd.propbaser, + GICR_PROPBASER_default()); + atomic_store_relaxed(&gicr->rd.pendbaser, + GICR_PENDBASER_default()); + } +} +#endif // GICV3_HAS_LPI + +void +gicv3_handle_boot_hypervisor_handover(void) +{ + // Disable group 1 interrupts on the local CPU + ICC_IGRPEN_EL1_t icc_grpen1 = ICC_IGRPEN_EL1_default(); + ICC_IGRPEN_EL1_set_Enable(&icc_grpen1, false); + register_ICC_IGRPEN1_EL1_write_ordered(icc_grpen1, &asm_ordering); + + // Disable affinity-routed group 1 interrupts at the distributor + GICD_CTLR_t ctlr = atomic_load_relaxed(&gicd->ctlr); + GICD_CTLR_NS_set_EnableGrp1A(&ctlr.ns, false); + atomic_store_relaxed(&gicd->ctlr, ctlr); + (void)gicd_wait_for_write(); + + // Deactivate and clear edge pending for all SPIs + for (index_t i = 0U; i < 32U; i++) { + atomic_store_relaxed(&gicd->icactiver[i], 0xffffffffU); + atomic_store_relaxed(&gicd->icpendr[i], 0xffffffffU); + } + + // Deactivate and clear edge pending for PPIs on all GICRs + for (index_t i = 0U; i < PLATFORM_GICR_COUNT; i++) { + gicr_t *gicr = mapped_gicrs[i]; + atomic_store_relaxed(&gicr->sgi.icactiver0, 0xffffffffU); + atomic_store_relaxed(&gicr->sgi.icpendr0, 0xffffffffU); + } +#if GICV3_EXT_IRQS +#error Hand-over clean up of the extended IRQs +#endif + +#if GICV3_HAS_LPI + gicv3_lpi_disable_all(); +#endif + +#if GICV3_HAS_ITS + gicv3_its_disable_all(&mapped_gitss); +#endif +} + +void +gicv3_handle_power_cpu_online(void) +{ + gicr_cpu_t *gc = &CPULOCAL(gicr_cpu); + + // Mark this CPU as online and availbable for SPI route migration + assert_preempt_disabled(); + spinlock_acquire_nopreempt(&spi_route_lock); + gc->online = true; + spinlock_release_nopreempt(&spi_route_lock); +} + +static bool +gicv3_try_move_spi_to_cpu(cpu_index_t target) REQUIRE_PREEMPT_DISABLED +{ + // Hold the spi_route_lock before scanning the SPI route table so it + // does not change while we are working with it. Holding the lock also + // prevents the target CPU from going offline while setting routes. + + bool moved = false; + gicr_cpu_t *gc = &CPULOCAL_BY_INDEX(gicr_cpu, target); + + assert_preempt_disabled(); + + // Take the SPI lock so we can search the route table safely + spinlock_acquire_nopreempt(&spi_route_lock); + + if (gc->online) { + // We have an online target CPU we can use. Search SPI routes + // that need migration + GICD_IROUTER_t route = GICD_IROUTER_default(); + GICD_IROUTER_t route_cmp = GICD_IROUTER_default(); + + // Create a route with our affinity to compare against + // Ignore 1:N routes + gicv3_spi_set_route_cpu_affinity(&route_cmp, + cpulocal_get_index()); + + // To advance 32 IRQs at a time, our base should start on a + // boundary + static_assert(GIC_SPI_BASE % 32U == 0, + "GIC_SPI_BASE not 32bit aligned"); + + for (irq_t irq_base = GIC_SPI_BASE; + irq_base <= gicv3_spi_max_cache; irq_base += 32U) { + uint32_t isenabler = atomic_load_relaxed( + &gicd->isenabler[GICD_ENABLE_GET_N(irq_base)]); + while (isenabler != 0U) { + index_t i = compiler_ctz(isenabler); + isenabler &= ~((index_t)util_bit(i)); + + irq_t irq = irq_base + i; + + // Special or reserved IRQs should never be + // enabled + assert(gicv3_get_irq_type(irq) == + GICV3_IRQ_TYPE_SPI); + + route = atomic_load_relaxed( + &gicd->irouter[irq]); + if (GICD_IROUTER_is_equal(route_cmp, route)) { + gicv3_spi_set_route_cpu_affinity( + &route, target); + error_t ret = + gicv3_spi_set_route_internal( + irq, route); + if (ret != OK) { + panic("Failed to move SPI"); + } + } + } + } + +#if GICV3_EXT_IRQS +#error Migration of the extended IRQs +#endif + + moved = true; + } + + spinlock_release_nopreempt(&spi_route_lock); + + return moved; +} + +void +gicv3_handle_power_cpu_offline(void) +{ + // Migrate any SPIs routed to this CPU to another online CPU. + // If the IRQ is mapped to the wrong CPU it will get fixed the next + // time the IRQ occurs. + + // Try to move any SPI IRQs to the next CPU up from this one. + // If this is the last CPU, wrap around + cpu_index_t my_index = cpulocal_get_index(); + cpu_index_t target = + (cpu_index_t)((my_index + 1U) % PLATFORM_MAX_CORES); + gicr_cpu_t *gc = &CPULOCAL(gicr_cpu); + + assert_preempt_disabled(); + + spinlock_acquire_nopreempt(&spi_route_lock); + + gc->online = false; + + spinlock_release_nopreempt(&spi_route_lock); + + // Find an online target CPU before we scan SPI routes + bool found_target = false; + while (!found_target) { + if (platform_cpu_exists(target)) { + if (gicv3_try_move_spi_to_cpu(target)) { + found_target = true; + break; + } + } + + // Try the next CPU + target = (cpu_index_t)((target + 1U) % PLATFORM_MAX_CORES); + if (target == my_index) { + // we looped around without finding a target, + // this should never happen. + break; + } + } + + if (!found_target) { + panic("Could not find target CPU for SPI migration"); + } +} diff --git a/hyp/platform/gicv3/templates/gich_lrs.h.tmpl b/hyp/platform/gicv3/templates/gich_lrs.h.tmpl index 27d4488..78d3919 100644 --- a/hyp/platform/gicv3/templates/gich_lrs.h.tmpl +++ b/hyp/platform/gicv3/templates/gich_lrs.h.tmpl @@ -3,10 +3,10 @@ // SPDX-License-Identifier: BSD-3-Clause static inline ICH_LR_EL2_t -gicv3_read_ich_lr(index_t i, struct asm_ordering_dummy *ordering) +gicv3_read_ich_lr(index_t i, asm_ordering_dummy_t *ordering) { ICH_LR_EL2_t ret; - assert(i < CPU_GICH_LR_COUNT); + switch (i) { #for lr in range(0, $CPU_GICH_LR_COUNT) case ${lr}U: @@ -23,8 +23,7 @@ gicv3_read_ich_lr(index_t i, struct asm_ordering_dummy *ordering) static inline void gicv3_write_ich_lr(index_t i, ICH_LR_EL2_t val, - struct asm_ordering_dummy *ordering) { - assert(i < CPU_GICH_LR_COUNT); + asm_ordering_dummy_t *ordering) { switch (i) { #for lr in range(0, $CPU_GICH_LR_COUNT) case ${lr}U: @@ -48,7 +47,7 @@ gicv3_read_ich_aprs(uint32_t *ap0rs, uint32_t *ap1rs) { static inline void gicv3_write_ich_aprs(const uint32_t *ap0rs, const uint32_t *ap1rs) { - struct asm_ordering_dummy apr_write_ordering; + asm_ordering_dummy_t apr_write_ordering; #for group in (0, 1) #for i in range(0, $CPU_GICH_APR_COUNT) register_ICH_AP${group}R${i}_EL2_write_ordered(ap${group}rs[${i}], diff --git a/hyp/platform/psci_smc/src/psci_smc.c b/hyp/platform/psci_smc/src/psci_smc.c index 07f2e7d..9cb4825 100644 --- a/hyp/platform/psci_smc/src/psci_smc.c +++ b/hyp/platform/psci_smc/src/psci_smc.c @@ -6,14 +6,16 @@ #include +#include #include +#include #include #include "psci_smc.h" #include "psci_smc_arch.h" uint32_t -psci_smc_psci_version() +psci_smc_psci_version(void) { psci_ret_t ret = psci_smc_fn_call32(PSCI_FUNCTION_PSCI_VERSION, 0, 0, 0); @@ -113,7 +115,7 @@ psci_smc_system_reset(void) } error_t -psci_smc_cpu_off() +psci_smc_cpu_off(void) { error_t err; @@ -145,17 +147,21 @@ psci_smc_cpu_on(psci_mpidr_t cpu_id, paddr_t entry_point, register_t context_id) switch (psci_smc_fn_call(PSCI_FUNCTION_CPU_ON, psci_mpidr_raw(cpu_id), entry_point, context_id)) { case PSCI_RET_SUCCESS: - case PSCI_RET_ALREADY_ON: case PSCI_RET_ON_PENDING: err = OK; break; + case PSCI_RET_ALREADY_ON: + err = ERROR_RETRY; + break; case PSCI_RET_INVALID_PARAMETERS: case PSCI_RET_INVALID_ADDRESS: err = ERROR_ARGUMENT_INVALID; break; + case PSCI_RET_INTERNAL_FAILURE: + err = ERROR_FAILURE; + break; case PSCI_RET_DENIED: case PSCI_RET_NOT_SUPPORTED: - case PSCI_RET_INTERNAL_FAILURE: case PSCI_RET_NOT_PRESENT: case PSCI_RET_DISABLED: default: @@ -175,9 +181,10 @@ psci_smc_psci_features(psci_function_t fn, bool smc64) smccc_function_id_set_function(&fn_id, (smccc_function_t)fn); sint32_result_t ret; - - ret.r = psci_smc_fn_call32(PSCI_FUNCTION_PSCI_FEATURES, - smccc_function_id_raw(fn_id), 0, 0); + psci_ret_t psci_ret = psci_smc_fn_call32(PSCI_FUNCTION_PSCI_FEATURES, + smccc_function_id_raw(fn_id), + 0, 0); + ret.r = (int32_t)psci_ret; switch ((psci_ret_t)ret.r) { case PSCI_RET_NOT_SUPPORTED: @@ -204,7 +211,7 @@ psci_smc_psci_features(psci_function_t fn, bool smc64) } error_t -psci_smc_cpu_freeze() +psci_smc_cpu_freeze(void) { error_t err; @@ -235,8 +242,8 @@ psci_smc_psci_set_suspend_mode(psci_mode_t mode) { error_t err; - switch (psci_smc_fn_call32(PSCI_FUNCTION_PSCI_SET_SUSPEND_MODE, mode, 0, - 0)) { + switch (psci_smc_fn_call32(PSCI_FUNCTION_PSCI_SET_SUSPEND_MODE, + (uint32_t)mode, 0, 0)) { case PSCI_RET_SUCCESS: err = OK; break; diff --git a/hyp/platform/soc_qemu/build.conf b/hyp/platform/soc_qemu/build.conf index c78c429..e8089fb 100644 --- a/hyp/platform/soc_qemu/build.conf +++ b/hyp/platform/soc_qemu/build.conf @@ -9,7 +9,7 @@ base_module hyp/platform/gicv3 base_module hyp/platform/psci_smc local_include source boot.c cpu.c irq.c platform_psci.c prng.c soc_qemu.c head.S abort.c uart.c -source cpu_features.c +source cpu_features.c addrspace.c types soc_qemu.tc events soc_qemu.ev diff --git a/hyp/platform/soc_qemu/soc_qemu.ev b/hyp/platform/soc_qemu/soc_qemu.ev index 897f568..e03bade 100644 --- a/hyp/platform/soc_qemu/soc_qemu.ev +++ b/hyp/platform/soc_qemu/soc_qemu.ev @@ -4,14 +4,17 @@ module soc_qemu -subscribe rootvm_init(root_partition, root_cspace, env_data) - priority -10 - -subscribe boot_cpu_cold_init - subscribe boot_cold_init handler soc_qemu_uart_init() priority 5 + require_preempt_disabled + +subscribe boot_cpu_cold_init + +#if !defined(UNIT_TESTS) +subscribe rootvm_init(root_partition, root_cspace, hyp_env, qcbor_enc_ctxt) + priority -10 +#endif #if !defined(UNIT_TESTS) subscribe vcpu_activate_thread diff --git a/hyp/platform/soc_qemu/soc_qemu.tc b/hyp/platform/soc_qemu/soc_qemu.tc index 2fd1200..196e2eb 100644 --- a/hyp/platform/soc_qemu/soc_qemu.tc +++ b/hyp/platform/soc_qemu/soc_qemu.tc @@ -2,17 +2,9 @@ // // SPDX-License-Identifier: BSD-3-Clause -extend boot_env_data structure { +extend hyp_env_data structure { device_me_capid type cap_id_t; - device_me_base type vmaddr_t; entry_hlos type vmaddr_t; - hlos_vm_base type vmaddr_t; - hlos_vm_size size; - hlos_dt_base type vmaddr_t; - hlos_ramfs_base type vmaddr_t; - watchdog_supported bool; - uart_me_capid type cap_id_t; - uart_address type paddr_t; }; #if defined (PLATFORM_PSCI_USE_ORIGINAL_POWERSTATE_FORMAT) @@ -23,11 +15,13 @@ extend psci_suspend_powerstate bitfield { define psci_suspend_powerstate_stateid bitfield<16> { 3:0 cpu type psci_cpu_state_t; + 7:4 cluster type psci_cluster_state_L3_t; others unknown=0; }; #endif -define psci_cpu_state_t newtype uint8; +define psci_cpu_state_t newtype uint32; +define psci_cluster_state_L3_t newtype uint32; define CLIENT_ID_HYP constant uint32 = 0; diff --git a/hyp/platform/soc_qemu/src/addrspace.c b/hyp/platform/soc_qemu/src/addrspace.c new file mode 100644 index 0000000..35953a1 --- /dev/null +++ b/hyp/platform/soc_qemu/src/addrspace.c @@ -0,0 +1,14 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include + +bool +platform_pgtable_undergoing_bbm(void) +{ + return false; +} diff --git a/hyp/platform/soc_qemu/src/boot.c b/hyp/platform/soc_qemu/src/boot.c index a15dcf9..e562e74 100644 --- a/hyp/platform/soc_qemu/src/boot.c +++ b/hyp/platform/soc_qemu/src/boot.c @@ -5,7 +5,6 @@ #include #include -#if !defined(UNIT_TESTS) #include #include #include @@ -29,9 +28,10 @@ platform_ram_probe(void) // system-device-tree approach. We need to make sure that hyp RAM memory // ranges do not overlap with the ranges specified in the QEMU start // command. - ram_info.num_ranges = 0x1; - ram_info.ram_range[0].base = 0x40000000; - ram_info.ram_range[0].size = 0x80000000; // 1Gb of RAM + ram_info.num_ranges = 0x1; + // TODO: Get info from DT + ram_info.ram_range[0].base = PLATFORM_DDR_BASE; + ram_info.ram_range[0].size = PLATFORM_DDR_SIZE; return OK; } @@ -59,7 +59,8 @@ platform_add_root_heap(partition_t *partition) // FIXME: Currently using the end memory of the hardcoded 1Gb hyp RAM // memory size. We need to find a better solution for this, possibly by // dynamically reading the RAM memory end address from a device tree. - paddr_t base = PLATFORM_LMA_BASE + 0x40000000 - alloc_size; + + paddr_t base = PLATFORM_DDR_BASE + PLATFORM_DDR_SIZE - alloc_size; // Add 1MiB to the hypervisor private partition error_t err = partition_mem_donate(partition, base, priv_size, @@ -87,6 +88,7 @@ platform_add_root_heap(partition_t *partition) trace_init(partition, trace_size); } +#if !defined(UNIT_TESTS) static memextent_t * create_memextent(partition_t *root_partition, cspace_t *root_cspace, paddr_t phys_base, size_t size, pgtable_access_t access, @@ -94,7 +96,7 @@ create_memextent(partition_t *root_partition, cspace_t *root_cspace, { bool device_mem = (memtype == MEMEXTENT_MEMTYPE_DEVICE); - memextent_create_t params_me = { .memextent = NULL, + memextent_create_t params_me = { .memextent = NULL, .memextent_device_mem = device_mem }; memextent_ptr_result_t me_ret; me_ret = partition_allocate_memextent(root_partition, params_me); @@ -106,6 +108,11 @@ create_memextent(partition_t *root_partition, cspace_t *root_cspace, memextent_attrs_t attrs = memextent_attrs_default(); memextent_attrs_set_access(&attrs, access); memextent_attrs_set_memtype(&attrs, memtype); +#if defined(MODULE_MEM_MEMEXTENT_SPARSE) + if (device_mem) { + memextent_attrs_set_type(&attrs, MEMEXTENT_TYPE_SPARSE); + } +#endif spinlock_acquire(&me->header.lock); error_t ret = memextent_configure(me, phys_base, size, attrs); @@ -135,7 +142,8 @@ create_memextent(partition_t *root_partition, cspace_t *root_cspace, void soc_qemu_handle_rootvm_init(partition_t *root_partition, cspace_t *root_cspace, - boot_env_data_t *env_data) + hyp_env_data_t *hyp_env, + qcbor_enc_ctxt_t *qcbor_enc_ctxt) { // FIXME: The memory layout for QEMU is hardcoded here. We need to find a // better solution for this, possibly by using a system-device-tree @@ -143,33 +151,31 @@ soc_qemu_handle_rootvm_init(partition_t *root_partition, cspace_t *root_cspace, // device-tree. We will also need to get the addresses such as // hlos-entry from this config such that ultimately these can all be // inputs from QEMU/user. + paddr_t hlos_vm_base = HLOS_VM_DDR_BASE; + paddr_t hlos_vm_size = HLOS_VM_DDR_SIZE; + + assert(qcbor_enc_ctxt != NULL); // VM memory node. Includes entry point, DT, and rootfs - env_data->hlos_vm_base = 0x40000000; - env_data->hlos_vm_size = 0x20000000; - env_data->entry_hlos = 0x41080000; - env_data->hlos_dt_base = 0x44200000; - env_data->hlos_ramfs_base = 0x44400000; - env_data->device_me_base = PLATFORM_DEVICES_BASE; + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "hlos_vm_base", + hlos_vm_base); + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "hlos_vm_size", + hlos_vm_size); + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "entry_hlos", + HLOS_ENTRY_POINT); + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "hlos_dt_base", + HLOS_DT_BASE); + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "hlos_ramfs_base", + HLOS_RAM_FS_BASE); + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "device_me_base", + PLATFORM_DEVICES_BASE); + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "device_me_size", + PLATFORM_DEVICES_SIZE); #if defined(WATCHDOG_DISABLE) - env_data->watchdog_supported = false; + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "watchdog_supported", false); #endif - // Add memory of VM to memdb first, so that we can use it to create a - // non-device memextent - //paddr_t phys_start = env_data->hlos_vm_base; - //paddr_t phys_end = - // env_data->hlos_vm_base + (env_data->hlos_vm_size - 1U); - - //partition_t *hyp_partition = partition_get_private(); - //error_t err = memdb_insert(hyp_partition, phys_start, phys_end, - // (uintptr_t)root_partition, - // MEMDB_TYPE_PARTITION); - //if (err != OK) { - // panic("Error adding VM memory to hyp_partition"); - //} - // Create a device memextent to cover the full HW physical address // space reserved for devices, so that the resource manager can derive // device memextents. @@ -179,7 +185,10 @@ soc_qemu_handle_rootvm_init(partition_t *root_partition, cspace_t *root_cspace, memextent_t *me = create_memextent( root_partition, root_cspace, PLATFORM_DEVICES_BASE, PLATFORM_DEVICES_SIZE, PGTABLE_ACCESS_RW, - MEMEXTENT_MEMTYPE_DEVICE, &env_data->device_me_capid); + MEMEXTENT_MEMTYPE_DEVICE, &hyp_env->device_me_capid); + + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "device_me_capid", + hyp_env->device_me_capid); // Derive memextents for GICD, GICR and watchdog to effectively remove // them from the device memextent we provide to the rootvm. @@ -213,7 +222,9 @@ soc_qemu_handle_rootvm_init(partition_t *root_partition, cspace_t *root_cspace, panic("Error create memextent cap id."); } - env_data->uart_address = PLATFORM_UART_BASE; - env_data->uart_me_capid = capid_ret.r; + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "uart_address", + PLATFORM_UART_BASE); + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "uart_me_capid", + capid_ret.r); } #endif diff --git a/hyp/platform/soc_qemu/src/cpu.c b/hyp/platform/soc_qemu/src/cpu.c index b801457..f8214f6 100644 --- a/hyp/platform/soc_qemu/src/cpu.c +++ b/hyp/platform/soc_qemu/src/cpu.c @@ -5,14 +5,17 @@ #include #include +#include #include #include #include #include #include #include +#include #include #include +#include #include "event_handlers.h" #include "psci_smc.h" @@ -31,43 +34,29 @@ soc_qemu_handle_boot_cpu_cold_init(cpu_index_t cpu) CPULOCAL_BY_INDEX(cpu_started, cpu) = true; } -psci_mpidr_t -platform_cpu_index_to_mpidr(cpu_index_t cpu) +bool +platform_cpu_exists(cpu_index_t cpu) { - psci_mpidr_t ret = psci_mpidr_default(); - assert(cpulocal_index_valid(cpu)); - psci_mpidr_set_Aff0(&ret, (uint8_t)cpu); - return ret; -} + assert(cpu < PLATFORM_MAX_CORES); -cpu_index_result_t -platform_cpu_mpidr_to_index(psci_mpidr_t mpidr) -{ - cpu_index_result_t result = - cpu_index_result_error(ERROR_ARGUMENT_INVALID); - - if ((psci_mpidr_get_Aff1(&mpidr) == 0U) && - (psci_mpidr_get_Aff2(&mpidr) == 0U) && - (psci_mpidr_get_Aff3(&mpidr) == 0U)) { - cpu_index_t index = (cpu_index_t)psci_mpidr_get_Aff0(&mpidr); - if (cpulocal_index_valid(index)) { - result = cpu_index_result_ok(index); - } - } - - return result; + return compiler_expected((util_bit(cpu) & PLATFORM_USABLE_CORES) != 0U); } error_t platform_cpu_on(cpu_index_t cpu) { - psci_mpidr_t mpidr = platform_cpu_index_to_mpidr(cpu); - thread_t *thread = idle_thread_for(cpu); - uintptr_t entry_virt = - CPULOCAL_BY_INDEX(cpu_started, cpu) - ? (uintptr_t)&soc_qemu_entry_warm - : (uintptr_t)&soc_qemu_entry_cold_secondary; - return psci_smc_cpu_on(mpidr, + MPIDR_EL1_t mpidr = platform_cpu_index_to_mpidr(cpu); + thread_t *thread = idle_thread_for(cpu); + uintptr_t entry_virt = + CPULOCAL_BY_INDEX(cpu_started, cpu) + ? (uintptr_t)&soc_qemu_entry_warm + : (uintptr_t)&soc_qemu_entry_cold_secondary; + psci_mpidr_t psci_mpidr = psci_mpidr_default(); + psci_mpidr_set_Aff0(&psci_mpidr, MPIDR_EL1_get_Aff0(&mpidr)); + psci_mpidr_set_Aff1(&psci_mpidr, MPIDR_EL1_get_Aff1(&mpidr)); + psci_mpidr_set_Aff2(&psci_mpidr, MPIDR_EL1_get_Aff2(&mpidr)); + psci_mpidr_set_Aff3(&psci_mpidr, MPIDR_EL1_get_Aff3(&mpidr)); + return psci_smc_cpu_on(psci_mpidr, partition_virt_to_phys(partition_get_private(), entry_virt), (uintptr_t)thread); @@ -108,7 +97,7 @@ platform_cpu_off(void) } static register_t -psci_smc_cpu_suspend_arg(register_t power_state) +psci_smc_cpu_suspend_arg(register_t power_state) REQUIRE_PREEMPT_DISABLED { thread_t *idle = idle_thread(); @@ -129,11 +118,11 @@ platform_cpu_suspend(psci_suspend_powerstate_t power_state) assert(idle_is_current()); ret = thread_freeze(psci_smc_cpu_suspend_arg, - psci_suspend_powerstate_raw(power_state), ~0U); + psci_suspend_powerstate_raw(power_state), ~0UL); - return (ret == 0U) ? bool_result_ok(false) - : (ret == ~0U) ? bool_result_ok(true) - : bool_result_error((error_t)ret); + return (ret == 0UL) ? bool_result_ok(false) + : (ret == ~0UL) ? bool_result_ok(true) + : bool_result_error((error_t)ret); } error_t @@ -165,27 +154,10 @@ platform_cpu_default_suspend(void) register_t ret; assert(idle_is_current()); - ret = thread_freeze(psci_smc_cpu_default_suspend_arg, 0U, ~0U); - - return (ret == 0U) ? bool_result_ok(false) - : (ret == ~0U) ? bool_result_ok(true) - : bool_result_error((error_t)ret); -} -#endif - -#if defined(SOC_QEMU_START_ALL_CORES) -void -soc_qemu_start_all_cores(void) -{ - cpu_index_t boot_cpu = cpulocal_get_index(); - - // Temporary for debugging: power on all CPUs - for (cpu_index_t cpu = 0U; cpulocal_index_valid(cpu); cpu++) { - if (cpu == boot_cpu) { - continue; - } + ret = thread_freeze(psci_smc_cpu_default_suspend_arg, 0UL, ~0UL); - platform_cpu_on(cpu); - } + return (ret == 0UL) ? bool_result_ok(false) + : (ret == ~0UL) ? bool_result_ok(true) + : bool_result_error((error_t)ret); } #endif diff --git a/hyp/platform/soc_qemu/src/cpu_features.c b/hyp/platform/soc_qemu/src/cpu_features.c index 3156567..707e336 100644 --- a/hyp/platform/soc_qemu/src/cpu_features.c +++ b/hyp/platform/soc_qemu/src/cpu_features.c @@ -19,10 +19,10 @@ platform_get_cpu_features(void) platform_cpu_features_set_trace_disable(&features, false); #endif #if defined(INTERFACE_DEBUG) - platform_cpu_features_set_debug_disable(&features, true); + platform_cpu_features_set_debug_disable(&features, false); #endif #if defined(MODULE_VM_ARM_VM_SVE_SIMPLE) - platform_cpu_features_set_sve_disable(&features, true); + platform_cpu_features_set_sve_disable(&features, false); #endif return features; diff --git a/hyp/platform/soc_qemu/src/platform_psci.c b/hyp/platform/soc_qemu/src/platform_psci.c index 41b2e93..964f6de 100644 --- a/hyp/platform/soc_qemu/src/platform_psci.c +++ b/hyp/platform/soc_qemu/src/platform_psci.c @@ -28,6 +28,13 @@ platform_psci_is_cpu_poweroff(psci_cpu_state_t cpu_state) return false; } +bool +platform_psci_is_cluster_active(psci_cluster_state_L3_t cluster_state) +{ + (void)cluster_state; + return true; +} + psci_cpu_state_t platform_psci_get_cpu_state(psci_suspend_powerstate_t suspend_state) { @@ -85,4 +92,25 @@ platform_psci_suspend_state_validation(psci_suspend_powerstate_t suspend_state, // QEMU does not care about suspend states since it only goes to WFI. return PSCI_RET_SUCCESS; } + +// Returns the cluster indices +uint32_t +platform_psci_get_cluster_index(cpu_index_t cpu) +{ + (void)cpu; + return 0U; +} + +error_t +platform_psci_get_index_by_level(cpu_index_t cpu, uint32_t *start_idx, + uint32_t *children_counts, uint32_t level) +{ + (void)cpu; + (void)level; + *start_idx = 0U; + *children_counts = PLATFORM_MAX_CORES; + + return OK; +} + #endif diff --git a/hyp/platform/soc_qemu/src/soc_qemu.c b/hyp/platform/soc_qemu/src/soc_qemu.c index eff6324..aad0d04 100644 --- a/hyp/platform/soc_qemu/src/soc_qemu.c +++ b/hyp/platform/soc_qemu/src/soc_qemu.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "event_handlers.h" @@ -24,11 +25,28 @@ platform_cpu_stack_size(void) return 0; } +bool +smccc_handle_smc_platform_call(register_t args[7], bool is_hvc) + EXCLUDE_PREEMPT_DISABLED +{ + (void)is_hvc; + args[0] = (register_t)SMCCC_UNKNOWN_FUNCTION64; + return true; +} + +// Overrides the weak imlementation in core_id.c +core_id_t +platform_cpu_get_coreid(MIDR_EL1_t midr) +{ + (void)midr; + return CORE_ID_QEMU; +} + #if !defined(UNIT_TESTS) static _Atomic BITMAP_DECLARE(PLATFORM_MAX_CORES, hlos_vm_cpus); bool -soc_qemu_handle_vcpu_activate_thread(thread_t *thread, +soc_qemu_handle_vcpu_activate_thread(thread_t *thread, vcpu_option_flags_t options) { bool ret = true; diff --git a/hyp/platform/soc_qemu/src/uart.c b/hyp/platform/soc_qemu/src/uart.c index 8eff177..fa719d9 100644 --- a/hyp/platform/soc_qemu/src/uart.c +++ b/hyp/platform/soc_qemu/src/uart.c @@ -12,17 +12,19 @@ #include #include #include +#include #include "event_handlers.h" #include "uart.h" static soc_qemu_uart_t *uart; +static spinlock_t uart_lock; static void uart_putc(const char c) { - while ((atomic_load_relaxed(&uart->tfr) & ((uint32_t)1U << 5)) != 0U) - ; + while ((atomic_load_relaxed(&uart->tfr) & ((uint32_t)1U << 5)) != 0U) { + } atomic_store_relaxed(&uart->dr, c); } @@ -59,11 +61,11 @@ uart_write(const char *out, size_t size) void soc_qemu_console_puts(const char *msg) { - preempt_disable(); + spinlock_acquire(&uart_lock); if (uart != NULL) { uart_write(msg, strlen(msg)); } - preempt_enable(); + spinlock_release(&uart_lock); } void @@ -88,6 +90,8 @@ soc_qemu_handle_log_message(trace_id_t id, const char *str) void soc_qemu_uart_init(void) { + spinlock_init(&uart_lock); + virt_range_result_t range = hyp_aspace_allocate(PLATFORM_UART_SIZE); if (range.e != OK) { panic("uart: Address allocation failed."); diff --git a/hyp/platform/tbre/aarch64/tbre.tc b/hyp/platform/tbre/aarch64/tbre.tc new file mode 100644 index 0000000..bdc89ef --- /dev/null +++ b/hyp/platform/tbre/aarch64/tbre.tc @@ -0,0 +1,25 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +extend MDCR_EL2 bitfield { + 25:24 E2TB uint8; +}; + +define TRBLIMITR_EL1 bitfield<64> { + 0 E bool; + 2:1 FM uint8; + 4:3 TM uint8; + 5 nVM bool; + 63:12 LIMIT uint64 lsl(12); + others unknown = 0; +}; + +define tbre_context structure { + TRBLIMITR_EL1 bitfield TRBLIMITR_EL1; + TRBPTR_EL1 uint64; + TRBBASER_EL1 uint64; + TRBSR_EL1 uint64; + TRBMAR_EL1 uint64; + TRBTRG_EL1 uint64; +}; diff --git a/hyp/platform/tbre/build.conf b/hyp/platform/tbre/build.conf new file mode 100644 index 0000000..e37a8b7 --- /dev/null +++ b/hyp/platform/tbre/build.conf @@ -0,0 +1,7 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +arch_types aarch64 tbre.tc +local_include +source tbre.c diff --git a/hyp/platform/tbre/include/tbre.h b/hyp/platform/tbre/include/tbre.h new file mode 100644 index 0000000..8241880 --- /dev/null +++ b/hyp/platform/tbre/include/tbre.h @@ -0,0 +1,9 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +void +tbre_save_context_percpu(cpu_index_t cpu); + +void +tbre_restore_context_percpu(cpu_index_t cpu); diff --git a/hyp/platform/tbre/src/tbre.c b/hyp/platform/tbre/src/tbre.c new file mode 100644 index 0000000..9473f18 --- /dev/null +++ b/hyp/platform/tbre/src/tbre.c @@ -0,0 +1,68 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include + +#include + +#include +#include +#include +#include +#include + +#include + +#include "tbre.h" + +CPULOCAL_DECLARE_STATIC(tbre_context_t, tbre_contexts); + +void +tbre_save_context_percpu(cpu_index_t cpu) +{ + CPULOCAL_BY_INDEX(tbre_contexts, cpu).TRBLIMITR_EL1 = + register_TRBLIMITR_EL1_read_ordered(&vet_ordering); + + CPULOCAL_BY_INDEX(tbre_contexts, cpu).TRBPTR_EL1 = + register_TRBPTR_EL1_read_ordered(&vet_ordering); + + CPULOCAL_BY_INDEX(tbre_contexts, cpu).TRBBASER_EL1 = + register_TRBBASER_EL1_read_ordered(&vet_ordering); + + CPULOCAL_BY_INDEX(tbre_contexts, cpu).TRBSR_EL1 = + register_TRBSR_EL1_read_ordered(&vet_ordering); + + CPULOCAL_BY_INDEX(tbre_contexts, cpu).TRBMAR_EL1 = + register_TRBMAR_EL1_read_ordered(&vet_ordering); + + CPULOCAL_BY_INDEX(tbre_contexts, cpu).TRBTRG_EL1 = + register_TRBTRG_EL1_read_ordered(&vet_ordering); +} + +void +tbre_restore_context_percpu(cpu_index_t cpu) +{ + register_TRBLIMITR_EL1_write_ordered( + CPULOCAL_BY_INDEX(tbre_contexts, cpu).TRBLIMITR_EL1, + &vet_ordering); + + register_TRBPTR_EL1_write_ordered( + CPULOCAL_BY_INDEX(tbre_contexts, cpu).TRBPTR_EL1, + &vet_ordering); + + register_TRBBASER_EL1_write_ordered( + CPULOCAL_BY_INDEX(tbre_contexts, cpu).TRBBASER_EL1, + &vet_ordering); + + register_TRBSR_EL1_write_ordered( + CPULOCAL_BY_INDEX(tbre_contexts, cpu).TRBSR_EL1, &vet_ordering); + + register_TRBMAR_EL1_write_ordered( + CPULOCAL_BY_INDEX(tbre_contexts, cpu).TRBMAR_EL1, + &vet_ordering); + + register_TRBTRG_EL1_write_ordered( + CPULOCAL_BY_INDEX(tbre_contexts, cpu).TRBTRG_EL1, + &vet_ordering); +} diff --git a/hyp/vm/arm_pv_time/arm_pv_time.ev b/hyp/vm/arm_pv_time/arm_pv_time.ev new file mode 100644 index 0000000..2aa0c10 --- /dev/null +++ b/hyp/vm/arm_pv_time/arm_pv_time.ev @@ -0,0 +1,22 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module arm_pv_time + +#include + +SMCCC_STANDARD_HYP_FUNCTION_64(PV_TIME_FEATURES, 0, pv_time_features, arg1, ret0) +SMCCC_STANDARD_HYP_FUNCTION_64(PV_TIME_ST, 0, pv_time_st, arg1, ret0) + +subscribe object_create_thread(thread_create) + +subscribe vcpu_activate_thread(thread) + +subscribe scheduler_schedule + +subscribe thread_context_switch_post(curticks, prevticks) + +subscribe scheduler_blocked(thread, block) + +subscribe scheduler_unblocked(thread, block) diff --git a/hyp/vm/arm_pv_time/arm_pv_time.tc b/hyp/vm/arm_pv_time/arm_pv_time.tc new file mode 100644 index 0000000..6b40cd3 --- /dev/null +++ b/hyp/vm/arm_pv_time/arm_pv_time.tc @@ -0,0 +1,71 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +define pv_time_data structure(aligned(64)) { + revision uint32; + attributes uint32; + + // Stolen time after conversion to nanoseconds. + // + // The specification requires accesses to this variable to be + // (single-copy) atomic, but does not require explicit ordering. + stolen_ns uint64(atomic); +}; + +// Packed self-unblock state. +define arm_pv_time_self_block_state bitfield<64> { + // The last block state the thread put itself into. This is only valid + // if last_unblocked is 0. + // + // At initialisation, this is set to SCHEDULER_BLOCK_THREAD_LIFECYCLE. + // VCPU activation changes it to SCHEDULER_BLOCK_VCPU_OFF. + auto block enumeration scheduler_block; + + // Time at which the thread was last unblocked from a self-imposed + // block state (i.e. one in in which the thread voluntarily gave up + // the CPU and which should not be counted as stolen). + // + // This is zero while the thread is in a self-imposed block state. + // The thread's initial block state (prior to first starting) is + // considered self-imposed for this purpose. + auto<60> last_unblocked type ticks_t; +}; + +define arm_pv_time structure { + // Pointer to the hypervisor mapping of the VM-accessible stolen time. + // + // If non-NULL, this has the same lifetime as the thread itself, and + // should only be accessed while holding a reference to the thread. + data pointer structure pv_time_data; + + // Stolen time in ticks. Must only be accessed by the thread. + stolen_ticks type ticks_t; + + // Time spent in a directed yield since the thread last ran. Must only + // be accessed by the thread, or by a CPU that is in a directed yield + // from the thread. + yield_time type ticks_t; + + // The packed last-block state, as defined above. + // + // This can only be accessed by the thread itself during context + // switch or entry into a block state, or while unblocking the thread + // while holding its scheduler lock. It is therefore ordered, but not + // directly protected, by the scheduler lock, so it needs to be atomic + // but can use relaxed ordering. + self_block bitfield arm_pv_time_self_block_state(atomic); +}; + +extend thread object { + arm_pv_time structure arm_pv_time; +}; + +extend smccc_standard_hyp_function enumeration { + PV_TIME_FEATURES = 0x20; + PV_TIME_ST = 0x21; +}; + +extend addrspace_info_area_layout structure { + pv_time_data array(PLATFORM_MAX_CORES) structure pv_time_data; +}; diff --git a/hyp/vm/arm_pv_time/build.conf b/hyp/vm/arm_pv_time/build.conf new file mode 100644 index 0000000..788a6a4 --- /dev/null +++ b/hyp/vm/arm_pv_time/build.conf @@ -0,0 +1,7 @@ +# © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +events arm_pv_time.ev +types arm_pv_time.tc +source arm_pv_time.c diff --git a/hyp/vm/arm_pv_time/src/arm_pv_time.c b/hyp/vm/arm_pv_time/src/arm_pv_time.c new file mode 100644 index 0000000..6a6a7cd --- /dev/null +++ b/hyp/vm/arm_pv_time/src/arm_pv_time.c @@ -0,0 +1,222 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "event_handlers.h" + +#if !defined(MODULE_VM_VGIC) +#error Unable to determine a unique VCPU index (vgic_gicr_index not present) +#endif + +bool +smccc_pv_time_features(uint64_t arg1, uint64_t *ret0) +{ + smccc_function_id_t fn_id = smccc_function_id_cast((uint32_t)arg1); + bool is_smc64 = smccc_function_id_get_is_smc64(&fn_id); + bool is_fast = smccc_function_id_get_is_fast(&fn_id); + bool res0 = smccc_function_id_get_res0(&fn_id); + smccc_function_t fn = smccc_function_id_get_function(&fn_id); + smccc_interface_id_t interface_id = + smccc_function_id_get_interface_id(&fn_id); + uint64_t ret = SMCCC_UNKNOWN_FUNCTION64; + + if ((interface_id == SMCCC_INTERFACE_ID_STANDARD_HYP) && (res0 == 0U) && + is_fast && is_smc64) { + switch ((smccc_standard_hyp_function_t)fn) { + case SMCCC_STANDARD_HYP_FUNCTION_PV_TIME_FEATURES: + ret = 0; + break; + case SMCCC_STANDARD_HYP_FUNCTION_PV_TIME_ST: { + thread_t *current = thread_get_self(); + if (current->addrspace->info_area.me != NULL) { + ret = 0; + } + break; + } + case SMCCC_STANDARD_HYP_FUNCTION_CALL_COUNT: + case SMCCC_STANDARD_HYP_FUNCTION_CALL_UID: + case SMCCC_STANDARD_HYP_FUNCTION_REVISION: + default: + // Nothing to do. + break; + } + } + + *ret0 = ret; + return true; +} + +bool +smccc_pv_time_st(uint64_t arg1, uint64_t *ret0) +{ + (void)arg1; + + thread_t *current = thread_get_self(); + uint64_t ret = SMCCC_UNKNOWN_FUNCTION64; + + if (current->addrspace->info_area.me != NULL) { + index_t index = current->vgic_gicr_index; + assert(index < PLATFORM_MAX_CORES); + size_t offset = offsetof(addrspace_info_area_layout_t, + pv_time_data[index]); + assert((offset + sizeof(pv_time_data_t)) <= + current->addrspace->info_area.me->size); + ret = current->addrspace->info_area.ipa + offset; + } + + *ret0 = ret; + return true; +} + +error_t +arm_pv_time_handle_object_create_thread(thread_create_t thread_create) +{ + thread_t *thread = thread_create.thread; + assert(thread != NULL); + + arm_pv_time_self_block_state_t new_state = + arm_pv_time_self_block_state_default(); + arm_pv_time_self_block_state_set_block( + &new_state, SCHEDULER_BLOCK_THREAD_LIFECYCLE); + atomic_store_relaxed(&thread->arm_pv_time.self_block, new_state); + + return OK; +} + +bool +arm_pv_time_handle_vcpu_activate_thread(thread_t *thread) +{ + assert(thread != NULL); + + arm_pv_time_self_block_state_t new_state = + arm_pv_time_self_block_state_default(); + arm_pv_time_self_block_state_set_block(&new_state, + SCHEDULER_BLOCK_VCPU_OFF); + atomic_store_relaxed(&thread->arm_pv_time.self_block, new_state); + + if ((thread->addrspace->info_area.me != NULL)) { + index_t index = thread->vgic_gicr_index; + assert(index < PLATFORM_MAX_CORES); + assert(thread->addrspace->info_area.hyp_va != NULL); + thread->arm_pv_time.data = &thread->addrspace->info_area.hyp_va + ->pv_time_data[index]; + + thread->arm_pv_time.data->revision = 0U; + thread->arm_pv_time.data->attributes = 0U; + atomic_init(&thread->arm_pv_time.data->stolen_ns, 0U); + } + + return true; +} + +void +arm_pv_time_handle_scheduler_schedule(thread_t *current, thread_t *yielded_from, + ticks_t schedtime, ticks_t curticks) +{ + assert(current == thread_get_self()); + assert(curticks >= schedtime); + + // Avoid counting time in directed yields as stolen + if (yielded_from != NULL) { + yielded_from->arm_pv_time.yield_time += curticks - schedtime; + TRACE(DEBUG, INFO, + "arm_pv_time: {:#x} added yield time {:d}, curticks {:d}", + (uintptr_t)yielded_from, curticks - schedtime, + yielded_from->arm_pv_time.yield_time); + } +} + +void +arm_pv_time_handle_thread_context_switch_post(ticks_t curticks, + ticks_t prevticks) +{ + thread_t *current = thread_get_self(); + + // The start of the stolen time period is the absolute time the thread + // stopped running plus the relative time it spent yielding, or the + // absolute time the thread was last unblocked after blocking itself, + // whichever is later. + ticks_t adjusted_last_run = prevticks + current->arm_pv_time.yield_time; + // Note that the scheduler reads curticks before acquiring any locks, + // so it is possible that the last unblock occurred on a remote CPU + // _after_ curticks. Checking for this explicitly is probably cheaper + // than the synchronisation required to prevent it. + arm_pv_time_self_block_state_t state = + atomic_load_relaxed(¤t->arm_pv_time.self_block); + ticks_t last_self_unblock = util_min( + curticks, + arm_pv_time_self_block_state_get_last_unblocked(&state)); + ticks_t steal_start = util_max(last_self_unblock, adjusted_last_run); + + TRACE(DEBUG, INFO, + "arm_pv_time: {:#x} increment steal time by {:d}ns; " + "last run at {:d}ns (+ {:d}ns yielding), unblocked at {:d}ns", + (uintptr_t)current, + platform_convert_ticks_to_ns(curticks - steal_start), + platform_convert_ticks_to_ns(prevticks), + platform_convert_ticks_to_ns(current->arm_pv_time.yield_time), + platform_convert_ticks_to_ns(last_self_unblock)); + + assert((curticks >= adjusted_last_run) && + (curticks >= last_self_unblock)); + + current->arm_pv_time.yield_time = 0; + current->arm_pv_time.stolen_ticks += curticks - steal_start; + if (current->arm_pv_time.data != NULL) { + uint64_t stolen_ns = platform_convert_ticks_to_ns( + current->arm_pv_time.stolen_ticks); + atomic_store_relaxed(¤t->arm_pv_time.data->stolen_ns, + stolen_ns); + } +} + +void +arm_pv_time_handle_scheduler_blocked(thread_t *thread, scheduler_block_t block) +{ + if (thread == thread_get_self()) { + // Thread has blocked itself, presumably voluntarily. Reset the + // last-unblock time. + TRACE(DEBUG, INFO, "arm_pv_time: blocking self {:#x} ({:d})", + (uintptr_t)thread, (register_t)block); + arm_pv_time_self_block_state_t new_state = + arm_pv_time_self_block_state_default(); + arm_pv_time_self_block_state_set_block(&new_state, block); + atomic_store_relaxed(&thread->arm_pv_time.self_block, + new_state); + } +} + +void +arm_pv_time_handle_scheduler_unblocked(thread_t *thread, + scheduler_block_t block) +{ + arm_pv_time_self_block_state_t state = + atomic_load_relaxed(&thread->arm_pv_time.self_block); + ticks_t last_self_unblock = + arm_pv_time_self_block_state_get_last_unblocked(&state); + scheduler_block_t self_block = + arm_pv_time_self_block_state_get_block(&state); + if ((last_self_unblock == 0U) && (block == self_block)) { + // Thread has been woken after blocking itself, or is becoming + // runnable for the first time. + TRACE(DEBUG, INFO, "arm_pv_time: unblocking {:#x}", + (uintptr_t)thread); + arm_pv_time_self_block_state_t new_state = + arm_pv_time_self_block_state_default(); + arm_pv_time_self_block_state_set_last_unblocked( + &new_state, platform_timer_get_current_ticks()); + atomic_store_relaxed(&thread->arm_pv_time.self_block, + new_state); + } +} diff --git a/hyp/vm/arm_vm_amu/aarch64/arm_vm_amu_aarch64.ev b/hyp/vm/arm_vm_amu/aarch64/arm_vm_amu_aarch64.ev index 9da071a..ff9a232 100644 --- a/hyp/vm/arm_vm_amu/aarch64/arm_vm_amu_aarch64.ev +++ b/hyp/vm/arm_vm_amu/aarch64/arm_vm_amu_aarch64.ev @@ -4,6 +4,18 @@ module arm_vm_amu -#if defined(ARCH_ARM_8_4_AMU) -subscribe object_activate_thread +#if defined(ARCH_ARM_FEAT_AMUv1) || defined(ARCH_ARM_FEAT_AMUv1p1) +subscribe boot_cpu_cold_init + +subscribe vcpu_activate_thread + +subscribe thread_context_switch_pre(next) + +subscribe thread_context_switch_post(prev) + +// The AMU counters are read often, give them a higher priority +subscribe vcpu_trap_sysreg_read + priority 10 + +subscribe vcpu_trap_sysreg_write #endif diff --git a/hyp/vm/arm_vm_amu/aarch64/src/arm_vm_amu.c b/hyp/vm/arm_vm_amu/aarch64/src/arm_vm_amu.c index 0d36a74..6c79f0d 100644 --- a/hyp/vm/arm_vm_amu/aarch64/src/arm_vm_amu.c +++ b/hyp/vm/arm_vm_amu/aarch64/src/arm_vm_amu.c @@ -5,33 +5,322 @@ #include #include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "arm_vm_amu.h" #include "event_handlers.h" -#if (ARCH_ARM_VER < 84) && defined(ARCH_ARM_8_4_AMU) -// AMU should not be enabled on ARM cores earlier than v8.4 -#error AMU configuration error +// The design: +// Only HLOS is given access to the AMU component. However, HLOS should not see +// how much the counters increment during the execution of the sensitive VMs. +// +// Unfortunately, only the highest EL can write to the AMU control registers and +// counters; therefore we can't protect against the AMU cross-exposure +// by simply disabling the counters dynamically or context switching them. +// +// Therefore we use a set of CPU-local variables to keep track of how much each +// counter increments during the sensitive threads. We do this by subtracting +// the counter value from our variable before switching to a sensitive thread, +// and adding the counter value when switching away from it. +// +// All the AMU accesses from HLOS are trapped. When HLOS tries to read a counter +// we return the hardware value minus our internal offset from above. +// The AMU counters take centuries to overflow, so arithmetic overflows are not +// a concern. +// +// This is not needed for counter 1 ("constant frequency cycles" counter), +// which is essentially defined the same as the ARM physical counter and +// virtualising it does not provide any additional security. + +#if defined(ARCH_ARM_FEAT_AMUv1p1) +#error Implement the AMU virtual offset registers +#error Investigate the need to support non-consecutive auxiliary counters #endif -#if defined(ARCH_ARM_8_4_AMU) -error_t -arm_vm_amu_handle_object_activate_thread(thread_t *thread) +#if defined(ARCH_ARM_FEAT_AMUv1) || defined(ARCH_ARM_FEAT_AMUv1p1) +CPULOCAL_DECLARE_STATIC(uint64_t, amu_counter_offsets)[PLATFORM_AMU_CNT_NUM]; +CPULOCAL_DECLARE_STATIC(uint64_t, amu_aux_counter_offsets) +[PLATFORM_AMU_AUX_CNT_NUM]; + +void +arm_vm_amu_handle_boot_cpu_cold_init(cpu_index_t cpu_index) +{ + uint64_t *amu_counter_offsets = + CPULOCAL_BY_INDEX(amu_counter_offsets, cpu_index); + uint64_t *amu_aux_counter_offsets = + CPULOCAL_BY_INDEX(amu_aux_counter_offsets, cpu_index); + + for (index_t i = 0; i < PLATFORM_AMU_CNT_NUM; i++) { + amu_counter_offsets[i] = 0; + } + for (index_t i = 0; i < PLATFORM_AMU_AUX_CNT_NUM; i++) { + amu_aux_counter_offsets[i] = 0; + } + + AMCFGR_EL0_t amcfgr = register_AMCFGR_EL0_read(); + AMCGCR_EL0_t amcgcr = register_AMCGCR_EL0_read(); + +#if defined(ARCH_ARM_FEAT_AMUv1p1) +#error TODO: Check the AMU counter bitmap +#else + if ((AMCFGR_EL0_get_N(&amcfgr) + 1U) != + (PLATFORM_AMU_CNT_NUM + PLATFORM_AMU_AUX_CNT_NUM)) { + panic("Incorrect CPU AMU count"); + } +#endif + if ((AMCGCR_EL0_get_CG0NC(&amcgcr) != PLATFORM_AMU_CNT_NUM) || + (AMCGCR_EL0_get_CG1NC(&amcgcr) != PLATFORM_AMU_AUX_CNT_NUM)) { + panic("Incorrect CPU AMU group counts"); + } +} + +bool +arm_vm_amu_handle_vcpu_activate_thread(thread_t *thread, + vcpu_option_flags_t options) { assert(thread != NULL); + assert(thread->kind == THREAD_KIND_VCPU); + + // Trap accesses to AMU registers. For HLOS we will emulate + // them, for the rest of the VMs we will leave them unhandled + // and inject an abort. + CPTR_EL2_E2H1_set_TAM(&thread->vcpu_regs_el2.cptr_el2, true); + + vcpu_option_flags_set_amu_counting_disabled( + &thread->vcpu_options, + vcpu_option_flags_get_amu_counting_disabled(&options)); + + return true; +} + +error_t +arm_vm_amu_handle_thread_context_switch_pre(thread_t *next) +{ + // If about to switch to a sensitive thread, take a snapshot of the AMU + // counters by subtracting them from the offsets. + // In theory it is not necessary to do this if we are coming from + // another sensitive thread, but adding the required extra checks will + // likely degrade the performance as this will be a rare occurrence. + if (compiler_unexpected((next->kind == THREAD_KIND_VCPU) && + (vcpu_option_flags_get_amu_counting_disabled( + &next->vcpu_options)))) { + cpulocal_begin(); + arm_vm_amu_subtract_counters(&CPULOCAL(amu_counter_offsets)); + arm_vm_amu_subtract_aux_counters( + &CPULOCAL(amu_aux_counter_offsets)); + cpulocal_end(); + } + + return OK; +} + +void +arm_vm_amu_handle_thread_context_switch_post(thread_t *prev) +{ + // If about to switch away from a sensitive thread, take a snapshot of + // the AMU counters by adding them to the offsets. + // In theory it is not necessary to do this if we are switching to + // another sensitive thread, but adding the required extra checks will + // likely degrade the performance as this will be a rare occurrence. + if (compiler_unexpected((prev->kind == THREAD_KIND_VCPU) && + (vcpu_option_flags_get_amu_counting_disabled( + &prev->vcpu_options)))) { + cpulocal_begin(); + arm_vm_amu_add_counters(&CPULOCAL(amu_counter_offsets)); + arm_vm_amu_add_aux_counters(&CPULOCAL(amu_aux_counter_offsets)); + cpulocal_end(); + } +} + +static vcpu_trap_result_t +arm_vm_amu_get_event_register(ESR_EL2_ISS_MSR_MRS_t iss, uint64_t *val) +{ + vcpu_trap_result_t ret; + uint8_t opc0 = ESR_EL2_ISS_MSR_MRS_get_Op0(&iss); + uint8_t opc1 = ESR_EL2_ISS_MSR_MRS_get_Op1(&iss); + uint8_t crn = ESR_EL2_ISS_MSR_MRS_get_CRn(&iss); - if (thread->kind == THREAD_KIND_VCPU) { - // Give HLOS full access to AMU registers. For the rest of the - // VMs, trap accesses to AMU registers without handling them, so - // an abort gets injected to any non-HLOS guest that tries to - // use AMU. - if (vcpu_option_flags_get_hlos_vm(&thread->vcpu_options)) { - CPTR_EL2_E2H1_set_TAM(&thread->vcpu_regs_el2.cptr_el2, - false); + if ((opc0 == 3U) && (opc1 == 3U) && (crn == 13U)) { + uint8_t crm = ESR_EL2_ISS_MSR_MRS_get_CRm(&iss); + uint8_t opc2 = ESR_EL2_ISS_MSR_MRS_get_Op2(&iss); + uint8_t index = (uint8_t)((crm & 1U) << 2U) | opc2; + + if (((crm == 4U) || (crm == 5U)) && + (index < PLATFORM_AMU_CNT_NUM)) { + // Event counter registers + cpulocal_begin(); + uint64_t *offsets = CPULOCAL(amu_counter_offsets); + *val = arm_vm_amu_get_counter(index); + if (index != 1U) { + // Adjust the counter value + *val -= offsets[index]; + } + cpulocal_end(); + + ret = VCPU_TRAP_RESULT_EMULATED; + } else if (((crm == 6U) || (crm == 7U)) && + (index < PLATFORM_AMU_CNT_NUM)) { + // Event type registers + *val = arm_vm_amu_get_event_type(index); + + ret = VCPU_TRAP_RESULT_EMULATED; + } else if (((crm == 12U) || (crm == 13U)) && + (index < PLATFORM_AMU_AUX_CNT_NUM)) { + // Auxiliary event counter registers + cpulocal_begin(); + uint64_t *offsets = CPULOCAL(amu_aux_counter_offsets); + *val = arm_vm_amu_get_aux_counter(index); + // Adjust the counter value + *val -= offsets[index]; + cpulocal_end(); + + ret = VCPU_TRAP_RESULT_EMULATED; + } else if (((crm == 14U) || (crm == 15U)) && + (index < PLATFORM_AMU_AUX_CNT_NUM)) { + // Auxiliary event type registers + *val = arm_vm_amu_get_aux_event_type(index); + + ret = VCPU_TRAP_RESULT_EMULATED; } else { - CPTR_EL2_E2H1_set_TAM(&thread->vcpu_regs_el2.cptr_el2, - true); + // Not an AMU event register + ret = VCPU_TRAP_RESULT_UNHANDLED; } + } else { + ret = VCPU_TRAP_RESULT_UNHANDLED; } - return OK; + return ret; +} + +vcpu_trap_result_t +arm_vm_amu_handle_vcpu_trap_sysreg_read(ESR_EL2_ISS_MSR_MRS_t iss) +{ + register_t val = 0U; + vcpu_trap_result_t ret = VCPU_TRAP_RESULT_EMULATED; + thread_t *thread = thread_get_self(); + + if (!vcpu_option_flags_get_hlos_vm(&thread->vcpu_options)) { + // Only HLOS is allowed to read the AMU registers + ret = VCPU_TRAP_RESULT_UNHANDLED; + goto out; + } + + // Assert this is a read + assert(ESR_EL2_ISS_MSR_MRS_get_Direction(&iss)); + + uint8_t reg_num = ESR_EL2_ISS_MSR_MRS_get_Rt(&iss); + + // Remove the fields that are not used in the comparison + ESR_EL2_ISS_MSR_MRS_t temp_iss = iss; + ESR_EL2_ISS_MSR_MRS_set_Rt(&temp_iss, 0U); + ESR_EL2_ISS_MSR_MRS_set_Direction(&temp_iss, false); + + switch (ESR_EL2_ISS_MSR_MRS_raw(temp_iss)) { + case ISS_MRS_MSR_AMCR_EL0: { + AMCR_EL0_t amcr = AMCR_EL0_default(); + val = AMCR_EL0_raw(amcr); + break; + } + case ISS_MRS_MSR_AMCFGR_EL0: { + AMCFGR_EL0_t amcfgr = AMCFGR_EL0_default(); + AMCFGR_EL0_t amcfgr_hw = register_AMCFGR_EL0_read(); + + AMCFGR_EL0_copy_HDBG(&amcfgr, &amcfgr_hw); + AMCFGR_EL0_set_Size(&amcfgr, 63U); + // With traps, it is possible to virtualise the number of HW + // counters; return the number of emulated counters. + AMCFGR_EL0_set_N(&amcfgr, + (uint16_t)(PLATFORM_AMU_CNT_NUM + + PLATFORM_AMU_AUX_CNT_NUM - 1U)); + AMCFGR_EL0_set_NCG(&amcfgr, + PLATFORM_AMU_AUX_CNT_NUM > 0U ? 1U : 0U); + val = AMCFGR_EL0_raw(amcfgr); + break; + } + case ISS_MRS_MSR_AMCGCR_EL0: { + AMCGCR_EL0_t amcgcr = AMCGCR_EL0_default(); + + // With traps, it is possible to virtualise the number of HW + // counters; return the number of emulated counters. + AMCGCR_EL0_set_CG0NC(&amcgcr, PLATFORM_AMU_CNT_NUM); + AMCGCR_EL0_set_CG1NC(&amcgcr, PLATFORM_AMU_AUX_CNT_NUM); + val = AMCGCR_EL0_raw(amcgcr); + break; + } + case ISS_MRS_MSR_AMUSERENR_EL0: + val = register_AMUSERENR_EL0_read(); + break; +#if defined(ARCH_ARM_FEAT_AMUv1p1) + case ISS_MRS_MSR_AMCG1IDR_EL0: + val = register_AMCG1IDR_EL0_read(); + break; +#endif + default: + ret = arm_vm_amu_get_event_register(iss, &val); + break; + } + + // Update the thread's register + if (ret == VCPU_TRAP_RESULT_EMULATED) { + vcpu_gpr_write(thread, reg_num, val); + } + +out: + return ret; +} + +vcpu_trap_result_t +arm_vm_amu_handle_vcpu_trap_sysreg_write(ESR_EL2_ISS_MSR_MRS_t iss) +{ + vcpu_trap_result_t ret = VCPU_TRAP_RESULT_UNHANDLED; + thread_t *thread = thread_get_self(); + + if (!vcpu_option_flags_get_hlos_vm(&thread->vcpu_options)) { + // Only HLOS is allowed to modify the AMU registers + goto out; + } + + // Assert this is a write + assert(!ESR_EL2_ISS_MSR_MRS_get_Direction(&iss)); + + // Read the thread's register + uint8_t reg_num = ESR_EL2_ISS_MSR_MRS_get_Rt(&iss); + register_t val = vcpu_gpr_read(thread, reg_num); + + // Remove the fields that are not used in the comparison + ESR_EL2_ISS_MSR_MRS_t temp_iss = iss; + ESR_EL2_ISS_MSR_MRS_set_Rt(&temp_iss, 0U); + ESR_EL2_ISS_MSR_MRS_set_Direction(&temp_iss, false); + + // Only AMUSERNR_EL0 may be written by the guest + switch (ESR_EL2_ISS_MSR_MRS_raw(temp_iss)) { + case ISS_MRS_MSR_AMUSERENR_EL0: { + SPSR_EL2_A64_t spsr_el2 = thread->vcpu_regs_gpr.spsr_el2.a64; + spsr_64bit_mode_t spsr_m = SPSR_EL2_A64_get_M(&spsr_el2); + // A trap from EL0 means a HW bug + assert((spsr_m & 0xfU) != 0U); + + register_AMUSERENR_EL0_write(val); + ret = VCPU_TRAP_RESULT_EMULATED; + break; + } + default: + // Do Nothing + break; + } + +out: + return ret; } #endif diff --git a/hyp/vm/arm_vm_amu/aarch64/templates/arm_vm_amu_counter_regs.c.tmpl b/hyp/vm/arm_vm_amu/aarch64/templates/arm_vm_amu_counter_regs.c.tmpl new file mode 100644 index 0000000..dda5045 --- /dev/null +++ b/hyp/vm/arm_vm_amu/aarch64/templates/arm_vm_amu_counter_regs.c.tmpl @@ -0,0 +1,169 @@ +// Automatically generated. Do not modify. +// +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +\#include +\#include + +\#include + +\#include +\#include +\#include + +\#include +\#include + +\#include "arm_vm_amu.h" + +\#if defined(ARCH_ARM_FEAT_AMUv1) || defined(ARCH_ARM_FEAT_AMUv1p1) +#set $cnt_num = $PLATFORM_AMU_CNT_NUM +#set $aux_cnt_num = $PLATFORM_AMU_AUX_CNT_NUM + +uint64_t +arm_vm_amu_get_counter(index_t index) +{ + uint64_t val; + + switch (index) + { +#for i in range(0, $cnt_num) + case ${i}: + sysreg64_read_ordered(AMEVCNTR0${i}_EL0, val, asm_ordering); + break; +#end for + default: + TRACE_AND_LOG(DEBUG, WARN, + "Read of non-existing AMU counter {:d} ", index); + val = 0; + break; + } + + return val; + +} + +uint64_t +arm_vm_amu_get_aux_counter(index_t index) +{ + uint64_t val; + + switch (index) + { +#for i in range(0, $aux_cnt_num) + case ${i}: + sysreg64_read_ordered(AMEVCNTR1${i}_EL0, val, asm_ordering); + break; +#end for + default: + TRACE_AND_LOG( + DEBUG, WARN, + "Read of non-existing AMU auxiliary counter {:d} ", + index); + val = 0; + break; + } + + return val; + +} + +uint64_t +arm_vm_amu_get_event_type(index_t index) +{ + uint64_t val; + + switch (index) + { +#for i in range(0, $cnt_num) + case ${i}: + sysreg64_read_ordered(AMEVTYPER0${i}_EL0, val, asm_ordering); + break; +#end for + default: + TRACE_AND_LOG(DEBUG, WARN, + "Read of non-existing AMU event type {:d} ", + index); + val = 0; + break; + } + + return val; + +} + +uint64_t +arm_vm_amu_get_aux_event_type(index_t index) +{ + uint64_t val; + + switch (index) + { +#for i in range(0, $aux_cnt_num) + case ${i}: + sysreg64_read_ordered(AMEVTYPER1${i}_EL0, val, asm_ordering); + break; +#end for + default: + TRACE_AND_LOG( + DEBUG, WARN, + "Read of non-existing AMU auxiliary event type {:d} ", + index); + val = 0; + break; + } + + return val; + +} + +void +arm_vm_amu_add_counters(arm_vm_amu_offsets_t *offsets) +{ + uint64_t val; + +#for i in range(0, $cnt_num) +#if i != 1 + sysreg64_read_ordered(AMEVCNTR0${i}_EL0, val, asm_ordering); + (*offsets)[${i}] += val; +#end if +#end for +} + +void +arm_vm_amu_subtract_counters(arm_vm_amu_offsets_t *offsets) +{ + uint64_t val; + +#for i in range(0, $cnt_num) +#if i != 1 + sysreg64_read_ordered(AMEVCNTR0${i}_EL0, val, asm_ordering); + (*offsets)[${i}] -= val; +#end if +#end for +} + +void +arm_vm_amu_add_aux_counters(arm_vm_amu_aux_offsets_t *offsets) +{ + uint64_t val; + +#for i in range(0, $aux_cnt_num) + sysreg64_read_ordered(AMEVCNTR1${i}_EL0, val, asm_ordering); + (*offsets)[${i}] += val; +#end for +} + +void +arm_vm_amu_subtract_aux_counters(arm_vm_amu_aux_offsets_t *offsets) +{ + uint64_t val; + +#for i in range(0, $aux_cnt_num) + sysreg64_read_ordered(AMEVCNTR1${i}_EL0, val, asm_ordering); + (*offsets)[${i}] -= val; +#end for +} +\#endif diff --git a/hyp/vm/arm_vm_amu/build.conf b/hyp/vm/arm_vm_amu/build.conf index 2f4b47a..153cb7d 100644 --- a/hyp/vm/arm_vm_amu/build.conf +++ b/hyp/vm/arm_vm_amu/build.conf @@ -2,5 +2,7 @@ # # SPDX-License-Identifier: BSD-3-Clause +local_include arch_events aarch64 arm_vm_amu_aarch64.ev arch_source aarch64 arm_vm_amu.c +arch_template simple aarch64 arm_vm_amu_counter_regs.c.tmpl diff --git a/hyp/vm/arm_vm_amu/include/arm_vm_amu.h b/hyp/vm/arm_vm_amu/include/arm_vm_amu.h new file mode 100644 index 0000000..04b1b5c --- /dev/null +++ b/hyp/vm/arm_vm_amu/include/arm_vm_amu.h @@ -0,0 +1,32 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#if defined(ARCH_ARM_FEAT_AMUv1) || defined(ARCH_ARM_FEAT_AMUv1p1) +typedef uint64_t arm_vm_amu_offsets_t[PLATFORM_AMU_CNT_NUM]; +typedef uint64_t arm_vm_amu_aux_offsets_t[PLATFORM_AMU_AUX_CNT_NUM]; + +uint64_t +arm_vm_amu_get_counter(index_t index); + +uint64_t +arm_vm_amu_get_aux_counter(index_t index); + +uint64_t +arm_vm_amu_get_event_type(index_t index); + +uint64_t +arm_vm_amu_get_aux_event_type(index_t index); + +void +arm_vm_amu_add_counters(arm_vm_amu_offsets_t *offsets); + +void +arm_vm_amu_subtract_counters(arm_vm_amu_offsets_t *offsets); + +void +arm_vm_amu_add_aux_counters(arm_vm_amu_aux_offsets_t *offsets); + +void +arm_vm_amu_subtract_aux_counters(arm_vm_amu_aux_offsets_t *offsets); +#endif diff --git a/hyp/vm/arm_vm_pmu/aarch64/arm_vm_pmu_aarch64.tc b/hyp/vm/arm_vm_pmu/aarch64/arm_vm_pmu_aarch64.tc index 9fd632c..6b71ebe 100644 --- a/hyp/vm/arm_vm_pmu/aarch64/arm_vm_pmu_aarch64.tc +++ b/hyp/vm/arm_vm_pmu/aarch64/arm_vm_pmu_aarch64.tc @@ -11,7 +11,7 @@ define vcpu_pmu_registers structure { pmccfiltr_el0 uint32; pmintenset_el1 uint32; pmcntenset_el0 uint32; -#if defined(ARCH_ARM_8_5_PMU) +#if defined(ARCH_ARM_FEAT_PMUv3p5) pmevcntr_el0 array(PLATFORM_PMU_CNT_NUM) uint64; pmevtyper_el0 array(PLATFORM_PMU_CNT_NUM) uint64; #else diff --git a/hyp/vm/arm_vm_pmu/aarch64/src/arm_vm_pmu.c b/hyp/vm/arm_vm_pmu/aarch64/src/arm_vm_pmu.c index 568e863..8ac52cf 100644 --- a/hyp/vm/arm_vm_pmu/aarch64/src/arm_vm_pmu.c +++ b/hyp/vm/arm_vm_pmu/aarch64/src/arm_vm_pmu.c @@ -9,9 +9,11 @@ #include #include +#include #include #include #include +#include #include #include @@ -44,6 +46,7 @@ // the context switching of the PMU registers. Currently it looks like Linux // does not actually comply with this standard anyway, except for writing the // debugger claim bits in the statistical profiling driver. +// FIXME: #if (ARCH_ARM_PMU_VER < 3) #error Only PMUv3 and above can be implemented in ARMv8/ARMv9. @@ -55,7 +58,7 @@ arm_vm_pmu_counters_enabled(thread_t *current) // Check if the global enable flag is set and at least one counter is // enabled. return (PMCR_EL0_get_E(¤t->pmu.pmu_regs.pmcr_el0) && - (current->pmu.pmu_regs.pmcntenset_el0 != 0)); + (current->pmu.pmu_regs.pmcntenset_el0 != 0U)); } static bool @@ -80,7 +83,7 @@ arm_vm_pmu_aarch64_handle_object_activate_thread(thread_t *thread) PMCR_EL0_t pmcr_el0 = register_PMCR_EL0_read(); MDCR_EL2_set_HPMN(&thread->vcpu_regs_el2.mdcr_el2, PMCR_EL0_get_N(&pmcr_el0)); -#if defined(ARCH_ARM_8_1_PMU) +#if defined(ARCH_ARM_FEAT_PMUv3p1) // Prohibit event counting at EL2 MDCR_EL2_set_HPMD(&thread->vcpu_regs_el2.mdcr_el2, true); #endif @@ -118,7 +121,7 @@ arm_vm_pmu_save_state(thread_t *thread) sysreg64_read_ordered(PMOVSSET_EL0, thread->pmu.pmu_regs.pmovsset_el0, asm_ordering); -#if !defined(ARCH_ARM_8_1_PMU) +#if !defined(ARCH_ARM_FEAT_PMUv3p1) // Event counting cannot be prohibited at EL2. Do an ISB to make sure // the operation above completes before we continue. This to ensure that // the register reads above are not delayed until after some sensitive @@ -170,19 +173,19 @@ arm_vm_pmu_handle_thread_save_state(void) { thread_t *thread = thread_get_self(); - if ((thread->kind == THREAD_KIND_VCPU) && - (!arm_vm_pmu_is_el1_trap_enabled(thread))) { + if (compiler_expected(thread->kind == THREAD_KIND_VCPU) && + compiler_unexpected(!arm_vm_pmu_is_el1_trap_enabled(thread))) { // PMU access was enabled for this timeslice, save the state arm_vm_pmu_save_state(thread); } } void -arm_vm_pmu_handle_thread_context_switch_post() +arm_vm_pmu_handle_thread_context_switch_post(void) { thread_t *thread = thread_get_self(); - if (thread->kind == THREAD_KIND_VCPU) { + if (compiler_expected(thread->kind == THREAD_KIND_VCPU)) { bool_result_t asserted = virq_query(&thread->pmu.pmu_virq_src); if ((asserted.e == OK) && !asserted.r) { platform_pmu_hw_irq_deactivate(); @@ -207,14 +210,17 @@ arm_vm_pmu_handle_thread_load_state(void) { thread_t *thread = thread_get_self(); - if ((thread->kind == THREAD_KIND_VCPU) && - arm_vm_pmu_counters_enabled(thread)) { + if (compiler_unexpected(thread->kind != THREAD_KIND_VCPU)) { + // Idle thread. Turn off the counters and the interrupts. + sysreg64_write_ordered(PMINTENCLR_EL1, ~0UL, asm_ordering); + sysreg64_write_ordered(PMCNTENCLR_EL0, ~0UL, asm_ordering); + } else if (compiler_unexpected(arm_vm_pmu_counters_enabled(thread))) { // The thread is actively using PMU. The context_switch_post // has already disabled traps for this thread above, and it will // get loaded into MDCR_EL2 during the context switch load // process in the generic module. Load its PMU context here. arm_vm_pmu_load_state(thread); - } else if (thread->kind == THREAD_KIND_VCPU) { + } else { // The thread is not actively using PMU. The context_switch_post // has already enabled traps for this thread above, and it will // get loaded into MDCR_EL2 during the context switch load @@ -226,12 +232,8 @@ arm_vm_pmu_handle_thread_load_state(void) // doesn't currently have access to thems. // // Turn off the counters and the interrupts. - sysreg64_write_ordered(PMINTENCLR_EL1, ~0U, asm_ordering); - sysreg64_write_ordered(PMCNTENCLR_EL0, ~0U, asm_ordering); - } else { - // Idle thread. Turn off the counters and the interrupts. - sysreg64_write_ordered(PMINTENCLR_EL1, ~0U, asm_ordering); - sysreg64_write_ordered(PMCNTENCLR_EL0, ~0U, asm_ordering); + sysreg64_write_ordered(PMINTENCLR_EL1, ~0UL, asm_ordering); + sysreg64_write_ordered(PMCNTENCLR_EL0, ~0UL, asm_ordering); } } @@ -240,7 +242,7 @@ arm_vm_pmu_handle_virq_check_pending(virq_source_t *source) { bool ret = true; - pmu_t *pmu = pmu_container_of_pmu_virq_src(source); + pmu_t *pmu = pmu_container_of_pmu_virq_src(source); thread_t *thread = thread_container_of_pmu(pmu); assert(thread != NULL); @@ -266,7 +268,7 @@ vcpu_trap_result_t arm_vm_pmu_handle_vcpu_trap_sysreg_access(ESR_EL2_ISS_MSR_MRS_t iss) { vcpu_trap_result_t ret = VCPU_TRAP_RESULT_UNHANDLED; - thread_t *thread = thread_get_self(); + thread_t *thread = thread_get_self(); // Remove the fields that are not used in the comparison ESR_EL2_ISS_MSR_MRS_t temp_iss = iss; @@ -300,7 +302,8 @@ arm_vm_pmu_handle_vcpu_trap_sysreg_access(ESR_EL2_ISS_MSR_MRS_t iss) crn = ESR_EL2_ISS_MSR_MRS_get_CRn(&iss); crm = ESR_EL2_ISS_MSR_MRS_get_CRm(&iss); - if ((opc0 == 3) && (opc1 == 3) && (crn == 14) && (crm >= 8)) { + if ((opc0 == 3U) && (opc1 == 3U) && (crn == 14U) && + (crm >= 8U)) { // PMEVCNTR and PMEVTYPER registers ret = VCPU_TRAP_RESULT_RETRY; } diff --git a/hyp/vm/arm_vm_pmu/arm_vm_pmu.ev b/hyp/vm/arm_vm_pmu/arm_vm_pmu.ev index 513f9ad..0dd718b 100644 --- a/hyp/vm/arm_vm_pmu/arm_vm_pmu.ev +++ b/hyp/vm/arm_vm_pmu/arm_vm_pmu.ev @@ -9,6 +9,7 @@ subscribe thread_save_state() subscribe thread_load_state() subscribe thread_context_switch_post() + require_preempt_disabled subscribe object_activate_thread unwinder arm_vm_pmu_handle_object_deactivate_thread(thread) @@ -18,3 +19,4 @@ subscribe object_deactivate_thread subscribe platform_pmu_counter_overflow subscribe virq_check_pending[VIRQ_TRIGGER_PMU](source) + require_preempt_disabled diff --git a/hyp/vm/arm_vm_sve_simple/aarch64/arm_vm_sve_aarch64.ev b/hyp/vm/arm_vm_sve_simple/aarch64/arm_vm_sve_aarch64.ev new file mode 100644 index 0000000..a94ad98 --- /dev/null +++ b/hyp/vm/arm_vm_sve_simple/aarch64/arm_vm_sve_aarch64.ev @@ -0,0 +1,20 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module arm_vm_sve_simple + +#if defined(ARCH_ARM_FEAT_SVE) +subscribe boot_runtime_first_init + handler arm_vm_sve_simple_handle_boot_runtime_init() +subscribe boot_runtime_warm_init + handler arm_vm_sve_simple_handle_boot_runtime_init() +subscribe boot_cold_init() +subscribe boot_cpu_warm_init() + +subscribe rootvm_init(qcbor_enc_ctxt) + +subscribe vcpu_activate_thread + +subscribe vcpu_trap_sysreg_read +#endif diff --git a/hyp/vm/arm_vm_sve_simple/aarch64/arm_vm_sve_aarch64.tc b/hyp/vm/arm_vm_sve_simple/aarch64/arm_vm_sve_aarch64.tc new file mode 100644 index 0000000..94b8c6e --- /dev/null +++ b/hyp/vm/arm_vm_sve_simple/aarch64/arm_vm_sve_aarch64.tc @@ -0,0 +1,13 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#if defined(ARCH_ARM_FEAT_SVE) +define ZCR_EL2 bitfield<64> { + 3:0 LEN uint8; + others unknown=0; +}; + +define SVE_Z_MIN_REG_SIZE constant type count_t = 16; +define SVE_Z_REG_SIZE constant type count_t = PLATFORM_SVE_REG_SIZE; +#endif diff --git a/hyp/vm/arm_vm_sve_simple/aarch64/src/arm_vm_sve.c b/hyp/vm/arm_vm_sve_simple/aarch64/src/arm_vm_sve.c new file mode 100644 index 0000000..da3b829 --- /dev/null +++ b/hyp/vm/arm_vm_sve_simple/aarch64/src/arm_vm_sve.c @@ -0,0 +1,173 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#if defined(ARCH_ARM_FEAT_SVE) + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "event_handlers.h" + +// A simple SVE module that allows SVE access to HLOS only. + +static bool sve_disabled = false; + +// Ensure the value of SVE_Z_REG_SIZE (PLATFORM_SVE_REG_SIZE) is sane +static_assert(SVE_Z_REG_SIZE >= SVE_Z_MIN_REG_SIZE, + "SVE register size should be minimum 16 bytes"); + +// Due a LLVM12.0 design choice, "-mgeneral-regs-only" also excludes the SVE +// registers. Therefore ".arch_extension sve;" needs to be added to all the +// inline "asm" statements that access SVE. +// For this reason the SVE code uses MSR/MRS directly instead of generated +// read/write accessors. +static inline void +register_ZCR_EL2_write(const ZCR_EL2_t val) +{ + register_t raw = (register_t)ZCR_EL2_raw(val); + __asm__ volatile(".arch_extension sve;" + "msr ZCR_EL2, %[r]" + : + : [r] "rz"(raw)); +} + +static inline uint64_t +register_ID_AA64ZFR0_EL1_read(void) +{ + register_t val; + __asm__ volatile(".arch_extension sve;" + "mrs %0, ID_AA64ZFR0_EL1;" + : "=r"(val)); + return (uint64_t)(val); +} + +void +arm_vm_sve_simple_handle_boot_runtime_init(void) +{ + // Before writing ZCR_EL2, make sure EL2 has access to SVE subsystem. + // Unfortunately the same bit that controls EL1/EL0 access also controls + // EL2 access. + CPTR_EL2_E2H1_t cptr = + register_CPTR_EL2_E2H1_read_ordered(&asm_ordering); + CPTR_EL2_E2H1_set_ZEN(&cptr, CPTR_ZEN_TRAP_NONE); + register_CPTR_EL2_E2H1_write_ordered(cptr, &asm_ordering); +} + +void +arm_vm_sve_simple_handle_boot_cold_init(void) +{ + platform_cpu_features_t features = platform_get_cpu_features(); + + sve_disabled = platform_cpu_features_get_sve_disable(&features); +} + +void +arm_vm_sve_simple_handle_boot_cpu_warm_init(void) +{ + if (!sve_disabled) { + // Initialise ZCR_EL2 as its reset value is architecturally + // UNKNOWN. SVE register size is (ZCR_EL2.LEN + 1) * 128 bits. + // SVE_Z_REG_SIZE is in bytes. + ZCR_EL2_t zcr = ZCR_EL2_default(); + ZCR_EL2_set_LEN(&zcr, + (uint8_t)(((SVE_Z_REG_SIZE << 3) >> 7) - 1U)); + + register_ZCR_EL2_write(zcr); + // No need to disable SVE access, the context-switch code will + // do it if necessary (if we are switching to a non-HLOS VM) + } +} + +void +arm_vm_sve_simple_handle_rootvm_init(qcbor_enc_ctxt_t *qcbor_enc_ctxt) +{ + assert(qcbor_enc_ctxt != NULL); + + QCBOREncode_AddBoolToMap(qcbor_enc_ctxt, "sve_supported", + !sve_disabled); +} + +bool +arm_vm_sve_simple_handle_vcpu_activate_thread(thread_t *thread, + vcpu_option_flags_t options) +{ + assert(thread != NULL); + bool ret; + + if (thread->kind == THREAD_KIND_VCPU) { + bool hlos = vcpu_option_flags_get_hlos_vm(&options); + bool sve_allowed = vcpu_option_flags_get_sve_allowed(&options); + + if (sve_allowed && sve_disabled) { + // Not permitted + ret = false; + } else if (hlos && sve_allowed) { + // Give HLOS threads SVE access + CPTR_EL2_E2H1_set_ZEN(&thread->vcpu_regs_el2.cptr_el2, + CPTR_ZEN_TRAP_NONE); + vcpu_option_flags_set_sve_allowed(&thread->vcpu_options, + true); + ret = true; + } else if (!hlos && sve_allowed) { + // Not supported + ret = false; + } else { + CPTR_EL2_E2H1_set_ZEN(&thread->vcpu_regs_el2.cptr_el2, + CPTR_ZEN_TRAP_ALL); + ret = true; + } + } else { + ret = true; + } + + return ret; +} + +// ID_AA64ZFR0_EL1 is trapped through HCR_EL2.TID3 +vcpu_trap_result_t +arm_vm_sve_simple_handle_vcpu_trap_sysreg_read(ESR_EL2_ISS_MSR_MRS_t iss) +{ + vcpu_trap_result_t ret; + thread_t *thread = thread_get_self(); + + // Assert this is a read + assert(ESR_EL2_ISS_MSR_MRS_get_Direction(&iss)); + + // Remove the fields that are not used in the comparison + ESR_EL2_ISS_MSR_MRS_t temp_iss = iss; + ESR_EL2_ISS_MSR_MRS_set_Rt(&temp_iss, 0U); + ESR_EL2_ISS_MSR_MRS_set_Direction(&temp_iss, false); + + switch (ESR_EL2_ISS_MSR_MRS_raw(temp_iss)) { + case ISS_MRS_MSR_ID_AA64ZFR0_EL1: + ret = VCPU_TRAP_RESULT_EMULATED; + uint64_t val; + if (vcpu_option_flags_get_sve_allowed(&thread->vcpu_options)) { + val = register_ID_AA64ZFR0_EL1_read(); + } else { + val = 0U; + } + uint8_t reg_num = ESR_EL2_ISS_MSR_MRS_get_Rt(&iss); + vcpu_gpr_write(thread, reg_num, val); + break; + default: + ret = VCPU_TRAP_RESULT_UNHANDLED; + break; + } + + return ret; +} +#endif diff --git a/hyp/vm/arm_vm_sve_simple/build.conf b/hyp/vm/arm_vm_sve_simple/build.conf new file mode 100644 index 0000000..9a52722 --- /dev/null +++ b/hyp/vm/arm_vm_sve_simple/build.conf @@ -0,0 +1,8 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +interface arm_sve +arch_types aarch64 arm_vm_sve_aarch64.tc +arch_events aarch64 arm_vm_sve_aarch64.ev +arch_source aarch64 arm_vm_sve.c diff --git a/hyp/vm/arm_vm_timer/aarch64/src/arm_vm_timer.c b/hyp/vm/arm_vm_timer/aarch64/src/arm_vm_timer.c index a16d331..94d6a4c 100644 --- a/hyp/vm/arm_vm_timer/aarch64/src/arm_vm_timer.c +++ b/hyp/vm/arm_vm_timer/aarch64/src/arm_vm_timer.c @@ -43,16 +43,11 @@ arm_vm_timer_init(arm_vm_timer_type_t tt) CNT_CTL_init(&cnt_ctl); CNT_CTL_set_IMASK(&cnt_ctl, true); - switch (tt) { - case ARM_VM_TIMER_TYPE_VIRTUAL: + if (tt == ARM_VM_TIMER_TYPE_VIRTUAL) { register_CNTV_CTL_EL0_write_ordered(cnt_ctl, &asm_ordering); - break; - - case ARM_VM_TIMER_TYPE_PHYSICAL: + } else if (tt == ARM_VM_TIMER_TYPE_PHYSICAL) { register_CNTP_CTL_EL0_write_ordered(cnt_ctl, &asm_ordering); - break; - - default: + } else { panic("Invalid timer"); } } @@ -62,18 +57,13 @@ arm_vm_timer_is_irq_enabled(arm_vm_timer_type_t tt) { CNT_CTL_t cnt_ctl; - switch (tt) { - case ARM_VM_TIMER_TYPE_VIRTUAL: + if (tt == ARM_VM_TIMER_TYPE_VIRTUAL) { cnt_ctl = register_CNTV_CTL_EL0_read_volatile_ordered( &asm_ordering); - break; - - case ARM_VM_TIMER_TYPE_PHYSICAL: + } else if (tt == ARM_VM_TIMER_TYPE_PHYSICAL) { cnt_ctl = register_CNTP_CTL_EL0_read_volatile_ordered( &asm_ordering); - break; - - default: + } else { panic("Invalid timer"); } @@ -85,18 +75,13 @@ arm_vm_timer_is_irq_pending(arm_vm_timer_type_t tt) { CNT_CTL_t cnt_ctl; - switch (tt) { - case ARM_VM_TIMER_TYPE_VIRTUAL: + if (tt == ARM_VM_TIMER_TYPE_VIRTUAL) { cnt_ctl = register_CNTV_CTL_EL0_read_volatile_ordered( &asm_ordering); - break; - - case ARM_VM_TIMER_TYPE_PHYSICAL: + } else if (tt == ARM_VM_TIMER_TYPE_PHYSICAL) { cnt_ctl = register_CNTP_CTL_EL0_read_volatile_ordered( &asm_ordering); - break; - - default: + } else { panic("Invalid timer"); } @@ -112,16 +97,11 @@ arm_vm_timer_cancel_timeout(arm_vm_timer_type_t tt) CNT_CTL_init(&cnt_ctl); CNT_CTL_set_ENABLE(&cnt_ctl, false); - switch (tt) { - case ARM_VM_TIMER_TYPE_VIRTUAL: + if (tt == ARM_VM_TIMER_TYPE_VIRTUAL) { register_CNTV_CTL_EL0_write_ordered(cnt_ctl, &asm_ordering); - break; - - case ARM_VM_TIMER_TYPE_PHYSICAL: + } else if (tt == ARM_VM_TIMER_TYPE_PHYSICAL) { register_CNTP_CTL_EL0_write_ordered(cnt_ctl, &asm_ordering); - break; - - default: + } else { panic("Invalid timer"); } } @@ -131,18 +111,13 @@ arm_vm_timer_get_is_expired(arm_vm_timer_type_t tt) { CNT_CTL_t cnt_ctl; - switch (tt) { - case ARM_VM_TIMER_TYPE_VIRTUAL: + if (tt == ARM_VM_TIMER_TYPE_VIRTUAL) { cnt_ctl = register_CNTV_CTL_EL0_read_volatile_ordered( &asm_ordering); - break; - - case ARM_VM_TIMER_TYPE_PHYSICAL: + } else if (tt == ARM_VM_TIMER_TYPE_PHYSICAL) { cnt_ctl = register_CNTP_CTL_EL0_read_volatile_ordered( &asm_ordering); - break; - - default: + } else { panic("Invalid timer"); } @@ -151,7 +126,7 @@ arm_vm_timer_get_is_expired(arm_vm_timer_type_t tt) } uint32_t -arm_vm_timer_get_freqeuncy() +arm_vm_timer_get_freqeuncy(void) { CNTFRQ_EL0_t cntfrq = register_CNTFRQ_EL0_read(); @@ -159,7 +134,7 @@ arm_vm_timer_get_freqeuncy() } uint64_t -arm_vm_timer_get_ticks() +arm_vm_timer_get_ticks(void) { // This register read below is allowed to occur speculatively at any // time after the most recent context sync event. If caller the wants @@ -176,16 +151,11 @@ arm_vm_timer_get_timeout(arm_vm_timer_type_t tt) { CNT_CVAL_t cnt_cval; - switch (tt) { - case ARM_VM_TIMER_TYPE_VIRTUAL: + if (tt == ARM_VM_TIMER_TYPE_VIRTUAL) { cnt_cval = register_CNTV_CVAL_EL0_read_volatile(); - break; - - case ARM_VM_TIMER_TYPE_PHYSICAL: + } else if (tt == ARM_VM_TIMER_TYPE_PHYSICAL) { cnt_cval = register_CNTP_CVAL_EL0_read_volatile(); - break; - - default: + } else { panic("Invalid timer"); } @@ -195,8 +165,8 @@ arm_vm_timer_get_timeout(arm_vm_timer_type_t tt) void arm_vm_timer_arch_timer_hw_irq_activated(arm_vm_timer_type_t tt) { - if ((tt <= ENUM_ARM_VM_TIMER_TYPE_MAX_VALUE) && - (tt >= ENUM_ARM_VM_TIMER_TYPE_MIN_VALUE)) { + if ((tt == ARM_VM_TIMER_TYPE_PHYSICAL) || + (tt == ARM_VM_TIMER_TYPE_VIRTUAL)) { CPULOCAL(arm_vm_timer_irq_active)[tt] = true; } else { panic("Invalid timer"); @@ -206,14 +176,14 @@ arm_vm_timer_arch_timer_hw_irq_activated(arm_vm_timer_type_t tt) void arm_vm_timer_arch_timer_hw_irq_deactivate(arm_vm_timer_type_t tt) { - if ((tt <= ENUM_ARM_VM_TIMER_TYPE_MAX_VALUE) && - (tt >= ENUM_ARM_VM_TIMER_TYPE_MIN_VALUE)) { + if ((tt == ARM_VM_TIMER_TYPE_PHYSICAL) || + (tt == ARM_VM_TIMER_TYPE_VIRTUAL)) { if (CPULOCAL(arm_vm_timer_irq_active)[tt]) { CPULOCAL(arm_vm_timer_irq_active)[tt] = false; irq_deactivate(arm_vm_timer_hwirq[tt]); } } else { - panic("Invalis timer"); + panic("Invalid timer"); } } @@ -227,7 +197,7 @@ arm_vm_timer_handle_boot_cpu_cold_init(void) } void -arm_vm_timer_handle_boot_hypervisor_start() +arm_vm_timer_handle_boot_hypervisor_start(void) { hwirq_ptr_result_t ret; hwirq_create_t params[] = { @@ -270,7 +240,7 @@ arm_vm_timer_handle_boot_cpu_warm_init(void) arm_vm_timer_init(ARM_VM_TIMER_TYPE_VIRTUAL); arm_vm_timer_init(ARM_VM_TIMER_TYPE_PHYSICAL); -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) CNTHCTL_EL2_E2H1_t cnthctl; CNTHCTL_EL2_E2H1_init(&cnthctl); @@ -288,6 +258,14 @@ arm_vm_timer_handle_boot_cpu_warm_init(void) CNTHCTL_EL2_E2H1_set_EL0VCTEN(&cnthctl, true); CNTHCTL_EL2_E2H1_set_EL0PCTEN(&cnthctl, true); +#if defined(ARCH_ARM_FEAT_ECV) + // Explicitly disable the ECV feature and the access traps for the + // virtual timer and counter registers. + CNTHCTL_EL2_E2H1_set_ECV(&cnthctl, false); + CNTHCTL_EL2_E2H1_set_EL1TVT(&cnthctl, false); + CNTHCTL_EL2_E2H1_set_EL1TVCT(&cnthctl, false); +#endif + register_CNTHCTL_EL2_E2H1_write(cnthctl); #else CNTHCTL_EL2_E2H0_t cnthctl; @@ -305,6 +283,14 @@ arm_vm_timer_handle_boot_cpu_warm_init(void) CNTHCTL_EL2_E2H0_set_EVNTDIR(&cnthctl, false); CNTHCTL_EL2_E2H0_set_EVNTEN(&cnthctl, false); +#if defined(ARCH_ARM_FEAT_ECV) + // Explicitly disable the ECV feature and the access traps for the + // virtual timer and counter registers. + CNTHCTL_EL2_E2H0_set_ECV(&cnthctl, false); + CNTHCTL_EL2_E2H0_set_EL1TVT(&cnthctl, false); + CNTHCTL_EL2_E2H0_set_EL1TVCT(&cnthctl, false); +#endif + register_CNTHCTL_EL2_E2H0_write(cnthctl); #endif @@ -315,8 +301,10 @@ arm_vm_timer_handle_boot_cpu_warm_init(void) CNTPCT_EL0_raw(register_CNTPCT_EL0_read_volatile_ordered( &asm_ordering)), CNT_CTL_raw(register_CNTV_CTL_EL0_read_ordered(&asm_ordering)), - CPULOCAL(arm_vm_timer_irq_active)[ARM_VM_TIMER_TYPE_VIRTUAL], - CPULOCAL(arm_vm_timer_irq_active)[ARM_VM_TIMER_TYPE_PHYSICAL]); + (register_t)CPULOCAL( + arm_vm_timer_irq_active)[ARM_VM_TIMER_TYPE_VIRTUAL], + (register_t)CPULOCAL( + arm_vm_timer_irq_active)[ARM_VM_TIMER_TYPE_PHYSICAL]); #endif register_CNTVOFF_EL2_write(CNTVOFF_EL2_cast(0U)); @@ -334,16 +322,11 @@ arm_vm_timer_is_irq_enabled_thread(thread_t *thread, arm_vm_timer_type_t tt) { CNT_CTL_t cnt_ctl; - switch (tt) { - case ARM_VM_TIMER_TYPE_VIRTUAL: + if (tt == ARM_VM_TIMER_TYPE_VIRTUAL) { cnt_ctl = thread->vcpu_regs_el1.cntv_ctl_el0; - break; - - case ARM_VM_TIMER_TYPE_PHYSICAL: + } else if (tt == ARM_VM_TIMER_TYPE_PHYSICAL) { cnt_ctl = thread->vcpu_regs_el1.cntp_ctl_el0; - break; - - default: + } else { panic("Invalid timer"); } @@ -355,16 +338,11 @@ arm_vm_timer_get_timeout_thread(thread_t *thread, arm_vm_timer_type_t tt) { CNT_CVAL_t cnt_cval; - switch (tt) { - case ARM_VM_TIMER_TYPE_VIRTUAL: + if (tt == ARM_VM_TIMER_TYPE_VIRTUAL) { cnt_cval = thread->vcpu_regs_el1.cntv_cval_el0; - break; - - case ARM_VM_TIMER_TYPE_PHYSICAL: - cnt_cval = thread->vcpu_regs_el1.cntv_cval_el0; - break; - - default: + } else if (tt == ARM_VM_TIMER_TYPE_PHYSICAL) { + cnt_cval = thread->vcpu_regs_el1.cntp_cval_el0; + } else { panic("Invalid timer"); } @@ -391,7 +369,7 @@ arm_vm_timer_handle_thread_save_state(void) { thread_t *thread = thread_get_self(); - if (thread->kind == THREAD_KIND_VCPU && + if ((compiler_expected(thread->kind == THREAD_KIND_VCPU)) && !scheduler_is_blocked(thread, SCHEDULER_BLOCK_VCPU_OFF)) { thread->vcpu_regs_el1.cntkctl_el1 = register_CNTKCTL_EL1_read(); thread->vcpu_regs_el1.cntv_ctl_el0 = diff --git a/hyp/vm/arm_vm_timer/arm_vm_timer.ev b/hyp/vm/arm_vm_timer/arm_vm_timer.ev index 4e6f66d..f76ddc3 100644 --- a/hyp/vm/arm_vm_timer/arm_vm_timer.ev +++ b/hyp/vm/arm_vm_timer/arm_vm_timer.ev @@ -14,10 +14,13 @@ subscribe boot_cpu_warm_init require_preempt_disabled subscribe thread_save_state() + require_preempt_disabled subscribe thread_context_switch_pre() + require_preempt_disabled subscribe thread_context_switch_post() + require_preempt_disabled subscribe object_create_thread @@ -34,14 +37,18 @@ subscribe timer_action[TIMER_ACTION_PHYSICAL_TIMER] subscribe virq_check_pending[VIRQ_TRIGGER_VIRTUAL_TIMER] handler arm_vm_timer_handle_virq_check_pending(trigger, source) + require_preempt_disabled subscribe virq_check_pending[VIRQ_TRIGGER_PHYSICAL_TIMER] handler arm_vm_timer_handle_virq_check_pending(trigger, source) + require_preempt_disabled subscribe irq_received[HWIRQ_ACTION_VM_TIMER](irq) + require_preempt_disabled -subscribe vcpu_poweroff() +subscribe vcpu_stopped() subscribe power_cpu_suspend() + require_preempt_disabled subscribe vcpu_suspend() diff --git a/hyp/vm/arm_vm_timer/include/arm_vm_timer.h b/hyp/vm/arm_vm_timer/include/arm_vm_timer.h index 1c090e1..48d07a7 100644 --- a/hyp/vm/arm_vm_timer/include/arm_vm_timer.h +++ b/hyp/vm/arm_vm_timer/include/arm_vm_timer.h @@ -38,7 +38,9 @@ ticks_t arm_vm_timer_get_timeout_thread(thread_t *thread, arm_vm_timer_type_t tt); void -arm_vm_timer_arch_timer_hw_irq_activated(arm_vm_timer_type_t tt); +arm_vm_timer_arch_timer_hw_irq_activated(arm_vm_timer_type_t tt) + REQUIRE_PREEMPT_DISABLED; void -arm_vm_timer_arch_timer_hw_irq_deactivate(arm_vm_timer_type_t tt); +arm_vm_timer_arch_timer_hw_irq_deactivate(arm_vm_timer_type_t tt) + REQUIRE_PREEMPT_DISABLED; diff --git a/hyp/vm/arm_vm_timer/src/arm_vm_timer_irq.c b/hyp/vm/arm_vm_timer/src/arm_vm_timer_irq.c index b586eba..39b70ea 100644 --- a/hyp/vm/arm_vm_timer/src/arm_vm_timer_irq.c +++ b/hyp/vm/arm_vm_timer/src/arm_vm_timer_irq.c @@ -20,16 +20,11 @@ static void arm_vm_timer_inject_timer_virq(thread_t *thread, arm_vm_timer_type_t tt) { - switch (tt) { - case ARM_VM_TIMER_TYPE_VIRTUAL: + if (tt == ARM_VM_TIMER_TYPE_VIRTUAL) { (void)virq_assert(&thread->virtual_timer_virq_src, false); - break; - - case ARM_VM_TIMER_TYPE_PHYSICAL: + } else if (tt == ARM_VM_TIMER_TYPE_PHYSICAL) { (void)virq_assert(&thread->physical_timer_virq_src, false); - break; - - default: + } else { panic("Invalid timer"); } } @@ -71,6 +66,7 @@ arm_vm_timer_handle_timer_action(timer_action_t action_type, timer_t *timer) // Handle the VM arch timer expiry static bool arm_vm_timer_type_irq_received(thread_t *thread, arm_vm_timer_type_t tt) + REQUIRE_PREEMPT_DISABLED { bool injected = false; @@ -106,6 +102,7 @@ arm_vm_timer_handle_irq_received(irq_t irq) static bool arm_vm_timer_virq_check_pending(thread_t *thread, arm_vm_timer_type_t tt) + REQUIRE_PREEMPT_DISABLED { bool ret = true; @@ -136,6 +133,7 @@ arm_vm_timer_handle_virq_check_pending(virq_trigger_t trigger, thread_container_of_physical_timer_virq_src(source), ARM_VM_TIMER_TYPE_PHYSICAL); } else { + /* Do Nothing */ } return ret; diff --git a/hyp/vm/arm_vm_timer/src/arm_vm_timer_thread.c b/hyp/vm/arm_vm_timer/src/arm_vm_timer_thread.c index cba689c..95025b7 100644 --- a/hyp/vm/arm_vm_timer/src/arm_vm_timer_thread.c +++ b/hyp/vm/arm_vm_timer/src/arm_vm_timer_thread.c @@ -7,6 +7,7 @@ #include +#include #include #include #include @@ -78,7 +79,8 @@ arm_vm_timer_handle_thread_context_switch_pre(void) // Enqueue thread's timeout if it is enabled, not already queued, and is // capable of waking the VCPU - if ((thread->kind == THREAD_KIND_VCPU) && vcpu_expects_wakeup(thread)) { + if ((compiler_expected(thread->kind == THREAD_KIND_VCPU) && + vcpu_expects_wakeup(thread))) { if (arm_vm_timer_is_irq_enabled_thread( thread, ARM_VM_TIMER_TYPE_VIRTUAL)) { timer_update(&thread->virtual_timer, @@ -103,7 +105,7 @@ void arm_vm_timer_handle_thread_context_switch_post(void) { thread_t *thread = thread_get_self(); - if (thread->kind == THREAD_KIND_VCPU) { + if (compiler_expected(thread->kind == THREAD_KIND_VCPU)) { arm_vm_timer_load_state(thread); bool_result_t asserted; @@ -126,8 +128,8 @@ arm_vm_timer_handle_thread_context_switch_post(void) } } -error_t -arm_vm_timer_handle_vcpu_poweroff(void) +void +arm_vm_timer_handle_vcpu_stopped(void) { thread_t *thread = thread_get_self(); @@ -139,8 +141,6 @@ arm_vm_timer_handle_vcpu_poweroff(void) // Ensure that the EL2 timer has not been lazily left queued. timer_dequeue(&thread->virtual_timer); timer_dequeue(&thread->physical_timer); - - return OK; } error_t diff --git a/hyp/vm/psci_pc/aarch64/psci.ev b/hyp/vm/psci/aarch64/psci.ev similarity index 94% rename from hyp/vm/psci_pc/aarch64/psci.ev rename to hyp/vm/psci/aarch64/psci.ev index c49d50e..981d6f4 100644 --- a/hyp/vm/psci_pc/aarch64/psci.ev +++ b/hyp/vm/psci/aarch64/psci.ev @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: BSD-3-Clause -module psci_pc +module psci subscribe vcpu_trap_wfi() diff --git a/hyp/vm/psci/aarch64/src/psci.c b/hyp/vm/psci/aarch64/src/psci.c new file mode 100644 index 0000000..0f71931 --- /dev/null +++ b/hyp/vm/psci/aarch64/src/psci.c @@ -0,0 +1,30 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include + +#include +#include + +#include + +#include "event_handlers.h" +#include "psci_arch.h" + +void +psci_handle_scheduler_selected_thread(thread_t *thread, bool *can_idle) +{ + if (thread->vpm_mode == VPM_MODE_IDLE) { + // This thread can't be allowed to disable the WFI trap, + // because WFI votes to suspend the physical CPU. + *can_idle = false; + } +} + +vcpu_trap_result_t +psci_handle_vcpu_trap_wfi(void) +{ + return psci_handle_trapped_idle() ? VCPU_TRAP_RESULT_EMULATED + : VCPU_TRAP_RESULT_UNHANDLED; +} diff --git a/hyp/vm/psci/build.conf b/hyp/vm/psci/build.conf new file mode 100644 index 0000000..28c98ae --- /dev/null +++ b/hyp/vm/psci/build.conf @@ -0,0 +1,12 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +base_module hyp/vm/vpm_base +interface psci +local_include +events psci.ev +types psci.tc +source psci_common.c psci_pm_list.c +arch_source aarch64 psci.c +arch_events aarch64 psci.ev diff --git a/hyp/vm/psci/include/psci_arch.h b/hyp/vm/psci/include/psci_arch.h new file mode 100644 index 0000000..ead07a1 --- /dev/null +++ b/hyp/vm/psci/include/psci_arch.h @@ -0,0 +1,6 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +bool +psci_handle_trapped_idle(void); diff --git a/hyp/vm/psci/include/psci_common.h b/hyp/vm/psci/include/psci_common.h new file mode 100644 index 0000000..c9c67de --- /dev/null +++ b/hyp/vm/psci/include/psci_common.h @@ -0,0 +1,53 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +// Called by psci_common + +error_t +psci_vcpu_suspend(thread_t *current) REQUIRE_PREEMPT_DISABLED; + +void +psci_vcpu_resume(thread_t *thread) REQUIRE_PREEMPT_DISABLED; + +void +psci_vcpu_clear_vcpu_state(thread_t *thread, cpu_index_t target_cpu) + REQUIRE_PREEMPT_DISABLED REQUIRE_SCHEDULER_LOCK(thread); + +uint32_t +psci_cpu_suspend_features(void); + +// Implemented by psci_common + +psci_ret_t +psci_suspend(psci_suspend_powerstate_t suspend_state, + paddr_t entry_point_address, register_t context_id) + EXCLUDE_PREEMPT_DISABLED; + +bool +psci_set_vpm_active_pcpus_bit(cpu_index_t bit); + +bool +psci_clear_vpm_active_pcpus_bit(cpu_index_t bit); + +void +psci_vpm_active_vcpus_get(cpu_index_t cpu, thread_t *vcpu) + REQUIRE_SCHEDULER_LOCK(vcpu); + +void +psci_vpm_active_vcpus_put(cpu_index_t cpu, thread_t *vcpu) + REQUIRE_SCHEDULER_LOCK(vcpu); + +bool +psci_vpm_active_vcpus_is_zero(cpu_index_t cpu); + +bool +vcpus_state_is_any_awake(vpm_group_suspend_state_t vm_state, uint32_t level, + cpu_index_t cpu); + +void +vcpus_state_set(vpm_group_suspend_state_t *vm_state, cpu_index_t cpu, + psci_cpu_state_t cpu_state); + +void +vcpus_state_clear(vpm_group_suspend_state_t *vm_state, cpu_index_t cpu); diff --git a/hyp/vm/psci/include/psci_events.h b/hyp/vm/psci/include/psci_events.h new file mode 100644 index 0000000..f6e1fa1 --- /dev/null +++ b/hyp/vm/psci/include/psci_events.h @@ -0,0 +1,43 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +// clang-format off + +#define PSCI_FUNCTION(fn, feat, h, ...) \ +subscribe smccc_call_fast_32_standard[(smccc_function_t)PSCI_FUNCTION_ ## fn]; \ + handler psci_ ## h ## _32(__VA_ARGS__); \ + exclude_preempt_disabled. \ +subscribe psci_features32[PSCI_FUNCTION_ ## fn]; \ + constant feat. \ +subscribe smccc_call_fast_64_standard[(smccc_function_t)PSCI_FUNCTION_ ## fn]; \ + handler psci_ ## h ## _64(__VA_ARGS__); \ + exclude_preempt_disabled. \ +subscribe psci_features64[PSCI_FUNCTION_ ## fn]; \ + constant feat. + +#define PSCI_FUNCTION32(fn, feat, h, ...) \ +subscribe smccc_call_fast_32_standard[(smccc_function_t)PSCI_FUNCTION_ ## fn]; \ + handler psci_ ## h(__VA_ARGS__); \ + exclude_preempt_disabled. \ +subscribe psci_features32[PSCI_FUNCTION_ ## fn]; \ + constant feat. + +#define PSCI_FUNCTION_PERVM(fn, h, ...) \ +subscribe smccc_call_fast_32_standard[(smccc_function_t)PSCI_FUNCTION_ ## fn]; \ + handler psci_ ## h ## _32(__VA_ARGS__); \ + exclude_preempt_disabled. \ +subscribe psci_features32[PSCI_FUNCTION_ ## fn]; \ + handler psci_ ## h ## _32_features(). \ +subscribe smccc_call_fast_64_standard[(smccc_function_t)PSCI_FUNCTION_ ## fn]; \ + handler psci_ ## h ## _64(__VA_ARGS__); \ + exclude_preempt_disabled. \ +subscribe psci_features64[PSCI_FUNCTION_ ## fn]; \ + handler psci_ ## h ## _64_features(). \ + +#define PSCI_FUNCTION32_PERVM(fn, h, ...) \ +subscribe smccc_call_fast_32_standard[(smccc_function_t)PSCI_FUNCTION_ ## fn]; \ + handler psci_ ## h(__VA_ARGS__); \ + exclude_preempt_disabled. \ +subscribe psci_features32[PSCI_FUNCTION_ ## fn]; \ + handler psci_ ## h ## _features(). diff --git a/hyp/vm/psci_pc/include/psci_pm_list.h b/hyp/vm/psci/include/psci_pm_list.h similarity index 90% rename from hyp/vm/psci_pc/include/psci_pm_list.h rename to hyp/vm/psci/include/psci_pm_list.h index ce6b5c1..0cb49a5 100644 --- a/hyp/vm/psci_pc/include/psci_pm_list.h +++ b/hyp/vm/psci/include/psci_pm_list.h @@ -9,7 +9,7 @@ psci_pm_list_init(void); // Get the current cpu list of vcpus that participate in power management // decisions list_t * -psci_pm_list_get_self(void); +psci_pm_list_get_self(void) REQUIRE_PREEMPT_DISABLED; // Add vcpu to specified cpu pm list void diff --git a/hyp/vm/psci/psci.ev b/hyp/vm/psci/psci.ev new file mode 100644 index 0000000..b95cc61 --- /dev/null +++ b/hyp/vm/psci/psci.ev @@ -0,0 +1,94 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module psci + +#include "psci_events.h" + +PSCI_FUNCTION32(PSCI_VERSION, 0U, version, ret0) +PSCI_FUNCTION_PERVM(CPU_SUSPEND, cpu_suspend, arg1, arg2, arg3, ret0) +PSCI_FUNCTION32(CPU_OFF, 0U, cpu_off, ret0) +PSCI_FUNCTION(CPU_ON, 0U, cpu_on, arg1, arg2, arg3, ret0) +PSCI_FUNCTION(AFFINITY_INFO, 0U, affinity_info, arg1, arg2, ret0) +//PSCI_FUNCTION32(MIGRATE, 0U, migrate, arg1, ret0) +//PSCI_FUNCTION32(MIGRATE_INFO_TYPE, 0U, migrate_info_type, ret0) +//PSCI_FUNCTION(MIGRATE_INFO_UP_CPU, 0U, migrate_info_up_cpu, ret0) +PSCI_FUNCTION32(SYSTEM_OFF, 0U, system_off) +PSCI_FUNCTION32(SYSTEM_RESET, 0U, system_reset) +PSCI_FUNCTION32(PSCI_FEATURES, 0U, features, arg1, ret0) +//PSCI_FUNCTION32(CPU_FREEZE, 0U, cpu_freeze, ret0) +PSCI_FUNCTION(CPU_DEFAULT_SUSPEND, 0U, cpu_default_suspend, arg1, arg2, ret0) +//PSCI_FUNCTION(NODE_HW_STATE, 0U, node_hw_state, arg1, arg2, ret0) +//PSCI_FUNCTION(SYSTEM_SUSPEND, 0U, system_suspend, arg1, arg2, ret0) +//PSCI_FUNCTION32(PSCI_SET_SUSPEND_MODE, 0U, set_suspend_mode, arg1, ret0) +//PSCI_FUNCTION(PSCI_STAT_RESIDENCY, 0U, stat_residency, arg1, arg2, ret0) +//PSCI_FUNCTION(PSCI_STAT_COUNT, 0U, stat_count, arg1, arg2, ret0) +PSCI_FUNCTION(SYSTEM_RESET2, 0U, system_reset2, arg1, arg2, ret0) +//PSCI_FUNCTION32(MEM_PROTECT, 0U, mem_protect, arg1, ret0) +//PSCI_FUNCTION(MEM_PROTECT_CHECK_RANGE, 0U, mem_protect_check_range, arg1, arg2, ret0) + +subscribe object_create_thread + +subscribe object_activate_thread + // Run early to ensure that MPIDR is set correctly, since other + // modules may rely on it (especially VGIC, which is priority 1) + priority 50 + +subscribe object_deactivate_thread + +subscribe object_deactivate_vpm_group + +subscribe vcpu_suspend + unwinder(current) + require_preempt_disabled + +subscribe vcpu_started + +subscribe vcpu_wakeup + require_scheduler_lock(vcpu) + +subscribe vcpu_wakeup_self + +subscribe vcpu_expects_wakeup + +#if defined(INTERFACE_VCPU_RUN) +subscribe vcpu_run_check(vcpu, state_data_0, state_data_1) +#endif + +subscribe vcpu_poweron + priority last + require_scheduler_lock(vcpu) + +subscribe vcpu_resume + require_preempt_disabled + +subscribe vcpu_poweroff + // First so it can deny poweroff without unwinding other modules. + priority first + require_scheduler_lock(current) + +subscribe vcpu_stopped + +subscribe vcpu_activate_thread(thread) + unwinder psci_handle_object_deactivate_thread(thread) + // Run after the scheduler handler + priority -100 + +subscribe boot_cold_init() + +subscribe boot_cpu_cold_init + +subscribe scheduler_affinity_changed(thread, prev_cpu, next_cpu, need_sync) + require_scheduler_lock(thread) + +subscribe scheduler_affinity_changed_sync(thread, next_cpu) + require_preempt_disabled + +subscribe task_queue_execute[TASK_QUEUE_CLASS_VPM_GROUP_VIRQ](entry) + +subscribe power_cpu_online() + require_preempt_disabled + +subscribe power_cpu_offline() + require_preempt_disabled diff --git a/hyp/vm/psci_pc/psci.tc b/hyp/vm/psci/psci.tc similarity index 68% rename from hyp/vm/psci_pc/psci.tc rename to hyp/vm/psci/psci.tc index b600b45..aadc3ce 100644 --- a/hyp/vm/psci_pc/psci.tc +++ b/hyp/vm/psci/psci.tc @@ -11,21 +11,30 @@ define PSCI_VERSION constant uint32 = PSCI_VER; // We need only 3 bits to encode the cpu level state of a vcpu define PSCI_VCPUS_STATE_PER_VCPU_BITS constant type count_t = 3; +define PSCI_PER_CLUSTER_STATE_BITS constant type count_t = 3; define PSCI_VCPUS_STATE_PER_VCPU_MASK constant uint8 = 0x7; -define PSCI_VCPUS_STATE_BITS constant type count_t = 32; +define PSCI_PER_CLUSTER_STATE_BITS_MASK constant uint8 = 0x7; +define PSCI_VCPUS_STATE_BITS constant type count_t = (PLATFORM_MAX_CORES * PSCI_VCPUS_STATE_PER_VCPU_BITS); +define PSCI_CLUSTER_STATE_BITS constant type count_t = (PLATFORM_MAX_CLUSTERS * PSCI_PER_CLUSTER_STATE_BITS); +define PSCI_SYSTEM_STATE_BITS constant type count_t = 8; +define PSCI_CLUSTER_STATE_SUSPEND_BITS constant type count_t = PLATFORM_MAX_CLUSTERS; define PSCI_VCPUS_STATE_MAX_VCPUS constant type count_t = PSCI_VCPUS_STATE_BITS/PSCI_VCPUS_STATE_PER_VCPU_BITS; define PSCI_VCPUS_STATE_MAX_INDEX constant type count_t = PLATFORM_MAX_CORES*PSCI_VCPUS_STATE_PER_VCPU_BITS; -define vpm_group_suspend_state bitfield<64> { - auto vcpus_state uint32; +define vpm_group_suspend_state bitfield<128> { + auto vcpus_state uint64; + auto cluster_state uint16; + auto system_state uint8; + auto is_cluster_suspended uint16; others unknown=0; }; +// FIXME: extend vpm_group object module psci { cpus array(PLATFORM_MAX_CORES) pointer(atomic) object thread; online_count type count_t(atomic); mode enumeration psci_mode; - vm_suspend_state bitfield vpm_group_suspend_state(atomic); + vm_suspend_state bitfield vpm_group_suspend_state(atomic, aligned(16)); system_suspend_virq structure virq_source(contained); virq_task structure task_queue_entry(contained); lock structure spinlock; @@ -36,6 +45,10 @@ extend task_queue_class enumeration { vpm_group_virq; }; +extend vpm_mode enumeration { + PSCI; +}; + extend thread object module psci { // Reference-counted virtual PM group pointer group pointer object vpm_group; @@ -52,30 +65,23 @@ extend thread object module psci { pm_list_node structure list_node(contained); - mode enumeration vpm_mode; - // True if the VCPU is being migrated between PM lists. migrate bool; +#if defined(INTERFACE_VCPU_RUN) + // True if the VCPU has called PSCI_SYSTEM_RESET. + system_reset bool; + // PSCI_SYSTEM_RESET2 parameters, returned in state data. + system_reset_type uint64; + system_reset_cookie uint64; +#endif + // Tracks when the VCPU becomes inactive due to being off, suspended // or migrating. When zero, the VCPU is considered active and will be // accounted for in vpm_active_vcpus. inactive_count type count_t; }; -#if defined(ROOTVM_IS_HLOS) && ROOTVM_IS_HLOS -extend boot_env_data structure module psci { - group type cap_id_t; - secondary_vcpus array(PLATFORM_MAX_CORES) type cap_id_t; -}; -#endif - -extend cap_rights_vpm_group bitfield { - 0 attach_vcpu bool; - 1 bind_virq bool; - 2 query bool; -}; - extend virq_trigger enumeration { vpm_group; }; @@ -85,7 +91,7 @@ extend trace_ids bitfield { }; extend trace_class enumeration { - PSCI; + PSCI = 16; }; extend trace_id enumeration { diff --git a/hyp/vm/psci/src/psci_common.c b/hyp/vm/psci/src/psci_common.c new file mode 100644 index 0000000..01b00c4 --- /dev/null +++ b/hyp/vm/psci/src/psci_common.c @@ -0,0 +1,1313 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "event_handlers.h" +#include "psci_arch.h" +#include "psci_common.h" +#include "psci_pm_list.h" + +CPULOCAL_DECLARE_STATIC(_Atomic count_t, vpm_active_vcpus); + +// vpm_active_pcpus_bitmap could be made a count to avoid a bitmap. +#define REGISTER_BITS (sizeof(register_t) * (size_t)CHAR_BIT) +static_assert(PLATFORM_MAX_CORES <= REGISTER_BITS, + "PLATFORM_MAX_CORES > REGISTER_BITS"); +static _Atomic register_t vpm_active_pcpus_bitmap; + +// Set to 1 to boot enable the PSCI tracepoints +#if defined(VERBOSE_TRACE) && VERBOSE_TRACE +#define DEBUG_PSCI_TRACES 1 +#else +#define DEBUG_PSCI_TRACES 0 +#endif + +void +psci_handle_boot_cold_init(void) +{ +#if !defined(NDEBUG) && DEBUG_PSCI_TRACES + register_t flags = 0U; + TRACE_SET_CLASS(flags, PSCI); + trace_set_class_flags(flags); +#endif + + psci_pm_list_init(); +} + +bool +psci_set_vpm_active_pcpus_bit(cpu_index_t bit) +{ + register_t old = atomic_fetch_or_explicit( + &vpm_active_pcpus_bitmap, util_bit(bit), memory_order_relaxed); + + return old == 0U; +} + +// Returns true if bitmap becomes zero after clearing bit +bool +psci_clear_vpm_active_pcpus_bit(cpu_index_t bit) +{ + register_t cleared_bit = ~util_bit(bit); + + register_t old = atomic_fetch_and_explicit( + &vpm_active_pcpus_bitmap, cleared_bit, memory_order_relaxed); + + return (old & cleared_bit) == 0U; +} + +void +psci_handle_boot_cpu_cold_init(cpu_index_t cpu) +{ + atomic_store_relaxed(&CPULOCAL_BY_INDEX(vpm_active_vcpus, cpu), 0U); + (void)psci_set_vpm_active_pcpus_bit(cpu); +} + +void +psci_vpm_active_vcpus_get(cpu_index_t cpu, thread_t *vcpu) +{ + assert(cpulocal_index_valid(cpu)); + assert(vcpu->psci_inactive_count != 0U); + + vcpu->psci_inactive_count--; + if (vcpu->psci_inactive_count == 0U) { + (void)atomic_fetch_add_explicit( + &CPULOCAL_BY_INDEX(vpm_active_vcpus, cpu), 1U, + memory_order_relaxed); + } +} + +void +psci_vpm_active_vcpus_put(cpu_index_t cpu, thread_t *vcpu) +{ + assert(cpulocal_index_valid(cpu)); + + vcpu->psci_inactive_count++; + if (vcpu->psci_inactive_count == 1U) { + count_t old = atomic_fetch_sub_explicit( + &CPULOCAL_BY_INDEX(vpm_active_vcpus, cpu), 1U, + memory_order_relaxed); + assert(old != 0U); + } +} + +bool +psci_vpm_active_vcpus_is_zero(cpu_index_t cpu) +{ + assert(cpulocal_index_valid(cpu)); + + return atomic_load_relaxed(&CPULOCAL_BY_INDEX(vpm_active_vcpus, cpu)) == + 0; +} + +bool +psci_handle_vcpu_activate_thread(thread_t *thread) +{ + bool ret = true; + + assert(thread != NULL); + assert(thread->kind == THREAD_KIND_VCPU); + + scheduler_lock(thread); + + // Determine the initial inactive count for the VCPU. + thread->psci_inactive_count = 0U; + + if (scheduler_is_blocked(thread, SCHEDULER_BLOCK_VCPU_OFF)) { + // VCPU is inactive because it is powered off. + thread->psci_inactive_count++; + } + // VCPU can't be suspended or in WFI yet. + assert(!scheduler_is_blocked(thread, SCHEDULER_BLOCK_VCPU_SUSPEND)); + assert(!scheduler_is_blocked(thread, SCHEDULER_BLOCK_VCPU_WFI)); + + cpu_index_t cpu = scheduler_get_affinity(thread); + if (cpulocal_index_valid(cpu)) { + if (thread->psci_group != NULL) { + psci_pm_list_insert(cpu, thread); + } + } else { + // VCPU is inactive because it has no valid affinity. + thread->psci_inactive_count++; + } + + // If the VCPU is initially active, make sure the CPU stays awake. + if (thread->psci_inactive_count == 0U) { + assert(cpulocal_index_valid(cpu)); + (void)atomic_fetch_add_explicit( + &CPULOCAL_BY_INDEX(vpm_active_vcpus, cpu), 1U, + memory_order_relaxed); + } + + scheduler_unlock(thread); + + return ret; +} + +void +psci_handle_scheduler_affinity_changed(thread_t *thread, cpu_index_t prev_cpu, + cpu_index_t next_cpu, bool *need_sync) +{ + object_state_t state = atomic_load_acquire(&thread->header.state); + + if ((state == OBJECT_STATE_ACTIVE) && + (thread->vpm_mode != VPM_MODE_NONE)) { + if (cpulocal_index_valid(prev_cpu)) { + if (thread->vpm_mode == VPM_MODE_PSCI) { + psci_pm_list_delete(prev_cpu, thread); + } + psci_vpm_active_vcpus_put(prev_cpu, thread); + } + + if (cpulocal_index_valid(next_cpu)) { + psci_vpm_active_vcpus_get(next_cpu, thread); + if (thread->vpm_mode == VPM_MODE_PSCI) { + thread->psci_migrate = true; + *need_sync = true; + } + } + } +} + +void +psci_handle_scheduler_affinity_changed_sync(thread_t *thread, + cpu_index_t next_cpu) +{ + if (thread->psci_migrate) { + assert(thread->kind == THREAD_KIND_VCPU); + assert(thread->vpm_mode == VPM_MODE_PSCI); + assert(cpulocal_index_valid(next_cpu)); + + psci_pm_list_insert(next_cpu, thread); + + thread->psci_migrate = false; + } +} + +static bool +psci_mpidr_matches_thread(MPIDR_EL1_t a, psci_mpidr_t b) +{ + return (MPIDR_EL1_get_Aff0(&a) == psci_mpidr_get_Aff0(&b)) && + (MPIDR_EL1_get_Aff1(&a) == psci_mpidr_get_Aff1(&b)) && + (MPIDR_EL1_get_Aff2(&a) == psci_mpidr_get_Aff2(&b)) && + (MPIDR_EL1_get_Aff3(&a) == psci_mpidr_get_Aff3(&b)); +} + +static MPIDR_EL1_t +psci_mpidr_to_cpu(psci_mpidr_t psci_mpidr) +{ + MPIDR_EL1_t mpidr = MPIDR_EL1_default(); + + MPIDR_EL1_set_Aff0(&mpidr, psci_mpidr_get_Aff0(&psci_mpidr)); + MPIDR_EL1_set_Aff1(&mpidr, psci_mpidr_get_Aff1(&psci_mpidr)); + MPIDR_EL1_set_Aff2(&mpidr, psci_mpidr_get_Aff2(&psci_mpidr)); + MPIDR_EL1_set_Aff3(&mpidr, psci_mpidr_get_Aff3(&psci_mpidr)); + + return mpidr; +} + +static thread_t * +psci_get_thread_by_mpidr(psci_mpidr_t mpidr) +{ + thread_t *current = thread_get_self(); + thread_t *result = NULL; + vpm_group_t *psci_group = current->psci_group; + + assert(psci_group != NULL); + + // This function is not performance-critical; it is only called during + // PSCI_CPU_ON and PSCI_AFFINITY_INFO. A simple linear search of the VPM + // group is good enough. + rcu_read_start(); + for (index_t i = 0U; i < util_array_size(psci_group->psci_cpus); i++) { + thread_t *thread = + atomic_load_consume(&psci_group->psci_cpus[i]); + if ((thread != NULL) && + psci_mpidr_matches_thread(thread->vcpu_regs_mpidr_el1, + mpidr) && + object_get_thread_safe(thread)) { + result = thread; + break; + } + } + rcu_read_finish(); + + return result; +} + +bool +psci_version(uint32_t *ret0) +{ + bool handled; + thread_t *current = thread_get_self(); + + if (compiler_unexpected(current->psci_group == NULL)) { + handled = false; + } else { + *ret0 = PSCI_VERSION; + handled = true; + } + return handled; +} + +psci_ret_t +psci_suspend(psci_suspend_powerstate_t suspend_state, + paddr_t entry_point_address, register_t context_id) +{ + psci_ret_t ret; + thread_t *current = thread_get_self(); + + current->psci_suspend_state = suspend_state; + + error_t err = vcpu_suspend(); + if (err == ERROR_DENIED) { + TRACE(PSCI, PSCI_PSTATE_VALIDATION, + "psci_suspend: DENIED - pstate {:#x} - VM {:d}", + psci_suspend_powerstate_raw(suspend_state), + current->addrspace->vmid); + ret = PSCI_RET_DENIED; + goto out; + } else if (err == ERROR_ARGUMENT_INVALID) { + TRACE(PSCI, PSCI_PSTATE_VALIDATION, + "psci suspend: INVALID_PARAMETERS - pstate {:#x} - VM {:d}", + psci_suspend_powerstate_raw(suspend_state), + current->addrspace->vmid); + ret = PSCI_RET_INVALID_PARAMETERS; + goto out; + } else if (err == ERROR_BUSY) { + // It did not suspend due to a pending interrupt + ret = PSCI_RET_SUCCESS; + goto out; + } else if (err == OK) { + ret = PSCI_RET_SUCCESS; + } else { + panic("unhandled vcpu_suspend error"); + } + + // Warm reset VCPU unconditionally from the psci mode to make the + // cpuidle stats work + if ((psci_suspend_powerstate_get_StateType(&suspend_state) == + PSCI_SUSPEND_POWERSTATE_TYPE_POWERDOWN)) { + vcpu_warm_reset(entry_point_address, context_id); + } + +out: + return ret; +} + +static psci_ret_t +psci_cpu_suspend(psci_suspend_powerstate_t suspend_state, + paddr_t entry_point_address, register_t context_id) + EXCLUDE_PREEMPT_DISABLED +{ + psci_ret_t ret; + thread_t *current = thread_get_self(); + + // If the VCPU is participating in aggregation, we need to check with + // platform code that the requested state is valid. Otherwise, all + // requested states are accepted and treated equally. + if (current->vpm_mode == VPM_MODE_PSCI) { + assert(current->psci_group != NULL); + + // FIXME: + cpulocal_begin(); + ret = platform_psci_suspend_state_validation( + suspend_state, cpulocal_get_index(), + current->psci_group->psci_mode); + cpulocal_end(); + if (ret != PSCI_RET_SUCCESS) { + TRACE(PSCI, PSCI_PSTATE_VALIDATION, + "psci_cpu_suspend: INVALID_PARAMETERS - pstate {:#x} - VM {:d}", + psci_suspend_powerstate_raw(suspend_state), + current->addrspace->vmid); + goto out; + } + } + + ret = psci_suspend(suspend_state, entry_point_address, context_id); + +out: + return ret; +} + +uint32_t +psci_cpu_suspend_32_features(void) +{ + return psci_cpu_suspend_features(); +} + +uint32_t +psci_cpu_suspend_64_features(void) +{ + return psci_cpu_suspend_features(); +} + +bool +psci_cpu_suspend_32(uint32_t arg1, uint32_t arg2, uint32_t arg3, uint32_t *ret0) +{ + bool handled; + thread_t *current = thread_get_self(); + + if (compiler_unexpected(current->psci_group == NULL)) { + handled = false; + } else { + psci_ret_t ret = psci_cpu_suspend( + psci_suspend_powerstate_cast(arg1), arg2, arg3); + *ret0 = (uint32_t)ret; + handled = true; + } + return handled; +} + +bool +psci_cpu_suspend_64(uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t *ret0) +{ + bool handled; + thread_t *current = thread_get_self(); + + if (compiler_unexpected(current->psci_group == NULL)) { + handled = false; + } else { + psci_ret_t ret = psci_cpu_suspend( + psci_suspend_powerstate_cast((uint32_t)arg1), arg2, + arg3); + *ret0 = (uint64_t)ret; + handled = true; + } + return handled; +} + +// Same as psci_cpu_suspend, but it sets the suspend state to the deepest +// cpu-level. +static psci_ret_t +psci_cpu_default_suspend(paddr_t entry_point_address, register_t context_id) + EXCLUDE_PREEMPT_DISABLED +{ + psci_ret_t ret; + + psci_suspend_powerstate_t pstate = psci_suspend_powerstate_default(); + + // FIXME: + cpulocal_begin(); + psci_suspend_powerstate_stateid_t stateid = + platform_psci_deepest_cpu_level_stateid(cpulocal_get_index()); + cpulocal_end(); + + psci_suspend_powerstate_set_StateID(&pstate, stateid); + psci_suspend_powerstate_set_StateType( + &pstate, PSCI_SUSPEND_POWERSTATE_TYPE_POWERDOWN); + + ret = psci_suspend(pstate, entry_point_address, context_id); + + return ret; +} + +bool +psci_cpu_default_suspend_32(uint32_t arg1, uint32_t arg2, uint32_t *ret0) +{ + bool handled; + thread_t *current = thread_get_self(); + + if (compiler_unexpected(current->psci_group == NULL)) { + handled = false; + } else { + psci_ret_t ret = psci_cpu_default_suspend(arg1, arg2); + *ret0 = (uint32_t)ret; + handled = true; + } + return handled; +} + +bool +psci_cpu_default_suspend_64(uint64_t arg1, uint64_t arg2, uint64_t *ret0) +{ + bool handled; + thread_t *current = thread_get_self(); + + if (compiler_unexpected(current->psci_group == NULL)) { + handled = false; + } else { + psci_ret_t ret = psci_cpu_default_suspend(arg1, arg2); + *ret0 = (uint64_t)ret; + handled = true; + } + return handled; +} + +bool +psci_cpu_off(uint32_t *ret0) +{ + bool handled; + thread_t *current = thread_get_self(); + + if (compiler_unexpected(current->psci_group == NULL)) { + handled = false; + } else { + error_t ret = vcpu_poweroff(false, false); + // If we return, the only reason should be DENIED + assert(ret == ERROR_DENIED); + *ret0 = (uint32_t)PSCI_RET_DENIED; + handled = true; + } + return handled; +} + +static psci_ret_t +psci_cpu_on(psci_mpidr_t cpu, paddr_t entry_point_address, + register_t context_id) +{ + thread_t *thread = psci_get_thread_by_mpidr(cpu); + psci_ret_t ret; + + if (compiler_unexpected(thread == NULL)) { + thread_t *current = thread_get_self(); + vic_t *vic = vic_get_vic(current); + if (vic == NULL) { + ret = PSCI_RET_INVALID_PARAMETERS; + } else { + // Check whether MPIDR was valid or not. Note, we + // currently use PLATFORM_MAX_CORES instead of a per + // psci group + MPIDR_EL1_t mpidr = psci_mpidr_to_cpu(cpu); + + const platform_mpidr_mapping_t *mpidr_mapping = + vgic_get_mpidr_mapping(vic); + + bool valid = platform_cpu_map_mpidr_valid(mpidr_mapping, + mpidr); + + index_t index = platform_cpu_map_mpidr_to_index( + mpidr_mapping, mpidr); + + if (!valid || (index > PLATFORM_MAX_CORES)) { + ret = PSCI_RET_INVALID_PARAMETERS; + } else { + ret = PSCI_RET_INTERNAL_FAILURE; + } + } + } else { + bool reschedule = false; + + scheduler_lock(thread); + if (vcpu_option_flags_get_pinned(&thread->vcpu_options) && + !platform_cpu_exists(thread->scheduler_affinity)) { + ret = PSCI_RET_INTERNAL_FAILURE; + goto unlock; + } + + if (scheduler_is_blocked(thread, SCHEDULER_BLOCK_VCPU_OFF)) { + bool_result_t result = vcpu_poweron( + thread, vmaddr_result_ok(entry_point_address), + register_result_ok(context_id)); + reschedule = result.r; + ret = (result.e == OK) ? PSCI_RET_SUCCESS + : (result.e == ERROR_FAILURE) + ? PSCI_RET_INTERNAL_FAILURE + : (result.e == ERROR_RETRY) + ? PSCI_RET_ALREADY_ON + : PSCI_RET_INVALID_PARAMETERS; + } else { + ret = PSCI_RET_ALREADY_ON; + } + unlock: + scheduler_unlock(thread); + object_put_thread(thread); + + if (reschedule) { + (void)scheduler_schedule(); + } + } + + return ret; +} + +bool +psci_cpu_on_32(uint32_t arg1, uint32_t arg2, uint32_t arg3, uint32_t *ret0) +{ + bool handled; + thread_t *current = thread_get_self(); + + if (compiler_unexpected(current->psci_group == NULL)) { + handled = false; + } else { + psci_ret_t ret = psci_cpu_on(psci_mpidr_cast(arg1), arg2, arg3); + *ret0 = (uint32_t)ret; + handled = true; + } + return handled; +} + +bool +psci_cpu_on_64(uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t *ret0) +{ + bool handled; + thread_t *current = thread_get_self(); + + if (compiler_unexpected(current->psci_group == NULL)) { + handled = false; + } else { + psci_ret_t ret = psci_cpu_on(psci_mpidr_cast(arg1), arg2, arg3); + *ret0 = (uint64_t)ret; + handled = true; + } + return handled; +} + +static psci_ret_t +psci_affinity_info(psci_mpidr_t affinity, uint32_t lowest_affinity_level) +{ + psci_ret_t ret; + + thread_t *thread = psci_get_thread_by_mpidr(affinity); + if (thread == NULL) { + ret = PSCI_RET_INVALID_PARAMETERS; + } else if (lowest_affinity_level != 0U) { + // lowest_affinity_level is legacy from PSCI 0.2; we are + // allowed to fail if it is nonzero (which indicates a + // query of the cluster-level state). + ret = PSCI_RET_INVALID_PARAMETERS; + } else { + // Don't bother locking, this is inherently racy anyway + psci_ret_affinity_info_t info = + scheduler_is_blocked(thread, SCHEDULER_BLOCK_VCPU_OFF) + ? PSCI_RET_AFFINITY_INFO_OFF + : PSCI_RET_AFFINITY_INFO_ON; + ret = (psci_ret_t)info; + } + + if (thread != NULL) { + object_put_thread(thread); + } + + return ret; +} + +bool +psci_affinity_info_32(uint32_t arg1, uint32_t arg2, uint32_t *ret0) +{ + bool handled; + thread_t *current = thread_get_self(); + + if (compiler_unexpected(current->psci_group == NULL)) { + handled = false; + } else { + psci_ret_t ret = + psci_affinity_info(psci_mpidr_cast(arg1), arg2); + *ret0 = (uint32_t)ret; + handled = true; + } + return handled; +} + +bool +psci_affinity_info_64(uint64_t arg1, uint64_t arg2, uint64_t *ret0) +{ + bool handled; + thread_t *current = thread_get_self(); + + if (compiler_unexpected(current->psci_group == NULL)) { + handled = false; + } else { + psci_ret_t ret = psci_affinity_info(psci_mpidr_cast(arg1), + (uint32_t)arg2); + *ret0 = (uint64_t)ret; + handled = true; + } + return handled; +} + +static noreturn void +psci_stop_all_vcpus(void) +{ + thread_t *current = thread_get_self(); + assert(current != NULL); + assert(current->kind == THREAD_KIND_VCPU); + + vpm_group_t *psci_group = current->psci_group; + if (psci_group != NULL) { + for (index_t i = 0U; i < util_array_size(psci_group->psci_cpus); + i++) { + thread_t *thread = + atomic_load_consume(&psci_group->psci_cpus[i]); + if ((thread != NULL) && (thread != current)) { + error_t err = thread_kill(thread); + if (err != OK) { + panic("Unable to kill VCPU"); + } + } + } + } + + // Force power off + (void)vcpu_poweroff(false, true); + // We should not be denied when force is true + panic("vcpu_poweroff(force=true) returned"); +} + +bool +psci_system_off(void) +{ + bool handled; + thread_t *current = thread_get_self(); + + if (compiler_unexpected(current->psci_group == NULL)) { + handled = false; + } else { + if (vcpu_option_flags_get_critical(¤t->vcpu_options)) { + // HLOS VM calls to this function are passed directly to + // the firmware, to power off the physical device. + trigger_power_system_off_event(); + panic("system_off event returned"); + } + + psci_stop_all_vcpus(); + } + + return handled; +} + +bool +psci_system_reset(void) +{ + bool handled; + thread_t *current = thread_get_self(); + + if (compiler_unexpected(current->psci_group == NULL)) { + handled = false; + } else { + if (vcpu_option_flags_get_critical(¤t->vcpu_options)) { + // HLOS VM calls to this function are passed directly to + // the firmware, to reset the physical device. + error_t ret = OK; + (void)trigger_power_system_reset_event( + PSCI_REQUEST_SYSTEM_RESET, 0U, &ret); + panic("system_reset event returned"); + } + +#if defined(INTERFACE_VCPU_RUN) + // Tell the proxy thread that a reset was requested + thread_get_self()->psci_system_reset = true; + thread_get_self()->psci_system_reset_type = + PSCI_REQUEST_SYSTEM_RESET; + thread_get_self()->psci_system_reset_cookie = 0U; +#endif + + psci_stop_all_vcpus(); + } + + return handled; +} + +static uint32_t +psci_system_reset2(uint64_t reset_type, uint64_t cookie) +{ + uint32_t ret; + thread_t *current = thread_get_self(); + + if (vcpu_option_flags_get_critical(¤t->vcpu_options)) { + // HLOS VM calls to this function are passed directly to the + // firmware, to reset the physical device. + error_t error = OK; + (void)trigger_power_system_reset_event(reset_type, cookie, + &error); + + if (error == ERROR_ARGUMENT_INVALID) { + ret = (uint32_t)PSCI_RET_INVALID_PARAMETERS; + } else { + ret = (uint32_t)PSCI_RET_NOT_SUPPORTED; + } + } else { +#if defined(INTERFACE_VCPU_RUN) + // Tell the proxy thread that a reset was requested + thread_get_self()->psci_system_reset = true; + thread_get_self()->psci_system_reset_type = reset_type; + thread_get_self()->psci_system_reset_cookie = cookie; +#endif + + psci_stop_all_vcpus(); + } + + return ret; +} + +bool +psci_system_reset2_32(uint32_t arg1, uint32_t arg2, uint32_t *ret0) +{ + bool handled; + thread_t *current = thread_get_self(); + + if (compiler_unexpected(current->psci_group == NULL)) { + handled = false; + } else { + *ret0 = psci_system_reset2(arg1, arg2); + handled = true; + } + + return handled; +} + +bool +psci_system_reset2_64(uint64_t arg1, uint64_t arg2, uint64_t *ret0) +{ + bool handled; + thread_t *current = thread_get_self(); + + if (compiler_unexpected(current->psci_group == NULL)) { + handled = false; + } else { + *ret0 = psci_system_reset2( + (uint32_t)arg1 | PSCI_REQUEST_SYSTEM_RESET2_64, arg2); + handled = true; + } + + return handled; +} + +bool +psci_features(uint32_t arg1, uint32_t *ret0) +{ + thread_t *current = thread_get_self(); + bool has_psci = current->psci_group != NULL; + + smccc_function_id_t fn_id = smccc_function_id_cast(arg1); + uint32_t ret = SMCCC_UNKNOWN_FUNCTION32; + smccc_function_t fn = smccc_function_id_get_function(&fn_id); + + if (has_psci && + (smccc_function_id_get_interface_id(&fn_id) == + SMCCC_INTERFACE_ID_STANDARD) && + smccc_function_id_get_is_fast(&fn_id) && + (smccc_function_id_get_res0(&fn_id) == 0U)) { + ret = smccc_function_id_get_is_smc64(&fn_id) + ? trigger_psci_features64_event( + (psci_function_t)fn) + : trigger_psci_features32_event( + (psci_function_t)fn); + } else if ((smccc_function_id_get_interface_id(&fn_id) == + SMCCC_INTERFACE_ID_ARCH) && + smccc_function_id_get_is_fast(&fn_id) && + !smccc_function_id_get_is_smc64(&fn_id) && + (smccc_function_id_get_res0(&fn_id) == 0U) && + (fn == (smccc_function_t)SMCCC_ARCH_FUNCTION_VERSION)) { + // SMCCC>=1.1 is implemented and SMCCC_VERSION is safe to call. + ret = (uint32_t)PSCI_RET_SUCCESS; + } else { + /* Do Nothing */ + } + + *ret0 = ret; + return true; +} + +error_t +psci_handle_object_create_thread(thread_create_t thread_create) +{ + thread_t *thread = thread_create.thread; + assert(thread != NULL); + + // FIXME: + psci_suspend_powerstate_t pstate = psci_suspend_powerstate_default(); +#if !defined(PSCI_AFFINITY_LEVELS_NOT_SUPPORTED) || \ + !PSCI_AFFINITY_LEVELS_NOT_SUPPORTED + psci_suspend_powerstate_stateid_t stateid = + platform_psci_deepest_cluster_level_stateid( + thread->scheduler_affinity); +#else + psci_suspend_powerstate_stateid_t stateid = + platform_psci_deepest_cpu_level_stateid( + thread->scheduler_affinity); +#endif + psci_suspend_powerstate_set_StateID(&pstate, stateid); + psci_suspend_powerstate_set_StateType( + &pstate, PSCI_SUSPEND_POWERSTATE_TYPE_POWERDOWN); + + // Initialize to deepest possible state + thread->psci_suspend_state = pstate; + + return OK; +} + +error_t +psci_handle_object_activate_thread(thread_t *thread) +{ + error_t err; + if (thread->kind != THREAD_KIND_VCPU) { + thread->vpm_mode = VPM_MODE_NONE; + err = OK; + } else if (thread->psci_group == NULL) { + thread->vpm_mode = VPM_MODE_IDLE; + err = OK; + } else { + assert(scheduler_is_blocked(thread, SCHEDULER_BLOCK_VCPU_OFF)); + + thread->vpm_mode = vpm_group_option_flags_get_no_aggregation( + &thread->psci_group->options) + ? VPM_MODE_NONE + : VPM_MODE_PSCI; + cpu_index_t index = thread->psci_index; + thread_t *tmp_null = NULL; + + if (!cpulocal_index_valid(index)) { + err = ERROR_OBJECT_CONFIG; + } else if (!atomic_compare_exchange_strong_explicit( + &thread->psci_group->psci_cpus[index], + &tmp_null, thread, memory_order_release, + memory_order_relaxed)) { + err = ERROR_DENIED; + } else { + err = OK; + } + } + return err; +} + +void +psci_handle_object_deactivate_thread(thread_t *thread) +{ + assert(thread != NULL); + + if (thread->psci_group != NULL) { + thread_t *tmp = thread; + cpu_index_t index = thread->psci_index; + + (void)atomic_compare_exchange_strong_explicit( + &thread->psci_group->psci_cpus[index], &tmp, NULL, + memory_order_relaxed, memory_order_relaxed); + object_put_vpm_group(thread->psci_group); + } + + if (thread->vpm_mode == VPM_MODE_PSCI) { + scheduler_lock(thread); + psci_pm_list_delete(scheduler_get_affinity(thread), thread); + scheduler_unlock(thread); + } +} + +void +psci_handle_object_deactivate_vpm_group(vpm_group_t *pg) +{ + for (cpu_index_t i = 0; cpulocal_index_valid(i); i++) { + assert(atomic_load_relaxed(&pg->psci_cpus[i]) == NULL); + } + + cpulocal_begin(); + ipi_one_relaxed(IPI_REASON_IDLE, cpulocal_get_index()); + ipi_others_idle(IPI_REASON_IDLE); + cpulocal_end(); +} + +error_t +vpm_group_configure(vpm_group_t *vpm_group, vpm_group_option_flags_t flags) +{ + vpm_group->options = flags; + + return OK; +} + +error_t +vpm_attach(vpm_group_t *pg, thread_t *thread, index_t index) +{ + assert(pg != NULL); + assert(thread != NULL); + assert(atomic_load_relaxed(&thread->header.state) == OBJECT_STATE_INIT); + assert(atomic_load_relaxed(&pg->header.state) == OBJECT_STATE_ACTIVE); + + error_t err; + + if (!cpulocal_index_valid((cpu_index_t)index)) { + err = ERROR_ARGUMENT_INVALID; + } else if (thread->kind != THREAD_KIND_VCPU) { + err = ERROR_ARGUMENT_INVALID; + } else { + if (thread->psci_group != NULL) { + object_put_vpm_group(thread->psci_group); + } + + thread->psci_group = object_get_vpm_group_additional(pg); + thread->psci_index = (cpu_index_t)index; + trace_ids_set_vcpu_index(&thread->trace_ids, + (cpu_index_t)index); + + err = OK; + } + + return err; +} + +error_t +psci_handle_task_queue_execute(task_queue_entry_t *task_entry) +{ + assert(task_entry != NULL); + vpm_group_t *vpm_group = + vpm_group_container_of_psci_virq_task(task_entry); + + (void)virq_assert(&vpm_group->psci_system_suspend_virq, true); + object_put_vpm_group(vpm_group); + + return OK; +} + +error_t +vpm_bind_virq(vpm_group_t *vpm_group, vic_t *vic, virq_t virq) +{ + error_t ret; + + assert(vpm_group != NULL); + assert(vic != NULL); + + ret = vic_bind_shared(&vpm_group->psci_system_suspend_virq, vic, virq, + VIRQ_TRIGGER_VPM_GROUP); + + return ret; +} + +void +vpm_unbind_virq(vpm_group_t *vpm_group) +{ + assert(vpm_group != NULL); + + vic_unbind_sync(&vpm_group->psci_system_suspend_virq); +} + +bool +vcpus_state_is_any_awake(vpm_group_suspend_state_t vm_state, uint32_t level, + cpu_index_t cpu) +{ + bool vcpu_awake = false; + error_t ret; + uint32_t start_idx = 0, children_counts = 0; + + uint64_t vcpus_state = + vpm_group_suspend_state_get_vcpus_state(&vm_state); + + uint16_t vcluster_state = + vpm_group_suspend_state_get_cluster_state(&vm_state); + + ret = platform_psci_get_index_by_level(cpu, &start_idx, + &children_counts, level); + + index_t idle_state = 0U, psci_index; + if (ret != OK) { + goto out; + } + + for (index_t i = 0U; i < children_counts; i++) { + // Check if another vcpu is awake + psci_index = start_idx + i; + + if (level == 1U) { + idle_state = + (index_t)(vcpus_state >> + (psci_index * + PSCI_VCPUS_STATE_PER_VCPU_BITS)) & + PSCI_VCPUS_STATE_PER_VCPU_MASK; + if (platform_psci_is_cpu_active( + (psci_cpu_state_t)idle_state)) { + vcpu_awake = true; + goto out; + } + } else if (level == 2U) { + idle_state = ((index_t)vcluster_state >> + ((psci_index % (PLATFORM_MAX_CORES)) * + PSCI_PER_CLUSTER_STATE_BITS)) & + PSCI_PER_CLUSTER_STATE_BITS_MASK; + if (platform_psci_is_cluster_active( + (psci_cluster_state_L3_t)idle_state)) { + vcpu_awake = true; + goto out; + } + } else { + // Only two levels are implemented. Return false + } + } +out: + return vcpu_awake; +} + +void +vcpus_state_set(vpm_group_suspend_state_t *vm_state, cpu_index_t cpu, + psci_cpu_state_t cpu_state) +{ + uint64_t vcpus_state = + vpm_group_suspend_state_get_vcpus_state(vm_state); + + vcpus_state &= ~((uint64_t)PSCI_VCPUS_STATE_PER_VCPU_MASK + << (cpu * PSCI_VCPUS_STATE_PER_VCPU_BITS)); + vcpus_state |= (uint64_t)cpu_state + << (cpu * PSCI_VCPUS_STATE_PER_VCPU_BITS); + + vpm_group_suspend_state_set_vcpus_state(vm_state, vcpus_state); +} + +void +vcpus_state_clear(vpm_group_suspend_state_t *vm_state, cpu_index_t cpu) +{ + uint64_t vcpus_state = + vpm_group_suspend_state_get_vcpus_state(vm_state); + + vcpus_state &= ~((uint64_t)PSCI_VCPUS_STATE_PER_VCPU_MASK + << (cpu * PSCI_VCPUS_STATE_PER_VCPU_BITS)); + + vpm_group_suspend_state_set_vcpus_state(vm_state, vcpus_state); +} + +error_t +psci_handle_vcpu_suspend(thread_t *current) +{ + error_t ret; + + if (current->vpm_mode != VPM_MODE_NONE) { + ret = psci_vcpu_suspend(current); + } else { + ret = OK; + } + + if (ret == OK) { + TRACE(PSCI, PSCI_VPM_VCPU_SUSPEND, + "psci vcpu suspend: {:#x} - VM {:d}", (uintptr_t)current, + current->addrspace->vmid); + } + + return ret; +} + +void +psci_unwind_vcpu_suspend(thread_t *current) +{ + if (current->vpm_mode != VPM_MODE_NONE) { + psci_vcpu_resume(current); + } +} + +bool +psci_handle_trapped_idle(void) +{ + thread_t *current = thread_get_self(); + bool handled = false; + + if (current->vpm_mode == VPM_MODE_IDLE) { + error_t err = vcpu_suspend(); + if ((err != OK) && (err != ERROR_BUSY)) { + panic("unhandled vcpu_suspend error (WFI)"); + } + handled = true; + } + + return handled; +} + +void +psci_handle_vcpu_resume(thread_t *vcpu) +{ + TRACE(PSCI, PSCI_VPM_VCPU_RESUME, + "psci vcpu resume: {:#x} - VM {:d} - VCPU {:d}", (uintptr_t)vcpu, + vcpu->addrspace->vmid, vcpu->psci_index); + + if (vcpu->vpm_mode != VPM_MODE_NONE) { + psci_vcpu_resume(vcpu); + } +} + +void +psci_handle_vcpu_started(bool warm_reset) +{ + // If the VCPU has been warm-reset, there was no vcpu_stopped event and + // no automatic psci_vcpu_suspend() call, so there's no need for a + // wakeup here. + if (!warm_reset) { + thread_t *current = thread_get_self(); + + TRACE(PSCI, PSCI_VPM_VCPU_RESUME, + "psci vcpu started: {:#x} - VM {:d}", (uintptr_t)current, + current->addrspace->vmid); + + if (current->vpm_mode != VPM_MODE_NONE) { + preempt_disable(); + psci_vcpu_resume(current); + preempt_enable(); + } + } +} + +void +psci_handle_vcpu_wakeup(thread_t *vcpu) +{ + if (scheduler_is_blocked(vcpu, SCHEDULER_BLOCK_VCPU_SUSPEND)) { + vcpu_resume(vcpu); + } +} + +void +psci_handle_vcpu_wakeup_self(void) +{ + assert(!scheduler_is_blocked(thread_get_self(), + SCHEDULER_BLOCK_VCPU_SUSPEND)); +} + +bool +psci_handle_vcpu_expects_wakeup(const thread_t *thread) +{ + return scheduler_is_blocked(thread, SCHEDULER_BLOCK_VCPU_SUSPEND); +} + +#if defined(INTERFACE_VCPU_RUN) +vcpu_run_state_t +psci_handle_vcpu_run_check(const thread_t *thread, register_t *state_data_0, + register_t *state_data_1) +{ + vcpu_run_state_t ret; + + if (thread->psci_system_reset) { + ret = VCPU_RUN_STATE_PSCI_SYSTEM_RESET; + *state_data_0 = thread->psci_system_reset_type; + *state_data_1 = thread->psci_system_reset_cookie; + } else if (psci_handle_vcpu_expects_wakeup(thread)) { + ret = VCPU_RUN_STATE_EXPECTS_WAKEUP; + *state_data_0 = + psci_suspend_powerstate_raw(thread->psci_suspend_state); + } else { + ret = VCPU_RUN_STATE_BLOCKED; + } + + return ret; +} +#endif + +error_t +psci_handle_vcpu_poweron(thread_t *vcpu) +{ + if (compiler_unexpected(vcpu->psci_group == NULL)) { + goto out; + } + + (void)atomic_fetch_add_explicit(&vcpu->psci_group->psci_online_count, + 1U, memory_order_relaxed); + cpu_index_t cpu = vcpu->scheduler_affinity; + if (cpulocal_index_valid(cpu)) { + psci_vcpu_clear_vcpu_state(vcpu, cpu); + } + +out: + return OK; +} + +error_t +psci_handle_vcpu_poweroff(thread_t *vcpu, bool last_cpu, bool force) +{ + error_t ret; + vpm_group_t *psci_group = vcpu->psci_group; + + if (psci_group == NULL) { + // This is always the last CPU in the VM, so permit the poweroff + // request if and only if it is intended for the last CPU or is + // forced. + ret = (last_cpu || force) ? OK : ERROR_DENIED; + } else if (vcpu->vpm_mode == VPM_MODE_PSCI) { + count_t online_cpus = + atomic_load_relaxed(&psci_group->psci_online_count); + do { + assert(online_cpus > 0U); + if (!force && (last_cpu != (online_cpus == 1U))) { + ret = ERROR_DENIED; + goto out; + } + } while (!atomic_compare_exchange_weak_explicit( + &psci_group->psci_online_count, &online_cpus, + online_cpus - 1U, memory_order_relaxed, + memory_order_relaxed)); + + ret = OK; + } else { + assert(vcpu->vpm_mode == VPM_MODE_NONE); + ret = OK; + } + +out: + return ret; +} + +void +psci_handle_vcpu_stopped(void) +{ + thread_t *vcpu = thread_get_self(); + + if (vcpu->psci_group != NULL) { + // Stopping a VCPU forces it into a power-off suspend state. + psci_suspend_powerstate_t pstate = + psci_suspend_powerstate_default(); + psci_suspend_powerstate_set_StateType( + &pstate, PSCI_SUSPEND_POWERSTATE_TYPE_POWERDOWN); + psci_suspend_powerstate_stateid_t stateid; + + preempt_disable(); + cpu_index_t cpu = cpulocal_get_index(); + +#if !defined(PSCI_AFFINITY_LEVELS_NOT_SUPPORTED) || \ + !PSCI_AFFINITY_LEVELS_NOT_SUPPORTED + // FIXME: + if (vcpu->psci_group->psci_mode == PSCI_MODE_PC) { + stateid = platform_psci_deepest_cluster_level_stateid( + cpu); + } else +#endif + { + stateid = platform_psci_deepest_cpu_level_stateid(cpu); + } + preempt_enable(); + + psci_suspend_powerstate_set_StateID(&pstate, stateid); + vcpu->psci_suspend_state = pstate; + } + + if (vcpu->vpm_mode != VPM_MODE_NONE) { + preempt_disable(); + error_t ret = psci_vcpu_suspend(vcpu); + preempt_enable(); + // Note that psci_vcpu_suspend can only fail if we are in OSI + // mode and requesting a cluster suspend state, which can't + // happen here because we set a non-cluster state above. + assert(ret == OK); + } +} + +void +psci_handle_power_cpu_online(void) +{ + psci_set_vpm_active_pcpus_bit(cpulocal_get_index()); +} + +void +psci_handle_power_cpu_offline(void) +{ + (void)psci_clear_vpm_active_pcpus_bit(cpulocal_get_index()); +} diff --git a/hyp/vm/psci_pc/src/psci_pm_list.c b/hyp/vm/psci/src/psci_pm_list.c similarity index 92% rename from hyp/vm/psci_pc/src/psci_pm_list.c rename to hyp/vm/psci/src/psci_pm_list.c index 35fad82..ce556ba 100644 --- a/hyp/vm/psci_pc/src/psci_pm_list.c +++ b/hyp/vm/psci/src/psci_pm_list.c @@ -47,8 +47,8 @@ psci_pm_list_delete(cpu_index_t cpu_index, thread_t *vcpu) list_t *list = &CPULOCAL_BY_INDEX(vcpu_pm_list, cpu_index); spinlock_acquire(&CPULOCAL_BY_INDEX(vcpu_pm_list_lock, cpu_index)); - list_delete_node(list, &vcpu->psci_pm_list_node); + (void)list_delete_node(list, &vcpu->psci_pm_list_node); spinlock_release(&CPULOCAL_BY_INDEX(vcpu_pm_list_lock, cpu_index)); - ipi_one_idle(IPI_REASON_IDLE, cpulocal_get_index()); + ipi_one_idle(IPI_REASON_IDLE, cpu_index); } diff --git a/hyp/vm/psci_pc/aarch64/src/psci.c b/hyp/vm/psci_pc/aarch64/src/psci.c deleted file mode 100644 index 5280030..0000000 --- a/hyp/vm/psci_pc/aarch64/src/psci.c +++ /dev/null @@ -1,61 +0,0 @@ -// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. -// -// SPDX-License-Identifier: BSD-3-Clause - -#include - -#include -#include - -#include - -#include "event_handlers.h" -#include "psci_arch.h" - -// The CPU ID values have the same format as MPIDR, but with all other fields -// masked out. This includes a bit that is forced to 1 in MPIDR_EL1_t, so we -// must mask off the affinity fields. -static const register_t mpidr_mask = MPIDR_EL1_AFF0_MASK | MPIDR_EL1_AFF1_MASK | - MPIDR_EL1_AFF2_MASK | MPIDR_EL1_AFF3_MASK; - -psci_mpidr_t -psci_thread_get_mpidr(thread_t *thread) -{ - return psci_mpidr_cast(MPIDR_EL1_raw(thread->vcpu_regs_mpidr_el1) & - mpidr_mask); -} - -psci_mpidr_t -psci_thread_set_mpidr_by_index(thread_t *thread, cpu_index_t index) -{ - psci_mpidr_t ret = platform_cpu_index_to_mpidr(index); - MPIDR_EL1_t real = register_MPIDR_EL1_read(); - thread->vcpu_regs_mpidr_el1 = MPIDR_EL1_default(); - MPIDR_EL1_set_Aff0(&thread->vcpu_regs_mpidr_el1, - psci_mpidr_get_Aff0(&ret)); - MPIDR_EL1_set_Aff1(&thread->vcpu_regs_mpidr_el1, - psci_mpidr_get_Aff1(&ret)); - MPIDR_EL1_set_Aff2(&thread->vcpu_regs_mpidr_el1, - psci_mpidr_get_Aff2(&ret)); - MPIDR_EL1_set_Aff3(&thread->vcpu_regs_mpidr_el1, - psci_mpidr_get_Aff3(&ret)); - MPIDR_EL1_set_MT(&thread->vcpu_regs_mpidr_el1, MPIDR_EL1_get_MT(&real)); - return ret; -} - -void -psci_pc_handle_scheduler_selected_thread(thread_t *thread, bool *can_idle) -{ - if (thread->psci_mode == VPM_MODE_IDLE) { - // This thread can't be allowed to disable the WFI trap, - // because WFI votes to suspend the physical CPU. - *can_idle = false; - } -} - -vcpu_trap_result_t -psci_pc_handle_vcpu_trap_wfi(void) -{ - return psci_pc_handle_trapped_idle() ? VCPU_TRAP_RESULT_EMULATED - : VCPU_TRAP_RESULT_UNHANDLED; -} diff --git a/hyp/vm/psci_pc/build.conf b/hyp/vm/psci_pc/build.conf index eba6c31..97d96db 100644 --- a/hyp/vm/psci_pc/build.conf +++ b/hyp/vm/psci_pc/build.conf @@ -1,12 +1,7 @@ -# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# © 2022 Qualcomm Innovation Center, Inc. All rights reserved. # # SPDX-License-Identifier: BSD-3-Clause -interface vpm -interface psci -events psci.ev -types psci.tc -local_include -source psci.c hypercalls.c psci_pm_list.c -arch_source aarch64 psci.c -arch_events aarch64 psci.ev +base_module hyp/vm/psci +events psci_pc.ev +source psci_pc.c diff --git a/hyp/vm/psci_pc/include/psci_arch.h b/hyp/vm/psci_pc/include/psci_arch.h deleted file mode 100644 index 2243928..0000000 --- a/hyp/vm/psci_pc/include/psci_arch.h +++ /dev/null @@ -1,12 +0,0 @@ -// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. -// -// SPDX-License-Identifier: BSD-3-Clause - -psci_mpidr_t -psci_thread_get_mpidr(thread_t *thread); - -psci_mpidr_t -psci_thread_set_mpidr_by_index(thread_t *thread, cpu_index_t index); - -bool -psci_pc_handle_trapped_idle(void); diff --git a/hyp/vm/psci_pc/psci.ev b/hyp/vm/psci_pc/psci.ev deleted file mode 100644 index c80295f..0000000 --- a/hyp/vm/psci_pc/psci.ev +++ /dev/null @@ -1,134 +0,0 @@ -// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. -// -// SPDX-License-Identifier: BSD-3-Clause - -module psci_pc - -#define PSCI_FUNCTION(fn, feat, h, ...) \ -subscribe smccc_call_fast_32_standard[PSCI_FUNCTION_ ## fn]; \ - handler psci_ ## h ## _32(__VA_ARGS__); \ - exclude_preempt_disabled. \ -subscribe psci_features32[PSCI_FUNCTION_ ## fn]; \ - constant feat. \ -subscribe smccc_call_fast_64_standard[PSCI_FUNCTION_ ## fn]; \ - handler psci_ ## h ## _64(__VA_ARGS__); \ - exclude_preempt_disabled. \ -subscribe psci_features64[PSCI_FUNCTION_ ## fn]; \ - constant feat. - -#define PSCI_FUNCTION32(fn, feat, h, ...) \ -subscribe smccc_call_fast_32_standard[PSCI_FUNCTION_ ## fn]; \ - handler psci_ ## h(__VA_ARGS__); \ - exclude_preempt_disabled. \ -subscribe psci_features32[PSCI_FUNCTION_ ## fn]; \ - constant feat. - -#define PSCI_FUNCTION_PERVM(fn, h, ...) \ -subscribe smccc_call_fast_32_standard[PSCI_FUNCTION_ ## fn]; \ - handler psci_ ## h ## _32(__VA_ARGS__); \ - exclude_preempt_disabled. \ -subscribe psci_features32[PSCI_FUNCTION_ ## fn]; \ - handler psci_ ## h ## _32_features(). \ -subscribe smccc_call_fast_64_standard[PSCI_FUNCTION_ ## fn]; \ - handler psci_ ## h ## _64(__VA_ARGS__); \ - exclude_preempt_disabled. \ -subscribe psci_features64[PSCI_FUNCTION_ ## fn]; \ - handler psci_ ## h ## _64_features(). \ - -#define PSCI_FUNCTION32_PERVM(fn, h, ...) \ -subscribe smccc_call_fast_32_standard[PSCI_FUNCTION_ ## fn]; \ - handler psci_ ## h(__VA_ARGS__); \ - exclude_preempt_disabled. \ -subscribe psci_features32[PSCI_FUNCTION_ ## fn]; \ - handler psci_ ## h ## _features(). - -PSCI_FUNCTION32(PSCI_VERSION, 0U, version, ret0) -PSCI_FUNCTION_PERVM(CPU_SUSPEND, cpu_suspend, arg1, arg2, arg3, ret0) -PSCI_FUNCTION32(CPU_OFF, 0U, cpu_off, ret0) -PSCI_FUNCTION(CPU_ON, 0U, cpu_on, arg1, arg2, arg3, ret0) -PSCI_FUNCTION(AFFINITY_INFO, 0U, affinity_info, arg1, arg2, ret0) -//PSCI_FUNCTION32(MIGRATE, 0U, migrate, arg1, ret0) -//PSCI_FUNCTION32(MIGRATE_INFO_TYPE, 0U, migrate_info_type, ret0) -//PSCI_FUNCTION(MIGRATE_INFO_UP_CPU, 0U, migrate_info_up_cpu, ret0) -PSCI_FUNCTION32(SYSTEM_OFF, 0U, system_off) -PSCI_FUNCTION32(SYSTEM_RESET, 0U, system_reset) -PSCI_FUNCTION32(PSCI_FEATURES, 0U, features, arg1, ret0) -//PSCI_FUNCTION32(CPU_FREEZE, 0U, cpu_freeze, ret0) -PSCI_FUNCTION(CPU_DEFAULT_SUSPEND, 0U, cpu_default_suspend, arg1, arg2, ret0) -//PSCI_FUNCTION(NODE_HW_STATE, 0U, node_hw_state, arg1, arg2, ret0) -//PSCI_FUNCTION(SYSTEM_SUSPEND, 0U, system_suspend, arg1, arg2, ret0) -PSCI_FUNCTION32(PSCI_SET_SUSPEND_MODE, 0U, set_suspend_mode, arg1, ret0) -//PSCI_FUNCTION(PSCI_STAT_RESIDENCY, 0U, stat_residency, arg1, arg2, ret0) -//PSCI_FUNCTION(PSCI_STAT_COUNT, 0U, stat_count, arg1, arg2, ret0) -PSCI_FUNCTION(SYSTEM_RESET2, 0U, system_reset2, arg1, arg2, ret0) -//PSCI_FUNCTION32(MEM_PROTECT, 0U, mem_protect, arg1, ret0) -//PSCI_FUNCTION(MEM_PROTECT_CHECK_RANGE, 0U, mem_protect_check_range, arg1, arg2, ret0) - -subscribe object_create_thread - -subscribe object_activate_thread - // Run early to ensure that MPIDR is set correctly, since other - // modules may rely on it (especially VGIC, which is priority 1) - priority 50 - -subscribe object_deactivate_thread - -subscribe object_activate_vpm_group - -subscribe object_deactivate_vpm_group - -subscribe idle_yield - // Run late, but before handlers that may sleep, to check - // whether we should suspend the physical CPU instead - priority -10 - require_preempt_disabled - -subscribe vcpu_suspend - unwinder () - -subscribe vcpu_started - -subscribe vcpu_wakeup - require_scheduler_lock(vcpu) - -subscribe vcpu_wakeup_self - -subscribe vcpu_expects_wakeup - -subscribe vcpu_poweron - -subscribe vcpu_resume - - // This is first so that it can deny poweroff without unwinding other - // modules. -subscribe vcpu_poweroff - priority first - -subscribe vcpu_activate_thread(thread) - unwinder psci_handle_object_deactivate_thread(thread) - // Run after the scheduler handler - priority -100 - -subscribe boot_cold_init() - -subscribe boot_cpu_cold_init - -subscribe scheduler_affinity_changed(thread, prev_cpu, need_sync) - -subscribe scheduler_affinity_changed_sync(thread, next_cpu) - -subscribe task_queue_execute[TASK_QUEUE_CLASS_VPM_GROUP_VIRQ](entry) - -#if defined(ROOTVM_IS_HLOS) && ROOTVM_IS_HLOS -subscribe rootvm_init - handler psci_handle_rootvm_init_early - // Run early to ensure that all the secondary threads exist for - // modules that need them (especially VGIC, which is priority 1) - priority first - -subscribe rootvm_init - handler psci_handle_rootvm_init_late(root_cspace, env_data) - // Run last to activate all the group's threads, to ensure that other - // modules don't try to configure activated threads - priority last -#endif diff --git a/hyp/vm/psci_pc/psci_pc.ev b/hyp/vm/psci_pc/psci_pc.ev new file mode 100644 index 0000000..28a9512 --- /dev/null +++ b/hyp/vm/psci_pc/psci_pc.ev @@ -0,0 +1,19 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module psci_pc + +#include "psci_events.h" + +PSCI_FUNCTION32(PSCI_SET_SUSPEND_MODE, 0U, pc_set_suspend_mode, arg1, ret0) + +subscribe boot_cold_init() + +subscribe object_activate_vpm_group + +subscribe idle_yield + // Run late, but before handlers that may sleep, to check + // whether we should suspend the physical CPU instead + priority -10 + require_preempt_disabled diff --git a/hyp/vm/psci_pc/src/psci.c b/hyp/vm/psci_pc/src/psci.c deleted file mode 100644 index f0c56fc..0000000 --- a/hyp/vm/psci_pc/src/psci.c +++ /dev/null @@ -1,1399 +0,0 @@ -// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. -// -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "event_handlers.h" -#include "psci_arch.h" -#include "psci_pm_list.h" - -// In this psci simple version, no affinity levels are supported - -CPULOCAL_DECLARE_STATIC(_Atomic count_t, vpm_active_vcpus); -static _Atomic register_t vpm_active_pcpus_bitmap; - -extern list_t partition_list; - -// Set to 1 to boot enable the PSCI tracepoints -#if defined(VERBOSE_TRACE) && VERBOSE_TRACE -#define DEBUG_PSCI_TRACES 1 -#else -#define DEBUG_PSCI_TRACES 0 -#endif - -void -psci_pc_handle_boot_cold_init(void) -{ -#if !defined(NDEBUG) && DEBUG_PSCI_TRACES - register_t flags = 0U; - TRACE_SET_CLASS(flags, PSCI); - trace_set_class_flags(flags); -#endif - - psci_pm_list_init(); - -#if !defined(PSCI_SET_SUSPEND_MODE_NOT_SUPPORTED) || \ - !PSCI_SET_SUSPEND_MODE_NOT_SUPPORTED - error_t ret = platform_psci_set_suspend_mode(PSCI_MODE_OSI); - assert(ret == OK); -#endif -} - -static bool -psci_set_vpm_active_pcpus_bit(index_t bit) -{ - register_t old = atomic_fetch_or_explicit( - &vpm_active_pcpus_bitmap, util_bit(bit), memory_order_relaxed); - - return old == 0U; -} - -// Returns true if bitmap becomes zero after clearing bit -static bool -psci_clear_vpm_active_pcpus_bit(index_t bit) -{ - register_t cleared_bit = ~util_bit(bit); - - register_t old = atomic_fetch_and_explicit( - &vpm_active_pcpus_bitmap, cleared_bit, memory_order_relaxed); - - return (old & cleared_bit) == 0U; -} - -void -psci_pc_handle_boot_cpu_cold_init(cpu_index_t cpu) -{ - atomic_store_relaxed(&CPULOCAL_BY_INDEX(vpm_active_vcpus, cpu), 0U); - (void)psci_set_vpm_active_pcpus_bit(cpu); -} - -static void -psci_vpm_active_vcpus_get(cpu_index_t cpu, thread_t *vcpu) -{ - assert(cpulocal_index_valid(cpu)); - assert(vcpu->psci_inactive_count != 0U); - - vcpu->psci_inactive_count--; - if (vcpu->psci_inactive_count == 0U) { - (void)atomic_fetch_add_explicit( - &CPULOCAL_BY_INDEX(vpm_active_vcpus, cpu), 1U, - memory_order_relaxed); - } -} - -static void -psci_vpm_active_vcpus_put(cpu_index_t cpu, thread_t *vcpu) -{ - assert(cpulocal_index_valid(cpu)); - - vcpu->psci_inactive_count++; - if (vcpu->psci_inactive_count == 1U) { - count_t old = atomic_fetch_sub_explicit( - &CPULOCAL_BY_INDEX(vpm_active_vcpus, cpu), 1U, - memory_order_relaxed); - assert(old != 0U); - } -} - -static bool -psci_vpm_active_vcpus_is_zero(cpu_index_t cpu) -{ - assert(cpulocal_index_valid(cpu)); - - return atomic_load_relaxed(&CPULOCAL_BY_INDEX(vpm_active_vcpus, cpu)) == - 0; -} - -bool -psci_pc_handle_vcpu_activate_thread(thread_t *thread) -{ - bool ret = true; - - assert(thread != NULL); - assert(thread->kind == THREAD_KIND_VCPU); - - scheduler_lock(thread); - - // Determine the initial inactive count for the VCPU. - thread->psci_inactive_count = 0U; - - if (scheduler_is_blocked(thread, SCHEDULER_BLOCK_VCPU_OFF)) { - // VCPU is inactive because it is powered off. - thread->psci_inactive_count++; - } - // VCPU can't be suspended or in WFI yet. - assert(!scheduler_is_blocked(thread, SCHEDULER_BLOCK_VCPU_SUSPEND)); - assert(!scheduler_is_blocked(thread, SCHEDULER_BLOCK_VCPU_WFI)); - - cpu_index_t cpu = scheduler_get_affinity(thread); - if (cpulocal_index_valid(cpu)) { - if (thread->psci_group != NULL) { - psci_pm_list_insert(cpu, thread); - } - } else { - // VCPU is inactive because it has no valid affinity. - thread->psci_inactive_count++; - } - - // If the VCPU is initially active, make sure the CPU stays awake. - if (thread->psci_inactive_count == 0U) { - assert(cpulocal_index_valid(cpu)); - (void)atomic_fetch_add_explicit( - &CPULOCAL_BY_INDEX(vpm_active_vcpus, cpu), 1U, - memory_order_relaxed); - } - - scheduler_unlock(thread); - - return ret; -} - -void -psci_pc_handle_scheduler_affinity_changed(thread_t *thread, - cpu_index_t prev_cpu, bool *need_sync) -{ - object_state_t state = atomic_load_acquire(&thread->header.state); - - if ((thread->kind == THREAD_KIND_VCPU) && - (state == OBJECT_STATE_ACTIVE)) { - if (cpulocal_index_valid(prev_cpu)) { - if (thread->psci_group != NULL) { - psci_pm_list_delete(prev_cpu, thread); - } - if ((thread->psci_mode == VPM_MODE_PSCI) || - (thread->psci_mode == VPM_MODE_IDLE)) { - psci_vpm_active_vcpus_put(prev_cpu, thread); - } - } - - thread->psci_migrate = true; - *need_sync = true; - } -} - -void -psci_pc_handle_scheduler_affinity_changed_sync(thread_t *thread, - cpu_index_t next_cpu) -{ - if (thread->psci_migrate) { - assert(thread->kind == THREAD_KIND_VCPU); - - if (cpulocal_index_valid(next_cpu)) { - if (thread->psci_group != NULL) { - psci_pm_list_insert(next_cpu, thread); - } - if ((thread->psci_mode == VPM_MODE_PSCI) || - (thread->psci_mode == VPM_MODE_IDLE)) { - scheduler_lock(thread); - psci_vpm_active_vcpus_get(next_cpu, thread); - scheduler_unlock(thread); - } - } - - thread->psci_migrate = false; - } -} - -static thread_t * -psci_get_thread_by_mpidr(psci_mpidr_t mpidr) -{ - thread_t *current = thread_get_self(); - thread_t *result = NULL; - - if (psci_mpidr_is_equal(psci_thread_get_mpidr(current), mpidr)) { - result = object_get_thread_additional(current); - } else { - vpm_group_t *psci_group = current->psci_group; - if (psci_group != NULL) { - cpu_index_result_t index = - platform_cpu_mpidr_to_index(mpidr); - if (index.e == OK) { - // RCU protects psci_group->psci_cpus[i] - rcu_read_start(); - - result = atomic_load_consume( - &psci_group->psci_cpus[index.r]); - if ((result != NULL) && - !object_get_thread_safe(result)) { - result = NULL; - } - - rcu_read_finish(); - } - } - } - - if (result != NULL) { - assert(psci_mpidr_is_equal(psci_thread_get_mpidr(result), - mpidr)); - } - - return result; -} - -static bool -psci_is_hlos(void) -{ - thread_t *vcpu = thread_get_self(); - - return vcpu_option_flags_get_hlos_vm(&vcpu->vcpu_options); -} - -bool -psci_version(uint32_t *ret0) -{ - *ret0 = PSCI_VERSION; - return true; -} - -static psci_ret_t -psci_suspend(psci_suspend_powerstate_t suspend_state, - paddr_t entry_point_address, register_t context_id) - EXCLUDE_PREEMPT_DISABLED -{ - psci_ret_t ret = PSCI_RET_SUCCESS; - thread_t *current = thread_get_self(); - - assert(current->psci_group != NULL); - - current->psci_suspend_state = suspend_state; - - error_t err = vcpu_suspend(); - if (err == ERROR_DENIED) { - TRACE(PSCI, PSCI_PSTATE_VALIDATION, - "psci_suspend: DENIED - pstate {:#x} - VM {:d}", - psci_suspend_powerstate_raw(suspend_state), - current->addrspace->vmid); - ret = PSCI_RET_DENIED; - goto out; - } else if (err == ERROR_ARGUMENT_INVALID) { - TRACE(PSCI, PSCI_PSTATE_VALIDATION, - "psci suspend: INVALID_PARAMETERS - pstate {:#x} - VM {:d}", - psci_suspend_powerstate_raw(suspend_state), - current->addrspace->vmid); - ret = PSCI_RET_INVALID_PARAMETERS; - goto out; - } else if (err == ERROR_BUSY) { - // It did not suspend due to a pending interrupt - ret = PSCI_RET_SUCCESS; - goto out; - } else if (err == OK) { - ret = PSCI_RET_SUCCESS; - } else { - panic("unhandled vcpu_suspend error"); - } - - // Warm reset VCPU unconditionally from the psci mode to make the - // cpuidle stats work - if ((psci_suspend_powerstate_get_StateType(&suspend_state) == - PSCI_SUSPEND_POWERSTATE_TYPE_POWERDOWN)) { - vcpu_warm_reset(entry_point_address, context_id); - } - -out: - return ret; -} - -static psci_ret_t -psci_cpu_suspend(psci_suspend_powerstate_t suspend_state, - paddr_t entry_point_address, register_t context_id) - EXCLUDE_PREEMPT_DISABLED -{ - psci_ret_t ret; - thread_t *current = thread_get_self(); - - if (current->psci_group == NULL) { - ret = PSCI_RET_NOT_SUPPORTED; - goto out; - } - - ret = psci_suspend(suspend_state, entry_point_address, context_id); - -out: - return ret; -} - -uint32_t -psci_cpu_suspend_32_features(void) -{ - uint32_t ret; - - // Only Platform-coordinated mode, extended StateID - ret = 2U; - - return ret; -} - -uint32_t -psci_cpu_suspend_64_features(void) -{ - return psci_cpu_suspend_32_features(); -} - -bool -psci_cpu_suspend_32(uint32_t arg1, uint32_t arg2, uint32_t arg3, uint32_t *ret0) -{ - psci_ret_t ret = psci_cpu_suspend(psci_suspend_powerstate_cast(arg1), - arg2, arg3); - *ret0 = (uint32_t)ret; - return true; -} - -bool -psci_cpu_suspend_64(uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t *ret0) -{ - psci_ret_t ret = psci_cpu_suspend( - psci_suspend_powerstate_cast((uint32_t)arg1), arg2, arg3); - *ret0 = (uint64_t)ret; - return true; -} - -// Same as psci_cpu_suspend, but it sets the suspend state to the deepest -// cpu-level. -static psci_ret_t -psci_cpu_default_suspend(paddr_t entry_point_address, register_t context_id) - EXCLUDE_PREEMPT_DISABLED -{ - psci_ret_t ret; - thread_t *current = thread_get_self(); - cpu_index_t cpu = cpulocal_get_index(); - - if (current->psci_group == NULL) { - ret = PSCI_RET_NOT_SUPPORTED; - goto out; - } - - psci_suspend_powerstate_t pstate = psci_suspend_powerstate_default(); - psci_suspend_powerstate_stateid_t stateid = - platform_psci_deepest_cpu_level_stateid(cpu); - psci_suspend_powerstate_set_StateID(&pstate, stateid); - psci_suspend_powerstate_set_StateType( - &pstate, PSCI_SUSPEND_POWERSTATE_TYPE_POWERDOWN); - - ret = psci_suspend(pstate, entry_point_address, context_id); -out: - return ret; -} - -bool -psci_cpu_default_suspend_32(uint32_t arg1, uint32_t arg2, uint32_t *ret0) -{ - psci_ret_t ret = psci_cpu_default_suspend(arg1, arg2); - *ret0 = (uint32_t)ret; - return true; -} - -bool -psci_cpu_default_suspend_64(uint64_t arg1, uint64_t arg2, uint64_t *ret0) -{ - psci_ret_t ret = psci_cpu_default_suspend(arg1, arg2); - *ret0 = (uint64_t)ret; - return true; -} - -static psci_ret_t -psci_switch_suspend_mode(psci_mode_t new_mode) -{ - psci_ret_t ret = PSCI_RET_SUCCESS; - - thread_t *thread = thread_get_self(); - vpm_group_t *psci_group = thread->psci_group; - cpu_index_t vcpu_id = thread->psci_index; - - assert(psci_group != NULL); - - rcu_read_start(); - - vpm_group_suspend_state_t vm_state = - atomic_load_acquire(&psci_group->psci_vm_suspend_state); - - cpu_index_t cpu_index = 0U; - psci_cpu_state_t cpu_state = 0U; - - vpm_vcpus_state_foreach ( - cpu_index, cpu_state, - vpm_group_suspend_state_get_vcpus_state(&vm_state)) { - if (vcpu_id == cpu_index) { - continue; - } - if (((new_mode == PSCI_MODE_OSI) && - !platform_psci_is_cpu_poweroff(cpu_state) && - !platform_psci_is_cpu_active(cpu_state)) || - ((new_mode == PSCI_MODE_PC) && - !platform_psci_is_cpu_poweroff(cpu_state))) { - ret = PSCI_RET_DENIED; - goto out; - } - } - - // If conditions met, change suspend mode of vpm group - psci_group->psci_mode = new_mode; - -out: - rcu_read_finish(); - - return ret; -} - -bool -psci_set_suspend_mode(uint32_t arg1, uint32_t *ret0) -{ - psci_ret_t ret; - - thread_t *current = thread_get_self(); - - if (arg1 == current->psci_group->psci_mode) { - ret = PSCI_RET_SUCCESS; - goto out; - } - - switch (arg1) { - case PSCI_MODE_PC: - case PSCI_MODE_OSI: - ret = psci_switch_suspend_mode(arg1); - if (ret == PSCI_RET_DENIED) { - TRACE(PSCI, INFO, - "psci_set_suspend_mode - DENIED - VM {:d}", - current->addrspace->vmid); - } - - if (ret == PSCI_RET_DENIED) { - TRACE(PSCI, INFO, - "psci_set_suspend_mode - DENIED - VM {:d}", - current->addrspace->vmid); - } - - break; - default: - ret = PSCI_RET_INVALID_PARAMETERS; - TRACE(PSCI, INFO, - "psci_set_suspend_mode - INVALID_PARAMETERS - VM {:d}", - current->addrspace->vmid); - break; - } - -out: - *ret0 = (uint32_t)ret; - return true; -} - -bool -psci_cpu_off(uint32_t *ret0) -{ - thread_t *current = thread_get_self(); - cpu_index_t cpu = cpulocal_get_index(); - vpm_group_t *psci_group = current->psci_group; - - if (psci_group != NULL) { - psci_suspend_powerstate_t pstate = - psci_suspend_powerstate_default(); - psci_suspend_powerstate_set_StateType( - &pstate, PSCI_SUSPEND_POWERSTATE_TYPE_POWERDOWN); - psci_suspend_powerstate_stateid_t stateid; - - stateid = platform_psci_deepest_cpu_level_stateid(cpu); - - psci_suspend_powerstate_set_StateID(&pstate, stateid); - current->psci_suspend_state = pstate; - - error_t ret = vcpu_poweroff(); - // If we return, the only reason should be DENIED - assert(ret == ERROR_DENIED); - } - *ret0 = (uint32_t)PSCI_RET_DENIED; - return true; -} - -static psci_ret_t -psci_cpu_on(psci_mpidr_t cpu, paddr_t entry_point_address, - register_t context_id) -{ - thread_t *thread = psci_get_thread_by_mpidr(cpu); - psci_ret_t ret; - - if (thread == NULL) { - ret = PSCI_RET_INVALID_PARAMETERS; - } else { - bool reschedule = false; - - scheduler_lock(thread); - if (scheduler_is_blocked(thread, SCHEDULER_BLOCK_VCPU_OFF)) { - reschedule = vcpu_poweron(thread, entry_point_address, - context_id); - ret = PSCI_RET_SUCCESS; - } else { - ret = PSCI_RET_ALREADY_ON; - } - scheduler_unlock(thread); - object_put_thread(thread); - - if (reschedule) { - scheduler_schedule(); - } - } - - return ret; -} - -bool -psci_cpu_on_32(uint32_t arg1, uint32_t arg2, uint32_t arg3, uint32_t *ret0) -{ - psci_ret_t ret = psci_cpu_on(psci_mpidr_cast(arg1), arg2, arg3); - *ret0 = (uint32_t)ret; - return true; -} - -bool -psci_cpu_on_64(uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t *ret0) -{ - psci_ret_t ret = psci_cpu_on(psci_mpidr_cast(arg1), arg2, arg3); - *ret0 = (uint64_t)ret; - return true; -} - -static psci_ret_t -psci_affinity_info(psci_mpidr_t affinity, uint32_t lowest_affinity_level) -{ - psci_ret_t ret; - - thread_t *thread = psci_get_thread_by_mpidr(affinity); - if (thread == NULL) { - ret = PSCI_RET_INVALID_PARAMETERS; - } else if (lowest_affinity_level != 0U) { - // lowest_affinity_level is legacy from PSCI 0.2; we are - // allowed to fail if it is nonzero (which indicates a - // query of the cluster-level state). - ret = PSCI_RET_INVALID_PARAMETERS; - } else { - // Don't bother locking, this is inherently racy anyway - psci_ret_affinity_info_t info = - scheduler_is_blocked(thread, SCHEDULER_BLOCK_VCPU_OFF) - ? PSCI_RET_AFFINITY_INFO_OFF - : PSCI_RET_AFFINITY_INFO_ON; - ret = (psci_ret_t)info; - } - - if (thread != NULL) { - object_put_thread(thread); - } - - return ret; -} - -bool -psci_affinity_info_32(uint32_t arg1, uint32_t arg2, uint32_t *ret0) -{ - psci_ret_t ret = psci_affinity_info(psci_mpidr_cast(arg1), arg2); - *ret0 = (uint32_t)ret; - return true; -} - -bool -psci_affinity_info_64(uint64_t arg1, uint64_t arg2, uint64_t *ret0) -{ - psci_ret_t ret = - psci_affinity_info(psci_mpidr_cast(arg1), (uint32_t)arg2); - *ret0 = (uint64_t)ret; - return true; -} - -bool -psci_system_off(void) -{ - if (!psci_is_hlos()) { - return (uint32_t)PSCI_RET_NOT_SUPPORTED; - } - - trigger_power_system_off_event(); - - panic("system_off event returned"); -} - -bool -psci_system_reset(void) -{ - if (!psci_is_hlos()) { - return (uint32_t)PSCI_RET_NOT_SUPPORTED; - } - - error_t ret = OK; - (void)trigger_power_system_reset_event(PSCI_REQUEST_SYSTEM_RESET, 0U, - &ret); - panic("system_reset event returned"); -} - -static uint32_t -psci_system_reset2(uint64_t reset_type, uint64_t cookie) -{ - uint32_t ret; - - if (psci_is_hlos()) { - error_t error = OK; - trigger_power_system_reset_event(reset_type, cookie, &error); - - if (error == ERROR_ARGUMENT_INVALID) { - ret = (uint32_t)PSCI_RET_INVALID_PARAMETERS; - } else { - ret = (uint32_t)PSCI_RET_NOT_SUPPORTED; - } - } else { - ret = (uint32_t)PSCI_RET_NOT_SUPPORTED; - } - - return ret; -} - -bool -psci_system_reset2_32(uint32_t arg1, uint32_t arg2, uint32_t *ret0) -{ - *ret0 = psci_system_reset2(arg1, arg2); - return true; -} - -bool -psci_system_reset2_64(uint64_t arg1, uint64_t arg2, uint64_t *ret0) -{ - *ret0 = psci_system_reset2( - (uint32_t)arg1 | PSCI_REQUEST_SYSTEM_RESET2_64, arg2); - return true; -} - -bool -psci_features(uint32_t arg1, uint32_t *ret0) -{ - smccc_function_id_t fn_id = smccc_function_id_cast(arg1); - uint32_t ret = SMCCC_UNKNOWN_FUNCTION32; - smccc_function_t fn = smccc_function_id_get_function(&fn_id); - - // We need to handle discovery of SMCCC_VERSION here - - if ((smccc_function_id_get_interface_id(&fn_id) == - SMCCC_INTERFACE_ID_STANDARD) && - smccc_function_id_get_is_fast(&fn_id) && - (smccc_function_id_get_res0(&fn_id) == 0U)) { - ret = smccc_function_id_get_is_smc64(&fn_id) - ? trigger_psci_features64_event(fn) - : trigger_psci_features32_event(fn); - } - - *ret0 = ret; - return true; -} - -error_t -psci_pc_handle_object_create_thread(thread_create_t thread_create) -{ - thread_t *thread = thread_create.thread; - assert(thread != NULL); - - // Default thread to be of IDLE mode - thread->psci_mode = VPM_MODE_IDLE; - - psci_suspend_powerstate_t pstate = psci_suspend_powerstate_default(); - psci_suspend_powerstate_stateid_t stateid = - platform_psci_deepest_cpu_level_stateid( - thread->scheduler_affinity); - psci_suspend_powerstate_set_StateID(&pstate, stateid); - psci_suspend_powerstate_set_StateType( - &pstate, PSCI_SUSPEND_POWERSTATE_TYPE_POWERDOWN); - - // Initialize to deepest possible state - thread->psci_suspend_state = pstate; - - return OK; -} - -error_t -psci_pc_handle_object_activate_thread(thread_t *thread) -{ - error_t err; - if (thread->kind != THREAD_KIND_VCPU) { - err = OK; - } else if (thread->psci_group == NULL) { - err = OK; - } else { - assert(scheduler_is_blocked(thread, SCHEDULER_BLOCK_VCPU_OFF)); - - cpu_index_t index = thread->psci_index; - thread_t *tmp_null = NULL; - - if (!cpulocal_index_valid(index)) { - err = ERROR_OBJECT_CONFIG; - } else if (!atomic_compare_exchange_strong_explicit( - &thread->psci_group->psci_cpus[index], - &tmp_null, thread, memory_order_release, - memory_order_relaxed)) { - err = ERROR_DENIED; - } else { - psci_thread_set_mpidr_by_index(thread, index); - err = OK; - } - } - return err; -} - -void -psci_pc_handle_object_deactivate_thread(thread_t *thread) -{ - assert(thread != NULL); - - if (thread->psci_group != NULL) { - thread_t *tmp = thread; - cpu_index_t index = thread->psci_index; - - atomic_compare_exchange_strong_explicit( - &thread->psci_group->psci_cpus[index], &tmp, NULL, - memory_order_relaxed, memory_order_relaxed); - object_put_vpm_group(thread->psci_group); - - scheduler_lock(thread); - psci_pm_list_delete(scheduler_get_affinity(thread), thread); - scheduler_unlock(thread); - } -} - -error_t -vpm_bind_virq(vpm_group_t *vpm_group, vic_t *vic, virq_t virq) -{ - error_t ret = OK; - - assert(vpm_group != NULL); - assert(vic != NULL); - - ret = vic_bind_shared(&vpm_group->psci_system_suspend_virq, vic, virq, - VIRQ_TRIGGER_VPM_GROUP); - - return ret; -} - -void -vpm_unbind_virq(vpm_group_t *vpm_group) -{ - assert(vpm_group != NULL); - - vic_unbind_sync(&vpm_group->psci_system_suspend_virq); -} - -static bool -vcpus_state_is_any_awake(vpm_group_suspend_state_t vm_state) -{ - bool vcpu_awake = false; - cpu_index_t cpu_index = 0U; - psci_cpu_state_t cpu_state = 0U; - - vpm_vcpus_state_foreach ( - cpu_index, cpu_state, - vpm_group_suspend_state_get_vcpus_state(&vm_state)) { - if (platform_psci_is_cpu_active(cpu_state)) { - vcpu_awake = true; - goto out; - } - } - -out: - return vcpu_awake; -} - -static void -vcpus_state_set(vpm_group_suspend_state_t *vm_state, cpu_index_t cpu, - psci_cpu_state_t cpu_state) -{ - uint32_t vcpus_state = - vpm_group_suspend_state_get_vcpus_state(vm_state); - - vcpus_state &= (uint32_t) ~(PSCI_VCPUS_STATE_PER_VCPU_MASK - << (cpu * PSCI_VCPUS_STATE_PER_VCPU_BITS)); - vcpus_state |= - (uint32_t)(cpu_state << (cpu * PSCI_VCPUS_STATE_PER_VCPU_BITS)); - - vpm_group_suspend_state_set_vcpus_state(vm_state, vcpus_state); -} - -static void -vcpus_state_clear(vpm_group_suspend_state_t *vm_state, cpu_index_t cpu) -{ - uint32_t vcpus_state = - vpm_group_suspend_state_get_vcpus_state(vm_state); - - vcpus_state &= (uint32_t) ~(PSCI_VCPUS_STATE_PER_VCPU_MASK - << (cpu * PSCI_VCPUS_STATE_PER_VCPU_BITS)); - - vpm_group_suspend_state_set_vcpus_state(vm_state, vcpus_state); -} - -error_t -psci_pc_handle_object_activate_vpm_group(vpm_group_t *pg) -{ - spinlock_init(&pg->psci_lock); - pg->psci_system_suspend_count = 0; - task_queue_init(&pg->psci_virq_task, TASK_QUEUE_CLASS_VPM_GROUP_VIRQ); - - // Default psci mode to be platfom-coordinated - pg->psci_mode = PSCI_MODE_PC; - - // Initialize vcpus states of the vpm to the deepest suspend state - psci_cpu_state_t cpu_state = - platform_psci_deepest_cpu_state(cpulocal_get_index()); - vpm_group_suspend_state_t vm_state = vpm_group_suspend_state_default(); - - for (cpu_index_t i = 0; - i < (PSCI_VCPUS_STATE_BITS / PSCI_VCPUS_STATE_PER_VCPU_BITS); - i++) { - vcpus_state_set(&vm_state, i, cpu_state); - } - - atomic_store_release(&pg->psci_vm_suspend_state, vm_state); - - return OK; -} - -void -psci_pc_handle_object_deactivate_vpm_group(vpm_group_t *pg) -{ - for (cpu_index_t i = 0; cpulocal_index_valid(i); i++) { - assert(atomic_load_relaxed(&pg->psci_cpus[i]) == NULL); - } - - ipi_one_idle(IPI_REASON_IDLE, cpulocal_get_index()); -} - -error_t -vpm_attach(vpm_group_t *pg, thread_t *thread, index_t index) -{ - assert(pg != NULL); - assert(thread != NULL); - assert(atomic_load_relaxed(&thread->header.state) == OBJECT_STATE_INIT); - assert(atomic_load_relaxed(&pg->header.state) == OBJECT_STATE_ACTIVE); - - error_t err; - - if (!cpulocal_index_valid((cpu_index_t)index)) { - err = ERROR_ARGUMENT_INVALID; - } else if (thread->kind != THREAD_KIND_VCPU) { - err = ERROR_ARGUMENT_INVALID; - } else { - if (thread->psci_group != NULL) { - object_put_vpm_group(thread->psci_group); - } - - thread->psci_group = object_get_vpm_group_additional(pg); - thread->psci_index = (cpu_index_t)index; - trace_ids_set_vcpu_index(&thread->trace_ids, - (cpu_index_t)index); - - thread->psci_mode = VPM_MODE_PSCI; - - err = OK; - } - - return err; -} - -error_t -psci_pc_handle_task_queue_execute(task_queue_entry_t *task_entry) -{ - assert(task_entry != NULL); - vpm_group_t *vpm_group = - vpm_group_container_of_psci_virq_task(task_entry); - - virq_assert(&vpm_group->psci_system_suspend_virq, true); - object_put_vpm_group(vpm_group); - - return OK; -} - -vpm_state_t -vpm_get_state(vpm_group_t *vpm_group) -{ - vpm_state_t vpm_state = VPM_STATE_NO_STATE; - - vpm_group_suspend_state_t vm_state = - atomic_load_acquire(&vpm_group->psci_vm_suspend_state); - - if (vcpus_state_is_any_awake(vm_state)) { - vpm_state = VPM_STATE_RUNNING; - } else { - vpm_state = VPM_STATE_CPUS_SUSPENDED; - } - - return vpm_state; -} - -static void -psci_vcpu_wakeup(thread_t *thread, cpu_index_t target_cpu) -{ - if (cpulocal_index_valid(target_cpu)) { - psci_vpm_active_vcpus_get(target_cpu, thread); - } - - if (thread->psci_mode != VPM_MODE_PSCI) { - goto out; - } - - assert(thread->psci_group != NULL); - - vpm_group_t *vpm_group = thread->psci_group; - cpu_index_t vcpu_id = thread->psci_index; - - thread->psci_suspend_state = psci_suspend_powerstate_default(); - - vpm_group_suspend_state_t old_state = - atomic_load_relaxed(&vpm_group->psci_vm_suspend_state); - vpm_group_suspend_state_t new_state; - - do { - new_state = old_state; - - vcpus_state_clear(&new_state, vcpu_id); - - } while (!atomic_compare_exchange_strong_explicit( - &vpm_group->psci_vm_suspend_state, &old_state, new_state, - memory_order_relaxed, memory_order_relaxed)); - -out: - // Nothing to do for non PSCI threads - return; -} - -static error_t -psci_vcpu_suspend(thread_t *current) -{ - if (current->psci_mode != VPM_MODE_PSCI) { - goto out; - } - - assert(current->psci_group != NULL); - - // Decrement refcount of the PCPU - psci_vpm_active_vcpus_put(cpulocal_get_index(), current); - - vpm_group_t *vpm_group = current->psci_group; - cpu_index_t vcpu_id = current->psci_index; - psci_cpu_state_t cpu_state = - platform_psci_get_cpu_state(current->psci_suspend_state); - - vpm_group_suspend_state_t new_state; - vpm_group_suspend_state_t old_state; - - // Set vcpus_state of corresponding cpu. - old_state = atomic_load_relaxed(&vpm_group->psci_vm_suspend_state); - - do { - new_state = old_state; - vcpus_state_set(&new_state, vcpu_id, cpu_state); - - } while (!atomic_compare_exchange_strong_explicit( - &vpm_group->psci_vm_suspend_state, &old_state, new_state, - memory_order_relaxed, memory_order_relaxed)); - -out: - return OK; -} - -error_t -psci_pc_handle_vcpu_suspend(void) -{ - error_t ret = OK; - thread_t *current = thread_get_self(); - - ret = psci_vcpu_suspend(current); - if (ret == OK) { - TRACE(PSCI, PSCI_VPM_VCPU_SUSPEND, - "psci vcpu suspend: {:#x} - VM {:d}", (uintptr_t)current, - current->addrspace->vmid); - } - - return ret; -} - -void -psci_pc_unwind_vcpu_suspend(void) -{ - thread_t *current = thread_get_self(); - - psci_vcpu_wakeup(current, cpulocal_get_index()); -} - -bool -psci_pc_handle_trapped_idle(void) -{ - thread_t *current = thread_get_self(); - bool handled = false; - - if (current->psci_mode == VPM_MODE_IDLE) { - psci_vpm_active_vcpus_put(cpulocal_get_index(), current); - error_t err = vcpu_suspend(); - if ((err != OK) && (err != ERROR_BUSY)) { - panic("unhandled vcpu_suspend error (WFI)"); - } - handled = true; - } - - return handled; -} - -void -psci_pc_handle_vcpu_resume(void) -{ - thread_t *vcpu = thread_get_self(); - - TRACE(PSCI, PSCI_VPM_VCPU_RESUME, - "psci vcpu resume: {:#x} - VM {:d} - VCPU {:d}", (uintptr_t)vcpu, - vcpu->addrspace->vmid, vcpu->psci_index); - - psci_vcpu_wakeup(vcpu, cpulocal_get_index()); -} - -void -psci_pc_handle_vcpu_started(void) -{ - thread_t *current = thread_get_self(); - - // If the VCPU has been warm-reset, it has already called - // psci_vcpu_wakeup in the above vcpu_resume event handler. - if (!current->vcpu_warm_reset) { - TRACE(PSCI, PSCI_VPM_VCPU_RESUME, - "psci vcpu started: {:#x} - VM {:d}", (uintptr_t)current, - current->addrspace->vmid); - - scheduler_lock(current); - psci_vcpu_wakeup(current, cpulocal_get_index()); - scheduler_unlock(current); - } -} - -void -psci_pc_handle_vcpu_wakeup(thread_t *vcpu) -{ - if (scheduler_is_blocked(vcpu, SCHEDULER_BLOCK_VCPU_SUSPEND)) { - vcpu_resume(vcpu); - } -} - -void -psci_pc_handle_vcpu_wakeup_self(void) -{ - assert(!scheduler_is_blocked(thread_get_self(), - SCHEDULER_BLOCK_VCPU_SUSPEND)); -} - -bool -psci_pc_handle_vcpu_expects_wakeup(const thread_t *thread) -{ - return scheduler_is_blocked(thread, SCHEDULER_BLOCK_VCPU_SUSPEND); -} - -void -psci_pc_handle_vcpu_poweron(thread_t *vcpu) -{ - if (vcpu->psci_group == NULL) { - goto out; - } - - (void)atomic_fetch_add_explicit(&vcpu->psci_group->psci_online_count, - 1U, memory_order_relaxed); - - cpu_index_t cpu = vcpu->scheduler_affinity; - if (cpulocal_index_valid(cpu)) { - if (platform_cpu_on(cpu) != OK) { - // Note that already-on and on-pending results - // from EL3 PSCI are treated as success. - panic("Failed to power on secondary CPU"); - } - } - -out: - return; -} - -error_t -psci_pc_handle_vcpu_poweroff(thread_t *vcpu, bool force) -{ - error_t ret = OK; - vpm_group_t *psci_group = vcpu->psci_group; - - if (psci_group != NULL) { - count_t online_cpus = - atomic_load_relaxed(&psci_group->psci_online_count); - do { - assert(online_cpus > 0U); - if (!force && (online_cpus == 1U)) { - ret = ERROR_DENIED; - goto out; - } - } while (!atomic_compare_exchange_weak_explicit( - &psci_group->psci_online_count, &online_cpus, - online_cpus - 1U, memory_order_relaxed, - memory_order_relaxed)); - - ret = psci_vcpu_suspend(vcpu); - } -out: - return ret; -} - -idle_state_t -psci_pc_handle_idle_yield(bool in_idle_thread) -{ - assert_preempt_disabled(); - - idle_state_t idle_state = IDLE_STATE_IDLE; - - if (!in_idle_thread) { - goto out; - } - - cpu_index_t cpu = cpulocal_get_index(); - - // Check if there is any vcpu running in this cpu - if (!psci_vpm_active_vcpus_is_zero(cpu)) { - goto out; - } - - thread_t *vcpu = NULL; - list_t *psci_pm_list = psci_pm_list_get_self(); - assert(psci_pm_list != NULL); - - psci_suspend_powerstate_t pstate = psci_suspend_powerstate_default(); - psci_cpu_state_t cpu_state = platform_psci_deepest_cpu_state(cpu); - - // Iterate through affine VCPUs and get the shallowest cpu-level state - rcu_read_start(); - list_foreach_container_consume (vcpu, psci_pm_list, thread, - psci_pm_list_node) { - psci_cpu_state_t cpu1 = - platform_psci_get_cpu_state(vcpu->psci_suspend_state); - cpu_state = platform_psci_shallowest_cpu_state(cpu_state, cpu1); - } - rcu_read_finish(); - - // Do not go to suspend if shallowest cpu state is zero. This may happen - // if a vcpu has started after doing the initial check of 'any vcpu - // running in this cpu' and therefore has been added to the psci_pm_list - // with a psci_suspend_state of 0. - if (cpu_state == 0U) { - goto out; - } - - platform_psci_set_cpu_state(&pstate, cpu_state); - - if (platform_psci_is_cpu_poweroff(cpu_state)) { - psci_suspend_powerstate_set_StateType( - &pstate, PSCI_SUSPEND_POWERSTATE_TYPE_POWERDOWN); - } else { - psci_suspend_powerstate_set_StateType( - &pstate, - PSCI_SUSPEND_POWERSTATE_TYPE_STANDBY_OR_RETENTION); - } - - bool last_cpu = false; - - if (psci_clear_vpm_active_pcpus_bit(cpu)) { - last_cpu = true; - } - - // Fence to prevent any power_cpu_suspend event handlers conditional on - // last_cpu (especially the trigger of power_system_suspend) being - // reordered before the psci_clear_vpm_active_pcpus_bit() above. This - // matches the fence before the resume event below. - atomic_thread_fence(memory_order_seq_cst); - - error_t suspend_result = trigger_power_cpu_suspend_event( - pstate, - psci_suspend_powerstate_get_StateType(&pstate) == - PSCI_SUSPEND_POWERSTATE_TYPE_POWERDOWN, - last_cpu); - if (suspend_result == OK) { - bool_result_t ret; - - TRACE(PSCI, INFO, "psci power_cpu_suspend {:#x}", - psci_suspend_powerstate_raw(pstate)); - - ret = platform_cpu_suspend(pstate); - - // Check if this is the first cpu to wake up - bool first_cpu = psci_set_vpm_active_pcpus_bit(cpu); - suspend_result = ret.e; - - // Fence to prevent any power_cpu_resume event handlers - // conditional on first_cpu (especially the trigger of - // power_system_resume) being reordered before the - // psci_set_vpm_active_pcpus_bit() above. This matches the - // fence before the suspend event above. - atomic_thread_fence(memory_order_seq_cst); - - trigger_power_cpu_resume_event((ret.e == OK) && ret.r, - first_cpu); - TRACE(PSCI, INFO, - "psci power_cpu_suspend wakeup; poweroff {:d} system_resume {:d} error {:d}", - ret.r, first_cpu, (register_t)ret.e); - } else { - TRACE(PSCI, INFO, "psci power_cpu_suspend failed: {:d}", - (unsigned int)suspend_result); - (void)psci_set_vpm_active_pcpus_bit(cpu); - } - - if (suspend_result == OK) { - // Return from successful suspend. We were presumably woken by - // an interrupt; handle it now and reschedule if required. - idle_state = irq_interrupt_dispatch() ? IDLE_STATE_RESCHEDULE - : IDLE_STATE_WAKEUP; - } else if (suspend_result == ERROR_BUSY) { - // An interrupt will arrive soon, continue with idle. - } else if (suspend_result != ERROR_DENIED) { - TRACE_AND_LOG(ERROR, WARN, "ERROR: psci suspend error {:d}", - (register_t)suspend_result); - panic("unhandled suspend error"); - } else { - // suspend state was denied, re-run psci aggregation. - idle_state = IDLE_STATE_WAKEUP; - } - -out: - return idle_state; -} - -#if defined(ROOTVM_IS_HLOS) && ROOTVM_IS_HLOS -void -psci_pc_handle_rootvm_init_early(partition_t *root_partition, - thread_t *root_thread, cspace_t *root_cspace, - boot_env_data_t *env_data) -{ - // Create the PSCI group for the root VM - vpm_group_create_t pg_params = { 0 }; - vpm_group_ptr_result_t pg_r = - partition_allocate_vpm_group(root_partition, pg_params); - if (pg_r.e != OK) { - panic("Unable to create root VM's PSCI group"); - } - - if (object_activate_vpm_group(pg_r.r) != OK) { - panic("Error activating root PSCI group"); - } - - // Create a master cap for the PSCI group - object_ptr_t optr = { .vpm_group = pg_r.r }; - cap_id_result_t cid_r = cspace_create_master_cap(root_cspace, optr, - OBJECT_TYPE_VPM_GROUP); - if (cid_r.e != OK) { - panic("Unable to create cap to root VM's PSCI group"); - } - env_data->psci_group = cid_r.r; - - // Attach the root VM's main VCPU to the group - assert(root_thread->scheduler_affinity == cpulocal_get_index()); - if (vpm_attach(pg_r.r, root_thread, root_thread->scheduler_affinity) != - OK) { - panic("Unable to attach root thread to its PSCI group"); - } - - // Create new powered-off VCPUs for every other CPU - for (cpu_index_t i = 0; cpulocal_index_valid(i); i++) { - if (i == root_thread->scheduler_affinity) { - env_data->psci_secondary_vcpus[i] = CSPACE_CAP_INVALID; - continue; - } - - thread_create_t thread_params = { - .scheduler_affinity = i, - .scheduler_affinity_valid = true, - .kind = THREAD_KIND_VCPU, - }; - - thread_ptr_result_t thread_r = partition_allocate_thread( - root_partition, thread_params); - if (thread_r.e != OK) { - panic("Unable to create root VM secondary VCPU"); - } - - vcpu_option_flags_t vcpu_options = vcpu_option_flags_default(); - vcpu_option_flags_set_hlos_vm(&vcpu_options, true); - - if (vcpu_configure(thread_r.r, vcpu_options) != OK) { - panic("Error configuring secondary VCPU"); - } - - // Attach thread to root cspace - if (cspace_attach_thread(root_cspace, thread_r.r) != OK) { - panic("Error attaching cspace to secondary VCPU"); - } - - optr.thread = thread_r.r; - cid_r = cspace_create_master_cap(root_cspace, optr, - OBJECT_TYPE_THREAD); - if (cid_r.e != OK) { - panic("Unable to create cap to root VM secondary VCPU"); - } - env_data->psci_secondary_vcpus[i] = cid_r.r; - - if (vpm_attach(pg_r.r, thread_r.r, i) != OK) { - panic("Unable to attach root VCPU to the PSCI group"); - } - } -} - -void -psci_pc_handle_rootvm_init_late(cspace_t *root_cspace, - boot_env_data_t *env_data) -{ - // Activate the secondary VCPU objects - for (cpu_index_t i = 0; cpulocal_index_valid(i); i++) { - cap_id_t thread_cap = env_data->psci_secondary_vcpus[i]; - if (thread_cap == CSPACE_CAP_INVALID) { - continue; - } - - object_type_t type; - object_ptr_result_t o = cspace_lookup_object_any( - root_cspace, thread_cap, - CAP_RIGHTS_GENERIC_OBJECT_ACTIVATE, &type); - - if ((o.e != OK) || (type != OBJECT_TYPE_THREAD) || - (object_activate_thread(o.r.thread) != OK)) { - panic("Error activating secondary VCPU"); - } - - object_put(type, o.r); - } -} -#endif diff --git a/hyp/vm/psci_pc/src/psci_pc.c b/hyp/vm/psci_pc/src/psci_pc.c new file mode 100644 index 0000000..de59f61 --- /dev/null +++ b/hyp/vm/psci_pc/src/psci_pc.c @@ -0,0 +1,378 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "event_handlers.h" +#include "psci_arch.h" +#include "psci_common.h" +#include "psci_pm_list.h" + +void +psci_pc_handle_boot_cold_init(void) +{ +#if !defined(PSCI_SET_SUSPEND_MODE_NOT_SUPPORTED) || \ + !PSCI_SET_SUSPEND_MODE_NOT_SUPPORTED + error_t ret = platform_psci_set_suspend_mode(PSCI_MODE_PC); + assert(ret == OK); +#endif +} + +uint32_t +psci_cpu_suspend_features(void) +{ + uint32_t ret; + + // Only Platform Co-ordinated mode, extended StateID + ret = 2U; + + return ret; +} + +bool +psci_pc_set_suspend_mode(uint32_t arg1, uint32_t *ret0) +{ + bool handled; + psci_ret_t ret; + + thread_t *current = thread_get_self(); + + if (current->psci_group == NULL) { + handled = false; + } else { + if (arg1 == PSCI_MODE_PC) { + ret = PSCI_RET_SUCCESS; + } else { + ret = PSCI_RET_INVALID_PARAMETERS; + TRACE(PSCI, INFO, + "psci_set_suspend_mode - INVALID_PARAMETERS - VM {:d}", + current->addrspace->vmid); + } + *ret0 = (uint32_t)ret; + handled = true; + } + + return handled; +} + +error_t +psci_pc_handle_object_activate_vpm_group(vpm_group_t *pg) +{ + spinlock_init(&pg->psci_lock); + task_queue_init(&pg->psci_virq_task, TASK_QUEUE_CLASS_VPM_GROUP_VIRQ); + + // Default psci mode to be platfom-coordinated + pg->psci_mode = PSCI_MODE_PC; + + // Initialize vcpus states of the vpm to the deepest suspend state + // FIXME: + cpulocal_begin(); + psci_cpu_state_t cpu_state = + platform_psci_deepest_cpu_state(cpulocal_get_index()); + cpulocal_end(); + + vpm_group_suspend_state_t vm_state = vpm_group_suspend_state_default(); + + for (cpu_index_t i = 0; + i < (PSCI_VCPUS_STATE_BITS / PSCI_VCPUS_STATE_PER_VCPU_BITS); + i++) { + vcpus_state_set(&vm_state, i, cpu_state); + } + + atomic_store_release(&pg->psci_vm_suspend_state, vm_state); + + return OK; +} + +vpm_state_t +vpm_get_state(vpm_group_t *vpm_group) +{ + vpm_state_t vpm_state = VPM_STATE_NO_STATE; + + vpm_group_suspend_state_t vm_state = + atomic_load_acquire(&vpm_group->psci_vm_suspend_state); + + if (vcpus_state_is_any_awake(vm_state, PLATFORM_MAX_HIERARCHY, 0)) { + vpm_state = VPM_STATE_RUNNING; + } else { + vpm_state = VPM_STATE_CPUS_SUSPENDED; + } + + return vpm_state; +} + +/* + * this clears the vcpu state for core which has started to boot from + * hw followed by firmware cluster and suspend states are still + * cleared by the same in wake-up path by calling into psci_vcpu_wakeup + */ +void +psci_vcpu_clear_vcpu_state(thread_t *thread, cpu_index_t target_cpu) +{ + (void)target_cpu; + if (thread->vpm_mode != VPM_MODE_PSCI) { + goto out; + } + + assert(thread->psci_group != NULL); + + vpm_group_t *vpm_group = thread->psci_group; + cpu_index_t vcpu_id = thread->psci_index; + + thread->psci_suspend_state = psci_suspend_powerstate_default(); + + vpm_group_suspend_state_t old_state = + atomic_load_relaxed(&vpm_group->psci_vm_suspend_state); + vpm_group_suspend_state_t new_state; + + new_state = old_state; + vcpus_state_clear(&new_state, vcpu_id); + +out: + // Nothing to do for non PSCI threads + return; +} + +void +psci_vcpu_resume(thread_t *thread) +{ + assert(thread->vpm_mode != VPM_MODE_NONE); + + scheduler_lock_nopreempt(thread); + psci_vpm_active_vcpus_get(scheduler_get_active_affinity(thread), + thread); + scheduler_unlock_nopreempt(thread); + + if (thread->vpm_mode != VPM_MODE_PSCI) { + goto out; + } + + assert(thread->psci_group != NULL); + + vpm_group_t *vpm_group = thread->psci_group; + cpu_index_t vcpu_id = thread->psci_index; + + thread->psci_suspend_state = psci_suspend_powerstate_default(); + + vpm_group_suspend_state_t old_state = + atomic_load_relaxed(&vpm_group->psci_vm_suspend_state); + vpm_group_suspend_state_t new_state; + + do { + new_state = old_state; + + vcpus_state_clear(&new_state, vcpu_id); + + } while (!atomic_compare_exchange_strong_explicit( + &vpm_group->psci_vm_suspend_state, &old_state, new_state, + memory_order_relaxed, memory_order_relaxed)); + +out: + // Nothing to do for non PSCI threads + return; +} + +error_t +psci_vcpu_suspend(thread_t *current) +{ + assert(current->vpm_mode != VPM_MODE_NONE); + + // Decrement refcount of the PCPU + scheduler_lock_nopreempt(current); + psci_vpm_active_vcpus_put(scheduler_get_active_affinity(current), + current); + scheduler_unlock_nopreempt(current); + + if (current->vpm_mode != VPM_MODE_PSCI) { + goto out; + } + + assert(current->psci_group != NULL); + + vpm_group_t *vpm_group = current->psci_group; + cpu_index_t vcpu_id = current->psci_index; + psci_cpu_state_t cpu_state = + platform_psci_get_cpu_state(current->psci_suspend_state); + + vpm_group_suspend_state_t new_state; + vpm_group_suspend_state_t old_state; + + // Set vcpus_state of corresponding cpu. + old_state = atomic_load_relaxed(&vpm_group->psci_vm_suspend_state); + + do { + new_state = old_state; + vcpus_state_set(&new_state, vcpu_id, cpu_state); + + } while (!atomic_compare_exchange_strong_explicit( + &vpm_group->psci_vm_suspend_state, &old_state, new_state, + memory_order_relaxed, memory_order_relaxed)); + +out: + return OK; +} + +idle_state_t +psci_pc_handle_idle_yield(bool in_idle_thread) +{ + assert_preempt_disabled(); + + idle_state_t idle_state = IDLE_STATE_IDLE; + + if (!in_idle_thread) { + goto out; + } + + if (rcu_has_pending_updates()) { + goto out; + } + + cpu_index_t cpu = cpulocal_get_index(); + + // Check if there is any vcpu running in this cpu + if (!psci_vpm_active_vcpus_is_zero(cpu)) { + goto out; + } + + thread_t *vcpu = NULL; + list_t *psci_pm_list = psci_pm_list_get_self(); + assert(psci_pm_list != NULL); + + psci_suspend_powerstate_t pstate = psci_suspend_powerstate_default(); + psci_cpu_state_t cpu_state = platform_psci_deepest_cpu_state(cpu); + + // Iterate through affine VCPUs and get the shallowest cpu-level state + rcu_read_start(); + list_foreach_container_consume (vcpu, psci_pm_list, thread, + psci_pm_list_node) { + psci_cpu_state_t cpu1 = + platform_psci_get_cpu_state(vcpu->psci_suspend_state); + cpu_state = platform_psci_shallowest_cpu_state(cpu_state, cpu1); + } + rcu_read_finish(); + + // Do not go to suspend if shallowest cpu state is zero. This may happen + // if a vcpu has started after doing the initial check of 'any vcpu + // running in this cpu' and therefore has been added to the psci_pm_list + // with a psci_suspend_state of 0. + if (cpu_state == 0U) { + goto out; + } + + platform_psci_set_cpu_state(&pstate, cpu_state); + + if (platform_psci_is_cpu_poweroff(cpu_state)) { + psci_suspend_powerstate_set_StateType( + &pstate, PSCI_SUSPEND_POWERSTATE_TYPE_POWERDOWN); + } else { + psci_suspend_powerstate_set_StateType( + &pstate, + PSCI_SUSPEND_POWERSTATE_TYPE_STANDBY_OR_RETENTION); + } + + bool last_cpu = false; + + if (psci_clear_vpm_active_pcpus_bit(cpu)) { + last_cpu = true; + } + + // Fence to prevent any power_cpu_suspend event handlers conditional on + // last_cpu (especially the trigger of power_system_suspend) being + // reordered before the psci_clear_vpm_active_pcpus_bit() above. This + // matches the fence before the resume event below. + atomic_thread_fence(memory_order_seq_cst); + + error_t suspend_result = trigger_power_cpu_suspend_event( + pstate, + psci_suspend_powerstate_get_StateType(&pstate) == + PSCI_SUSPEND_POWERSTATE_TYPE_POWERDOWN, + last_cpu); + if (suspend_result == OK) { + bool_result_t ret; + + TRACE(PSCI, INFO, "psci power_cpu_suspend {:#x}", + psci_suspend_powerstate_raw(pstate)); + + ret = platform_cpu_suspend(pstate); + + // Check if this is the first cpu to wake up + bool first_cpu = psci_set_vpm_active_pcpus_bit(cpu); + suspend_result = ret.e; + + // Fence to prevent any power_cpu_resume event handlers + // conditional on first_cpu (especially the trigger of + // power_system_resume) being reordered before the + // psci_set_vpm_active_pcpus_bit() above. This matches the + // fence before the suspend event above. + atomic_thread_fence(memory_order_seq_cst); + + trigger_power_cpu_resume_event((ret.e == OK) && ret.r, + first_cpu); + TRACE(PSCI, INFO, + "psci power_cpu_suspend wakeup; poweroff {:d} system_resume {:d} error {:d}", + ret.r, first_cpu, (register_t)ret.e); + } else { + TRACE(PSCI, INFO, "psci power_cpu_suspend failed: {:d}", + (unsigned int)suspend_result); + (void)psci_set_vpm_active_pcpus_bit(cpu); + } + + if (suspend_result == OK) { + // Return from successful suspend. We were presumably woken by + // an interrupt; handle it now and reschedule if required. + idle_state = irq_interrupt_dispatch() ? IDLE_STATE_RESCHEDULE + : IDLE_STATE_WAKEUP; + } else if (suspend_result == ERROR_BUSY) { + // An interrupt will arrive soon, continue with idle. + } else if (suspend_result != ERROR_DENIED) { + TRACE_AND_LOG(ERROR, WARN, "ERROR: psci suspend error {:d}", + (register_t)suspend_result); + panic("unhandled suspend error"); + } else { + // suspend state was denied, re-run psci aggregation. + idle_state = IDLE_STATE_WAKEUP; + } + +out: + return idle_state; +} diff --git a/hyp/vm/rootvm/rootvm.ev b/hyp/vm/rootvm/rootvm.ev index 2657b56..9b8e826 100644 --- a/hyp/vm/rootvm/rootvm.ev +++ b/hyp/vm/rootvm/rootvm.ev @@ -4,14 +4,27 @@ interface rootvm +include + event rootvm_init param root_partition: partition_t * param root_thread: thread_t * param root_cspace: cspace_t * - param env_data: boot_env_data_t * + param hyp_env: hyp_env_data_t * + param qcbor_enc_ctxt: qcbor_enc_ctxt_t * + +event rootvm_init_late + param root_partition: partition_t * + param root_thread: thread_t * + param root_cspace: cspace_t * + param hyp_env: const hyp_env_data_t * + +event rootvm_started + param root_thread: thread_t * module rootvm subscribe boot_hypervisor_start handler rootvm_init() priority last + require_preempt_disabled diff --git a/hyp/vm/rootvm/rootvm.tc b/hyp/vm/rootvm/rootvm.tc index a5d5972..eac613f 100644 --- a/hyp/vm/rootvm/rootvm.tc +++ b/hyp/vm/rootvm/rootvm.tc @@ -2,10 +2,51 @@ // // SPDX-License-Identifier: BSD-3-Clause -extend boot_env_data structure { - partition_capid type cap_id_t; - cspace_capid type cap_id_t; +// Place the root VM at the default scheduling priority. This will typically +// be shared with most HLOS VMs. +// +// Note: VMs with higher priority than this must perform directed yields to +// the root VM while waiting for it to complete an operation. VMs with lower +// priority than this may be blocked by long-running root VM operations. +define ROOTVM_PRIORITY public constant type priority_t = + SCHEDULER_DEFAULT_PRIORITY; + +extend hyp_env_data structure { vcpu_capid type cap_id_t; entry_ipa type vmaddr_t; env_ipa type vmaddr_t; + env_data_size size; +}; + +define ROOTVM_ENV_DATA_SIGNATURE public constant = 0x454D5652; +define ROOTVM_ENV_DATA_VERSION public constant = 0x1000; + +// Do NOT extend this structure, since its cross image interface +define rt_env_data public structure { + signature uint32; + version uint16; + + runtime_ipa type vmaddr_t; + app_ipa type vmaddr_t; + app_heap_ipa type vmaddr_t; + app_heap_size size; + + vcpu_capid type cap_id_t; + timer_freq uint64; + gicd_base type paddr_t; + gicr_base type paddr_t; + + rm_config_offset size; + rm_config_size size; +}; + +define RM_ENV_DATA_SIGNATURE public constant = 0x524D4544; +define RM_ENV_DATA_VERSION public constant = 0x1000; + +// Do NOT extend this structure, since its cross image interface +define rm_env_data_hdr public structure { + signature uint32; + version uint16; + data_payload_offset uint32; + data_payload_size uint32; }; diff --git a/hyp/vm/rootvm/src/rootvm_init.c b/hyp/vm/rootvm/src/rootvm_init.c index 111995c..ebecc89 100644 --- a/hyp/vm/rootvm/src/rootvm_init.c +++ b/hyp/vm/rootvm/src/rootvm_init.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -44,13 +45,13 @@ rootvm_init(void) { static_assert(SCHEDULER_NUM_PRIORITIES >= (priority_t)3U, "unexpected scheduler configuration"); - static_assert(SCHEDULER_MAX_PRIORITY - 2U > SCHEDULER_DEFAULT_PRIORITY, + static_assert(ROOTVM_PRIORITY <= VCPU_MAX_PRIORITY, "unexpected scheduler configuration"); thread_create_t params = { .scheduler_affinity = cpulocal_get_index(), .scheduler_affinity_valid = true, - .scheduler_priority = SCHEDULER_MAX_PRIORITY - 2U, + .scheduler_priority = ROOTVM_PRIORITY, .scheduler_priority_valid = true, }; @@ -70,12 +71,12 @@ rootvm_init(void) } cspace_t *root_cspace = cspace_ret.r; - spinlock_acquire(&root_cspace->header.lock); + spinlock_acquire_nopreempt(&root_cspace->header.lock); if (cspace_configure(root_cspace, MAX_CAPS) != OK) { - spinlock_release(&root_cspace->header.lock); + spinlock_release_nopreempt(&root_cspace->header.lock); goto cspace_fail; } - spinlock_release(&root_cspace->header.lock); + spinlock_release_nopreempt(&root_cspace->header.lock); if (object_activate_cspace(root_cspace) != OK) { goto cspace_fail; @@ -93,9 +94,7 @@ rootvm_init(void) vcpu_option_flags_t vcpu_options = vcpu_option_flags_default(); -#if defined(ROOTVM_IS_HLOS) && ROOTVM_IS_HLOS - vcpu_option_flags_set_hlos_vm(&vcpu_options, true); -#endif + vcpu_option_flags_set_critical(&vcpu_options, true); if (vcpu_configure(root_thread, vcpu_options) != OK) { panic("Error configuring vcpu"); @@ -117,18 +116,68 @@ rootvm_init(void) } void_ptr_result_t alloc_ret; - boot_env_data_t *env_data; - size_t env_data_size = sizeof(*env_data); + hyp_env_data_t hyp_env; // Local on stack used as context + qcbor_enc_ctxt_t *qcbor_enc_ctxt; + + rm_env_data_hdr_t *rm_env_data; + rt_env_data_t *crt_env; + uint32_t env_data_size = 0x4000; + uint32_t remaining_size; alloc_ret = partition_alloc(root_partition, env_data_size, - alignof(*env_data)); + PGTABLE_VM_PAGE_SIZE); if (alloc_ret.e != OK) { panic("Allocate env_data failed"); } - env_data = (boot_env_data_t *)alloc_ret.r; - memset(env_data, 0, env_data_size); + crt_env = (rt_env_data_t *)alloc_ret.r; + (void)memset_s(crt_env, env_data_size, 0, env_data_size); + + alloc_ret = partition_alloc(root_partition, sizeof(*qcbor_enc_ctxt), + alignof(*qcbor_enc_ctxt)); + if (alloc_ret.e != OK) { + panic("Allocate cbor_ctxt failed"); + } + + qcbor_enc_ctxt = (qcbor_enc_ctxt_t *)alloc_ret.r; + memset_s(qcbor_enc_ctxt, sizeof(*qcbor_enc_ctxt), 0, + sizeof(*qcbor_enc_ctxt)); + + memset_s(&hyp_env, sizeof(hyp_env), 0, sizeof(hyp_env)); + + hyp_env.env_data_size = env_data_size; + remaining_size = env_data_size; + + crt_env->signature = ROOTVM_ENV_DATA_SIGNATURE; + crt_env->version = 1; + + size_t rm_config_offset = + util_balign_up(sizeof(*crt_env), alignof(*rm_env_data)); + assert(remaining_size >= (rm_config_offset + sizeof(*rm_env_data))); + + remaining_size -= rm_config_offset; + rm_env_data = + (rm_env_data_hdr_t *)((uintptr_t)crt_env + rm_config_offset); + + crt_env->rm_config_offset = rm_config_offset; + crt_env->rm_config_size = remaining_size; + + rm_env_data->signature = RM_ENV_DATA_SIGNATURE; + rm_env_data->version = 1; + rm_env_data->data_payload_offset = sizeof(*rm_env_data); + rm_env_data->data_payload_size = 0U; + + remaining_size -= sizeof(*rm_env_data); + + useful_buff_t qcbor_data_buff; + qcbor_data_buff.ptr = + (((uint8_t *)rm_env_data) + rm_env_data->data_payload_offset); + qcbor_data_buff.len = remaining_size; - env_data->cspace_capid = capid_ret.r; + QCBOREncode_Init(qcbor_enc_ctxt, qcbor_data_buff); + + QCBOREncode_OpenMap(qcbor_enc_ctxt); + + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "cspace_capid", capid_ret.r); // Take extra reference so that the deletion of the master cap does not // accidentally destroy the partition. @@ -141,7 +190,8 @@ rootvm_init(void) if (capid_ret.e != OK) { panic("Error creating root partition cap"); } - env_data->partition_capid = capid_ret.r; + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "partition_capid", + capid_ret.r); obj_ptr.thread = root_thread; capid_ret = cspace_create_master_cap(root_cspace, obj_ptr, @@ -149,54 +199,85 @@ rootvm_init(void) if (capid_ret.e != OK) { panic("Error creating root partition cap"); } - env_data->vcpu_capid = capid_ret.r; + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "vcpu_capid", capid_ret.r); + crt_env->vcpu_capid = capid_ret.r; // Do a memdb walk to get all the available memory ranges of the root - // partition and save in the boot_env_data - if (memdb_walk((uintptr_t)root_partition, MEMDB_TYPE_PARTITION, - boot_add_free_range, (void *)env_data) != OK) { + // partition and save in the rm_env_data + if (boot_add_free_range((uintptr_t)root_partition, MEMDB_TYPE_PARTITION, + qcbor_enc_ctxt) != OK) { panic("Error doing the memory database walk"); } - // FIXME: add event for converting env_data structure to a DTB trigger_rootvm_init_event(root_partition, root_thread, root_cspace, - env_data); - -#if !defined(ROOTVM_IS_HLOS) || !ROOTVM_IS_HLOS - // Copy the boot_env_data to the root VM memory - paddr_t rootvm_env_phys = env_data->env_ipa - env_data->me_ipa_base + - PLATFORM_ROOTVM_LMA_BASE; - void *va = partition_phys_map(rootvm_env_phys, - util_balign_up(env_data_size, - PGTABLE_VM_PAGE_SIZE)); + &hyp_env, qcbor_enc_ctxt); + + QCBOREncode_CloseMap(qcbor_enc_ctxt); + { + qcbor_err_t cb_err; + const_useful_buff_t payload_out_buff; + + payload_out_buff.ptr = NULL; + payload_out_buff.len = 0; + + cb_err = QCBOREncode_Finish(qcbor_enc_ctxt, &payload_out_buff); + + if (cb_err != QCBOR_SUCCESS) { + panic("Env data encoding error, increase the buffer size"); + } + + rm_env_data->data_payload_size = (uint32_t)payload_out_buff.len; + } + + crt_env->runtime_ipa = hyp_env.runtime_ipa; + crt_env->app_ipa = hyp_env.app_ipa; + crt_env->app_heap_ipa = hyp_env.app_heap_ipa; + crt_env->app_heap_size = hyp_env.app_heap_size; + crt_env->timer_freq = hyp_env.timer_freq; + crt_env->gicd_base = hyp_env.gicd_base; + crt_env->gicr_base = hyp_env.gicr_base; + + // Copy the rm_env_data to the root VM memory + paddr_t hyp_env_phys = hyp_env.env_ipa - hyp_env.me_ipa_base + + PLATFORM_ROOTVM_LMA_BASE; + assert(util_is_baligned(hyp_env_phys, PGTABLE_VM_PAGE_SIZE)); + + void *va = partition_phys_map(hyp_env_phys, env_data_size); partition_phys_access_enable(va); - memcpy(va, (void *)env_data, env_data_size); - CACHE_CLEAN_RANGE((boot_env_data_t *)va, env_data_size); + (void)memcpy(va, (void *)crt_env, + (rm_env_data->data_payload_size + sizeof(*rm_env_data) + + sizeof(*crt_env))); + CACHE_CLEAN_RANGE((rm_env_data_hdr_t *)va, + (rm_env_data->data_payload_size + + sizeof(*rm_env_data) + sizeof(*crt_env))); partition_phys_access_disable(va); - partition_phys_unmap(va, rootvm_env_phys, - util_balign_up(env_data_size, - PGTABLE_VM_PAGE_SIZE)); -#endif + partition_phys_unmap(va, hyp_env_phys, env_data_size); // Setup the root VM thread if (object_activate_thread(root_thread) != OK) { panic("Error activating root thread"); } - scheduler_lock(root_thread); -#if defined(ROOTVM_IS_HLOS) && ROOTVM_IS_HLOS - // FIXME: add a platform interface for configuring root thread - vcpu_poweron(root_thread, env_data->entry_hlos, 0U); -#else - // FIXME: eventually pass as dtb, for now the boot_env_data ipa is passed + trigger_rootvm_init_late_event(root_partition, root_thread, root_cspace, + &hyp_env); + + scheduler_lock_nopreempt(root_thread); + // FIXME: eventually pass as dtb, for now the rm_env_data ipa is passed // directly. - vcpu_poweron(root_thread, env_data->entry_ipa, env_data->env_ipa); -#endif - scheduler_unlock(root_thread); - partition_free(root_partition, env_data, env_data_size); - env_data = NULL; + bool_result_t power_ret = + vcpu_poweron(root_thread, vmaddr_result_ok(hyp_env.entry_ipa), + register_result_ok(hyp_env.env_ipa)); + if (power_ret.e != OK) { + panic("Error vcpu poweron"); + } + + // Allow other modules to clean up after root VM creation. + trigger_rootvm_started_event(root_thread); + scheduler_unlock_nopreempt(root_thread); + (void)partition_free(root_partition, crt_env, env_data_size); + rm_env_data = NULL; return; diff --git a/hyp/vm/rootvm_package/rootvm_package.ev b/hyp/vm/rootvm_package/rootvm_package.ev index 97182b8..dd1dc34 100644 --- a/hyp/vm/rootvm_package/rootvm_package.ev +++ b/hyp/vm/rootvm_package/rootvm_package.ev @@ -4,4 +4,5 @@ module rootvm_package -subscribe rootvm_init(root_partition, root_thread, root_cspace, env_data) +subscribe rootvm_init(root_partition, root_thread, root_cspace, hyp_env, qcbor_enc_ctxt) + require_preempt_disabled diff --git a/hyp/vm/rootvm_package/rootvm_package.tc b/hyp/vm/rootvm_package/rootvm_package.tc index 6bd7c6b..1a7d2a4 100644 --- a/hyp/vm/rootvm_package/rootvm_package.tc +++ b/hyp/vm/rootvm_package/rootvm_package.tc @@ -23,19 +23,11 @@ define rootvm_package_header structure { list array(ROOTVM_PACKAGE_ITEMS_MAX) structure rootvm_package_entry; }; - -extend boot_env_data structure { - me_capid type cap_id_t; +extend hyp_env_data structure { me_ipa_base type vmaddr_t; - me_size size; - ipa_offset uintptr; app_ipa type vmaddr_t; runtime_ipa type vmaddr_t; - - usable_cores uint64; - boot_core type cpu_index_t; - app_heap_ipa type vmaddr_t; app_heap_size size; }; diff --git a/hyp/vm/rootvm_package/src/package.c b/hyp/vm/rootvm_package/src/package.c index f1722e9..2fb7b9e 100644 --- a/hyp/vm/rootvm_package/src/package.c +++ b/hyp/vm/rootvm_package/src/package.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -93,8 +94,7 @@ rootvm_package_load_elf(void *elf, size_t elf_max_size, addrspace_t *addrspace, size_t offset = phys_offset - PLATFORM_ROOTVM_LMA_BASE; paddr_t range_start = PLATFORM_ROOTVM_LMA_BASE; - paddr_t range_end = - (paddr_t)PLATFORM_ROOTVM_LMA_BASE + PLATFORM_ROOTVM_LMA_SIZE; + paddr_t range_end = PLATFORM_ROOTVM_LMA_BASE + PLATFORM_ROOTVM_LMA_SIZE; for (i = 0; i < elf_get_num_phdrs(elf); i++) { Elf_Phdr *phdr = elf_get_phdr(elf, i); @@ -123,10 +123,12 @@ rootvm_package_load_elf(void *elf, size_t elf_max_size, addrspace_t *addrspace, pgtable_access_t access = PGTABLE_ACCESS_R; assert((phdr->p_flags & PF_R) != 0U); if ((phdr->p_flags & PF_W) != 0U) { - access |= PGTABLE_ACCESS_W; + access = pgtable_access_combine(access, + PGTABLE_ACCESS_W); } if ((phdr->p_flags & PF_X) != 0U) { - access |= PGTABLE_ACCESS_X; + access = pgtable_access_combine(access, + PGTABLE_ACCESS_X); } // Derive extents from RM memory extent @@ -171,16 +173,25 @@ rootvm_package_load_elf(void *elf, size_t elf_max_size, addrspace_t *addrspace, } static void -update_cores_info(boot_env_data_t *env_data) +update_cores_info(qcbor_enc_ctxt_t *qcbor_enc_ctxt) REQUIRE_PREEMPT_DISABLED { - env_data->boot_core = cpulocal_get_index(); - assert(PLATFORM_MAX_CORES > env_data->boot_core); + cpu_index_t boot_core; + uint64_t usable_cores; - env_data->usable_cores = PLATFORM_USABLE_CORES; - assert((env_data->usable_cores & (1UL << env_data->boot_core)) != 0); + assert(qcbor_enc_ctxt != NULL); - index_t max_idx = ((sizeof(env_data->usable_cores) * 8U) - 1U) - - compiler_clz(env_data->usable_cores); + boot_core = cpulocal_get_index(); + assert(PLATFORM_MAX_CORES > boot_core); + + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "boot_core", boot_core); + + usable_cores = PLATFORM_USABLE_CORES; + assert((usable_cores & util_bit(boot_core)) != 0); + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "usable_cores", + usable_cores); + + index_t max_idx = (index_t)((sizeof(usable_cores) * 8U) - 1U) - + compiler_clz(usable_cores); // can be a static assertion assert(max_idx < PLATFORM_MAX_CORES); } @@ -188,10 +199,13 @@ update_cores_info(boot_env_data_t *env_data) void rootvm_package_handle_rootvm_init(partition_t *root_partition, thread_t *root_thread, cspace_t *root_cspace, - boot_env_data_t *env_data) + hyp_env_data_t *hyp_env, + qcbor_enc_ctxt_t *qcbor_enc_ctxt) { error_t ret; + assert(qcbor_enc_ctxt != NULL); + assert(root_partition != NULL); assert(root_thread != NULL); assert(root_cspace != NULL); @@ -217,7 +231,7 @@ rootvm_package_handle_rootvm_init(partition_t *root_partition, panic("Invalid pkg_hdr"); } - paddr_t load_base = (paddr_t)PLATFORM_ROOTVM_LMA_BASE; + paddr_t load_base = PLATFORM_ROOTVM_LMA_BASE; paddr_t load_next = load_base; // Create memory extent for the RM with randomized base @@ -231,9 +245,10 @@ rootvm_package_handle_rootvm_init(partition_t *root_partition, #endif #if 0 + // FIXME: // Root VM address space could be smaller // Currently limit usable address space to 1GiB - vmaddr_t addr_limit = (vmaddr_t)1 << 30; + vmaddr_t addr_limit = (vmaddr_t)util_bit(30); vmaddr_t ipa = (vmaddr_t)rand % (addr_limit - PLATFORM_ROOTVM_LMA_SIZE - PGTABLE_VM_PAGE_SIZE); @@ -282,10 +297,10 @@ rootvm_package_handle_rootvm_init(partition_t *root_partition, if (t == ROOTVM_PACKAGE_IMAGE_TYPE_RUNTIME) { runtime_ipa = ipa + offset; - if (env_data->entry_ipa != 0U) { + if (hyp_env->entry_ipa != 0U) { panic("Multiple RootVM runtime images"); } - env_data->entry_ipa = + hyp_env->entry_ipa = elf_get_entry(elf) + runtime_ipa; } else { app_ipa = ipa + offset; @@ -315,33 +330,44 @@ rootvm_package_handle_rootvm_init(partition_t *root_partition, panic("Error mapping to root VM address space"); } + assert(util_is_baligned(offset, PGTABLE_VM_PAGE_SIZE)); + vmaddr_t env_data_ipa = ipa + offset; - size_t env_data_size = - util_balign_up(sizeof(boot_env_data_t), PGTABLE_VM_PAGE_SIZE); - offset += env_data_size; + offset += util_balign_up(hyp_env->env_data_size, PGTABLE_VM_PAGE_SIZE); vmaddr_t app_heap_ipa = ipa + offset; - size_t app_heap_size = util_balign_down( - PLATFORM_ROOTVM_LMA_SIZE - offset, PGTABLE_VM_PAGE_SIZE); + size_t app_heap_size = PLATFORM_ROOTVM_LMA_SIZE - offset; + + // The C runtime expects the heap to be page aligned. + assert(util_is_baligned(app_heap_ipa, PGTABLE_VM_PAGE_SIZE)); + assert(util_is_baligned(app_heap_size, PGTABLE_VM_PAGE_SIZE)); - // Add info of the memory left in RM to boot_env_data, so that it can be + // Add info of the memory left in RM to hyp_env_data, so that it can be // later used for the boot info structure for example. - env_data->me_capid = me_cap; - env_data->me_ipa_base = ipa; - env_data->me_size = PLATFORM_ROOTVM_LMA_SIZE; - env_data->env_ipa = env_data_ipa; + hyp_env->me_ipa_base = ipa; + hyp_env->env_ipa = env_data_ipa; + + hyp_env->app_ipa = app_ipa; + hyp_env->runtime_ipa = runtime_ipa; + hyp_env->ipa_offset = ipa - PLATFORM_ROOTVM_LMA_BASE; + hyp_env->app_heap_ipa = app_heap_ipa; + hyp_env->app_heap_size = app_heap_size; - env_data->app_ipa = app_ipa; - env_data->runtime_ipa = runtime_ipa; - env_data->ipa_offset = ipa - PLATFORM_ROOTVM_LMA_BASE; + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "me_ipa_base", + hyp_env->me_ipa_base); + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "ipa_offset", + hyp_env->ipa_offset); - env_data->app_heap_ipa = app_heap_ipa; - env_data->app_heap_size = app_heap_size; + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "me_capid", me_cap); + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "me_size", + PLATFORM_ROOTVM_LMA_SIZE); - update_cores_info(env_data); + update_cores_info(qcbor_enc_ctxt); LOG(DEBUG, INFO, "runtime_ipa: {:#x}", runtime_ipa); LOG(DEBUG, INFO, "app_ipa: {:#x}", app_ipa); + LOG(DEBUG, INFO, "env_data_ipa: {:#x}", env_data_ipa); + LOG(DEBUG, INFO, "app_heap_ipa: {:#x}", app_heap_ipa); ret = hyp_aspace_unmap_direct(map_base, map_size); assert(ret == OK); diff --git a/hyp/vm/smccc/aarch64/src/smccc_64.c b/hyp/vm/smccc/aarch64/src/smccc_64.c index 8dcef9a..7984f36 100644 --- a/hyp/vm/smccc/aarch64/src/smccc_64.c +++ b/hyp/vm/smccc/aarch64/src/smccc_64.c @@ -9,18 +9,34 @@ #include #include "event_handlers.h" +#include "smccc_hypercall.h" static bool smccc_handle_call(bool is_hvc) EXCLUDE_PREEMPT_DISABLED { bool handled; - thread_t *current = thread_get_self(); + thread_t *current = thread_get_self(); smccc_function_id_t function_id = smccc_function_id_cast((uint32_t)current->vcpu_regs_gpr.x[0]); uint32_t res0 = smccc_function_id_get_res0(&function_id); if (res0 != 0U) { - handled = false; + current->vcpu_regs_gpr.x[0] = + (register_t)SMCCC_UNKNOWN_FUNCTION64; + handled = true; + goto out; + } + + // TODO: the smccc handling below needs to be refactored, to permit + // registering ranges of service IDs, rather than registering + // individual calls directly. The current approach allows for unknown + // call IDs to be unhandled and fallthrough to a later module, which is + // undesirable. + // + // For SMCCC based hypercalls, we need function ID range-base handling, + // so its currently called directly here. + if (smccc_handle_hypercall_wrapper(function_id, is_hvc)) { + handled = true; goto out; } diff --git a/hyp/vm/smccc/aarch64/templates/hyp_wrapper.c.tmpl b/hyp/vm/smccc/aarch64/templates/hyp_wrapper.c.tmpl new file mode 100644 index 0000000..1dcb7e2 --- /dev/null +++ b/hyp/vm/smccc/aarch64/templates/hyp_wrapper.c.tmpl @@ -0,0 +1,121 @@ + +#def prefix: hypercall_ +#set $wrapper_suffix = "__hyp_wrapper" + +\#include +\#include + +\#include +\#include +\#include +\#include + +#def return_type(hypcall) +#if len(hypcall.outputs) > 1 +${prefix}${hypcall.name}_result_t#slurp +#else +${hypcall.outputs[0][1].ctype}#slurp +#end if +#end def + +#def register_expr(variable) +#if variable.category == 'bitfield' +## FIXME: this is an implementation detail of the type system +${variable.name}.bf[0]#slurp +#else +${variable.name}#slurp +#end if +#end def + +#def input_cast(variable, val) +#if variable.category == 'bitfield' +## FIXME: this is an implementation detail of the type system +${variable.type_name}_cast((uint${8 * variable.size}_t)${val})#slurp +#else +(${variable.ctype})${val}#slurp +#end if +#end def + +#for hypcall_num in sorted($hypcall_dict.keys()) +#set $hypcall = $hypcall_dict[$hypcall_num] + +#set $num_in = len($hypcall.inputs) +#set $num_out = len($hypcall.outputs) + +#if $num_in > 8 or $num_out > 8 +#error too many hypcall arguments: ${hypcall.name}: input $num_in, output $num_out +#end if + +static void +${hypcall.name}${wrapper_suffix}(register_t *args) { + #if $hypcall.outputs + $return_type($hypcall) ret_; + #end if + + ## call the implementation + #if $hypcall.outputs + ret_ = + #end if + $prefix${hypcall.name}(#slurp + #set xar=1 + #set sep='' + #for i, input in $hypcall.inputs + #if not $input.ignore +#set $val = "args[{:d}]".format($xar) + ${sep}$input_cast(input, $val) + #set sep=', ' + #end if + #set xar=xar+1 + #end for + ); + + ## return the result, if any + #if len(hypcall.outputs) > 1 + #set xar=0 + #for i, output in hypcall.outputs + ## + ## assuming complex struct, we only return the first + ## 'register_t'. + ## + args[$xar] = (register_t)(ret_.$register_expr($output)); + #set xar=xar+1 + #end for + #else if hypcall.outputs + args[0] = (register_t)ret_; + #end if +} + +#end for + +void +smccc_hypercall_table_wrapper(count_t hyp_num, register_t *args); + +void +smccc_hypercall_table_wrapper(count_t hyp_num, register_t *args) { + TRACE(USER, HYPERCALL, "smccc hyp: {:#x}: {:#x} {:#x} {:#x}, {:#x}", + (register_t)(hyp_num), args[1], args[2], args[3], args[4]); + bool trace_ret = true; + + switch (hyp_num) { +#for hypcall_num in sorted($hypcall_dict.keys()) +#set $hypcall = $hypcall_dict[$hypcall_num] +#set $sensitive = $hypcall.properties.get('sensitive', False) + case $hypcall_num: +#if $sensitive + // Sensitive hypercall + trace_ret = false; +#end if + ${hypcall.name}${wrapper_suffix}(args); + break; +#end for + default: + args[0] = (register_t)SMCCC_UNKNOWN_FUNCTION64; + break; + } + + if (compiler_expected(trace_ret)) { + TRACE(USER, HYPERCALL, + "smccc ret: {:#x} {:#x} {:#x} {:#x} {:#x}", + args[0], args[1], args[2], args[3], args[4]); + } +} diff --git a/hyp/vm/smccc/build.conf b/hyp/vm/smccc/build.conf index 8d5c0ec..ea1a060 100644 --- a/hyp/vm/smccc/build.conf +++ b/hyp/vm/smccc/build.conf @@ -4,6 +4,8 @@ interface smccc events smccc.ev -source smccc.c +local_include +source smccc.c smccc_hypercalls.c arch_events aarch64 smccc_64.ev arch_source aarch64 smccc_64.c +arch_template hypercalls aarch64 hyp_wrapper.c diff --git a/hyp/vm/smccc/include/smccc_hypercall.h b/hyp/vm/smccc/include/smccc_hypercall.h new file mode 100644 index 0000000..cd1f868 --- /dev/null +++ b/hyp/vm/smccc/include/smccc_hypercall.h @@ -0,0 +1,9 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +void +smccc_hypercall_table_wrapper(count_t hyp_num, register_t args[7]); + +bool +smccc_handle_hypercall_wrapper(smccc_function_id_t smc_id, bool is_hvc); diff --git a/hyp/vm/smccc/src/smccc.c b/hyp/vm/smccc/src/smccc.c index 4538ac0..d9e6e35 100644 --- a/hyp/vm/smccc/src/smccc.c +++ b/hyp/vm/smccc/src/smccc.c @@ -28,9 +28,11 @@ smccc_arch_features(uint32_t arg1, uint32_t *ret0) smccc_function_id_get_is_fast(&fn_id) && (smccc_function_id_get_res0(&fn_id) == 0U)) { if (is_smc64) { - ret = trigger_smccc_arch_features_fast64_event(fn); + ret = trigger_smccc_arch_features_fast64_event( + (smccc_arch_function_t)fn); } else { - ret = trigger_smccc_arch_features_fast32_event(fn); + ret = trigger_smccc_arch_features_fast32_event( + (smccc_arch_function_t)fn); } } else if ((smccc_function_id_get_interface_id(&fn_id) == SMCCC_INTERFACE_ID_STANDARD_HYP) && @@ -38,10 +40,10 @@ smccc_arch_features(uint32_t arg1, uint32_t *ret0) (smccc_function_id_get_res0(&fn_id) == 0U)) { if (is_smc64) { ret = trigger_smccc_standard_hyp_features_fast64_event( - fn); + (smccc_standard_hyp_function_t)fn); } else { ret = trigger_smccc_standard_hyp_features_fast32_event( - fn); + (smccc_standard_hyp_function_t)fn); } } else { ret = SMCCC_UNKNOWN_FUNCTION32; @@ -55,12 +57,10 @@ bool smccc_std_hyp_call_uid(uint32_t *ret0, uint32_t *ret1, uint32_t *ret2, uint32_t *ret3) { - // uuidgen -s -n @url -N "qualcomm.com/gunyah/smccc/standard_hyp" - // UUID: c1d58fcd-a453-5fdb-9265-ce36673d5f14 - *ret0 = 0x673d5f14U; - *ret1 = 0x9265ce36U; - *ret2 = 0xa4535fdbU; - *ret3 = 0xc1d58fcdU; + *ret0 = (uint32_t)SMCCC_GUNYAH_UID0; + *ret1 = (uint32_t)SMCCC_GUNYAH_UID1; + *ret2 = (uint32_t)SMCCC_GUNYAH_UID2; + *ret3 = (uint32_t)SMCCC_GUNYAH_UID3; return true; } diff --git a/hyp/vm/smccc/src/smccc_hypercalls.c b/hyp/vm/smccc/src/smccc_hypercalls.c new file mode 100644 index 0000000..ea3ea82 --- /dev/null +++ b/hyp/vm/smccc/src/smccc_hypercalls.c @@ -0,0 +1,87 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include + +#include +#include + +#include + +#include "smccc_hypercall.h" + +bool +smccc_handle_hypercall_wrapper(smccc_function_id_t smc_id, bool is_hvc) +{ + bool handled; + + smccc_function_t smc_func = smccc_function_id_get_function(&smc_id); + smccc_interface_id_t smc_iface = + smccc_function_id_get_interface_id(&smc_id); + + if (smc_iface != SMCCC_INTERFACE_ID_VENDOR_HYP) { + handled = false; + goto out; + } + + bool is_smc64 = smccc_function_id_get_is_smc64(&smc_id); + bool is_fast = smccc_function_id_get_is_fast(&smc_id); + + thread_t *current = thread_get_self(); + register_t *args = ¤t->vcpu_regs_gpr.x[0]; + + smccc_vendor_hyp_function_id_t smc_type = + smccc_vendor_hyp_function_id_cast((uint16_t)(smc_func)); + + switch (smccc_vendor_hyp_function_id_get_call_class(&smc_type)) { + case SMCCC_VENDOR_HYP_FUNCTION_CLASS_PLATFORM_CALL: + handled = smccc_handle_smc_platform_call(args, is_hvc); + break; + case SMCCC_VENDOR_HYP_FUNCTION_CLASS_HYPERCALL: + if (is_fast && is_smc64) { + uint32_t hyp_num = + smccc_vendor_hyp_function_id_get_function( + &smc_type); + smccc_hypercall_table_wrapper(hyp_num, args); + } else { + args[0] = (register_t)SMCCC_UNKNOWN_FUNCTION64; + } + handled = true; + break; + case SMCCC_VENDOR_HYP_FUNCTION_CLASS_SERVICE: + if (is_fast && !is_smc64) { + uint16_t func = + smccc_vendor_hyp_function_id_get_function( + &smc_type); + switch ((smccc_vendor_hyp_function_t)func) { + case SMCCC_VENDOR_HYP_FUNCTION_CALL_UID: + args[0] = SMCCC_GUNYAH_UID0; + args[1] = SMCCC_GUNYAH_UID1; + args[2] = SMCCC_GUNYAH_UID2; + args[3] = SMCCC_GUNYAH_UID3; + break; + case SMCCC_VENDOR_HYP_FUNCTION_REVISION: + args[0] = (register_t)hyp_api_info_raw( + hyp_api_info_default()); + break; + case SMCCC_VENDOR_HYP_FUNCTION_CALL_COUNT: + // Deprecated + default: + args[0] = (register_t)SMCCC_UNKNOWN_FUNCTION64; + break; + } + } else { + args[0] = (register_t)SMCCC_UNKNOWN_FUNCTION64; + } + handled = true; + break; + default: + args[0] = (register_t)SMCCC_UNKNOWN_FUNCTION64; + handled = true; + break; + } + +out: + return handled; +} diff --git a/hyp/vm/vcpu/aarch64/hypercalls.hvc b/hyp/vm/vcpu/aarch64/hypercalls.hvc index 8aecd64..5292653 100644 --- a/hyp/vm/vcpu/aarch64/hypercalls.hvc +++ b/hyp/vm/vcpu/aarch64/hypercalls.hvc @@ -6,7 +6,17 @@ define vcpu_configure hypercall { call_num 0x34; cap_id input type cap_id_t; vcpu_options input bitfield vcpu_option_flags; - _res0 input uregister; + res0 input uregister; + error output enumeration error; +}; + +define vcpu_register_write hypercall { + call_num 0x64; + vcpu input type cap_id_t; + register_set input enumeration vcpu_register_set; + register_index input type index_t; + value input uregister; + res0 input uregister; error output enumeration error; }; @@ -14,7 +24,7 @@ define vcpu_set_affinity hypercall { call_num 0x3d; cap_id input type cap_id_t; affinity input type cpu_index_t; - _res1 input uregister; + res1 input uregister; error output enumeration error; }; @@ -23,21 +33,21 @@ define vcpu_poweron hypercall { cap_id input type cap_id_t; entry_point input uregister; context input uregister; - _res0 input uregister; + flags input bitfield vcpu_poweron_flags; error output enumeration error; }; define vcpu_poweroff hypercall { call_num 0x39; cap_id input type cap_id_t; - _res0 input uregister; + flags input bitfield vcpu_poweroff_flags; error output enumeration error; }; define vcpu_kill hypercall { call_num 0x3a; cap_id input type cap_id_t; - _res0 input uregister; + res0 input uregister; error output enumeration error; }; @@ -54,3 +64,21 @@ define vcpu_set_timeslice hypercall { timeslice input type nanoseconds_t; error output enumeration error; }; + +define vcpu_bind_virq hypercall { + call_num 0x5c; + vcpu input type cap_id_t; + vic input type cap_id_t; + virq input type virq_t; + virq_type input enumeration vcpu_virq_type; + res0 input uregister; + error output enumeration error; +}; + +define vcpu_unbind_virq hypercall { + call_num 0x5d; + vcpu input type cap_id_t; + virq_type input enumeration vcpu_virq_type; + res0 input uregister; + error output enumeration error; +}; diff --git a/hyp/vm/vcpu/aarch64/include/exception_dispatch.h b/hyp/vm/vcpu/aarch64/include/exception_dispatch.h index 2d091e2..ad97a4a 100644 --- a/hyp/vm/vcpu/aarch64/include/exception_dispatch.h +++ b/hyp/vm/vcpu/aarch64/include/exception_dispatch.h @@ -7,3 +7,6 @@ vcpu_interrupt_dispatch(void) REQUIRE_PREEMPT_DISABLED; void vcpu_exception_dispatch(bool is_aarch64) REQUIRE_PREEMPT_DISABLED; + +void +vcpu_error_dispatch(void) REQUIRE_PREEMPT_DISABLED; diff --git a/hyp/vm/vcpu/aarch64/include/reg_access.h b/hyp/vm/vcpu/aarch64/include/reg_access.h new file mode 100644 index 0000000..c6a13b2 --- /dev/null +++ b/hyp/vm/vcpu/aarch64/include/reg_access.h @@ -0,0 +1,7 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +error_t +vcpu_register_write(thread_t *vcpu, vcpu_register_set_t register_set, + index_t register_index, register_t value); diff --git a/hyp/vm/vcpu/aarch64/include/vectors_vcpu.h b/hyp/vm/vcpu/aarch64/include/vectors_vcpu.h new file mode 100644 index 0000000..330b906 --- /dev/null +++ b/hyp/vm/vcpu/aarch64/include/vectors_vcpu.h @@ -0,0 +1,5 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +CPULOCAL_DECLARE_EXTERN(uintptr_t, vcpu_aarch64_vectors); diff --git a/hyp/vm/vcpu/aarch64/src/aarch64_init.c b/hyp/vm/vcpu/aarch64/src/aarch64_init.c index d1bdc4c..fc355a9 100644 --- a/hyp/vm/vcpu/aarch64/src/aarch64_init.c +++ b/hyp/vm/vcpu/aarch64/src/aarch64_init.c @@ -9,10 +9,14 @@ #include #include +#include +#include #include #include #include +#include #include +#include #include #include @@ -22,11 +26,20 @@ #include "event_handlers.h" +#if defined(ARCH_ARM_HAVE_SCXT) +#include +#include +#include + +static bool scxt_disabled = false; +static _Atomic uint64_t scxt_count; +#endif + void vcpu_handle_boot_runtime_init(void) { // Disable floating-point traps -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) CPTR_EL2_E2H1_t cptr = register_CPTR_EL2_E2H1_read_ordered(&asm_ordering); CPTR_EL2_E2H1_set_FPEN(&cptr, 3); @@ -42,7 +55,7 @@ vcpu_handle_boot_runtime_init(void) void vcpu_handle_boot_cpu_warm_init(void) { -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) CONTEXTIDR_EL2_t ctxidr = CONTEXTIDR_EL2_default(); register_CONTEXTIDR_EL2_write(ctxidr); #endif @@ -56,8 +69,32 @@ vcpu_handle_boot_cpu_warm_init(void) // it is implementation defined, so zero this register. HSTR_EL2_t hstr = HSTR_EL2_cast(0U); register_HSTR_EL2_write(hstr); +#if defined(ARCH_ARM_HAVE_SCXT) + if (!scxt_disabled) { + register_SCXTNUM_EL2_write(atomic_fetch_add_explicit( + &scxt_count, 1U, memory_order_relaxed)); + } +#endif } +#if defined(ARCH_ARM_HAVE_SCXT) +void +vcpu_handle_boot_cold_init(void) +{ + platform_cpu_features_t features = platform_get_cpu_features(); + + scxt_disabled = platform_cpu_features_get_scxt_disable(&features); + + if (!scxt_disabled) { + uint64_result_t rand_r = prng_get64(); + assert(rand_r.e == OK); + atomic_store_relaxed(&scxt_count, rand_r.r); + } else { + LOG(ERROR, WARN, "platform SCXTNUM_ELx access disabled!"); + } +} +#endif + static void arch_vcpu_el1_registers_init(thread_t *vcpu) { @@ -71,7 +108,7 @@ arch_vcpu_el1_registers_init(thread_t *vcpu) static void arch_vcpu_el2_registers_init(vcpu_el2_registers_t *el2_regs) { -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) CPTR_EL2_E2H1_init(&el2_regs->cptr_el2); CPTR_EL2_E2H1_set_FPEN(&el2_regs->cptr_el2, 3); #else @@ -124,37 +161,37 @@ arch_vcpu_el2_registers_init(vcpu_el2_registers_t *el2_regs) // attributes. HCR_EL2_set_MIOCNCE(&el2_regs->hcr_el2, true); -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) HCR_EL2_set_E2H(&el2_regs->hcr_el2, true); #endif HCR_EL2_set_TGE(&el2_regs->hcr_el2, false); -#if defined(ARCH_ARM_8_1_LOR) +#if defined(ARCH_ARM_FEAT_LOR) // FIXME: we could temporarily set TLOR to false if we encounter Linux // using these registers HCR_EL2_set_TLOR(&el2_regs->hcr_el2, true); #endif -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) HCR_EL2_set_APK(&el2_regs->hcr_el2, true); HCR_EL2_set_API(&el2_regs->hcr_el2, true); #endif -#if defined(ARCH_ARM_8_3_NV) - HCR_EL2_set_AT(&el2_regs->hcr_el2, true); -#endif - -#if defined(ARCH_ARM_8_4_NV) +#if defined(ARCH_ARM_FEAT_NV) + HCR_EL2_set_AT(&el2_regs->hcr_el2, false); HCR_EL2_set_NV(&el2_regs->hcr_el2, false); HCR_EL2_set_NV1(&el2_regs->hcr_el2, false); +#endif + +#if defined(ARCH_ARM_FEAT_NV2) HCR_EL2_set_NV2(&el2_regs->hcr_el2, false); #endif -#if defined(ARCH_ARM_8_4_S2FWB) +#if defined(ARCH_ARM_FEAT_S2FWB) HCR_EL2_set_FWB(&el2_regs->hcr_el2, false); #endif -#if defined(ARCH_ARM_8_4_RAS) +#if defined(ARCH_ARM_FEAT_RASv1p1) HCR_EL2_set_FIEN(&el2_regs->hcr_el2, false); #endif @@ -171,11 +208,11 @@ arch_vcpu_el2_registers_init(vcpu_el2_registers_t *el2_regs) MDCR_EL2_set_TPM(&el2_regs->mdcr_el2, true); MDCR_EL2_set_TPMCR(&el2_regs->mdcr_el2, true); #endif -#if defined(ARCH_ARM_SPE) +#if defined(ARCH_ARM_FEAT_SPEv1p1) // Enable SPE traps by default MDCR_EL2_set_TPMS(&el2_regs->mdcr_el2, true); #endif -#if defined(ARCH_ARM_8_4_TRACE) +#if defined(ARCH_ARM_FEAT_TRF) // Enable trace traps by default MDCR_EL2_set_TTRF(&el2_regs->mdcr_el2, true); #endif @@ -186,7 +223,6 @@ arch_vcpu_el2_registers_init(vcpu_el2_registers_t *el2_regs) void vcpu_handle_rootvm_init(thread_t *root_thread) { -#if !defined(ROOTVM_IS_HLOS) || !ROOTVM_IS_HLOS vcpu_el2_registers_t *el2_regs = &root_thread->vcpu_regs_el2; // Run the root VM with HCR.DC set, so we don't need a stg-1 page-table @@ -194,9 +230,6 @@ vcpu_handle_rootvm_init(thread_t *root_thread) // Note however we don't support switching off HCR.DC yet! HCR_EL2_set_DC(&el2_regs->hcr_el2, true); HCR_EL2_set_TVM(&el2_regs->hcr_el2, true); -#else - (void)root_thread; -#endif } error_t @@ -210,11 +243,17 @@ vcpu_arch_handle_object_create_thread(thread_create_t thread_create) // Set up nonzero init values for EL2 registers arch_vcpu_el2_registers_init(&thread->vcpu_regs_el2); - // Indicate that the VCPU is uniprocessor by default. The PSCI - // module will override this if the VCPU is attached to a PSCI - // group. + // Indicate that the VCPU is uniprocessor by default. The vgic + // module will override this if the VCPU is attached to a VIC. thread->vcpu_regs_mpidr_el1 = MPIDR_EL1_default(); MPIDR_EL1_set_U(&thread->vcpu_regs_mpidr_el1, true); + +#if defined(ARCH_ARM_HAVE_SCXT) + if (!scxt_disabled) { + vcpu_option_flags_set_scxt_allowed( + &thread->vcpu_options, true); + } +#endif } return err; @@ -261,24 +300,26 @@ noreturn void vcpu_exception_return(uintptr_t unused_param); static noreturn void -vcpu_thread_start(uintptr_t unused_param) EXCLUDE_PREEMPT_DISABLED +vcpu_thread_start(bool warm_reset) EXCLUDE_PREEMPT_DISABLED { - (void)unused_param; - trigger_vcpu_started_event(); - - thread_t *vcpu = thread_get_self(); - vcpu->vcpu_warm_reset = false; - + trigger_vcpu_started_event(warm_reset); trigger_thread_exit_to_user_event(THREAD_ENTRY_REASON_NONE); thread_reset_stack(vcpu_exception_return, 0U); } +static noreturn void +vcpu_thread_entry(uintptr_t unused_param) EXCLUDE_PREEMPT_DISABLED +{ + (void)unused_param; + vcpu_thread_start(false); +} + thread_func_t vcpu_handle_thread_get_entry_fn(void) { assert(thread_get_self()->kind == THREAD_KIND_VCPU); - return vcpu_thread_start; + return vcpu_thread_entry; } error_t @@ -294,52 +335,111 @@ vcpu_configure(thread_t *thread, vcpu_option_flags_t vcpu_options) return ret; } -bool -vcpu_poweron(thread_t *vcpu, paddr_t entry_point, register_t context) +static void +vcpu_reset_execution_context(thread_t *vcpu) { + assert((vcpu != NULL) && (vcpu->kind == THREAD_KIND_VCPU)); + assert((thread_get_self() == vcpu) || + scheduler_is_blocked(vcpu, SCHEDULER_BLOCK_VCPU_OFF)); + + // Reset the EL1 registers. + arch_vcpu_el1_registers_init(vcpu); + + // Reset the EL1 processor state: EL1H mode, all interrupts disabled. + SPSR_EL2_A64_t spsr_el2 = SPSR_EL2_A64_default(); + SPSR_EL2_A64_set_M(&spsr_el2, SPSR_64BIT_MODE_EL1H); + SPSR_EL2_A64_set_D(&spsr_el2, true); + SPSR_EL2_A64_set_A(&spsr_el2, true); + SPSR_EL2_A64_set_I(&spsr_el2, true); + SPSR_EL2_A64_set_F(&spsr_el2, true); + vcpu->vcpu_regs_gpr.spsr_el2.a64 = spsr_el2; +} + +bool_result_t +vcpu_poweron(thread_t *vcpu, vmaddr_result_t entry_point, + register_result_t context) +{ + error_t err = OK; + bool ret = false; + assert(vcpu != NULL); assert(vcpu->kind == THREAD_KIND_VCPU); assert(scheduler_is_blocked(vcpu, SCHEDULER_BLOCK_VCPU_OFF)); - trigger_vcpu_poweron_event(vcpu); + err = trigger_vcpu_poweron_event(vcpu); + if (err == OK) { + vcpu_reset_execution_context(vcpu); + if (entry_point.e == OK) { + vcpu->vcpu_regs_gpr.pc = ELR_EL2_cast(entry_point.r); + } + if (context.e == OK) { + vcpu->vcpu_regs_gpr.x[0] = context.r; + } - vcpu->vcpu_regs_gpr.pc = ELR_EL2_cast(entry_point); - vcpu->vcpu_regs_gpr.x[0] = context; + // We must have a valid address space and stage 2 must be + // enabled. Otherwise the guest can trivially take over the + // hypervisor. + assert(HCR_EL2_get_VM(&vcpu->vcpu_regs_el2.hcr_el2) && + (VTTBR_EL2_get_BADDR( + &vcpu->addrspace->vm_pgtable.vttbr_el2) != 0U)); - // We must have a valid address space and stage 2 must be enabled. - // Otherwise the guest can trivially take over the hypervisor. - assert(HCR_EL2_get_VM(&vcpu->vcpu_regs_el2.hcr_el2) && - (VTTBR_EL2_get_BADDR(&vcpu->addrspace->vm_pgtable.vttbr_el2) != - 0U)); + ret = scheduler_unblock(vcpu, SCHEDULER_BLOCK_VCPU_OFF); + } - return scheduler_unblock(vcpu, SCHEDULER_BLOCK_VCPU_OFF); + return (bool_result_t){ .e = err, .r = ret }; } error_t -vcpu_poweroff(void) +vcpu_poweroff(bool last_cpu, bool force) { thread_t *current = thread_get_self(); assert(current->kind == THREAD_KIND_VCPU); scheduler_lock(current); - error_t ret = trigger_vcpu_poweroff_event(current, false); - if (ret != OK) { - scheduler_unlock(current); - return ret; + error_t ret = trigger_vcpu_poweroff_event(current, last_cpu, force); + if (ret == OK) { + scheduler_block(current, SCHEDULER_BLOCK_VCPU_OFF); + scheduler_unlock_nopreempt(current); + + if (force) { + preempt_enable(); + vcpu_halted(); + // not reached + } else { + trigger_vcpu_stopped_event(); + scheduler_yield(); + + // If we get here, then someone has called + // vcpu_poweron() on us. + preempt_enable(); + vcpu_thread_start(false); + // not reached + } } - scheduler_block(current, SCHEDULER_BLOCK_VCPU_OFF); scheduler_unlock(current); + return ret; +} - trigger_vcpu_poweredoff_event(); - - scheduler_yield(); - - // If we get here, then someone has called vcpu_poweron() on us. - vcpu_thread_start(0U); - // not reached +#if defined(MODULE_VM_VCPU_RUN) +vcpu_run_state_t +vcpu_handle_vcpu_run_check(const thread_t *vcpu, register_t *state_data_0) +{ + vcpu_run_state_t ret = VCPU_RUN_STATE_BLOCKED; + if (scheduler_is_blocked(vcpu, SCHEDULER_BLOCK_VCPU_FAULT)) { + ret = VCPU_RUN_STATE_FAULT; + } else if (scheduler_is_blocked(vcpu, SCHEDULER_BLOCK_VCPU_OFF)) { + vcpu_run_poweroff_flags_t flags = + vcpu_run_poweroff_flags_default(); + ret = VCPU_RUN_STATE_POWERED_OFF; + *state_data_0 = vcpu_run_poweroff_flags_raw(flags); + } else { + // Nothing to do + } + return ret; } +#endif error_t vcpu_suspend(void) @@ -353,23 +453,20 @@ vcpu_suspend(void) // vcpu_wakeup_self(), but we want that function to be fast. preempt_disable(); - scheduler_lock_nopreempt(current); if (vcpu_pending_wakeup()) { ret = ERROR_BUSY; } else { - ret = trigger_vcpu_suspend_event(); + ret = trigger_vcpu_suspend_event(current); } + if (ret == OK) { + scheduler_lock_nopreempt(current); scheduler_block(current, SCHEDULER_BLOCK_VCPU_SUSPEND); - } - scheduler_unlock_nopreempt(current); + scheduler_unlock_nopreempt(current); - if (ret == OK) { scheduler_yield(); - scheduler_lock_nopreempt(current); - trigger_vcpu_resume_event(); - scheduler_unlock_nopreempt(current); + trigger_vcpu_resume_event(current); } preempt_enable(); @@ -400,31 +497,30 @@ vcpu_warm_reset(paddr_t entry_point, register_t context) trigger_vcpu_warm_reset_event(vcpu); // Set the thread's startup context + vcpu_reset_execution_context(vcpu); vcpu->vcpu_regs_gpr.pc = ELR_EL2_cast(entry_point); vcpu->vcpu_regs_gpr.x[0] = context; - vcpu->vcpu_warm_reset = true; - // We've been warm-reset; jump directly to the entry point. - vcpu_thread_start(0U); + vcpu_thread_start(true); } -void -vcpu_reset_execution_context(thread_t *vcpu) +noreturn void +vcpu_halted(void) { - assert((vcpu != NULL) && (vcpu->kind == THREAD_KIND_VCPU)); - assert((thread_get_self() == vcpu) || - scheduler_is_blocked(vcpu, SCHEDULER_BLOCK_VCPU_OFF)); + thread_t *current = thread_get_self(); - // Reset the EL1 registers. - arch_vcpu_el1_registers_init(vcpu); + assert(current->kind == THREAD_KIND_VCPU); - // Reset the EL1 processor state: EL1H mode, all interrupts disabled. - SPSR_EL2_A64_t spsr = SPSR_EL2_A64_default(); - SPSR_EL2_A64_set_M(&spsr, SPSR_64BIT_MODE_EL1H); - SPSR_EL2_A64_set_D(&spsr, true); - SPSR_EL2_A64_set_A(&spsr, true); - SPSR_EL2_A64_set_I(&spsr, true); - SPSR_EL2_A64_set_F(&spsr, true); - vcpu->vcpu_regs_gpr.spsr_el2 = spsr; + preempt_disable(); + + trigger_vcpu_stopped_event(); + + (void)virq_assert(¤t->vcpu_halt_virq_src, true); + + scheduler_yield(); + + // If we get here, then someone resumed the halted vcpu. + preempt_enable(); + vcpu_thread_start(false); } diff --git a/hyp/vm/vcpu/aarch64/src/context_switch.c b/hyp/vm/vcpu/aarch64/src/context_switch.c index ef8f3c6..347369d 100644 --- a/hyp/vm/vcpu/aarch64/src/context_switch.c +++ b/hyp/vm/vcpu/aarch64/src/context_switch.c @@ -7,19 +7,32 @@ #include +#include +#include #include #include +#include #include #include "event_handlers.h" +#include "vectors_vcpu.h" +// VCPU_TRACE_CONTEXT_SAVED and VCPU_DEBUG_CONTEXT_SAVED defines are used as +// sanity checks to test whether the configuration is correct and we don't leak +// trace and debug context registers between VMs, or permit tracing the +// hypervisor. +#if defined(MODULE_VM_VETM) || defined(MODULE_VM_VETM_NULL) +#define VCPU_TRACE_CONTEXT_SAVED 1 +#elif defined(PLATFORM_ETM_BASE) && \ + (defined(MODULE_PLATFORM_SOC_QCOM) && defined(MODULE_PLATFORM_QHEE)) +#define VCPU_TRACE_CONTEXT_SAVED 1 +#elif defined(PLATFORM_HAS_NO_ETM_BASE) && (PLATFORM_HAS_NO_ETM_BASE != 0) #pragma message( \ - "PLATFORM_HAS_NO_ETM_BASE is defined, if an ETM is present it may be" \ + "PLATFORM_HAS_NO_ETM_BASE is nonzero; if an ETM is present it may be" \ " accessible and could trace the hypervisor.") #define VCPU_TRACE_CONTEXT_SAVED 1 - -extern uintptr_t vcpu_aarch64_vectors; +#endif void vcpu_context_switch_load(void) @@ -29,13 +42,13 @@ vcpu_context_switch_load(void) thread_t *thread = thread_get_self(); -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) CONTEXTIDR_EL2_t ctxidr = CONTEXTIDR_EL2_default(); CONTEXTIDR_EL2_set_PROCID(&ctxidr, (uint32_t)(uintptr_t)thread); register_CONTEXTIDR_EL2_write(ctxidr); #endif - if (thread->kind == THREAD_KIND_VCPU) { + if (compiler_expected(thread->kind == THREAD_KIND_VCPU)) { register_CPACR_EL1_write(thread->vcpu_regs_el1.cpacr_el1); register_CSSELR_EL1_write(thread->vcpu_regs_el1.csselr_el1); register_CONTEXTIDR_EL1_write( @@ -60,33 +73,55 @@ vcpu_context_switch_load(void) #if SCHEDULER_CAN_MIGRATE register_VPIDR_EL2_write(thread->vcpu_regs_midr_el1); #endif +#if !defined(CPU_HAS_NO_ACTLR_EL1) + register_ACTLR_EL1_write(thread->vcpu_regs_el1.actlr_el1); +#endif +#if !defined(CPU_HAS_NO_AMAIR_EL1) + register_AMAIR_EL1_write(thread->vcpu_regs_el1.amair_el1); +#endif +#if !defined(CPU_HAS_NO_AFSR0_EL1) + register_AFSR0_EL1_write(thread->vcpu_regs_el1.afsr0_el1); +#endif +#if !defined(CPU_HAS_NO_AFSR1_EL1) + register_AFSR1_EL1_write(thread->vcpu_regs_el1.afsr1_el1); +#endif // Floating-point access should not be disabled for any VM -#if defined(ARCH_ARM_8_1_VHE) - assert(CPTR_EL2_E2H1_get_FPEN( - &thread->vcpu_regs_el2.cptr_el2) == 3); +#if defined(ARCH_ARM_FEAT_VHE) + assert_debug(CPTR_EL2_E2H1_get_FPEN( + &thread->vcpu_regs_el2.cptr_el2) == 3); register_CPTR_EL2_E2H1_write(thread->vcpu_regs_el2.cptr_el2); #else - assert(CPTR_EL2_E2H0_get_TFP(&thread->vcpu_regs_el2.cptr_el2) == - 0); + assert_debug(CPTR_EL2_E2H0_get_TFP( + &thread->vcpu_regs_el2.cptr_el2) == 0); register_CPTR_EL2_E2H0_write(thread->vcpu_regs_el2.cptr_el2); #endif -#if defined(ARCH_ARM_8_1_VHE) - assert(HCR_EL2_get_E2H(&thread->vcpu_regs_el2.hcr_el2) == true); - assert(HCR_EL2_get_TGE(&thread->vcpu_regs_el2.hcr_el2) == 0); +#if defined(VERBOSE) && VERBOSE +#if defined(ARCH_ARM_FEAT_VHE) + assert_debug(HCR_EL2_get_E2H(&thread->vcpu_regs_el2.hcr_el2)); + assert_debug(!HCR_EL2_get_TGE(&thread->vcpu_regs_el2.hcr_el2)); +#endif + assert_debug(HCR_EL2_get_VM(&thread->vcpu_regs_el2.hcr_el2)); #endif register_HCR_EL2_write(thread->vcpu_regs_el2.hcr_el2); + register_MDCR_EL2_write(thread->vcpu_regs_el2.mdcr_el2); - // FIXME: check disabled until Stage-2 is enabled - // assert(HCR_EL2_get_VM(&thread->vcpu_regs_el2.hcr_el2) == 1); register_VBAR_EL2_write( - VBAR_EL2_cast((uint64_t)&vcpu_aarch64_vectors)); + VBAR_EL2_cast(CPULOCAL(vcpu_aarch64_vectors))); register_FPCR_write(thread->vcpu_regs_fpr.fpcr); register_FPSR_write(thread->vcpu_regs_fpr.fpsr); +#if defined(ARCH_ARM_HAVE_SCXT) + if (vcpu_option_flags_get_scxt_allowed(&thread->vcpu_options)) { + register_SCXTNUM_EL0_write( + thread->vcpu_regs_el1.scxtnum_el0); + register_SCXTNUM_EL1_write( + thread->vcpu_regs_el1.scxtnum_el1); + } +#endif __asm__ volatile("ldp q0, q1, [%[q]] ;" "ldp q2, q3, [%[q], 32] ;" "ldp q4, q5, [%[q], 64] ;" @@ -112,7 +147,7 @@ vcpu_context_switch_load(void) HCR_EL2_set_FMO(&nonvm_hcr, true); HCR_EL2_set_IMO(&nonvm_hcr, true); HCR_EL2_set_AMO(&nonvm_hcr, true); -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) HCR_EL2_set_E2H(&nonvm_hcr, true); #endif HCR_EL2_set_TGE(&nonvm_hcr, true); @@ -125,8 +160,9 @@ vcpu_context_switch_save(void) { thread_t *thread = thread_get_self(); - if (thread->kind == THREAD_KIND_VCPU && - !scheduler_is_blocked(thread, SCHEDULER_BLOCK_VCPU_OFF)) { + if (compiler_expected( + (thread->kind == THREAD_KIND_VCPU) && + !scheduler_is_blocked(thread, SCHEDULER_BLOCK_VCPU_OFF))) { thread->vcpu_regs_el1.cpacr_el1 = register_CPACR_EL1_read(); thread->vcpu_regs_el1.csselr_el1 = register_CSSELR_EL1_read(); thread->vcpu_regs_el1.contextidr_el1 = @@ -148,9 +184,17 @@ vcpu_context_switch_save(void) thread->vcpu_regs_el1.ttbr0_el1 = register_TTBR0_EL1_read(); thread->vcpu_regs_el1.ttbr1_el1 = register_TTBR1_EL1_read(); thread->vcpu_regs_el1.vbar_el1 = register_VBAR_EL1_read(); - thread->vcpu_regs_mpidr_el1 = register_VMPIDR_EL2_read(); -#if SCHEDULER_CAN_MIGRATE - thread->vcpu_regs_midr_el1 = register_VPIDR_EL2_read(); +#if !defined(CPU_HAS_NO_ACTLR_EL1) + thread->vcpu_regs_el1.actlr_el1 = register_ACTLR_EL1_read(); +#endif +#if !defined(CPU_HAS_NO_AMAIR_EL1) + thread->vcpu_regs_el1.amair_el1 = register_AMAIR_EL1_read(); +#endif +#if !defined(CPU_HAS_NO_AFSR0_EL1) + thread->vcpu_regs_el1.afsr0_el1 = register_AFSR0_EL1_read(); +#endif +#if !defined(CPU_HAS_NO_AFSR1_EL1) + thread->vcpu_regs_el1.afsr1_el1 = register_AFSR1_EL1_read(); #endif // Read back HCR_EL2 as VSE may have been cleared. @@ -158,6 +202,14 @@ vcpu_context_switch_save(void) thread->vcpu_regs_fpr.fpcr = register_FPCR_read(); thread->vcpu_regs_fpr.fpsr = register_FPSR_read(); +#if defined(ARCH_ARM_HAVE_SCXT) + if (vcpu_option_flags_get_scxt_allowed(&thread->vcpu_options)) { + thread->vcpu_regs_el1.scxtnum_el0 = + register_SCXTNUM_EL0_read(); + thread->vcpu_regs_el1.scxtnum_el1 = + register_SCXTNUM_EL1_read(); + } +#endif __asm__ volatile("stp q0, q1, [%[q]] ;" "stp q2, q3, [%[q], 32] ;" "stp q4, q5, [%[q], 64] ;" @@ -176,5 +228,16 @@ vcpu_context_switch_save(void) "stp q30, q31, [%[q], 480] ;" : "=m"(thread->vcpu_regs_fpr) : [q] "r"(thread->vcpu_regs_fpr.q)); + +#if SCHEDULER_CAN_MIGRATE + if (!vcpu_option_flags_get_pinned(&thread->vcpu_options)) { + // We need a DSB to ensure that any cache or TLB op + // executed by the VCPU in EL1 is complete before the + // VCPU potentially migrates. Otherwise the VCPU may + // execute its own DSB on the wrong CPU, and proceed + // before the maintenance operation completes. + __asm__ volatile("dsb ish" ::"m"(asm_ordering)); + } +#endif } } diff --git a/hyp/vm/vcpu/aarch64/src/exception_inject.c b/hyp/vm/vcpu/aarch64/src/exception_inject.c index 9e0a110..5fc3bf3 100644 --- a/hyp/vm/vcpu/aarch64/src/exception_inject.c +++ b/hyp/vm/vcpu/aarch64/src/exception_inject.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "event_handlers.h" #include "exception_inject.h" @@ -27,18 +28,17 @@ static void exception_inject(ESR_EL1_t esr_el1) { - VBAR_EL1_t vbar; - ELR_EL2_t elr_el2; - SPSR_EL2_A64_t spsr_el2; - thread_t *thread = thread_get_self(); - register_t guest_vector; + VBAR_EL1_t vbar; + ELR_EL2_t elr_el2; + thread_t *thread = thread_get_self(); + register_t guest_vector; vbar = register_VBAR_EL1_read(); guest_vector = VBAR_EL1_get_VectorBase(&vbar); - spsr_el2 = thread->vcpu_regs_gpr.spsr_el2; - + SPSR_EL2_A64_t spsr_el2 = thread->vcpu_regs_gpr.spsr_el2.a64; // Adjust the vector based on the mode we came from + // FIXME: spsr_64bit_mode_t spsr_m = SPSR_EL2_A64_get_M(&spsr_el2); switch (spsr_m) { case SPSR_64BIT_MODE_EL0T: @@ -60,7 +60,7 @@ exception_inject(ESR_EL1_t esr_el1) default: // Either illegal M, or exceptions coming from 32-bit EL0/EL1 // For now we only support 32-bit EL0 - if (spsr_m == SPSR_32BIT_MODE_USER) { + if (spsr_m == (spsr_64bit_mode_t)SPSR_32BIT_MODE_USER) { // Exception from 32-bit EL0 User guest_vector += 0x600U; break; @@ -81,27 +81,27 @@ exception_inject(ESR_EL1_t esr_el1) SPSR_EL2_A64_set_IL(&spsr_el2, false); SPSR_EL2_A64_set_SS(&spsr_el2, false); SPSR_EL2_A64_set_M(&spsr_el2, SPSR_64BIT_MODE_EL1H); -#if defined(ARCH_ARM_8_0_SSBS) || defined(ARCH_ARM_8_1_PAN) +#if defined(ARCH_ARM_FEAT_SSBS) || defined(ARCH_ARM_FEAT_PAN) SCTLR_EL1_t sctlr_el1 = register_SCTLR_EL1_read(); -#if defined(ARCH_ARM_8_0_SSBS) +#if defined(ARCH_ARM_FEAT_SSBS) SPSR_EL2_A64_set_SSBS(&spsr_el2, SCTLR_EL1_get_DSSBS(&sctlr_el1)); #endif -#if defined(ARCH_ARM_8_1_PAN) +#if defined(ARCH_ARM_FEAT_PAN) if (!SCTLR_EL1_get_SPAN(&sctlr_el1)) { SPSR_EL2_A64_set_PAN(&spsr_el2, true); } #endif #endif -#if defined(ARCH_ARM_8_2_UAO) +#if defined(ARCH_ARM_FEAT_UAO) SPSR_EL2_A64_set_UAO(&spsr_el2, false); #endif -#if defined(ARCH_ARM_8_5_MEMTAG) +#if defined(ARCH_ARM_FEAT_MTE) SPSR_EL2_A64_set_TCO(&spsr_el2, true); #endif -#if defined(ARCH_ARM_8_5_BTI) +#if defined(ARCH_ARM_FEAT_BTI) SPSR_EL2_A64_set_BTYPE(&spsr_el2, 0); #endif - thread->vcpu_regs_gpr.spsr_el2 = spsr_el2; + thread->vcpu_regs_gpr.spsr_el2.a64 = spsr_el2; // Tell the guest where the exception came from elr_el2 = thread->vcpu_regs_gpr.pc; @@ -119,9 +119,10 @@ bool inject_inst_data_abort(ESR_EL2_t esr_el2, esr_ec_t ec, iss_da_ia_fsc_t fsc, FAR_EL2_t far, vmaddr_t ipa, bool is_data_abort) { - thread_t *thread = thread_get_self(); - SPSR_EL2_A64_t spsr = thread->vcpu_regs_gpr.spsr_el2; - ESR_EL1_t esr_el1 = ESR_EL1_cast(ESR_EL2_raw(esr_el2)); + thread_t *thread = thread_get_self(); + ESR_EL1_t esr_el1 = ESR_EL1_cast(ESR_EL2_raw(esr_el2)); + SPSR_EL2_A64_t spsr_el2 = thread->vcpu_regs_gpr.spsr_el2.a64; + gvaddr_t va = FAR_EL2_get_VirtualAddress(&far); assert(thread->kind == THREAD_KIND_VCPU); assert(thread->addrspace != NULL); @@ -147,7 +148,7 @@ inject_inst_data_abort(ESR_EL2_t esr_el2, esr_ec_t ec, iss_da_ia_fsc_t fsc, #if !defined(NDEBUG) // Injecting an abort from the guest EL1H sync vector will // cause an exception inject loop, so block the vcpu instead. - if (SPSR_EL2_A64_get_M(&spsr) == SPSR_64BIT_MODE_EL1H) { + if (SPSR_EL2_A64_get_M(&spsr_el2) == SPSR_64BIT_MODE_EL1H) { VBAR_EL1_t vbar = register_VBAR_EL1_read(); uint64_t pc = ELR_EL2_get_ReturnAddress( &thread->vcpu_regs_gpr.pc); @@ -158,23 +159,22 @@ inject_inst_data_abort(ESR_EL2_t esr_el2, esr_ec_t ec, iss_da_ia_fsc_t fsc, TRACE_AND_LOG( DEBUG, DEBUG, "Detected exception inject loop from " - "VM {:d}, original ESR_EL2 = {:#x}, " - "ELR_EL2 = {:#x}, VBAR_EL1 = {:#x}", + "VM {:d}, previous ESR_EL1 = {:#x}, " + "ELR_EL1 = {:#x}, VBAR_EL1 = {:#x}", VTTBR_EL2_get_VMID(&vttbr), - ESR_EL2_raw(esr_el2), - ELR_EL2_raw(thread->vcpu_regs_gpr.pc), + ESR_EL1_raw(register_ESR_EL1_read()), + ELR_EL1_raw(register_ELR_EL1_read()), VBAR_EL1_raw(vbar)); scheduler_lock(thread); scheduler_block(thread, SCHEDULER_BLOCK_VCPU_FAULT); scheduler_unlock(thread); - scheduler_yield(); - break; + vcpu_halted(); } } #endif // Inject a synchronous external abort - spsr_64bit_mode_t mode = SPSR_EL2_A64_get_M(&spsr); + spsr_64bit_mode_t mode = SPSR_EL2_A64_get_M(&spsr_el2); if ((mode == SPSR_64BIT_MODE_EL1T) || (mode == SPSR_64BIT_MODE_EL1H)) { if (is_data_abort) { @@ -214,8 +214,6 @@ inject_inst_data_abort(ESR_EL2_t esr_el2, esr_ec_t ec, iss_da_ia_fsc_t fsc, register_FAR_EL1_write(FAR_EL1_cast(FAR_EL2_raw(far))); - gvaddr_t va = FAR_EL2_get_VirtualAddress(&far); - TRACE_AND_LOG(ERROR, WARN, "Injecting instruction/data abort to VM {:d}, " "original ESR_EL2 = {:#x}, fault VA = {:#x}, " @@ -242,7 +240,7 @@ inject_inst_data_abort(ESR_EL2_t esr_el2, esr_ec_t ec, iss_da_ia_fsc_t fsc, case ISS_DA_IA_FSC_SYNC_PARITY_ECC_WALK_3: case ISS_DA_IA_FSC_SYNC_TAG_CHECK: case ISS_DA_IA_FSC_TLB_CONFLICT: -#if defined(ARCH_ARM_8_1_TTHM) +#if defined(ARCH_ARM_FEAT_HAFDBS) case ISS_DA_IA_FSC_ATOMIC_HW_UPDATE: #endif case ISS_DA_IA_FSC_PAGE_DOMAIN: @@ -251,8 +249,6 @@ inject_inst_data_abort(ESR_EL2_t esr_el2, esr_ec_t ec, iss_da_ia_fsc_t fsc, case ISS_DA_IA_FSC_IMP_DEF_LOCKDOWN: case ISS_DA_IA_FSC_IMP_DEF_ATOMIC: default: { - gvaddr_t va = FAR_EL2_get_VirtualAddress(&far); - TRACE_AND_LOG(ERROR, WARN, "instruction/data abort from VM {:d}, " "ESR_EL2 = {:#x}, fault VA = {:#x}, " @@ -268,10 +264,19 @@ inject_inst_data_abort(ESR_EL2_t esr_el2, esr_ec_t ec, iss_da_ia_fsc_t fsc, // - IMPLEMENTATION DEFINED fault // Also the following have already been checked by the caller: // - TLB conflict - // - Unsupported atomic hardware update file (ARMv8.1-TTHM) + // - Unsupported atomic hardware update file (ARM_FEAT_HAFDBS) - abort("Unhandled instruction/data abort", - ABORT_REASON_UNHANDLED_EXCEPTION); + if (vcpu_option_flags_get_critical(&thread->vcpu_options)) { + abort("Unhandled instruction/data abort", + ABORT_REASON_UNHANDLED_EXCEPTION); + } else { + // The above faults cannot be triggered by the VM, so + // halt the VCPU without revealing any fault state. + scheduler_lock(thread); + scheduler_block(thread, SCHEDULER_BLOCK_VCPU_FAULT); + scheduler_unlock(thread); + vcpu_halted(); + } } } diff --git a/hyp/vm/vcpu/aarch64/src/hypercalls.c b/hyp/vm/vcpu/aarch64/src/hypercalls.c index 7550bdb..16247f7 100644 --- a/hyp/vm/vcpu/aarch64/src/hypercalls.c +++ b/hyp/vm/vcpu/aarch64/src/hypercalls.c @@ -15,12 +15,15 @@ #include #include #include +#include #include #include #include +#include #include #include "event_handlers.h" +#include "reg_access.h" // This hypercall should be called before the vCPU is activated. It copies the // provided flags into a variable called vcpu_options in the thread structure. @@ -41,7 +44,7 @@ hypercall_vcpu_configure(cap_id_t cap_id, vcpu_option_flags_t vcpu_options) goto out; } - cspace_t *cspace = cspace_get_self(); + cspace_t *cspace = cspace_get_self(); object_type_t type; object_ptr_result_t result = cspace_lookup_object_any( cspace, cap_id, CAP_RIGHTS_GENERIC_OBJECT_ACTIVATE, &type); @@ -77,18 +80,108 @@ hypercall_vcpu_configure(cap_id_t cap_id, vcpu_option_flags_t vcpu_options) } error_t -hypercall_vcpu_set_affinity(cap_id_t cap_id, cpu_index_t affinity) +hypercall_vcpu_register_write(cap_id_t vcpu_cap, + vcpu_register_set_t register_set, + index_t register_index, register_t value) { error_t ret; cspace_t *cspace = cspace_get_self(); - if (((1UL << affinity) & PLATFORM_USABLE_CORES) == 0) { + thread_ptr_result_t result = cspace_lookup_thread_any( + cspace, vcpu_cap, CAP_RIGHTS_THREAD_WRITE_CONTEXT); + if (compiler_unexpected(result.e != OK)) { + ret = result.e; + goto out; + } + + thread_t *vcpu = result.r; + + ret = vcpu_register_write(vcpu, register_set, register_index, value); + + object_put_thread(vcpu); +out: + return ret; +} + +error_t +hypercall_vcpu_bind_virq(cap_id_t vcpu_cap, cap_id_t vic_cap, virq_t virq, + vcpu_virq_type_t virq_type) +{ + error_t err; + cspace_t *cspace = cspace_get_self(); + + thread_ptr_result_t result = cspace_lookup_thread( + cspace, vcpu_cap, CAP_RIGHTS_THREAD_BIND_VIRQ); + if (compiler_unexpected(result.e != OK)) { + err = result.e; + goto out; + } + thread_t *vcpu = result.r; + + vic_ptr_result_t v = + cspace_lookup_vic(cspace, vic_cap, CAP_RIGHTS_VIC_BIND_SOURCE); + if (compiler_unexpected(v.e != OK)) { + err = v.e; + goto out_release_vcpu; + } + vic_t *vic = v.r; + + err = vcpu_bind_virq(vcpu, vic, virq, virq_type); + + object_put_vic(vic); +out_release_vcpu: + object_put_thread(vcpu); +out: + return err; +} + +error_t +hypercall_vcpu_unbind_virq(cap_id_t vcpu_cap, vcpu_virq_type_t virq_type) +{ + error_t err = OK; + cspace_t *cspace = cspace_get_self(); + + thread_ptr_result_t result = cspace_lookup_thread( + cspace, vcpu_cap, CAP_RIGHTS_THREAD_BIND_VIRQ); + if (compiler_unexpected(result.e != OK)) { + err = result.e; + goto out; + } + thread_t *vcpu = result.r; + + err = vcpu_unbind_virq(vcpu, virq_type); + + object_put_thread(vcpu); +out: + return err; +} + +error_t +hypercall_vcpu_set_affinity(cap_id_t cap_id, cpu_index_t affinity) +{ + error_t ret; + cspace_t *cspace = cspace_get_self(); + cap_rights_thread_t required_rights; + + if (affinity == CPU_INDEX_INVALID) { +#if SCHEDULER_CAN_MIGRATE + // Thread will become non-runnable + required_rights = cap_rights_thread_union( + CAP_RIGHTS_THREAD_AFFINITY, CAP_RIGHTS_THREAD_DISABLE); +#else + ret = ERROR_UNIMPLEMENTED; + goto out; +#endif + } else if (!platform_cpu_exists(affinity)) { ret = ERROR_ARGUMENT_INVALID; goto out; + } else { + // Affinity is valid + required_rights = CAP_RIGHTS_THREAD_AFFINITY; } - thread_ptr_result_t result = cspace_lookup_thread_any( - cspace, cap_id, CAP_RIGHTS_THREAD_AFFINITY); + thread_ptr_result_t result = + cspace_lookup_thread_any(cspace, cap_id, required_rights); if (compiler_unexpected(result.e != OK)) { ret = result.e; goto out; @@ -123,11 +216,17 @@ hypercall_vcpu_set_affinity(cap_id_t cap_id, cpu_index_t affinity) } error_t -hypercall_vcpu_poweron(cap_id_t cap_id, uint64_t entry_point, uint64_t context) +hypercall_vcpu_poweron(cap_id_t cap_id, uint64_t entry_point, uint64_t context, + vcpu_poweron_flags_t flags) { error_t ret = OK; cspace_t *cspace = cspace_get_self(); + if (!vcpu_poweron_flags_is_clean(flags)) { + ret = ERROR_ARGUMENT_INVALID; + goto out; + } + thread_ptr_result_t result = cspace_lookup_thread(cspace, cap_id, CAP_RIGHTS_THREAD_POWER); if (compiler_unexpected(result.e != OK)) { @@ -142,7 +241,19 @@ hypercall_vcpu_poweron(cap_id_t cap_id, uint64_t entry_point, uint64_t context) scheduler_lock(vcpu); if (scheduler_is_blocked(vcpu, SCHEDULER_BLOCK_VCPU_OFF)) { - reschedule = vcpu_poweron(vcpu, entry_point, context); + bool_result_t poweron_result = vcpu_poweron( + vcpu, + vcpu_poweron_flags_get_preserve_entry_point( + &flags) + ? vmaddr_result_error( + ERROR_ARGUMENT_INVALID) + : vmaddr_result_ok(entry_point), + vcpu_poweron_flags_get_preserve_context(&flags) + ? register_result_error( + ERROR_ARGUMENT_INVALID) + : register_result_ok(context)); + reschedule = poweron_result.r; + ret = poweron_result.e; } else { ret = ERROR_BUSY; } @@ -150,7 +261,7 @@ hypercall_vcpu_poweron(cap_id_t cap_id, uint64_t entry_point, uint64_t context) object_put_thread(vcpu); if (reschedule) { - scheduler_schedule(); + (void)scheduler_schedule(); } } else { ret = ERROR_ARGUMENT_INVALID; @@ -161,11 +272,16 @@ hypercall_vcpu_poweron(cap_id_t cap_id, uint64_t entry_point, uint64_t context) } error_t -hypercall_vcpu_poweroff(cap_id_t cap_id) +hypercall_vcpu_poweroff(cap_id_t cap_id, vcpu_poweroff_flags_t flags) { error_t ret = OK; cspace_t *cspace = cspace_get_self(); + if (!vcpu_poweroff_flags_is_clean(flags)) { + ret = ERROR_ARGUMENT_INVALID; + goto out; + } + thread_ptr_result_t result = cspace_lookup_thread(cspace, cap_id, CAP_RIGHTS_THREAD_POWER); if (compiler_unexpected(result.e != OK)) { @@ -177,8 +293,15 @@ hypercall_vcpu_poweroff(cap_id_t cap_id) if (compiler_expected(vcpu->kind == THREAD_KIND_VCPU) && (vcpu == thread_get_self())) { + // We can (and must) safely release our reference to the VCPU + // here, because we know it's the current thread so the + // scheduler will keep a reference to it. Since vcpu_poweroff() + // does not return, failing to release this reference will + // leave the thread as a zombie after it halts. object_put_thread(vcpu); - ret = vcpu_poweroff(); + + ret = vcpu_poweroff(vcpu_poweroff_flags_get_last_vcpu(&flags), + false); // It will not reach here if it succeeded } else { ret = ERROR_ARGUMENT_INVALID; @@ -210,6 +333,12 @@ hypercall_vcpu_set_priority(cap_id_t cap_id, priority_t priority) goto out; } + if (priority > VCPU_MAX_PRIORITY) { + ret = ERROR_DENIED; + object_put_thread(vcpu); + goto out; + } + spinlock_acquire(&vcpu->header.lock); object_state_t state = atomic_load_relaxed(&vcpu->header.state); if (state == OBJECT_STATE_INIT) { diff --git a/hyp/vm/vcpu/aarch64/src/reg_access.c b/hyp/vm/vcpu/aarch64/src/reg_access.c index 8fd8360..5d16b90 100644 --- a/hyp/vm/vcpu/aarch64/src/reg_access.c +++ b/hyp/vm/vcpu/aarch64/src/reg_access.c @@ -5,19 +5,24 @@ #include #include +#include #include #include +#include #include +#include #include +#include "reg_access.h" + register_t vcpu_gpr_read(thread_t *thread, uint8_t reg_num) { register_t value; - assert(reg_num <= 31); + assert(reg_num <= 31U); - if (compiler_expected(reg_num != 31)) { + if (compiler_expected(reg_num != 31U)) { value = thread->vcpu_regs_gpr.x[reg_num]; } else { value = 0; @@ -29,9 +34,79 @@ vcpu_gpr_read(thread_t *thread, uint8_t reg_num) void vcpu_gpr_write(thread_t *thread, uint8_t reg_num, register_t value) { - assert(reg_num <= 31); + assert(reg_num <= 31U); - if (compiler_expected(reg_num != 31)) { + if (compiler_expected(reg_num != 31U)) { thread->vcpu_regs_gpr.x[reg_num] = value; } } + +error_t +vcpu_register_write(thread_t *vcpu, vcpu_register_set_t register_set, + index_t register_index, register_t value) +{ + error_t err; + + if (compiler_expected(vcpu->kind != THREAD_KIND_VCPU)) { + err = ERROR_ARGUMENT_INVALID; + goto out; + } + + scheduler_lock(vcpu); + + thread_state_t state = atomic_load_relaxed(&vcpu->state); + if ((state != THREAD_STATE_INIT) && (state != THREAD_STATE_READY)) { + // Thread has been killed or exited + err = ERROR_OBJECT_STATE; + goto out_locked; + } + + if (!scheduler_is_blocked(vcpu, SCHEDULER_BLOCK_VCPU_OFF)) { + // Not safe to access registers of a runnable VCPU + err = ERROR_BUSY; + goto out_locked; + } + + switch (register_set) { + case VCPU_REGISTER_SET_X: + if (register_index < 31U) { + vcpu_gpr_write(vcpu, (uint8_t)register_index, value); + err = OK; + } else { + err = ERROR_ARGUMENT_INVALID; + } + break; + case VCPU_REGISTER_SET_PC: +#if ARCH_AARCH64_32BIT_EL1 +#error alignment check is not correct for AArch32 +#endif + if ((register_index == 0U) && util_is_baligned(value, 4U)) { + vcpu->vcpu_regs_gpr.pc = ELR_EL2_cast(value); + err = OK; + } else { + err = ERROR_ARGUMENT_INVALID; + } + break; + case VCPU_REGISTER_SET_SP_EL: + if (!util_is_baligned(value, 16U)) { + err = ERROR_ARGUMENT_INVALID; + } else if (register_index == 0U) { + vcpu->vcpu_regs_el1.sp_el0 = SP_EL0_cast(value); + err = OK; + } else if (register_index == 1U) { + vcpu->vcpu_regs_el1.sp_el1 = SP_EL1_cast(value); + err = OK; + } else { + err = ERROR_ARGUMENT_INVALID; + } + break; + default: + err = ERROR_ARGUMENT_INVALID; + break; + } + +out_locked: + scheduler_unlock(vcpu); +out: + return err; +} diff --git a/hyp/vm/vcpu/aarch64/src/sysreg_traps.c b/hyp/vm/vcpu/aarch64/src/sysreg_traps.c index 2d3d498..d02c67d 100644 --- a/hyp/vm/vcpu/aarch64/src/sysreg_traps.c +++ b/hyp/vm/vcpu/aarch64/src/sysreg_traps.c @@ -13,7 +13,10 @@ #include #include #include -#if defined(ARCH_ARM_8_5_MEMTAG) +#if defined(ARCH_ARM_FEAT_MPAM) +#include +#endif +#if defined(ARCH_ARM_FEAT_MTE) #include #endif @@ -22,13 +25,21 @@ #include "event_handlers.h" +static_assert((ARCH_AARCH64_32BIT_EL1 && ARCH_AARCH64_32BIT_EL0) || + !ARCH_AARCH64_32BIT_EL1, + "32BIT_EL1 implies 32BIT_EL0"); + +#if defined(ARCH_ARM_FEAT_SHA512) != defined(ARCH_ARM_FEAT_SHA3) +#error ARCH_ARM_FEAT_SHA512 and ARCH_ARM_FEAT_SHA3 mismatch +#endif + #if SCHEDULER_CAN_MIGRATE static bool read_virtual_id_register(ESR_EL2_ISS_MSR_MRS_t iss, uint8_t reg_num) { - register_t val = 0U; + register_t reg_val = 0U; bool handled = true; - thread_t *thread = thread_get_self(); + thread_t *thread = thread_get_self(); switch (ESR_EL2_ISS_MSR_MRS_raw(iss)) { // Trapped with HCR_EL2.TID1 @@ -40,9 +51,39 @@ read_virtual_id_register(ESR_EL2_ISS_MSR_MRS_t iss, uint8_t reg_num) break; // Trapped with HCR_EL2.TID2 // Trapped with HCR_EL2.TID3 + case ISS_MRS_MSR_MVFR0_EL1: + // TODO - it is possible that not all cores support the same + // features. For non-pinned vcpus, we return the HW MVFRx_EL1 + // values, which has potential to return incorrect values. If + // this becomes a problem, we need to define a subset ID value + // per machine. +#if ARCH_AARCH64_32BIT_EL0 && ARCH_AARCH64_32BIT_EL0_ALL_CORES + sysreg64_read(MVFR0_EL1, reg_val); +#else + reg_val = 0U; // Return defined as UNKNOWN +#endif + break; + case ISS_MRS_MSR_MVFR1_EL1: +#if ARCH_AARCH64_32BIT_EL0 && ARCH_AARCH64_32BIT_EL0_ALL_CORES + sysreg64_read(MVFR1_EL1, reg_val); +#else + reg_val = 0U; // Return defined as UNKNOWN +#endif + break; + case ISS_MRS_MSR_MVFR2_EL1: +#if ARCH_AARCH64_32BIT_EL0 && ARCH_AARCH64_32BIT_EL0_ALL_CORES + sysreg64_read(MVFR2_EL1, reg_val); +#else + reg_val = 0U; // Return defined as UNKNOWN +#endif + break; case ISS_MRS_MSR_ID_AA64PFR0_EL1: { ID_AA64PFR0_EL1_t pfr0 = ID_AA64PFR0_EL1_default(); +#if ARCH_AARCH64_32BIT_EL0 && ARCH_AARCH64_32BIT_EL0_ALL_CORES ID_AA64PFR0_EL1_set_EL0(&pfr0, 2U); +#else + ID_AA64PFR0_EL1_set_EL0(&pfr0, 1U); +#endif #if ARCH_AARCH64_32BIT_EL1 ID_AA64PFR0_EL1_set_EL1(&pfr0, 2U); #else @@ -50,7 +91,7 @@ read_virtual_id_register(ESR_EL2_ISS_MSR_MRS_t iss, uint8_t reg_num) #endif ID_AA64PFR0_EL1_set_EL2(&pfr0, 1U); ID_AA64PFR0_EL1_set_EL3(&pfr0, 1U); -#if defined(ARCH_ARM_8_2_FP16) +#if defined(ARCH_ARM_FEAT_FP16) ID_AA64PFR0_EL1_set_FP(&pfr0, 1U); ID_AA64PFR0_EL1_set_AdvSIMD(&pfr0, 1U); #endif @@ -58,250 +99,355 @@ read_virtual_id_register(ESR_EL2_ISS_MSR_MRS_t iss, uint8_t reg_num) if (vcpu_option_flags_get_ras_error_handler( &thread->vcpu_options)) { -#if defined(ARCH_ARM_8_4_RAS) +#if defined(ARCH_ARM_FEAT_RASv1p1) ID_AA64PFR0_EL1_set_RAS(&pfr0, 2); -#elif defined(ARCH_ARM_8_2_RAS) +#elif defined(ARCH_ARM_FEAT_RAS) ID_AA64PFR0_EL1_set_RAS(&pfr0, 1); #else // Nothing to do, the field is already 0 #endif } -#if defined(ARCH_ARM_8_4_SECEL2) +#if defined(ARCH_ARM_FEAT_SEL2) ID_AA64PFR0_EL1_set_SEL2(&pfr0, 1U); #endif -#if defined(ARCH_ARM_8_4_DIT) +#if defined(ARCH_ARM_FEAT_DIT) ID_AA64PFR0_EL1_set_DIT(&pfr0, 1U); #endif -#if defined(ARCH_ARM_8_0_CSV2) -#if defined(ARM_ARM_8_0_CSV2_SCXTNUM) - ID_AA64PFR0_EL1_set_CSV2(&pfr0, 2U); +#if defined(ARCH_ARM_HAVE_SCXT) && \ + (defined(ARCH_ARM_FEAT_CSV2_2) || defined(ARCH_ARM_FEAT_CSV2_3)) + if (vcpu_option_flags_get_scxt_allowed(&thread->vcpu_options)) { +#if defined(ARCH_ARM_FEAT_CSV2_3) + ID_AA64PFR0_EL1_set_CSV2(&pfr0, 3U); #else - ID_AA64PFR0_EL1_set_CSV2(&pfr0, 1U); + ID_AA64PFR0_EL1_set_CSV2(&pfr0, 2U); #endif + } else { + ID_AA64PFR0_EL1_set_CSV2(&pfr0, 1U); + } +#elif defined(ARCH_ARM_FEAT_CSV2) + ID_AA64PFR0_EL1_set_CSV2(&pfr0, 1U); #endif -#if defined(ARCH_ARM_8_0_CSV3) +#if defined(ARCH_ARM_FEAT_CSV3) ID_AA64PFR0_EL1_set_CSV3(&pfr0, 1U); #endif - val = ID_AA64PFR0_EL1_raw(pfr0); +#if defined(ARCH_ARM_FEAT_MPAM) + if (arm_mpam_is_allowed() && + vcpu_option_flags_get_mpam_allowed(&thread->vcpu_options)) { + ID_AA64PFR0_EL1_t hw_pfr0 = + register_ID_AA64PFR0_EL1_read(); + ID_AA64PFR0_EL1_copy_MPAM(&pfr0, &hw_pfr0); + } +#endif + reg_val = ID_AA64PFR0_EL1_raw(pfr0); break; } case ISS_MRS_MSR_ID_AA64PFR1_EL1: { ID_AA64PFR1_EL1_t pfr1 = ID_AA64PFR1_EL1_default(); -#if defined(ARCH_ARM_8_5_BTI) +#if defined(ARCH_ARM_FEAT_BTI) ID_AA64PFR1_EL1_set_BT(&pfr1, 1U); #endif -#if defined(ARCH_ARM_8_0_SSBS) -#if defined(ARCH_ARM_8_0_SSBS_MSR_MRS) +#if defined(ARCH_ARM_FEAT_SSBS) +#if defined(ARCH_ARM_FEAT_SSBS_MSR_MRS) ID_AA64PFR1_EL1_set_SSBS(&pfr1, 2U); #else ID_AA64PFR1_EL1_set_SSBS(&pfr1, 1U); #endif #endif -#if defined(ARCH_ARM_8_5_MEMTAG) +#if defined(ARCH_ARM_HAVE_SCXT) && defined(ARCH_ARM_FEAT_CSV2_1p2) + if (vcpu_option_flags_get_scxt_allowed(&thread->vcpu_options)) { + ID_AA64PFR1_EL1_set_CSV2_frac(&pfr0, 2U); + } else { + ID_AA64PFR1_EL1_set_CSV2_frac(&pfr0, 1U); + } +#elif defined(ARCH_ARM_FEAT_CSV2) +#if defined(ARCH_ARM_FEAT_CSV2_1p1) || defined(ARCH_ARM_FEAT_CSV2_1p2) + ID_AA64PFR1_EL1_set_CSV2_frac(&pfr0, 1U); +#endif +#endif +#if defined(ARCH_ARM_FEAT_MTE) if (arm_mte_is_allowed()) { ID_AA64PFR1_EL1_t hw_pfr1 = register_ID_AA64PFR1_EL1_read(); ID_AA64PFR1_EL1_copy_MTE(&pfr1, &hw_pfr1); } #endif - val = ID_AA64PFR1_EL1_raw(pfr1); +#if defined(ARCH_ARM_FEAT_MPAM) + if (arm_mpam_is_allowed() && + vcpu_option_flags_get_mpam_allowed(&thread->vcpu_options)) { + ID_AA64PFR1_EL1_t hw_pfr1 = + register_ID_AA64PFR1_EL1_read(); + ID_AA64PFR1_EL1_copy_MPAM_frac(&pfr1, &hw_pfr1); + } +#endif + reg_val = ID_AA64PFR1_EL1_raw(pfr1); break; } case ISS_MRS_MSR_ID_AA64ISAR0_EL1: { ID_AA64ISAR0_EL1_t isar0 = ID_AA64ISAR0_EL1_default(); -#if defined(ARCH_ARM_8_0_AES_PMULL) +#if defined(ARCH_ARM_FEAT_PMULL) ID_AA64ISAR0_EL1_set_AES(&isar0, 2U); -#elif defined(ARCH_ARM_8_0_AES) +#elif defined(ARCH_ARM_FEAT_AES) ID_AA64ISAR0_EL1_set_AES(&isar0, 1U); #endif -#if defined(ARCH_ARM_8_0_SHA) +#if defined(ARCH_ARM_FEAT_SHA1) ID_AA64ISAR0_EL1_set_SHA1(&isar0, 1U); #endif -#if defined(ARCH_ARM_8_2_SHA) +#if defined(ARCH_ARM_FEAT_SHA512) ID_AA64ISAR0_EL1_set_SHA2(&isar0, 2U); -#elif defined(ARCH_ARM_8_0_SHA) +#elif defined(ARCH_ARM_FEAT_SHA256) ID_AA64ISAR0_EL1_set_SHA2(&isar0, 1U); #endif +#if defined(ARCH_ARM_FEAT_CRC32) ID_AA64ISAR0_EL1_set_CRC32(&isar0, 1U); -#if defined(ARCH_ARM_8_1_VHE) +#endif +#if defined(ARCH_ARM_FEAT_VHE) ID_AA64ISAR0_EL1_set_Atomic(&isar0, 2U); #endif -#if defined(ARCH_ARM_8_1_RDMA) +#if defined(ARCH_ARM_FEAT_RDM) ID_AA64ISAR0_EL1_set_RDM(&isar0, 1U); #endif -#if defined(ARCH_ARM_8_2_SHA) +#if defined(ARCH_ARM_FEAT_SHA3) ID_AA64ISAR0_EL1_set_SHA3(&isar0, 1U); #endif -#if defined(ARCH_ARM_8_2_SM) +#if defined(ARCH_ARM_FEAT_SM3) ID_AA64ISAR0_EL1_set_SM3(&isar0, 1U); +#endif +#if defined(ARCH_ARM_FEAT_SM4) ID_AA64ISAR0_EL1_set_SM4(&isar0, 1U); #endif -#if defined(ARCH_ARM_8_2_DOTPROD) +#if defined(ARCH_ARM_FEAT_DotProd) ID_AA64ISAR0_EL1_set_DP(&isar0, 1U); #endif -#if defined(ARCH_ARM_8_2_FHM) +#if defined(ARCH_ARM_FEAT_FHM) ID_AA64ISAR0_EL1_set_FHM(&isar0, 1U); #endif -#if defined(ARCH_ARM_8_5_CONDM) +#if defined(ARCH_ARM_FEAT_FlagM2) ID_AA64ISAR0_EL1_set_TS(&isar0, 2U); -#elif defined(ARCH_ARM_8_4_CONDM) +#elif defined(ARCH_ARM_FEAT_FlagM) ID_AA64ISAR0_EL1_set_TS(&isar0, 1U); #endif -#if defined(ARCH_ARM_8_4_TLBI) +#if defined(ARCH_ARM_FEAT_TLBIRANGE) ID_AA64ISAR0_EL1_set_TLB(&isar0, 2U); +#elif defined(ARCH_ARM_FEAT_TLBIOS) + ID_AA64ISAR0_EL1_set_TLB(&isar0, 1U); #endif -#if defined(ARCH_ARM_8_5_RNG) +#if defined(ARCH_ARM_FEAT_RNG) ID_AA64ISAR0_EL1_set_RNDR(&isar0, 2U); #endif - val = ID_AA64ISAR0_EL1_raw(isar0); + reg_val = ID_AA64ISAR0_EL1_raw(isar0); break; } case ISS_MRS_MSR_ID_AA64ISAR1_EL1: { ID_AA64ISAR1_EL1_t isar1 = ID_AA64ISAR1_EL1_default(); ID_AA64ISAR1_EL1_t hw_isar1 = register_ID_AA64ISAR1_EL1_read(); (void)hw_isar1; -#if defined(ARCH_ARM_8_2_DCCVADP) +#if defined(ARCH_ARM_FEAT_DPB2) ID_AA64ISAR1_EL1_set_DPB(&isar1, 2U); -#elif defined(ARCH_ARM_8_2_DCPOP) +#elif defined(ARCH_ARM_FEAT_DPB) ID_AA64ISAR1_EL1_set_DPB(&isar1, 1U); #endif -#if defined(ARCH_ARM_8_3_JSCONV) +#if defined(ARCH_ARM_FEAT_JSCVT) ID_AA64ISAR1_EL1_set_JSCVT(&isar1, 1U); #endif -#if defined(ARCH_ARM_8_3_COMPNUM) +#if defined(ARCH_ARM_FEAT_FCMA) ID_AA64ISAR1_EL1_set_FCMA(&isar1, 1U); #endif -#if defined(ARCH_ARM_8_4_RCPC) +#if defined(ARCH_ARM_FEAT_LRCPC2) ID_AA64ISAR1_EL1_set_LRCPC(&isar1, 2U); -#elif defined(ARCH_ARM_8_3_RCPC) +#elif defined(ARCH_ARM_FEAT_LRCPC) ID_AA64ISAR1_EL1_set_LRCPC(&isar1, 1U); #endif -#if defined(ARCH_ARM_8_5_FRINT) +#if defined(ARCH_ARM_FEAT_FRINTTS) ID_AA64ISAR1_EL1_set_FRINTTS(&isar1, 1U); #endif -#if defined(ARCH_ARM_8_0_SB) +#if defined(ARCH_ARM_FEAT_SB) ID_AA64ISAR1_EL1_set_SB(&isar1, 1U); #endif -#if defined(ARCH_ARM_8_0_PREDINV) +#if defined(ARCH_ARM_FEAT_SPECRES) ID_AA64ISAR1_EL1_set_SPECRES(&isar1, 1U); #endif -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) ID_AA64ISAR1_EL1_copy_APA(&isar1, &hw_isar1); ID_AA64ISAR1_EL1_copy_API(&isar1, &hw_isar1); ID_AA64ISAR1_EL1_copy_GPA(&isar1, &hw_isar1); ID_AA64ISAR1_EL1_copy_GPI(&isar1, &hw_isar1); #endif -#if defined(ARCH_ARM_8_0_DGH) +#if defined(ARCH_ARM_FEAT_DGH) ID_AA64ISAR1_EL1_set_DGH(&isar1, 1U); #endif -#if defined(ARCH_ARM_8_2_BF16) +#if defined(ARCH_ARM_FEAT_BF16) ID_AA64ISAR1_EL1_set_BF16(&isar1, 1U); #endif -#if defined(ARCH_ARM_8_2_I8MM) +#if defined(ARCH_ARM_FEAT_I8MM) ID_AA64ISAR1_EL1_set_I8MM(&isar1, 1U); #endif - val = ID_AA64ISAR1_EL1_raw(isar1); + reg_val = ID_AA64ISAR1_EL1_raw(isar1); + break; + } + case ISS_MRS_MSR_ID_AA64ISAR2_EL1: { + ID_AA64ISAR2_EL1_t isar2 = ID_AA64ISAR2_EL1_default(); + ID_AA64ISAR2_EL1_t hw_isar2 = register_ID_AA64ISAR2_EL1_read(); + (void)hw_isar2; +#if defined(ARCH_ARM_FEAT_PAuth) + ID_AA64ISAR2_EL1_copy_APA3(&isar2, &hw_isar2); + ID_AA64ISAR2_EL1_copy_GPA3(&isar2, &hw_isar2); + ID_AA64ISAR2_EL1_copy_PAC_frac(&isar2, &hw_isar2); +#endif +#if defined(ARCH_ARM_FEAT_CLRBHB) + ID_AA64ISAR2_EL1_copy_CLRBHB(&isar2, &hw_isar2); +#endif +#if defined(ARCH_ARM_FEAT_WFxT) + // Copy the hardware values across once FEAT_WFxT is implemented + // FIXME: + // ID_AA64ISAR2_EL1_copy_WFxT(&isar2, &hw_isar2); +#endif + + reg_val = ID_AA64ISAR2_EL1_raw(isar2); break; } + case ISS_MRS_MSR_ID_AA64MMFR0_EL1: { ID_AA64MMFR0_EL1_t mmfr0 = ID_AA64MMFR0_EL1_default(); + // FIXME: match PLATFORM_VM_ADDRESS_SPACE_BITS ID_AA64MMFR0_EL1_set_PARange(&mmfr0, TCR_PS_SIZE_36BITS); #if defined(ARCH_AARCH64_ASID16) ID_AA64MMFR0_EL1_set_ASIDBits(&mmfr0, 2U); #endif - ID_AA64MMFR0_EL1_set_TGran16(&mmfr0, 1U); -#if defined(ARCH_ARM_8_5_CSEH) + ID_AA64MMFR0_EL1_set_SNSMem(&mmfr0, 1U); +#if defined(ARCH_AARCH64_BIG_END_ALL_CORES) && ARCH_AARCH64_BIG_END_ALL_CORES + ID_AA64MMFR0_EL1_set_BigEnd(&mmfr0, 1U); + ID_AA64MMFR0_EL1_set_BigEndEL0(&mmfr0, 0U); +#elif defined(ARCH_AARCH64_BIG_END_EL0_ALL_CORES) && \ + ARCH_AARCH64_BIG_END_EL0_ALL_CORES + ID_AA64MMFR0_EL1_set_BigEnd(&mmfr0, 0U); + ID_AA64MMFR0_EL1_set_BigEndEL0(&mmfr0, 1U); +#endif + ID_AA64MMFR0_EL1_set_TGran4(&mmfr0, 0U); + ID_AA64MMFR0_EL1_set_TGran16(&mmfr0, 0U); + ID_AA64MMFR0_EL1_set_TGran64(&mmfr0, 0xfU); +#if defined(ARCH_ARM_FEAT_ExS) ID_AA64MMFR0_EL1_set_ExS(&mmfr0, 1U); #endif - val = ID_AA64MMFR0_EL1_raw(mmfr0); +#if defined(ARCH_ARM_FEAT_ECV) + ID_AA64MMFR0_EL1_set_ECV(&mmfr0, 1U); +#endif + + reg_val = ID_AA64MMFR0_EL1_raw(mmfr0); break; } case ISS_MRS_MSR_ID_AA64MMFR1_EL1: { - ID_AA64MMFR1_EL1_t mmfr1 = ID_AA64MMFR1_EL1_default(); -#if defined(ARCH_ARM_8_1_TTHM) ID_AA64MMFR1_EL1_t hw_mmfr1 = register_ID_AA64MMFR1_EL1_read(); + ID_AA64MMFR1_EL1_t mmfr1 = ID_AA64MMFR1_EL1_default(); +#if defined(ARCH_ARM_FEAT_HPDS2) + ID_AA64MMFR1_EL1_set_HPDS(&mmfr1, 2U); +#elif defined(ARCH_ARM_FEAT_HPDS) + ID_AA64MMFR1_EL1_set_HPDS(&mmfr1, 1U); +#endif +#if defined(ARCH_ARM_FEAT_HAFDBS) ID_AA64MMFR1_EL1_copy_HAFDBS(&mmfr1, &hw_mmfr1); #endif -#if defined(ARCH_ARM_8_1_VMID16) +#if defined(ARCH_ARM_FEAT_VMID16) ID_AA64MMFR1_EL1_set_VMIDBits(&mmfr1, 2U); #endif -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) ID_AA64MMFR1_EL1_set_VH(&mmfr1, 1U); #endif -#if defined(ARCH_ARM_8_2_TTPBHA) - ID_AA64MMFR1_EL1_set_HPDS(&mmfr1, 2U); -#elif defined(ARCH_ARM_8_1_HPD) - ID_AA64MMFR1_EL1_set_HPDS(&mmfr1, 1U); -#endif -#if defined(ARCH_ARM_8_1_LOR) +#if defined(ARCH_ARM_FEAT_LOR) ID_AA64MMFR1_EL1_set_LO(&mmfr1, 1U); #endif -#if defined(ARCH_ARM_8_2_ATS1E1) +#if defined(ARCH_ARM_FEAT_PAN3) + ID_AA64MMFR1_EL1_set_PAN(&mmfr1, 3U); +#elif defined(ARCH_ARM_FEAT_PAN2) // now known as FEAT_PAN2 ID_AA64MMFR1_EL1_set_PAN(&mmfr1, 2U); -#elif defined(ARCH_ARM_8_1_PAN) +#elif defined(ARCH_ARM_FEAT_PAN) ID_AA64MMFR1_EL1_set_PAN(&mmfr1, 1U); #endif -#if defined(ARCH_ARM_8_2_TTS2UXN) +#if defined(ARCH_ARM_FEAT_XNX) ID_AA64MMFR1_EL1_set_XNX(&mmfr1, 1U); #endif - val = ID_AA64MMFR1_EL1_raw(mmfr1); +#if defined(ARCH_ARM_FEAT_TWED) + ID_AA64MMFR1_EL1_set_TWED(&mmfr1, 1U); +#endif +#if defined(ARCH_ARM_FEAT_ETS) + ID_AA64MMFR1_EL1_set_ETS(&mmfr1, 1U); +#endif +#if defined(ARCH_ARM_FEAT_HCX) + ID_AA64MMFR1_EL1_set_HCX(&mmfr1, 1U); +#endif +#if defined(ARCH_ARM_FEAT_AFP) + ID_AA64MMFR1_EL1_set_AFP(&mmfr1, 1U); +#endif +#if defined(ARCH_ARM_FEAT_nTLBPA) + ID_AA64MMFR1_EL1_set_nTLBPAwAFP(&mmfr1, 1U); +#endif +#if defined(ARCH_ARM_FEAT_TIDCP1) + ID_AA64MMFR1_EL1_set_TIDCP1(&mmfr1, 1U); +#endif +#if defined(ARCH_ARM_FEAT_CMOW) + ID_AA64MMFR1_EL1_set_CMOW(&mmfr1, 1U); +#endif +#if defined(ARCH_ARM_FEAT_ECBHB) + ID_AA64MMFR1_EL1_copy_ECBHB(&mmfr1, &hw_mmfr1); +#endif + reg_val = ID_AA64MMFR1_EL1_raw(mmfr1); + (void)hw_mmfr1; break; } case ISS_MRS_MSR_ID_AA64MMFR2_EL1: { ID_AA64MMFR2_EL1_t mmfr2 = ID_AA64MMFR2_EL1_default(); -#if defined(ARCH_ARM_8_2_TTCNP) +#if defined(ARCH_ARM_FEAT_TTCNP) ID_AA64MMFR2_EL1_set_CnP(&mmfr2, 1U); #endif -#if defined(ARCH_ARM_8_2_UAO) +#if defined(ARCH_ARM_FEAT_UAO) ID_AA64MMFR2_EL1_set_UAO(&mmfr2, 1U); #endif -#if defined(ARCH_ARM_8_2_LSMAOC) +#if defined(ARCH_ARM_FEAT_LSMAOC) ID_AA64MMFR2_EL1_set_LSM(&mmfr2, 1U); #endif -#if defined(ARCH_ARM_8_2_IESB) +#if defined(ARCH_ARM_FEAT_IESB) ID_AA64MMFR2_EL1_set_IESB(&mmfr2, 1U); #endif -#if defined(ARCH_ARM_8_2_LVA) +#if defined(ARCH_ARM_FEAT_LVA) ID_AA64MMFR2_EL1_set_VARange(&mmfr2, 1U); #endif -#if defined(ARCH_ARM_8_3_CCIDX) +#if defined(ARCH_ARM_FEAT_CCIDX) ID_AA64MMFR2_EL1_set_CCIDX(&mmfr2, 1U); #endif -#if defined(ARCH_ARM_8_4_NV) +#if defined(ARCH_ARM_FEAT_NV2) ID_AA64MMFR2_EL1_set_NV(&mmfr2, 2U); -#elif defined(ARCH_ARM_8_3_NV) +#elif defined(ARCH_ARM_FEAT_NV) ID_AA64MMFR2_EL1_set_NV(&mmfr2, 1U); #endif -#if defined(ARCH_ARM_8_4_TTST) +#if defined(ARCH_ARM_FEAT_TTST) ID_AA64MMFR2_EL1_set_ST(&mmfr2, 1U); #endif -#if defined(ARCH_ARM_8_4_LSE) +#if defined(ARCH_ARM_FEAT_LSE2) ID_AA64MMFR2_EL1_set_AT(&mmfr2, 1U); #endif -#if defined(ARCH_ARM_8_4_IDST) +#if defined(ARCH_ARM_FEAT_IDST) ID_AA64MMFR2_EL1_set_IDS(&mmfr2, 1U); #endif -#if defined(ARCH_ARM_8_4_SECEL2) +#if defined(ARCH_ARM_FEAT_SEL2) ID_AA64MMFR2_EL1_set_FWB(&mmfr2, 1U); #endif -#if defined(ARCH_ARM_8_4_TTL) +#if defined(ARCH_ARM_FEAT_TTL) ID_AA64MMFR2_EL1_set_IDS(&mmfr2, 1U); #endif -#if defined(ARCH_ARM_8_4_TTREM) || defined(ARCH_ARM_8_2_EVT) +#if defined(ARCH_ARM_FEAT_BBM) || defined(ARCH_ARM_FEAT_EVT) ID_AA64MMFR2_EL1_t hw_mmfr2 = register_ID_AA64MMFR2_EL1_read(); -#if defined(ARCH_ARM_8_4_TTREM) +#if defined(ARCH_ARM_FEAT_BBM) ID_AA64MMFR2_EL1_copy_BBM(&mmfr2, &hw_mmfr2); #endif -#if defined(ARCH_ARM_8_2_EVT) +#if defined(ARCH_ARM_FEAT_EVT) ID_AA64MMFR2_EL1_copy_EVT(&mmfr2, &hw_mmfr2); #endif #endif -#if defined(ARCH_ARM_8_5_E0PD) +#if defined(ARCH_ARM_FEAT_E0PD) ID_AA64MMFR2_EL1_set_E0PD(&mmfr2, 1U); #endif - val = ID_AA64MMFR2_EL1_raw(mmfr2); + reg_val = ID_AA64MMFR2_EL1_raw(mmfr2); break; } case ISS_MRS_MSR_ID_PFR0_EL1: { @@ -309,23 +455,32 @@ read_virtual_id_register(ESR_EL2_ISS_MSR_MRS_t iss, uint8_t reg_num) ID_PFR0_EL1_set_State0(&pfr0, 1U); ID_PFR0_EL1_set_State1(&pfr0, 3U); ID_PFR0_EL1_set_State2(&pfr0, 1U); -#if defined(ARCH_ARM_8_0_CSV2) +#if defined(ARCH_ARM_HAVE_SCXT) && \ + (defined(ARCH_ARM_FEAT_CSV2_2) || defined(ARCH_ARM_FEAT_CSV2_3)) + if (vcpu_option_flags_get_scxt_allowed(&thread->vcpu_options)) { + // At the time of writing, ARM does not have CSV2_3 + // encoding for ID_PFR0_EL1.CSV2 + ID_PFR0_EL1_set_CSV2(&pfr0, 2U); + } else { + ID_PFR0_EL1_set_CSV2(&pfr0, 1U); + } +#elif defined(ARCH_ARM_FEAT_CSV2) ID_PFR0_EL1_set_CSV2(&pfr0, 1U); #endif -#if defined(ARCH_ARM_8_4_DIT) +#if defined(ARCH_ARM_FEAT_DIT) ID_PFR0_EL1_set_DIT(&pfr0, 1U); #endif if (vcpu_option_flags_get_ras_error_handler( &thread->vcpu_options)) { -#if defined(ARCH_ARM_8_4_RAS) +#if defined(ARCH_ARM_FEAT_RASv1p1) ID_PFR0_EL1_set_RAS(&pfr0, 2); -#elif defined(ARCH_ARM_8_2_RAS) +#elif defined(ARCH_ARM_FEAT_RAS) ID_PFR0_EL1_set_RAS(&pfr0, 1); #else // Nothing to do, the field is already 0 #endif } - val = ID_PFR0_EL1_raw(pfr0); + reg_val = ID_PFR0_EL1_raw(pfr0); break; } case ISS_MRS_MSR_ID_PFR1_EL1: { @@ -337,18 +492,38 @@ read_virtual_id_register(ESR_EL2_ISS_MSR_MRS_t iss, uint8_t reg_num) #endif ID_PFR1_EL1_set_GenTimer(&pfr1, 1U); ID_PFR1_EL1_set_GIC(&pfr1, 1U); - val = ID_PFR1_EL1_raw(pfr1); + reg_val = ID_PFR1_EL1_raw(pfr1); break; } case ISS_MRS_MSR_ID_PFR2_EL1: { ID_PFR2_EL1_t pfr2 = ID_PFR2_EL1_default(); -#if defined(ARCH_ARM_8_0_CSV3) +#if defined(ARCH_ARM_FEAT_CSV3) ID_PFR2_EL1_set_CSV3(&pfr2, 1U); #endif -#if defined(ARCH_ARM_8_0_SSBS) +#if defined(ARCH_ARM_FEAT_SSBS) ID_PFR2_EL1_set_SSBS(&pfr2, 1U); #endif - val = ID_PFR2_EL1_raw(pfr2); + reg_val = ID_PFR2_EL1_raw(pfr2); + break; + } + case ISS_MRS_MSR_ID_DFR0_EL1: { + ID_DFR0_EL1_t hw_dfr0 = ID_DFR0_EL1_default(); + ID_DFR0_EL1_t dfr0 = register_ID_DFR0_EL1_read(); + + // The debug, trace, PMU and SPE modules must correctly support + // the values reported by the hardware. All we do here is to + // zero out fields for features we don't support. + +#if defined(MODULE_VM_VDEBUG) + ID_DFR0_EL1_copy_CopDbg(&dfr0, &hw_dfr0); + ID_DFR0_EL1_copy_CopSDbg(&dfr0, &hw_dfr0); + ID_DFR0_EL1_copy_MMapDbg(&dfr0, &hw_dfr0); +#endif +#if defined(MODULE_PLATFORM_ARM_PMU) + ID_DFR0_EL1_copy_PerfMon(&dfr0, &hw_dfr0); +#endif + + reg_val = ID_DFR0_EL1_raw(dfr0); break; } case ISS_MRS_MSR_ID_ISAR0_EL1: { @@ -357,7 +532,7 @@ read_virtual_id_register(ESR_EL2_ISS_MSR_MRS_t iss, uint8_t reg_num) ID_ISAR0_EL1_set_BitField(&isar0, 1U); ID_ISAR0_EL1_set_CmpBranch(&isar0, 1U); ID_ISAR0_EL1_set_Divide(&isar0, 2U); - val = ID_ISAR0_EL1_raw(isar0); + reg_val = ID_ISAR0_EL1_raw(isar0); break; } case ISS_MRS_MSR_ID_ISAR1_EL1: { @@ -369,7 +544,7 @@ read_virtual_id_register(ESR_EL2_ISS_MSR_MRS_t iss, uint8_t reg_num) ID_ISAR1_EL1_set_Immediate(&isar1, 1U); ID_ISAR1_EL1_set_Interwork(&isar1, 3U); ID_ISAR1_EL1_set_Jazelle(&isar1, 1U); - val = ID_ISAR1_EL1_raw(isar1); + reg_val = ID_ISAR1_EL1_raw(isar1); break; } case ISS_MRS_MSR_ID_ISAR2_EL1: { @@ -382,7 +557,7 @@ read_virtual_id_register(ESR_EL2_ISS_MSR_MRS_t iss, uint8_t reg_num) ID_ISAR2_EL1_set_MultU(&isar2, 2U); ID_ISAR2_EL1_set_PSR_AR(&isar2, 1U); ID_ISAR2_EL1_set_Reversal(&isar2, 2U); - val = ID_ISAR2_EL1_raw(isar2); + reg_val = ID_ISAR2_EL1_raw(isar2); break; } case ISS_MRS_MSR_ID_ISAR3_EL1: { @@ -394,7 +569,7 @@ read_virtual_id_register(ESR_EL2_ISS_MSR_MRS_t iss, uint8_t reg_num) ID_ISAR3_EL1_set_TabBranch(&isar3, 1U); ID_ISAR3_EL1_set_T32Copy(&isar3, 1U); ID_ISAR3_EL1_set_TrueNOP(&isar3, 1U); - val = ID_ISAR3_EL1_raw(isar3); + reg_val = ID_ISAR3_EL1_raw(isar3); break; } case ISS_MRS_MSR_ID_ISAR4_EL1: { @@ -406,49 +581,51 @@ read_virtual_id_register(ESR_EL2_ISS_MSR_MRS_t iss, uint8_t reg_num) ID_ISAR4_EL1_set_SMC(&isar4, 1U); #endif ID_ISAR4_EL1_set_Barrier(&isar4, 1U); - val = ID_ISAR4_EL1_raw(isar4); + reg_val = ID_ISAR4_EL1_raw(isar4); break; } case ISS_MRS_MSR_ID_ISAR5_EL1: { ID_ISAR5_EL1_t isar5 = ID_ISAR5_EL1_default(); ID_ISAR5_EL1_set_SEVL(&isar5, 1U); -#if defined(ARCH_ARM_8_0_AES_PMULL) +#if defined(ARCH_ARM_FEAT_PMULL) ID_ISAR5_EL1_set_AES(&isar5, 2U); -#elif defined(ARCH_ARM_8_0_AES) +#elif defined(ARCH_ARM_FEAT_AES) ID_ISAR5_EL1_set_AES(&isar5, 1U); #endif -#if defined(ARCH_ARM_8_0_SHA) +#if defined(ARCH_ARM_FEAT_SHA1) ID_ISAR5_EL1_set_SHA1(&isar5, 1U); ID_ISAR5_EL1_set_SHA2(&isar5, 1U); #endif +#if defined(ARCH_ARM_FEAT_CRC32) ID_ISAR5_EL1_set_CRC32(&isar5, 1U); -#if defined(ARCH_ARM_8_1_RDMA) +#endif +#if defined(ARCH_ARM_FEAT_RDM) ID_ISAR5_EL1_set_RDM(&isar5, 1U); #endif -#if defined(ARCH_ARM_8_3_COMPNUM) +#if defined(ARCH_ARM_FEAT_FCMA) ID_ISAR5_EL1_set_VCMA(&isar5, 2U); #endif - val = ID_ISAR5_EL1_raw(isar5); + reg_val = ID_ISAR5_EL1_raw(isar5); break; } case ISS_MRS_MSR_ID_ISAR6_EL1: { ID_ISAR6_EL1_t isar6 = ID_ISAR6_EL1_default(); -#if defined(ARCH_ARM_8_3_JSCONV) +#if defined(ARCH_ARM_FEAT_JSCVT) ID_ISAR6_EL1_set_JSCVT(&isar6, 1U); #endif -#if defined(ARCH_ARM_8_2_DOTPROD) +#if defined(ARCH_ARM_FEAT_DotProd) ID_ISAR6_EL1_set_DP(&isar6, 1U); #endif -#if defined(ARCH_ARM_8_2_FHM) +#if defined(ARCH_ARM_FEAT_FHM) ID_ISAR6_EL1_set_FHM(&isar6, 1U); #endif -#if defined(ARCH_ARM_8_0_SB) +#if defined(ARCH_ARM_FEAT_SB) ID_ISAR6_EL1_set_SB(&isar6, 1U); #endif -#if defined(ARCH_ARM_8_0_PREDINV) +#if defined(ARCH_ARM_FEAT_SPECRES) ID_ISAR6_EL1_set_SPECRES(&isar6, 1U); #endif - val = ID_ISAR6_EL1_raw(isar6); + reg_val = ID_ISAR6_EL1_raw(isar6); break; } case ISS_MRS_MSR_ID_MMFR0_EL1: { @@ -458,13 +635,13 @@ read_virtual_id_register(ESR_EL2_ISS_MSR_MRS_t iss, uint8_t reg_num) ID_MMFR0_EL1_set_ShareLvl(&mmfr0, 1U); ID_MMFR0_EL1_set_AuxReg(&mmfr0, 2U); ID_MMFR0_EL1_set_InnerShr(&mmfr0, 1U); - val = ID_MMFR0_EL1_raw(mmfr0); + reg_val = ID_MMFR0_EL1_raw(mmfr0); break; } case ISS_MRS_MSR_ID_MMFR1_EL1: { ID_MMFR1_EL1_t mmfr1 = ID_MMFR1_EL1_default(); ID_MMFR1_EL1_set_BPred(&mmfr1, 4U); - val = ID_MMFR1_EL1_raw(mmfr1); + reg_val = ID_MMFR1_EL1_raw(mmfr1); break; } case ISS_MRS_MSR_ID_MMFR2_EL1: { @@ -472,7 +649,7 @@ read_virtual_id_register(ESR_EL2_ISS_MSR_MRS_t iss, uint8_t reg_num) ID_MMFR2_EL1_set_UniTLB(&mmfr2, 6U); ID_MMFR2_EL1_set_MemBarr(&mmfr2, 2U); ID_MMFR2_EL1_set_WFIStall(&mmfr2, 1U); - val = ID_MMFR2_EL1_raw(mmfr2); + reg_val = ID_MMFR2_EL1_raw(mmfr2); break; } case ISS_MRS_MSR_ID_MMFR3_EL1: { @@ -481,47 +658,50 @@ read_virtual_id_register(ESR_EL2_ISS_MSR_MRS_t iss, uint8_t reg_num) ID_MMFR3_EL1_set_CMaintSW(&mmfr3, 1U); ID_MMFR3_EL1_set_BPMaint(&mmfr3, 2U); ID_MMFR3_EL1_set_MaintBcst(&mmfr3, 2U); -#if defined(ARCH_ARM_8_2_ATS1E1) +#if defined(ARCH_ARM_FEAT_PAN3) + ID_MMFR3_EL1_set_PAN(&mmfr3, 3U); +#elif defined(ARCH_ARM_FEAT_PAN2) // now known as FEAT_PAN2 ID_MMFR3_EL1_set_PAN(&mmfr3, 2U); -#elif defined(ARCH_ARM_8_1_PAN) +#elif defined(ARCH_ARM_FEAT_PAN) ID_MMFR3_EL1_set_PAN(&mmfr3, 1U); #endif ID_MMFR3_EL1_set_CohWalk(&mmfr3, 1U); ID_MMFR3_EL1_set_CMemSz(&mmfr3, 2U); - val = ID_MMFR3_EL1_raw(mmfr3); + reg_val = ID_MMFR3_EL1_raw(mmfr3); break; } case ISS_MRS_MSR_ID_MMFR4_EL1: { ID_MMFR4_EL1_t mmfr4 = ID_MMFR4_EL1_default(); ID_MMFR4_EL1_set_AC2(&mmfr4, 1U); -#if defined(ARCH_ARM_8_2_TTS2UXN) +#if defined(ARCH_ARM_FEAT_XNX) ID_MMFR4_EL1_set_XNX(&mmfr4, 1U); #endif -#if defined(ARCH_ARM_8_2_TTCNP) +#if defined(ARCH_ARM_FEAT_TTCNP) ID_MMFR4_EL1_set_CnP(&mmfr4, 1U); #endif -#if defined(ARCH_ARM_8_2_TTPBHA) +#if defined(ARCH_ARM_FEAT_HPDS2) ID_MMFR4_EL1_set_HPDS(&mmfr4, 2U); -#elif defined(ARCH_ARM_8_2_AA32HPD) +#elif defined(ARCH_ARM_FEAT_AA32HPD) ID_MMFR4_EL1_set_HPDS(&mmfr4, 1U); #endif -#if defined(ARCH_ARM_8_2_LSMAOC) +#if defined(ARCH_ARM_FEAT_LSMAOC) ID_MMFR4_EL1_set_LSM(&mmfr4, 1U); #endif -#if defined(ARCH_ARM_8_3_CCIDX) +#if defined(ARCH_ARM_FEAT_CCIDX) ID_MMFR4_EL1_set_CCIDX(&mmfr4, 1U); #endif -#if defined(ARCH_ARM_8_2_EVT) +#if defined(ARCH_ARM_FEAT_EVT) ID_MMFR4_EL1_t hw_mmfr4 = register_ID_MMFR4_EL1_read(); ID_MMFR4_EL1_copy_EVT(&mmfr4, &hw_mmfr4); #endif - val = ID_MMFR4_EL1_raw(mmfr4); + reg_val = ID_MMFR4_EL1_raw(mmfr4); break; } case ISS_MRS_MSR_ID_AA64DFR1_EL1: case ISS_MRS_MSR_ID_AA64AFR0_EL1: case ISS_MRS_MSR_ID_AA64AFR1_EL1: case ISS_MRS_MSR_ID_AFR0_EL1: + case ISS_MRS_MSR_ID_AA64SMFR0_EL1: // RAZ break; default: @@ -530,7 +710,7 @@ read_virtual_id_register(ESR_EL2_ISS_MSR_MRS_t iss, uint8_t reg_num) } if (handled) { - vcpu_gpr_write(thread, reg_num, val); + vcpu_gpr_write(thread, reg_num, reg_val); } return handled; @@ -544,9 +724,9 @@ read_virtual_id_register(ESR_EL2_ISS_MSR_MRS_t iss, uint8_t reg_num) vcpu_trap_result_t sysreg_read(ESR_EL2_ISS_MSR_MRS_t iss) { - register_t val = 0ULL; // Default action is RAZ - vcpu_trap_result_t ret = VCPU_TRAP_RESULT_EMULATED; - thread_t *thread = thread_get_self(); + register_t reg_val = 0ULL; // Default action is RAZ + vcpu_trap_result_t ret = VCPU_TRAP_RESULT_EMULATED; + thread_t *thread = thread_get_self(); // Assert this is a read assert(ESR_EL2_ISS_MSR_MRS_get_Direction(&iss)); @@ -569,44 +749,52 @@ sysreg_read(ESR_EL2_ISS_MSR_MRS_t iss) switch (ESR_EL2_ISS_MSR_MRS_raw(temp_iss)) { // The registers trapped with HCR_EL2.TID3 case ISS_MRS_MSR_ID_PFR0_EL1: { - ID_PFR0_EL1_t id_pfr0 = register_ID_PFR0_EL1_read(); + ID_PFR0_EL1_t pfr0 = register_ID_PFR0_EL1_read(); -#if defined(ARCH_ARM_8_2_RAS) || defined(ARCH_ARM_8_4_RAS) +#if defined(ARCH_ARM_FEAT_RAS) || defined(ARCH_ARM_FEAT_RASv1p1) // Tell non-RAS handler guests there is no RAS. if (!vcpu_option_flags_get_ras_error_handler( &thread->vcpu_options)) { - ID_PFR0_EL1_set_RAS(&id_pfr0, 0); + ID_PFR0_EL1_set_RAS(&pfr0, 0); } #endif -#if defined(ARCH_ARM_8_4_AMU) || defined(ARCH_ARM_8_6_AMU) +#if defined(ARCH_ARM_FEAT_AMUv1) || defined(ARCH_ARM_FEAT_AMUv1p1) // Tell non-HLOS guests that there is no AMU if (!vcpu_option_flags_get_hlos_vm(&thread->vcpu_options)) { - ID_PFR0_EL1_set_AMU(&id_pfr0, 0); + ID_PFR0_EL1_set_AMU(&pfr0, 0); + } +#endif +#if defined(ARCH_ARM_HAVE_SCXT) + if (!vcpu_option_flags_get_scxt_allowed( + &thread->vcpu_options)) { + ID_PFR0_EL1_set_CSV2(&pfr0, 1U); } +#elif defined(ARCH_ARM_FEAT_CSV2) + ID_PFR0_EL1_set_CSV2(&pfr0, 1U); #endif - val = ID_PFR0_EL1_raw(id_pfr0); + reg_val = ID_PFR0_EL1_raw(pfr0); break; } case ISS_MRS_MSR_ID_PFR1_EL1: { - ID_PFR1_EL1_t id_pfr1 = register_ID_PFR1_EL1_read(); + ID_PFR1_EL1_t pfr1 = register_ID_PFR1_EL1_read(); - val = ID_PFR1_EL1_raw(id_pfr1); + reg_val = ID_PFR1_EL1_raw(pfr1); break; } case ISS_MRS_MSR_ID_PFR2_EL1: { ID_PFR2_EL1_t pfr2 = ID_PFR2_EL1_default(); -#if defined(ARCH_ARM_8_0_CSV3) +#if defined(ARCH_ARM_FEAT_CSV3) ID_PFR2_EL1_set_CSV3(&pfr2, 1U); #endif -#if defined(ARCH_ARM_8_0_SSBS) +#if defined(ARCH_ARM_FEAT_SSBS) ID_PFR2_EL1_set_SSBS(&pfr2, 1U); #endif - val = ID_PFR2_EL1_raw(pfr2); + reg_val = ID_PFR2_EL1_raw(pfr2); break; } case ISS_MRS_MSR_ID_DFR0_EL1: { - ID_DFR0_EL1_t id_dfr0 = register_ID_DFR0_EL1_read(); + ID_DFR0_EL1_t dfr0 = register_ID_DFR0_EL1_read(); // The debug, trace, PMU and SPE modules must correctly support // the values reported by the hardware. All we do here is to @@ -615,146 +803,228 @@ sysreg_read(ESR_EL2_ISS_MSR_MRS_t iss) #if !defined(MODULE_VM_VDEBUG) // Note that ARMv8-A does not allow 0 (not implemented) in the // CopDbg field. So this configuration is not really supported. - ID_DFR0_EL1_set_CopDbg(&id_dfr0, 0U); - ID_DFR0_EL1_set_CopSDbg(&id_dfr0, 0U); - ID_DFR0_EL1_set_MMapDbg(&id_dfr0, 0U); - ID_DFR0_EL1_set_MProfDbg(&id_dfr0, 0U); + ID_DFR0_EL1_set_CopDbg(&dfr0, 0U); + ID_DFR0_EL1_set_CopSDbg(&dfr0, 0U); + ID_DFR0_EL1_set_MMapDbg(&dfr0, 0U); + ID_DFR0_EL1_set_MProfDbg(&dfr0, 0U); #endif #if defined(MODULE_VM_VETE) // Only the HLOS VM is allowed to trace if (!vcpu_option_flags_get_trace_allowed( &thread->vcpu_options)) { - ID_DFR0_EL1_set_CopTrc(&id_dfr0, 0U); - ID_DFR0_EL1_set_TraceFilt(&id_dfr0, 0U); + ID_DFR0_EL1_set_CopTrc(&dfr0, 0U); + ID_DFR0_EL1_set_TraceFilt(&dfr0, 0U); } #else - ID_DFR0_EL1_set_CopTrc(&id_dfr0, 0U); - ID_DFR0_EL1_set_TraceFilt(&id_dfr0, 0U); + ID_DFR0_EL1_set_CopTrc(&dfr0, 0U); + ID_DFR0_EL1_set_TraceFilt(&dfr0, 0U); #endif #if defined(MODULE_VM_VETM) // Only the HLOS VM is allowed to trace if (!vcpu_option_flags_get_trace_allowed( &thread->vcpu_options)) { - ID_DFR0_EL1_set_MMapTrc(&id_dfr0, 0U); + ID_DFR0_EL1_set_MMapTrc(&dfr0, 0U); } #else - ID_DFR0_EL1_set_MMapTrc(&id_dfr0, 0U); + ID_DFR0_EL1_set_MMapTrc(&dfr0, 0U); +#endif +#if !defined(MODULE_PLATFORM_ARM_PMU) + ID_DFR0_EL1_set_PerfMon(&dfr0, 0U); #endif - // In the first release we will give all VMs full control of PMU - // hardware, therefore the code below is commented out until - // advance PMU support has been added to the hypervisor. - // No PMU support - // ID_DFR0_EL1_set_PerfMon(&id_dfr0, 0); - - val = ID_DFR0_EL1_raw(id_dfr0); + reg_val = ID_DFR0_EL1_raw(dfr0); break; } case ISS_MRS_MSR_ID_AFR0_EL1: - sysreg64_read(ID_AFR0_EL1, val); + // RES0 - We don't know any AFR0 bits break; case ISS_MRS_MSR_ID_MMFR0_EL1: - sysreg64_read(ID_MMFR0_EL1, val); + sysreg64_read(ID_MMFR0_EL1, reg_val); break; case ISS_MRS_MSR_ID_MMFR1_EL1: - sysreg64_read(ID_MMFR1_EL1, val); + sysreg64_read(ID_MMFR1_EL1, reg_val); break; case ISS_MRS_MSR_ID_MMFR2_EL1: - sysreg64_read(ID_MMFR2_EL1, val); + sysreg64_read(ID_MMFR2_EL1, reg_val); break; - case ISS_MRS_MSR_ID_MMFR3_EL1: - sysreg64_read(ID_MMFR3_EL1, val); + case ISS_MRS_MSR_ID_MMFR3_EL1: { + sysreg64_read(ID_MMFR3_EL1, reg_val); + ID_MMFR3_EL1_t mmfr1 = ID_MMFR3_EL1_cast(reg_val); +#if defined(ARCH_ARM_FEAT_PAN3) + assert(ID_MMFR3_EL1_get_PAN(&mmfr1) >= 3U); + ID_MMFR3_EL1_set_PAN(&mmfr1, 3U); +#elif defined(ARCH_ARM_FEAT_PAN2) // now known as FEAT_PAN2 + assert(ID_MMFR3_EL1_get_PAN(&mmfr1) >= 2U); + ID_MMFR3_EL1_set_PAN(&mmfr1, 2U); +#elif defined(ARCH_ARM_FEAT_PAN) + assert(ID_MMFR3_EL1_get_PAN(&mmfr1) >= 1U); + ID_MMFR3_EL1_set_PAN(&mmfr1, 1U); +#else + ID_MMFR3_EL1_set_PAN(&mmfr1, 0U); +#endif + reg_val = ID_MMFR3_EL1_raw(mmfr1); break; + } case ISS_MRS_MSR_ID_MMFR4_EL1: - sysreg64_read(ID_MMFR4_EL1, val); + sysreg64_read(ID_MMFR4_EL1, reg_val); break; case ISS_MRS_MSR_ID_ISAR0_EL1: - sysreg64_read(ID_ISAR0_EL1, val); + sysreg64_read(ID_ISAR0_EL1, reg_val); break; case ISS_MRS_MSR_ID_ISAR1_EL1: - sysreg64_read(ID_ISAR1_EL1, val); + sysreg64_read(ID_ISAR1_EL1, reg_val); break; case ISS_MRS_MSR_ID_ISAR2_EL1: - sysreg64_read(ID_ISAR2_EL1, val); + sysreg64_read(ID_ISAR2_EL1, reg_val); break; case ISS_MRS_MSR_ID_ISAR3_EL1: - sysreg64_read(ID_ISAR3_EL1, val); + sysreg64_read(ID_ISAR3_EL1, reg_val); break; case ISS_MRS_MSR_ID_ISAR4_EL1: - sysreg64_read(ID_ISAR4_EL1, val); + sysreg64_read(ID_ISAR4_EL1, reg_val); break; case ISS_MRS_MSR_ID_ISAR5_EL1: - sysreg64_read(ID_ISAR5_EL1, val); + sysreg64_read(ID_ISAR5_EL1, reg_val); break; case ISS_MRS_MSR_ID_ISAR6_EL1: - sysreg64_read(S3_0_C0_C2_7, val); + sysreg64_read(S3_0_C0_C2_7, reg_val); break; case ISS_MRS_MSR_MVFR0_EL1: - sysreg64_read(MVFR0_EL1, val); + sysreg64_read(MVFR0_EL1, reg_val); break; case ISS_MRS_MSR_MVFR1_EL1: - sysreg64_read(MVFR1_EL1, val); + sysreg64_read(MVFR1_EL1, reg_val); break; case ISS_MRS_MSR_MVFR2_EL1: - sysreg64_read(MVFR2_EL1, val); + sysreg64_read(MVFR2_EL1, reg_val); break; case ISS_MRS_MSR_ID_AA64PFR0_EL1: { - ID_AA64PFR0_EL1_t id_aa64pfr0 = register_ID_AA64PFR0_EL1_read(); + ID_AA64PFR0_EL1_t pfr0 = register_ID_AA64PFR0_EL1_read(); + pfr0 = ID_AA64PFR0_EL1_clean(pfr0); +#if !ARCH_AARCH64_32BIT_EL0 + // Require EL0 to be 64-bit only, even if core supports 32-bit + ID_AA64PFR0_EL1_set_EL0(&pfr0, 1U); +#endif +#if !ARCH_AARCH64_32BIT_EL1 + // Require EL1 to be 64-bit only, even if core supports 32-bit + ID_AA64PFR0_EL1_set_EL1(&pfr0, 1U); +#endif + ID_AA64PFR0_EL1_set_EL2(&pfr0, 1U); + ID_AA64PFR0_EL1_set_EL3(&pfr0, 1U); +#if defined(ARCH_ARM_HAVE_SCXT) + if (!vcpu_option_flags_get_scxt_allowed( + &thread->vcpu_options)) { + ID_AA64PFR0_EL1_set_CSV2(&pfr0, 1U); + } +#elif defined(ARCH_ARM_FEAT_CSV2) + ID_AA64PFR0_EL1_set_CSV2(&pfr0, 1U); +#endif + +#if defined(ARCH_ARM_FEAT_MPAM) + if (!arm_mpam_is_allowed() || + !vcpu_option_flags_get_mpam_allowed( + &thread->vcpu_options)) { + // No MPAM + ID_AA64PFR0_EL1_set_MPAM(&pfr0, 0); + } +#else // No MPAM - ID_AA64PFR0_EL1_set_MPAM(&id_aa64pfr0, 0); + ID_AA64PFR0_EL1_set_MPAM(&pfr0, 0); +#endif -#if defined(ARCH_ARM_8_2_SVE) +#if defined(ARCH_ARM_FEAT_SVE) // Tell non-SVE allowed guests that there is no SVE if (!vcpu_option_flags_get_sve_allowed(&thread->vcpu_options)) { - ID_AA64PFR0_EL1_set_SVE(&id_aa64pfr0, 0); + ID_AA64PFR0_EL1_set_SVE(&pfr0, 0); } #else // No SVE - ID_AA64PFR0_EL1_set_SVE(&id_aa64pfr0, 0); + ID_AA64PFR0_EL1_set_SVE(&pfr0, 0); #endif -#if defined(ARCH_ARM_8_2_RAS) || defined(ARCH_ARM_8_4_RAS) +#if defined(ARCH_ARM_FEAT_RAS) || defined(ARCH_ARM_FEAT_RASv1p1) // Tell non-RAS handler guests there is no RAS if (!vcpu_option_flags_get_ras_error_handler( &thread->vcpu_options)) { - ID_AA64PFR0_EL1_set_RAS(&id_aa64pfr0, 0); + ID_AA64PFR0_EL1_set_RAS(&pfr0, 0); } #endif -#if defined(ARCH_ARM_8_4_AMU) || defined(ARCH_ARM_8_6_AMU) +#if defined(ARCH_ARM_FEAT_AMUv1) || defined(ARCH_ARM_FEAT_AMUv1p1) // Tell non-HLOS guests that there is no AMU if (!vcpu_option_flags_get_hlos_vm(&thread->vcpu_options)) { - ID_AA64PFR0_EL1_set_AMU(&id_aa64pfr0, 0); + ID_AA64PFR0_EL1_set_AMU(&pfr0, 0); } #endif +#if !defined(ARCH_ARM_FEAT_SEL2) + ID_AA64PFR0_EL1_set_SEL2(&pfr0, 0U); +#endif + ID_AA64PFR0_EL1_set_RME(&pfr0, 0U); - val = ID_AA64PFR0_EL1_raw(id_aa64pfr0); + reg_val = ID_AA64PFR0_EL1_raw(pfr0); break; } case ISS_MRS_MSR_ID_AA64PFR1_EL1: { ID_AA64PFR1_EL1_t pfr1 = register_ID_AA64PFR1_EL1_read(); -#if defined(ARCH_ARM_8_5_MEMTAG) + + pfr1 = ID_AA64PFR1_EL1_clean(pfr1); +#if defined(ARCH_ARM_FEAT_MTE) if (!arm_mte_is_allowed()) { ID_AA64PFR1_EL1_set_MTE(&pfr1, 0); } #else ID_AA64PFR1_EL1_set_MTE(&pfr1, 0); #endif - val = ID_AA64PFR1_EL1_raw(pfr1); +#if defined(ARCH_ARM_FEAT_RAS) || defined(ARCH_ARM_FEAT_RASv1p1) + if (!vcpu_option_flags_get_ras_error_handler( + &thread->vcpu_options)) { + ID_AA64PFR1_EL1_set_RAS_frac(&pfr1, 0); + } +#endif +#if defined(ARCH_ARM_HAVE_SCXT) && defined(ARCH_ARM_FEAT_CSV2_1p2) + if (!vcpu_option_flags_get_scxt_allowed( + &thread->vcpu_options)) { + ID_AA64PFR1_EL1_set_CSV2_frac(&pfr1, 1U); + } +#elif defined(ARCH_ARM_FEAT_CSV2_1p1) + ID_AA64PFR1_EL1_set_CSV2_frac(&pfr1, 1U); +#else + ID_AA64PFR1_EL1_set_CSV2_frac(&pfr1, 0U); +#endif + +#if defined(ARCH_ARM_FEAT_MPAM) + if (!arm_mpam_is_allowed() || + !vcpu_option_flags_get_mpam_allowed( + &thread->vcpu_options)) { + // No MPAM + ID_AA64PFR1_EL1_set_MPAM_frac(&pfr1, 0); + } +#else + // No MPAM + ID_AA64PFR1_EL1_set_MPAM_frac(&pfr1, 0); +#endif + // No SME / NMI + ID_AA64PFR1_EL1_set_SME(&pfr1, 0); + ID_AA64PFR1_EL1_set_NMI(&pfr1, 0); + + reg_val = ID_AA64PFR1_EL1_raw(pfr1); break; } case ISS_MRS_MSR_ID_AA64ZFR0_EL1: -#if defined(ARCH_ARM_8_2_SVE) +#if defined(ARCH_ARM_FEAT_SVE) // The SVE module will handle this register ret = VCPU_TRAP_RESULT_UNHANDLED; #else // When SVE is not implemented this register is RAZ, do nothing #endif break; + case ISS_MRS_MSR_ID_AA64SMFR0_EL1: + // No Scalable Matrix Extension support for now + break; case ISS_MRS_MSR_ID_AA64DFR0_EL1: { - ID_AA64DFR0_EL1_t id_ret = ID_AA64DFR0_EL1_default(); - ID_AA64DFR0_EL1_t id_aa64dfr0 = register_ID_AA64DFR0_EL1_read(); + ID_AA64DFR0_EL1_t dfr0 = ID_AA64DFR0_EL1_default(); + ID_AA64DFR0_EL1_t hw_dfr0 = register_ID_AA64DFR0_EL1_read(); // The debug, trace, PMU and SPE modules must correctly support // the values reported by the hardware. All we do here is to @@ -763,69 +1033,132 @@ sysreg_read(ESR_EL2_ISS_MSR_MRS_t iss) #if defined(MODULE_VM_VDEBUG) // Note that ARMv8-A does not allow 0 (not implemented) in this // field. So without this module is not really supported. - ID_AA64DFR0_EL1_copy_DebugVer(&id_ret, &id_aa64dfr0); + ID_AA64DFR0_EL1_copy_DebugVer(&dfr0, &hw_dfr0); - ID_AA64DFR0_EL1_copy_BRPs(&id_ret, &id_aa64dfr0); - ID_AA64DFR0_EL1_copy_WRPs(&id_ret, &id_aa64dfr0); - ID_AA64DFR0_EL1_copy_CTX_CMPs(&id_ret, &id_aa64dfr0); - ID_AA64DFR0_EL1_copy_DoubleLock(&id_ret, &id_aa64dfr0); + ID_AA64DFR0_EL1_copy_BRPs(&dfr0, &hw_dfr0); + ID_AA64DFR0_EL1_copy_WRPs(&dfr0, &hw_dfr0); + ID_AA64DFR0_EL1_copy_CTX_CMPs(&dfr0, &hw_dfr0); + ID_AA64DFR0_EL1_copy_DoubleLock(&dfr0, &hw_dfr0); #endif #if defined(MODULE_VM_ARM_VM_PMU) - ID_AA64DFR0_EL1_copy_PMUVer(&id_ret, &id_aa64dfr0); + ID_AA64DFR0_EL1_copy_PMUVer(&dfr0, &hw_dfr0); #endif #if defined(INTERFACE_VET) // Set IDs for VMs allowed to trace if (vcpu_option_flags_get_trace_allowed( &thread->vcpu_options)) { #if defined(MODULE_VM_VETE) - ID_AA64DFR0_EL1_copy_TraceVer(&id_ret, &id_aa64dfr0); - ID_AA64DFR0_EL1_copy_TraceFilt(&id_ret, &id_aa64dfr0); + ID_AA64DFR0_EL1_copy_TraceVer(&dfr0, &hw_dfr0); + ID_AA64DFR0_EL1_copy_TraceFilt(&dfr0, &hw_dfr0); #endif #if defined(MODULE_VM_VTBRE) - ID_AA64DFR0_EL1_copy_TraceBuffer(&id_ret, &id_aa64dfr0); + ID_AA64DFR0_EL1_copy_TraceBuffer(&dfr0, &hw_dfr0); #endif } #endif #if defined(MODULE_SPE) - ID_AA64DFR0_EL1_copy_PMSVer(&id_ret, &id_aa64dfr0); + ID_AA64DFR0_EL1_copy_PMSVer(&dfr0, &hw_dfr0); #endif - val = ID_AA64DFR0_EL1_raw(id_ret); + reg_val = ID_AA64DFR0_EL1_raw(dfr0); break; } case ISS_MRS_MSR_ID_AA64DFR1_EL1: - sysreg64_read(ID_AA64DFR1_EL1, val); + // RES0 - We don't know any AA64DFR1 bits break; case ISS_MRS_MSR_ID_AA64AFR0_EL1: - sysreg64_read(ID_AA64AFR0_EL1, val); + // RES0 - We don't know any AA64AFR0 bits break; case ISS_MRS_MSR_ID_AA64AFR1_EL1: + // RES0 - We don't know any AA64AFR1 bits break; - case ISS_MRS_MSR_ID_AA64ISAR0_EL1: - sysreg64_read(ID_AA64ISAR0_EL1, val); + case ISS_MRS_MSR_ID_AA64ISAR0_EL1: { + ID_AA64ISAR0_EL1_t isar0 = register_ID_AA64ISAR0_EL1_read(); + + isar0 = ID_AA64ISAR0_EL1_clean(isar0); + + reg_val = ID_AA64ISAR0_EL1_raw(isar0); break; - case ISS_MRS_MSR_ID_AA64ISAR1_EL1: - sysreg64_read(ID_AA64ISAR1_EL1, val); -#if !defined(ARCH_ARM_8_3_PAUTH) + } + case ISS_MRS_MSR_ID_AA64ISAR1_EL1: { + ID_AA64ISAR1_EL1_t isar1 = register_ID_AA64ISAR1_EL1_read(); + + isar1 = ID_AA64ISAR1_EL1_clean(isar1); +#if !defined(ARCH_ARM_FEAT_BF16) + ID_AA64ISAR1_EL1_set_BF16(&isar1, 0U); +#endif +#if !defined(ARCH_ARM_FEAT_PAuth) // When no PAUTH is enabled, hide it from the VM - ID_AA64ISAR1_EL1_t isar1 = ID_AA64ISAR1_EL1_cast(val); ID_AA64ISAR1_EL1_set_APA(&isar1, 0U); ID_AA64ISAR1_EL1_set_API(&isar1, 0U); ID_AA64ISAR1_EL1_set_GPA(&isar1, 0U); ID_AA64ISAR1_EL1_set_GPI(&isar1, 0U); - val = ID_AA64ISAR1_EL1_raw(isar1); #endif + reg_val = ID_AA64ISAR1_EL1_raw(isar1); break; - case ISS_MRS_MSR_ID_AA64MMFR0_EL1: - sysreg64_read(ID_AA64MMFR0_EL1, val); + } + case ISS_MRS_MSR_ID_AA64ISAR2_EL1: { + ID_AA64ISAR2_EL1_t isar2 = register_ID_AA64ISAR2_EL1_read(); + + isar2 = ID_AA64ISAR2_EL1_clean(isar2); + +#if !defined(ARCH_ARM_FEAT_PAuth) + // When PAUTH using QARMA3 is disabled, hide it from the VM + ID_AA64ISAR2_EL1_set_APA3(&isar2, 0U); + ID_AA64ISAR2_EL1_set_GPA3(&isar2, 0U); + ID_AA64ISAR2_EL1_set_PAC_frac(&isar2, 0U); +#endif +#if defined(ARCH_ARM_FEAT_WFxT) + // Remove once FEAT_WFxT is implemented + // FIXME: + ID_AA64ISAR2_EL1_set_WFxT(&isar2, 0U); +#endif + reg_val = ID_AA64ISAR2_EL1_raw(isar2); break; - case ISS_MRS_MSR_ID_AA64MMFR1_EL1: - sysreg64_read(ID_AA64MMFR1_EL1, val); + } + case ISS_MRS_MSR_ID_AA64MMFR0_EL1: { + ID_AA64MMFR0_EL1_t mmfr0 = register_ID_AA64MMFR0_EL1_read(); + + mmfr0 = ID_AA64MMFR0_EL1_clean(mmfr0); + + reg_val = ID_AA64MMFR0_EL1_raw(mmfr0); break; - case ISS_MRS_MSR_ID_AA64MMFR2_EL1: - sysreg64_read(ID_AA64MMFR2_EL1, val); + } + case ISS_MRS_MSR_ID_AA64MMFR1_EL1: { + ID_AA64MMFR1_EL1_t mmfr1 = register_ID_AA64MMFR1_EL1_read(); + + mmfr1 = ID_AA64MMFR1_EL1_clean(mmfr1); + +#if defined(ARCH_ARM_FEAT_PAN3) + assert(ID_AA64MMFR1_EL1_get_PAN(&mmfr1) >= 3U); + ID_AA64MMFR1_EL1_set_PAN(&mmfr1, 3U); +#elif defined(ARCH_ARM_FEAT_PAN2) // now known as FEAT_PAN2 + assert(ID_AA64MMFR1_EL1_get_PAN(&mmfr1) >= 2U); + ID_AA64MMFR1_EL1_set_PAN(&mmfr1, 2U); +#elif defined(ARCH_ARM_FEAT_PAN) + assert(ID_AA64MMFR1_EL1_get_PAN(&mmfr1) >= 1U); + ID_AA64MMFR1_EL1_set_PAN(&mmfr1, 1U); +#else + ID_AA64MMFR1_EL1_set_PAN(&mmfr1, 0U); +#endif + reg_val = ID_AA64MMFR1_EL1_raw(mmfr1); + break; + } + case ISS_MRS_MSR_ID_AA64MMFR2_EL1: { + ID_AA64MMFR2_EL1_t mmfr2 = register_ID_AA64MMFR2_EL1_read(); + + mmfr2 = ID_AA64MMFR2_EL1_clean(mmfr2); + + reg_val = ID_AA64MMFR2_EL1_raw(mmfr2); break; + } + // The trapped ACTLR_EL1 by default returns 0 for reads. + // The particular access should be handled in sysreg_read_cpu. + case ISS_MRS_MSR_ACTLR_EL1: { + reg_val = 0U; + break; + } default: { uint8_t opc0, opc1, crn, crm; @@ -834,8 +1167,8 @@ sysreg_read(ESR_EL2_ISS_MSR_MRS_t iss) crn = ESR_EL2_ISS_MSR_MRS_get_CRn(&iss); crm = ESR_EL2_ISS_MSR_MRS_get_CRm(&iss); - if ((opc0 == 3) && (opc1 == 0) && (crn == 0) && (crm >= 1) && - (crm <= 7)) { + if ((opc0 == 3U) && (opc1 == 0U) && (crn == 0U) && + (crm >= 1U) && (crm <= 7U)) { // It is IMPLEMENTATION DEFINED whether HCR_EL2.TID3 // traps MRS accesses to the registers in this range // (that have not been handled above). If we ever get @@ -843,7 +1176,7 @@ sysreg_read(ESR_EL2_ISS_MSR_MRS_t iss) TRACE_AND_LOG(DEBUG, WARN, "Emulated RAZ for ID register: ISS {:#x}", ESR_EL2_ISS_MSR_MRS_raw(iss)); - val = 0U; + reg_val = 0U; } else { ret = VCPU_TRAP_RESULT_UNHANDLED; } @@ -853,7 +1186,7 @@ sysreg_read(ESR_EL2_ISS_MSR_MRS_t iss) // Update the thread's register if (ret == VCPU_TRAP_RESULT_EMULATED) { - vcpu_gpr_write(thread, reg_num, val); + vcpu_gpr_write(thread, reg_num, reg_val); } #if SCHEDULER_CAN_MIGRATE @@ -866,7 +1199,7 @@ vcpu_trap_result_t sysreg_read_fallback(ESR_EL2_ISS_MSR_MRS_t iss) { vcpu_trap_result_t ret = VCPU_TRAP_RESULT_UNHANDLED; - thread_t *thread = thread_get_self(); + thread_t *thread = thread_get_self(); if (ESR_EL2_ISS_MSR_MRS_get_Op0(&iss) == 2U) { // Debug registers, RAZ by default @@ -881,7 +1214,7 @@ vcpu_trap_result_t sysreg_write(ESR_EL2_ISS_MSR_MRS_t iss) { vcpu_trap_result_t ret = VCPU_TRAP_RESULT_EMULATED; - thread_t *thread = thread_get_self(); + thread_t *thread = thread_get_self(); if (compiler_expected(ESR_EL2_ISS_MSR_MRS_get_Op0(&iss) != 1U)) { ret = VCPU_TRAP_RESULT_UNHANDLED; @@ -913,6 +1246,7 @@ sysreg_write(ESR_EL2_ISS_MSR_MRS_t iss) // (because DC ISW is upgraded to DC CISW in hardware) so we // disable the trap after the first warning (except on physical // CPUs with an erratum that makes all set/way ops unsafe). + // FIXME: preempt_disable(); thread->vcpu_regs_el2.hcr_el2 = register_HCR_EL2_read(); HCR_EL2_set_TSW(&thread->vcpu_regs_el2.hcr_el2, false); @@ -933,11 +1267,11 @@ vcpu_trap_result_t sysreg_write_fallback(ESR_EL2_ISS_MSR_MRS_t iss) { vcpu_trap_result_t ret = VCPU_TRAP_RESULT_EMULATED; - thread_t *thread = thread_get_self(); + thread_t *thread = thread_get_self(); // Read the thread's register uint8_t reg_num = ESR_EL2_ISS_MSR_MRS_get_Rt(&iss); - register_t val = vcpu_gpr_read(thread, reg_num); + register_t reg_val = vcpu_gpr_read(thread, reg_num); // Remove the fields that are not used in the comparison ESR_EL2_ISS_MSR_MRS_t temp_iss = iss; @@ -947,7 +1281,7 @@ sysreg_write_fallback(ESR_EL2_ISS_MSR_MRS_t iss) switch (ESR_EL2_ISS_MSR_MRS_raw(temp_iss)) { // The registers trapped with HCR_EL2.TVM case ISS_MRS_MSR_SCTLR_EL1: { - SCTLR_EL1_t sctrl = SCTLR_EL1_cast(val); + SCTLR_EL1_t sctrl = SCTLR_EL1_cast(reg_val); // If HCR_EL2.DC is set, prevent VM's enabling Stg-1 MMU if (HCR_EL2_get_DC(&thread->vcpu_regs_el2.hcr_el2) && SCTLR_EL1_get_M(&sctrl)) { @@ -958,38 +1292,43 @@ sysreg_write_fallback(ESR_EL2_ISS_MSR_MRS_t iss) break; } case ISS_MRS_MSR_TTBR0_EL1: - register_TTBR0_EL1_write(TTBR0_EL1_cast(val)); + register_TTBR0_EL1_write(TTBR0_EL1_cast(reg_val)); break; case ISS_MRS_MSR_TTBR1_EL1: - register_TTBR1_EL1_write(TTBR1_EL1_cast(val)); + register_TTBR1_EL1_write(TTBR1_EL1_cast(reg_val)); break; case ISS_MRS_MSR_TCR_EL1: - register_TCR_EL1_write(TCR_EL1_cast(val)); + register_TCR_EL1_write(TCR_EL1_cast(reg_val)); break; case ISS_MRS_MSR_ESR_EL1: - register_ESR_EL1_write(ESR_EL1_cast(val)); + register_ESR_EL1_write(ESR_EL1_cast(reg_val)); break; case ISS_MRS_MSR_FAR_EL1: - register_FAR_EL1_write(FAR_EL1_cast(val)); + register_FAR_EL1_write(FAR_EL1_cast(reg_val)); break; case ISS_MRS_MSR_AFSR0_EL1: - register_AFSR0_EL1_write(AFSR0_EL1_cast(val)); + register_AFSR0_EL1_write(AFSR0_EL1_cast(reg_val)); break; case ISS_MRS_MSR_AFSR1_EL1: - register_AFSR1_EL1_write(AFSR1_EL1_cast(val)); + register_AFSR1_EL1_write(AFSR1_EL1_cast(reg_val)); break; case ISS_MRS_MSR_MAIR_EL1: - register_MAIR_EL1_write(MAIR_EL1_cast(val)); + register_MAIR_EL1_write(MAIR_EL1_cast(reg_val)); break; case ISS_MRS_MSR_AMAIR_EL1: // WI break; + // The trapped ACTLR_EL1 by default will be ignored for writes. + // The particular access should be handled in sysreg_read_cpu. + case ISS_MRS_MSR_ACTLR_EL1: + // WI + break; case ISS_MRS_MSR_CONTEXTIDR_EL1: - register_CONTEXTIDR_EL1_write(CONTEXTIDR_EL1_cast(val)); + register_CONTEXTIDR_EL1_write(CONTEXTIDR_EL1_cast(reg_val)); break; default: { uint8_t opc0 = ESR_EL2_ISS_MSR_MRS_get_Op0(&iss); - if (opc0 == 2) { + if (opc0 == 2U) { // Debug registers, WI by default } else { ret = VCPU_TRAP_RESULT_UNHANDLED; diff --git a/hyp/vm/vcpu/aarch64/src/trap_dispatch.c b/hyp/vm/vcpu/aarch64/src/trap_dispatch.c index 93f9e61..e023c2c 100644 --- a/hyp/vm/vcpu/aarch64/src/trap_dispatch.c +++ b/hyp/vm/vcpu/aarch64/src/trap_dispatch.c @@ -36,34 +36,31 @@ static inline void exception_skip_inst(bool is_il32) { - thread_t *thread = thread_get_self(); + thread_t *thread = thread_get_self(); register_t pc = ELR_EL2_get_ReturnAddress(&thread->vcpu_regs_gpr.pc); #if ARCH_AARCH64_32BIT_EL0 pc += is_il32 ? 4U : 2U; - SPSR_EL2_A64_t spsr_el2 = thread->vcpu_regs_gpr.spsr_el2; - spsr_64bit_mode_t spsr_m = SPSR_EL2_A64_get_M(&spsr_el2); + SPSR_EL2_base_t spsr_base = thread->vcpu_regs_gpr.spsr_el2.base; - if ((spsr_m & 0x10) != 0U) { + if (SPSR_EL2_base_get_M4(&spsr_base)) { // Exception was in AArch32 execution. Update PSTATE.IT - SPSR_EL2_A32_t spsr32 = - SPSR_EL2_A32_cast(SPSR_EL2_A64_raw(spsr_el2)); + SPSR_EL2_A32_t spsr32 = thread->vcpu_regs_gpr.spsr_el2.a32; if (SPSR_EL2_A32_get_T(&spsr32)) { uint8_t IT = SPSR_EL2_A32_get_IT(&spsr32); - if ((IT & 0xf) == 0x8) { + if ((IT & 0xfU) == 0x8U) { // Was the last instruction in IT block IT = 0; } else { // Otherwise shift bits. This is safe even if // not in an IT block. - IT = (uint8_t)((IT & 0xe0) | ((IT & 0xf) << 1)); + IT = (uint8_t)((IT & 0xe0U) | + ((IT & 0xfU) << 1U)); } SPSR_EL2_A32_set_IT(&spsr32, IT); - spsr_el2 = SPSR_EL2_A64_cast(SPSR_EL2_A32_raw(spsr32)); - - thread->vcpu_regs_gpr.spsr_el2 = spsr_el2; + thread->vcpu_regs_gpr.spsr_el2.a32 = spsr32; } else { assert(is_il32); } @@ -77,101 +74,47 @@ exception_skip_inst(bool is_il32) ELR_EL2_set_ReturnAddress(&thread->vcpu_regs_gpr.pc, pc); } -static bool -handle_tlb_conflict() -{ - // FIXME: - // First check if a page table update is already in progress. If this is - // the case, flush the TLBs and return true (to retry the instruction). - // Otherwise return false. - - return false; -} - -static bool -handle_break_before_make() -{ - // FIXME: - // First check if a page table update is already in progress. If this is - // the case return true (to retry the instruction). Otherwise return - // false. - - return false; -} - static vcpu_trap_result_t handle_inst_data_abort(ESR_EL2_t esr, esr_ec_t ec, FAR_EL2_t far, - HPFAR_EL2_t hpfar, iss_da_ia_fsc_t fsc, + HPFAR_EL2_t hpfar, iss_da_ia_fsc_t fsc, bool is_s1ptw, bool is_data_abort) { vcpu_trap_result_t ret = VCPU_TRAP_RESULT_UNHANDLED; - - if (fsc == ISS_DA_IA_FSC_TLB_CONFLICT) { - if (handle_tlb_conflict()) { - ret = VCPU_TRAP_RESULT_RETRY; - } -#if defined(ARCH_ARM_8_1_TTHM) - } else if (fsc == ISS_DA_IA_FSC_ATOMIC_HW_UPDATE) { - // Unsupported atomic hardware update fail - if (handle_break_before_make()) { - ret = VCPU_TRAP_RESULT_RETRY; - } -#endif + gvaddr_t va = FAR_EL2_get_VirtualAddress(&far); + vmaddr_result_t ipa_r; + + if (is_s1ptw || (fsc == ISS_DA_IA_FSC_ADDR_SIZE_0) || + (fsc == ISS_DA_IA_FSC_ADDR_SIZE_1) || + (fsc == ISS_DA_IA_FSC_ADDR_SIZE_2) || + (fsc == ISS_DA_IA_FSC_ADDR_SIZE_3) || + (fsc == ISS_DA_IA_FSC_TRANSLATION_0) || + (fsc == ISS_DA_IA_FSC_TRANSLATION_1) || + (fsc == ISS_DA_IA_FSC_TRANSLATION_2) || + (fsc == ISS_DA_IA_FSC_TRANSLATION_3) || + (fsc == ISS_DA_IA_FSC_ACCESS_FLAG_1) || + (fsc == ISS_DA_IA_FSC_ACCESS_FLAG_2) || + (fsc == ISS_DA_IA_FSC_ACCESS_FLAG_3)) { + // HPFAR_EL2 is valid; combine it with the sub-page bits + // of the VA to find the exact IPA. + ipa_r = vmaddr_result_ok(HPFAR_EL2_get_FIPA(&hpfar) | + (va & 0xfffU)); } else { - gvaddr_t va = FAR_EL2_get_VirtualAddress(&far); - vmaddr_result_t ipa_r; - - if ((fsc == ISS_DA_IA_FSC_ADDR_SIZE_0) || - (fsc == ISS_DA_IA_FSC_ADDR_SIZE_1) || - (fsc == ISS_DA_IA_FSC_ADDR_SIZE_2) || - (fsc == ISS_DA_IA_FSC_ADDR_SIZE_3) || - (fsc == ISS_DA_IA_FSC_TRANSLATION_0) || - (fsc == ISS_DA_IA_FSC_TRANSLATION_1) || - (fsc == ISS_DA_IA_FSC_TRANSLATION_2) || - (fsc == ISS_DA_IA_FSC_TRANSLATION_3) || - (fsc == ISS_DA_IA_FSC_ACCESS_FLAG_1) || - (fsc == ISS_DA_IA_FSC_ACCESS_FLAG_2) || - (fsc == ISS_DA_IA_FSC_ACCESS_FLAG_3) || - (fsc == ISS_DA_IA_FSC_SYNC_EXTERN_WALK_0) || - (fsc == ISS_DA_IA_FSC_SYNC_EXTERN_WALK_1) || - (fsc == ISS_DA_IA_FSC_SYNC_EXTERN_WALK_2) || - (fsc == ISS_DA_IA_FSC_SYNC_EXTERN_WALK_3)) { - // HPFAR_EL2 is valid - ipa_r = vmaddr_result_ok(HPFAR_EL2_get_FIPA(&hpfar) | - (va & 0xfff)); - } else { - // HPFAR_EL2 is invalid, translate - ipa_r = addrspace_va_to_ipa_read(va); - } - - // Call the event handlers for the DA/PA - if (compiler_unexpected(ipa_r.e != OK)) { - // This can happen if the guest unmapped the faulting - // VA in stage 1 on another CPU after the stage 2 - // fault was triggered. In that case, we must retry the - // faulting instruction; it should fault in stage 1. - ret = VCPU_TRAP_RESULT_RETRY; - } else if (is_data_abort) { - ret = trigger_vcpu_trap_data_abort_guest_event( - esr, ipa_r.r, far); - } else { - ret = trigger_vcpu_trap_pf_abort_guest_event( - esr, ipa_r.r, far); - } + // HPFAR_EL2 was not set by the fault; we can't rely on it. + ipa_r = vmaddr_result_error(ERROR_ADDR_INVALID); + } - // If not handled, check if we are in the middle of a page - // table update - if ((ret == VCPU_TRAP_RESULT_UNHANDLED) && - handle_break_before_make()) { - ret = VCPU_TRAP_RESULT_RETRY; - } + // Call the event handlers for the DA/PA + if (is_data_abort) { + ret = trigger_vcpu_trap_data_abort_guest_event(esr, ipa_r, far); + } else { + ret = trigger_vcpu_trap_pf_abort_guest_event(esr, ipa_r, far); + } - // If still not handled inject the abort to the guest - if ((ret == VCPU_TRAP_RESULT_UNHANDLED) && - inject_inst_data_abort(esr, ec, fsc, far, ipa_r.r, - is_data_abort)) { - ret = VCPU_TRAP_RESULT_RETRY; - } + // If faulting or still not handled, inject the abort to the guest + if (((ret == VCPU_TRAP_RESULT_UNHANDLED) || + (ret == VCPU_TRAP_RESULT_FAULT)) && + inject_inst_data_abort(esr, ec, fsc, far, ipa_r.r, is_data_abort)) { + ret = VCPU_TRAP_RESULT_RETRY; } return ret; @@ -186,7 +129,7 @@ vcpu_interrupt_dispatch(void) preempt_disable_in_irq(); if (irq_interrupt_dispatch()) { - scheduler_schedule(); + (void)scheduler_schedule(); } preempt_enable_in_irq(); @@ -194,7 +137,7 @@ vcpu_interrupt_dispatch(void) trigger_thread_exit_to_user_event(THREAD_ENTRY_REASON_INTERRUPT); } -// Dispatching of guest synchronous exceptions and asynchronous system errors +// Dispatching of guest synchronous exceptions void vcpu_exception_dispatch(bool is_aarch64) { @@ -204,10 +147,12 @@ vcpu_exception_dispatch(bool is_aarch64) trigger_thread_entry_from_user_event(THREAD_ENTRY_REASON_EXCEPTION); + bool fatal = false; vcpu_trap_result_t result = VCPU_TRAP_RESULT_UNHANDLED; esr_ec_t ec = ESR_EL2_get_EC(&esr); bool is_il32 = true; + // FIXME: // For exceptions AArch32 execution, we need to determine whether the // trapped instruction passed its condition code. If it did not pass, // then skip the instruction. Remember special cases, such as BKPT in @@ -230,37 +175,58 @@ vcpu_exception_dispatch(bool is_aarch64) ESR_EL2_ISS_WFI_WFE_t iss = ESR_EL2_ISS_WFI_WFE_cast(ESR_EL2_get_ISS(&esr)); #if ARCH_AARCH64_32BIT_EL1 + // FIXME: #error Check the condition code #endif - if (ESR_EL2_ISS_WFI_WFE_get_TI(&iss)) { + switch (ESR_EL2_ISS_WFI_WFE_get_TI(&iss)) { + case ISS_WFX_TI_WFE: result = trigger_vcpu_trap_wfe_event(iss); - } else { + break; + case ISS_WFX_TI_WFI: result = trigger_vcpu_trap_wfi_event(iss); + break; +#if defined(ARCH_ARM_FEAT_WFxT) + // These need events updated to pass timeout for FEAT_WFxT + // support + // FIXME: + case ISS_WFX_TI_WFET: + result = trigger_vcpu_trap_wfe_event(iss); + break; + case ISS_WFX_TI_WFIT: + result = trigger_vcpu_trap_wfi_event(iss); + break; +#endif + default: + // should not happen + // result = VCPU_TRAP_RESULT_UNHANDLED + break; } break; } - case ESR_EC_FPEN: + case ESR_EC_FPEN: { #if ARCH_AARCH64_32BIT_EL1 + // FIXME: #error Check the condition code #endif result = trigger_vcpu_trap_fp_enabled_event(esr); break; + } -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) case ESR_EC_PAUTH: if (trigger_vcpu_trap_pauth_event()) { result = VCPU_TRAP_RESULT_RETRY; } break; -#if defined(ARCH_ARM_8_3_NV) +#if defined(ARCH_ARM_FEAT_NV) case ESR_EC_ERET: if (trigger_vcpu_trap_eret_event(esr)) { result = VCPU_TRAP_RESULT_RETRY; } break; #endif -#endif // defined(ARCH_ARM_8_3_FPAC) +#endif // defined(ARCH_ARM_FEAT_FPAC) case ESR_EC_ILLEGAL: if (trigger_vcpu_trap_illegal_state_event()) { @@ -322,7 +288,7 @@ vcpu_exception_dispatch(bool is_aarch64) } break; } -#if defined(ARCH_ARM_8_2_SVE) +#if defined(ARCH_ARM_FEAT_SVE) case ESR_EC_SVE: result = trigger_vcpu_trap_sve_access_event(); break; @@ -330,10 +296,11 @@ vcpu_exception_dispatch(bool is_aarch64) case ESR_EC_INST_ABT_LO: { ESR_EL2_ISS_INST_ABORT_t iss = ESR_EL2_ISS_INST_ABORT_cast(ESR_EL2_get_ISS(&esr)); - iss_da_ia_fsc_t fsc = ESR_EL2_ISS_INST_ABORT_get_IFSC(&iss); + iss_da_ia_fsc_t fsc = ESR_EL2_ISS_INST_ABORT_get_IFSC(&iss); + bool s1ptw = ESR_EL2_ISS_INST_ABORT_get_S1PTW(&iss); - result = - handle_inst_data_abort(esr, ec, far, hpfar, fsc, false); + result = handle_inst_data_abort(esr, ec, far, hpfar, fsc, s1ptw, + false); break; } case ESR_EC_PC_ALIGN: @@ -345,9 +312,11 @@ vcpu_exception_dispatch(bool is_aarch64) case ESR_EC_DATA_ABT_LO: { ESR_EL2_ISS_DATA_ABORT_t iss = ESR_EL2_ISS_DATA_ABORT_cast(ESR_EL2_get_ISS(&esr)); - iss_da_ia_fsc_t fsc = ESR_EL2_ISS_DATA_ABORT_get_DFSC(&iss); + iss_da_ia_fsc_t fsc = ESR_EL2_ISS_DATA_ABORT_get_DFSC(&iss); + bool s1ptw = ESR_EL2_ISS_DATA_ABORT_get_S1PTW(&iss); - result = handle_inst_data_abort(esr, ec, far, hpfar, fsc, true); + result = handle_inst_data_abort(esr, ec, far, hpfar, fsc, s1ptw, + true); break; } case ESR_EC_SP_ALIGN: @@ -360,12 +329,6 @@ vcpu_exception_dispatch(bool is_aarch64) result = trigger_vcpu_trap_fp64_event(esr); break; - case ESR_EC_SERROR: { - ESR_EL2_ISS_SERROR_t iss = - ESR_EL2_ISS_SERROR_cast(ESR_EL2_get_ISS(&esr)); - result = trigger_vcpu_trap_serror_event(iss); - break; - } case ESR_EC_BREAK_LO: result = trigger_vcpu_trap_breakpoint_guest_event(esr); break; @@ -417,6 +380,8 @@ vcpu_exception_dispatch(bool is_aarch64) case ESR_EC_BKPT: break; #endif + /* Asynchronous traps which don't come through this path */ + case ESR_EC_SERROR: /* AArch32 traps which may come when TGE=1 */ case ESR_EC_FP32: /* AArch32 traps which may come only from EL1 */ @@ -434,14 +399,45 @@ vcpu_exception_dispatch(bool is_aarch64) case ESR_EC_BREAK: case ESR_EC_STEP: case ESR_EC_WATCH: -#if defined(ARCH_ARM_8_5_BTI) +#if defined(ARCH_ARM_FEAT_BTI) case ESR_EC_BTI: #endif -#if defined(ARCH_ARM_8_3_PAUTH) && defined(ARCH_ARM_8_3_FPAC) +#if defined(ARCH_ARM_FEAT_PAuth) && defined(ARCH_ARM_FEAT_FPAC) case ESR_EC_FPAC: #endif - default: { - thread_t *thread = thread_get_self(); +#if defined(ARCH_ARM_FEAT_LS64) + case ESR_EC_LD64B_ST64B: +#endif +#if defined(ARCH_ARM_FEAT_TME) + case ESR_EC_TSTART: +#endif +#if defined(ARCH_ARM_FEAT_SME) + case ESR_EC_SME: +#endif +#if defined(ARCH_ARM_FEAT_RME) + case ESR_EC_RME: +#endif +#if defined(ARCH_ARM_FEAT_MOPS) + case ESR_EC_MOPS: +#endif +#if defined(VERBOSE) && VERBOSE + // Cause a fatal error in verbose builds so we can detect any unhandled + // ECs + default: +#endif + fatal = true; + break; +#if !(defined(VERBOSE) && VERBOSE) + // On non-verbose builds pass the unexpected ECs back to the VM + default: + result = VCPU_TRAP_RESULT_UNHANDLED; + break; +#endif + } + + thread_t *thread = thread_get_self(); + + if (compiler_unexpected(fatal)) { TRACE_AND_LOG(ERROR, WARN, "Unexpected trap from VM {:d}, ESR_EL2 = {:#x}, " "ELR_EL2 = {:#x}", @@ -450,11 +446,9 @@ vcpu_exception_dispatch(bool is_aarch64) abort("Unexpected guest trap", ABORT_REASON_UNHANDLED_EXCEPTION); } - } switch (result) { - case VCPU_TRAP_RESULT_UNHANDLED: { - thread_t *thread = thread_get_self(); + case VCPU_TRAP_RESULT_UNHANDLED: TRACE_AND_LOG(ERROR, WARN, "Unhandled trap from VM {:d}, ESR_EL2 = {:#x}, " "ELR_EL2 = {:#x}", @@ -462,7 +456,6 @@ vcpu_exception_dispatch(bool is_aarch64) ELR_EL2_raw(thread->vcpu_regs_gpr.pc)); inject_undef_abort(esr); break; - } case VCPU_TRAP_RESULT_FAULT: inject_undef_abort(esr); break; @@ -470,9 +463,29 @@ vcpu_exception_dispatch(bool is_aarch64) exception_skip_inst(is_il32); break; case VCPU_TRAP_RESULT_RETRY: + default: // Nothing to do here. break; } trigger_thread_exit_to_user_event(THREAD_ENTRY_REASON_EXCEPTION); } + +// Dispatching of guest asynchronous system errors +void +vcpu_error_dispatch(void) +{ + ESR_EL2_t esr = register_ESR_EL2_read_ordered(&asm_ordering); + + trigger_thread_entry_from_user_event(THREAD_ENTRY_REASON_INTERRUPT); + + preempt_disable_in_irq(); + + ESR_EL2_ISS_SERROR_t iss = + ESR_EL2_ISS_SERROR_cast(ESR_EL2_get_ISS(&esr)); + (void)trigger_vcpu_trap_serror_event(iss); + + preempt_enable_in_irq(); + + trigger_thread_exit_to_user_event(THREAD_ENTRY_REASON_INTERRUPT); +} diff --git a/hyp/vm/vcpu/aarch64/src/wfi.c b/hyp/vm/vcpu/aarch64/src/wfi.c index fd3ced9..57cb5e5 100644 --- a/hyp/vm/vcpu/aarch64/src/wfi.c +++ b/hyp/vm/vcpu/aarch64/src/wfi.c @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -32,14 +33,31 @@ vcpu_handle_scheduler_selected_thread(thread_t *thread, bool *can_idle) } vcpu_trap_result_t -vcpu_handle_vcpu_trap_wfi(void) +vcpu_handle_vcpu_trap_wfi(ESR_EL2_ISS_WFI_WFE_t iss) { + vcpu_trap_result_t ret = VCPU_TRAP_RESULT_EMULATED; + thread_t *current = thread_get_self(); assert(current->kind == THREAD_KIND_VCPU); assert_preempt_enabled(); preempt_disable(); +#if defined(ARCH_ARM_FEAT_WFxT) + // Inject a trap to the guest if it uses the WFIT instruction + // without checking their availability in the ID registers first. + // Remove once support for FEAT_WFxT is added to the hypervisor. + // FIXME: + if (ESR_EL2_ISS_WFI_WFE_get_TI(&iss) == ISS_WFX_TI_WFIT) { + TRACE_AND_LOG(ERROR, WARN, "WFIT trap from thread {:#x}", + (register_t)current); + ret = VCPU_TRAP_RESULT_FAULT; + goto out; + } +#else + (void)iss; +#endif + #if !defined(PREEMPT_NULL) #if !defined(VCPU_IDLE_IN_EL1) || !VCPU_IDLE_IN_EL1 if (current->vcpu_can_idle && !current->vcpu_interrupted) { @@ -83,7 +101,7 @@ vcpu_handle_vcpu_trap_wfi(void) out: preempt_enable(); - return VCPU_TRAP_RESULT_EMULATED; + return ret; } #if defined(VCPU_IDLE_IN_EL1) && VCPU_IDLE_IN_EL1 @@ -91,7 +109,7 @@ void vcpu_handle_scheduler_quiescent(void) { thread_t *current = thread_get_self(); - if (current->kind == THREAD_KIND_VCPU) { + if (compiler_expected(current->kind == THREAD_KIND_VCPU)) { current->vcpu_regs_el2.hcr_el2 = register_HCR_EL2_read(); HCR_EL2_set_TWI(¤t->vcpu_regs_el2.hcr_el2, !current->vcpu_can_idle); @@ -149,6 +167,16 @@ vcpu_expects_wakeup(const thread_t *thread) trigger_vcpu_expects_wakeup_event(thread); } +#if defined(MODULE_VM_VCPU_RUN) +vcpu_run_state_t +vcpu_arch_handle_vcpu_run_check(const thread_t *thread) +{ + return scheduler_is_blocked(thread, SCHEDULER_BLOCK_VCPU_WFI) + ? VCPU_RUN_STATE_EXPECTS_WAKEUP + : VCPU_RUN_STATE_BLOCKED; +} +#endif + bool vcpu_pending_wakeup(void) { diff --git a/hyp/vm/vcpu/aarch64/vcpu_aarch64.ev b/hyp/vm/vcpu/aarch64/vcpu_aarch64.ev index 99d21cd..f75cf7e 100644 --- a/hyp/vm/vcpu/aarch64/vcpu_aarch64.ev +++ b/hyp/vm/vcpu/aarch64/vcpu_aarch64.ev @@ -3,6 +3,7 @@ // SPDX-License-Identifier: BSD-3-Clause // Add the events for the traps coming from AArch32 +// FIXME: module vcpu @@ -13,9 +14,17 @@ subscribe boot_runtime_first_init subscribe boot_runtime_warm_init handler vcpu_handle_boot_runtime_init() subscribe boot_cpu_warm_init() +#if defined(ARCH_ARM_FEAT_CSV2_2) || defined(ARCH_ARM_FEAT_CSV2_1p2) || \ + defined(ARCH_ARM_FEAT_CSV2_3) +subscribe boot_cold_init() +#endif // new vcpu handlers +subscribe object_deactivate_thread + +subscribe vcpu_activate_thread + subscribe thread_get_entry_fn[THREAD_KIND_VCPU] () subscribe object_create_thread @@ -30,15 +39,19 @@ subscribe thread_start subscribe thread_save_state handler vcpu_context_switch_save() + require_preempt_disabled subscribe thread_save_state handler vcpu_context_switch_cpu_save() + require_preempt_disabled subscribe thread_load_state handler vcpu_context_switch_load() + require_preempt_disabled subscribe thread_load_state handler vcpu_context_switch_cpu_load() + require_preempt_disabled // vcpu register trap handlers @@ -47,6 +60,7 @@ subscribe vcpu_trap_sysreg_read subscribe vcpu_trap_sysreg_read handler sysreg_read_cpu + priority 1 subscribe vcpu_trap_sysreg_read handler sysreg_read_fallback @@ -58,6 +72,7 @@ subscribe vcpu_trap_sysreg_write subscribe vcpu_trap_sysreg_write handler sysreg_write_cpu + priority 1 subscribe vcpu_trap_sysreg_write handler sysreg_write_fallback @@ -70,7 +85,7 @@ subscribe vcpu_trap_brk_instruction_guest subscribe scheduler_selected_thread(thread, can_idle) -subscribe vcpu_trap_wfi() +subscribe vcpu_trap_wfi priority last exclude_preempt_disabled @@ -82,12 +97,11 @@ subscribe scheduler_quiescent() subscribe thread_context_switch_post() #endif +#if defined(MODULE_VM_VCPU_RUN) +subscribe vcpu_run_check + handler vcpu_arch_handle_vcpu_run_check(vcpu) +#endif + // VCPU lifecycle and power management subscribe rootvm_init(root_thread) - -subscribe vcpu_poweron - handler vcpu_reset_execution_context(vcpu) - -subscribe vcpu_warm_reset - handler vcpu_reset_execution_context(vcpu) diff --git a/hyp/vm/vcpu/aarch64/vcpu_aarch64.tc b/hyp/vm/vcpu/aarch64/vcpu_aarch64.tc index 3a66691..a4c0590 100644 --- a/hyp/vm/vcpu/aarch64/vcpu_aarch64.tc +++ b/hyp/vm/vcpu/aarch64/vcpu_aarch64.tc @@ -3,9 +3,9 @@ // SPDX-License-Identifier: BSD-3-Clause define vcpu_gpr structure { - x array(31) type register_t; + x array(31) type register_t(aligned(16)); pc bitfield ELR_EL2; - spsr_el2 bitfield SPSR_EL2_A64; + spsr_el2 union SPSR_EL2; }; define vector_register_t newtype array(16) uint8(); @@ -36,21 +36,27 @@ define vcpu_el1_registers structure { tpidr_el1 bitfield TPIDR_EL1; tcr_el1 bitfield TCR_EL1; vbar_el1 bitfield VBAR_EL1; -// zce_el1 -// trfcr_el1 -// apxxKeyxx_el1 -// RAS registers ERRxxx_el1, DISR_el1 -// SPE PMxxx_el1 -// PMU PMxx_el1 -// LOxxx_el1 -// vgic registers -// timer registers cntxxx_el1 -// AMU -// aarch32 guest registers +#if defined(ARCH_ARM_FEAT_CSV2_2) || defined(ARCH_ARM_FEAT_CSV2_1p2) || \ + defined(ARCH_ARM_FEAT_CSV2_3) + scxtnum_el0 uint64; + scxtnum_el1 uint64; +#endif +#if !defined(CPU_HAS_NO_ACTLR_EL1) + actlr_el1 bitfield ACTLR_EL1; +#endif +#if !defined(CPU_HAS_NO_AMAIR_EL1) + amair_el1 bitfield AMAIR_EL1; +#endif +#if !defined(CPU_HAS_NO_AFSR0_EL1) + afsr0_el1 bitfield AFSR0_EL1; +#endif +#if !defined(CPU_HAS_NO_AFSR1_EL1) + afsr1_el1 bitfield AFSR1_EL1; +#endif }; define vcpu_el2_registers structure { -#if defined(ARCH_ARM_8_1_VHE) +#if defined(ARCH_ARM_FEAT_VHE) cptr_el2 bitfield CPTR_EL2_E2H1; #else cptr_el2 bitfield CPTR_EL2_E2H0; @@ -60,17 +66,17 @@ define vcpu_el2_registers structure { }; extend vcpu object { - gpr structure vcpu_gpr; -#if defined(ARCH_ARM_8_3_PAUTH) - pauth structure aarch64_pauth_keys; + gpr structure vcpu_gpr(group(context_switch, registers, a)); +#if defined(ARCH_ARM_FEAT_PAuth) + pauth structure aarch64_pauth_keys(group(context_switch, registers, b)); #endif - fpr structure vcpu_vfp_registers; - el1 structure vcpu_el1_registers; - el2 structure vcpu_el2_registers; + fpr structure vcpu_vfp_registers(group(context_switch, registers, c)); + el1 structure vcpu_el1_registers(group(context_switch, registers, d)); + el2 structure vcpu_el2_registers(group(context_switch, registers, e)); - mpidr_el1 bitfield MPIDR_EL1; + mpidr_el1 bitfield MPIDR_EL1(group(context_switch, registers, d)); #if SCHEDULER_CAN_MIGRATE - midr_el1 bitfield MIDR_EL1; + midr_el1 bitfield MIDR_EL1(group(context_switch, registers, d)); #endif }; @@ -81,25 +87,18 @@ extend thread object module vcpu { }; #endif -// Relevant modules (such as the debug module) need to extend this bitfield -// and add their configuration flags for the hypercall_vcpu_configure -// hypercall. Then in their thread_activate handlers they need to check the -// values of these flags and act on them. -define vcpu_option_flags public bitfield<64> { - 0 pinned bool = 0; +extend vcpu_option_flags bitfield { 1 ras_error_handler bool = 0; 2 amu_counting_disabled bool = 0; 3 sve_allowed bool = 0; // 4 reserved: debug_allowed // 5 reserved: trace_allowed +#if defined(ARCH_ARM_FEAT_CSV2_2) || defined(ARCH_ARM_FEAT_CSV2_1p2) || \ + defined(ARCH_ARM_FEAT_CSV2_3) + 6 scxt_allowed bool = 0; +#endif + // 7 reserved: mpam_allowed + // 8 reserved: critical 63 hlos_vm bool = 0; others unknown = 0; }; - -extend thread object module vcpu { - // The option variable used for the hypercall_vcpu_configure hypercall. - options bitfield vcpu_option_flags; - - // Flag to indicate that the VCPU has been warm-reset - warm_reset bool; -}; diff --git a/hyp/vm/vcpu/armv8-64/include/vectors_vcpu.inc b/hyp/vm/vcpu/armv8-64/include/vectors_vcpu.inc index 1b6bdf6..fe2fa4b 100644 --- a/hyp/vm/vcpu/armv8-64/include/vectors_vcpu.inc +++ b/hyp/vm/vcpu/armv8-64/include/vectors_vcpu.inc @@ -8,7 +8,7 @@ #error The layout of vcpu_gpr_t has changed #endif -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) // Define symbols for pointer auth offsets so we can access them from macros .equ vcpu_pauth_tp_pauth_ofs, \ OFS_THREAD_VCPU_REGS_PAUTH - OFS_THREAD_VCPU_REGS_GPR @@ -38,14 +38,14 @@ #endif .macro vcpu_pauth_entry tp:req, kp:req, tl:req, th:req, kl:req, kh:req -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) thread_get_self \tp, offset=OFS_THREAD_VCPU_REGS_GPR vcpu_pauth_entry_thread \tp, \kp, \tl, \th, \kl, \kh #endif .endm .macro vcpu_pauth_entry_thread tp:req, kp:req, tl:req, th:req, kl:req, kh:req -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) adrl \kp, aarch64_pauth_keys vcpu_pauth_entry_key DA, \tp, \kp, \tl, \th, \kl, \kh vcpu_pauth_entry_key DB, \tp, \kp, \tl, \th, \kl, \kh @@ -57,14 +57,14 @@ .endm .macro vcpu_pauth_exit tp:req, tl:req, th:req, spsr:req, tmp:req -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) thread_get_self \tp, offset=OFS_THREAD_VCPU_REGS_GPR vcpu_pauth_exit_thread \tp, \tl, \th, \spsr, \tmp #endif .endm .macro vcpu_pauth_exit_thread tp:req, tl:req, th:req, spsr:req, tmp:req -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) // There is no need to sign userspace return addresses, but we do need // to check that we are actually returning to userspace here: either // SPSR_EL2.M[4]=1 (for 32-bit mode) or SPSR_EL2.M[3]=0 (for EL1 or diff --git a/hyp/vm/vcpu/armv8-64/src/return.S b/hyp/vm/vcpu/armv8-64/src/return.S index c069428..fdecc1d 100644 --- a/hyp/vm/vcpu/armv8-64/src/return.S +++ b/hyp/vm/vcpu/armv8-64/src/return.S @@ -93,6 +93,7 @@ function_chain vcpu_hypercall_return_sanitize_x2, vcpu_hypercall_return_sanitize mov x17, xzr + // FIXME: // Temporarily preserve x8 for backwards compatibility. It should be // sanitised once this is removed. //mov x8, xzr @@ -108,7 +109,7 @@ function_chain vcpu_hypercall_return_sanitize_x2, vcpu_hypercall_return_sanitize eret function_end vcpu_hypercall_return_sanitize_x8 -#if defined(ARCH_ARM_8_3_PAUTH) +#if defined(ARCH_ARM_FEAT_PAuth) function vcpu_pauth_exit_failed panic "Invalid SPSR_EL2.M in VCPU exception return" function_end vcpu_pauth_exit_failed diff --git a/hyp/vm/vcpu/armv8-64/src/vectors.S b/hyp/vm/vcpu/armv8-64/src/vectors.S index 9144d65..68f663c 100644 --- a/hyp/vm/vcpu/armv8-64/src/vectors.S +++ b/hyp/vm/vcpu/armv8-64/src/vectors.S @@ -34,6 +34,7 @@ cmp x18, HYPERCALL_NUM b.hs LOCAL(\el\()_non_hypercall) + // FIXME: // Temporarily preserve x8 for backwards compatibility. stp x8, x19, [sp, #-16]! @@ -70,7 +71,7 @@ local \el\()_non_hypercall: .global vcpu_aarch64_vectors vcpu_aarch64_vectors: -el2_vectors vcpu +el2_vectors vcpu 1 // Guest vectors // The assumption is that upon entry, SP (SP_EL2) points to the kernel stack, @@ -107,9 +108,8 @@ vector vector_guest64_fiq vector_end vector_guest64_fiq vector vector_guest64_serror - disable_phys_access - - b exception_64_entry + save_guest_context_vcpu_zero_0_13_irq + b error_64_entry vector_end vector_guest64_serror @@ -145,9 +145,8 @@ vector vector_guest32_fiq vector_end vector_guest32_fiq vector vector_guest32_serror - disable_phys_access - - bl exception_32_entry + save_kernel_context_vcpu_zero_0_13_irq + bl error_32_entry vector_end vector_guest32_serror #else // !ARCH_AARCH64_32BIT_EL1 @@ -202,7 +201,7 @@ function hypercall_64_entry local, align=(1 << CPU_L1D_LINE_BITS) stp x10, x11, [sp, #-16]! // Compute hypercall jump table index. -#if defined(ARCH_ARM_8_5_BTI) +#if defined(ARCH_ARM_FEAT_BTI) add x9, x9, x18, lsl 4 #else add x9, x9, x18, lsl 3 @@ -225,6 +224,13 @@ function exception_64_entry local, align=(1 << CPU_L1D_LINE_BITS) b vcpu_exception_return function_end exception_64_entry +function error_64_entry local, align=(1 << CPU_L1D_LINE_BITS) + save_guest_context_vcpu_zero_14_29_irq + + bl vcpu_error_dispatch + b vcpu_exception_return +function_end error_64_entry + function irq_64_entry local, align=(1 << CPU_L1D_LINE_BITS) save_guest_context_vcpu_zero_14_29_irq @@ -241,6 +247,13 @@ function exception_32_entry local, align=(1 << CPU_L1D_LINE_BITS) b vcpu_exception_return function_end exception_32_entry +function error_32_entry local, align=(1 << CPU_L1D_LINE_BITS) + save_guest_context_vcpu_zero_14_29_irq + + bl vcpu_error_dispatch + b vcpu_exception_return +function_end error_32_entry + function irq_32_entry local, align=(1 << CPU_L1D_LINE_BITS) save_guest_context_vcpu_zero_14_29_irq diff --git a/hyp/vm/vcpu/armv8-64/templates/vectors_tramp.S.tmpl b/hyp/vm/vcpu/armv8-64/templates/vectors_tramp.S.tmpl new file mode 100644 index 0000000..c1f1cc1 --- /dev/null +++ b/hyp/vm/vcpu/armv8-64/templates/vectors_tramp.S.tmpl @@ -0,0 +1,81 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +\#include + +\#include +\#include + +\#include "vectors_el2.inc" + +.macro mitigate_spectre_bhb_loop count:req + stp x18, xzr, [sp, #-16]! + mov x18, #\count + +local bhb_flush_loop\@: + b LOCAL(bhb_flush_forward_jump\@) +local bhb_flush_forward_jump\@: + subs x18, x18, #1 + b.ne LOCAL(bhb_flush_loop\@) + +\#if defined(ARCH_ARM_FEAT_SB) + sb +\#else + dsb nsh + isb +\#endif + ldp x18, xzr, [sp], #16 +.endm + +.macro mitigate_spectre_bhb_clrbhb + hint #22 + isb +.endm + +.macro vector_tramp name:req label:req content:req content_args:vararg +vector vector_\name\()_vcpu_tramp_\label + \content \content_args + b vector_\name +vector_end vector_\name\()_vcpu_tramp_\label +.endm + +.macro vcpu_vector name:req content:req content_args:vararg + .section .text.vectors; + .balign 2048; +.global vcpu_aarch64_vectors_tramp_\name +vcpu_aarch64_vectors_tramp_\name: + + el2_vectors vcpu_tramp_\name 1 + + vector_tramp guest64_sync \name \content \content_args + vector_tramp guest64_irq \name \content \content_args + vector_tramp guest64_fiq \name \content \content_args + vector_tramp guest64_serror \name \content \content_args + + vector_tramp guest32_sync \name \content \content_args + vector_tramp guest32_irq \name \content \content_args + vector_tramp guest32_fiq \name \content \content_args + vector_tramp guest32_serror \name \content \content_args +.endm + +\#if defined(ARCH_ARM_FEAT_CLRBHB) +vcpu_vector clrbhb mitigate_spectre_bhb_clrbhb +\#endif +## +#set loop_counts = set() +#set target_cpu_ids = $ARCH_CORE_IDS.split(',') +## +#for cpu_id in $ARCH_CORE_IDS.split(',') +#set _loops = 'SPECTRE_{:s}_BHB_LOOP_FLUSH'.format(cpu_id) +#if self.varExists($_loops) +#set loops=int(self.getVar($_loops)) +#silent loop_counts.add($loops) +#end if +#end for +## +#for loop_count in sorted(loop_counts) +#if loop_count > 0 +vcpu_vector bhb_loop_${loop_count} mitigate_spectre_bhb_loop ${loop_count} +#end if +#end for diff --git a/hyp/vm/vcpu/armv8-64/templates/vectors_tramp.c.tmpl b/hyp/vm/vcpu/armv8-64/templates/vectors_tramp.c.tmpl new file mode 100644 index 0000000..d41b9a0 --- /dev/null +++ b/hyp/vm/vcpu/armv8-64/templates/vectors_tramp.c.tmpl @@ -0,0 +1,134 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#set loop_counts = set() +#set loop_cores = dict() +#set unaffected_cores = dict() +#set unhandled_cores = dict() +#set hwworkaround_cores = dict() +#set target_cpu_ids = $ARCH_CORE_IDS.split(',') +## +#for cpu_id in $ARCH_CORE_IDS.split(',') +#set _nospec = 'SPECTRE_{:s}_NO_SPECULATION'.format(cpu_id) +#set _loops = 'SPECTRE_{:s}_BHB_LOOP_FLUSH'.format(cpu_id) +#set _bpiall = 'SPECTRE_{:s}_BPIALL'.format(cpu_id) +#set _hwwrk = 'SPECTRE_{:s}_BHB_HW_WORKAROUND'.format(cpu_id) +#if self.varExists($_loops) +#set loops=int(self.getVar($_loops)) +#silent loop_counts.add($loops) +#silent loop_cores[$cpu_id] = $loops +#elif self.varExists($_bpiall) +#silent unhandled_cores[$cpu_id] = 1 +#elif self.varExists($_hwwrk) +#silent hwworkaround_cores[$cpu_id] = 1 +#elif self.varExists($_nospec) +#silent unaffected_cores[$cpu_id] = 1 +#else +#silent print('Spectre BHB config missing for', cpu_id) +#silent sys.exit(1) +#end if +#end for +\#include +\#include + +\#include + +\#include +\#include +\#include +\#include +\#include +\#include + +\#include "event_handlers.h" +\#include "vectors_vcpu.h" + +CPULOCAL_DECLARE(uintptr_t, vcpu_aarch64_vectors); + +extern uintptr_t vcpu_aarch64_vectors; +extern uintptr_t vcpu_aarch64_vectors_tramp_clrbhb; + +#for loop_count in sorted(loop_counts) +extern uintptr_t vcpu_aarch64_vectors_tramp_bhb_loop_${loop_count}; +#end for + +void +vcpu_arch_handle_boot_cpu_cold_init(cpu_index_t cpu) +{ + core_id_t core_id = get_current_core_id(); + + ID_AA64MMFR1_EL1_t mmfr1 = register_ID_AA64MMFR1_EL1_read(); +\#if defined(ARCH_ARM_FEAT_CLRBHB) + ID_AA64ISAR2_EL1_t isar2 = register_ID_AA64ISAR2_EL1_read(); +\#endif + + if (ID_AA64MMFR1_EL1_get_ECBHB(&mmfr1) == 1U) { + CPULOCAL_BY_INDEX(vcpu_aarch64_vectors, cpu) = + (uintptr_t)&vcpu_aarch64_vectors; + } +\#if defined(ARCH_ARM_FEAT_CLRBHB) + else if (ID_AA64ISAR2_EL1_get_CLRBHB(&isar2) == 1U) { + CPULOCAL_BY_INDEX(vcpu_aarch64_vectors, cpu) = + (uintptr_t)&vcpu_aarch64_vectors_tramp_clrbhb; + } +\#endif + else { +\#pragma clang diagnostic push +\#pragma clang diagnostic ignored "-Wswitch-enum" + switch(core_id) { +#for cpu_id in sorted(loop_cores): + case CORE_ID_${cpu_id}: + // BHB eviction loop in vector entry, ${loop_cores[cpu_id]} iterations + CPULOCAL_BY_INDEX(vcpu_aarch64_vectors, cpu) = + (uintptr_t)&vcpu_aarch64_vectors_tramp_bhb_loop_${loop_cores[cpu_id]}; + break; +#end for +#if len(unaffected_cores) +#for cpu_id in sorted(unaffected_cores): + case CORE_ID_${cpu_id}: +#end for + // Not vulnerable to Spectre-BHB + CPULOCAL_BY_INDEX(vcpu_aarch64_vectors, cpu) = + (uintptr_t)&vcpu_aarch64_vectors; + break; +#end if +#if len(unhandled_cores): +#for cpu_id in sorted(unhandled_cores): + case CORE_ID_${cpu_id}: +#end for + // Needs an SMC call or switch to AArch32 + panic("No firmware support for spectre-BHB yet!"); +#end if +#if len($hwworkaround_cores) +#for cpu_id in sorted(hwworkaround_cores): + case CORE_ID_${cpu_id}: +#end for + // Core is expected to have a HW workaround, ECBHB or + // CLRBHB +\#if defined(ARCH_ARM_SPECTRE_BHB_WARN) && !defined(NDEBUG) && defined(VERBOSE) && VERBOSE + CPULOCAL_BY_INDEX(vcpu_aarch64_vectors, cpu) = + (uintptr_t)&vcpu_aarch64_vectors; + LOG(ERROR, WARN, + "No spectre-BHB mitigation for unexpected core {:d}:{:d}", + cpu, (register_t)core_id); + break; +\#else + panic("No spectre-BHB mitigation for unexpected core"); +\#endif +#end if + default: +\#if defined(ARCH_ARM_SPECTRE_BHB_WARN) && !defined(NDEBUG) && defined(VERBOSE) && VERBOSE + CPULOCAL_BY_INDEX(vcpu_aarch64_vectors, cpu) = + (uintptr_t)&vcpu_aarch64_vectors; + LOG(ERROR, WARN, + "No spectre-BHB mitigation registered for unknown core {:d}:{:d}", + cpu, (register_t)core_id); + break; +\#else + panic("No spectre-BHB mitigation registered for unknown core"); +\#endif + } +\#pragma clang diagnostic pop + } +} diff --git a/hyp/vm/vcpu/armv8-64/vcpu_aarch64.ev b/hyp/vm/vcpu/armv8-64/vcpu_aarch64.ev new file mode 100644 index 0000000..ec82c6b --- /dev/null +++ b/hyp/vm/vcpu/armv8-64/vcpu_aarch64.ev @@ -0,0 +1,8 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module vcpu + +subscribe boot_cpu_cold_init + handler vcpu_arch_handle_boot_cpu_cold_init diff --git a/hyp/vm/vcpu/build.conf b/hyp/vm/vcpu/build.conf index 93faa9d..131e517 100644 --- a/hyp/vm/vcpu/build.conf +++ b/hyp/vm/vcpu/build.conf @@ -3,19 +3,35 @@ # SPDX-License-Identifier: BSD-3-Clause interface vcpu + +local_include types vcpu.tc events vcpu.ev source vcpu.c + base_module hyp/core/vectors + arch_types aarch64 vcpu_aarch64.tc arch_events aarch64 vcpu_aarch64.ev arch_local_include aarch64 arch_source aarch64 sysreg_traps.c exception_inject.c reg_access.c wfi.c arch_source aarch64 trap_dispatch.c aarch64_init.c context_switch.c +arch_events armv8-64 vcpu_aarch64.ev arch_local_include armv8-64 arch_source armv8-64 vectors.S return.S + +arch_hypercalls aarch64 hypercalls.hvc +arch_source aarch64 hypercalls.c + +arch_template simple armv8-64 vectors_tramp.S.tmpl vectors_tramp.c.tmpl +arch_source cortex-a-v8_0 sysreg_traps_cpu.c +arch_source cortex-a-v8_0 context_switch.c +arch_types cortex-a-v8_0 vcpu_aarch64.tc arch_source cortex-a-v8_2 sysreg_traps_cpu.c arch_source cortex-a-v8_2 context_switch.c arch_types cortex-a-v8_2 vcpu_aarch64.tc -arch_hypercalls aarch64 hypercalls.hvc -arch_source aarch64 hypercalls.c +arch_source cortex-a-v9 sysreg_traps_cpu.c +arch_source cortex-a-v9 context_switch.c +arch_types cortex-a-v9 vcpu_aarch64.tc +arch_source qemu-armv8-5a-rng sysreg_traps_cpu.c +arch_source qemu-armv8-5a-rng context_switch.c diff --git a/hyp/vm/vcpu/cortex-a-v8_0/src/context_switch.c b/hyp/vm/vcpu/cortex-a-v8_0/src/context_switch.c new file mode 100644 index 0000000..2e01b28 --- /dev/null +++ b/hyp/vm/vcpu/cortex-a-v8_0/src/context_switch.c @@ -0,0 +1,34 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include + +#include + +#include +#include + +#include + +#include "event_handlers.h" + +void +vcpu_context_switch_cpu_load(void) +{ + thread_t *thread = thread_get_self(); + + if (compiler_expected(thread->kind == THREAD_KIND_VCPU)) { + // No-op + } +} + +void +vcpu_context_switch_cpu_save(void) +{ + thread_t *thread = thread_get_self(); + + if (compiler_expected(thread->kind == THREAD_KIND_VCPU)) { + // No-op + } +} diff --git a/hyp/vm/vcpu/cortex-a-v8_0/src/sysreg_traps_cpu.c b/hyp/vm/vcpu/cortex-a-v8_0/src/sysreg_traps_cpu.c new file mode 100644 index 0000000..29f3ccf --- /dev/null +++ b/hyp/vm/vcpu/cortex-a-v8_0/src/sysreg_traps_cpu.c @@ -0,0 +1,67 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include +#include + +#include +#include + +#include "event_handlers.h" + +vcpu_trap_result_t +sysreg_read_cpu(ESR_EL2_ISS_MSR_MRS_t iss) +{ + (void)iss; + return VCPU_TRAP_RESULT_UNHANDLED; +} + +// ACTLR_EL2 defaults to zero on reset, which disables write access to these +// registers and traps them to EL2. We want to keep it that way for now as +// writing to these registers generally has dangerous side effects and we don't +// want the guest to mess with them. +vcpu_trap_result_t +sysreg_write_cpu(ESR_EL2_ISS_MSR_MRS_t iss) +{ + uint8_t opc0, opc1, crn, crm; + vcpu_trap_result_t ret = VCPU_TRAP_RESULT_EMULATED; + + // Assert this is a write + assert(!ESR_EL2_ISS_MSR_MRS_get_Direction(&iss)); + + // Remove the fields that are not used in the comparison + ESR_EL2_ISS_MSR_MRS_t temp_iss = iss; + ESR_EL2_ISS_MSR_MRS_set_Rt(&temp_iss, 0U); + ESR_EL2_ISS_MSR_MRS_set_Direction(&temp_iss, false); + + switch (ESR_EL2_ISS_MSR_MRS_raw(temp_iss)) { + case ISS_MRS_MSR_CPUACTLR_EL1: + // CPUACTLR2_EL1 does not exist on A55 + case ISS_MRS_MSR_CPUECTLR_EL1: + // WI + break; + + default: + opc0 = ESR_EL2_ISS_MSR_MRS_get_Op0(&iss); + opc1 = ESR_EL2_ISS_MSR_MRS_get_Op1(&iss); + crn = ESR_EL2_ISS_MSR_MRS_get_CRn(&iss); + crm = ESR_EL2_ISS_MSR_MRS_get_CRm(&iss); + + if ((opc0 == 3) && (opc1 == 0) && (crn == 15) && (crm >= 3) && + (crm <= 4)) { + // CLUSTER* registers, all WI. + } else if ((opc0 == 3) && ((opc1 == 0) || (opc1 == 6)) && + (crn == 15) && (crm >= 5) && (crm <= 6)) { + // CLUSTERPM* registers, all WI. + } else { + ret = VCPU_TRAP_RESULT_UNHANDLED; + } + break; + } + + return ret; +} diff --git a/hyp/vm/vcpu/cortex-a-v8_0/vcpu_aarch64.tc b/hyp/vm/vcpu/cortex-a-v8_0/vcpu_aarch64.tc new file mode 100644 index 0000000..b590786 --- /dev/null +++ b/hyp/vm/vcpu/cortex-a-v8_0/vcpu_aarch64.tc @@ -0,0 +1,6 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +// Extend vcpu_el1_registers struct here if required for cpu/family specific +// EL1/0 registers, and add context switch handling. diff --git a/hyp/vm/vcpu/cortex-a-v8_2/src/context_switch.c b/hyp/vm/vcpu/cortex-a-v8_2/src/context_switch.c index 35bccdf..2e01b28 100644 --- a/hyp/vm/vcpu/cortex-a-v8_2/src/context_switch.c +++ b/hyp/vm/vcpu/cortex-a-v8_2/src/context_switch.c @@ -6,6 +6,7 @@ #include +#include #include #include @@ -17,11 +18,8 @@ vcpu_context_switch_cpu_load(void) { thread_t *thread = thread_get_self(); - if (thread->kind == THREAD_KIND_VCPU) { - register_ACTLR_EL1_write(thread->vcpu_regs_el1.actlr_el1); - register_AMAIR_EL1_write(thread->vcpu_regs_el1.amair_el1); - register_AFSR0_EL1_write(thread->vcpu_regs_el1.afsr0_el1); - register_AFSR1_EL1_write(thread->vcpu_regs_el1.afsr1_el1); + if (compiler_expected(thread->kind == THREAD_KIND_VCPU)) { + // No-op } } @@ -30,10 +28,7 @@ vcpu_context_switch_cpu_save(void) { thread_t *thread = thread_get_self(); - if (thread->kind == THREAD_KIND_VCPU) { - thread->vcpu_regs_el1.actlr_el1 = register_ACTLR_EL1_read(); - thread->vcpu_regs_el1.amair_el1 = register_AMAIR_EL1_read(); - thread->vcpu_regs_el1.afsr0_el1 = register_AFSR0_EL1_read(); - thread->vcpu_regs_el1.afsr1_el1 = register_AFSR1_EL1_read(); + if (compiler_expected(thread->kind == THREAD_KIND_VCPU)) { + // No-op } } diff --git a/hyp/vm/vcpu/cortex-a-v8_2/vcpu_aarch64.tc b/hyp/vm/vcpu/cortex-a-v8_2/vcpu_aarch64.tc index 8867ddc..b590786 100644 --- a/hyp/vm/vcpu/cortex-a-v8_2/vcpu_aarch64.tc +++ b/hyp/vm/vcpu/cortex-a-v8_2/vcpu_aarch64.tc @@ -2,9 +2,5 @@ // // SPDX-License-Identifier: BSD-3-Clause -extend vcpu_el1_registers structure { - actlr_el1 bitfield ACTLR_EL1; - amair_el1 bitfield AMAIR_EL1; - afsr0_el1 bitfield AFSR0_EL1; - afsr1_el1 bitfield AFSR1_EL1; -}; +// Extend vcpu_el1_registers struct here if required for cpu/family specific +// EL1/0 registers, and add context switch handling. diff --git a/hyp/vm/vcpu/cortex-a-v9/src/context_switch.c b/hyp/vm/vcpu/cortex-a-v9/src/context_switch.c new file mode 100644 index 0000000..2e01b28 --- /dev/null +++ b/hyp/vm/vcpu/cortex-a-v9/src/context_switch.c @@ -0,0 +1,34 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include + +#include + +#include +#include + +#include + +#include "event_handlers.h" + +void +vcpu_context_switch_cpu_load(void) +{ + thread_t *thread = thread_get_self(); + + if (compiler_expected(thread->kind == THREAD_KIND_VCPU)) { + // No-op + } +} + +void +vcpu_context_switch_cpu_save(void) +{ + thread_t *thread = thread_get_self(); + + if (compiler_expected(thread->kind == THREAD_KIND_VCPU)) { + // No-op + } +} diff --git a/hyp/vm/vcpu/cortex-a-v9/src/sysreg_traps_cpu.c b/hyp/vm/vcpu/cortex-a-v9/src/sysreg_traps_cpu.c new file mode 100644 index 0000000..3748f6a --- /dev/null +++ b/hyp/vm/vcpu/cortex-a-v9/src/sysreg_traps_cpu.c @@ -0,0 +1,68 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include +#include + +#include +#include + +#include "event_handlers.h" + +vcpu_trap_result_t +sysreg_read_cpu(ESR_EL2_ISS_MSR_MRS_t iss) +{ + (void)iss; + return VCPU_TRAP_RESULT_UNHANDLED; +} + +// ACTLR_EL2 defaults to zero on reset, which disables write access to these +// registers and traps them to EL2. We want to keep it that way for now as +// writing to these registers generally has dangerous side effects and we don't +// want the guest to mess with them. +vcpu_trap_result_t +sysreg_write_cpu(ESR_EL2_ISS_MSR_MRS_t iss) +{ + uint8_t opc0, opc1, crn, crm; + vcpu_trap_result_t ret = VCPU_TRAP_RESULT_EMULATED; + + // Assert this is a write + assert(!ESR_EL2_ISS_MSR_MRS_get_Direction(&iss)); + + // Remove the fields that are not used in the comparison + ESR_EL2_ISS_MSR_MRS_t temp_iss = iss; + ESR_EL2_ISS_MSR_MRS_set_Rt(&temp_iss, 0U); + ESR_EL2_ISS_MSR_MRS_set_Direction(&temp_iss, false); + + switch (ESR_EL2_ISS_MSR_MRS_raw(temp_iss)) { + case ISS_MRS_MSR_CPUACTLR_EL1: + case ISS_MRS_MSR_A7X_CPUACTLR2_EL1: + case ISS_MRS_MSR_CPUECTLR_EL1: + case ISS_MRS_MSR_CPUPWRCTLR_EL1: + // WI + break; + + default: + opc0 = ESR_EL2_ISS_MSR_MRS_get_Op0(&iss); + opc1 = ESR_EL2_ISS_MSR_MRS_get_Op1(&iss); + crn = ESR_EL2_ISS_MSR_MRS_get_CRn(&iss); + crm = ESR_EL2_ISS_MSR_MRS_get_CRm(&iss); + + if ((opc0 == 3U) && (opc1 == 0U) && (crn == 15U) && + (crm >= 3U) && (crm <= 4U)) { + // CLUSTER* registers, all WI. + } else if ((opc0 == 3U) && ((opc1 == 0U) || (opc1 == 6U)) && + (crn == 15U) && (crm >= 5U) && (crm <= 6U)) { + // CLUSTERPM* registers, all WI. + } else { + ret = VCPU_TRAP_RESULT_UNHANDLED; + } + break; + } + + return ret; +} diff --git a/hyp/vm/vcpu/cortex-a-v9/vcpu_aarch64.tc b/hyp/vm/vcpu/cortex-a-v9/vcpu_aarch64.tc new file mode 100644 index 0000000..b590786 --- /dev/null +++ b/hyp/vm/vcpu/cortex-a-v9/vcpu_aarch64.tc @@ -0,0 +1,6 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +// Extend vcpu_el1_registers struct here if required for cpu/family specific +// EL1/0 registers, and add context switch handling. diff --git a/hyp/vm/vcpu/include/vcpu.h b/hyp/vm/vcpu/include/vcpu.h new file mode 100644 index 0000000..bc237aa --- /dev/null +++ b/hyp/vm/vcpu/include/vcpu.h @@ -0,0 +1,10 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +error_t +vcpu_bind_virq(thread_t *vcpu, vic_t *vic, virq_t virq, + vcpu_virq_type_t virq_type); + +error_t +vcpu_unbind_virq(thread_t *vcpu, vcpu_virq_type_t virq_type); diff --git a/hyp/vm/vcpu/qemu-armv8-5a-rng/src/context_switch.c b/hyp/vm/vcpu/qemu-armv8-5a-rng/src/context_switch.c new file mode 100644 index 0000000..8418624 --- /dev/null +++ b/hyp/vm/vcpu/qemu-armv8-5a-rng/src/context_switch.c @@ -0,0 +1,36 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include + +#include + +#include +#include + +#include + +#include "event_handlers.h" + +// There are no implementation specific EL1 registers to switch for QEMU. + +void +vcpu_context_switch_cpu_load(void) +{ + thread_t *thread = thread_get_self(); + + if (compiler_expected(thread->kind == THREAD_KIND_VCPU)) { + // No-op + } +} + +void +vcpu_context_switch_cpu_save(void) +{ + thread_t *thread = thread_get_self(); + + if (compiler_expected(thread->kind == THREAD_KIND_VCPU)) { + // No-op + } +} diff --git a/hyp/vm/vcpu/qemu-armv8-5a-rng/src/sysreg_traps_cpu.c b/hyp/vm/vcpu/qemu-armv8-5a-rng/src/sysreg_traps_cpu.c new file mode 100644 index 0000000..4b96576 --- /dev/null +++ b/hyp/vm/vcpu/qemu-armv8-5a-rng/src/sysreg_traps_cpu.c @@ -0,0 +1,30 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include +#include + +#include +// #include + +#include "event_handlers.h" + +// There are no implementation specific EL1 registers to emulate for QEMU. + +vcpu_trap_result_t +sysreg_read_cpu(ESR_EL2_ISS_MSR_MRS_t iss) +{ + (void)iss; + return VCPU_TRAP_RESULT_UNHANDLED; +} + +vcpu_trap_result_t +sysreg_write_cpu(ESR_EL2_ISS_MSR_MRS_t iss) +{ + (void)iss; + return VCPU_TRAP_RESULT_UNHANDLED; +} diff --git a/hyp/vm/vcpu/src/vcpu.c b/hyp/vm/vcpu/src/vcpu.c index 52471d5..903b2ee 100644 --- a/hyp/vm/vcpu/src/vcpu.c +++ b/hyp/vm/vcpu/src/vcpu.c @@ -5,13 +5,22 @@ #include #include +#include + +#include +#include #include +#include #include #include +#include +#include +#include #include #include "event_handlers.h" +#include "vcpu.h" void vcpu_handle_object_get_defaults_thread(thread_create_t *create) @@ -19,11 +28,14 @@ vcpu_handle_object_get_defaults_thread(thread_create_t *create) uint32_t stack_size; assert(create != NULL); - assert(create->kind == THREAD_KIND_NONE); // This may be 0, which will fall back to the global default stack_size = platform_cpu_stack_size(); +#if defined(VCPU_MIN_STACK_SIZE) + stack_size = util_max(stack_size, VCPU_MIN_STACK_SIZE); +#endif + assert((stack_size == 0U) || util_is_baligned(stack_size, PGTABLE_HYP_PAGE_SIZE)); assert(stack_size <= THREAD_STACK_MAX_SIZE); @@ -37,12 +49,20 @@ vcpu_handle_object_create_thread(thread_create_t create) { thread_t *thread = create.thread; assert(thread != NULL); + error_t ret; if (thread->kind == THREAD_KIND_VCPU) { scheduler_block_init(thread, SCHEDULER_BLOCK_VCPU_OFF); } - return OK; + if (create.scheduler_priority_valid && + (create.scheduler_priority > VCPU_MAX_PRIORITY)) { + ret = ERROR_DENIED; + } else { + ret = OK; + } + + return ret; } error_t @@ -58,8 +78,8 @@ vcpu_handle_object_activate_thread(thread_t *thread) goto out; } - if (((1UL << thread->scheduler_affinity) & - PLATFORM_USABLE_CORES) == 0) { + if (cpulocal_index_valid(thread->scheduler_affinity) && + !platform_cpu_exists(thread->scheduler_affinity)) { ret = ERROR_OBJECT_CONFIG; goto out; } @@ -78,3 +98,89 @@ vcpu_handle_object_activate_thread(thread_t *thread) out: return ret; } + +void +vcpu_handle_thread_exited(void) +{ + thread_t *current = thread_get_self(); + assert(current != NULL); + + assert_preempt_disabled(); + + if (current->kind == THREAD_KIND_VCPU) { + if (vcpu_option_flags_get_critical(¤t->vcpu_options)) { + panic("Critical VCPU exited"); + } + + trigger_vcpu_stopped_event(); + } +} + +bool +vcpu_handle_vcpu_activate_thread(thread_t *thread, vcpu_option_flags_t options) +{ + bool ret = false; + + assert(thread != NULL); + assert(thread->kind == THREAD_KIND_VCPU); + + // Check that the partition has the right to mark the VCPU as critical. + if (vcpu_option_flags_get_critical(&options) || + vcpu_option_flags_get_hlos_vm(&options)) { + if (!partition_option_flags_get_privileged( + &thread->header.partition->options)) { + goto out; + } + + vcpu_option_flags_set_critical(&thread->vcpu_options, true); + } + + ret = true; + +out: + return ret; +} + +void +vcpu_handle_object_deactivate_thread(thread_t *thread) +{ + if (thread->kind == THREAD_KIND_VCPU) { + vic_unbind(&thread->vcpu_halt_virq_src); + } +} + +error_t +vcpu_bind_virq(thread_t *vcpu, vic_t *vic, virq_t virq, + vcpu_virq_type_t virq_type) +{ + return trigger_vcpu_bind_virq_event(virq_type, vcpu, vic, virq); +} + +error_t +vcpu_unbind_virq(thread_t *vcpu, vcpu_virq_type_t virq_type) +{ + return trigger_vcpu_unbind_virq_event(virq_type, vcpu); +} + +error_t +vcpu_handle_vcpu_bind_virq(thread_t *vcpu, vic_t *vic, virq_t virq) +{ + error_t err = vic_bind_shared(&vcpu->vcpu_halt_virq_src, vic, virq, + VIRQ_TRIGGER_VCPU_HALT); + + return err; +} + +error_t +vcpu_handle_vcpu_unbind_virq(thread_t *vcpu) +{ + vic_unbind_sync(&vcpu->vcpu_halt_virq_src); + + return OK; +} + +irq_trigger_result_t +vcpu_handle_virq_set_mode(void) +{ + return irq_trigger_result_ok(IRQ_TRIGGER_EDGE_RISING); +} diff --git a/hyp/vm/vcpu/vcpu.ev b/hyp/vm/vcpu/vcpu.ev index 1312ff8..ec1fccb 100644 --- a/hyp/vm/vcpu/vcpu.ev +++ b/hyp/vm/vcpu/vcpu.ev @@ -13,6 +13,21 @@ subscribe object_create_thread subscribe object_activate_thread priority -100 +subscribe thread_exited + require_preempt_disabled + +#if defined(MODULE_VM_VCPU_RUN) +subscribe vcpu_run_check(vcpu, state_data_0) + priority last +#endif + +subscribe virq_set_mode[VIRQ_TRIGGER_VCPU_HALT] + handler vcpu_handle_virq_set_mode() + +subscribe vcpu_bind_virq[VCPU_VIRQ_TYPE_HALT](vcpu, vic, virq) + +subscribe vcpu_unbind_virq[VCPU_VIRQ_TYPE_HALT](vcpu) + interface vcpu setup_event vcpu_activate_thread diff --git a/hyp/vm/vcpu/vcpu.tc b/hyp/vm/vcpu/vcpu.tc index 4c879a5..d4fd405 100644 --- a/hyp/vm/vcpu/vcpu.tc +++ b/hyp/vm/vcpu/vcpu.tc @@ -2,24 +2,34 @@ // // SPDX-License-Identifier: BSD-3-Clause +// Reserve highest possible priority for EL2 tasks only (typically used for +// tasks that execute prior to boot) +define VCPU_MAX_PRIORITY public constant type priority_t = + SCHEDULER_MAX_PRIORITY - 1; + define vcpu object { }; extend thread object module vcpu { - regs object vcpu; + regs object vcpu; + // The option variable used for the hypercall_vcpu_configure hypercall. + options bitfield vcpu_option_flags; + halt_virq_src structure virq_source(contained); }; extend thread_kind enumeration { - vcpu; + // Having THREAD_KIND_VCPU as 0 and using the correct compiler expected + // and unexpected directives helps with optimising the context-switch + // path and any other code that checks the thread kind against this + // value, replacing the "cmp/b.ne" pair with a single "cbz" or "cbnz". + vcpu = 0; }; extend scheduler_block enumeration { vcpu_off; vcpu_suspend; vcpu_wfi; -#if !defined(NDEBUG) vcpu_fault; -#endif }; extend hyp_api_flags0 bitfield { @@ -30,3 +40,21 @@ extend hyp_api_flags0 bitfield { extend abort_reason enumeration { UNHANDLED_EXCEPTION; }; + +// Relevant modules (such as the debug module) need to extend this bitfield +// and add their configuration flags for the hypercall_vcpu_configure +// hypercall. Then in their thread_activate handlers they need to check the +// values of these flags and act on them. +define vcpu_option_flags public bitfield<64> { + 0 pinned bool = 0; + 8 critical bool = 0; + others unknown = 0; +}; + +extend vcpu_virq_type enumeration { + halt = 0; +}; + +extend virq_trigger enumeration { + vcpu_halt; +}; diff --git a/hyp/vm/vcpu_power/build.conf b/hyp/vm/vcpu_power/build.conf new file mode 100644 index 0000000..db47606 --- /dev/null +++ b/hyp/vm/vcpu_power/build.conf @@ -0,0 +1,7 @@ +# © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +types vcpu_power.tc +events vcpu_power.ev +source vcpu_power.c diff --git a/hyp/vm/vcpu_power/src/vcpu_power.c b/hyp/vm/vcpu_power/src/vcpu_power.c new file mode 100644 index 0000000..3d72e40 --- /dev/null +++ b/hyp/vm/vcpu_power/src/vcpu_power.c @@ -0,0 +1,162 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include +#include +#include +#include + +#if defined(INTERFACE_VCPU_RUN) +#include +#include +#endif + +#include "event_handlers.h" + +error_t +vcpu_power_handle_vcpu_poweron(thread_t *vcpu) +{ + cpu_index_t cpu = scheduler_get_affinity(vcpu); + bool should_vote = cpulocal_index_valid(cpu); + +#if defined(INTERFACE_VCPU_RUN) + if (vcpu_run_is_enabled(vcpu)) { + should_vote = false; + } +#endif + + error_t ret; + if (should_vote) { + ret = power_vote_cpu_on(cpu); + } else { + ret = OK; + } + + return ret; +} + +error_t +vcpu_power_handle_vcpu_poweroff(thread_t *vcpu) +{ + cpu_index_t cpu = scheduler_get_affinity(vcpu); + bool should_vote = cpulocal_index_valid(cpu); + +#if defined(INTERFACE_VCPU_RUN) + if (vcpu_run_is_enabled(vcpu)) { + should_vote = false; + } +#endif + + if (should_vote) { + power_vote_cpu_off(cpu); + } + + return OK; +} + +void +vcpu_power_handle_vcpu_stopped(void) +{ + thread_t *vcpu = thread_get_self(); + assert((vcpu != NULL) && (vcpu->kind == THREAD_KIND_VCPU)); + + scheduler_lock_nopreempt(vcpu); + + cpu_index_t cpu = scheduler_get_affinity(vcpu); + bool should_vote = cpulocal_index_valid(cpu); + + if (scheduler_is_blocked(vcpu, SCHEDULER_BLOCK_VCPU_OFF)) { + // If the VCPU is already powered off, it does not hold a vote. + should_vote = false; + } + + if (should_vote) { + power_vote_cpu_off(cpu); + } + + scheduler_unlock_nopreempt(vcpu); +} + +#if defined(INTERFACE_VCPU_RUN) +void +vcpu_power_handle_vcpu_run_disabled(thread_t *vcpu) +{ + cpu_index_t cpu = scheduler_get_affinity(vcpu); + bool should_vote = cpulocal_index_valid(cpu); + + if (scheduler_is_blocked(vcpu, SCHEDULER_BLOCK_VCPU_OFF)) { + should_vote = false; + } + + error_t err; + if (should_vote) { + err = power_vote_cpu_on(cpu); + } else { + err = OK; + } + + if (err != OK) { + // Note: vcpu_run is still enabled when this event is triggered, + // so the affinity change handler won't cast a duplicate vote. + err = scheduler_set_affinity(vcpu, CPU_INDEX_INVALID); + + // If there's already an affinity change in progress for the + // VCPU it is not possible to retry at this point. + if (err == ERROR_RETRY) { + // scheduler_lock(vcpu) already held here + scheduler_block(vcpu, SCHEDULER_BLOCK_VCPU_FAULT); + vcpu_halted(); + } + } +} + +void +vcpu_power_handle_vcpu_run_enabled(thread_t *vcpu) +{ + cpu_index_t cpu = scheduler_get_affinity(vcpu); + bool should_vote = cpulocal_index_valid(cpu); + + if (scheduler_is_blocked(vcpu, SCHEDULER_BLOCK_VCPU_OFF)) { + should_vote = false; + } + + if (should_vote) { + power_vote_cpu_off(cpu); + } +} +#endif + +error_t +vcpu_power_handle_scheduler_set_affinity_prepare(thread_t *vcpu, + cpu_index_t prev_cpu, + cpu_index_t next_cpu) +{ + error_t ret = OK; + assert(prev_cpu != next_cpu); + + if (vcpu->kind != THREAD_KIND_VCPU) { + goto out; + } + +#if defined(INTERFACE_VCPU_RUN) + if (vcpu_run_is_enabled(vcpu)) { + goto out; + } +#endif + + if (!scheduler_is_blocked(vcpu, SCHEDULER_BLOCK_VCPU_OFF)) { + if (cpulocal_index_valid(next_cpu)) { + ret = power_vote_cpu_on(next_cpu); + } + if ((ret == OK) && cpulocal_index_valid(prev_cpu)) { + power_vote_cpu_off(prev_cpu); + } + } + +out: + return ret; +} diff --git a/hyp/vm/vcpu_power/vcpu_power.ev b/hyp/vm/vcpu_power/vcpu_power.ev new file mode 100644 index 0000000..aa8e720 --- /dev/null +++ b/hyp/vm/vcpu_power/vcpu_power.ev @@ -0,0 +1,30 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module vcpu_power + +subscribe vcpu_poweron(vcpu) + priority first + require_scheduler_lock(vcpu) + +subscribe vcpu_poweroff(current) + require_scheduler_lock(current) + +subscribe vcpu_stopped() + require_preempt_disabled + +#if defined(INTERFACE_VCPU_RUN) + +subscribe vcpu_run_enabled + require_scheduler_lock(vcpu) + +subscribe vcpu_run_disabled + require_scheduler_lock(vcpu) + +#endif + +subscribe scheduler_set_affinity_prepare(thread, prev_cpu, next_cpu) + // Run late to avoid unwinding. + priority last + require_scheduler_lock(thread) diff --git a/hyp/vm/vcpu_power/vcpu_power.tc b/hyp/vm/vcpu_power/vcpu_power.tc new file mode 100644 index 0000000..f4680e8 --- /dev/null +++ b/hyp/vm/vcpu_power/vcpu_power.tc @@ -0,0 +1,3 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause diff --git a/hyp/vm/vcpu_run/build.conf b/hyp/vm/vcpu_run/build.conf new file mode 100644 index 0000000..11578f7 --- /dev/null +++ b/hyp/vm/vcpu_run/build.conf @@ -0,0 +1,8 @@ +# © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +interface vcpu_run +types vcpu_run.tc +events vcpu_run.ev +source vcpu_run.c diff --git a/hyp/vm/vcpu_run/src/vcpu_run.c b/hyp/vm/vcpu_run/src/vcpu_run.c new file mode 100644 index 0000000..e59f33a --- /dev/null +++ b/hyp/vm/vcpu_run/src/vcpu_run.c @@ -0,0 +1,286 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "event_handlers.h" + +error_t +vcpu_run_handle_object_activate_thread(thread_t *thread) +{ + assert(thread != NULL); + + if (thread->kind == THREAD_KIND_VCPU) { + task_queue_init(&thread->vcpu_run_wakeup_virq_task, + TASK_QUEUE_CLASS_VCPU_RUN_WAKEUP_VIRQ); + + thread->vcpu_run_last_state = VCPU_RUN_STATE_READY; + } + + return OK; +} + +bool +vcpu_run_is_enabled(const thread_t *vcpu) +{ + return vcpu->vcpu_run_enabled; +} + +hypercall_vcpu_run_result_t +hypercall_vcpu_run(cap_id_t vcpu_cap_id, register_t resume_data_0, + register_t resume_data_1, register_t resume_data_2) +{ + hypercall_vcpu_run_result_t ret = { 0 }; + cspace_t *cspace = cspace_get_self(); + + thread_ptr_result_t thread_r = cspace_lookup_thread( + cspace, vcpu_cap_id, + cap_rights_thread_union(CAP_RIGHTS_THREAD_AFFINITY, + CAP_RIGHTS_THREAD_YIELD_TO)); + if (compiler_unexpected(thread_r.e != OK)) { + ret.error = thread_r.e; + goto out_err; + } + + thread_t *vcpu = thread_r.r; + if (compiler_unexpected(vcpu->kind != THREAD_KIND_VCPU)) { + ret.error = ERROR_ARGUMENT_INVALID; + goto out_obj_put_thread; + } + + scheduler_lock(vcpu); + if (!scheduler_is_blocked(vcpu, SCHEDULER_BLOCK_VCPU_RUN)) { + // VCPU not proxy-scheduled, or is being run by another caller + ret.error = ERROR_BUSY; + goto unlock; + } + + ret.error = trigger_vcpu_run_resume_event(vcpu->vcpu_run_last_state, + vcpu, resume_data_0, + resume_data_1, resume_data_2); + if (ret.error != OK) { + goto unlock; + } + + (void)scheduler_unblock(vcpu, SCHEDULER_BLOCK_VCPU_RUN); + + if (scheduler_is_runnable(vcpu)) { + assert_cpulocal_safe(); + cpu_index_t this_pcpu = cpulocal_get_index(); + // Make sure the vCPU will run on this PCPU. Note that this + // might block the thread for an RCU grace period, which will + // show up as a brief transient VCPU_RUN_STATE_BLOCKED. To + // prevent that persisting indefinitely, the caller should avoid + // migration as much as possible. + ret.error = scheduler_set_affinity(vcpu, this_pcpu); + if (ret.error != OK) { + goto unlock; + } + + // Use a nopreempt unlock to make sure we don't get migrated + scheduler_unlock_nopreempt(vcpu); + scheduler_yield_to(vcpu); + scheduler_lock_nopreempt(vcpu); + } + + if (scheduler_is_runnable(vcpu)) { + ret.vcpu_state = VCPU_RUN_STATE_READY; + } else { + ret.vcpu_state = trigger_vcpu_run_check_event( + vcpu, &ret.state_data_0, &ret.state_data_1, + &ret.state_data_2); + } + vcpu->vcpu_run_last_state = ret.vcpu_state; + + scheduler_block(vcpu, SCHEDULER_BLOCK_VCPU_RUN); +unlock: + scheduler_unlock(vcpu); +out_obj_put_thread: + object_put_thread(vcpu); +out_err: + return ret; +} + +vcpu_run_state_t +vcpu_run_handle_vcpu_run_check(const thread_t *vcpu, register_t *state_data_0) +{ + vcpu_run_state_t ret = VCPU_RUN_STATE_BLOCKED; + if (compiler_unexpected(thread_has_exited(vcpu))) { + vcpu_run_poweroff_flags_t flags = + vcpu_run_poweroff_flags_default(); + ret = VCPU_RUN_STATE_POWERED_OFF; + vcpu_run_poweroff_flags_set_exited(&flags, true); + *state_data_0 = vcpu_run_poweroff_flags_raw(flags); + } + return ret; +} + +hypercall_vcpu_run_check_result_t +hypercall_vcpu_run_check(cap_id_t vcpu_cap_id) +{ + hypercall_vcpu_run_check_result_t ret = { 0 }; + cspace_t *cspace = cspace_get_self(); + + thread_ptr_result_t thread_r = cspace_lookup_thread( + cspace, vcpu_cap_id, + cap_rights_thread_union(CAP_RIGHTS_THREAD_BIND_VIRQ, + CAP_RIGHTS_THREAD_STATE)); + if (compiler_unexpected(thread_r.e != OK)) { + ret.error = thread_r.e; + goto out_err; + } + + thread_t *vcpu = thread_r.r; + if (compiler_unexpected(vcpu->kind != THREAD_KIND_VCPU)) { + ret.error = ERROR_ARGUMENT_INVALID; + goto out_obj_put_thread; + } + + scheduler_lock(vcpu); + if (scheduler_is_runnable(vcpu)) { + ret.error = ERROR_BUSY; + } else { + ret.vcpu_state = trigger_vcpu_run_check_event( + vcpu, &ret.state_data_0, &ret.state_data_1, + &ret.state_data_2); + if (ret.vcpu_state == VCPU_RUN_STATE_BLOCKED) { + ret.error = ERROR_BUSY; + } + } + scheduler_unlock(vcpu); + +out_obj_put_thread: + object_put_thread(vcpu); +out_err: + return ret; +} + +error_t +vcpu_run_handle_vcpu_bind_virq(thread_t *vcpu, vic_t *vic, virq_t virq) +{ + scheduler_lock(vcpu); + + error_t err = vic_bind_shared(&vcpu->vcpu_run_wakeup_virq, vic, virq, + VIRQ_TRIGGER_VCPU_RUN_WAKEUP); + if (err == OK) { + scheduler_block(vcpu, SCHEDULER_BLOCK_VCPU_RUN); + vcpu->vcpu_run_enabled = true; + trigger_vcpu_run_enabled_event(vcpu); + } + + scheduler_unlock(vcpu); + + return err; +} + +error_t +vcpu_run_handle_vcpu_unbind_virq(thread_t *vcpu) +{ + scheduler_lock(vcpu); + if (vcpu->vcpu_run_enabled) { + trigger_vcpu_run_disabled_event(vcpu); + vcpu->vcpu_run_enabled = false; + if (scheduler_unblock(vcpu, SCHEDULER_BLOCK_VCPU_RUN)) { + scheduler_trigger(); + } + } + scheduler_unlock(vcpu); + + vic_unbind_sync(&vcpu->vcpu_run_wakeup_virq); + + return OK; +} + +error_t +vcpu_run_handle_task_queue_execute(task_queue_entry_t *task_entry) +{ + assert(task_entry != NULL); + thread_t *vcpu = + thread_container_of_vcpu_run_wakeup_virq_task(task_entry); + + assert(vcpu != NULL); + assert(vcpu->kind == THREAD_KIND_VCPU); + + (void)virq_assert(&vcpu->vcpu_run_wakeup_virq, true); + object_put_thread(vcpu); + + return OK; +} + +void +vcpu_run_trigger_virq(thread_t *vcpu) +{ + assert(vcpu != NULL); + assert(vcpu->kind == THREAD_KIND_VCPU); + + if (scheduler_is_blocked(vcpu, SCHEDULER_BLOCK_VCPU_RUN)) { + (void)object_get_thread_additional(vcpu); + if (task_queue_schedule(&vcpu->vcpu_run_wakeup_virq_task) != + OK) { + object_put_thread(vcpu); + } + } +} + +error_t +vcpu_run_handle_vcpu_poweron(thread_t *vcpu) +{ + vcpu_run_trigger_virq(vcpu); + return OK; +} + +void +vcpu_run_handle_thread_killed(thread_t *thread) +{ + assert(thread != NULL); + if (thread->kind == THREAD_KIND_VCPU) { + // Killing the VCPU may have made it temporarily runnable so + // it can unwind its EL2 stack. Raise a scheduling doorbell. + vcpu_run_trigger_virq(thread); + } +} + +void +vcpu_run_handle_object_deactivate_thread(thread_t *thread) +{ + if (thread->kind == THREAD_KIND_VCPU) { + vic_unbind(&thread->vcpu_run_wakeup_virq); + } +} + +scheduler_block_properties_t +vcpu_run_handle_scheduler_get_block_properties(scheduler_block_t block) +{ + assert(block == SCHEDULER_BLOCK_VCPU_RUN); + + // Set the vcpu_run block flag as non-killable to ensure that killed + // VCPUs continue to be scheduled normally. + scheduler_block_properties_t props = + scheduler_block_properties_default(); + scheduler_block_properties_set_non_killable(&props, true); + + return props; +} diff --git a/hyp/vm/vcpu_run/vcpu_run.ev b/hyp/vm/vcpu_run/vcpu_run.ev new file mode 100644 index 0000000..eb6561e --- /dev/null +++ b/hyp/vm/vcpu_run/vcpu_run.ev @@ -0,0 +1,70 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +interface vcpu_run + +// Triggered at the start of vcpu_run() to handle resume data specified by a +// VCPU run state. A handler is needed for any run state that defines the +// semantics of the resume data words. +// +// The run state is the last value returned by vcpu_run_check for this VCPU. +// If vcpu_run_check has never been triggered before, the run state is +// VCPU_RUN_STATE_READY, which should not have a handler for this event. +// +// This event is called with the VCPU's scheduler lock held. +selector_event vcpu_run_resume + selector run_state: vcpu_run_state_t + param vcpu: thread_t * + param resume_data_0: register_t + param resume_data_1: register_t + param resume_data_2: register_t + return: error_t = OK + +// Triggered by vcpu_run() if the targeted thread is found to be blocked, +// either before or after attempting to yield to it. +// +// Handlers for this event should try to determine why the VCPU cannot +// continue to run. If the reason is known, the corresponding vcpu_run_state_t +// value should be returned; otherwise, return VCPU_RUN_STATE_BLOCKED. +// +// A handler is needed in any module that either defines a vcpu_run_state +// value, or implements the vcpu_expects_wakeup event (which has a +// corresponding run state, VCPU_RUN_STATE_EXPECTS_WAKEUP). +// +// If the returned run state value defines extra data, e.g. the address and +// size of a faulting memory access, it should be returned in the state_data_* +// pointers. +// +// This event is called with the VCPU's scheduler lock held. +handled_event vcpu_run_check + param vcpu: const thread_t * + param state_data_0: register_t * + param state_data_1: register_t * + param state_data_2: register_t * + return: vcpu_run_state_t = VCPU_RUN_STATE_BLOCKED + +module vcpu_run + +subscribe object_activate_thread(thread) + +subscribe object_deactivate_thread(thread) + +subscribe task_queue_execute[TASK_QUEUE_CLASS_VCPU_RUN_WAKEUP_VIRQ](entry) + +subscribe vcpu_wakeup + handler vcpu_run_trigger_virq + +subscribe vcpu_poweron(vcpu) + +subscribe thread_killed + +subscribe vcpu_bind_virq[VCPU_VIRQ_TYPE_VCPU_RUN_WAKEUP](vcpu, vic, virq) + +subscribe vcpu_unbind_virq[VCPU_VIRQ_TYPE_VCPU_RUN_WAKEUP](vcpu) + +subscribe scheduler_get_block_properties[SCHEDULER_BLOCK_VCPU_RUN] + +subscribe vcpu_run_check(vcpu, state_data_0) + priority first + require_scheduler_lock(vcpu) diff --git a/hyp/vm/vcpu_run/vcpu_run.tc b/hyp/vm/vcpu_run/vcpu_run.tc new file mode 100644 index 0000000..1df0c01 --- /dev/null +++ b/hyp/vm/vcpu_run/vcpu_run.tc @@ -0,0 +1,30 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +extend thread object module vcpu_run { + wakeup_virq structure virq_source(contained); + wakeup_virq_task structure task_queue_entry(contained); + + // Protected by the scheduler lock and the SCHEDULER_BLOCK_VCPU_RUN + // flag; only a thread that has cleared that flag can run the VCPU and + // update this state (even after dropping the lock). Also, any state + // variables maintained by handlers for the resume & check events are + // protected the same way. + last_state enumeration vcpu_run_state; + + // Protected by the scheduler lock. + enabled bool; +}; + +extend virq_trigger enumeration { + vcpu_run_wakeup; +}; + +extend task_queue_class enumeration { + vcpu_run_wakeup_virq; +}; + +extend scheduler_block enumeration { + vcpu_run; +}; diff --git a/hyp/vm/vdebug/aarch64/src/vdebug.c b/hyp/vm/vdebug/aarch64/src/vdebug.c index f6debbb..6a9d101 100644 --- a/hyp/vm/vdebug/aarch64/src/vdebug.c +++ b/hyp/vm/vdebug/aarch64/src/vdebug.c @@ -19,7 +19,7 @@ #include "debug_bps.h" #include "event_handlers.h" -static struct asm_ordering_dummy vdebug_asm_order; +static asm_ordering_dummy_t vdebug_asm_order; void vdebug_handle_boot_cpu_cold_init(void) @@ -50,7 +50,7 @@ vdebug_handle_object_create_thread(thread_create_t thread_create) } bool -vdebug_handle_vcpu_activate_thread(thread_t *thread, +vdebug_handle_vcpu_activate_thread(thread_t *thread, vcpu_option_flags_t options) { assert(thread != NULL); @@ -117,7 +117,7 @@ vdebug_handle_thread_context_switch_post(thread_t *prev) } void -vdebug_handle_thread_load_state() +vdebug_handle_thread_load_state(void) { thread_t *current = thread_get_self(); @@ -148,7 +148,7 @@ static vcpu_trap_result_t vdebug_handle_vcpu_debug_trap(void) { vcpu_trap_result_t ret; - thread_t *current = thread_get_self(); + thread_t *current = thread_get_self(); #if defined(PLATFORM_HAS_NO_DBGCLAIM_EL1) && PLATFORM_HAS_NO_DBGCLAIM_EL1 DBGCLAIM_EL1_t dbgclaim = DBGCLAIM_EL1_default(); @@ -224,7 +224,7 @@ vdebug_handle_vcpu_trap_mcrmrc14_guest(ESR_EL2_ISS_MCR_MRC_t iss) { vcpu_trap_result_t ret; - if (ESR_EL2_ISS_MCR_MRC_get_Opc1(&iss) != 0) { + if (ESR_EL2_ISS_MCR_MRC_get_Opc1(&iss) != 0U) { // Not a debug register ret = VCPU_TRAP_RESULT_UNHANDLED; } else { @@ -236,7 +236,7 @@ vdebug_handle_vcpu_trap_mcrmrc14_guest(ESR_EL2_ISS_MCR_MRC_t iss) if (ESR_EL2_ISS_MCR_MRC_get_Direction(&iss) == 1) { if ((ESR_EL2_ISS_MCR_MRC_get_CV(&iss) == 0) || - (ESR_EL2_ISS_MCR_MRC_get_COND(&iss) != 0xe)) { + (ESR_EL2_ISS_MCR_MRC_get_COND(&iss) != 0xeU)) { // TODO: Need to read COND/ITState/condition // flags to determined whether to emulate or // ignore. diff --git a/hyp/vm/vdebug/aarch64/vdebug.ev b/hyp/vm/vdebug/aarch64/vdebug.ev index bc7ddee..d145d3f 100644 --- a/hyp/vm/vdebug/aarch64/vdebug.ev +++ b/hyp/vm/vdebug/aarch64/vdebug.ev @@ -14,7 +14,7 @@ subscribe vcpu_activate_thread subscribe thread_save_state -subscribe thread_context_switch_post +subscribe thread_context_switch_post(prev) subscribe thread_load_state() diff --git a/hyp/vm/vdebug/aarch64/vdebug.tc b/hyp/vm/vdebug/aarch64/vdebug.tc index 327ad40..c9d1883 100644 --- a/hyp/vm/vdebug/aarch64/vdebug.tc +++ b/hyp/vm/vdebug/aarch64/vdebug.tc @@ -4,9 +4,6 @@ extend thread object module vdebug { state structure debug_common_registers; -#if ARCH_AARCH64_32BIT_EL1 -#error Context switch DBGVCR32_EL2 for 32-bit guests -#endif enabled bool; }; diff --git a/hyp/vm/vete/build.conf b/hyp/vm/vete/build.conf new file mode 100644 index 0000000..18d63e2 --- /dev/null +++ b/hyp/vm/vete/build.conf @@ -0,0 +1,9 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +base_module hyp/platform/ete +base_module hyp/misc/vet +types vete.tc +events vete.ev +source vete.c diff --git a/hyp/vm/vete/src/vete.c b/hyp/vm/vete/src/vete.c new file mode 100644 index 0000000..93e261c --- /dev/null +++ b/hyp/vm/vete/src/vete.c @@ -0,0 +1,167 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "ete.h" +#include "event_handlers.h" + +#define ISS_TRFCR_EL1 ISS_OP0_OP1_CRN_CRM_OP2(3, 0, 1, 2, 1) + +void +vete_handle_boot_cpu_cold_init(void) +{ + ID_AA64DFR0_EL1_t id_aa64dfr0 = register_ID_AA64DFR0_EL1_read(); + // NOTE: ID_AA64DFR0.TraceVer just indicates if trace is implemented, + // so here we use equal for assertion. + assert(ID_AA64DFR0_EL1_get_TraceVer(&id_aa64dfr0) == 1U); +} + +void +vete_handle_boot_cpu_warm_init(void) +{ + TRFCR_EL2_t trfcr = TRFCR_EL2_default(); + // prohibit trace of EL2 + TRFCR_EL2_set_E2TRE(&trfcr, 0); + register_TRFCR_EL2_write_ordered(trfcr, &vet_ordering); +} + +void +vet_update_trace_unit_status(thread_t *self) +{ + assert(self != NULL); + + TRCPRGCTLR_t trcprgctlr = TRCPRGCTLR_cast( + register_TRCPRGCTLR_read_ordered(&vet_ordering)); + self->vet_trace_unit_enabled = TRCPRGCTLR_get_EN(&trcprgctlr); +} + +void +vet_flush_trace(thread_t *self) +{ + assert(self != NULL); + + if (compiler_unexpected(self->vet_trace_unit_enabled)) { + __asm__ volatile("tsb csync" : "+m"(vet_ordering)); + } +} + +void +vet_disable_trace(void) +{ + TRCPRGCTLR_t trcprg_ctlr = TRCPRGCTLR_default(); + TRCPRGCTLR_set_EN(&trcprg_ctlr, false); + register_TRCPRGCTLR_write_ordered(TRCPRGCTLR_raw(trcprg_ctlr), + &vet_ordering); +} + +static void +vete_prohibit_registers_access(bool prohibit) +{ + thread_t *current = thread_get_self(); + + MDCR_EL2_set_TTRF(¤t->vcpu_regs_el2.mdcr_el2, prohibit); + register_MDCR_EL2_write_ordered(current->vcpu_regs_el2.mdcr_el2, + &vet_ordering); +} + +void +vet_save_trace_thread_context(thread_t *self) +{ + (void)self; + // disable trace register access by set CPTR_EL2.TTA=1 + vete_prohibit_registers_access(true); +} + +void +vet_restore_trace_thread_context(thread_t *self) +{ + (void)self; + // enable trace register access by clear CPTR_EL2.TAA=0 + vete_prohibit_registers_access(false); +} + +void +vet_enable_trace(void) +{ + TRCPRGCTLR_t trcprg_ctlr = TRCPRGCTLR_default(); + TRCPRGCTLR_set_EN(&trcprg_ctlr, true); + register_TRCPRGCTLR_write_ordered(TRCPRGCTLR_raw(trcprg_ctlr), + &vet_ordering); +} + +void +vet_restore_trace_power_context(bool was_poweroff) +{ + // enable trace register access by clear CPTR_EL2.TAA=0 + vete_prohibit_registers_access(false); + asm_context_sync_ordered(&vet_ordering); + + ete_restore_context_percpu(cpulocal_get_index(), was_poweroff); + + // disable trace register access by clear CPTR_EL2.TAA=1 + vete_prohibit_registers_access(true); +} + +void +vet_save_trace_power_context(bool was_poweroff) +{ + vete_prohibit_registers_access(false); + asm_context_sync_ordered(&vet_ordering); + + ete_save_context_percpu(cpulocal_get_index(), was_poweroff); + + // disable trace register access by clear CPTR_EL2.TAA=1 + vete_prohibit_registers_access(true); +} + +vcpu_trap_result_t +vete_handle_vcpu_trap_sysreg(ESR_EL2_ISS_MSR_MRS_t iss) +{ + thread_t *current = thread_get_self(); + vcpu_trap_result_t ret; + + // Remove the fields that are not used in the comparison + ESR_EL2_ISS_MSR_MRS_set_Rt(&iss, 0); + ESR_EL2_ISS_MSR_MRS_set_Direction(&iss, false); + + if (((ESR_EL2_ISS_MSR_MRS_get_Op0(&iss) != 2U) || + (ESR_EL2_ISS_MSR_MRS_get_Op1(&iss) != 1U)) && + (ESR_EL2_ISS_MSR_MRS_raw(iss) != ISS_TRFCR_EL1)) { + // Not a TBRE register access. + ret = VCPU_TRAP_RESULT_UNHANDLED; + } else if (!vcpu_option_flags_get_trace_allowed( + ¤t->vcpu_options)) { + // This VCPU isn't allowed to access debug. Fault immediately. + ret = VCPU_TRAP_RESULT_FAULT; + } else if (!current->vet_trace_unit_enabled) { + // Lazily enable trace register access and restore context. + current->vet_trace_unit_enabled = true; + + // only enable the register access + vete_prohibit_registers_access(false); + + ret = VCPU_TRAP_RESULT_RETRY; + } else { + // Probably an attempted OS lock; fall back to default RAZ/WI. + ret = VCPU_TRAP_RESULT_UNHANDLED; + } + + return ret; +} diff --git a/hyp/vm/vete/vete.ev b/hyp/vm/vete/vete.ev new file mode 100644 index 0000000..4bf31ec --- /dev/null +++ b/hyp/vm/vete/vete.ev @@ -0,0 +1,15 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module vete + +subscribe boot_cpu_cold_init() + +subscribe boot_cpu_warm_init() + +subscribe vcpu_trap_sysreg_read + handler vete_handle_vcpu_trap_sysreg + +subscribe vcpu_trap_sysreg_write + handler vete_handle_vcpu_trap_sysreg diff --git a/hyp/vm/vete/vete.tc b/hyp/vm/vete/vete.tc new file mode 100644 index 0000000..af497cd --- /dev/null +++ b/hyp/vm/vete/vete.tc @@ -0,0 +1,7 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +extend vcpu_option_flags bitfield { + 5 trace_allowed bool = 0; +}; diff --git a/hyp/vm/vetm/src/vetm.c b/hyp/vm/vetm/src/vetm.c index dd23aec..60f83b4 100644 --- a/hyp/vm/vetm/src/vetm.c +++ b/hyp/vm/vetm/src/vetm.c @@ -88,7 +88,7 @@ vetm_protect_trcvi_ctlr(ETM_TRCVI_CTLR_t trcvi_ctlr) return trcvi_ctlr; } -static bool +static vcpu_trap_result_t vetm_vdevice_write(thread_t *vcpu, cpu_index_t pcpu, size_t offset, register_t val, size_t access_size) { @@ -107,10 +107,10 @@ vetm_vdevice_write(thread_t *vcpu, cpu_index_t pcpu, size_t offset, etm_set_reg(pcpu, offset, write_val, access_size); - return true; + return VCPU_TRAP_RESULT_EMULATED; } -static bool +static vcpu_trap_result_t vetm_vdevice_read(thread_t *vcpu, cpu_index_t pcpu, size_t offset, register_t *val, size_t access_size) { @@ -118,14 +118,14 @@ vetm_vdevice_read(thread_t *vcpu, cpu_index_t pcpu, size_t offset, etm_get_reg(pcpu, offset, val, access_size); - return true; + return VCPU_TRAP_RESULT_EMULATED; } -bool +vcpu_trap_result_t vetm_handle_vdevice_access_fixed_addr(vmaddr_t ipa, size_t access_size, register_t *value, bool is_write) { - bool ret; + vcpu_trap_result_t ret = VCPU_TRAP_RESULT_UNHANDLED; cpulocal_begin(); @@ -133,16 +133,17 @@ vetm_handle_vdevice_access_fixed_addr(vmaddr_t ipa, size_t access_size, assert(vcpu != NULL); if (!vcpu_option_flags_get_hlos_vm(&vcpu->vcpu_options)) { - ret = false; goto out; } if ((ipa >= PLATFORM_ETM_BASE) && - (ipa < PLATFORM_ETM_BASE + ETM_STRIDE * PLATFORM_MAX_CORES)) { + (ipa < + PLATFORM_ETM_BASE + PLATFORM_ETM_STRIDE * PLATFORM_MAX_CORES)) { size_t base_offset = (size_t)(ipa - PLATFORM_ETM_BASE); cpu_index_t access_pcpu = - (cpu_index_t)(base_offset / ETM_STRIDE); - size_t offset = ipa - PLATFORM_ETM_BASE - pcpu * ETM_STRIDE; + (cpu_index_t)(base_offset / PLATFORM_ETM_STRIDE); + size_t offset = + ipa - PLATFORM_ETM_BASE - pcpu * PLATFORM_ETM_STRIDE; if ((pcpu == access_pcpu) && vetm_access_allowed(access_size, offset)) { @@ -153,11 +154,7 @@ vetm_handle_vdevice_access_fixed_addr(vmaddr_t ipa, size_t access_size, ret = vetm_vdevice_read(vcpu, pcpu, offset, value, access_size); } - } else { - ret = false; } - } else { - ret = false; } out: cpulocal_end(); diff --git a/hyp/vm/vetm/vetm.tc b/hyp/vm/vetm/vetm.tc index 5c930aa..3a2bb8c 100644 --- a/hyp/vm/vetm/vetm.tc +++ b/hyp/vm/vetm/vetm.tc @@ -15,5 +15,5 @@ extend thread object module vetm { }; extend trace_class enumeration { - VETM; + VETM = 20; }; diff --git a/hyp/vm/vetm_null/build.conf b/hyp/vm/vetm_null/build.conf index 4f402ed..9f358c2 100644 --- a/hyp/vm/vetm_null/build.conf +++ b/hyp/vm/vetm_null/build.conf @@ -2,6 +2,5 @@ # # SPDX-License-Identifier: BSD-3-Clause -base_module hyp/platform/etm events vetm_null.ev source vetm_null.c diff --git a/hyp/vm/vetm_null/src/vetm_null.c b/hyp/vm/vetm_null/src/vetm_null.c index 8b46486..bf8eb4c 100644 --- a/hyp/vm/vetm_null/src/vetm_null.c +++ b/hyp/vm/vetm_null/src/vetm_null.c @@ -11,14 +11,13 @@ #include #include -#include "etm.h" #include "event_handlers.h" -bool +vcpu_trap_result_t vetm_null_handle_vdevice_access_fixed_addr(vmaddr_t ipa, size_t access_size, register_t *value, bool is_write) { - bool ret; + vcpu_trap_result_t ret = VCPU_TRAP_RESULT_UNHANDLED; (void)access_size; thread_t *vcpu = thread_get_self(); @@ -30,14 +29,13 @@ vetm_null_handle_vdevice_access_fixed_addr(vmaddr_t ipa, size_t access_size, } if ((ipa >= PLATFORM_ETM_BASE) && - (ipa < (PLATFORM_ETM_BASE + (ETM_STRIDE * PLATFORM_MAX_CORES)))) { + (ipa < (PLATFORM_ETM_BASE + + (PLATFORM_ETM_STRIDE * PLATFORM_MAX_CORES)))) { // Treat the entire ETM region as RAZ/WI if (!is_write) { *value = 0U; } - ret = true; - } else { - ret = false; + ret = VCPU_TRAP_RESULT_EMULATED; } out: diff --git a/hyp/vm/vgic/build.conf b/hyp/vm/vgic/build.conf index a6d1101..7f2794b 100644 --- a/hyp/vm/vgic/build.conf +++ b/hyp/vm/vgic/build.conf @@ -3,15 +3,21 @@ # SPDX-License-Identifier: BSD-3-Clause local_include +interface vgic base_module hyp/vm/vic_base base_module hyp/mem/useraccess base_module hyp/platform/gicv3 types vgic.tc events vgic.ev -source deliver.c distrib.c vdevice.c sysregs.c util.c +hypercalls vgic.hvc +source deliver.c distrib.c vdevice.c sysregs.c util.c vpe.c vgic.c configs VGIC_HAS_EXT_IRQS=0 configs VGIC_HAS_1N=GICV3_HAS_1N configs VGIC_HAS_1N_PRIORITY_CHECK=0 +configs VGIC_HAS_LPI=GICV3_HAS_VLPI_V4_1 +configs GICV3_ENABLE_VPE=VGIC_HAS_LPI +# Avoid trapping WFI if GICv4 is used, because doorbell wakeup latency is high +configs VCPU_IDLE_IN_EL1=VGIC_HAS_LPI # Workaround for broken max IRQs calculation (1024 instead of 1020) in UEFI configs VGIC_IGNORE_ARRAY_OVERFLOWS=1 configs VIC_BASE_FORWARD_PRIVATE=1 diff --git a/hyp/vm/vgic/include/internal.h b/hyp/vm/vgic/include/internal.h index 787831c..251ebd3 100644 --- a/hyp/vm/vgic/include/internal.h +++ b/hyp/vm/vgic/include/internal.h @@ -55,6 +55,11 @@ vgic_sync_all(vic_t *vic, bool wakeup); void vgic_update_enables(vic_t *vic, GICD_CTLR_DS_t gicd_ctlr); +#if VGIC_HAS_LPI && GICV3_HAS_VLPI +void +vgic_vpe_schedule_current(void); +#endif + void vgic_retry_unrouted(vic_t *vic); @@ -74,6 +79,10 @@ void vgic_lr_owner_unlock_nopreempt(thread_t *vcpu) RELEASE_LOCK(vcpu->vgic_lr_owner_lock) REQUIRE_PREEMPT_DISABLED; +index_result_t +vgic_get_index_for_mpidr(vic_t *vic, uint8_t aff0, uint8_t aff1, uint8_t aff2, + uint8_t aff3); + // // Utility functions (IRQ types, bit manipulations etc) // @@ -172,6 +181,17 @@ vgic_gicr_rd_set_statusr(thread_t *gicr_vcpu, GICR_STATUSR_t statusr, bool set); bool vgic_gicr_rd_check_sleep(thread_t *gicr_vcpu); +#if VGIC_HAS_LPI +// Note: the GICR_PROPBASER is shared between all VCPUs (as permitted by the +// GICv3 spec) so there is no gicr_vcpu argument here +void +vgic_gicr_rd_set_propbase(vic_t *vic, GICR_PROPBASER_t propbase); + +void +vgic_gicr_rd_set_pendbase(vic_t *vic, thread_t *gicr_vcpu, + GICR_PENDBASER_t pendbase); +#endif + void vgic_gicr_sgi_change_sgi_ppi_enable(vic_t *vic, thread_t *gicr_vcpu, irq_t irq_num, bool set); @@ -205,3 +225,6 @@ vgic_icc_irq_deactivate(vic_t *vic, irq_t irq_num); void vgic_icc_generate_sgi(vic_t *vic, ICC_SGIR_EL1_t sgir, bool is_group_1); + +error_t +vgic_vsgi_assert(thread_t *gicr_vcpu, irq_t irq_num); diff --git a/hyp/vm/vgic/include/vgic.h b/hyp/vm/vgic/include/vgic.h index e69de29..2560a77 100644 --- a/hyp/vm/vgic/include/vgic.h +++ b/hyp/vm/vgic/include/vgic.h @@ -0,0 +1,21 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +// Interfaces used by the VGIC ITS for LPI cache invalidation. + +#if VGIC_HAS_LPI +void +vgic_gicr_copy_propbase_one(vic_t *vic, thread_t *gicr_vcpu, irq_t vlpi); + +#if GICV3_HAS_VLPI_V4_1 +void +vgic_gicr_rd_invlpi(vic_t *vic, thread_t *gicr_vcpu, virq_t vlpi_num); + +void +vgic_gicr_rd_invall(vic_t *vic, thread_t *gicr_vcpu); + +bool +vgic_gicr_get_inv_pending(vic_t *vic, thread_t *gicr_vcpu); +#endif +#endif diff --git a/hyp/vm/vgic/src/deliver.c b/hyp/vm/vgic/src/deliver.c index 26e7f24..4c1b766 100644 --- a/hyp/vm/vgic/src/deliver.c +++ b/hyp/vm/vgic/src/deliver.c @@ -36,6 +36,10 @@ #include +#if defined(ARCH_ARM_FEAT_FGT) && ARCH_ARM_FEAT_FGT +#include +#endif + #include "event_handlers.h" #include "gich_lrs.h" #include "gicv3.h" @@ -43,7 +47,7 @@ static hwirq_t *vgic_maintenance_hwirq; -static struct asm_ordering_dummy gich_lr_ordering; +static asm_ordering_dummy_t gich_lr_ordering; // Set to 1 to boot enable the virtual GIC delivery tracepoints #if defined(VERBOSE_TRACE) && VERBOSE_TRACE @@ -52,6 +56,16 @@ static struct asm_ordering_dummy gich_lr_ordering; #define VGIC_TRACES 0 #endif +static bool +vgic_fgt_allowed(void) +{ +#if defined(ARCH_ARM_FEAT_FGT) && ARCH_ARM_FEAT_FGT + return compiler_expected(arm_fgt_is_allowed()); +#else + return false; +#endif +} + void vgic_handle_boot_hypervisor_start(void) { @@ -102,10 +116,10 @@ vgic_handle_boot_cpu_warm_init(void) void vgic_read_lr_state(index_t i) { - assert(i < CPU_GICH_LR_COUNT); thread_t *current = thread_get_self(); assert((current != NULL) && (current->kind == THREAD_KIND_VCPU)); + assert_debug(i < CPU_GICH_LR_COUNT); vgic_lr_status_t *status = ¤t->vgic_lrs[i]; // Read back the hardware register if necessary @@ -118,16 +132,45 @@ vgic_read_lr_state(index_t i) static void vgic_write_lr(index_t i) { - assert(i < CPU_GICH_LR_COUNT); + assert_debug(i < CPU_GICH_LR_COUNT); thread_t *current = thread_get_self(); - assert((current != NULL) && (current->kind == THREAD_KIND_VCPU)); + assert_debug((current != NULL) && (current->kind == THREAD_KIND_VCPU)); vgic_lr_status_t *status = ¤t->vgic_lrs[i]; - assert(status != NULL); gicv3_write_ich_lr(i, status->lr, &gich_lr_ordering); } +#if VGIC_HAS_1N +static bool +vgic_get_delivery_state_is_class0(vgic_delivery_state_t *dstate) +{ + bool ret; +#if defined(GICV3_HAS_GICD_ICLAR) && GICV3_HAS_GICD_ICLAR + ret = !vgic_delivery_state_get_nclass0(dstate); +#else + (void)dstate; + ret = true; +#endif + + return ret; +} + +static bool +vgic_get_delivery_state_is_class1(vgic_delivery_state_t *dstate) +{ + bool ret; +#if defined(GICV3_HAS_GICD_ICLAR) && GICV3_HAS_GICD_ICLAR + ret = vgic_delivery_state_get_class1(dstate); +#else + (void)dstate; + ret = false; +#endif + + return ret; +} +#endif + // Determine whether a VCPU is a valid route for a given VIRQ. // // This is allowed to take the enabled groups into account, but must ignore the @@ -154,8 +197,8 @@ vgic_route_allowed(vic_t *vic, thread_t *vcpu, vgic_delivery_state_t dstate) // check the class bits. allowed = (platform_irq_cpu_class( (cpu_index_t)vcpu->vgic_gicr_index) == 0U) - ? !vgic_delivery_state_get_nclass0(&dstate) - : vgic_delivery_state_get_class1(&dstate); + ? vgic_get_delivery_state_is_class0(&dstate) + : vgic_get_delivery_state_is_class1(&dstate); } #endif else { @@ -180,72 +223,88 @@ vgic_spi_set_route_1n(irq_t irq, vgic_delivery_state_t dstate) // Restore the 1-of-N route GICD_IROUTER_t route_1n = GICD_IROUTER_default(); GICD_IROUTER_set_IRM(&route_1n, true); - gicv3_spi_set_route(irq, route_1n); + (void)gicv3_spi_set_route(irq, route_1n); #if GICV3_HAS_GICD_ICLAR // Set the HW IRQ's 1-of-N routing classes. Note that these are reset // in the hardware whenever the IRM bit is cleared. - gicv3_spi_set_classes(irq, !vgic_delivery_state_get_nclass0(&dstate), - vgic_delivery_state_get_class1(&dstate)); + (void)gicv3_spi_set_classes(irq, + !vgic_delivery_state_get_nclass0(&dstate), + vgic_delivery_state_get_class1(&dstate)); #else (void)dstate; #endif } #endif -// Clear pending or enable bits from a listed VIRQ, and then update its LR. -// -// This is used for all changes to a currently listed VIRQ other than a local -// redelivery: disabling, clearing, rerouting, remotely enabling or asserting, -// or releasing the source. Locally enabling or asserting a listed VIRQ is -// handled by vgic_redeliver_lr(). -// -// The flags that are set in the clear_dstate argument, if any, will be cleared -// in the delivery state. This value must not have any flags set other than the -// four pending flags, the enabled flag, and the hardware active flag. -// -// If the current delivery state has the enable bit clear or clear_dstate has -// the enable bit set, the pending state will be removed from the LR regardless -// of the pending state of the interrupt (though the active state can remain in -// the LR). -// -// If the current delivery state has the hw_detached bit set or clear_dstate has -// the hw_active bit set, the HW bit of the LR will be cleared even if it is -// left listed. The HW bit of the LR may also be cleared if it is necessary to -// trap EOI to guarantee delivery of the IRQ. -// -// The specified VCPU must either be the current thread, or LR-locked by -// the caller and known not to be running remotely. If the VCPU is the current -// thread, the caller is responsible for syncing and updating the physical LR. -// -// For hardware interrupts, the level_src flag in clear_dstate may be overridden -// by the hw_active flag, if it has been set by a concurrent remote delivery; -// this is unnecessary for software interrupts because level_src changes are -// required to be serialised. -// -// If the VIRQ is still enabled and pending after clearing the pending and -// enable bits, it will be set pending in the LR if possible, or otherwise -// rerouted. If it is 1-of-N, the use_local_vcpu flag determines whether the -// current VCPU is given routing priority. -// -// The result is true if the VIRQ has been unlisted. +// Check whether a level-triggered source is still asserting its interrupt. static bool -vgic_sync_lr(vic_t *vic, thread_t *vcpu, vgic_lr_status_t *status, - vgic_delivery_state_t clear_dstate, bool use_local_vcpu) - REQUIRE_PREEMPT_DISABLED EXCLUDE_SCHEDULER_LOCK(vcpu) +vgic_virq_check_pending(virq_source_t *source, bool reasserted) + REQUIRE_PREEMPT_DISABLED { - virq_t virq = ICH_LR_EL2_base_get_vINTID(&status->lr.base); + bool is_pending; - assert(status->dstate != NULL); + if (compiler_unexpected(source == NULL)) { + // Source has been detached since the IRQ was asserted. + is_pending = false; + } else { + // The virq_check_pending event must guarantee that all memory + // reads executed by the handler are ordered after the read that + // determined (a) that the IRQ was marked level-pending, and (b) + // the value of the reasserted argument. Since the callers of + // this function make those determinations using relaxed atomic + // reads of the delivery state, we need an acquire fence here to + // enforce the correct ordering. + atomic_thread_fence(memory_order_acquire); - bool lr_hw = ICH_LR_EL2_base_get_HW(&status->lr.base); - ICH_LR_EL2_State_t lr_state = - ICH_LR_EL2_base_get_State(&status->lr.base); - bool lr_pending = (lr_state == ICH_LR_EL2_STATE_PENDING) || - (lr_state == ICH_LR_EL2_STATE_PENDING_ACTIVE); - bool lr_active = (lr_state == ICH_LR_EL2_STATE_ACTIVE) || - (lr_state == ICH_LR_EL2_STATE_PENDING_ACTIVE); + is_pending = trigger_virq_check_pending_event( + source->trigger, source, reasserted); + } + + return is_pending; +} + +static bool +vgic_sync_lr_set_delivery_state(bool lr_active, bool virq_pending, + bool allow_pending, bool lr_pending, + vgic_delivery_state_t *new_dstate) +{ + bool remove_hw; + + if (!lr_active && (!virq_pending || !allow_pending)) { + vgic_delivery_state_set_listed(new_dstate, false); + vgic_delivery_state_set_active(new_dstate, false); + remove_hw = lr_pending; + } else if (virq_pending && allow_pending) { + // We're going to leave the LR in pending state, so + // clear the edge bit. + vgic_delivery_state_set_edge(new_dstate, false); + remove_hw = false; + } else { + // We are leaving the VIRQ listed in active state, and + // can't set the pending state in the LR. If the VIRQ is + // pending, we must trap EOI to deliver it elsewhere. + remove_hw = virq_pending; + } + + return remove_hw; +} +typedef struct { + vgic_delivery_state_t new_dstate; + bool virq_pending; + bool hw_detach; + bool allow_pending; + bool remove_hw; +} vgic_sync_lr_update_t; + +static vgic_sync_lr_update_t +vgic_sync_lr_update_delivery_state(vic_t *vic, thread_t *vcpu, + const vgic_lr_status_t *status, + vgic_delivery_state_t clear_dstate, + bool lr_hw, bool lr_pending, virq_t virq, + bool lr_active) REQUIRE_PREEMPT_DISABLED +{ vgic_delivery_state_t old_dstate = atomic_load_relaxed(status->dstate); bool hw_detach = vgic_delivery_state_get_hw_active(&clear_dstate); @@ -268,7 +327,8 @@ vgic_sync_lr(vic_t *vic, thread_t *vcpu, vgic_lr_status_t *status, } // If level_src is set and is not being explicitly cleared, - // check whether we need to clear it. + // check whether it should be cleared based on the LR's pending + // state. if (vgic_delivery_state_get_level_src(&old_dstate) && !vgic_delivery_state_get_level_src(&clear_dstate)) { if (lr_hw && (!lr_pending || hw_detach)) { @@ -282,11 +342,11 @@ vgic_sync_lr(vic_t *vic, thread_t *vcpu, vgic_lr_status_t *status, } else { virq_source_t *source = vgic_find_source(vic, vcpu, virq); - if (compiler_unexpected(source == NULL) || - !trigger_virq_check_pending_event( - source->trigger, source, - vgic_delivery_state_get_edge( - &old_dstate))) { + bool reassert = lr_pending || + vgic_delivery_state_get_edge( + &old_dstate); + if (!vgic_virq_check_pending(source, + reassert)) { vgic_delivery_state_set_level_src( &new_dstate, false); } @@ -312,21 +372,9 @@ vgic_sync_lr(vic_t *vic, thread_t *vcpu, vgic_lr_status_t *status, // Determine whether to delist the IRQ, and whether the HW=1 bit // is being removed from a valid LR (whether delisted or not). - if (!lr_active && (!virq_pending || !allow_pending)) { - vgic_delivery_state_set_listed(&new_dstate, false); - vgic_delivery_state_set_active(&new_dstate, false); - remove_hw = lr_pending; - } else if (virq_pending && allow_pending) { - // We're going to leave the LR in pending state, so - // clear the edge bit. - vgic_delivery_state_set_edge(&new_dstate, false); - remove_hw = false; - } else { - // We are leaving the VIRQ listed in active state, and - // can't set the pending state in the LR. If the VIRQ is - // pending, we must trap EOI to deliver it elsewhere. - remove_hw = virq_pending; - } + remove_hw = vgic_sync_lr_set_delivery_state( + lr_active, virq_pending, allow_pending, lr_pending, + &new_dstate); // If we're removing HW=1 from a valid LR, but not detaching // (and therefore deactivating) the HW IRQ, we need to reset the @@ -345,39 +393,36 @@ vgic_sync_lr(vic_t *vic, thread_t *vcpu, vgic_lr_status_t *status, virq, vgic_delivery_state_raw(old_dstate), vgic_delivery_state_raw(new_dstate)); - // If we're detaching a HW IRQ, clear the HW bit in the LR. - if (compiler_unexpected(lr_hw && hw_detach)) { - // If the LR was pending or active, the physical IRQ is still - // active. Clearing the HW bit destroys our record that this - // might be the case, so we have to deactivate at this point. - if (lr_pending || lr_active) { - assert(!vgic_delivery_state_get_hw_active(&new_dstate)); - irq_t irq = ICH_LR_EL2_HW1_get_pINTID(&status->lr.hw); - VGIC_TRACE(HWSTATE_CHANGED, vic, vcpu, - "sync_lr {:d}: deactivate HW IRQ {:d}", - ICH_LR_EL2_HW1_get_vINTID(&status->lr.hw), - irq); - gicv3_irq_deactivate(irq); - } - - ICH_LR_EL2_base_set_HW(&status->lr.base, false); - // If HW was 1 there must be no SW level assertion, so we don't - // need to trap EOI - ICH_LR_EL2_HW0_set_EOI(&status->lr.sw, false); - } + return (vgic_sync_lr_update_t){ + .new_dstate = new_dstate, + .virq_pending = virq_pending, + .hw_detach = hw_detach, + .allow_pending = allow_pending, + .remove_hw = remove_hw, + }; +} - // Update the LR. +static void +vgic_sync_lr_update_lr(vic_t *vic, thread_t *vcpu, vgic_lr_status_t *status, + bool lr_pending, virq_t virq, bool lr_active, + bool virq_pending, bool allow_pending, bool remove_hw, + bool lr_hw, vgic_delivery_state_t new_dstate, + bool use_local_vcpu) REQUIRE_PREEMPT_DISABLED +{ if (!vgic_delivery_state_get_listed(&new_dstate)) { VGIC_TRACE(HWSTATE_CHANGED, vic, vcpu, "sync_lr {:d}: delisted (pending {:d})", virq, - virq_pending); + (register_t)virq_pending); #if VGIC_HAS_1N if (lr_hw && vgic_delivery_state_get_route_1n(&new_dstate)) { + assert(ICH_LR_EL2_base_get_HW(&status->lr.base)); vgic_spi_set_route_1n( ICH_LR_EL2_HW1_get_pINTID(&status->lr.hw), new_dstate); } +#else + (void)lr_hw; #endif status->dstate = NULL; status->lr.base = ICH_LR_EL2_base_default(); @@ -449,10 +494,195 @@ vgic_sync_lr(vic_t *vic, thread_t *vcpu, vgic_lr_status_t *status, VGIC_TRACE(HWSTATE_CHANGED, vic, vcpu, "sync_lr {:d}: LR unchanged", virq); } +} + +// Clear pending or enable bits from a listed VIRQ, and then update its LR. +// +// This is used for all changes to a currently listed VIRQ other than a local +// redelivery: disabling, clearing, rerouting, remotely enabling or asserting, +// or releasing the source. Locally enabling or asserting a listed VIRQ is +// handled by vgic_redeliver_lr(). +// +// The flags that are set in the clear_dstate argument, if any, will be cleared +// in the delivery state. This value must not have any flags set other than the +// four pending flags, the enabled flag, and the hardware active flag. +// +// If the current delivery state has the enable bit clear or clear_dstate has +// the enable bit set, the pending state will be removed from the LR regardless +// of the pending state of the interrupt (though the active state can remain in +// the LR). +// +// If the current delivery state has the hw_detached bit set or clear_dstate has +// the hw_active bit set, the HW bit of the LR will be cleared even if it is +// left listed. The HW bit of the LR may also be cleared if it is necessary to +// trap EOI to guarantee delivery of the IRQ. +// +// The specified VCPU must either be the current thread, or LR-locked by +// the caller and known not to be running remotely. If the VCPU is the current +// thread, the caller is responsible for syncing and updating the physical LR. +// +// For hardware interrupts, the level_src flag in clear_dstate may be overridden +// by the hw_active flag, if it has been set by a concurrent remote delivery; +// this is unnecessary for software interrupts because level_src changes are +// required to be serialised. +// +// If the VIRQ is still enabled and pending after clearing the pending and +// enable bits, it will be set pending in the LR if possible, or otherwise +// rerouted. If it is 1-of-N, the use_local_vcpu flag determines whether the +// current VCPU is given routing priority. +// +// The result is true if the VIRQ has been unlisted. +static bool +vgic_sync_lr(vic_t *vic, thread_t *vcpu, vgic_lr_status_t *status, + vgic_delivery_state_t clear_dstate, bool use_local_vcpu) + REQUIRE_PREEMPT_DISABLED EXCLUDE_SCHEDULER_LOCK(vcpu) +{ + virq_t virq = ICH_LR_EL2_base_get_vINTID(&status->lr.base); + + assert(status->dstate != NULL); + + bool lr_hw = ICH_LR_EL2_base_get_HW(&status->lr.base); + ICH_LR_EL2_State_t lr_state = + ICH_LR_EL2_base_get_State(&status->lr.base); + bool lr_pending = (lr_state == ICH_LR_EL2_STATE_PENDING) || + (lr_state == ICH_LR_EL2_STATE_PENDING_ACTIVE); + bool lr_active = (lr_state == ICH_LR_EL2_STATE_ACTIVE) || + (lr_state == ICH_LR_EL2_STATE_PENDING_ACTIVE); + + vgic_delivery_state_t new_dstate; + bool virq_pending; + bool hw_detach; + bool allow_pending; + bool remove_hw; + { + vgic_sync_lr_update_t sync_lr_info = + vgic_sync_lr_update_delivery_state(vic, vcpu, status, + clear_dstate, lr_hw, + lr_pending, virq, + lr_active); + new_dstate = sync_lr_info.new_dstate; + virq_pending = sync_lr_info.virq_pending; + hw_detach = sync_lr_info.hw_detach; + allow_pending = sync_lr_info.allow_pending; + remove_hw = sync_lr_info.remove_hw; + } + + // If we're detaching a HW IRQ, clear the HW bit in the LR. + if (compiler_unexpected(lr_hw && hw_detach)) { + // If the LR was pending or active, the physical IRQ is still + // active. Clearing the HW bit destroys our record that this + // might be the case, so we have to deactivate at this point. + if (lr_pending || lr_active) { + assert(!vgic_delivery_state_get_hw_active(&new_dstate)); + irq_t irq = ICH_LR_EL2_HW1_get_pINTID(&status->lr.hw); + VGIC_TRACE(HWSTATE_CHANGED, vic, vcpu, + "sync_lr {:d}: deactivate HW IRQ {:d}", + ICH_LR_EL2_HW1_get_vINTID(&status->lr.hw), + irq); + gicv3_irq_deactivate(irq); + } + + // If the LR will remain valid, turn it into a SW IRQ. + if (vgic_delivery_state_get_listed(&new_dstate)) { + ICH_LR_EL2_base_set_HW(&status->lr.base, false); + // If HW was 1 there must be no SW level assertion, so + // we don't need to trap EOI + ICH_LR_EL2_HW0_set_EOI(&status->lr.sw, false); + } + } + + // Update the LR. + vgic_sync_lr_update_lr(vic, vcpu, status, lr_pending, virq, lr_active, + virq_pending, allow_pending, remove_hw, lr_hw, + new_dstate, use_local_vcpu); return !vgic_delivery_state_get_listed(&new_dstate); } +static bool +vgic_undeliver_update_hw_detach_and_sync(const vic_t *vic, const thread_t *vcpu, + virq_t virq, + _Atomic vgic_delivery_state_t *dstate, + vgic_delivery_state_t clear_dstate, + vgic_delivery_state_t old_dstate, + bool check_route) +{ + vgic_delivery_state_t new_dstate; + bool hw_detach = vgic_delivery_state_get_hw_active(&clear_dstate); + vgic_delivery_state_set_hw_active(&clear_dstate, false); + + do { + new_dstate = vgic_delivery_state_difference(old_dstate, + clear_dstate); + + if (!vgic_delivery_state_get_listed(&old_dstate)) { + // Delisted by another thread; no sync needed. + } else if (check_route) { + // Force a sync regardless of pending state. + vgic_delivery_state_set_need_sync(&new_dstate, true); + } else if (!vgic_delivery_state_get_enabled(&new_dstate)) { + // No longer enabled; a sync is required. + vgic_delivery_state_set_need_sync(&new_dstate, true); + } else if (!vgic_delivery_state_get_cfg_is_edge(&new_dstate) && + !vgic_delivery_state_is_level_asserted( + &new_dstate)) { + // No longer pending; a sync is required. + vgic_delivery_state_set_need_sync(&new_dstate, true); + } else { + // Still pending and not reclaimed; no sync needed. + } + + if (hw_detach && vgic_delivery_state_get_listed(&old_dstate)) { + vgic_delivery_state_set_hw_detached(&new_dstate, true); + } + } while (!atomic_compare_exchange_strong_explicit( + dstate, &old_dstate, new_dstate, memory_order_relaxed, + memory_order_relaxed)); + + VGIC_TRACE(DSTATE_CHANGED, vic, vcpu, + "undeliver-sync {:d}: {:#x} -> {:#x}", virq, + vgic_delivery_state_raw(old_dstate), + vgic_delivery_state_raw(new_dstate)); + + return !vgic_delivery_state_get_listed(&old_dstate); +} + +static vgic_delivery_state_t +vgic_undeliver_update_dstate(vic_t *vic, thread_t *vcpu, + _Atomic vgic_delivery_state_t *dstate, virq_t virq, + vgic_delivery_state_t clear_dstate, + vgic_delivery_state_t old_dstate) + REQUIRE_PREEMPT_DISABLED +{ + vgic_delivery_state_t new_dstate; + do { + // If the VIRQ is not listed, update its flags directly. + new_dstate = vgic_delivery_state_difference(old_dstate, + clear_dstate); + if (vgic_delivery_state_get_listed(&old_dstate)) { + break; + } + + // If level_src is set and is not being explicitly cleared, + // check whether we need to clear it. + if (vgic_delivery_state_get_level_src(&old_dstate) && + !vgic_delivery_state_get_level_src(&clear_dstate)) { + virq_source_t *source = + vgic_find_source(vic, vcpu, virq); + if (!vgic_virq_check_pending( + source, vgic_delivery_state_get_edge( + &old_dstate))) { + vgic_delivery_state_set_level_src(&new_dstate, + false); + } + } + } while (!atomic_compare_exchange_strong_explicit( + dstate, &old_dstate, new_dstate, memory_order_relaxed, + memory_order_relaxed)); + + return new_dstate; +} + // Clear pending bits from a given VIRQ, and abort its delivery if necessary. // // This is used when disabling, rerouting, manually clearing, or releasing the @@ -484,33 +714,8 @@ vgic_undeliver(vic_t *vic, thread_t *vcpu, cpu_index_t remote_cpu = vgic_lr_owner_lock(vcpu); vgic_delivery_state_t old_dstate = atomic_load_relaxed(dstate); - vgic_delivery_state_t new_dstate; - - do { - // If the VIRQ is not listed, update its flags directly. - new_dstate = vgic_delivery_state_difference(old_dstate, - clear_dstate); - if (vgic_delivery_state_get_listed(&old_dstate)) { - break; - } - - // If level_src is set and is not being explicitly cleared, - // check whether we need to clear it. - if (vgic_delivery_state_get_level_src(&old_dstate) && - !vgic_delivery_state_get_level_src(&clear_dstate)) { - virq_source_t *source = - vgic_find_source(vic, vcpu, virq); - if (compiler_unexpected(source == NULL) || - !trigger_virq_check_pending_event( - source->trigger, source, - vgic_delivery_state_get_edge(&old_dstate))) { - vgic_delivery_state_set_level_src(&new_dstate, - false); - } - } - } while (!atomic_compare_exchange_strong_explicit( - dstate, &old_dstate, new_dstate, memory_order_relaxed, - memory_order_relaxed)); + vgic_delivery_state_t new_dstate = vgic_undeliver_update_dstate( + vic, vcpu, dstate, virq, clear_dstate, old_dstate); bool unlisted = false; if (!vgic_delivery_state_get_listed(&old_dstate)) { @@ -524,9 +729,6 @@ vgic_undeliver(vic_t *vic, thread_t *vcpu, !vgic_delivery_state_get_hw_active(&new_dstate)) { virq_source_t *source = vgic_find_source(vic, vcpu, virq); - assert(source != NULL); - assert(source->trigger == - VIRQ_TRIGGER_VGIC_FORWARDED_SPI); hwirq_t *hwirq = hwirq_from_virq_source(source); @@ -587,48 +789,123 @@ vgic_undeliver(vic_t *vic, thread_t *vcpu, #endif // We can't directly clear hw_active on a remote CPU; we need to use // the hw_detached bit to ask the remote CPU to do it. - bool hw_detach = vgic_delivery_state_get_hw_active(&clear_dstate); - vgic_delivery_state_set_hw_active(&clear_dstate, false); + unlisted = vgic_undeliver_update_hw_detach_and_sync( + vic, vcpu, virq, dstate, clear_dstate, old_dstate, check_route); - do { - new_dstate = vgic_delivery_state_difference(old_dstate, - clear_dstate); +out: + vgic_lr_owner_unlock(vcpu); - if (!vgic_delivery_state_get_listed(&old_dstate)) { - // Delisted by another thread; no sync needed. - } else if (check_route) { - // Force a sync regardless of pending state. - vgic_delivery_state_set_need_sync(&new_dstate, true); - } else if (!vgic_delivery_state_get_enabled(&new_dstate)) { - // No longer enabled; a sync is required. - vgic_delivery_state_set_need_sync(&new_dstate, true); - } else if (!vgic_delivery_state_get_cfg_is_edge(&new_dstate) && - !vgic_delivery_state_is_level_asserted( - &new_dstate)) { - // No longer pending; a sync is required. - vgic_delivery_state_set_need_sync(&new_dstate, true); - } else { - // Still pending and not reclaimed; no sync needed. - } + return unlisted; +} - if (hw_detach && vgic_delivery_state_get_listed(&old_dstate)) { - vgic_delivery_state_set_hw_detached(&new_dstate, true); +typedef struct { + ICH_LR_EL2_t new_lr; + vgic_delivery_state_t new_dstate; + bool force_eoi_trap; + bool need_wakeup; + uint8_t pad[2]; +} vgic_redeliver_lr_info_t; + +static vgic_redeliver_lr_info_t +vgic_redeliver_lr_update_state(const vic_t *vic, const thread_t *vcpu, + virq_source_t *source, virq_t virq, + ICH_LR_EL2_State_t old_lr_state, + const vgic_lr_status_t *status, + vgic_delivery_state_t old_dstate, + vgic_delivery_state_t assert_dstate) +{ + vgic_delivery_state_t new_dstate = + vgic_delivery_state_union(old_dstate, assert_dstate); + bool is_hw = vgic_delivery_state_get_hw_active(&new_dstate); + ICH_LR_EL2_t new_lr = status->lr; + bool force_eoi_trap = false; + bool need_wakeup = false; + + if (compiler_expected(old_lr_state == ICH_LR_EL2_STATE_INVALID)) { + // Previous interrupt is gone; take the new one. Don't + // bother to recheck level triggering yet; that will be + // done when this interrupt ends. + ICH_LR_EL2_base_set_HW(&new_lr.base, is_hw); + if (ICH_LR_EL2_base_get_HW(&new_lr.base)) { + vgic_delivery_state_set_hw_active(&new_dstate, false); + ICH_LR_EL2_HW1_set_pINTID( + &new_lr.hw, + hwirq_from_virq_source(source)->irq); } - } while (!atomic_compare_exchange_strong_explicit( - dstate, &old_dstate, new_dstate, memory_order_relaxed, - memory_order_relaxed)); + ICH_LR_EL2_base_set_State(&new_lr.base, + ICH_LR_EL2_STATE_PENDING); - VGIC_TRACE(DSTATE_CHANGED, vic, vcpu, - "undeliver-sync {:d}: {:#x} -> {:#x}", virq, - vgic_delivery_state_raw(old_dstate), - vgic_delivery_state_raw(new_dstate)); + // Interrupt is newly pending; we need to wake the VCPU. + need_wakeup = true; + } else if (compiler_unexpected(is_hw != + ICH_LR_EL2_base_get_HW(&new_lr.base))) { + // If we have both a SW and a HW source, deliver the SW + // assertion first, and request an EOI maintenance + // interrupt to deliver (or trigger reassertion of) the + // HW source afterwards. + if (ICH_LR_EL2_base_get_HW(&new_lr.base)) { + ICH_LR_EL2_base_set_HW(&new_lr.base, false); + vgic_delivery_state_set_hw_active(&new_dstate, true); - unlisted = !vgic_delivery_state_get_listed(&old_dstate); + VGIC_DEBUG_TRACE( + HWSTATE_UNCHANGED, vic, vcpu, + "redeliver {:d}: hw + sw; relisting as sw", + virq); + } + force_eoi_trap = true; + + // Interrupt is either already pending (so the VCPU + // should be awake) or is active (so not deliverable, + // and the VCPU should not be woken); no need for a + // wakeup. + } +#if VGIC_HAS_LPI && GICV3_HAS_VLPI_V4_1 + else if ((old_lr_state == ICH_LR_EL2_STATE_ACTIVE) && + vic->vsgis_enabled && + (vgic_get_irq_type(virq) == VGIC_IRQ_TYPE_SGI)) { + // A vSGI delivered by the ITS does not have an active + // state, because it is really a vLPI in disguise. Make + // software-delivered SGIs behave the same way. + assert(!is_hw && !ICH_LR_EL2_base_get_HW(&new_lr.base)); + ICH_LR_EL2_base_set_State(&new_lr.base, + ICH_LR_EL2_STATE_PENDING); -out: - vgic_lr_owner_unlock(vcpu); + // Interrupt was previously active and is now pending, + // so it has just become deliverable and we need to wake + // the VCPU. + need_wakeup = true; + } +#endif + else { + // We should never get here for a hardware-mode LR, + // since it would mean that we were risking a double + // deactivate. + assert(!is_hw && !ICH_LR_EL2_base_get_HW(&new_lr.base)); - return unlisted; + // A software-mode LR that is in active state can me + // moved straight to active+pending. + if (old_lr_state == ICH_LR_EL2_STATE_ACTIVE) { + ICH_LR_EL2_base_set_State( + &new_lr.base, ICH_LR_EL2_STATE_PENDING_ACTIVE); + } else { + VGIC_DEBUG_TRACE( + HWSTATE_UNCHANGED, vic, vcpu, + "redeliver {:d}: redundant assertions merged", + virq); + } + + // Interrupt is already pending, so the VCPU should be + // awake; no need for a wakeup. + } + + vgic_delivery_state_set_edge(&new_dstate, force_eoi_trap); + + return (vgic_redeliver_lr_info_t){ + .new_dstate = new_dstate, + .force_eoi_trap = force_eoi_trap, + .need_wakeup = need_wakeup, + .new_lr = new_lr, + }; } static bool @@ -637,7 +914,7 @@ vgic_redeliver_lr(vic_t *vic, thread_t *vcpu, virq_source_t *source, vgic_delivery_state_t *old_dstate, vgic_delivery_state_t assert_dstate, index_t lr) { - assert(lr < CPU_GICH_LR_COUNT); + assert_debug(lr < CPU_GICH_LR_COUNT); // Merge the old and new LR states. vgic_lr_status_t *status = &vcpu->vgic_lrs[lr]; @@ -650,86 +927,27 @@ vgic_redeliver_lr(vic_t *vic, thread_t *vcpu, virq_source_t *source, bool need_wakeup; do { + ICH_LR_EL2_State_t trace_state; assert(vgic_delivery_state_get_listed(old_dstate)); - new_dstate = - vgic_delivery_state_union(*old_dstate, assert_dstate); - bool is_hw = vgic_delivery_state_get_hw_active(&new_dstate); - new_lr = status->lr; - force_eoi_trap = false; - need_wakeup = false; - ICH_LR_EL2_State_t old_state = + ICH_LR_EL2_State_t old_lr_state = ICH_LR_EL2_base_get_State(&status->lr.base); - if (compiler_expected(old_state == ICH_LR_EL2_STATE_INVALID)) { - // Previous interrupt is gone; take the new one. Don't - // bother to recheck level triggering yet; that will be - // done when this interrupt ends. - ICH_LR_EL2_base_set_HW(&new_lr.base, is_hw); - if (ICH_LR_EL2_base_get_HW(&new_lr.base)) { - vgic_delivery_state_set_hw_active(&new_dstate, - false); - ICH_LR_EL2_HW1_set_pINTID( - &new_lr.hw, - hwirq_from_virq_source(source)->irq); - } - ICH_LR_EL2_base_set_State(&new_lr.base, - ICH_LR_EL2_STATE_PENDING); - - // Interrupt is newly pending; we need to wake the VCPU. - need_wakeup = true; - } else if (compiler_unexpected( - is_hw != - ICH_LR_EL2_base_get_HW(&new_lr.base))) { - // If we have both a SW and a HW source, deliver the SW - // assertion first, and request an EOI maintenance - // interrupt to deliver (or trigger reassertion of) the - // HW source afterwards. - if (ICH_LR_EL2_base_get_HW(&new_lr.base)) { - ICH_LR_EL2_base_set_HW(&new_lr.base, false); - vgic_delivery_state_set_hw_active(&new_dstate, - true); - - VGIC_DEBUG_TRACE( - HWSTATE_UNCHANGED, vic, vcpu, - "redeliver {:d}: hw + sw; relisting as sw", - virq); - } - force_eoi_trap = true; - - // Interrupt is either already pending (so the VCPU - // should be awake) or is active (so not deliverable, - // and the VCPU should not be woken); no need for a - // wakeup. - } - else { - // We should never get here for a hardware-mode LR, - // since it would mean that we were risking a double - // deactivate. - assert(!is_hw && !ICH_LR_EL2_base_get_HW(&new_lr.base)); - - // A software-mode LR that is in active state can me - // moved straight to active+pending. - if (old_state == ICH_LR_EL2_STATE_ACTIVE) { - ICH_LR_EL2_base_set_State( - &new_lr.base, - ICH_LR_EL2_STATE_PENDING_ACTIVE); - } else { - VGIC_DEBUG_TRACE( - HWSTATE_UNCHANGED, vic, vcpu, - "redeliver {:d}: redundant assertions merged", - virq); - } - - // Interrupt is already pending, so the VCPU should be - // awake; no need for a wakeup. + { + vgic_redeliver_lr_info_t vgic_redeliver_lr_info = + vgic_redeliver_lr_update_state( + vic, vcpu, source, virq, old_lr_state, + status, *old_dstate, assert_dstate); + new_dstate = vgic_redeliver_lr_info.new_dstate; + force_eoi_trap = vgic_redeliver_lr_info.force_eoi_trap; + need_wakeup = vgic_redeliver_lr_info.need_wakeup; + new_lr = vgic_redeliver_lr_info.new_lr; } + trace_state = ICH_LR_EL2_base_get_State(&new_lr.base); VGIC_TRACE(HWSTATE_CHANGED, vic, vcpu, - "redeliver {:d}: lr {:d} -> {:d}", virq, old_state, - ICH_LR_EL2_base_get_State(&new_lr.base)); - - vgic_delivery_state_set_edge(&new_dstate, force_eoi_trap); + "redeliver {:d}: lr {:d} -> {:d}", virq, + (register_t)old_lr_state, (register_t)trace_state); } while (!atomic_compare_exchange_strong_explicit( dstate, old_dstate, new_dstate, memory_order_relaxed, memory_order_relaxed)); @@ -759,19 +977,17 @@ vgic_redeliver(vic_t *vic, thread_t *vcpu, virq_source_t *source, vgic_delivery_state_t *old_dstate, vgic_delivery_state_t assert_dstate) { - const bool to_self = vcpu == thread_get_self(); - bool found_lr = false; + const bool to_self = vcpu == thread_get_self(); index_t i; bool_result_t ret = bool_result_error(ERROR_BUSY); for (i = 0; i < CPU_GICH_LR_COUNT; i++) { if (dstate == vcpu->vgic_lrs[i].dstate) { - found_lr = true; break; } } - if (found_lr) { + if (i < CPU_GICH_LR_COUNT) { // If we are targeting ourselves, read the current state. if (to_self) { vgic_read_lr_state(i); @@ -895,12 +1111,25 @@ vgic_select_lr(thread_t *vcpu, uint8_t priority, uint8_t *lr_priority) return result; } -// The number of VIRQs in each low (SPI + PPI) range. +// The number of VIRQs in each low (SPI + PPI) range other than the last SPI +// range (which has 4 fewer because of the "special" IRQ numbers 1020-1023). #define VGIC_LOW_RANGE_SIZE \ - (count_t)((GIC_SPI_BASE + GIC_SPI_NUM + VGIC_LOW_RANGES - 1) / \ + (count_t)((GIC_SPI_BASE + GIC_SPI_NUM + VGIC_LOW_RANGES - 1U) / \ VGIC_LOW_RANGES) static_assert(util_is_p2(VGIC_LOW_RANGE_SIZE), "VGIC search ranges must have power-of-two sizes"); +static_assert(VGIC_LOW_RANGE_SIZE > GIC_SPECIAL_INTIDS_NUM, + "VGIC search ranges must have size greater than 4"); + +// The number of VIRQs in a specific low range, taking into account the special +// IRQ numbers that immediately follow the SPIs. +static count_t +vgic_low_range_size(index_t range) +{ + return (range == (VGIC_LOW_RANGES - 1U)) + ? (VGIC_LOW_RANGE_SIZE - GIC_SPECIAL_INTIDS_NUM) + : VGIC_LOW_RANGE_SIZE; +} // Mark an unlisted interrupt as pending on a VCPU. // @@ -918,7 +1147,7 @@ vgic_flag_locked(virq_t virq, thread_t *vcpu, uint8_t priority, bool group1, { assert_preempt_disabled(); - count_t priority_shifted = priority >> VGIC_PRIO_SHIFT; + count_t priority_shifted = (count_t)priority >> VGIC_PRIO_SHIFT; bitmap_atomic_set(vcpu->vgic_search_ranges_low[priority_shifted], virq / VGIC_LOW_RANGE_SIZE, memory_order_release); @@ -961,8 +1190,9 @@ vgic_flag_locked(virq_t virq, thread_t *vcpu, uint8_t priority, bool group1, // signalling is performed without having to acquire the LR lock. static void vgic_flag_unlocked(virq_t virq, thread_t *vcpu, uint8_t priority) + REQUIRE_PREEMPT_DISABLED { - count_t priority_shifted = priority >> VGIC_PRIO_SHIFT; + count_t priority_shifted = (count_t)priority >> VGIC_PRIO_SHIFT; if (!bitmap_atomic_test_and_set( vcpu->vgic_search_ranges_low[priority_shifted], @@ -986,9 +1216,9 @@ vgic_flag_unlocked(virq_t virq, thread_t *vcpu, uint8_t priority) ipi_one(IPI_REASON_VGIC_DELIVER, lr_owner); } else { - scheduler_lock(vcpu); + scheduler_lock_nopreempt(vcpu); vcpu_wakeup(vcpu); - scheduler_unlock(vcpu); + scheduler_unlock_nopreempt(vcpu); } } } @@ -1058,7 +1288,7 @@ vgic_route_1n_preference(vic_t *vic, thread_t *vcpu, REQUIRE_SCHEDULER_LOCK(vcpu) { vgic_route_preference_t ret = VGIC_ROUTE_DENIED; - thread_t *current = thread_get_self(); + thread_t *current = thread_get_self(); if (compiler_unexpected(!vgic_route_allowed(vic, vcpu, dstate))) { ret = VGIC_ROUTE_DENIED; @@ -1134,7 +1364,7 @@ vgic_wakeup_1n(vic_t *vic, bool class0, bool class1) // We always start this search from the VCPU corresponding to the // current physical CPU, to reduce the chances of waking a second // physical CPU if the GIC has just chosen to wake this one. - count_t start_point = cpulocal_get_index(); + count_t start_point = cpulocal_check_index(cpulocal_get_index_unsafe()); for (index_t i = 0U; i < vic->gicr_count; i++) { cpu_index_t cpu = (cpu_index_t)((i + start_point) % vic->gicr_count); @@ -1200,7 +1430,7 @@ vgic_get_route_from_state(vic_t *vic, vgic_delivery_state_t dstate, if (use_local_vcpu) { // Assuming that any VM receiving physical 1-of-N IRQs has a // 1:1 VCPU to PCPU mapping, start by checking the local VCPU. - start_point = cpulocal_get_index(); + start_point = cpulocal_check_index(cpulocal_get_index_unsafe()); } else { // Determine the starting point for VIRQ selection using // round-robin, if we didn't get a hint from the physical GIC. @@ -1228,7 +1458,8 @@ vgic_get_route_from_state(vic_t *vic, vgic_delivery_state_t dstate, target = candidate; VGIC_DEBUG_TRACE(ROUTE, vic, target, "route: {:d} immediate, checked {:d}", - target->vgic_gicr_index, i + 1); + target->vgic_gicr_index, + ((register_t)i + 1U)); goto out; } if (candidate_pref > target_pref) { @@ -1242,6 +1473,7 @@ vgic_get_route_from_state(vic_t *vic, vgic_delivery_state_t dstate, // This should be unconditional, and everything beyond this point // should be moved to after the VIRQ has been flagged as unrouted. // + // FIXME: if (compiler_expected(target != NULL)) { VGIC_DEBUG_TRACE(ROUTE, vic, target, "route: {:d} best ({:d})", target->vgic_gicr_index, @@ -1249,9 +1481,10 @@ vgic_get_route_from_state(vic_t *vic, vgic_delivery_state_t dstate, goto out; } - GICD_CTLR_DS_t gicd_ctlr = atomic_load_relaxed(&vic->gicd_ctlr); + GICD_CTLR_DS_t gicd_ctlr = atomic_load_relaxed(&vic->gicd_ctlr); + bool trace_IsE1NWF = GICD_CTLR_DS_get_E1NWF(&gicd_ctlr); VGIC_TRACE(ROUTE, vic, target, "route: none (E1NWF={:d})", - GICD_CTLR_DS_get_E1NWF(&gicd_ctlr)); + (register_t)trace_IsE1NWF); out: return target; @@ -1301,6 +1534,7 @@ vgic_get_route_for_spi(vic_t *vic, virq_t virq, bool use_local_vcpu) static bool vgic_try_route_and_flag(vic_t *vic, virq_t virq, vgic_delivery_state_t new_dstate, bool use_local_vcpu) + REQUIRE_PREEMPT_DISABLED { rcu_read_start(); thread_t *target = @@ -1321,73 +1555,36 @@ vgic_try_route_and_flag(vic_t *vic, virq_t virq, // failure, and triggers a 1-of-N wakeup. static void vgic_route_and_flag(vic_t *vic, virq_t virq, vgic_delivery_state_t new_dstate, - bool use_local_vcpu) + bool use_local_vcpu) REQUIRE_PREEMPT_DISABLED { if (!vgic_try_route_and_flag(vic, virq, new_dstate, use_local_vcpu)) { vgic_flag_unrouted(vic, virq); #if VGIC_HAS_1N vgic_wakeup_1n(vic, - !vgic_delivery_state_get_nclass0(&new_dstate), - vgic_delivery_state_get_class1(&new_dstate)); + vgic_get_delivery_state_is_class0(&new_dstate), + vgic_get_delivery_state_is_class1(&new_dstate)); #endif } } -// Clear out a VIRQ from a specified LR and flag it to be delivered later. -// -// This is used when there are no empty LRs available to deliver an IRQ, but -// an LR is occupied by an IRQ that is either lower-priority, or already -// acknowledged, or (in the current thread) already deactivated. It is also -// used when tearing down a VCPU permanently, so active IRQs can't be left -// in the LRs as they are for a normal group disable. In the latter case, the -// reroute argument should be true, to force the route to be recalculated. -// -// The specified VCPU must either be the current thread, or LR-locked by the -// caller and known not to be running remotely. If the specified VCPU is the -// current thread, the caller must rewrite the LR after calling this function. -// -// The specified LR must be occupied. If it contains an active interrupt -// (regardless of its pending state), it must be the lowest-priority listed -// active interrupt on the VCPU, to ensure that the active_unlisted stack is -// correctly ordered. -static void -vgic_reclaim_lr(vic_t *vic, thread_t *vcpu, index_t lr, bool reroute) - REQUIRE_LOCK(vcpu->vgic_lr_owner_lock) REQUIRE_PREEMPT_DISABLED +static vgic_delivery_state_t +vgic_reclaim_update_level_src_and_hw(const vic_t *vic, const thread_t *vcpu, + virq_t virq, + vgic_delivery_state_t *old_dstate, + bool lr_active, bool lr_hw, + vgic_lr_status_t *status, + virq_source_t *source) + REQUIRE_PREEMPT_DISABLED { - const bool from_self = vcpu == thread_get_self(); - vgic_lr_status_t *status = &vcpu->vgic_lrs[lr]; - assert(status->dstate != NULL); - - if (from_self) { - vgic_read_lr_state(lr); - } - - virq_t virq = ICH_LR_EL2_base_get_vINTID(&status->lr.base); - bool lr_hw = ICH_LR_EL2_base_get_HW(&status->lr.base); - bool lr_pending = (ICH_LR_EL2_base_get_State(&status->lr.base) == - ICH_LR_EL2_STATE_PENDING) || + bool lr_pending = (ICH_LR_EL2_base_get_State(&status->lr.base) == + ICH_LR_EL2_STATE_PENDING) || (ICH_LR_EL2_base_get_State(&status->lr.base) == ICH_LR_EL2_STATE_PENDING_ACTIVE); - bool lr_active = (ICH_LR_EL2_base_get_State(&status->lr.base) == - ICH_LR_EL2_STATE_ACTIVE) || - (ICH_LR_EL2_base_get_State(&status->lr.base) == - ICH_LR_EL2_STATE_PENDING_ACTIVE); - - - if (lr_active) { - index_t i = vcpu->vgic_active_unlisted_count % VGIC_PRIORITIES; - vcpu->vgic_active_unlisted[i] = virq; - vcpu->vgic_active_unlisted_count++; - } - - virq_source_t *source = vgic_find_source(vic, vcpu, virq); - vgic_delivery_state_t old_dstate = atomic_load_relaxed(status->dstate); - vgic_delivery_state_t new_dstate; bool need_deactivate; do { - new_dstate = old_dstate; + new_dstate = *old_dstate; need_deactivate = false; vgic_delivery_state_set_active(&new_dstate, lr_active); @@ -1399,15 +1596,15 @@ vgic_reclaim_lr(vic_t *vic, thread_t *vcpu, index_t lr, bool reroute) } // Update level_src and hw_active based on the LR state. - if (lr_hw && vgic_delivery_state_get_hw_active(&old_dstate)) { + if (lr_hw && vgic_delivery_state_get_hw_active(old_dstate)) { // If it's a hardware IRQ that has already been marked // active somewhere else, we don't need to change its // state beyond the abave. For this to happen, it must // have been inactive in the LR already. assert(!lr_pending && !lr_active); } else if (lr_hw && lr_pending && - vgic_delivery_state_get_need_sync(&old_dstate) && - !vgic_delivery_state_get_cfg_is_edge(&old_dstate)) { + vgic_delivery_state_get_need_sync(old_dstate) && + !vgic_delivery_state_get_cfg_is_edge(old_dstate)) { // If it's a pending hardware level-triggered interrupt // that has been marked for sync, we clear its pending // state and deactivate it early to force the hardware @@ -1430,16 +1627,14 @@ vgic_reclaim_lr(vic_t *vic, thread_t *vcpu, index_t lr, bool reroute) // any remote assertion). vgic_delivery_state_set_level_src( &new_dstate, - vgic_delivery_state_get_hw_active(&old_dstate)); - } else if (vgic_delivery_state_get_level_src(&old_dstate)) { + vgic_delivery_state_get_hw_active(old_dstate)); + } else if (vgic_delivery_state_get_level_src(old_dstate)) { // It's a software IRQ with level_src set; call the // source to check whether it's still pending, and order // the check_pending event after the dstate read. - atomic_thread_fence(memory_order_acquire); - if (compiler_unexpected(source == NULL) || - !trigger_virq_check_pending_event( - source->trigger, source, - vgic_delivery_state_get_edge(&old_dstate))) { + if (!vgic_virq_check_pending( + source, + vgic_delivery_state_get_edge(old_dstate))) { vgic_delivery_state_set_level_src(&new_dstate, false); } @@ -1447,11 +1642,11 @@ vgic_reclaim_lr(vic_t *vic, thread_t *vcpu, index_t lr, bool reroute) // Software IRQ with level_src clear; nothing to do. } } while (!atomic_compare_exchange_strong_explicit( - status->dstate, &old_dstate, new_dstate, memory_order_relaxed, + status->dstate, old_dstate, new_dstate, memory_order_relaxed, memory_order_relaxed)); VGIC_TRACE(DSTATE_CHANGED, vic, vcpu, "reclaim_lr {:d}: {:#x} -> {:#x}", - virq, vgic_delivery_state_raw(old_dstate), + virq, vgic_delivery_state_raw(*old_dstate), vgic_delivery_state_raw(new_dstate)); if (need_deactivate) { @@ -1462,6 +1657,65 @@ vgic_reclaim_lr(vic_t *vic, thread_t *vcpu, index_t lr, bool reroute) gicv3_irq_deactivate(ICH_LR_EL2_HW1_get_pINTID(&status->lr.hw)); } + return new_dstate; +} + +// Clear out a VIRQ from a specified LR and flag it to be delivered later. +// +// This is used when there are no empty LRs available to deliver an IRQ, but +// an LR is occupied by an IRQ that is either lower-priority, or already +// acknowledged, or (in the current thread) already deactivated. It is also +// used when tearing down a VCPU permanently, so active IRQs can't be left +// in the LRs as they are for a normal group disable. In the latter case, the +// reroute argument should be true, to force the route to be recalculated. +// +// The specified VCPU must either be the current thread, or LR-locked by the +// caller and known not to be running remotely. If the specified VCPU is the +// current thread, the caller must rewrite the LR after calling this function. +// +// The specified LR must be occupied. If it contains an active interrupt +// (regardless of its pending state), it must be the lowest-priority listed +// active interrupt on the VCPU, to ensure that the active_unlisted stack is +// correctly ordered. +static void +vgic_reclaim_lr(vic_t *vic, thread_t *vcpu, index_t lr, bool reroute) + REQUIRE_LOCK(vcpu->vgic_lr_owner_lock) REQUIRE_PREEMPT_DISABLED +{ + const bool from_self = vcpu == thread_get_self(); + vgic_lr_status_t *status = &vcpu->vgic_lrs[lr]; + assert(status->dstate != NULL); + + if (from_self) { + vgic_read_lr_state(lr); + } + + virq_t virq = ICH_LR_EL2_base_get_vINTID(&status->lr.base); + bool lr_hw = ICH_LR_EL2_base_get_HW(&status->lr.base); + bool lr_active = (ICH_LR_EL2_base_get_State(&status->lr.base) == + ICH_LR_EL2_STATE_ACTIVE) || + (ICH_LR_EL2_base_get_State(&status->lr.base) == + ICH_LR_EL2_STATE_PENDING_ACTIVE); + +#if VGIC_HAS_LPI && GICV3_HAS_VLPI_V4_1 + if (vic->vsgis_enabled && + (vgic_get_irq_type(virq) == VGIC_IRQ_TYPE_SGI)) { + // vSGIs have no active state. + lr_active = false; + } +#endif + + if (lr_active) { + index_t i = vcpu->vgic_active_unlisted_count % VGIC_PRIORITIES; + vcpu->vgic_active_unlisted[i] = virq; + vcpu->vgic_active_unlisted_count++; + } + + virq_source_t *source = vgic_find_source(vic, vcpu, virq); + vgic_delivery_state_t old_dstate = atomic_load_relaxed(status->dstate); + + vgic_delivery_state_t new_dstate = vgic_reclaim_update_level_src_and_hw( + vic, vcpu, virq, &old_dstate, lr_active, lr_hw, status, source); + #if VGIC_HAS_1N if (lr_hw && vgic_delivery_state_get_route_1n(&new_dstate)) { vgic_spi_set_route_1n(ICH_LR_EL2_HW1_get_pINTID(&status->lr.hw), @@ -1494,103 +1748,133 @@ vgic_reclaim_lr(vic_t *vic, thread_t *vcpu, index_t lr, bool reroute) static bool vgic_sync_vcpu(thread_t *vcpu, bool hw_access); -// Try to deliver a VIRQ to a specified target for a specified reason. -// -// The specified VCPU is the current route of the VIRQ if it is shared (in which -// case it may be NULL), or the owner of the VIRQ if it is private. -// -// The pending flags in assert_dstate will be asserted in the delivery state. -// This may be 0 if pending flags have already been set by the caller. This -// value must not have any flags set other than the four pending flags and the -// enabled flag. -// -// If the level_src pending bit or the hw_active bit is being set, the VIRQ -// source must be specified. Otherwise, the source may be NULL, even if a -// registered source exists for the VIRQ. -// -// The is_private flag should be set if the delivered interrupt cannot possibly -// be rerouted. This is used to reduce the set of VCPUs that receive IPIs when a -// currently listed interrupt is redelivered, e.g. on an SGI to a busy VCPU. -// -// If it is not possible to immediately list the VIRQ, the target's -// pending-check flags will be updated so it will find the VIRQ next time it -// goes looking for pending interrupts to assert. -// -// This function returns the previous delivery state. -vgic_delivery_state_t -vgic_deliver(virq_t virq, vic_t *vic, thread_t *vcpu, virq_source_t *source, - _Atomic vgic_delivery_state_t *dstate, - vgic_delivery_state_t assert_dstate, bool is_private) +static void +vgic_list_irq(vgic_delivery_state_t new_dstate, index_t lr, bool is_hw, + uint8_t priority, _Atomic vgic_delivery_state_t *dstate, + virq_t virq, vic_t *vic, thread_t *vcpu, virq_source_t *source, + bool to_self) REQUIRE_LOCK(vcpu->vgic_lr_owner_lock) + REQUIRE_PREEMPT_DISABLED { - bool to_self = vcpu == thread_get_self(); - bool need_wakeup = true; - bool need_sync_all = false; + assert(vgic_delivery_state_get_listed(&new_dstate)); + assert_debug(lr < CPU_GICH_LR_COUNT); - assert((source != NULL) || - !vgic_delivery_state_get_level_src(&assert_dstate)); - assert((source == NULL) || - (vgic_get_irq_type(source->virq) == VGIC_IRQ_TYPE_PPI) || - (vgic_get_irq_type(source->virq) == VGIC_IRQ_TYPE_SPI)); - - cpu_index_t remote_cpu = vgic_lr_owner_lock(vcpu); + vgic_lr_status_t *status = &vcpu->vgic_lrs[lr]; + if (status->dstate != NULL) { + vgic_reclaim_lr(vic, vcpu, lr, false); + assert(status->dstate == NULL); + } - vgic_delivery_state_t old_dstate = atomic_load_relaxed(dstate); - vgic_delivery_state_t new_dstate = - vgic_delivery_state_union(old_dstate, assert_dstate); + status->dstate = dstate; + ICH_LR_EL2_base_set_HW(&status->lr.base, is_hw); + if (is_hw) { + ICH_LR_EL2_HW1_set_pINTID(&status->lr.hw, + hwirq_from_virq_source(source)->irq); + } else { + ICH_LR_EL2_HW0_set_EOI( + &status->lr.sw, + !vgic_delivery_state_get_cfg_is_edge(&new_dstate) && + vgic_delivery_state_is_level_asserted( + &new_dstate)); + } + ICH_LR_EL2_base_set_vINTID(&status->lr.base, virq); + ICH_LR_EL2_base_set_Priority(&status->lr.base, priority); + ICH_LR_EL2_base_set_Group(&status->lr.base, + vgic_delivery_state_get_group1(&new_dstate)); + ICH_LR_EL2_base_set_State(&status->lr.base, ICH_LR_EL2_STATE_PENDING); - if (vgic_delivery_state_get_listed(&old_dstate) && - vgic_delivery_state_is_pending(&new_dstate) && - vgic_delivery_state_get_enabled(&new_dstate) && (vcpu != NULL) && - !cpulocal_index_valid(remote_cpu)) { - // Fast path: try to reset the pending state in the LR. This can - // fail if the LR is not found, e.g. because the routing has - // changed. Note that this function updates dstate if it - // succeeds, so we can skip the updates below. - // - // We don't check the route, priority or group enables here - // because listed IRQs affected by changes in those since they - // were first listed either don't need an immediate update, or - // else will be updated by whoever is changing them. - // - // We only need to try this once, because the listed bit can't - // be changed by anyone else while we're holding the LR lock. - bool_result_t redeliver_wakeup = vgic_redeliver( - vic, vcpu, source, dstate, &old_dstate, assert_dstate); - if (redeliver_wakeup.e == OK) { - need_wakeup = redeliver_wakeup.r; - goto out; - } + if (to_self) { + vgic_write_lr(lr); } +} - // If this is a physical SPI assertion, we may need to update the route - // of the physical SPI. - if (vgic_delivery_state_get_hw_active(&assert_dstate)) { -#if VGIC_HAS_1N - if (vgic_delivery_state_get_route_1n(&old_dstate) && - (vcpu != NULL)) { - // Make the IRQ stick to the physical CPU that is - // handling it, until it is delisted. This is because a - // delivery on another CPU while it is still listed will - // require a sync IPI to delist it first. - hwirq_t *hwirq = hwirq_from_virq_source(source); - gicv3_spi_set_route(hwirq->irq, vcpu->vgic_irouter); - } else -#endif - if (cpulocal_index_valid(remote_cpu)) { - assert(vcpu != NULL); - // IRQ was HW-delivered to the wrong CPU, probably - // because the VCPU was migrated. Update the route. - hwirq_t *hwirq = hwirq_from_virq_source(source); - gicv3_spi_set_route(hwirq->irq, vcpu->vgic_irouter); +typedef struct { + bool need_wakeup; + bool need_sync_all; +} vgic_deliver_list_or_flag_info_t; + +static vgic_deliver_list_or_flag_info_t +vgic_deliver_list_or_flag(vic_t *vic, thread_t *vcpu, virq_source_t *source, + vgic_delivery_state_t old_dstate, + vgic_delivery_state_t new_dstate, index_result_t lr_r, + _Atomic vgic_delivery_state_t *dstate, virq_t virq, + cpu_index_t remote_cpu, uint8_t lr_priority, + bool is_private, bool to_self, bool is_hw, + uint8_t priority, bool pending, bool enabled, + bool route_valid) + REQUIRE_LOCK(vcpu->vgic_lr_owner_lock) REQUIRE_PREEMPT_DISABLED +{ + bool need_wakeup = true; + bool need_sync_all = false; - VGIC_TRACE(HWSTATE_CHANGED, vic, vcpu, - "lazy reroute {:d}: to cpu {:d}", hwirq->irq, - remote_cpu); + assert(vcpu != NULL); + thread_t *target = vcpu; + + if (!pending) { + // Not pending; nothing more to do. + need_wakeup = false; + } else if (vgic_delivery_state_get_listed(&old_dstate)) { + // IRQ is already listed remotely; send a sync IPI. + assert(vgic_delivery_state_get_need_sync(&new_dstate)); + if (!is_private) { + need_sync_all = true; + need_wakeup = false; + } else if (cpulocal_index_valid(remote_cpu)) { + ipi_one(IPI_REASON_VGIC_SYNC, remote_cpu); } else { - // Directly routed to the correct CPU; do nothing + TRACE(DEBUG, INFO, + "vgic sync after failed redeliver of {:#x}: dstate {:#x} -> {:#x}", + virq, vgic_delivery_state_raw(old_dstate), + vgic_delivery_state_raw(new_dstate)); + + (void)vgic_sync_vcpu(target, to_self); } + } else if (!enabled) { + // Not enabled; nothing more to do. + need_wakeup = false; + } else if (!route_valid) { + // The route became invalid after it was selected. Try to + // re-route and flag it, and if that fails, flag it as unrouted. + // This function issues a wakeup, so we don't need to do it + // below. + vgic_route_and_flag(vic, virq, new_dstate, false); + need_wakeup = false; + } else if ((lr_r.e == OK) && (priority < lr_priority)) { + // List the IRQ immediately. + vgic_list_irq(new_dstate, lr_r.r, is_hw, priority, dstate, virq, + vic, vcpu, source, to_self); + } else { + assert(route_valid); + // We have a valid route, but can't immediately list; set the + // search flags in the target VCPU so it finds this VIRQ next + // time it goes looking for something to deliver. A delivery IPI + // is sent if the target is currently running. + vgic_flag_locked(virq, target, priority, + vgic_delivery_state_get_group1(&new_dstate), + remote_cpu); } + return (vgic_deliver_list_or_flag_info_t){ + .need_wakeup = need_wakeup, + .need_sync_all = need_sync_all, + }; +} + +typedef struct { + vgic_delivery_state_t new_dstate; + vgic_delivery_state_t old_dstate; + bool need_wakeup; + bool need_sync_all; + uint8_t pad[2]; +} vgic_deliver_info_t; + +static vgic_deliver_info_t +vgic_deliver_update_state(virq_t virq, vgic_delivery_state_t prev_dstate, + vgic_delivery_state_t assert_dstate, + _Atomic vgic_delivery_state_t *dstate, vic_t *vic, + thread_t *vcpu, cpu_index_t remote_cpu, + virq_source_t *source, bool is_private, bool to_self) + REQUIRE_LOCK(vcpu->vgic_lr_owner_lock) REQUIRE_PREEMPT_DISABLED +{ // Keep track of the LR allocated for delivery (if any) and the priority // of the VIRQ currently in it (if any). index_result_t lr_r = index_result_error(ERROR_BUSY); @@ -1606,6 +1890,11 @@ vgic_deliver(virq_t virq, vic_t *vic, thread_t *vcpu, virq_source_t *source, // at this point. assert(lr_r.e != OK); + vgic_delivery_state_t new_dstate; + vgic_delivery_state_t old_dstate = prev_dstate; + bool need_wakeup = true; + bool need_sync_all = false; + do { new_dstate = vgic_delivery_state_union(old_dstate, assert_dstate); @@ -1672,84 +1961,144 @@ vgic_deliver(virq_t virq, vic_t *vic, thread_t *vcpu, virq_source_t *source, goto out; } - assert(vcpu != NULL); - thread_t *target = vcpu; + vgic_deliver_list_or_flag_info_t info = vgic_deliver_list_or_flag( + vic, vcpu, source, old_dstate, new_dstate, lr_r, dstate, virq, + remote_cpu, lr_priority, is_private, to_self, is_hw, priority, + pending, enabled, route_valid); - if (!pending) { - // Not pending; nothing more to do. - need_wakeup = false; - } else if (vgic_delivery_state_get_listed(&old_dstate)) { - // IRQ is already listed remotely; send a sync IPI. - assert(vgic_delivery_state_get_need_sync(&new_dstate)); - if (!is_private) { - need_sync_all = true; - need_wakeup = false; - } else if (cpulocal_index_valid(remote_cpu)) { - ipi_one(IPI_REASON_VGIC_SYNC, remote_cpu); - } else { - TRACE(DEBUG, INFO, - "vgic sync after failed redeliver of {:#x}: dstate {:#x} -> {:#x}", - virq, vgic_delivery_state_raw(old_dstate), - vgic_delivery_state_raw(new_dstate)); + need_wakeup = info.need_wakeup; + need_sync_all = info.need_sync_all; - vgic_sync_vcpu(target, to_self); - } - } else if (!enabled) { - // Not enabled; nothing more to do. - need_wakeup = false; - } else if (!route_valid) { - // The route became invalid after it was selected. Try to - // re-route and flag it, and if that fails, flag it as unrouted. - // This function issues a wakeup, so we don't need to do it - // below. - vgic_route_and_flag(vic, virq, new_dstate, false); - need_wakeup = false; - } else if ((lr_r.e == OK) && (priority < lr_priority)) { - // List the IRQ immediately. - assert(vgic_delivery_state_get_listed(&new_dstate)); +out: + return (vgic_deliver_info_t){ + .new_dstate = new_dstate, + .old_dstate = old_dstate, + .need_wakeup = need_wakeup, + .need_sync_all = need_sync_all, + }; +} - vgic_lr_status_t *status = &vcpu->vgic_lrs[lr_r.r]; - if (status->dstate != NULL) { - vgic_reclaim_lr(vic, vcpu, lr_r.r, false); - assert(status->dstate == NULL); - } +static void +vgic_deliver_update_spi_route(vgic_delivery_state_t assert_dstate, + vgic_delivery_state_t old_dstate, + const vic_t *vic, const thread_t *vcpu, + cpu_index_t remote_cpu, virq_source_t *source) +{ + if (vgic_delivery_state_get_hw_active(&assert_dstate)) { +#if VGIC_HAS_1N + if (vgic_delivery_state_get_route_1n(&old_dstate) && + (vcpu != NULL)) { + // Make the IRQ stick to the physical CPU that is + // handling it, until it is delisted. This is because a + // delivery on another CPU while it is still listed + // will require a sync IPI to delist it first. + hwirq_t *hwirq = hwirq_from_virq_source(source); + (void)gicv3_spi_set_route(hwirq->irq, + vcpu->vgic_irouter); + } else +#else + (void)old_dstate; +#endif + if (cpulocal_index_valid(remote_cpu)) { + assert(vcpu != NULL); + // IRQ was HW-delivered to the wrong CPU, probably + // because the VCPU was migrated. Update the route. + hwirq_t *hwirq = hwirq_from_virq_source(source); + (void)gicv3_spi_set_route(hwirq->irq, + vcpu->vgic_irouter); - status->dstate = dstate; - ICH_LR_EL2_base_set_HW(&status->lr.base, is_hw); - if (is_hw) { - ICH_LR_EL2_HW1_set_pINTID( - &status->lr.hw, - hwirq_from_virq_source(source)->irq); + VGIC_TRACE(HWSTATE_CHANGED, vic, vcpu, + "lazy reroute {:d}: to cpu {:d}", hwirq->irq, + remote_cpu); } else { - ICH_LR_EL2_HW0_set_EOI( - &status->lr.sw, - !vgic_delivery_state_get_cfg_is_edge( - &new_dstate) && - vgic_delivery_state_is_level_asserted( - &new_dstate)); + // Directly routed to the correct CPU; do nothing } - ICH_LR_EL2_base_set_vINTID(&status->lr.base, virq); - ICH_LR_EL2_base_set_Priority(&status->lr.base, priority); - ICH_LR_EL2_base_set_Group( - &status->lr.base, - vgic_delivery_state_get_group1(&new_dstate)); - ICH_LR_EL2_base_set_State(&status->lr.base, - ICH_LR_EL2_STATE_PENDING); + } +} - if (to_self) { - vgic_write_lr(lr_r.r); +// Try to deliver a VIRQ to a specified target for a specified reason. +// +// The specified VCPU is the current route of the VIRQ if it is shared (in which +// case it may be NULL), or the owner of the VIRQ if it is private. +// +// The pending flags in assert_dstate will be asserted in the delivery state. +// This may be 0 if pending flags have already been set by the caller. This +// value must not have any flags set other than the four pending flags and the +// enabled flag. +// +// If the level_src pending bit or the hw_active bit is being set, the VIRQ +// source must be specified. Otherwise, the source may be NULL, even if a +// registered source exists for the VIRQ. +// +// The is_private flag should be set if the delivered interrupt cannot possibly +// be rerouted. This is used to reduce the set of VCPUs that receive IPIs when a +// currently listed interrupt is redelivered, e.g. on an SGI to a busy VCPU. +// +// If it is not possible to immediately list the VIRQ, the target's +// pending-check flags will be updated so it will find the VIRQ next time it +// goes looking for pending interrupts to assert. +// +// This function returns the previous delivery state. +vgic_delivery_state_t +vgic_deliver(virq_t virq, vic_t *vic, thread_t *vcpu, virq_source_t *source, + _Atomic vgic_delivery_state_t *dstate, + vgic_delivery_state_t assert_dstate, bool is_private) +{ + bool to_self = vcpu == thread_get_self(); + bool need_wakeup = true; + bool need_sync_all = false; + + assert((source != NULL) || + !vgic_delivery_state_get_level_src(&assert_dstate)); + assert((source == NULL) || + (vgic_get_irq_type(source->virq) == VGIC_IRQ_TYPE_PPI) || + (vgic_get_irq_type(source->virq) == VGIC_IRQ_TYPE_SPI)); + + cpu_index_t remote_cpu = vgic_lr_owner_lock(vcpu); + + vgic_delivery_state_t old_dstate = atomic_load_relaxed(dstate); + vgic_delivery_state_t new_dstate = + vgic_delivery_state_union(old_dstate, assert_dstate); + + if (vgic_delivery_state_get_listed(&old_dstate) && + vgic_delivery_state_is_pending(&new_dstate) && + vgic_delivery_state_get_enabled(&new_dstate) && (vcpu != NULL) && + !cpulocal_index_valid(remote_cpu)) { + // Fast path: try to reset the pending state in the LR. This can + // fail if the LR is not found, e.g. because the routing has + // changed. Note that this function updates dstate if it + // succeeds, so we can skip the updates below. + // + // We don't check the route, priority or group enables here + // because listed IRQs affected by changes in those since they + // were first listed either don't need an immediate update, or + // else will be updated by whoever is changing them. + // + // We only need to try this once, because the listed bit can't + // be changed by anyone else while we're holding the LR lock. + bool_result_t redeliver_wakeup = vgic_redeliver( + vic, vcpu, source, dstate, &old_dstate, assert_dstate); + if (redeliver_wakeup.e == OK) { + need_wakeup = redeliver_wakeup.r; + goto out; } - } else { - assert(route_valid); - // We have a valid route, but can't immediately list; set the - // search flags in the target VCPU so it finds this VIRQ next - // time it goes looking for something to deliver. A delivery IPI - // is sent if the target is currently running. - vgic_flag_locked(virq, target, priority, - vgic_delivery_state_get_group1(&new_dstate), - remote_cpu); } + // If this is a physical SPI assertion, we may need to update the route + // of the physical SPI. + vgic_deliver_update_spi_route(assert_dstate, old_dstate, vic, vcpu, + remote_cpu, source); + + // Update the dstate and deliver the interrupt + vgic_deliver_info_t vgic_deliver_info = vgic_deliver_update_state( + virq, old_dstate, assert_dstate, dstate, vic, vcpu, remote_cpu, + source, is_private, to_self); + + new_dstate = vgic_deliver_info.new_dstate; + old_dstate = vgic_deliver_info.old_dstate; + need_wakeup = vgic_deliver_info.need_wakeup; + need_sync_all = vgic_deliver_info.need_sync_all; + out: vgic_lr_owner_unlock(vcpu); @@ -1764,8 +2113,8 @@ vgic_deliver(virq_t virq, vic_t *vic, thread_t *vcpu, virq_source_t *source, #if VGIC_HAS_1N vgic_wakeup_1n( vic, - !vgic_delivery_state_get_nclass0(&new_dstate), - vgic_delivery_state_get_class1(&new_dstate)); + vgic_get_delivery_state_is_class0(&new_dstate), + vgic_get_delivery_state_is_class1(&new_dstate)); #else // VIRQ is unrouted; there is no VCPU we can wake. assert(!need_wakeup); @@ -1832,7 +2181,7 @@ vgic_update_enables(vic_t *vic, GICD_CTLR_DS_t gicd_ctlr) rcu_read_start(); for (index_t i = 0; i < vic->gicr_count; i++) { - thread_t *vcpu = atomic_load_consume(&vic->gicr_vcpus[i]); + thread_t *vcpu = atomic_load_consume(&vic->gicr_vcpus[i]); cpu_index_t lr_owner = vgic_lr_owner_lock_nopreempt(vcpu); if (thread_get_self() == vcpu) { if (vgic_gicr_update_group_enables(vic, vcpu, @@ -1964,7 +2313,7 @@ vgic_handle_eoi_lr(vic_t *vic, thread_t *vcpu, index_t lr) REQUIRE_PREEMPT_DISABLED { assert(thread_get_self() == vcpu); - assert(lr < CPU_GICH_LR_COUNT); + assert_debug(lr < CPU_GICH_LR_COUNT); // The specified LR should have a software delivery listed in it vgic_lr_status_t *status = &vcpu->vgic_lrs[lr]; @@ -1986,8 +2335,6 @@ vgic_handle_eoi_lr(vic_t *vic, thread_t *vcpu, index_t lr) // If level_src is set, check that the source is still pending // before we set the LR state back to pending. if (vgic_delivery_state_get_level_src(&old_dstate)) { - // Order the check_pending event after the dstate read. - atomic_thread_fence(memory_order_acquire); // If the source is a physical SPI, assume it is no // longer pending. If it is, then the deactivation below // will reassert it. Note that this is different to the @@ -1998,11 +2345,9 @@ vgic_handle_eoi_lr(vic_t *vic, thread_t *vcpu, index_t lr) VIRQ_TRIGGER_VGIC_FORWARDED_SPI)) { vgic_delivery_state_set_level_src(&new_dstate, false); - } else if (compiler_unexpected(source == NULL) || - !trigger_virq_check_pending_event( - source->trigger, source, - vgic_delivery_state_get_edge( - &new_dstate))) { + } else if (!vgic_virq_check_pending( + source, vgic_delivery_state_get_edge( + &new_dstate))) { vgic_delivery_state_set_level_src(&new_dstate, false); } else { @@ -2119,7 +2464,8 @@ vgic_deactivate(vic_t *vic, thread_t *vcpu, virq_t virq, if (local_listed) { // Nobody else should delist the IRQ from under us. - assert(vgic_delivery_state_get_listed(&old_dstate)); + assert(vgic_delivery_state_get_listed(&old_dstate) == + true); vgic_delivery_state_set_listed(&new_dstate, false); vgic_delivery_state_set_need_sync(&new_dstate, false); vgic_delivery_state_set_hw_detached(&new_dstate, false); @@ -2132,11 +2478,21 @@ vgic_deactivate(vic_t *vic, thread_t *vcpu, virq_t virq, // already. It must have been deactivated some // other way, e.g. by a previous ICACTIVE write, // so we have nothing to do here. + VGIC_TRACE( + DSTATE_CHANGED, vic, vcpu, + "deactivate {:d}: already listed {:#x}", + virq, + vgic_delivery_state_raw(old_dstate)); goto out; } if (!vgic_delivery_state_get_active(&old_dstate)) { // Interrupt is already inactive; we have // nothing to do. + VGIC_TRACE( + DSTATE_CHANGED, vic, vcpu, + "deactivate {:d}: already inactive {:#x}", + virq, + vgic_delivery_state_raw(old_dstate)); goto out; } assert(!set_edge && !hw_active); @@ -2155,12 +2511,9 @@ vgic_deactivate(vic_t *vic, thread_t *vcpu, virq_t virq, // If level_src is set, check that the source is still pending // before we try to deliver it. if (vgic_delivery_state_get_level_src(&old_dstate)) { - // Order the check_pending event after the dstate read. - atomic_thread_fence(memory_order_acquire); - if (compiler_unexpected(source == NULL) || - !trigger_virq_check_pending_event( - source->trigger, source, - vgic_delivery_state_get_edge(&new_dstate))) { + if (!vgic_virq_check_pending( + source, vgic_delivery_state_get_edge( + &new_dstate))) { vgic_delivery_state_set_level_src(&new_dstate, false); } @@ -2216,6 +2569,9 @@ vgic_deactivate_unlisted(vic_t *vic, thread_t *vcpu, virq_t virq) if (vgic_delivery_state_get_listed(&old_dstate)) { // Somebody else must have deactivated it already, so ignore the // deactivate. + VGIC_TRACE(DSTATE_CHANGED, vic, vcpu, + "deactivate {:d}: already re-listed ({:#x})", virq, + vgic_delivery_state_raw(old_dstate)); } else { vgic_deactivate(vic, vcpu, virq, dstate, old_dstate, false, false); @@ -2362,7 +2718,7 @@ vgic_find_pending_and_list(vic_t *vic, thread_t *vcpu, uint8_t priority_mask, REQUIRE_PREEMPT_DISABLED { bool listed = false; - index_t prio_mask_index = priority_mask >> VGIC_PRIO_SHIFT; + index_t prio_mask_index = (index_t)priority_mask >> VGIC_PRIO_SHIFT; _Atomic BITMAP_DECLARE_PTR(VGIC_PRIORITIES, prios) = &vcpu->vgic_search_prios; @@ -2374,6 +2730,9 @@ vgic_find_pending_and_list(vic_t *vic, thread_t *vcpu, uint8_t priority_mask, bool reset_prio = false; uint8_t priority = (uint8_t)(prio_index << VGIC_PRIO_SHIFT); +#if !GICV3_HAS_VLPI && VGIC_HAS_LPI +#error lpi search ranges not implemented +#endif _Atomic BITMAP_DECLARE_PTR(VGIC_LOW_RANGES, ranges) = &vcpu->vgic_search_ranges_low[prio_index]; BITMAP_ATOMIC_FOREACH_SET_BEGIN(range, *ranges, VGIC_LOW_RANGES) @@ -2383,7 +2742,8 @@ vgic_find_pending_and_list(vic_t *vic, thread_t *vcpu, uint8_t priority_mask, } bool reset_range = false; - for (index_t i = 0; i < VGIC_LOW_RANGE_SIZE; i++) { + for (index_t i = 0; i < vgic_low_range_size(range); + i++) { virq_t virq = (virq_t)((range * VGIC_LOW_RANGE_SIZE) + i); @@ -2396,6 +2756,8 @@ vgic_find_pending_and_list(vic_t *vic, thread_t *vcpu, uint8_t priority_mask, } else if (err == ERROR_DENIED) { reset_range = true; reset_prio = true; + } else { + // Unable to list } } @@ -2444,9 +2806,10 @@ vgic_handle_irq_received_maintenance(void) assert_preempt_disabled(); thread_t *vcpu = thread_get_self(); - vic_t *vic = vcpu->vgic_vic; + vic_t *vic = vcpu->vgic_vic; - if ((vcpu->kind != THREAD_KIND_VCPU) || (vic == NULL)) { + if (compiler_unexpected((vcpu->kind != THREAD_KIND_VCPU) || + (vic == NULL))) { // Spurious IRQ; this can happen if a maintenance interrupt // is asserted shortly before a context switch, and the GICR // hasn't yet that it is no longer asserted by the time we @@ -2487,18 +2850,29 @@ vgic_handle_irq_received_maintenance(void) register_ICH_HCR_EL2_write(vcpu->vgic_ich_hcr); } - // Check for enable bit changes. This will clear out all of the LRs - // and redo any deliveries, so we can skip the none-pending handling. - if (ICH_MISR_EL2_get_VGrp0D(&misr) || ICH_MISR_EL2_get_VGrp1D(&misr) || - ICH_MISR_EL2_get_VGrp0E(&misr) || ICH_MISR_EL2_get_VGrp1E(&misr)) { - GICD_CTLR_DS_t gicd_ctlr = atomic_load_acquire(&vic->gicd_ctlr); - cpu_index_t lr_owner = vgic_lr_owner_lock_nopreempt(vcpu); - assert(lr_owner == CPU_INDEX_INVALID); - if (vgic_gicr_update_group_enables(vic, vcpu, gicd_ctlr)) { - vcpu_wakeup_self(); + if (!vgic_fgt_allowed()) { + // Check for enable bit changes. This will clear out all of the + // LRs and redo any deliveries, so we can skip the none-pending + // handling. + if (ICH_MISR_EL2_get_VGrp0D(&misr) || + ICH_MISR_EL2_get_VGrp1D(&misr) || + ICH_MISR_EL2_get_VGrp0E(&misr) || + ICH_MISR_EL2_get_VGrp1E(&misr)) { + GICD_CTLR_DS_t gicd_ctlr = + atomic_load_acquire(&vic->gicd_ctlr); + cpu_index_t lr_owner = + vgic_lr_owner_lock_nopreempt(vcpu); + assert(lr_owner == CPU_INDEX_INVALID); + VGIC_TRACE(ASYNC_EVENT, vic, vcpu, + "group enable maintenance: {:#x}", + ICH_MISR_EL2_raw(misr)); + if (vgic_gicr_update_group_enables(vic, vcpu, + gicd_ctlr)) { + vcpu_wakeup_self(); + } + vgic_lr_owner_unlock_nopreempt(vcpu); + goto out; } - vgic_lr_owner_unlock_nopreempt(vcpu); - goto out; } // Always try to deliver more interrupts if the NP interrupt is enabled, @@ -2530,6 +2904,8 @@ vgic_handle_irq_received_maintenance(void) index_t lr = compiler_ctz(elrsr); elrsr &= ~util_bit(lr); + assert_debug(lr < CPU_GICH_LR_COUNT); + if (vgic_find_pending_and_list( vic, vcpu, GIC_PRIORITY_LOWEST, lr)) { vcpu_wakeup_self(); @@ -2562,7 +2938,7 @@ vgic_sync_one(vic_t *vic, thread_t *vcpu, index_t lr) REQUIRE_LOCK(vcpu->vgic_lr_owner_lock) REQUIRE_PREEMPT_DISABLED EXCLUDE_SCHEDULER_LOCK(vcpu) { - assert(lr < CPU_GICH_LR_COUNT); + assert_debug(lr < CPU_GICH_LR_COUNT); vgic_lr_status_t *status = &vcpu->vgic_lrs[lr]; assert(status->dstate != NULL); bool need_update = false; @@ -2692,10 +3068,14 @@ vgic_do_delivery_check(vic_t *vic, thread_t *vcpu) } } - if ((lowest_prio > GIC_PRIORITY_HIGHEST) && - vgic_find_pending_and_list(vic, vcpu, lowest_prio, - lowest_prio_lr)) { - wakeup = true; + if (lowest_prio > GIC_PRIORITY_HIGHEST) { + if (vgic_find_pending_and_list(vic, vcpu, lowest_prio, + lowest_prio_lr)) { + wakeup = true; + } else { + break; + } + } else { break; } @@ -2703,7 +3083,7 @@ vgic_do_delivery_check(vic_t *vic, thread_t *vcpu) // We can't deliver IRQs that are equal or lower (numerically // greater) priority than the lowest-priority pending LR, so // exclude them from the next vgic_search_prios check. - prio_index_cutoff = lowest_prio >> VGIC_PRIO_SHIFT; + prio_index_cutoff = (index_t)lowest_prio >> VGIC_PRIO_SHIFT; } ICH_HCR_EL2_set_NPIE(&vcpu->vgic_ich_hcr, @@ -2715,7 +3095,7 @@ vgic_do_delivery_check(vic_t *vic, thread_t *vcpu) } static bool -vgic_retry_unrouted_virq(vic_t *vic, virq_t virq) +vgic_retry_unrouted_virq(vic_t *vic, virq_t virq) REQUIRE_PREEMPT_DISABLED { assert(vic != NULL); // Only SPIs can be unrouted @@ -2757,7 +3137,7 @@ vgic_retry_unrouted(vic_t *vic) range); bool unclaimed = false; - for (index_t i = 0; i < VGIC_LOW_RANGE_SIZE; i++) { + for (index_t i = 0; i < vgic_low_range_size(range); i++) { virq_t virq = (virq_t)((range * VGIC_LOW_RANGE_SIZE) + i); if (vgic_retry_unrouted_virq(vic, virq)) { @@ -2804,7 +3184,8 @@ vgic_undeliver_all(vic_t *vic, thread_t *vcpu) BITMAP_ATOMIC_FOREACH_SET_BEGIN( range, vcpu->vgic_search_ranges_low[prio], VGIC_LOW_RANGES) - for (index_t i = 0; i < VGIC_LOW_RANGE_SIZE; i++) { + for (index_t i = 0; i < vgic_low_range_size(range); + i++) { virq_t virq = (virq_t)((range * VGIC_LOW_RANGE_SIZE) + i); @@ -2861,7 +3242,8 @@ vgic_reroute_all(vic_t *vic, thread_t *vcpu) BITMAP_ATOMIC_FOREACH_SET_BEGIN( range, vcpu->vgic_search_ranges_low[prio_index], VGIC_LOW_RANGES) - for (index_t i = 0; i < VGIC_LOW_RANGE_SIZE; i++) { + for (index_t i = 0; i < vgic_low_range_size(range); + i++) { virq_t virq = (virq_t)((range * VGIC_LOW_RANGE_SIZE) + i); @@ -2931,7 +3313,7 @@ vgic_reroute_all(vic_t *vic, thread_t *vcpu) // before acquiring the LR lock; any subsequent change to the GICD_CTLR by // another CPU must trigger another call to this function, typically by sending // an IPI. -bool +static bool vgic_gicr_update_group_enables(vic_t *vic, thread_t *gicr_vcpu, GICD_CTLR_DS_t gicd_ctlr) { @@ -2974,15 +3356,26 @@ vgic_gicr_update_group_enables(vic_t *vic, thread_t *gicr_vcpu, gicr_vcpu->vgic_ich_hcr = register_ICH_HCR_EL2_read(); } +#if VGIC_HAS_LPI && GICV3_HAS_VLPI_V4_1 + // The vSGIEOICount flag is set for every VCPU based on the nASSGIreq + // flag in GICD_CTLR, which the VM can only update while the groups + // are disabled in GICD_CTLR. Updating it unconditionally here is + // probably faster than checking whether we need to update it. + ICH_HCR_EL2_set_vSGIEOICount(&gicr_vcpu->vgic_ich_hcr, + vic->vsgis_enabled); +#endif + // Update the group enable / disable traps. This isn't needed if we have // ARMv8.6-FGT, because we can unconditionally trap all ICC_IGRPENn_EL1 // writes in that case. -#if !defined(ARCH_ARM_8_6_FGT) || !ARCH_ARM_8_6_FGT - ICH_HCR_EL2_set_TALL0(&gicr_vcpu->vgic_ich_hcr, !group0_enable); - ICH_HCR_EL2_set_TALL1(&gicr_vcpu->vgic_ich_hcr, !group1_enable); - ICH_HCR_EL2_set_VGrp0DIE(&gicr_vcpu->vgic_ich_hcr, group0_enable); - ICH_HCR_EL2_set_VGrp1DIE(&gicr_vcpu->vgic_ich_hcr, group1_enable); -#endif + if (!vgic_fgt_allowed()) { + ICH_HCR_EL2_set_TALL0(&gicr_vcpu->vgic_ich_hcr, !group0_enable); + ICH_HCR_EL2_set_TALL1(&gicr_vcpu->vgic_ich_hcr, !group1_enable); + ICH_HCR_EL2_set_VGrp0DIE(&gicr_vcpu->vgic_ich_hcr, + group0_enable); + ICH_HCR_EL2_set_VGrp1DIE(&gicr_vcpu->vgic_ich_hcr, + group1_enable); + } // Now search for and list all deliverable VIRQs. if (group0_enable || group1_enable) { @@ -3038,14 +3431,18 @@ vgic_handle_thread_context_switch_post(thread_t *prev) cpu_index_t lr_owner = vgic_lr_owner_lock(prev); assert(lr_owner == cpulocal_get_index()); - if (ipi_clear(IPI_REASON_VGIC_SYNC) && - vgic_sync_vcpu(prev, false)) { - wakeup_prev = true; + if (ipi_clear(IPI_REASON_VGIC_SYNC)) { + if (vgic_sync_vcpu(prev, false)) { + wakeup_prev = true; + } } - if (ipi_clear(IPI_REASON_VGIC_ENABLE) && - vgic_gicr_update_group_enables( - vic, prev, atomic_load_acquire(&vic->gicd_ctlr))) { - wakeup_prev = true; + + if (ipi_clear(IPI_REASON_VGIC_ENABLE)) { + if (vgic_gicr_update_group_enables( + vic, prev, + atomic_load_acquire(&vic->gicd_ctlr))) { + wakeup_prev = true; + } } atomic_store_relaxed(&prev->vgic_lr_owner_lock.owner, CPU_INDEX_INVALID); @@ -3126,24 +3523,36 @@ vgic_gicr_rd_check_sleep(thread_t *gicr_vcpu) bool is_asleep; if (atomic_load_relaxed(&gicr_vcpu->vgic_sleep)) { -#if !defined(ARCH_ARM_8_6_FGT) || !ARCH_ARM_8_6_FGT - cpu_index_t lr_owner = vgic_lr_owner_lock(gicr_vcpu); - // We might not have received the maintenance interrupt yet - // after the VM cleared the group enable bits. Synchronise the - // group enables before checking them. - if (lr_owner == CPU_INDEX_INVALID) { - (void)vgic_gicr_update_group_enables( - gicr_vcpu->vgic_vic, gicr_vcpu, - atomic_load_acquire( - &gicr_vcpu->vgic_vic->gicd_ctlr)); + if (!vgic_fgt_allowed()) { + cpu_index_t lr_owner = vgic_lr_owner_lock(gicr_vcpu); + // We might not have received the maintenance interrupt + // yet after the VM cleared the group enable bits. + // Synchronise the group enables before checking them. + if (lr_owner == CPU_INDEX_INVALID) { + (void)vgic_gicr_update_group_enables( + gicr_vcpu->vgic_vic, gicr_vcpu, + atomic_load_acquire( + &gicr_vcpu->vgic_vic + ->gicd_ctlr)); + } + vgic_lr_owner_unlock(gicr_vcpu); } - vgic_lr_owner_unlock(gicr_vcpu); -#endif // We can only sleep if the groups are disabled. is_asleep = !gicr_vcpu->vgic_group0_enabled && !gicr_vcpu->vgic_group1_enabled; } else { is_asleep = false; +#if VGIC_HAS_LPI && GICV3_HAS_VLPI_V4_1 + if (gicv3_vpe_check_wakeup(false)) { + // The GICR hasn't finished scheduling the vPE yet. + // Returning true here means that the GICR_WAKER poll + // on VCPU resume will effectively prevent the VCPU + // entering its idle loop (and maybe suspending again) + // until the GICR has had an opportunity to forward any + // pending SGIs and LPIs. + is_asleep = true; + } +#endif } return is_asleep; @@ -3183,7 +3592,7 @@ vgic_handle_vcpu_pending_wakeup(void) } void -vgic_handle_vcpu_poweredoff(void) +vgic_handle_vcpu_stopped(void) { thread_t *vcpu = thread_get_self(); @@ -3214,7 +3623,7 @@ vgic_handle_vcpu_trap_wfi(void) assert(vcpu != NULL); if (vcpu->vgic_vic != NULL) { - vgic_lr_owner_lock(vcpu); + (void)vgic_lr_owner_lock(vcpu); #if VGIC_HAS_1N // Eagerly release invalid LRs. This increases the likelihood @@ -3226,6 +3635,8 @@ vgic_handle_vcpu_trap_wfi(void) index_t lr = compiler_ctz(elrsr); elrsr &= ~util_bit(lr); + assert_debug(lr < CPU_GICH_LR_COUNT); + vgic_lr_status_t *status = &vcpu->vgic_lrs[lr]; if (status->dstate != NULL) { vgic_reclaim_lr(vcpu->vgic_vic, vcpu, lr, @@ -3257,7 +3668,7 @@ vgic_handle_ipi_received_enable(void) { thread_t *current = thread_get_self(); assert(current->vgic_vic != NULL); - vgic_lr_owner_lock_nopreempt(current); + (void)vgic_lr_owner_lock_nopreempt(current); bool wakeup = vgic_gicr_update_group_enables( current->vgic_vic, current, atomic_load_acquire(¤t->vgic_vic->gicd_ctlr)); @@ -3269,7 +3680,7 @@ bool vgic_handle_ipi_received_sync(void) { thread_t *current = thread_get_self(); - vgic_lr_owner_lock_nopreempt(current); + (void)vgic_lr_owner_lock_nopreempt(current); bool wakeup = vgic_sync_vcpu(current, true); vgic_lr_owner_unlock_nopreempt(current); return wakeup; @@ -3283,7 +3694,7 @@ vgic_handle_ipi_received_deliver(void) vic_t *vic = current->vgic_vic; if (vic != NULL) { - vgic_lr_owner_lock_nopreempt(current); + (void)vgic_lr_owner_lock_nopreempt(current); current->vgic_ich_hcr = register_ICH_HCR_EL2_read(); for (index_t i = 0; i < CPU_GICH_LR_COUNT; i++) { @@ -3335,12 +3746,14 @@ vgic_icc_set_group_enable(bool is_group_1, ICC_IGRPEN_EL1_t igrpen) assert(remote_cpu == CPU_INDEX_INVALID); current->vgic_ich_vmcr = register_ICH_VMCR_EL2_read(); + bool enabled = ICC_IGRPEN_EL1_get_Enable(&igrpen); + VGIC_TRACE(ICC_WRITE, vic, current, "group {:d} {:s}", + (register_t)is_group_1, + (uintptr_t)(enabled ? "enabled" : "disabled")); if (is_group_1) { - ICH_VMCR_EL2_set_VENG1(¤t->vgic_ich_vmcr, - ICC_IGRPEN_EL1_get_Enable(&igrpen)); + ICH_VMCR_EL2_set_VENG1(¤t->vgic_ich_vmcr, enabled); } else { - ICH_VMCR_EL2_set_VENG0(¤t->vgic_ich_vmcr, - ICC_IGRPEN_EL1_get_Enable(&igrpen)); + ICH_VMCR_EL2_set_VENG0(¤t->vgic_ich_vmcr, enabled); } register_ICH_VMCR_EL2_write_ordered(current->vgic_ich_vmcr, &asm_ordering); @@ -3356,7 +3769,7 @@ vgic_icc_set_group_enable(bool is_group_1, ICC_IGRPEN_EL1_t igrpen) void vgic_icc_irq_deactivate(vic_t *vic, irq_t irq_num) { - thread_t *vcpu = thread_get_self(); + thread_t *vcpu = thread_get_self(); _Atomic vgic_delivery_state_t *dstate = vgic_find_dstate(vic, vcpu, irq_num); assert(dstate != NULL); @@ -3427,6 +3840,7 @@ vgic_icc_irq_deactivate(vic_t *vic, irq_t irq_num) // work for this. However, few VMs will use EOImode=1 so we don't care // very much just yet. For now, warn and do nothing. // + // FIXME: #if !defined(NDEBUG) static _Thread_local bool warned_about_ignored_dir = false; if (!warned_about_ignored_dir) { @@ -3443,94 +3857,115 @@ vgic_icc_irq_deactivate(vic_t *vic, irq_t irq_num) preempt_enable(); } -void -vgic_icc_generate_sgi(vic_t *vic, ICC_SGIR_EL1_t sgir, bool is_group_1) +static void +vgic_send_sgi(vic_t *vic, thread_t *vcpu, virq_t virq, bool is_group_1) + REQUIRE_RCU_READ { - register_t target_list = ICC_SGIR_EL1_get_TargetList(&sgir); - index_t target_offset = 16U * ICC_SGIR_EL1_get_RS(&sgir); - virq_t virq = ICC_SGIR_EL1_get_INTID(&sgir); - - assert(virq < GIC_SGI_NUM); + _Atomic vgic_delivery_state_t *dstate = + &vcpu->vgic_private_states[virq]; + vgic_delivery_state_t old_dstate = atomic_load_relaxed(dstate); - if (compiler_unexpected(ICC_SGIR_EL1_get_IRM(&sgir))) { - TRACE_AND_LOG(DEBUG, WARN, - "vcpu {:#x}: SGIR write with IRM set ignored", - (uintptr_t)(thread_t *)thread_get_self()); + if (!is_group_1 && vgic_delivery_state_get_group1(&old_dstate)) { + // SGI0R & ASGI1R do not generate group 1 SGIs goto out; } - psci_mpidr_t route_id = psci_mpidr_default(); - psci_mpidr_set_Aff1(&route_id, ICC_SGIR_EL1_get_Aff1(&sgir)); - psci_mpidr_set_Aff2(&route_id, ICC_SGIR_EL1_get_Aff2(&sgir)); - psci_mpidr_set_Aff3(&route_id, ICC_SGIR_EL1_get_Aff3(&sgir)); - - rcu_read_start(); - - while (target_list != 0U) { - index_t target_bit = compiler_ctz(target_list); - target_list &= ~util_bit(target_bit); +#if GICV3_HAS_VLPI_V4_1 && VGIC_HAS_LPI + // Raise SGI using direct injection through the ITS if possible. + // + // We can only use direct injection if: + // - The SGI is not listed in an LR (which has unpredictable behaviour + // when combined with direct injection of the same SGI) + // - The VM has permitted vSGI delivery with no active state, by setting + // GICD_CTLR.nASSGIreq (cached in vic->vsgis_enabled) + // - The VCPU has enabled vLPIs, and the ITS commands to sync the SGI + // configuration into the LPI tables have completed + if (!vgic_delivery_state_get_listed(&old_dstate) && + vic->vsgis_enabled && (vgic_vsgi_assert(vcpu, virq) == OK)) { + goto out; + } +#endif - index_t target = target_bit + target_offset; - psci_mpidr_set_Aff0(&route_id, (uint8_t)target); + if ((vcpu != thread_get_self()) && + vgic_delivery_state_get_enabled(&old_dstate)) { + VGIC_TRACE(SGI, vic, vcpu, "sgi fast: {:d}", virq); - cpu_index_result_t cpu_r = - platform_cpu_mpidr_to_index(route_id); - if ((cpu_r.e != OK) || (cpu_r.r >= vic->gicr_count)) { - // ignore invalid target - continue; - } + // Mark the SGI as pending delivery, and wake + // the target VCPU for delivery. + bitmap_atomic_set(vcpu->vgic_pending_sgis, virq, + memory_order_relaxed); - thread_t *vcpu = atomic_load_consume(&vic->gicr_vcpus[cpu_r.r]); - if (vcpu == NULL) { - // ignore missing target - continue; - } + // Match the seq_cst fences when the owner is changed + // during the context switch. + atomic_thread_fence(memory_order_seq_cst); - _Atomic vgic_delivery_state_t *dstate = - &vcpu->vgic_private_states[virq]; - vgic_delivery_state_t old_dstate = atomic_load_relaxed(dstate); + cpu_index_t lr_owner = + atomic_load_relaxed(&vcpu->vgic_lr_owner_lock.owner); - if (!is_group_1 && - vgic_delivery_state_get_group1(&old_dstate)) { - // SGI0R & ASGI1R do not generate group 1 SGIs - continue; + if (cpulocal_index_valid(lr_owner)) { + ipi_one(IPI_REASON_VGIC_SGI, lr_owner); + } else { + scheduler_lock(vcpu); + vcpu_wakeup(vcpu); + scheduler_unlock(vcpu); } + } else { + // Deliver the interrupt to the target + vgic_delivery_state_t assert_dstate = + vgic_delivery_state_default(); + vgic_delivery_state_set_edge(&assert_dstate, true); - if ((vcpu != thread_get_self()) && - vgic_delivery_state_get_enabled(&old_dstate)) { - VGIC_TRACE(SGI, vic, vcpu, "sgi fast: {:d}", virq); + (void)vgic_deliver(virq, vic, vcpu, NULL, dstate, assert_dstate, + true); + } - // Mark the SGI as pending delivery, and wake - // the target VCPU for delivery. - bitmap_atomic_set(vcpu->vgic_pending_sgis, virq, - memory_order_relaxed); +out: + return; +} - // Match the seq_cst fences when the owner is changed - // during the context switch. - atomic_thread_fence(memory_order_seq_cst); +void +vgic_icc_generate_sgi(vic_t *vic, ICC_SGIR_EL1_t sgir, bool is_group_1) +{ + register_t target_list = ICC_SGIR_EL1_get_TargetList(&sgir); + index_t target_offset = 16U * (index_t)ICC_SGIR_EL1_get_RS(&sgir); + virq_t virq = ICC_SGIR_EL1_get_INTID(&sgir); - cpu_index_t lr_owner = atomic_load_relaxed( - &vcpu->vgic_lr_owner_lock.owner); + assert(virq < GIC_SGI_NUM); - if (cpulocal_index_valid(lr_owner)) { - ipi_one(IPI_REASON_VGIC_SGI, lr_owner); - } else { - scheduler_lock(vcpu); - vcpu_wakeup(vcpu); - scheduler_unlock(vcpu); + if (compiler_unexpected(ICC_SGIR_EL1_get_IRM(&sgir))) { + thread_t *current = thread_get_self(); + for (index_t i = 0U; i < vic->gicr_count; i++) { + rcu_read_start(); + thread_t *vcpu = + atomic_load_consume(&vic->gicr_vcpus[i]); + if ((vcpu != NULL) && (vcpu != current)) { + vgic_send_sgi(vic, vcpu, virq, is_group_1); } - } else { - // Deliver the interrupt to the target - vgic_delivery_state_t assert_dstate = - vgic_delivery_state_default(); - vgic_delivery_state_set_edge(&assert_dstate, true); + rcu_read_finish(); + } + } else { + while (target_list != 0U) { + index_t target_bit = compiler_ctz(target_list); + target_list &= ~util_bit(target_bit); + + index_result_t cpu_r = vgic_get_index_for_mpidr( + vic, (uint8_t)(target_bit + target_offset), + ICC_SGIR_EL1_get_Aff1(&sgir), + ICC_SGIR_EL1_get_Aff2(&sgir), + ICC_SGIR_EL1_get_Aff3(&sgir)); + if (cpu_r.e != OK) { + // ignore invalid target + continue; + } + assert(cpu_r.r < vic->gicr_count); - (void)vgic_deliver(virq, vic, vcpu, NULL, dstate, - assert_dstate, true); + rcu_read_start(); + thread_t *vcpu = + atomic_load_consume(&vic->gicr_vcpus[cpu_r.r]); + if (vcpu != NULL) { + vgic_send_sgi(vic, vcpu, virq, is_group_1); + } + rcu_read_finish(); } } - - rcu_read_finish(); -out: - return; } diff --git a/hyp/vm/vgic/src/distrib.c b/hyp/vm/vgic/src/distrib.c index 53e805c..681fa56 100644 --- a/hyp/vm/vgic/src/distrib.c +++ b/hyp/vm/vgic/src/distrib.c @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -16,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -32,12 +34,17 @@ #include #include #include +#include #include #include #include #include +#if defined(ARCH_ARM_FEAT_FGT) && ARCH_ARM_FEAT_FGT +#include +#endif + #include "event_handlers.h" #include "gicv3.h" #include "internal.h" @@ -48,9 +55,9 @@ error_t vgic_handle_object_create_vic(vic_create_t vic_create) { - struct vic *vic = vic_create.vic; + vic_t *vic = vic_create.vic; assert(vic != NULL); - struct partition *partition = vic->header.partition; + partition_t *partition = vic->header.partition; assert(partition != NULL); vic->gicr_count = 1U; @@ -67,19 +74,26 @@ vgic_handle_object_create_vic(vic_create_t vic_create) GICD_CTLR_DS_set_ARE(&ctlr, true); #if VGIC_HAS_1N // We currently don't implement E1NWF=0. + // FIXME: GICD_CTLR_DS_set_E1NWF(&ctlr, true); #endif atomic_init(&vic->gicd_ctlr, ctlr); + // If not configured otherwise, default to using the same MPIDR mapping + // as the hardware + vic->mpidr_mapping = platform_cpu_get_mpidr_mapping(); + return OK; } error_t vic_configure(vic_t *vic, count_t max_vcpus, count_t max_virqs, - count_t max_msis) + count_t max_msis, bool allow_fixed_vmaddr) { error_t err = OK; + vic->allow_fixed_vmaddr = allow_fixed_vmaddr; + if ((max_vcpus == 0U) || (max_vcpus > PLATFORM_MAX_CORES)) { err = ERROR_ARGUMENT_INVALID; goto out; @@ -92,10 +106,18 @@ vic_configure(vic_t *vic, count_t max_vcpus, count_t max_virqs, } vic->sources_count = max_virqs; +#if VGIC_HAS_LPI + if ((max_msis + GIC_LPI_BASE) >= util_bit(VGIC_IDBITS)) { + err = ERROR_ARGUMENT_INVALID; + goto out; + } + vic->gicd_idbits = compiler_msb(max_msis + GIC_LPI_BASE - 1U) + 1U; +#else if (max_msis != 0U) { err = ERROR_ARGUMENT_INVALID; goto out; } +#endif out: return err; @@ -104,14 +126,18 @@ vic_configure(vic_t *vic, count_t max_vcpus, count_t max_virqs, bool vgic_has_lpis(vic_t *vic) { +#if VGIC_HAS_LPI + return vic->gicd_idbits >= 14U; +#else (void)vic; return false; +#endif } error_t vgic_handle_object_activate_vic(vic_t *vic) { - struct partition *partition = vic->header.partition; + partition_t *partition = vic->header.partition; assert(partition != NULL); error_t err = OK; void_ptr_result_t alloc_r; @@ -123,6 +149,27 @@ vgic_handle_object_activate_vic(vic_t *vic) assert(vic->gicr_count <= PLATFORM_MAX_CORES); size_t vcpus_size = sizeof(vic->gicr_vcpus[0U]) * vic->gicr_count; +#if VGIC_HAS_LPI + if (vgic_has_lpis(vic)) { + size_t vlpi_propbase_size = + util_bit(vic->gicd_idbits) - GIC_LPI_BASE; + size_t vlpi_propbase_align = + util_bit(GIC_ITS_CMD_VMAPP_VCONF_ADDR_PRESHIFT); + alloc_r = partition_alloc(vic->header.partition, + vlpi_propbase_size, + vlpi_propbase_align); + if (alloc_r.e != OK) { + err = alloc_r.e; + goto out; + } + // No need for a memset here; the first time a VM enables LPIs + // we will memcpy the table from VM memory (and zero the rest + // of the table if necessary) before sending a VMAPP command. + // The vlpi_config_valid flag indicates that this has been done + vic->vlpi_config_table = alloc_r.r; + } +#endif + if (sources_size != 0U) { alloc_r = partition_alloc(partition, sources_size, alignof(vic->sources[0U])); @@ -130,7 +177,7 @@ vgic_handle_object_activate_vic(vic_t *vic) err = alloc_r.e; goto out; } - memset(alloc_r.r, 0, sources_size); + (void)memset_s(alloc_r.r, sources_size, 0, sources_size); vic->sources = (virq_source_t *_Atomic *)alloc_r.r; } @@ -140,7 +187,8 @@ vgic_handle_object_activate_vic(vic_t *vic) err = alloc_r.e; goto out; } - memset(alloc_r.r, 0, vcpus_size); + (void)memset_s(alloc_r.r, vcpus_size, 0, vcpus_size); + vic->gicr_vcpus = (thread_t *_Atomic *)alloc_r.r; out: @@ -149,6 +197,76 @@ vgic_handle_object_activate_vic(vic_t *vic) return err; } +error_t +vgic_handle_addrspace_attach_vdevice(addrspace_t *addrspace, + cap_id_t vdevice_object_cap, index_t index, + vmaddr_t vbase, size_t size) +{ + error_t err; + cspace_t *cspace = cspace_get_self(); + + vic_ptr_result_t vic_r = cspace_lookup_vic( + cspace, vdevice_object_cap, CAP_RIGHTS_VIC_ATTACH_VDEVICE); + if (compiler_unexpected(vic_r.e) != OK) { + err = vic_r.e; + goto out; + } + + if (index > vic_r.r->gicr_count + 1U) { + err = ERROR_ARGUMENT_INVALID; + goto out_ref; + } + + spinlock_acquire(&vic_r.r->gicd_lock); + + if (index == 0U) { + // Attaching the GICD registers. + if (vic_r.r->gicd_device.type != VDEVICE_TYPE_NONE) { + err = ERROR_BUSY; + goto out_locked; + } + vic_r.r->gicd_device.type = VDEVICE_TYPE_VGIC_GICD; + + err = vdevice_attach_vmaddr(&vic_r.r->gicd_device, addrspace, + vbase, size); + if (err != OK) { + vic_r.r->gicd_device.type = VDEVICE_TYPE_NONE; + } + } else { + // Attaching GICR registers for a specific VCPU. + rcu_read_start(); + thread_t *gicr_vcpu = + atomic_load_consume(&vic_r.r->gicr_vcpus[index - 1U]); + + if (gicr_vcpu == NULL) { + err = ERROR_IDLE; + goto out_gicr_rcu; + } + + if (gicr_vcpu->vgic_gicr_device.type != VDEVICE_TYPE_NONE) { + err = ERROR_BUSY; + goto out_gicr_rcu; + } + + gicr_vcpu->vgic_gicr_device.type = VDEVICE_TYPE_VGIC_GICR; + err = vdevice_attach_vmaddr(&gicr_vcpu->vgic_gicr_device, + addrspace, vbase, size); + if (err != OK) { + gicr_vcpu->vgic_gicr_device.type = VDEVICE_TYPE_NONE; + } + + out_gicr_rcu: + rcu_read_finish(); + } + +out_locked: + spinlock_release(&vic_r.r->gicd_lock); +out_ref: + object_put_vic(vic_r.r); +out: + return err; +} + void vgic_handle_object_deactivate_vic(vic_t *vic) { @@ -169,26 +287,41 @@ vgic_handle_object_deactivate_vic(vic_t *vic) vic_unbind(virq_source); } rcu_read_finish(); + + if (vic->gicd_device.type != VDEVICE_TYPE_NONE) { + vdevice_detach_vmaddr(&vic->gicd_device); + } } void vgic_handle_object_cleanup_vic(vic_t *vic) { - struct partition *partition = vic->header.partition; + partition_t *partition = vic->header.partition; if (vic->gicr_vcpus != NULL) { size_t vcpus_size = sizeof(vic->gicr_vcpus[0]) * vic->gicr_count; - partition_free(partition, vic->gicr_vcpus, vcpus_size); + (void)partition_free(partition, vic->gicr_vcpus, vcpus_size); vic->gicr_vcpus = NULL; } if (vic->sources != NULL) { size_t sources_size = sizeof(vic->sources[0]) * vic->sources_count; - partition_free(partition, vic->sources, sources_size); + (void)partition_free(partition, vic->sources, sources_size); vic->sources = NULL; } + +#if VGIC_HAS_LPI + if (vic->vlpi_config_table != NULL) { + size_t vlpi_propbase_size = + util_bit(vic->gicd_idbits) - GIC_LPI_BASE; + (void)partition_free(vic->header.partition, + vic->vlpi_config_table, + vlpi_propbase_size); + vic->vlpi_config_table = NULL; + } +#endif } error_t @@ -233,6 +366,12 @@ vgic_handle_object_create_thread(thread_create_t thread_create) CPU_INDEX_INVALID); if (vcpu->kind == THREAD_KIND_VCPU) { +#if VGIC_HAS_LPI + GICR_CTLR_t ctlr = GICR_CTLR_default(); + GICR_CTLR_set_IR(&ctlr, true); + atomic_store_relaxed(&vcpu->vgic_gicr_rd_ctlr, ctlr); +#endif + // The sleep flag is initially clear. This has no real effect on // guests with GICR_WAKER awareness (like Linux), but allows // interrupt delivery to work correctly for guests that assume @@ -242,25 +381,29 @@ vgic_handle_object_create_thread(thread_create_t thread_create) vcpu->vgic_ich_hcr = ICH_HCR_EL2_default(); // Trap changes to the group enable bits. -#if defined(ARCH_ARM_8_6_FGT) && ARCH_ARM_8_6_FGT - // Use fine-grained traps of the enable registers if they are - // available, so we don't have to emulate the other registers - // trapped by TALL[01]. - HFGWTR_EL2_set_ICC_IGRPENn_EL1(&vcpu->vcpu_regs_el2.hfgwtr_el2, - true); -#else - // Trap all accesses for disabled groups. Note that these traps - // and the group disable maintenance IRQs are toggled every time - // we update the group enables. - // - // We can't use the group enable maintenance IRQs, because their - // latency is high enough that a VCPU's idle loop might enable - // the groups and then disable them again before we know they've - // been enabled, causing it to get stuck in a loop being woken - // by IRQs that are never delivered. - ICH_HCR_EL2_set_TALL0(&vcpu->vgic_ich_hcr, true); - ICH_HCR_EL2_set_TALL1(&vcpu->vgic_ich_hcr, true); +#if defined(ARCH_ARM_FEAT_FGT) && ARCH_ARM_FEAT_FGT + if (arm_fgt_is_allowed()) { + // Use fine-grained traps of the enable registers if + // they are available, so we don't have to emulate the + // other registers trapped by TALL[01]. + HFGWTR_EL2_set_ICC_IGRPENn_EL1( + &vcpu->vcpu_regs_el2.hfgwtr_el2, true); + } else #endif + { + // Trap all accesses for disabled groups. Note that + // these traps and the group disable maintenance IRQs + // are toggled every time we update the group enables. + // + // We can't use the group enable maintenance IRQs, + // because their latency is high enough that a VCPU's + // idle loop might enable the groups and then disable + // them again before we know they've been enabled, + // causing it to get stuck in a loop being woken by IRQs + // that are never delivered. + ICH_HCR_EL2_set_TALL0(&vcpu->vgic_ich_hcr, true); + ICH_HCR_EL2_set_TALL1(&vcpu->vgic_ich_hcr, true); + } // Always set LRENPIE, and keep UIE off. This is because we // don't reload active interrupts into the LRs once they've been @@ -271,6 +414,13 @@ vgic_handle_object_create_thread(thread_create_t thread_create) // this in the first place. ICH_HCR_EL2_set_UIE(&vcpu->vgic_ich_hcr, false); ICH_HCR_EL2_set_LRENPIE(&vcpu->vgic_ich_hcr, true); +#if VGIC_HAS_LPI && GICV3_HAS_VLPI_V4_1 + // We don't know whether to set vSGIEOICount until the VM + // enables groups in GICD_CTLR, at which point we must propagate + // the nASSGIreq bit from the same register to all the vCPUs. + // That is done in vgic_gicr_update_group_enables(). + ICH_HCR_EL2_set_vSGIEOICount(&vcpu->vgic_ich_hcr, false); +#endif // Always trap DIR, so we know which IRQs are being deactivated // when the VM uses EOImode=1. We can't rely on LRENPIE/EOIcount // in this case (as opposed to EOImode=0, when we can assume the @@ -285,22 +435,35 @@ vgic_handle_object_create_thread(thread_create_t thread_create) return OK; } -static void -vic_set_mpidr_by_index(thread_t *thread, cpu_index_t index) +index_result_t +vgic_get_index_for_mpidr(vic_t *vic, uint8_t aff0, uint8_t aff1, uint8_t aff2, + uint8_t aff3) { - psci_mpidr_t ret = platform_cpu_index_to_mpidr(index); - MPIDR_EL1_t real = register_MPIDR_EL1_read(); + platform_mpidr_mapping_t mapping = vic->mpidr_mapping; + index_result_t ret; + + if (compiler_unexpected(((~mapping.aff_mask[0] & aff0) != 0U) || + ((~mapping.aff_mask[1] & aff1) != 0U) || + ((~mapping.aff_mask[2] & aff2) != 0U) || + ((~mapping.aff_mask[3] & aff3) != 0U))) { + ret = index_result_error(ERROR_ARGUMENT_INVALID); + goto out; + } - thread->vcpu_regs_mpidr_el1 = MPIDR_EL1_default(); - MPIDR_EL1_set_Aff0(&thread->vcpu_regs_mpidr_el1, - psci_mpidr_get_Aff0(&ret)); - MPIDR_EL1_set_Aff1(&thread->vcpu_regs_mpidr_el1, - psci_mpidr_get_Aff1(&ret)); - MPIDR_EL1_set_Aff2(&thread->vcpu_regs_mpidr_el1, - psci_mpidr_get_Aff2(&ret)); - MPIDR_EL1_set_Aff3(&thread->vcpu_regs_mpidr_el1, - psci_mpidr_get_Aff3(&ret)); - MPIDR_EL1_set_MT(&thread->vcpu_regs_mpidr_el1, MPIDR_EL1_get_MT(&real)); + index_t index = 0U; + index |= ((index_t)aff0 << mapping.aff_shift[0]); + index |= ((index_t)aff1 << mapping.aff_shift[1]); + index |= ((index_t)aff2 << mapping.aff_shift[2]); + index |= ((index_t)aff3 << mapping.aff_shift[3]); + + if (compiler_unexpected(index >= vic->gicr_count)) { + ret = index_result_error(ERROR_ARGUMENT_INVALID); + goto out; + } + + ret = index_result_ok(index); +out: + return ret; } error_t @@ -310,52 +473,11 @@ vgic_handle_object_activate_thread(thread_t *vcpu) vic_t *vic = vcpu->vgic_vic; if (vic != NULL) { - if (vic->gicr_count > 1U) { - // When there is no vpm_group (psci) attached, we need - // to update the vcpu's MPIDR to the vgic - // configuration. - // The default MPIDR is flagged as uniprocessor when - // not initialized by vpm_group. - MPIDR_EL1_t mpidr_default = MPIDR_EL1_default(); - MPIDR_EL1_set_U(&mpidr_default, true); - - if (MPIDR_EL1_is_equal(mpidr_default, - vcpu->vcpu_regs_mpidr_el1)) { - vic_set_mpidr_by_index( - vcpu, - (cpu_index_t)vcpu->vgic_gicr_index); - } - } - spinlock_acquire(&vic->gicd_lock); - psci_mpidr_t route_id = psci_mpidr_default(); - psci_mpidr_set_Aff0( - &route_id, - MPIDR_EL1_get_Aff0(&vcpu->vcpu_regs_mpidr_el1)); - psci_mpidr_set_Aff1( - &route_id, - MPIDR_EL1_get_Aff1(&vcpu->vcpu_regs_mpidr_el1)); - psci_mpidr_set_Aff2( - &route_id, - MPIDR_EL1_get_Aff2(&vcpu->vcpu_regs_mpidr_el1)); - psci_mpidr_set_Aff3( - &route_id, - MPIDR_EL1_get_Aff3(&vcpu->vcpu_regs_mpidr_el1)); - - cpu_index_result_t cpu_r = - platform_cpu_mpidr_to_index(route_id); - if (cpu_r.e != OK) { - err = cpu_r.e; - goto out_locked; - } - if (cpu_r.r != vcpu->vgic_gicr_index) { - err = ERROR_OBJECT_CONFIG; - goto out_locked; - } - assert(cpu_r.r < vic->gicr_count); + index_t index = vcpu->vgic_gicr_index; - if (atomic_load_relaxed(&vic->gicr_vcpus[cpu_r.r]) != NULL) { + if (atomic_load_relaxed(&vic->gicr_vcpus[index]) != NULL) { err = ERROR_BUSY; goto out_locked; } @@ -369,16 +491,14 @@ vgic_handle_object_activate_thread(thread_t *vcpu) vgic_delivery_state_t sgi_dstate = vgic_delivery_state_default(); vgic_delivery_state_set_cfg_is_edge(&sgi_dstate, true); - vgic_delivery_state_set_route(&sgi_dstate, - vcpu->vgic_gicr_index); + vgic_delivery_state_set_route(&sgi_dstate, index); for (index_t i = 0; i < GIC_SGI_NUM; i++) { atomic_init(&vcpu->vgic_private_states[i], sgi_dstate); } // PPIs are normally level-triggered. vgic_delivery_state_t ppi_dstate = vgic_delivery_state_default(); - vgic_delivery_state_set_route(&ppi_dstate, - vcpu->vgic_gicr_index); + vgic_delivery_state_set_route(&ppi_dstate, index); for (index_t i = 0; i < GIC_PPI_NUM; i++) { atomic_init( &vcpu->vgic_private_states[GIC_PPI_BASE + i], @@ -388,27 +508,75 @@ vgic_handle_object_activate_thread(thread_t *vcpu) // Determine the physical interrupt route that should be used // for interrupts that target this VCPU. scheduler_lock_nopreempt(vcpu); - cpu_index_t affinity = scheduler_get_affinity(vcpu); - psci_mpidr_t mpidr = platform_cpu_index_to_mpidr( + cpu_index_t affinity = scheduler_get_affinity(vcpu); + MPIDR_EL1_t mpidr = platform_cpu_index_to_mpidr( cpulocal_index_valid(affinity) ? affinity : 0U); GICD_IROUTER_t phys_route = GICD_IROUTER_default(); GICD_IROUTER_set_IRM(&phys_route, false); - GICD_IROUTER_set_Aff0(&phys_route, psci_mpidr_get_Aff0(&mpidr)); - GICD_IROUTER_set_Aff1(&phys_route, psci_mpidr_get_Aff1(&mpidr)); - GICD_IROUTER_set_Aff2(&phys_route, psci_mpidr_get_Aff2(&mpidr)); - GICD_IROUTER_set_Aff3(&phys_route, psci_mpidr_get_Aff3(&mpidr)); + GICD_IROUTER_set_Aff0(&phys_route, MPIDR_EL1_get_Aff0(&mpidr)); + GICD_IROUTER_set_Aff1(&phys_route, MPIDR_EL1_get_Aff1(&mpidr)); + GICD_IROUTER_set_Aff2(&phys_route, MPIDR_EL1_get_Aff2(&mpidr)); + GICD_IROUTER_set_Aff3(&phys_route, MPIDR_EL1_get_Aff3(&mpidr)); vcpu->vgic_irouter = phys_route; +#if VGIC_HAS_LPI && GICV3_HAS_VLPI +#if GICV3_HAS_VLPI_V4_1 + // VSGI setup has not been done yet; set the sequence + // number to one that will never be complete. + atomic_init(&vcpu->vgic_vsgi_setup_seq, ~(count_t)0U); +#endif + + if (vgic_has_lpis(vic)) { + size_t vlpi_pendbase_size = + BITMAP_NUM_WORDS(util_bit(vic->gicd_idbits)) * + sizeof(register_t); + size_t vlpi_pendbase_align = + util_bit(GIC_ITS_CMD_VMAPP_VPT_ADDR_PRESHIFT); + void_ptr_result_t alloc_r = partition_alloc( + vcpu->header.partition, vlpi_pendbase_size, + vlpi_pendbase_align); + if (alloc_r.e != OK) { + err = alloc_r.e; + goto out_vcpu_locked; + } + + // Call the ITS driver to allocate a vPE ID and a + // doorbell LPI for this VCPU. We do this before we + // save the pending table pointer so the cleanup + // function can use the pointer to decide whether + // to call gicv3_its_vpe_cleanup(vcpu). + err = gicv3_its_vpe_activate(vcpu); + if (err != OK) { + (void)partition_free(vcpu->header.partition, + alloc_r.r, + vlpi_pendbase_size); + goto out_vcpu_locked; + } + + // No need to memset here; it will be done (with a + // possible partial memcpy from the VM) before we issue + // a VMAPP, when the VM writes 1 to EnableLPIs. + vcpu->vgic_vlpi_pending_table = alloc_r.r; + } +#endif + // Set the GICD's pointer to the VCPU. This is a store release // so we can be sure that all of the thread's initialisation is // complete before the VGIC tries to use it. - atomic_store_release(&vic->gicr_vcpus[cpu_r.r], vcpu); + atomic_store_release(&vic->gicr_vcpus[index], vcpu); +#if VGIC_HAS_LPI && GICV3_HAS_VLPI + out_vcpu_locked: +#endif scheduler_unlock_nopreempt(vcpu); out_locked: spinlock_release(&vic->gicd_lock); if (err == OK) { + vcpu->vcpu_regs_mpidr_el1 = + platform_cpu_map_index_to_mpidr( + &vic->mpidr_mapping, index); + // Check for IRQs that were routed to this CPU and // delivered before it was attached, to make sure they // are flagged locally. @@ -422,13 +590,13 @@ vgic_handle_object_activate_thread(thread_t *vcpu) void vgic_handle_scheduler_affinity_changed(thread_t *vcpu, cpu_index_t next_cpu) { - psci_mpidr_t mpidr = platform_cpu_index_to_mpidr(next_cpu); + MPIDR_EL1_t mpidr = platform_cpu_index_to_mpidr(next_cpu); GICD_IROUTER_t phys_route = GICD_IROUTER_default(); GICD_IROUTER_set_IRM(&phys_route, false); - GICD_IROUTER_set_Aff0(&phys_route, psci_mpidr_get_Aff0(&mpidr)); - GICD_IROUTER_set_Aff1(&phys_route, psci_mpidr_get_Aff1(&mpidr)); - GICD_IROUTER_set_Aff2(&phys_route, psci_mpidr_get_Aff2(&mpidr)); - GICD_IROUTER_set_Aff3(&phys_route, psci_mpidr_get_Aff3(&mpidr)); + GICD_IROUTER_set_Aff0(&phys_route, MPIDR_EL1_get_Aff0(&mpidr)); + GICD_IROUTER_set_Aff1(&phys_route, MPIDR_EL1_get_Aff1(&mpidr)); + GICD_IROUTER_set_Aff2(&phys_route, MPIDR_EL1_get_Aff2(&mpidr)); + GICD_IROUTER_set_Aff3(&phys_route, MPIDR_EL1_get_Aff3(&mpidr)); vcpu->vgic_irouter = phys_route; } @@ -465,10 +633,35 @@ vgic_handle_object_deactivate_thread(thread_t *thread) NULL); } +#if VGIC_HAS_LPI + if (vgic_has_lpis(vic) && + (thread->vgic_vlpi_pending_table != NULL)) { + // Ensure that any outstanding unmap has finished + GICR_CTLR_t old_ctlr = + atomic_load_relaxed(&thread->vgic_gicr_rd_ctlr); + if (GICR_CTLR_get_Enable_LPIs(&old_ctlr)) { + count_result_t count_r = + gicv3_its_vpe_unmap(thread); + assert(count_r.e == OK); + thread->vgic_vlpi_unmap_seq = count_r.r; + } + } +#endif + + if (thread->vgic_gicr_device.type != VDEVICE_TYPE_NONE) { + vdevice_detach_vmaddr(&thread->vgic_gicr_device); + } + spinlock_release(&vic->gicd_lock); } } +void +vgic_unwind_object_activate_thread(thread_t *thread) +{ + vgic_handle_object_deactivate_thread(thread); +} + void vgic_handle_object_cleanup_thread(thread_t *thread) { @@ -484,6 +677,34 @@ vgic_handle_object_cleanup_thread(thread_t *thread) // Clear out all LRs and re-route all pending IRQs vgic_undeliver_all(vic, thread); +#if VGIC_HAS_LPI && GICV3_HAS_VLPI + if (vgic_has_lpis(vic) && + (thread->vgic_vlpi_pending_table != NULL)) { + // Ensure that any outstanding unmap has finished + GICR_CTLR_t old_ctlr = + atomic_load_relaxed(&thread->vgic_gicr_rd_ctlr); + if (GICR_CTLR_get_Enable_LPIs(&old_ctlr)) { + (void)gicv3_its_wait( + 0U, thread->vgic_vlpi_unmap_seq); + } + + // Discard the pending table + size_t vlpi_pendbase_size = + BITMAP_NUM_WORDS(util_bit(vic->gicd_idbits)) * + sizeof(register_t); + (void)partition_free(thread->header.partition, + thread->vgic_vlpi_pending_table, + vlpi_pendbase_size); + thread->vgic_vlpi_pending_table = NULL; + + // Tell the ITS driver to release the allocated vPE ID + // and doorbell IRQ. + gicv3_its_vpe_cleanup(thread); + } else { + assert(thread->vgic_vlpi_pending_table == NULL); + } +#endif + #if VGIC_HAS_1N // Wake any other threads on the GIC, in case the deferred IRQs // can be rerouted. @@ -496,7 +717,8 @@ vgic_handle_object_cleanup_thread(thread_t *thread) void vgic_handle_rootvm_init(partition_t *root_partition, thread_t *root_thread, - cspace_t *root_cspace, boot_env_data_t *env_data) + cspace_t *root_cspace, hyp_env_data_t *hyp_env, + qcbor_enc_ctxt_t *qcbor_enc_ctxt) { // Create the VIC object for the root VM vic_create_t vic_params = { 0 }; @@ -506,20 +728,18 @@ vgic_handle_rootvm_init(partition_t *root_partition, thread_t *root_thread, goto vic_fail; } spinlock_acquire(&vic_r.r->header.lock); -#if defined(ROOTVM_IS_HLOS) && ROOTVM_IS_HLOS - count_t max_vcpus = PLATFORM_MAX_CORES; - count_t max_virqs = GIC_SPI_NUM; - count_t max_msis = 0U; -#else count_t max_vcpus = 1U; count_t max_virqs = 64U; - count_t max_msis = 0U; // FIXME: for testing -#endif - env_data->gicd_base = PLATFORM_GICD_BASE; - env_data->gicr_base = PLATFORM_GICR_BASE; - env_data->gicr_stride = (size_t)util_bit(GICR_STRIDE_SHIFT); + count_t max_msis = 0U; + + assert(qcbor_enc_ctxt != NULL); - if (vic_configure(vic_r.r, max_vcpus, max_virqs, max_msis) != OK) { + hyp_env->gicd_base = PLATFORM_GICD_BASE; + hyp_env->gicr_base = PLATFORM_GICR_BASE; + hyp_env->gicr_stride = (size_t)util_bit(GICR_STRIDE_SHIFT); + + if (vic_configure(vic_r.r, max_vcpus, max_virqs, max_msis, false) != + OK) { spinlock_release(&vic_r.r->header.lock); goto vic_fail; } @@ -536,44 +756,15 @@ vgic_handle_rootvm_init(partition_t *root_partition, thread_t *root_thread, if (cid_r.e != OK) { goto vic_fail; } - env_data->vic = cid_r.r; + hyp_env->vic = cid_r.r; + QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "vic", cid_r.r); -#if defined(ROOTVM_IS_HLOS) && ROOTVM_IS_HLOS - index_t vic_index = root_thread->scheduler_affinity; -#else index_t vic_index = 0U; -#endif if (vic_attach_vcpu(vic_r.r, root_thread, vic_index) != OK) { panic("VIC couldn't attach root VM thread"); } -#if defined(ROOTVM_IS_HLOS) && ROOTVM_IS_HLOS - // Attach all secondary root VM threads to the VIC - for (cpu_index_t i = 0; cpulocal_index_valid(i); i++) { - thread_t *thread; - if (i == root_thread->scheduler_affinity) { - continue; - } else { - cap_id_t thread_cap = env_data->psci_secondary_vcpus[i]; - object_type_t type; - object_ptr_result_t o = cspace_lookup_object_any( - root_cspace, thread_cap, - CAP_RIGHTS_GENERIC_OBJECT_ACTIVATE, &type); - if ((o.e != OK) || (type != OBJECT_TYPE_THREAD)) { - panic("VIC couldn't attach root VM thread"); - } - thread = o.r.thread; - } - - if (vic_attach_vcpu(vic_r.r, thread, i) != OK) { - panic("VIC couldn't attach root VM thread"); - } - - object_put_thread(thread); - } -#endif - // Create a HWIRQ object for every SPI index_t i; #if GICV3_EXT_IRQS @@ -581,7 +772,8 @@ vgic_handle_rootvm_init(partition_t *root_partition, thread_t *root_thread, #endif index_t last_spi = util_min((count_t)platform_irq_max(), GIC_SPI_BASE + GIC_SPI_NUM - 1U); - assert(last_spi < util_array_size(env_data->vic_hwirq)); + + QCBOREncode_OpenArrayInMap(qcbor_enc_ctxt, "vic_hwirq"); for (i = 0; i <= last_spi; i++) { hwirq_create_t hwirq_params = { .irq = i, @@ -593,8 +785,8 @@ vgic_handle_rootvm_init(partition_t *root_partition, thread_t *root_thread, hwirq_params.action = HWIRQ_ACTION_VIC_BASE_FORWARD_PRIVATE; } else { - // Don't try to register unhandled interrupt types - env_data->vic_hwirq[i] = CSPACE_CAP_INVALID; + QCBOREncode_AddUInt64(qcbor_enc_ctxt, + CSPACE_CAP_INVALID); continue; } @@ -609,7 +801,8 @@ vgic_handle_rootvm_init(partition_t *root_partition, thread_t *root_thread, if ((err == ERROR_DENIED) || (err == ERROR_ARGUMENT_INVALID) || (err == ERROR_BUSY)) { - env_data->vic_hwirq[i] = CSPACE_CAP_INVALID; + QCBOREncode_AddUInt64(qcbor_enc_ctxt, + CSPACE_CAP_INVALID); object_put_hwirq(hwirq_r.r); continue; } else { @@ -624,40 +817,12 @@ vgic_handle_rootvm_init(partition_t *root_partition, thread_t *root_thread, if (cid_r.e != OK) { panic("Unable to create cap to HWIRQ"); } - env_data->vic_hwirq[i] = cid_r.r; - -#if defined(ROOTVM_IS_HLOS) && ROOTVM_IS_HLOS - if (gicv3_get_irq_type(i) == GICV3_IRQ_TYPE_SPI) { - // Bind the HW IRQ to the HLOS VIC - error_t err = vgic_bind_hwirq_spi(vic_r.r, hwirq_r.r, - hwirq_params.irq); - if (err != OK) { - panic("Unable to bind HW SPI to HLOS VGIC"); - } - } else if (gicv3_get_irq_type(i) == GICV3_IRQ_TYPE_PPI) { - // Bind the HW IRQ to the HLOS VIC - error_t err = vgic_bind_hwirq_forward_private( - vic_r.r, hwirq_r.r, hwirq_params.irq); - if (err != OK) { - panic("Unable to bind HW PPI to HLOS VGIC"); - } - } -#endif - } - for (; i < util_array_size(env_data->vic_hwirq); i++) { - env_data->vic_hwirq[i] = CSPACE_CAP_INVALID; + QCBOREncode_AddUInt64(qcbor_enc_ctxt, cid_r.r); } + QCBOREncode_CloseArray(qcbor_enc_ctxt); - // Fill in the msi source array with invalid caps, and zero the ITS - // address range. The vgic_its module will write over these if necessary - // (note that this handler has elevated priority, so vgic_its will run - // later). They are part of this module's API to avoid an ABI dependency - // on the presence of the vgic_its module. - for (i = 0U; i < util_array_size(env_data->vic_msi_source); i++) { - env_data->vic_msi_source[i] = CSPACE_CAP_INVALID; - } - env_data->gits_base = 0U; - env_data->gits_stride = 0U; + hyp_env->gits_base = 0U; + hyp_env->gits_stride = 0U; return; @@ -665,6 +830,46 @@ vgic_handle_rootvm_init(partition_t *root_partition, thread_t *root_thread, panic("Unable to create root VM's virtual GIC"); } +void +vgic_handle_rootvm_init_late(thread_t *root_thread, + const hyp_env_data_t *hyp_env) +{ + assert(root_thread != NULL); + assert(hyp_env != NULL); + + addrspace_t *root_addrspace = root_thread->addrspace; + if (root_addrspace == NULL) { + panic("vgic rootvm_init_late: addrspace not yet created\n"); + } + + vic_t *root_vic = root_thread->vgic_vic; + spinlock_acquire(&root_vic->gicd_lock); + + root_vic->gicd_device.type = VDEVICE_TYPE_VGIC_GICD; + if (vdevice_attach_vmaddr(&root_vic->gicd_device, root_addrspace, + hyp_env->gicd_base, sizeof(gicd_t)) != OK) { + panic("vgic rootvm_init_late: unable to map GICD\n"); + } + + rcu_read_start(); + for (index_t i = 0U; i < root_vic->gicr_count; i++) { + thread_t *gicr_vcpu = + atomic_load_consume(&root_vic->gicr_vcpus[i]); + if (gicr_vcpu == NULL) { + continue; + } + gicr_vcpu->vgic_gicr_device.type = VDEVICE_TYPE_VGIC_GICR; + if (vdevice_attach_vmaddr( + &gicr_vcpu->vgic_gicr_device, root_addrspace, + hyp_env->gicr_base + (i * hyp_env->gicr_stride), + hyp_env->gicr_stride) != OK) { + panic("vgic rootvm_init_late: unable to map GICR\n"); + } + } + rcu_read_finish(); + spinlock_release(&root_vic->gicd_lock); +} + error_t vgic_handle_object_create_hwirq(hwirq_create_t hwirq_create) { @@ -686,6 +891,8 @@ vgic_handle_object_create_hwirq(hwirq_create_t hwirq_create) GICV3_IRQ_TYPE_PPI) { err = ERROR_ARGUMENT_INVALID; } + } else { + // Not a forwarded IRQ } return err; @@ -757,15 +964,22 @@ vgic_bind_hwirq_spi(vic_t *vic, hwirq_t *hwirq, virq_t virq) rcu_read_finish(); // Set the chosen physical route - gicv3_spi_set_route(hwirq->irq, physical_router); + err = gicv3_spi_set_route(hwirq->irq, physical_router); + if (err != OK) { + goto release_lock; + } #if GICV3_HAS_GICD_ICLAR if (GICD_IROUTER_get_IRM(&physical_router)) { // Set the HW IRQ's 1-of-N routing classes. - gicv3_spi_set_classes( + err = gicv3_spi_set_classes( hwirq->irq, !vgic_delivery_state_get_nclass0(¤t_dstate), vgic_delivery_state_get_class1(¤t_dstate)); + + if (err != OK) { + goto release_lock; + } } #endif @@ -784,10 +998,10 @@ vgic_bind_hwirq_spi(vic_t *vic, hwirq_t *hwirq, virq_t virq) // Mode change failed; the hardware config must be fixed to the // other mode. Flip the software mode. if (is_edge) { - vgic_delivery_state_atomic_intersection( + (void)vgic_delivery_state_atomic_intersection( dstate, cfg_is_edge, memory_order_relaxed); } else { - vgic_delivery_state_atomic_difference( + (void)vgic_delivery_state_atomic_difference( dstate, cfg_is_edge, memory_order_relaxed); } } @@ -799,6 +1013,8 @@ vgic_bind_hwirq_spi(vic_t *vic, hwirq_t *hwirq, virq_t virq) } hwirq->vgic_enable_hw = true; + +release_lock: spinlock_release(&vic->gicd_lock); out: @@ -1118,8 +1334,17 @@ vgic_gicd_set_control(vic_t *vic, GICD_CTLR_DS_t ctlr) GICD_CTLR_DS_copy_EnableGrp0(&new_ctlr, &ctlr); GICD_CTLR_DS_copy_EnableGrp1(&new_ctlr, &ctlr); +#if GICV3_HAS_VLPI_V4_1 && VGIC_HAS_LPI + if (!GICD_CTLR_DS_get_EnableGrp0(&old_ctlr) && + !GICD_CTLR_DS_get_EnableGrp1(&old_ctlr)) { + GICD_CTLR_DS_copy_nASSGIreq(&new_ctlr, &ctlr); + } +#endif if (!GICD_CTLR_DS_is_equal(new_ctlr, old_ctlr)) { +#if GICV3_HAS_VLPI_V4_1 && VGIC_HAS_LPI + vic->vsgis_enabled = GICD_CTLR_DS_get_nASSGIreq(&new_ctlr); +#endif atomic_store_relaxed(&vic->gicd_ctlr, new_ctlr); vgic_update_enables(vic, new_ctlr); } @@ -1276,14 +1501,11 @@ vgic_gicd_set_irq_router(vic_t *vic, irq_t irq_num, uint8_t aff0, uint8_t aff1, assert(dstate != NULL); // Find the new target index - psci_mpidr_t route_id = psci_mpidr_default(); - psci_mpidr_set_Aff0(&route_id, aff0); - psci_mpidr_set_Aff1(&route_id, aff1); - psci_mpidr_set_Aff2(&route_id, aff2); - psci_mpidr_set_Aff3(&route_id, aff3); - cpu_index_result_t cpu_r = platform_cpu_mpidr_to_index(route_id); - index_t route_index; - if ((cpu_r.e == OK) && (cpu_r.r < vic->gicr_count)) { + index_result_t cpu_r = + vgic_get_index_for_mpidr(vic, aff0, aff1, aff2, aff3); + index_t route_index; + if (cpu_r.e == OK) { + assert(cpu_r.r < vic->gicr_count); route_index = cpu_r.r; } else { // Use an out-of-range value to indicate an invalid route. @@ -1358,6 +1580,8 @@ vgic_gicd_set_irq_router(vic_t *vic, irq_t irq_num, uint8_t aff0, uint8_t aff1, #endif if (new_target != NULL) { physical_router = new_target->vgic_irouter; + } else { + // No valid target } // Set the chosen physical route @@ -1365,12 +1589,12 @@ vgic_gicd_set_irq_router(vic_t *vic, irq_t irq_num, uint8_t aff0, uint8_t aff1, irq_num, route_index, GICD_IROUTER_raw(physical_router)); irq_t irq = hwirq_from_virq_source(source)->irq; - gicv3_spi_set_route(irq, physical_router); + (void)gicv3_spi_set_route(irq, physical_router); #if GICV3_HAS_GICD_ICLAR if (GICD_IROUTER_get_IRM(&physical_router)) { // Set the HW IRQ's 1-of-N routing classes. - gicv3_spi_set_classes( + (void)gicv3_spi_set_classes( irq, !vgic_delivery_state_get_nclass0(&new_dstate), vgic_delivery_state_get_class1(&new_dstate)); @@ -1441,12 +1665,335 @@ vgic_get_thread_by_gicr_index(vic_t *vic, index_t gicr_num) return atomic_load_consume(&vic->gicr_vcpus[gicr_num]); } +#if VGIC_HAS_LPI +// Copy part or all of an LPI config or pending table from VM memory. +static void +vgic_gicr_copy_in(addrspace_t *addrspace, uint8_t *hyp_table, + size_t hyp_table_size, vmaddr_t vm_table_ipa, size_t offset, + size_t vm_table_size) +{ + error_t err = OK; + + if (util_add_overflows((uintptr_t)hyp_table, offset) || + util_add_overflows(vm_table_ipa, offset)) { + err = ERROR_ADDR_OVERFLOW; + goto out; + } + + if ((offset >= hyp_table_size) || (offset >= vm_table_size)) { + err = ERROR_ADDR_UNDERFLOW; + goto out; + } + + err = useraccess_copy_from_guest_ipa(addrspace, hyp_table + offset, + hyp_table_size - offset, + vm_table_ipa + offset, + vm_table_size - offset, false, + false) + .e; + +out: + if (err != OK) { + // Copy failed. + // + // Note that GICv4.1 deprecates implementation of SError + // generation in the GICR & CPU interface (as opposed to the + // ITS), and recent CPUs don't implement it. So there is no way + // to report this to the VM. We just log it and continue. + TRACE_AND_LOG(DEBUG, WARN, + "vgicr: LPI table copy-in failed: {:d}", + (register_t)err); + } +} + +static bool +vgic_gicr_copy_pendbase(vic_t *vic, count_t idbits, thread_t *gicr_vcpu) +{ + assert(vic != NULL); + assert(gicr_vcpu != NULL); + + GICR_PENDBASER_t pendbaser = + atomic_load_relaxed(&gicr_vcpu->vgic_gicr_rd_pendbaser); + bool ptz = GICR_PENDBASER_get_PTZ(&pendbaser); + size_t pending_table_size = + BITMAP_NUM_WORDS(util_bit(vic->gicd_idbits)) * + sizeof(register_t); + const size_t pending_table_reserved = + BITMAP_NUM_WORDS(GIC_LPI_BASE) * sizeof(register_t); + + assert(gicr_vcpu->vgic_vlpi_pending_table != NULL); + assert(pending_table_size > pending_table_reserved); + + if (ptz) { + errno_t err_mem = memset_s(gicr_vcpu->vgic_vlpi_pending_table, + pending_table_size, 0, + pending_table_size); + if (err_mem != 0) { + panic("Error in memset_s operation!"); + } + } else { + // Zero the reserved part of the pending table + errno_t err_mem = memset_s(gicr_vcpu->vgic_vlpi_pending_table, + pending_table_reserved, 0, + pending_table_reserved); + if (err_mem != 0) { + panic("Error in memset_s operation!"); + } + + // Look up the physical address of the IPA range specified in + // the GICR_PENDBASER, and copy it into the pending table. If + // the lookup fails, or the permissions are wrong, copy zeros. + vmaddr_t base = GICR_PENDBASER_get_PA(&pendbaser); + size_t vm_table_size = + BITMAP_NUM_WORDS(util_bit(idbits)) * sizeof(register_t); + assert(vm_table_size <= pending_table_size); + + vgic_gicr_copy_in(gicr_vcpu->addrspace, + gicr_vcpu->vgic_vlpi_pending_table, + pending_table_size, base, + pending_table_reserved, vm_table_size); + + // Zero the remainder of the pending table + if (vm_table_size < pending_table_size) { + err_mem = memset_s(gicr_vcpu->vgic_vlpi_pending_table + + vm_table_size, + pending_table_size - vm_table_size, + 0, + pending_table_size - vm_table_size); + if (err_mem != 0) { + panic("Error in memset_s operation!"); + } + } + } + return ptz; +} + +static void +vgic_gicr_copy_propbase_all(vic_t *vic, thread_t *gicr_vcpu, + bool zero_remainder) +{ + assert(vic != NULL); + + GICR_PROPBASER_t propbaser = + atomic_load_relaxed(&vic->gicr_rd_propbaser); + size_t config_table_size = util_bit(vic->gicd_idbits) - GIC_LPI_BASE; + + count_t idbits = util_min(GICR_PROPBASER_get_IDbits(&propbaser) + 1U, + vic->gicd_idbits); + vmaddr_t base = GICR_PROPBASER_get_PA(&propbaser); + size_t vm_table_size = (util_bit(idbits) >= GIC_LPI_BASE) + ? (util_bit(idbits) - GIC_LPI_BASE) + : 0U; + assert(vm_table_size <= config_table_size); + + vgic_gicr_copy_in(gicr_vcpu->addrspace, vic->vlpi_config_table, + config_table_size, base, 0U, vm_table_size); + + // Zero the remainder of the pending table + if (zero_remainder && (vm_table_size < config_table_size)) { + errno_t err_mem = + memset_s(vic->vlpi_config_table + vm_table_size, + config_table_size - vm_table_size, 0, + config_table_size - vm_table_size); + if (err_mem != 0) { + panic("Error in memset_s operation!"); + } + } +} + +void +vgic_gicr_copy_propbase_one(vic_t *vic, thread_t *gicr_vcpu, irq_t vlpi) +{ + GICR_PROPBASER_t propbaser = + atomic_load_relaxed(&vic->gicr_rd_propbaser); + size_t config_table_size = util_bit(vic->gicd_idbits) - GIC_LPI_BASE; + + count_t idbits = util_min(GICR_PROPBASER_get_IDbits(&propbaser) + 1U, + vic->gicd_idbits); + // Note that we only ever read these mappings (as writing back to them + // is strictly optional in the spec) so we don't require write access. + vmaddr_t base = GICR_PROPBASER_get_PA(&propbaser); + + // Ignore requests for out-of-range vLPI numbers + if ((vlpi >= GIC_LPI_BASE) && (vlpi < util_bit(idbits))) { + // Copy in a single byte + vgic_gicr_copy_in(gicr_vcpu->addrspace, vic->vlpi_config_table, + config_table_size, base, + ((size_t)vlpi - (size_t)GIC_LPI_BASE), + ((size_t)vlpi - (size_t)GIC_LPI_BASE + 1U)); + } +} + +#if GICV3_HAS_VLPI_V4_1 +static void +vgic_update_vsgi(thread_t *gicr_vcpu, irq_t irq_num) +{ + // Note: we don't check whether vSGI delivery is enabled here; that is + // only done when sending an SGI. + _Atomic vgic_delivery_state_t *dstate = + &gicr_vcpu->vgic_private_states[irq_num]; + vgic_delivery_state_t new_dstate = atomic_load_relaxed(dstate); + + // Note: as per the spec, this is a no-op if the vPE is not mapped. + // The gicv3 driver may ignore the call in that case. + (void)gicv3_its_vsgi_config( + gicr_vcpu, irq_num, + vgic_delivery_state_get_enabled(&new_dstate), + vgic_delivery_state_get_group1(&new_dstate), + vgic_delivery_state_get_priority(&new_dstate)); +} + +static void +vgic_setup_vcpu_vsgis(thread_t *vcpu) +{ + for (virq_t sgi = GIC_SGI_BASE; sgi < GIC_SGI_BASE + GIC_SGI_NUM; + sgi++) { + vgic_update_vsgi(vcpu, sgi); + } + + count_result_t sync_r = gicv3_its_vsgi_sync(vcpu); + assert(sync_r.e == OK); + atomic_store_release(&vcpu->vgic_vsgi_setup_seq, sync_r.r); +} + +error_t +vgic_vsgi_assert(thread_t *gicr_vcpu, irq_t irq_num) +{ + error_t err; + + count_t setup_seq = + atomic_load_acquire(&gicr_vcpu->vgic_vsgi_setup_seq); + + if (setup_seq == ~(count_t)0U) { + // VSGI setup not queued yet + err = ERROR_DENIED; + goto out; + } + + if (compiler_unexpected(setup_seq != 0U)) { + bool_result_t complete_r = + gicv3_its_vsgi_is_complete(setup_seq); + assert(complete_r.e == OK); + if (!complete_r.r) { + // VSGI setup queued but VSYNC not complete yet + err = ERROR_BUSY; + goto out; + } + atomic_store_release(&gicr_vcpu->vgic_vsgi_setup_seq, 0U); + } + + VGIC_TRACE(VIRQ_CHANGED, gicr_vcpu->vgic_vic, gicr_vcpu, + "sgi {:d}: send vsgi", irq_num); + err = gicv3_its_vsgi_assert(gicr_vcpu, irq_num); + +out: + return err; +} +#endif + +static error_t +vgic_gicr_enable_lpis(vic_t *vic, thread_t *gicr_vcpu) +{ + assert(vic != NULL); + assert(vgic_has_lpis(vic)); + assert(vic->vlpi_config_table != NULL); + assert(gicr_vcpu != NULL); + assert(gicr_vcpu->vgic_vlpi_pending_table != NULL); + + GICR_PROPBASER_t propbaser = + atomic_load_relaxed(&vic->gicr_rd_propbaser); + count_t idbits = util_min(GICR_PROPBASER_get_IDbits(&propbaser) + 1U, + vic->gicd_idbits); + + // If this is the first VCPU to enable LPIs, we need to copy the + // LPI configurations from the virtual GICR_PROPBASER. This is not + // done for subsequent enables; LPI configuration changes must raise + // explicit invalidates after that point. + spinlock_acquire(&vic->gicd_lock); + if (!vic->vlpi_config_valid) { + vgic_gicr_copy_propbase_all(vic, gicr_vcpu, true); + vic->vlpi_config_valid = true; + } + spinlock_release(&vic->gicd_lock); + + // If the virtual GICR_PENDBASER has the PTZ bit clear when LPIs are + // enabled, we need to copy the VCPU's VLPI pending states from the + // virtual GICR_PENDBASER. Otherwise we just zero the VLPI pending + // states and ignore the GICR_PENDBASER PA entirely. + // + // Note that the spec does not require us to ever write back to the + // pending table. + bool pending_zeroed = vgic_gicr_copy_pendbase(vic, idbits, gicr_vcpu); + + // Call the ITS driver to map the VCPU into the VPE table. + paddr_t config_table_phys = partition_virt_to_phys( + vic->header.partition, (uintptr_t)vic->vlpi_config_table); + assert(config_table_phys != PADDR_INVALID); + size_t config_table_size = util_bit(vic->gicd_idbits) - GIC_LPI_BASE; + paddr_t pending_table_phys = partition_virt_to_phys( + gicr_vcpu->header.partition, + (uintptr_t)gicr_vcpu->vgic_vlpi_pending_table); + assert(pending_table_phys != PADDR_INVALID); + size_t pending_table_size = + BITMAP_NUM_WORDS(util_bit(vic->gicd_idbits)) * + sizeof(register_t); + error_t err = gicv3_its_vpe_map(gicr_vcpu, vic->gicd_idbits, + config_table_phys, config_table_size, + pending_table_phys, pending_table_size, + pending_zeroed); + +#if GICV3_HAS_VLPI_V4_1 + if (err == OK) { + // Tell the ITS about the vPE's vSGI configuration. + spinlock_acquire(&vic->gicd_lock); + vgic_setup_vcpu_vsgis(gicr_vcpu); + spinlock_release(&vic->gicd_lock); + } +#endif + + if (gicr_vcpu == thread_get_self()) { + preempt_disable(); + vgic_vpe_schedule_current(); + preempt_enable(); + } + + return err; +} +#endif // VGIC_HAS_LPI + void vgic_gicr_rd_set_control(vic_t *vic, thread_t *gicr_vcpu, GICR_CTLR_t ctlr) { +#if VGIC_HAS_LPI + bool enable_lpis = GICR_CTLR_get_Enable_LPIs(&ctlr) && + vgic_has_lpis(vic); + + if (enable_lpis) { + GICR_CTLR_t ctlr_enable_lpis = GICR_CTLR_default(); + GICR_CTLR_set_Enable_LPIs(&ctlr_enable_lpis, true); + GICR_CTLR_t old_ctlr = GICR_CTLR_atomic_union( + &gicr_vcpu->vgic_gicr_rd_ctlr, ctlr_enable_lpis, + memory_order_acquire); + bool old_enable_lpis = GICR_CTLR_get_Enable_LPIs(&old_ctlr); + + if (!old_enable_lpis) { + error_t err = vgic_gicr_enable_lpis(vic, gicr_vcpu); + if (err != OK) { + // LPI enable failed; clear the enable bit. + TRACE_AND_LOG(DEBUG, WARN, + "vgicr: LPI enable failed: {:d}", + (register_t)err); + (void)GICR_CTLR_atomic_difference( + &gicr_vcpu->vgic_gicr_rd_ctlr, + ctlr_enable_lpis, memory_order_release); + } + } + } +#else (void)vic; (void)gicr_vcpu; (void)ctlr; +#endif } GICR_CTLR_t @@ -1454,8 +2001,19 @@ vgic_gicr_rd_get_control(vic_t *vic, thread_t *gicr_vcpu) { (void)vic; +#if VGIC_HAS_LPI + GICR_CTLR_t ctlr = atomic_load_relaxed(&gicr_vcpu->vgic_gicr_rd_ctlr); +#if GICV3_HAS_VLPI_V4_1 + bool_result_t disabled_r = + gicv3_its_vsgi_is_complete(gicr_vcpu->vgic_vsgi_disable_seq); + if ((disabled_r.e == OK) && !disabled_r.r) { + GICR_CTLR_set_RWP(&ctlr, true); + } +#endif +#else (void)gicr_vcpu; GICR_CTLR_t ctlr = GICR_CTLR_default(); +#endif return ctlr; } @@ -1464,25 +2022,125 @@ void vgic_gicr_rd_set_statusr(thread_t *gicr_vcpu, GICR_STATUSR_t statusr, bool set) { if (set) { - GICR_STATUSR_atomic_union(&gicr_vcpu->vgic_gicr_rd_statusr, - statusr, memory_order_relaxed); + (void)GICR_STATUSR_atomic_union( + &gicr_vcpu->vgic_gicr_rd_statusr, statusr, + memory_order_relaxed); } else { - GICR_STATUSR_atomic_difference(&gicr_vcpu->vgic_gicr_rd_statusr, - statusr, memory_order_relaxed); + (void)GICR_STATUSR_atomic_difference( + &gicr_vcpu->vgic_gicr_rd_statusr, statusr, + memory_order_relaxed); } } +#if VGIC_HAS_LPI +void +vgic_gicr_rd_set_propbase(vic_t *vic, GICR_PROPBASER_t propbase) +{ + GICR_PROPBASER_t new_propbase = GICR_PROPBASER_default(); + + // We implement the cache and shareability fields as read-only to + // reflect the fact that the hypervisor always accesses the table + // through its own shared cacheable mapping. + GICR_PROPBASER_set_OuterCache(&new_propbase, 0U); + GICR_PROPBASER_set_InnerCache(&new_propbase, 7U); + GICR_PROPBASER_set_Shareability(&new_propbase, 1U); + + // Use the physical address and size provided by the VM. + GICR_PROPBASER_copy_PA(&new_propbase, &propbase); + GICR_PROPBASER_copy_IDbits(&new_propbase, &propbase); + + // There is no need to synchronise or update anything else here. This + // value is only used when EnableLPIs changes to 1 or an explicit + // invalidate is processed. + atomic_store_relaxed(&vic->gicr_rd_propbaser, new_propbase); +} + +void +vgic_gicr_rd_set_pendbase(vic_t *vic, thread_t *gicr_vcpu, + GICR_PENDBASER_t pendbase) +{ + (void)vic; + + GICR_PENDBASER_t new_pendbase = GICR_PENDBASER_default(); + + // We implement the cache and shareability fields as read-only to + // reflect the fact that the hypervisor always accesses the table + // through its own shared cacheable mapping. + GICR_PENDBASER_set_OuterCache(&new_pendbase, 0U); + GICR_PENDBASER_set_InnerCache(&new_pendbase, 7U); + GICR_PENDBASER_set_Shareability(&new_pendbase, 1U); + + // Use the physical address provided by the VM. + GICR_PENDBASER_set_PA(&new_pendbase, GICR_PENDBASER_get_PA(&pendbase)); + + // Copy the PTZ bit. When the VM sets EnableLPIs to 1, this will + // determine the cache update behaviour and the VMAPP command's PTZ bit. + // However, the read trap will always zero this. + GICR_PENDBASER_set_PTZ(&new_pendbase, + GICR_PENDBASER_get_PTZ(&pendbase)); + + // There is no need to synchronise or update anything else here. This + // value is only used when EnableLPIs changes to 1 or an explicit + // invalidate is processed. + atomic_store_relaxed(&gicr_vcpu->vgic_gicr_rd_pendbaser, new_pendbase); +} + +void +vgic_gicr_rd_invlpi(vic_t *vic, thread_t *gicr_vcpu, virq_t vlpi_num) +{ + if (vic->vlpi_config_valid) { + vgic_gicr_copy_propbase_one(vic, gicr_vcpu, vlpi_num); + gicv3_vlpi_inv_by_id(gicr_vcpu, vlpi_num); + } +} + +void +vgic_gicr_rd_invall(vic_t *vic, thread_t *gicr_vcpu) +{ + if (vic->vlpi_config_valid) { + vgic_gicr_copy_propbase_all(vic, gicr_vcpu, false); + gicv3_vlpi_inv_all(gicr_vcpu); + } +} + +bool +vgic_gicr_get_inv_pending(vic_t *vic, thread_t *gicr_vcpu) +{ + return vic->vlpi_config_valid && gicv3_vlpi_inv_pending(gicr_vcpu); +} +#endif + void vgic_gicr_sgi_change_sgi_ppi_pending(vic_t *vic, thread_t *gicr_vcpu, irq_t irq_num, bool set) { assert(vgic_irq_is_private(irq_num)); +#if GICV3_HAS_VLPI_V4_1 && VGIC_HAS_LPI + if (!vgic_irq_is_ppi(irq_num) && vic->vsgis_enabled) { + if (set) { + if (vgic_vsgi_assert(gicr_vcpu, irq_num) == OK) { + // Delivered by ITS + goto out; + } + // Need to deliver in software instead; fall through + } else { + (void)gicv3_its_vsgi_clear(gicr_vcpu, irq_num); + // Might be pending in software too; fall through + } + } +#endif + rcu_read_start(); virq_source_t *source = vgic_find_source(vic, gicr_vcpu, irq_num); vgic_change_irq_pending(vic, gicr_vcpu, irq_num, true, source, set, false); rcu_read_finish(); + +#if GICV3_HAS_VLPI_V4_1 && VGIC_HAS_LPI +out: + return; +#endif } void @@ -1491,7 +2149,13 @@ vgic_gicr_sgi_change_sgi_ppi_enable(vic_t *vic, thread_t *gicr_vcpu, { assert(vgic_irq_is_private(irq_num)); +#if GICV3_HAS_VLPI_V4_1 && VGIC_HAS_LPI + // Take the distributor lock for SGIs to ensure that vSGI config changes + // by different CPUs don't end up out of order in the ITS. + spinlock_acquire(&vic->gicd_lock); +#else preempt_disable(); +#endif rcu_read_start(); virq_source_t *source = vgic_find_source(vic, gicr_vcpu, irq_num); @@ -1503,7 +2167,20 @@ vgic_gicr_sgi_change_sgi_ppi_enable(vic_t *vic, thread_t *gicr_vcpu, rcu_read_finish(); +#if GICV3_HAS_VLPI_V4_1 && VGIC_HAS_LPI + if (!vgic_irq_is_ppi(irq_num) && vgic_has_lpis(vic)) { + vgic_update_vsgi(gicr_vcpu, irq_num); + if (!set) { + count_result_t seq_r = gicv3_its_vsgi_sync(gicr_vcpu); + if (seq_r.e == OK) { + gicr_vcpu->vgic_vsgi_disable_seq = seq_r.r; + } + } + } + spinlock_release(&vic->gicd_lock); +#else preempt_enable(); +#endif } void @@ -1521,10 +2198,23 @@ vgic_gicr_sgi_set_sgi_ppi_group(vic_t *vic, thread_t *gicr_vcpu, irq_t irq_num, { assert(vgic_irq_is_private(irq_num)); +#if GICV3_HAS_VLPI_V4_1 + // Take the distributor lock for SGIs to ensure that two config changes + // by different CPUs don't end up out of order in the ITS. + spinlock_acquire(&vic->gicd_lock); +#endif + _Atomic vgic_delivery_state_t *dstate = &gicr_vcpu->vgic_private_states[irq_num]; vgic_sync_group_change(vic, irq_num, dstate, is_group_1); + +#if GICV3_HAS_VLPI_V4_1 && VGIC_HAS_LPI + if (!vgic_irq_is_ppi(irq_num) && vgic_has_lpis(vic)) { + vgic_update_vsgi(gicr_vcpu, irq_num); + } + spinlock_release(&vic->gicd_lock); +#endif } void @@ -1537,6 +2227,12 @@ vgic_gicr_sgi_set_sgi_ppi_priority(vic_t *vic, thread_t *gicr_vcpu, vgic_set_irq_priority(vic, gicr_vcpu, irq_num, priority); +#if GICV3_HAS_VLPI_V4_1 && VGIC_HAS_LPI + if (!vgic_irq_is_ppi(irq_num) && vgic_has_lpis(vic)) { + vgic_update_vsgi(gicr_vcpu, irq_num); + } +#endif + spinlock_release(&vic->gicd_lock); } @@ -1608,7 +2304,7 @@ vic_bind_shared(virq_source_t *source, vic_t *vic, virq_t virq, rcu_read_start(); virq_source_t *_Atomic *attach_ptr = &vic->sources[virq - GIC_SPI_BASE]; - virq_source_t *old_source = atomic_load_acquire(attach_ptr); + virq_source_t *old_source = atomic_load_acquire(attach_ptr); do { // If there is already a source bound, we can't bind another. if (old_source != NULL) { @@ -1800,10 +2496,10 @@ vic_bind_private_forward_private(virq_source_t *source, vic_t *vic, // Mode change failed; the hardware config must be fixed to the // other mode. Flip the software mode. if (is_edge) { - vgic_delivery_state_atomic_intersection( + (void)vgic_delivery_state_atomic_intersection( dstate, cfg_is_edge, memory_order_relaxed); } else { - vgic_delivery_state_atomic_difference( + (void)vgic_delivery_state_atomic_difference( dstate, cfg_is_edge, memory_order_relaxed); } } @@ -1864,7 +2560,7 @@ vic_do_unbind(virq_source_t *source) // be ordered after the level_src bit is cleared in the undeliver, to // ensure that other threads don't see this NULL pointer while the // level_src or hw_active bits are still set. - virq_source_t *registered_source = source; + virq_source_t *registered_source = source; virq_source_t *_Atomic *registered_source_ptr = source->is_private ? &vcpu->vgic_sources[source->virq - GIC_PPI_BASE] @@ -1993,3 +2689,103 @@ vgic_handle_irq_received_forward_spi(hwirq_t *hwirq) return deactivate; } + +static error_t +vgic_set_mpidr_mapping(vic_t *vic, MPIDR_EL1_t mask, count_t aff0_shift, + count_t aff1_shift, count_t aff2_shift, + count_t aff3_shift, bool mt) +{ + uint64_t cpuindex_mask = 0U; + const count_t shifts[4] = { aff0_shift, aff1_shift, aff2_shift, + aff3_shift }; + const uint8_t masks[4] = { MPIDR_EL1_get_Aff0(&mask), + MPIDR_EL1_get_Aff1(&mask), + MPIDR_EL1_get_Aff2(&mask), + MPIDR_EL1_get_Aff3(&mask) }; + error_t err; + + for (index_t i = 0U; i < 4U; i++) { + // Since there are only 32 significant affinity bits, a shift of + // more than 32 can't be useful, so don't allow it. + if (shifts[i] >= 32U) { + err = ERROR_ARGUMENT_INVALID; + goto out; + } + + // Collect the output bits, checking that there's no overlap. + uint64_t field_mask = (uint64_t)masks[i] << shifts[i]; + if ((cpuindex_mask & field_mask) != 0U) { + err = ERROR_ARGUMENT_INVALID; + goto out; + } + cpuindex_mask |= field_mask; + } + + // We don't allow sparse mappings, so check that the output bits are + // contiguous and start from the least significant bit. This is true if + // the mask is one less than a power of two. + // + // Also, the mask has to fit in cpu_index_t, and must not be able to + // produce CPU_INDEX_INVALID, which currently limits it to 15 bits. + if (!util_is_p2(cpuindex_mask + 1U) || + (cpuindex_mask >= CPU_INDEX_INVALID)) { + err = ERROR_ARGUMENT_INVALID; + goto out; + } + + // Note: we currently don't check that the mapping can assign unique + // MPIDR values to all VCPUs. If it doesn't, the VM will probably fail + // to boot or at least fail to start the VCPUs with duplicated values, + // but the hypervisor itself will not fail. + + // Construct and set the mapping. + vic->mpidr_mapping = (platform_mpidr_mapping_t){ + .aff_shift = { shifts[0], shifts[1], shifts[2], shifts[3] }, + .aff_mask = { masks[0], masks[1], masks[2], masks[3] }, + .multi_thread = mt, + .uniprocessor = (cpuindex_mask == 0U), + }; + err = OK; + +out: + return err; +} + +error_t +hypercall_vgic_set_mpidr_mapping(cap_id_t vic_cap, uint64_t mask, + count_t aff0_shift, count_t aff1_shift, + count_t aff2_shift, count_t aff3_shift, + bool mt) +{ + error_t err; + cspace_t *cspace = cspace_get_self(); + object_type_t type; + + object_ptr_result_t o = cspace_lookup_object_any( + cspace, vic_cap, CAP_RIGHTS_GENERIC_OBJECT_ACTIVATE, &type); + if (compiler_unexpected(o.e != OK)) { + err = o.e; + goto out_released; + } + if (type != OBJECT_TYPE_VIC) { + err = ERROR_CSPACE_WRONG_OBJECT_TYPE; + goto out_unlocked; + } + vic_t *vic = o.r.vic; + + spinlock_acquire(&vic->header.lock); + if (atomic_load_relaxed(&vic->header.state) == OBJECT_STATE_INIT) { + err = vgic_set_mpidr_mapping(vic, MPIDR_EL1_cast(mask), + aff0_shift, aff1_shift, aff2_shift, + aff3_shift, mt); + } else { + err = ERROR_OBJECT_STATE; + } + spinlock_release(&vic->header.lock); + +out_unlocked: + object_put(type, o.r); +out_released: + + return err; +} diff --git a/hyp/vm/vgic/src/sysregs.c b/hyp/vm/vgic/src/sysregs.c index d141186..bfb5c85 100644 --- a/hyp/vm/vgic/src/sysregs.c +++ b/hyp/vm/vgic/src/sysregs.c @@ -21,61 +21,25 @@ #include "gich_lrs.h" #include "internal.h" -vcpu_trap_result_t -vgic_handle_vcpu_trap_sysreg_write(ESR_EL2_ISS_MSR_MRS_t iss) +#if defined(ARCH_ARM_FEAT_FGT) && ARCH_ARM_FEAT_FGT +#include +#endif + +static vcpu_trap_result_t +vgic_handle_vcpu_trap_sysreg_write_no_fgt(ESR_EL2_ISS_MSR_MRS_t iss, vic_t *vic, + register_t val) { - vcpu_trap_result_t ret = VCPU_TRAP_RESULT_EMULATED; - thread_t *thread = thread_get_self(); - vic_t *vic = thread->vgic_vic; - if (vic == NULL) { + vcpu_trap_result_t ret = VCPU_TRAP_RESULT_EMULATED; + +#if defined(ARCH_ARM_FEAT_FGT) && ARCH_ARM_FEAT_FGT + if (arm_fgt_is_allowed()) { + // These registers should not be trapped when FGT is available ret = VCPU_TRAP_RESULT_UNHANDLED; goto out; } - - // Assert this is a write - assert(!ESR_EL2_ISS_MSR_MRS_get_Direction(&iss)); - - // Read the thread's register - uint8_t reg_num = ESR_EL2_ISS_MSR_MRS_get_Rt(&iss); - register_t val = vcpu_gpr_read(thread, reg_num); - - // Remove the fields that are not used in the comparison - ESR_EL2_ISS_MSR_MRS_set_Rt(&iss, 0); - ESR_EL2_ISS_MSR_MRS_set_Direction(&iss, false); +#endif switch (ESR_EL2_ISS_MSR_MRS_raw(iss)) { - case ISS_MRS_MSR_ICC_DIR_EL1: - vgic_icc_irq_deactivate( - vic, ICC_DIR_EL1_get_INTID(&ICC_DIR_EL1_cast(val))); - break; - - case ISS_MRS_MSR_ICC_ASGI1R_EL1: - // ICC_ASGI1R_EL1 is treated as an alias of ICC_SGI0R_EL1. - // This is because virtual accesses are always non-secure, and - // non-secure writes generate SGIs for group 0 or secure group - // 1, where the latter is treated as group 0 too because - // GICD_CTLR.DS=1. - case ISS_MRS_MSR_ICC_SGI0R_EL1: - vgic_icc_generate_sgi(vic, ICC_SGIR_EL1_cast(val), false); - break; - - case ISS_MRS_MSR_ICC_SGI1R_EL1: - vgic_icc_generate_sgi(vic, ICC_SGIR_EL1_cast(val), true); - break; - - case ISS_MRS_MSR_ICC_SRE_EL1: - // WI - break; - - case ISS_MRS_MSR_ICC_IGRPEN0_EL1: - vgic_icc_set_group_enable(false, ICC_IGRPEN_EL1_cast(val)); - break; - - case ISS_MRS_MSR_ICC_IGRPEN1_EL1: - vgic_icc_set_group_enable(true, ICC_IGRPEN_EL1_cast(val)); - break; - -#if !defined(ARCH_ARM_8_6_FGT) || !ARCH_ARM_8_6_FGT // Trapped by TALL[01] which are set to trap ICC_IGRPEN[01]_EL1 case ISS_MRS_MSR_ICC_EOIR0_EL1: { // Drop the highest active priority (which we are allowed to @@ -169,61 +133,95 @@ vgic_handle_vcpu_trap_sysreg_write(ESR_EL2_ISS_MSR_MRS_t iss) register_ICH_AP1R3_EL2_write((uint32_t)val); #endif break; -#endif - default: ret = VCPU_TRAP_RESULT_UNHANDLED; break; } +#if defined(ARCH_ARM_FEAT_FGT) && ARCH_ARM_FEAT_FGT out: +#endif return ret; } vcpu_trap_result_t -vgic_handle_vcpu_trap_sysreg_read(ESR_EL2_ISS_MSR_MRS_t iss) +vgic_handle_vcpu_trap_sysreg_write(ESR_EL2_ISS_MSR_MRS_t iss) { - register_t val = 0U; vcpu_trap_result_t ret = VCPU_TRAP_RESULT_EMULATED; - thread_t *thread = thread_get_self(); + thread_t *thread = thread_get_self(); + vic_t *vic = thread->vgic_vic; + if (vic == NULL) { + ret = VCPU_TRAP_RESULT_UNHANDLED; + goto out; + } - // Assert this is a read - assert(ESR_EL2_ISS_MSR_MRS_get_Direction(&iss)); + // Assert this is a write + assert(!ESR_EL2_ISS_MSR_MRS_get_Direction(&iss)); - uint8_t reg_num = ESR_EL2_ISS_MSR_MRS_get_Rt(&iss); + // Read the thread's register + uint8_t reg_num = ESR_EL2_ISS_MSR_MRS_get_Rt(&iss); + register_t val = vcpu_gpr_read(thread, reg_num); // Remove the fields that are not used in the comparison - ESR_EL2_ISS_MSR_MRS_t temp_iss = iss; - ESR_EL2_ISS_MSR_MRS_set_Rt(&temp_iss, 0U); - ESR_EL2_ISS_MSR_MRS_set_Direction(&temp_iss, false); + ESR_EL2_ISS_MSR_MRS_set_Rt(&iss, 0); + ESR_EL2_ISS_MSR_MRS_set_Direction(&iss, false); - switch (ESR_EL2_ISS_MSR_MRS_raw(temp_iss)) { - case ISS_MRS_MSR_ICC_SRE_EL1: { - // Return 1 for SRE, DFB and DIB - ICC_SRE_EL1_t sre; - ICC_SRE_EL1_init(&sre); - ICC_SRE_EL1_set_SRE(&sre, true); - ICC_SRE_EL1_set_DFB(&sre, true); - ICC_SRE_EL1_set_DIB(&sre, true); - val = ICC_SRE_EL1_raw(sre); + switch (ESR_EL2_ISS_MSR_MRS_raw(iss)) { + case ISS_MRS_MSR_ICC_DIR_EL1: + vgic_icc_irq_deactivate( + vic, ICC_DIR_EL1_get_INTID(&ICC_DIR_EL1_cast(val))); break; - } - case ISS_MRS_MSR_ICC_IGRPEN0_EL1: { - ICC_IGRPEN_EL1_t igrpen = ICC_IGRPEN_EL1_default(); - ICC_IGRPEN_EL1_set_Enable(&igrpen, thread->vgic_group0_enabled); - val = ICC_IGRPEN_EL1_raw(igrpen); + case ISS_MRS_MSR_ICC_ASGI1R_EL1: + // ICC_ASGI1R_EL1 is treated as an alias of ICC_SGI0R_EL1. + // This is because virtual accesses are always non-secure, and + // non-secure writes generate SGIs for group 0 or secure group + // 1, where the latter is treated as group 0 too because + // GICD_CTLR.DS=1. + case ISS_MRS_MSR_ICC_SGI0R_EL1: + vgic_icc_generate_sgi(vic, ICC_SGIR_EL1_cast(val), false); break; - } - case ISS_MRS_MSR_ICC_IGRPEN1_EL1: { - ICC_IGRPEN_EL1_t igrpen = ICC_IGRPEN_EL1_default(); - ICC_IGRPEN_EL1_set_Enable(&igrpen, thread->vgic_group1_enabled); - val = ICC_IGRPEN_EL1_raw(igrpen); + case ISS_MRS_MSR_ICC_SGI1R_EL1: + vgic_icc_generate_sgi(vic, ICC_SGIR_EL1_cast(val), true); + break; + + case ISS_MRS_MSR_ICC_SRE_EL1: + // WI + break; + + case ISS_MRS_MSR_ICC_IGRPEN0_EL1: + vgic_icc_set_group_enable(false, ICC_IGRPEN_EL1_cast(val)); + break; + + case ISS_MRS_MSR_ICC_IGRPEN1_EL1: + vgic_icc_set_group_enable(true, ICC_IGRPEN_EL1_cast(val)); + break; + + default: + ret = vgic_handle_vcpu_trap_sysreg_write_no_fgt(iss, vic, val); break; } -#if !defined(ARCH_ARM_8_6_FGT) || !ARCH_ARM_8_6_FGT +out: + return ret; +} + +static vcpu_trap_result_t +vgic_handle_vcpu_trap_sysreg_read_no_fgt(ESR_EL2_ISS_MSR_MRS_t iss, + thread_t *thread, register_t *val) +{ + vcpu_trap_result_t ret = VCPU_TRAP_RESULT_EMULATED; + +#if defined(ARCH_ARM_FEAT_FGT) && ARCH_ARM_FEAT_FGT + if (arm_fgt_is_allowed()) { + // These registers should not be trapped when FGT is available + ret = VCPU_TRAP_RESULT_UNHANDLED; + goto out; + } +#endif + + switch (ESR_EL2_ISS_MSR_MRS_raw(iss)) { // Trapped by TALL[01] which are set to trap ICC_IGRPEN[01]_EL1 case ISS_MRS_MSR_ICC_IAR0_EL1: case ISS_MRS_MSR_ICC_HPPIR0_EL1: @@ -235,36 +233,36 @@ vgic_handle_vcpu_trap_sysreg_read(ESR_EL2_ISS_MSR_MRS_t iss) // group 1 interrupt (1020 or 1021) can only be returned to EL3 // reads as of GICv3, so we don't need to check group 1. assert(!thread->vgic_group0_enabled); - val = 1023U; + *val = 1023U; break; case ISS_MRS_MSR_ICC_BPR0_EL1: { ICH_VMCR_EL2_t vmcr = register_ICH_VMCR_EL2_read(); - val = ICH_VMCR_EL2_get_VBPR0(&vmcr); + *val = ICH_VMCR_EL2_get_VBPR0(&vmcr); break; } case ISS_MRS_MSR_ICC_AP0R0_EL1: #if CPU_GICH_APR_COUNT >= 1U - val = register_ICH_AP0R0_EL2_read(); + *val = register_ICH_AP0R0_EL2_read(); #endif break; case ISS_MRS_MSR_ICC_AP0R1_EL1: #if CPU_GICH_APR_COUNT >= 2U - val = register_ICH_AP0R1_EL2_read(); + *val = register_ICH_AP0R1_EL2_read(); #endif break; case ISS_MRS_MSR_ICC_AP0R2_EL1: #if CPU_GICH_APR_COUNT >= 4U - val = register_ICH_AP0R2_EL2_read(); + *val = register_ICH_AP0R2_EL2_read(); #endif break; case ISS_MRS_MSR_ICC_AP0R3_EL1: #if CPU_GICH_APR_COUNT >= 4U - val = register_ICH_AP0R3_EL2_read(); + *val = register_ICH_AP0R3_EL2_read(); #endif break; @@ -274,42 +272,95 @@ vgic_handle_vcpu_trap_sysreg_read(ESR_EL2_ISS_MSR_MRS_t iss) // there can't be any deliverable IRQs; return 1023, which is // the reserved value meaning no pending interrupt. assert(!thread->vgic_group1_enabled); - val = 1023U; + *val = 1023U; break; case ISS_MRS_MSR_ICC_BPR1_EL1: { ICH_VMCR_EL2_t vmcr = register_ICH_VMCR_EL2_read(); - val = ICH_VMCR_EL2_get_VBPR1(&vmcr); + *val = ICH_VMCR_EL2_get_VBPR1(&vmcr); break; } case ISS_MRS_MSR_ICC_AP1R0_EL1: #if CPU_GICH_APR_COUNT >= 1U - val = register_ICH_AP1R0_EL2_read(); + *val = register_ICH_AP1R0_EL2_read(); #endif break; case ISS_MRS_MSR_ICC_AP1R1_EL1: #if CPU_GICH_APR_COUNT >= 2U - val = register_ICH_AP1R1_EL2_read(); + *val = register_ICH_AP1R1_EL2_read(); #endif break; case ISS_MRS_MSR_ICC_AP1R2_EL1: #if CPU_GICH_APR_COUNT >= 4U - val = register_ICH_AP1R2_EL2_read(); + *val = register_ICH_AP1R2_EL2_read(); #endif break; case ISS_MRS_MSR_ICC_AP1R3_EL1: #if CPU_GICH_APR_COUNT >= 4U - val = register_ICH_AP1R3_EL2_read(); + *val = register_ICH_AP1R3_EL2_read(); #endif break; + default: + ret = VCPU_TRAP_RESULT_UNHANDLED; + break; + } + +#if defined(ARCH_ARM_FEAT_FGT) && ARCH_ARM_FEAT_FGT +out: #endif + return ret; +} + +vcpu_trap_result_t +vgic_handle_vcpu_trap_sysreg_read(ESR_EL2_ISS_MSR_MRS_t iss) +{ + register_t val = 0U; + vcpu_trap_result_t ret = VCPU_TRAP_RESULT_EMULATED; + thread_t *thread = thread_get_self(); + + // Assert this is a read + assert(ESR_EL2_ISS_MSR_MRS_get_Direction(&iss)); + + uint8_t reg_num = ESR_EL2_ISS_MSR_MRS_get_Rt(&iss); + + // Remove the fields that are not used in the comparison + ESR_EL2_ISS_MSR_MRS_t temp_iss = iss; + ESR_EL2_ISS_MSR_MRS_set_Rt(&temp_iss, 0U); + ESR_EL2_ISS_MSR_MRS_set_Direction(&temp_iss, false); + + switch (ESR_EL2_ISS_MSR_MRS_raw(temp_iss)) { + case ISS_MRS_MSR_ICC_SRE_EL1: { + // Return 1 for SRE, DFB and DIB + ICC_SRE_EL1_t sre; + ICC_SRE_EL1_init(&sre); + ICC_SRE_EL1_set_SRE(&sre, true); + ICC_SRE_EL1_set_DFB(&sre, true); + ICC_SRE_EL1_set_DIB(&sre, true); + val = ICC_SRE_EL1_raw(sre); + break; + } + + case ISS_MRS_MSR_ICC_IGRPEN0_EL1: { + ICC_IGRPEN_EL1_t igrpen = ICC_IGRPEN_EL1_default(); + ICC_IGRPEN_EL1_set_Enable(&igrpen, thread->vgic_group0_enabled); + val = ICC_IGRPEN_EL1_raw(igrpen); + break; + } + + case ISS_MRS_MSR_ICC_IGRPEN1_EL1: { + ICC_IGRPEN_EL1_t igrpen = ICC_IGRPEN_EL1_default(); + ICC_IGRPEN_EL1_set_Enable(&igrpen, thread->vgic_group1_enabled); + val = ICC_IGRPEN_EL1_raw(igrpen); + break; + } default: - ret = VCPU_TRAP_RESULT_UNHANDLED; + ret = vgic_handle_vcpu_trap_sysreg_read_no_fgt(iss, thread, + &val); break; } diff --git a/hyp/vm/vgic/src/util.c b/hyp/vm/vgic/src/util.c index 486aacf..89b1ae1 100644 --- a/hyp/vm/vgic/src/util.c +++ b/hyp/vm/vgic/src/util.c @@ -35,6 +35,11 @@ vgic_get_irq_type(virq_t irq) (irq < (virq_t)(GIC_SPI_EXT_BASE + GIC_SPI_EXT_NUM))) { type = VGIC_IRQ_TYPE_SPI_EXT; } +#endif +#if VGIC_HAS_LPI + else if (irq >= (virq_t)GIC_LPI_BASE) { + type = VGIC_IRQ_TYPE_LPI; + } #endif else { type = VGIC_IRQ_TYPE_RESERVED; @@ -56,6 +61,9 @@ vgic_irq_is_private(virq_t virq) break; case VGIC_IRQ_TYPE_SPI: case VGIC_IRQ_TYPE_RESERVED: +#if VGIC_HAS_LPI && GICV3_HAS_VLPI_V4_1 + case VGIC_IRQ_TYPE_LPI: +#endif default: result = false; break; @@ -76,6 +84,9 @@ vgic_irq_is_spi(virq_t virq) case VGIC_IRQ_TYPE_SGI: case VGIC_IRQ_TYPE_PPI: case VGIC_IRQ_TYPE_RESERVED: +#if VGIC_HAS_LPI && GICV3_HAS_VLPI_V4_1 + case VGIC_IRQ_TYPE_LPI: +#endif default: result = false; break; @@ -96,6 +107,9 @@ vgic_irq_is_ppi(virq_t virq) case VGIC_IRQ_TYPE_SGI: case VGIC_IRQ_TYPE_SPI: case VGIC_IRQ_TYPE_RESERVED: +#if VGIC_HAS_LPI && GICV3_HAS_VLPI_V4_1 + case VGIC_IRQ_TYPE_LPI: +#endif default: result = false; break; @@ -175,6 +189,9 @@ vgic_find_source(vic_t *vic, thread_t *vcpu, virq_t virq) break; case VGIC_IRQ_TYPE_SGI: case VGIC_IRQ_TYPE_RESERVED: +#if VGIC_HAS_LPI && GICV3_HAS_VLPI_V4_1 + case VGIC_IRQ_TYPE_LPI: +#endif default: source = NULL; break; @@ -197,6 +214,9 @@ vgic_find_dstate(vic_t *vic, thread_t *vcpu, virq_t virq) dstate = &vic->spi_states[virq - GIC_SPI_BASE]; break; case VGIC_IRQ_TYPE_RESERVED: +#if VGIC_HAS_LPI && GICV3_HAS_VLPI_V4_1 + case VGIC_IRQ_TYPE_LPI: +#endif default: // Invalid IRQ number dstate = NULL; diff --git a/hyp/vm/vgic/src/vdevice.c b/hyp/vm/vgic/src/vdevice.c index 68579a8..6ddd158 100644 --- a/hyp/vm/vgic/src/vdevice.c +++ b/hyp/vm/vgic/src/vdevice.c @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -13,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -29,12 +29,13 @@ // Qualcomm's JEP106 identifier is 0x70, with no continuation bytes. This is // used in the virtual GICD_IIDR and GICR_IIDR. -#define JEP106_IDENTITY 0x70U -#define JEP106_CONTCODE 0x0U -#define IIDR_IMPLEMENTER ((JEP106_CONTCODE << 8U) | JEP106_IDENTITY) -#define IIDR_PRODUCTID (uint8_t)'G' /* For "Gunyah" */ -#define IIDR_VARIANT 0U -#define IIDR_REVISION 0U +#define JEP106_IDENTITY 0x70U +#define JEP106_CONTCODE 0x0U +#define IIDR_IMPLEMENTER \ + (((uint16_t)JEP106_CONTCODE << 8U) | (uint16_t)JEP106_IDENTITY) +#define IIDR_PRODUCTID (uint8_t)'G' /* For "Gunyah" */ +#define IIDR_VARIANT 0U +#define IIDR_REVISION 0U static register_t vgic_read_irqbits(vic_t *vic, thread_t *vcpu, size_t base_offset, size_t offset) @@ -94,10 +95,22 @@ vgic_read_irqbits(vic_t *vic, thread_t *vcpu, size_t base_offset, size_t offset) } if (bit) { - bits |= (1U << i); + bits |= util_bit(i); } } +#if GICV3_HAS_VLPI_V4_1 && defined(GICV3_ENABLE_VPE) && GICV3_ENABLE_VPE + if ((range_base == GIC_SGI_BASE) && + ((base_offset == offsetof(gicd_t, ispendr)) || + (base_offset == offsetof(gicd_t, icpendr)))) { + // Query the hardware for the vSGI pending state + uint32_result_t bits_r = gicv3_vpe_vsgi_query(vcpu); + if (bits_r.e == OK) { + bits |= bits_r.r; + } + } +#endif // GICV3_HAS_VLPI_V4_1 && GICV3_ENABLE_VPE + if (compiler_expected(!listed)) { // We didn't try to read the pending or active state of a VIRQ // that is in list register, so the value we've read is @@ -242,7 +255,7 @@ vgic_read_config(vic_t *vic, thread_t *vcpu, size_t offset) atomic_load_relaxed(&dstates[i]); if (vgic_delivery_state_get_cfg_is_edge(&this_dstate)) { - bits |= 2U << (i * 2U); + bits |= util_bit((i * 2U) + 1U); } } @@ -251,20 +264,19 @@ vgic_read_config(vic_t *vic, thread_t *vcpu, size_t offset) } static bool -gicd_vdevice_read(size_t offset, register_t *val, size_t access_size) +gicd_vdevice_read(vic_t *vic, size_t offset, register_t *val, + size_t access_size) { bool ret = true; thread_t *thread = thread_get_self(); - vic_t *vic = thread->vgic_vic; - if (vic == NULL) { - ret = false; - goto out; - } + assert(vic != NULL); - if ((offset == OFS_GICD_SETSPI_NSR) || - (offset == OFS_GICD_CLRSPI_NSR) || (offset == OFS_GICD_SETSPI_SR) || - (offset == OFS_GICD_CLRSPI_SR) || (offset == OFS_GICD_SGIR)) { + if ((offset == offsetof(gicd_t, setspi_nsr)) || + (offset == offsetof(gicd_t, clrspi_nsr)) || + (offset == offsetof(gicd_t, setspi_sr)) || + (offset == offsetof(gicd_t, clrspi_sr)) || + (offset == offsetof(gicd_t, sgir))) { // WO registers, RAZ GICD_STATUSR_t statusr; GICD_STATUSR_init(&statusr); @@ -278,10 +290,11 @@ gicd_vdevice_read(size_t offset, register_t *val, size_t access_size) } else if (offset == offsetof(gicd_t, statusr)) { *val = GICD_STATUSR_raw(vic->gicd_statusr); - } else if (offset == OFS_GICD_TYPER) { + } else if (offset == offsetof(gicd_t, typer)) { GICD_TYPER_t typer = GICD_TYPER_default(); GICD_TYPER_set_ITLinesNumber( - &typer, util_balign_up(GIC_SPI_NUM, 32U) / 32U); + &typer, + (count_t)util_balign_up(GIC_SPI_NUM, 32U) / 32U); GICD_TYPER_set_MBIS(&typer, true); #if VGIC_HAS_EXT_IRQS #error Extended IRQs not yet implemented @@ -289,12 +302,17 @@ gicd_vdevice_read(size_t offset, register_t *val, size_t access_size) GICD_TYPER_set_ESPI(&typer, false); #endif +#if VGIC_HAS_LPI + GICD_TYPER_set_LPIS(&typer, vgic_has_lpis(vic)); + GICD_TYPER_set_IDbits(&typer, vic->gicd_idbits - 1U); +#else GICD_TYPER_set_IDbits(&typer, VGIC_IDBITS - 1U); +#endif GICD_TYPER_set_A3V(&typer, true); - GICD_TYPER_set_No1N(&typer, VGIC_HAS_1N == 0U); + GICD_TYPER_set_No1N(&typer, VGIC_HAS_1N == 0); *val = GICD_TYPER_raw(typer); - } else if (offset == OFS_GICD_IIDR) { + } else if (offset == offsetof(gicd_t, iidr)) { GICD_IIDR_t iidr = GICD_IIDR_default(); GICD_IIDR_set_Implementer(&iidr, IIDR_IMPLEMENTER); GICD_IIDR_set_ProductID(&iidr, IIDR_PRODUCTID); @@ -302,11 +320,14 @@ gicd_vdevice_read(size_t offset, register_t *val, size_t access_size) GICD_IIDR_set_Revision(&iidr, IIDR_REVISION); *val = GICD_IIDR_raw(iidr); - } else if (offset == OFS_GICD_TYPER2) { + } else if (offset == offsetof(gicd_t, typer2)) { GICD_TYPER2_t typer2 = GICD_TYPER2_default(); +#if GICV3_HAS_VLPI_V4_1 + GICD_TYPER2_set_nASSGIcap(&typer2, vgic_has_lpis(vic)); +#endif *val = GICD_TYPER2_raw(typer2); - } else if (offset == OFS_GICD_PIDR2) { + } else if (offset == (size_t)OFS_GICD_PIDR2) { *val = VGIC_PIDR2; } else if ((offset >= OFS_GICD_IGROUPR(0U)) && @@ -368,49 +389,46 @@ gicd_vdevice_read(size_t offset, register_t *val, size_t access_size) *val = 0U; } -out: return ret; } static bool -gicd_vdevice_write(size_t offset, register_t val, size_t access_size) +gicd_vdevice_write(vic_t *vic, size_t offset, register_t val, + size_t access_size) { - bool ret = true; - thread_t *thread = thread_get_self(); - vic_t *vic = thread->vgic_vic; + bool ret = true; - if (vic == NULL) { - ret = false; - goto out; - } + assert(vic != NULL); VGIC_TRACE(GICD_WRITE, vic, NULL, "GICD_WRITE reg = {:x}, val = {:#x}", offset, val); - if (offset == OFS_GICD_CTLR) { + if (offset == offsetof(gicd_t, ctlr)) { vgic_gicd_set_control(vic, GICD_CTLR_DS_cast((uint32_t)val)); - } else if ((offset == OFS_GICD_TYPER) || (offset == OFS_GICD_IIDR) || - (offset == OFS_GICD_PIDR2) || (offset == OFS_GICD_TYPER2)) { + } else if ((offset == offsetof(gicd_t, typer)) || + (offset == offsetof(gicd_t, iidr)) || + (offset == (size_t)OFS_GICD_PIDR2) || + (offset == offsetof(gicd_t, typer2))) { // RO registers GICD_STATUSR_t statusr; GICD_STATUSR_init(&statusr); GICD_STATUSR_set_WROD(&statusr, true); vgic_gicd_set_statusr(vic, statusr, true); - } else if (offset == OFS_GICD_STATUSR) { + } else if (offset == offsetof(gicd_t, statusr)) { GICD_STATUSR_t statusr = GICD_STATUSR_cast((uint32_t)val); vgic_gicd_set_statusr(vic, statusr, false); - } else if ((offset == OFS_GICD_SETSPI_NSR) || - (offset == OFS_GICD_CLRSPI_NSR)) { + } else if ((offset == offsetof(gicd_t, setspi_nsr)) || + (offset == offsetof(gicd_t, clrspi_nsr))) { vgic_gicd_change_irq_pending( vic, GICD_CLRSPI_SETSPI_NSR_SR_get_INTID( &GICD_CLRSPI_SETSPI_NSR_SR_cast((uint32_t)val)), - offset == OFS_GICD_SETSPI_NSR, true); + (offset == offsetof(gicd_t, setspi_nsr)), true); - } else if ((offset == OFS_GICD_SETSPI_SR) || - (offset == OFS_GICD_CLRSPI_SR)) { + } else if ((offset == offsetof(gicd_t, setspi_sr)) || + (offset == offsetof(gicd_t, clrspi_sr))) { // WI } else if ((offset >= OFS_GICD_IGROUPR(0U)) && @@ -421,8 +439,8 @@ gicd_vdevice_write(size_t offset, register_t val, size_t access_size) sizeof(uint32_t)); for (index_t i = util_max(n * 32U, GIC_SPI_BASE); i < util_min((n + 1U) * 32U, 1020U); i++) { - vgic_gicd_set_irq_group(vic, i, - (val & util_bit(i % 32)) != 0U); + vgic_gicd_set_irq_group( + vic, i, (val & util_bit(i % 32U)) != 0U); } } else if ((offset >= OFS_GICD_ISENABLER(0U)) && @@ -436,11 +454,11 @@ gicd_vdevice_write(size_t offset, register_t val, size_t access_size) uint32_t bits = (uint32_t)val; if (n == 31U) { // Ignore the bits for IRQs 1020-1023 - bits &= ~0xf0000000; + bits &= ~0xf0000000U; } while (bits != 0U) { index_t i = compiler_ctz(bits); - bits &= ~util_bit(i); + bits &= ~((index_t)util_bit(i)); vgic_gicd_change_irq_enable(vic, (n * 32U) + i, true); @@ -458,11 +476,11 @@ gicd_vdevice_write(size_t offset, register_t val, size_t access_size) uint32_t bits = (uint32_t)val; if (n == 31U) { // Ignore the bits for IRQs 1020-1023 - bits &= ~0xf0000000; + bits &= ~0xf0000000U; } while (bits != 0U) { index_t i = compiler_ctz(bits); - bits &= ~util_bit(i); + bits &= ~((index_t)util_bit(i)); vgic_gicd_change_irq_enable(vic, (n * 32U) + i, false); @@ -480,11 +498,11 @@ gicd_vdevice_write(size_t offset, register_t val, size_t access_size) uint32_t bits = (uint32_t)val; if (n == 31U) { // Ignore the bits for IRQs 1020-1023 - bits &= ~0xf0000000; + bits &= ~0xf0000000U; } while (bits != 0U) { index_t i = compiler_ctz(bits); - bits &= ~util_bit(i); + bits &= ~((index_t)util_bit(i)); vgic_gicd_change_irq_pending(vic, (n * 32U) + i, true, false); @@ -502,11 +520,11 @@ gicd_vdevice_write(size_t offset, register_t val, size_t access_size) uint32_t bits = (uint32_t)val; if (n == 31U) { // Ignore the bits for IRQs 1020-1023 - bits &= ~0xf0000000; + bits &= ~0xf0000000U; } while (bits != 0U) { index_t i = compiler_ctz(bits); - bits &= ~util_bit(i); + bits &= ~((index_t)util_bit(i)); vgic_gicd_change_irq_pending(vic, (n * 32U) + i, false, false); @@ -524,11 +542,11 @@ gicd_vdevice_write(size_t offset, register_t val, size_t access_size) uint32_t bits = (uint32_t)val; if (n == 31U) { // Ignore the bits for IRQs 1020-1023 - bits &= ~0xf0000000; + bits &= ~0xf0000000U; } while (bits != 0U) { index_t i = compiler_ctz(bits); - bits &= ~util_bit(i); + bits &= ~((index_t)util_bit(i)); vgic_gicd_change_irq_active(vic, (n * 32U) + i, true); @@ -546,11 +564,11 @@ gicd_vdevice_write(size_t offset, register_t val, size_t access_size) uint32_t bits = (uint32_t)val; if (n == 31U) { // Ignore the bits for IRQs 1020-1023 - bits &= ~0xf0000000; + bits &= ~0xf0000000U; } while (bits != 0U) { index_t i = compiler_ctz(bits); - bits &= ~util_bit(i); + bits &= ~((index_t)util_bit(i)); vgic_gicd_change_irq_active(vic, (n * 32U) + i, false); @@ -582,7 +600,7 @@ gicd_vdevice_write(size_t offset, register_t val, size_t access_size) sizeof(uint32_t)); // Ignore writes to the SGI and PPI bits for (index_t i = util_max(n * 16U, GIC_SPI_BASE); - i < util_min((n + 1U) * 16U, 1020); i++) { + i < util_min((n + 1U) * 16U, 1020U); i++) { vgic_gicd_set_irq_config( vic, i, (val & util_bit(((i % 16U) * 2U) + 1U)) != 0U); @@ -596,7 +614,7 @@ gicd_vdevice_write(size_t offset, register_t val, size_t access_size) (offset <= OFS_GICD_NSACR(63U))) { // WI - } else if (offset == OFS_GICD_SGIR) { + } else if (offset == offsetof(gicd_t, sgir)) { // WI } else if ((offset >= OFS_GICD_CPENDSGIR(0U)) && @@ -656,7 +674,6 @@ gicd_vdevice_write(size_t offset, register_t val, size_t access_size) ret = false; } -out: return ret; } @@ -685,8 +702,8 @@ gicd_access_allowed(size_t size, size_t offset) } else if (size == sizeof(uint16_t)) { // Half-word accesses are only allowed for the SETSPI and CLRSPI // registers - ret = ((offset == OFS_GICD_SETSPI_NSR) || - (offset == OFS_GICD_CLRSPI_NSR)); + ret = ((offset == offsetof(gicd_t, setspi_nsr)) || + (offset == offsetof(gicd_t, clrspi_nsr))); } else if (size == sizeof(uint8_t)) { // Byte accesses are only allowed for priority, target and // SGI pending registers @@ -721,10 +738,10 @@ gicr_vdevice_read(vic_t *vic, thread_t *gicr_vcpu, index_t gicr_num, (void)vic; - if ((offset == OFS_GICR_RD_SETLPIR) || - (offset == OFS_GICR_RD_CLRLPIR) || - (offset == OFS_GICR_RD_INVLPIR) || - (offset == OFS_GICR_RD_INVALLR)) { + if ((offset == offsetof(gicr_t, rd.setlpir)) || + (offset == offsetof(gicr_t, rd.clrlpir)) || + (offset == offsetof(gicr_t, rd.invlpir)) || + (offset == offsetof(gicr_t, rd.invallr))) { // WO registers, RAZ GICR_STATUSR_t statusr; GICR_STATUSR_init(&statusr); @@ -733,15 +750,21 @@ gicr_vdevice_read(vic_t *vic, thread_t *gicr_vcpu, index_t gicr_num, *val = 0U; } else if (util_balign_down(offset, sizeof(GICR_TYPER_t)) == - OFS_GICR_RD_TYPER) { - psci_mpidr_t route_id = - platform_cpu_index_to_mpidr((cpu_index_t)gicr_num); - + offsetof(gicr_t, rd.typer)) { GICR_TYPER_t typer = GICR_TYPER_default(); - GICR_TYPER_set_Aff0(&typer, psci_mpidr_get_Aff0(&route_id)); - GICR_TYPER_set_Aff1(&typer, psci_mpidr_get_Aff1(&route_id)); - GICR_TYPER_set_Aff2(&typer, psci_mpidr_get_Aff2(&route_id)); - GICR_TYPER_set_Aff3(&typer, psci_mpidr_get_Aff3(&route_id)); + GICR_TYPER_set_Aff0( + &typer, + MPIDR_EL1_get_Aff0(&gicr_vcpu->vcpu_regs_mpidr_el1)); + GICR_TYPER_set_Aff1( + &typer, + MPIDR_EL1_get_Aff1(&gicr_vcpu->vcpu_regs_mpidr_el1)); + GICR_TYPER_set_Aff2( + &typer, + MPIDR_EL1_get_Aff2(&gicr_vcpu->vcpu_regs_mpidr_el1)); + GICR_TYPER_set_Aff3( + &typer, + MPIDR_EL1_get_Aff3(&gicr_vcpu->vcpu_regs_mpidr_el1)); + // The last bit must indicate whether this is the last GICR in a // contiguous range. This is true either if it is at the end of // the VGIC's array, or if the next entry in the array is NULL. @@ -752,15 +775,18 @@ gicr_vdevice_read(vic_t *vic, thread_t *gicr_vcpu, index_t gicr_num, &vic->gicr_vcpus[gicr_num + 1U]) == NULL)); GICR_TYPER_set_Processor_Num(&typer, gicr_num); +#if VGIC_HAS_LPI + GICR_TYPER_set_PLPIS(&typer, vgic_has_lpis(vic)); +#endif *val = GICR_TYPER_raw(typer); - if (offset != OFS_GICR_RD_TYPER) { + if (offset != offsetof(gicr_t, rd.typer)) { // Must be a 32-bit access to the big end assert(offset == OFS_GICR_RD_TYPER + sizeof(uint32_t)); *val >>= 32U; } - } else if (offset == OFS_GICR_RD_IIDR) { + } else if (offset == offsetof(gicr_t, rd.iidr)) { GICR_IIDR_t iidr = GICR_IIDR_default(); GICR_IIDR_set_Implementer(&iidr, IIDR_IMPLEMENTER); GICR_IIDR_set_ProductID(&iidr, IIDR_PRODUCTID); @@ -768,17 +794,17 @@ gicr_vdevice_read(vic_t *vic, thread_t *gicr_vcpu, index_t gicr_num, GICR_IIDR_set_Revision(&iidr, IIDR_REVISION); *val = GICR_IIDR_raw(iidr); - } else if (offset == OFS_GICR_PIDR2) { + } else if (offset == offsetof(gicr_t, PIDR2)) { *val = VGIC_PIDR2; - } else if (offset == OFS_GICR_RD_CTLR) { + } else if (offset == offsetof(gicr_t, rd.ctlr)) { *val = GICR_CTLR_raw(vgic_gicr_rd_get_control(vic, gicr_vcpu)); - } else if (offset == OFS_GICR_RD_STATUSR) { + } else if (offset == offsetof(gicr_t, rd.statusr)) { *val = GICR_STATUSR_raw( atomic_load_relaxed(&gicr_vcpu->vgic_gicr_rd_statusr)); - } else if (offset == OFS_GICR_RD_WAKER) { + } else if (offset == offsetof(gicr_t, rd.waker)) { GICR_WAKER_t gicr_waker = GICR_WAKER_default(); GICR_WAKER_set_ProcessorSleep( &gicr_waker, @@ -788,27 +814,50 @@ gicr_vdevice_read(vic_t *vic, thread_t *gicr_vcpu, index_t gicr_num, *val = GICR_WAKER_raw(gicr_waker); - } else if (offset == OFS_GICR_RD_PROPBASER) { + } else if (offset == offsetof(gicr_t, rd.propbaser)) { +#if VGIC_HAS_LPI + *val = GICR_PROPBASER_raw( + atomic_load_relaxed(&vic->gicr_rd_propbaser)); +#else *val = 0U; +#endif - } else if (offset == OFS_GICR_RD_PENDBASER) { + } else if (offset == offsetof(gicr_t, rd.pendbaser)) { +#if VGIC_HAS_LPI + GICR_PENDBASER_t pendbase = + atomic_load_relaxed(&gicr_vcpu->vgic_gicr_rd_pendbaser); + // The PTZ bit is specified as WO/RAZ, but we use it to cache + // the written value which is used when EnableLPIs is set to 1. + // Therefore we must clear it here. + GICR_PENDBASER_set_PTZ(&pendbase, false); + *val = GICR_PENDBASER_raw(pendbase); +#else *val = 0U; +#endif - } else if (offset == OFS_GICR_RD_SYNCR) { + } else if (offset == offsetof(gicr_t, rd.syncr)) { +#if VGIC_HAS_LPI + GICR_SYNCR_t syncr = GICR_SYNCR_default(); + GICR_SYNCR_set_Busy(&syncr, + vgic_gicr_get_inv_pending(vic, gicr_vcpu)); + *val = GICR_SYNCR_raw(syncr); +#else *val = 0U; +#endif - } else if ((offset == OFS_GICR_SGI_IGROUPR0) || - (offset == OFS_GICR_SGI_ISENABLER0) || - (offset == OFS_GICR_SGI_ICENABLER0) || - (offset == OFS_GICR_SGI_ISPENDR0) || - (offset == OFS_GICR_SGI_ICPENDR0) || - (offset == OFS_GICR_SGI_ISACTIVER0) || - (offset == OFS_GICR_SGI_ICACTIVER0)) { - *val = vgic_read_irqbits(vic, gicr_vcpu, offset - OFS_GICR_SGI, - offset - OFS_GICR_SGI); - - } else if ((offset == OFS_GICR_SGI_IGRPMODR0) || - (offset == OFS_GICR_SGI_NSACR)) { + } else if ((offset == offsetof(gicr_t, sgi.igroupr0)) || + (offset == offsetof(gicr_t, sgi.isenabler0)) || + (offset == offsetof(gicr_t, sgi.icenabler0)) || + (offset == offsetof(gicr_t, sgi.ispendr0)) || + (offset == offsetof(gicr_t, sgi.icpendr0)) || + (offset == offsetof(gicr_t, sgi.isactiver0)) || + (offset == offsetof(gicr_t, sgi.icactiver0))) { + *val = vgic_read_irqbits(vic, gicr_vcpu, + offset - offsetof(gicr_t, sgi), + offset - offsetof(gicr_t, sgi)); + + } else if ((offset == offsetof(gicr_t, sgi.igrpmodr0)) || + (offset == offsetof(gicr_t, sgi.nsacr))) { // RAZ/WI because GICD_CTLR.DS==1 *val = 0U; @@ -842,25 +891,25 @@ gicr_vdevice_write(vic_t *vic, thread_t *gicr_vcpu, size_t offset, VGIC_TRACE(GICR_WRITE, vic, gicr_vcpu, "GICR_WRITE reg = {:x}, val = {:#x}", offset, val); - if (offset == OFS_GICR_RD_CTLR) { + if (offset == offsetof(gicr_t, rd.ctlr)) { vgic_gicr_rd_set_control(vic, gicr_vcpu, GICR_CTLR_cast((uint32_t)val)); - } else if ((offset == OFS_GICR_RD_IIDR) || - (offset == OFS_GICR_RD_TYPER) || - (offset == OFS_GICR_RD_SYNCR) || - (offset == OFS_GICR_PIDR2)) { + } else if ((offset == offsetof(gicr_t, rd.iidr)) || + (offset == offsetof(gicr_t, rd.typer)) || + (offset == offsetof(gicr_t, rd.syncr)) || + (offset == offsetof(gicr_t, PIDR2))) { // RO registers GICR_STATUSR_t statusr; GICR_STATUSR_init(&statusr); GICR_STATUSR_set_WROD(&statusr, true); vgic_gicr_rd_set_statusr(gicr_vcpu, statusr, true); - } else if (offset == OFS_GICR_RD_STATUSR) { + } else if (offset == offsetof(gicr_t, rd.statusr)) { GICR_STATUSR_t statusr = GICR_STATUSR_cast((uint32_t)val); vgic_gicr_rd_set_statusr(gicr_vcpu, statusr, false); - } else if (offset == OFS_GICR_RD_WAKER) { + } else if (offset == offsetof(gicr_t, rd.waker)) { bool new_sleep = GICR_WAKER_get_ProcessorSleep( &GICR_WAKER_cast((uint32_t)val)); #if VGIC_HAS_1N @@ -877,58 +926,82 @@ gicr_vdevice_write(vic_t *vic, thread_t *gicr_vcpu, size_t offset, atomic_store_relaxed(&gicr_vcpu->vgic_sleep, new_sleep); #endif - } else if ((offset == OFS_GICR_RD_SETLPIR) || - (offset == OFS_GICR_RD_CLRLPIR)) { + } else if ((offset == offsetof(gicr_t, rd.setlpir)) || + (offset == offsetof(gicr_t, rd.clrlpir))) { // Direct LPIs not implemented, WI // // Implementing these is strictly required by the GICv3 spec // when the VCPU has LPI support but no ITS. We define that to // be a configuration error in VM provisioning. - } else if (offset == OFS_GICR_SGI_IGROUPR0) { +#if VGIC_HAS_LPI + } else if (offset == offsetof(gicr_t, rd.propbaser)) { + vgic_gicr_rd_set_propbase(vic, GICR_PROPBASER_cast(val)); + + } else if (offset == offsetof(gicr_t, rd.pendbaser)) { + vgic_gicr_rd_set_pendbase(vic, gicr_vcpu, + GICR_PENDBASER_cast(val)); + + } else if (offset == offsetof(gicr_t, rd.invlpir)) { + GICR_INVLPIR_t invlpir = GICR_INVLPIR_cast(val); + // WI if the virtual bit is set + if (!GICR_INVLPIR_get_V(&invlpir)) { + vgic_gicr_rd_invlpi(vic, gicr_vcpu, + GICR_INVLPIR_get_pINTID(&invlpir)); + } + + } else if (offset == offsetof(gicr_t, rd.invallr)) { + GICR_INVALLR_t invallr = GICR_INVALLR_cast(val); + // WI if the virtual bit is set + if (!GICR_INVALLR_get_V(&invallr)) { + vgic_gicr_rd_invall(vic, gicr_vcpu); + } +#endif // VGIC_HAS_LPI + + } else if (offset == offsetof(gicr_t, sgi.igroupr0)) { // 32-bit register, 32-bit access only for (index_t i = 0U; i < 32U; i++) { vgic_gicr_sgi_set_sgi_ppi_group( vic, gicr_vcpu, i, (val & util_bit(i)) != 0U); } - } else if ((offset == OFS_GICR_SGI_ISENABLER0) || - (offset == OFS_GICR_SGI_ICENABLER0)) { + } else if ((offset == offsetof(gicr_t, sgi.isenabler0)) || + (offset == offsetof(gicr_t, sgi.icenabler0))) { // 32-bit registers, 32-bit access only uint32_t bits = (uint32_t)val; while (bits != 0U) { index_t i = compiler_ctz(bits); - bits &= ~util_bit(i); + bits &= ~((index_t)util_bit(i)); vgic_gicr_sgi_change_sgi_ppi_enable( vic, gicr_vcpu, i, - offset == OFS_GICR_SGI_ISENABLER0); + (offset == offsetof(gicr_t, sgi.isenabler0))); } - } else if ((offset == OFS_GICR_SGI_ISPENDR0) || - (offset == OFS_GICR_SGI_ICPENDR0)) { + } else if ((offset == offsetof(gicr_t, sgi.ispendr0)) || + (offset == offsetof(gicr_t, sgi.icpendr0))) { // 32-bit registers, 32-bit access only uint32_t bits = (uint32_t)val; while (bits != 0U) { index_t i = compiler_ctz(bits); - bits &= ~util_bit(i); + bits &= ~((index_t)util_bit(i)); vgic_gicr_sgi_change_sgi_ppi_pending( vic, gicr_vcpu, i, - offset == OFS_GICR_SGI_ISPENDR0); + (offset == offsetof(gicr_t, sgi.ispendr0))); } - } else if ((offset == OFS_GICR_SGI_ISACTIVER0) || - (offset == OFS_GICR_SGI_ICACTIVER0)) { + } else if ((offset == offsetof(gicr_t, sgi.isactiver0)) || + (offset == offsetof(gicr_t, sgi.icactiver0))) { // 32-bit registers, 32-bit access only uint32_t bits = (uint32_t)val; while (bits != 0U) { index_t i = compiler_ctz(bits); - bits &= ~util_bit(i); + bits &= ~((index_t)util_bit(i)); vgic_gicr_sgi_change_sgi_ppi_active( vic, gicr_vcpu, i, - offset == OFS_GICR_SGI_ISACTIVER0); + (offset == offsetof(gicr_t, sgi.isactiver0))); } } else if ((offset >= OFS_GICR_SGI_IPRIORITYR(0U)) && @@ -956,10 +1029,10 @@ gicr_vdevice_write(vic_t *vic, thread_t *gicr_vcpu, size_t offset, (val & util_bit((i * 2U) + 1U)) != 0U); } - } else if (offset == OFS_GICR_SGI_IGRPMODR0) { + } else if (offset == offsetof(gicr_t, sgi.igrpmodr0)) { // WI - } else if (offset == OFS_GICR_SGI_NSACR) { + } else if (offset == offsetof(gicr_t, sgi.nsacr)) { // WI } @@ -987,13 +1060,13 @@ gicr_access_allowed(size_t size, size_t offset) if ((offset & (size - 1U)) != 0UL) { ret = false; } else if (size == sizeof(uint64_t)) { - ret = ((offset == OFS_GICR_RD_INVALLR) || - (offset <= OFS_GICR_RD_INVLPIR) || - (offset == OFS_GICR_RD_PENDBASER) || - (offset == OFS_GICR_RD_PROPBASER) || - (offset == OFS_GICR_RD_SETLPIR) || - (offset == OFS_GICR_RD_CLRLPIR) || - (offset == OFS_GICR_RD_TYPER)); + ret = ((offset == offsetof(gicr_t, rd.invallr)) || + (offset <= offsetof(gicr_t, rd.invlpir)) || + (offset == offsetof(gicr_t, rd.pendbaser)) || + (offset == offsetof(gicr_t, rd.propbaser)) || + (offset == offsetof(gicr_t, rd.setlpir)) || + (offset == offsetof(gicr_t, rd.clrlpir)) || + (offset == offsetof(gicr_t, rd.typer))); } else if (size == sizeof(uint32_t)) { // Word accesses, always allowed ret = true; @@ -1012,38 +1085,93 @@ gicr_access_allowed(size_t size, size_t offset) return ret; } -bool -vgic_handle_vdevice_access(vmaddr_t ipa, size_t access_size, register_t *value, - bool is_write) +static vcpu_trap_result_t +vgic_handle_gicd_access(vic_t *vic, size_t offset, size_t access_size, + register_t *value, bool is_write) { - bool ret; + bool access_ok = false; - if ((ipa >= PLATFORM_GICD_BASE) && - (ipa < PLATFORM_GICD_BASE + 0x10000U)) { - size_t offset = (size_t)(ipa - PLATFORM_GICD_BASE); + if (gicd_access_allowed(access_size, offset)) { + if (is_write) { + access_ok = gicd_vdevice_write(vic, offset, *value, + access_size); + } else { + access_ok = gicd_vdevice_read(vic, offset, value, + access_size); + } + } + return access_ok ? VCPU_TRAP_RESULT_EMULATED : VCPU_TRAP_RESULT_FAULT; +} - if (gicd_access_allowed(access_size, offset)) { - if (is_write) { - ret = gicd_vdevice_write(offset, *value, - access_size); - } else { - ret = gicd_vdevice_read(offset, value, - access_size); - } +static vcpu_trap_result_t +vgic_handle_gicr_access(vic_t *vic, thread_t *thread, size_t offset, + size_t access_size, register_t *value, bool is_write) +{ + bool access_ok = false; + + if (gicr_access_allowed(access_size, offset)) { + if (is_write) { + access_ok = gicr_vdevice_write(vic, thread, offset, + *value, access_size); } else { - ret = false; + access_ok = gicr_vdevice_read(vic, thread, + thread->vgic_gicr_index, + offset, value, + access_size); } + } + + return access_ok ? VCPU_TRAP_RESULT_EMULATED : VCPU_TRAP_RESULT_FAULT; +} + +vcpu_trap_result_t +vgic_handle_vdevice_access(vdevice_type_t type, vdevice_t *vdevice, + size_t offset, size_t access_size, register_t *value, + bool is_write) +{ + assert(vdevice != NULL); + + vcpu_trap_result_t ret; + + if (type == VDEVICE_TYPE_VGIC_GICD) { + vic_t *vic = vic_container_of_gicd_device(vdevice); + ret = vgic_handle_gicd_access(vic, offset, access_size, value, + is_write); + } else { + assert(type == VDEVICE_TYPE_VGIC_GICR); + thread_t *gicr_vcpu = + thread_container_of_vgic_gicr_device(vdevice); + vic_t *vic = gicr_vcpu->vgic_vic; + assert(vic != NULL); + ret = vgic_handle_gicr_access(vic, gicr_vcpu, offset, + access_size, value, is_write); + } + + return ret; +} + +vcpu_trap_result_t +vgic_handle_vdevice_access_fixed_addr(vmaddr_t ipa, size_t access_size, + register_t *value, bool is_write) +{ + vcpu_trap_result_t ret; + + thread_t *thread = thread_get_self(); + vic_t *vic = thread->vgic_vic; + + if ((vic == NULL) || !vic->allow_fixed_vmaddr) { + ret = VCPU_TRAP_RESULT_UNHANDLED; + } else if ((ipa >= PLATFORM_GICD_BASE) && + (ipa < PLATFORM_GICD_BASE + 0x10000U)) { + size_t offset = (size_t)(ipa - PLATFORM_GICD_BASE); + ret = vgic_handle_gicd_access(vic, offset, access_size, value, + is_write); } else if ((ipa >= PLATFORM_GICR_BASE) && - (ipa < PLATFORM_GICR_BASE + - (PLATFORM_MAX_CORES << GICR_STRIDE_SHIFT))) { + (ipa < PLATFORM_GICR_BASE + ((vmaddr_t)PLATFORM_MAX_CORES + << GICR_STRIDE_SHIFT))) { index_t gicr_num = (index_t)((ipa - PLATFORM_GICR_BASE) >> GICR_STRIDE_SHIFT); - vic_t *vic = thread_get_self()->vgic_vic; - if (vic == NULL) { - ret = false; - } else if (gicr_num >= vic->gicr_count) { - ret = false; - } else { + if ((vic != NULL) && (gicr_num < vic->gicr_count)) { rcu_read_start(); thread_t *gicr_vcpu = @@ -1051,32 +1179,24 @@ vgic_handle_vdevice_access(vmaddr_t ipa, size_t access_size, register_t *value, if (gicr_vcpu != NULL) { vmaddr_t gicr_base = - PLATFORM_GICR_BASE + - (gicr_num << GICR_STRIDE_SHIFT); + ((vmaddr_t)PLATFORM_GICR_BASE + + ((vmaddr_t)gicr_num + << GICR_STRIDE_SHIFT)); size_t offset = (size_t)(ipa - gicr_base); - - if (gicr_access_allowed(access_size, offset)) { - if (is_write) { - ret = gicr_vdevice_write( - vic, gicr_vcpu, offset, - *value, access_size); - } else { - ret = gicr_vdevice_read( - vic, gicr_vcpu, - gicr_num, offset, value, - access_size); - } - } else { - ret = false; - } + ret = vgic_handle_gicr_access(vic, gicr_vcpu, + offset, + access_size, + value, is_write); } else { - ret = false; + ret = VCPU_TRAP_RESULT_UNHANDLED; } rcu_read_finish(); + } else { + ret = VCPU_TRAP_RESULT_UNHANDLED; } } else { - ret = false; + ret = VCPU_TRAP_RESULT_UNHANDLED; } return ret; diff --git a/hyp/vm/vgic/src/vgic.c b/hyp/vm/vgic/src/vgic.c new file mode 100644 index 0000000..c46be21 --- /dev/null +++ b/hyp/vm/vgic/src/vgic.c @@ -0,0 +1,20 @@ +// © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include + +#include +#include + +vic_t * +vic_get_vic(const thread_t *vcpu) +{ + return vcpu->vgic_vic; +} + +const platform_mpidr_mapping_t * +vgic_get_mpidr_mapping(const vic_t *vic) +{ + return &vic->mpidr_mapping; +} diff --git a/hyp/vm/vgic/src/vpe.c b/hyp/vm/vgic/src/vpe.c new file mode 100644 index 0000000..48c2033 --- /dev/null +++ b/hyp/vm/vgic/src/vpe.c @@ -0,0 +1,99 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include +#include +#include + +#include "event_handlers.h" +#include "gicv3.h" +#include "internal.h" + +#if VGIC_HAS_LPI && GICV3_HAS_VLPI + +error_t +vgic_handle_thread_context_switch_pre(void) +{ + thread_t *current = thread_get_self(); + + if (current->vgic_vic != NULL) { + bool expects_wakeup = vcpu_expects_wakeup(current); + if (gicv3_vpe_deschedule(expects_wakeup)) { + scheduler_lock_nopreempt(current); + vcpu_wakeup(current); + scheduler_unlock_nopreempt(current); + } + } + + return OK; +} + +void +vgic_handle_thread_load_state_vpe(void) +{ + thread_t *current = thread_get_self(); + + if (current->vgic_vic != NULL) { + vgic_vpe_schedule_current(); + } +} + +// Deschedule the vPE while blocked in EL2 / EL3. +// +// Note that vgic_vpe_schedule_current() is directly registered as both the +// unwinder for this event and the handler for vcpu_block_finish. +bool +vgic_handle_vcpu_block_start(void) +{ + bool wakeup = false; + + if (gicv3_vpe_deschedule(true)) { + wakeup = true; + vgic_vpe_schedule_current(); + } + + return wakeup; +} + +void +vgic_vpe_schedule_current(void) +{ + thread_t *current = thread_get_self(); + assert(current->kind == THREAD_KIND_VCPU); + + assert(current->vgic_vic != NULL); + + // While it is not especially clear from the spec, it seems that + // these two enable bits must be set specifically to the GICD_CTLR + // enable bits, without being masked by the ICV bits. + // + // This is because GIC-700 has been observed dropping any + // vSGI targeted to a disabled group on a scheduled vPE, and + // might do so for vLPIs too. This is allowed for a group + // disabled by GICD_CTLR, but not for a group disabled by + // ICV_IGRPEN*. + GICD_CTLR_DS_t gicd_ctlr = + atomic_load_acquire(¤t->vgic_vic->gicd_ctlr); + gicv3_vpe_schedule(GICD_CTLR_DS_get_EnableGrp0(&gicd_ctlr), + GICD_CTLR_DS_get_EnableGrp1(&gicd_ctlr)); +} + +vcpu_trap_result_t +vgic_vpe_handle_vcpu_trap_wfi(void) +{ + // FIXME: + return gicv3_vpe_check_wakeup(true) ? VCPU_TRAP_RESULT_RETRY + : VCPU_TRAP_RESULT_UNHANDLED; +} + +bool +vgic_vpe_handle_vcpu_pending_wakeup(void) +{ + return gicv3_vpe_check_wakeup(false); +} + +#endif // VGIC_HAS_LPI && GICV3_HAS_VLPI diff --git a/hyp/vm/vgic/vgic.ev b/hyp/vm/vgic/vgic.ev index bd95c51..582ba83 100644 --- a/hyp/vm/vgic/vgic.ev +++ b/hyp/vm/vgic/vgic.ev @@ -15,6 +15,8 @@ subscribe rootvm_init // which is priority 10. priority 1 +subscribe rootvm_init_late(root_thread, hyp_env) + subscribe object_create_vic priority last @@ -31,11 +33,13 @@ subscribe object_deactivate_hwirq subscribe object_create_thread subscribe object_activate_thread - unwinder vgic_handle_object_deactivate_thread(thread) + unwinder vgic_unwind_object_activate_thread(thread) // Run early so other modules (timer, etc) can bind to virtual PPIs. // Must run after PSCI, which is priority 10. priority 1 +subscribe object_deactivate_thread + subscribe object_cleanup_thread(thread) subscribe irq_received[HWIRQ_ACTION_VGIC_FORWARD_SPI] @@ -84,8 +88,13 @@ subscribe thread_load_state() subscribe scheduler_affinity_changed(thread, next_cpu) +subscribe addrspace_attach_vdevice + +subscribe vdevice_access[VDEVICE_TYPE_VGIC_GICD] + +subscribe vdevice_access[VDEVICE_TYPE_VGIC_GICR] + subscribe vdevice_access_fixed_addr - handler vgic_handle_vdevice_access // Raise priority as this is more likely to be performance-critical // than other vdevices. This can be removed once we have proper // device-kind tracking for vdevices @@ -104,4 +113,47 @@ subscribe vcpu_trap_wfi() subscribe vcpu_pending_wakeup -subscribe vcpu_poweredoff() +subscribe vcpu_stopped() + +#if VGIC_HAS_LPI && GICV3_HAS_VLPI + +// When switching away from a VCPU, deschedule it in the GICR. +subscribe thread_context_switch_pre() + // Run early, to give vPE descheduling time to complete during the + // rest of the context switch. + priority 100 + require_preempt_disabled + +// When switching to a VCPU, schedule it in the GICR. +subscribe thread_load_state + handler vgic_handle_thread_load_state_vpe() + // Run early, to give vPE scheduling time to complete before we return + // to userspace. + priority 100 + require_preempt_disabled + +// When blocking for the VCPU, mark the vPE as descheduled. This allows us to +// check for pending vLPIs / vSGIs now, and also to receive a doorbell if one +// arrives later. +subscribe vcpu_block_start() + unwinder vgic_vpe_schedule_current() + require_preempt_disabled + +subscribe vcpu_block_finish + handler vgic_vpe_schedule_current() + require_preempt_disabled + +subscribe vcpu_pending_wakeup + handler vgic_vpe_handle_vcpu_pending_wakeup() + // Run early, because this is a relatively cheap check and might fail + // repeatedly. + priority 100 + require_preempt_disabled + +subscribe vcpu_trap_wfi + handler vgic_vpe_handle_vcpu_trap_wfi() + // Run early, because this is a relatively cheap check and might fail + // repeatedly. + priority 200 + +#endif // VGIC_HAS_LPI && GICV3_HAS_VLPI diff --git a/hyp/vm/vgic/vgic.hvc b/hyp/vm/vgic/vgic.hvc new file mode 100644 index 0000000..a1f6de9 --- /dev/null +++ b/hyp/vm/vgic/vgic.hvc @@ -0,0 +1,15 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +define vgic_set_mpidr_mapping hypercall { + call_num 0x67; + vic input type cap_id_t; + mask input uint64; + aff0_shift input type count_t; + aff1_shift input type count_t; + aff2_shift input type count_t; + aff3_shift input type count_t; + mt input bool; + error output enumeration error; +}; diff --git a/hyp/vm/vgic/vgic.tc b/hyp/vm/vgic/vgic.tc index 7827438..6c04994 100644 --- a/hyp/vm/vgic/vgic.tc +++ b/hyp/vm/vgic/vgic.tc @@ -23,7 +23,14 @@ define VGIC_LOW_RANGES constant type count_t = 64; define VGIC_LPI_BITS constant type count_t = 16; #endif define VGIC_IDBITS constant type count_t = - VGIC_HAS_EXT_IRQS ? 13 : 10; + VGIC_HAS_LPI ? VGIC_LPI_BITS : VGIC_HAS_EXT_IRQS ? 13 : 10; + +#if VGIC_HAS_LPI +define VGIC_LPI_NUM constant type count_t = (1 << VGIC_IDBITS) - GIC_LPI_BASE; +#if !GICV3_HAS_VLPI +define VGIC_LPI_RANGES constant type count_t = BITMAP_NUM_WORDS(VGIC_LPI_NUM); +#endif +#endif // Delivery state for VIRQs. // @@ -64,6 +71,7 @@ define vgic_delivery_state bitfield<32>(set_ops) { 11 hw_active bool; // Add GICR index so we don't need to broadcast sync IPIs + // FIXME: // An interrupt has been rerouted, disabled or reasserted while it is // listed on a VCPU that is current on a remote CPU. The remote CPU @@ -92,6 +100,7 @@ define vgic_delivery_state bitfield<32>(set_ops) { 3 enabled bool; // Also migrate priority and route to this bitfield. + // FIXME: // The interrupt is currently active. This is only valid for an // unlisted interrupt; for a listed interrupt the active state is in @@ -205,11 +214,11 @@ extend virq_trigger enumeration { }; extend virq_source structure module vgic { - // Index of the virtual CPU for a private binding. Invalid if shared. - gicr_index type index_t; - // Flag to protect against concurrent binding of the source. is_bound bool(atomic); + + // Index of the virtual CPU for a private binding. Invalid if shared. + gicr_index type index_t; }; // HW IRQ extensions for forwarding through the virtual GIC. @@ -240,17 +249,22 @@ extend vic object { // acquired while holding a GICR's attochment or LR ownership lock. gicd_lock structure spinlock; + // Virtual device structure representing the GICD registers. If the + // type is set to VDEVICE_TYPE_NONE, it has not been mapped. Mapping + // is protected by the gicd lock. + gicd_device structure vdevice(contained); + // The current values of the virtual GICD_CTLR and GICD_STATUSR. // Updates are serialised by gicd_lock, but GICD_CTLR is atomic so we // can read it without taking the lock. gicd_ctlr bitfield GICD_CTLR_DS(atomic); gicd_statusr bitfield GICD_STATUSR; + // The size of the vcpus array. + gicr_count type count_t; // The array of VCPUs attached to this GICD. Protected by gicd_lock. // Weak references; pointers cleared by thread deactivate. gicr_vcpus pointer pointer(atomic) object thread; - // The size of the vcpus array. - gicr_count type count_t; // The array of shared VIRQ sources attached to this GICD, indexed by // VIRQ number minus GIC_SPI_BASE. The attachment pointers are @@ -271,6 +285,52 @@ extend vic object { rr_start_point type count_t(atomic); #endif +#if VGIC_HAS_LPI + // Maximum supported ID bits. Between 14 and VIC_IDBITS if LPIs are + // enabled for this VIC; otherwise 10. + gicd_idbits type count_t; + + // Virtual GICR_PROPBASER. As per the spec we are allowed to share a + // single copy of this across all virtual GICRs, since the table it + // points to is also shared. + gicr_rd_propbaser bitfield GICR_PROPBASER(atomic); + + // Cache of the VLPI configurations. + // + // This is a pointer to an array of 8-bit configuration fields for the + // virtual LPIs. Unlike the pending bitmap, there is no reserved space + // at the start corresponding to non-LPI IRQ numbers. + // + // Updates to this cache are not protected by any lock, based on the + // assumption that parallel updates will be copying from the same + // source anyway. + vlpi_config_table pointer uint8; + + // Validity flag for the vlpi config table. + // + // This is set the first time a VCPU enables VLPIs, to indicate that + // the initial copy of the config table from the VM has been done. + // Subsequent copies are triggered by invalidate commands. + // + // The GICD lock must be held while querying this, and the lock must + // be held across the initial copy-in of the config table. + vlpi_config_valid bool; + +#if GICV3_HAS_VLPI_V4_1 + // True if we can use the ITS to deliver virtual SGIs. This is only + // possible on GICv4.1, and only if this VIC has VLPI tables. Also, + // because an ITS-delivered SGI is really an LPI and therefore has no + // active state, this could potentially break the guest; so the guest + // must enable it by setting the virtual GICD_CTLR.nASSGIreq to 1. + vsgis_enabled bool; +#endif +#endif + + // True if we should handle fixed address vdevice accesses. This is + // temporary and will be removed once we have migrated away from fixed + // addresses. + allow_fixed_vmaddr bool; + // Lock to serialise unrouted-IRQ searches. This only needs to be held // while searching, not when flagging an IRQ in the search bitmaps. search_lock structure spinlock; @@ -290,10 +350,16 @@ extend vic object { // lock must be held, to prevent the transient clearing of a bit // hiding an IRQ from a concurrent search. search_ranges_low BITMAP(VGIC_LOW_RANGES, atomic); + + // Mapping between GICR indices and 4-level affinity values for + // MPIDR, GICR_TYPER etc. If not configured explicitly, this defaults + // to being the same as the underlying hardware's mapping. + mpidr_mapping structure platform_mpidr_mapping; }; extend cap_rights_vic bitfield { 1 attach_vcpu bool; + 2 attach_vdevice bool; }; // State tracking for a list register. @@ -324,6 +390,11 @@ extend thread object module vgic { // time. gicr_index type index_t; + // Virtual device structure representing the GICR registers. If the + // type is set to VDEVICE_TYPE_NONE, it has not been mapped. Mapping + // is protected by the gicd lock. + gicr_device structure vdevice(contained); + // Physical route register that should be used to target this thread. irouter bitfield GICD_IROUTER; @@ -341,6 +412,10 @@ extend thread object module vgic { // The GICR's rd_base register state. gicr_rd_statusr bitfield GICR_STATUSR(atomic); +#if VGIC_HAS_LPI + gicr_rd_ctlr bitfield GICR_CTLR(atomic); + gicr_rd_pendbaser bitfield GICR_PENDBASER(atomic); +#endif // The ICH EL2 register state. Parts of these are exposed to userspace // through the ICV EL1 registers. These are accessed only by the owner @@ -456,6 +531,44 @@ extend thread object module vgic { search_ranges_low array(VGIC_PRIORITIES) BITMAP(VGIC_LOW_RANGES, atomic); +#if VGIC_HAS_LPI && !GICV3_HAS_VLPI + // Extended search ranges for software-implemented LPIs. + search_ranges_lpi array(VGIC_PRIORITIES) + BITMAP(VGIC_LPI_RANGES, atomic); +#error Software VLPIs are not implemented yet +#elif VGIC_HAS_LPI && GICV3_HAS_VLPI + // Cache of the VLPI pending state. + // + // This is a pointer to a bitmap with 1 << gicd_idbits bits if LPIs + // are supported by this VIC, or otherwise NULL. It must be + // 64k-aligned, due to the 16-bit shift of the physical base in the + // VMAPP command. + // + // The contents of this table are undefined when EnableLPIs is false in + // gicr_rd_ctlr. When that bit is set to true, this table is either + // entirely zeroed, or else partially copied from VM memory and the + // remainder zeroed, depending on the PTZ bit of gicr_rd_pendbaser. + vlpi_pending_table pointer uint8; + + // Sequence number of the vPE unmap. Valid only after the VCPU is + // deactivated while its EnableLPIs bit is true. + vlpi_unmap_seq type count_t; + +#if GICV3_HAS_VLPI_V4_1 + // Sequence number of the VSYNC following the initial VSGI setup. + // If this is set to ~0, the VSGI setup commands have not all been + // enqueued in the ITS yet, and software delivery must be used; if it + // is set to 0, the setup commands are known to have completed, and + // polling the ITS is not necessary. Otherwise, software delivery must + // be used until the specified sequence number is complete. + vsgi_setup_seq type count_t(atomic); + + // Sequence number of the VSYNC following the most recent VSGI clear + // enable operation. This is used to set GICR_CTLR.RWP on trapped reads. + vsgi_disable_seq type count_t; +#endif +#endif + // Cache of shifted priorities that may have nonzero bits in their // search ranges. // @@ -495,11 +608,8 @@ extend ipi_reason enumeration { vgic_sgi; }; -extend boot_env_data structure { +extend hyp_env_data structure { vic type cap_id_t; - vic_hwirq array(GIC_SPI_BASE + GIC_SPI_NUM) type cap_id_t; - vic_msi_source array(16) type cap_id_t; - gicd_base type paddr_t; gicr_base type paddr_t; gicr_stride size; @@ -528,13 +638,16 @@ define vgic_irq_type enumeration { #if VGIC_HAS_EXT_IRQS ppi_ext; spi_ext; +#endif +#if VGIC_HAS_LPI + lpi; #endif reserved; }; extend trace_class enumeration { - VGIC; - VGIC_DEBUG; + VGIC = 17; + VGIC_DEBUG = 18; }; extend trace_id enumeration { @@ -546,4 +659,11 @@ extend trace_id enumeration { VGIC_GICR_WRITE = 0x25; VGIC_SGI = 0x26; VGIC_ROUTE = 0x28; + VGIC_ICC_WRITE = 0x29; + VGIC_ASYNC_EVENT = 0x2a; +}; + +extend vdevice_type enumeration { + vgic_gicd; + vgic_gicr; }; diff --git a/hyp/vm/vic_base/include/vic_base.h b/hyp/vm/vic_base/include/vic_base.h index 1a9dd9e..9e2b710 100644 --- a/hyp/vm/vic_base/include/vic_base.h +++ b/hyp/vm/vic_base/include/vic_base.h @@ -7,7 +7,7 @@ // Configure a new virtual interrupt controller object. error_t vic_configure(vic_t *vic, count_t max_vcpus, count_t max_virqs, - count_t max_msis); + count_t max_msis, bool allow_fixed_vmaddr); // Attach a new VCPU to an active virtual interrupt controller object. error_t diff --git a/hyp/vm/vic_base/src/forward_private.c b/hyp/vm/vic_base/src/forward_private.c index 1fefbaa..2b6b1a1 100644 --- a/hyp/vm/vic_base/src/forward_private.c +++ b/hyp/vm/vic_base/src/forward_private.c @@ -31,6 +31,7 @@ #include "event_handlers.h" #include "gicv3.h" +#include "panic.h" #include "vic_base.h" static vic_private_irq_info_t * @@ -87,7 +88,8 @@ vic_unbind_private_hwirq_helper(hwirq_t *hwirq) vic_unbind(&fp->irq_info[i].source); } - list_delete_node(&vic->forward_private_list, &fp->list_node); + (void)list_delete_node(&vic->forward_private_list, + &fp->list_node); spinlock_release(&vic->forward_private_lock); @@ -103,7 +105,7 @@ vic_bind_hwirq_forward_private(vic_t *vic, hwirq_t *hwirq, virq_t virq) assert(hwirq->action == HWIRQ_ACTION_VIC_BASE_FORWARD_PRIVATE); - struct partition *partition = vic->header.partition; + partition_t *partition = vic->header.partition; assert(partition != NULL); size_t size = sizeof(vic_forward_private_t); @@ -115,7 +117,8 @@ vic_bind_hwirq_forward_private(vic_t *vic, hwirq_t *hwirq, virq_t virq) } vic_forward_private_t *fp = (vic_forward_private_t *)alloc_r.r; - memset(fp, 0, size); + (void)memset_s(fp, sizeof(*fp), 0, sizeof(*fp)); + fp->vic = object_get_vic_additional(vic); fp->virq = virq; @@ -137,7 +140,7 @@ vic_bind_hwirq_forward_private(vic_t *vic, hwirq_t *hwirq, virq_t virq) &hwirq->vic_base_forward_private, &expected, fp, memory_order_release, memory_order_relaxed)) { spinlock_release(&vic->forward_private_lock); - partition_free(partition, fp, size); + (void)partition_free(partition, fp, size); err = ERROR_DENIED; goto out; } @@ -145,12 +148,16 @@ vic_bind_hwirq_forward_private(vic_t *vic, hwirq_t *hwirq, virq_t virq) list_insert_at_tail(&vic->forward_private_list, &fp->list_node); // Bind for VCPUs that are attached to the VIC and active. - for (cpu_index_t i = 0U; (err == OK) && (i < PLATFORM_MAX_CORES); i++) { + for (cpu_index_t i = 0U; i < PLATFORM_MAX_CORES; i++) { rcu_read_start(); thread_t *vcpu = atomic_load_consume(&vic->gicr_vcpus[i]); if ((vcpu != NULL) && vcpu->forward_private_active) { err = vic_bind_private_hwirq_helper(fp, vcpu); + if (err != OK) { + rcu_read_finish(); + break; + } } rcu_read_finish(); @@ -345,7 +352,7 @@ vic_handle_free_forward_private(rcu_entry_t *entry) partition_t *partition = vic->header.partition; assert(partition != NULL); - partition_free(partition, fp, sizeof(vic_forward_private_t)); + (void)partition_free(partition, fp, sizeof(vic_forward_private_t)); object_put_vic(vic); diff --git a/hyp/vm/vic_base/src/hypercalls.c b/hyp/vm/vic_base/src/hypercalls.c index 621dbff..bd55b87 100644 --- a/hyp/vm/vic_base/src/hypercalls.c +++ b/hyp/vm/vic_base/src/hypercalls.c @@ -74,7 +74,7 @@ hypercall_vic_configure(cap_id_t vic_cap, count_t max_vcpus, count_t max_virqs, vic_option_flags_t vic_options, count_t max_msis) { error_t err; - cspace_t *cspace = cspace_get_self(); + cspace_t *cspace = cspace_get_self(); object_type_t type; object_ptr_result_t o = cspace_lookup_object_any( @@ -102,7 +102,9 @@ hypercall_vic_configure(cap_id_t vic_cap, count_t max_vcpus, count_t max_virqs, spinlock_acquire(&vic->header.lock); if (atomic_load_relaxed(&vic->header.state) == OBJECT_STATE_INIT) { - err = vic_configure(vic, max_vcpus, max_virqs, max_msis); + err = vic_configure(vic, max_vcpus, max_virqs, max_msis, + !vic_option_flags_get_disable_default_addr( + &vic_options)); } else { err = ERROR_OBJECT_STATE; } diff --git a/hyp/vm/virtio_mmio/build.conf b/hyp/vm/virtio_mmio/build.conf new file mode 100644 index 0000000..c4e04f6 --- /dev/null +++ b/hyp/vm/virtio_mmio/build.conf @@ -0,0 +1,9 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +interface virtio_mmio +local_include +types virtio_mmio.tc +events virtio_mmio.ev +source virtio_mmio.c vdevice.c hypercalls.c diff --git a/hyp/vm/virtio_mmio/include/virtio_mmio.h b/hyp/vm/virtio_mmio/include/virtio_mmio.h new file mode 100644 index 0000000..2fe12cc --- /dev/null +++ b/hyp/vm/virtio_mmio/include/virtio_mmio.h @@ -0,0 +1,21 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +error_t +virtio_mmio_configure(virtio_mmio_t *virtio_mmio, memextent_t *memextent, + count_t vqs_num); + +error_t +virtio_mmio_backend_bind_virq(virtio_mmio_t *virtio_mmio, vic_t *vic, + virq_t virq); + +void +virtio_mmio_backend_unbind_virq(virtio_mmio_t *virtio_mmio); + +error_t +virtio_mmio_frontend_bind_virq(virtio_mmio_t *virtio_mmio, vic_t *vic, + virq_t virq); + +void +virtio_mmio_frontend_unbind_virq(virtio_mmio_t *virtio_mmio); diff --git a/hyp/vm/virtio_mmio/src/hypercalls.c b/hyp/vm/virtio_mmio/src/hypercalls.c new file mode 100644 index 0000000..1426b81 --- /dev/null +++ b/hyp/vm/virtio_mmio/src/hypercalls.c @@ -0,0 +1,445 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "virtio_mmio.h" + +error_t +hypercall_virtio_mmio_configure(cap_id_t virtio_mmio_cap, + cap_id_t memextent_cap, count_t vqs_num) +{ + error_t err; + cspace_t *cspace = cspace_get_self(); + object_type_t type; + + memextent_ptr_result_t m = cspace_lookup_memextent( + cspace, memextent_cap, CAP_RIGHTS_MEMEXTENT_ATTACH); + if (compiler_unexpected(m.e != OK)) { + err = m.e; + goto out; + } + + memextent_t *memextent = m.r; + + object_ptr_result_t o = cspace_lookup_object_any( + cspace, virtio_mmio_cap, CAP_RIGHTS_GENERIC_OBJECT_ACTIVATE, + &type); + if (compiler_unexpected(o.e != OK)) { + err = o.e; + goto out_memextent_release; + } + if (type != OBJECT_TYPE_VIRTIO_MMIO) { + err = ERROR_CSPACE_WRONG_OBJECT_TYPE; + goto out_virtio_mmio_release; + } + + virtio_mmio_t *virtio_mmio = o.r.virtio_mmio; + + spinlock_acquire(&virtio_mmio->header.lock); + + if (atomic_load_relaxed(&virtio_mmio->header.state) == + OBJECT_STATE_INIT) { + err = virtio_mmio_configure(virtio_mmio, memextent, vqs_num); + } else { + err = ERROR_OBJECT_STATE; + } + + spinlock_release(&virtio_mmio->header.lock); +out_virtio_mmio_release: + object_put(type, o.r); +out_memextent_release: + object_put_memextent(memextent); +out: + return err; +} + +error_t +hypercall_virtio_mmio_backend_bind_virq(cap_id_t virtio_mmio_cap, + cap_id_t vic_cap, virq_t virq) +{ + error_t err = OK; + cspace_t *cspace = cspace_get_self(); + + virtio_mmio_ptr_result_t p = cspace_lookup_virtio_mmio( + cspace, virtio_mmio_cap, + CAP_RIGHTS_VIRTIO_MMIO_BIND_BACKEND_VIRQ); + if (compiler_unexpected(p.e != OK)) { + err = p.e; + goto out; + } + virtio_mmio_t *virtio_mmio = p.r; + + vic_ptr_result_t v = + cspace_lookup_vic(cspace, vic_cap, CAP_RIGHTS_VIC_BIND_SOURCE); + if (compiler_unexpected(v.e != OK)) { + err = v.e; + goto out_virtio_mmio_release; + } + vic_t *vic = v.r; + + err = virtio_mmio_backend_bind_virq(virtio_mmio, vic, virq); + + object_put_vic(vic); +out_virtio_mmio_release: + object_put_virtio_mmio(virtio_mmio); +out: + return err; +} + +error_t +hypercall_virtio_mmio_backend_unbind_virq(cap_id_t virtio_mmio_cap) +{ + error_t err = OK; + cspace_t *cspace = cspace_get_self(); + + virtio_mmio_ptr_result_t p = cspace_lookup_virtio_mmio( + cspace, virtio_mmio_cap, + CAP_RIGHTS_VIRTIO_MMIO_BIND_BACKEND_VIRQ); + if (compiler_unexpected(p.e != OK)) { + err = p.e; + goto out; + } + virtio_mmio_t *virtio_mmio = p.r; + + virtio_mmio_backend_unbind_virq(virtio_mmio); + + object_put_virtio_mmio(virtio_mmio); +out: + return err; +} + +error_t +hypercall_virtio_mmio_backend_assert_virq(cap_id_t virtio_mmio_cap, + uint32_t interrupt_status) +{ + error_t err = OK; + cspace_t *cspace = cspace_get_self(); + + virtio_mmio_ptr_result_t p = cspace_lookup_virtio_mmio( + cspace, virtio_mmio_cap, CAP_RIGHTS_VIRTIO_MMIO_ASSERT_VIRQ); + if (compiler_unexpected(p.e != OK)) { + err = p.e; + goto out; + } + virtio_mmio_t *virtio_mmio = p.r; + + virtio_mmio_status_reg_t status = + atomic_load_relaxed(&virtio_mmio->regs->status); + + if (virtio_mmio_status_reg_get_device_needs_reset(&status)) { + err = ERROR_DENIED; + } else { + spinlock_acquire(&virtio_mmio->lock); + atomic_store_relaxed(&virtio_mmio->regs->interrupt_status, + interrupt_status); + spinlock_release(&virtio_mmio->lock); + + atomic_thread_fence(memory_order_release); + // Assert frontend's IRQ + (void)virq_assert(&virtio_mmio->backend_source, false); + } + + object_put_virtio_mmio(virtio_mmio); +out: + return err; +} + +error_t +hypercall_virtio_mmio_frontend_bind_virq(cap_id_t virtio_mmio_cap, + cap_id_t vic_cap, virq_t virq) +{ + error_t err = OK; + cspace_t *cspace = cspace_get_self(); + + virtio_mmio_ptr_result_t p = cspace_lookup_virtio_mmio( + cspace, virtio_mmio_cap, + CAP_RIGHTS_VIRTIO_MMIO_BIND_FRONTEND_VIRQ); + if (compiler_unexpected(p.e != OK)) { + err = p.e; + goto out; + } + virtio_mmio_t *virtio_mmio = p.r; + + vic_ptr_result_t v = + cspace_lookup_vic(cspace, vic_cap, CAP_RIGHTS_VIC_BIND_SOURCE); + if (compiler_unexpected(v.e != OK)) { + err = v.e; + goto out_virtio_mmio_release; + } + vic_t *vic = v.r; + + err = virtio_mmio_frontend_bind_virq(virtio_mmio, vic, virq); + + object_put_vic(vic); +out_virtio_mmio_release: + object_put_virtio_mmio(virtio_mmio); +out: + return err; +} + +error_t +hypercall_virtio_mmio_frontend_unbind_virq(cap_id_t virtio_mmio_cap) +{ + error_t err = OK; + cspace_t *cspace = cspace_get_self(); + + virtio_mmio_ptr_result_t p = cspace_lookup_virtio_mmio( + cspace, virtio_mmio_cap, + CAP_RIGHTS_VIRTIO_MMIO_BIND_FRONTEND_VIRQ); + if (compiler_unexpected(p.e != OK)) { + err = p.e; + goto out; + } + virtio_mmio_t *virtio_mmio = p.r; + + virtio_mmio_frontend_unbind_virq(virtio_mmio); + + object_put_virtio_mmio(virtio_mmio); +out: + return err; +} + +error_t +hypercall_virtio_mmio_backend_set_dev_features(cap_id_t virtio_mmio_cap, + uint32_t sel, uint32_t dev_feat) +{ + error_t ret = OK; + cspace_t *cspace = cspace_get_self(); + + virtio_mmio_ptr_result_t p = cspace_lookup_virtio_mmio( + cspace, virtio_mmio_cap, CAP_RIGHTS_VIRTIO_MMIO_CONFIG); + if (compiler_unexpected(p.e != OK)) { + ret = p.e; + goto out; + } + virtio_mmio_t *virtio_mmio = p.r; + + index_result_t res = nospec_range_check(sel, VIRTIO_MMIO_DEV_FEAT_NUM); + if (res.e != OK) { + ret = res.e; + goto set_failed; + } + + // Check features enforced by the hypervisor + if (res.r == 1U) { + uint32_t allow = + (uint32_t)util_bit((VIRTIO_F_VERSION_1 - 32U)) | + (uint32_t)util_bit((VIRTIO_F_ACCESS_PLATFORM - 32U)) | + dev_feat; + uint32_t forbid = ~(uint32_t)util_bit( + (VIRTIO_F_NOTIFICATION_DATA - 32U)) & + dev_feat; + + if ((allow != dev_feat) || (forbid != dev_feat)) { + ret = ERROR_DENIED; + goto set_failed; + } + } + + virtio_mmio->banked_dev_feat[res.r] = dev_feat; + +set_failed: + object_put_virtio_mmio(virtio_mmio); +out: + return ret; +} + +error_t +hypercall_virtio_mmio_backend_set_queue_num_max(cap_id_t virtio_mmio_cap, + uint32_t sel, + uint32_t queue_num_max) +{ + error_t ret = OK; + cspace_t *cspace = cspace_get_self(); + + virtio_mmio_ptr_result_t p = cspace_lookup_virtio_mmio( + cspace, virtio_mmio_cap, CAP_RIGHTS_VIRTIO_MMIO_CONFIG); + if (compiler_unexpected(p.e != OK)) { + ret = p.e; + goto out; + } + virtio_mmio_t *virtio_mmio = p.r; + + index_result_t res = nospec_range_check(sel, virtio_mmio->vqs_num); + if (res.e == OK) { + virtio_mmio->banked_queue_regs[res.r].num_max = queue_num_max; + } else { + ret = res.e; + } + + object_put_virtio_mmio(virtio_mmio); +out: + return ret; +} + +hypercall_virtio_mmio_backend_get_drv_features_result_t +hypercall_virtio_mmio_backend_get_drv_features(cap_id_t virtio_mmio_cap, + uint32_t sel) +{ + hypercall_virtio_mmio_backend_get_drv_features_result_t ret = { 0 }; + cspace_t *cspace = cspace_get_self(); + + virtio_mmio_ptr_result_t p = cspace_lookup_virtio_mmio( + cspace, virtio_mmio_cap, CAP_RIGHTS_VIRTIO_MMIO_CONFIG); + if (compiler_unexpected(p.e != OK)) { + ret.error = p.e; + goto out; + } + virtio_mmio_t *virtio_mmio = p.r; + + index_result_t res = nospec_range_check(sel, VIRTIO_MMIO_DRV_FEAT_NUM); + if (res.e == OK) { + ret.drv_feat = virtio_mmio->banked_drv_feat[res.r]; + ret.error = OK; + } else { + ret.error = res.e; + } + + object_put_virtio_mmio(virtio_mmio); +out: + return ret; +} + +hypercall_virtio_mmio_backend_get_queue_info_result_t +hypercall_virtio_mmio_backend_get_queue_info(cap_id_t virtio_mmio_cap, + uint32_t sel) +{ + hypercall_virtio_mmio_backend_get_queue_info_result_t ret = { 0 }; + cspace_t *cspace = cspace_get_self(); + + virtio_mmio_ptr_result_t p = cspace_lookup_virtio_mmio( + cspace, virtio_mmio_cap, CAP_RIGHTS_VIRTIO_MMIO_CONFIG); + if (compiler_unexpected(p.e != OK)) { + ret.error = p.e; + goto out; + } + virtio_mmio_t *virtio_mmio = p.r; + + index_result_t res = nospec_range_check(sel, virtio_mmio->vqs_num); + if (res.e != OK) { + object_put_virtio_mmio(virtio_mmio); + ret.error = res.e; + goto out; + } + + virtio_mmio_banked_queue_registers_t *queue_regs = + &virtio_mmio->banked_queue_regs[res.r]; + + ret.queue_num = queue_regs->num; + ret.queue_ready = queue_regs->ready; + + ret.queue_desc = queue_regs->desc_high; + ret.queue_desc = ret.queue_desc << 32; + ret.queue_desc |= queue_regs->desc_low; + + ret.queue_drv = queue_regs->drv_high; + ret.queue_drv = ret.queue_drv << 32; + ret.queue_drv |= queue_regs->drv_low; + + ret.queue_dev = queue_regs->dev_high; + ret.queue_dev = ret.queue_dev << 32; + ret.queue_dev |= queue_regs->dev_low; + + ret.error = OK; + + object_put_virtio_mmio(virtio_mmio); +out: + return ret; +} + +hypercall_virtio_mmio_backend_get_notification_result_t +hypercall_virtio_mmio_backend_get_notification(cap_id_t virtio_mmio_cap) +{ + hypercall_virtio_mmio_backend_get_notification_result_t ret = { 0 }; + cspace_t *cspace = cspace_get_self(); + + virtio_mmio_ptr_result_t p = cspace_lookup_virtio_mmio( + cspace, virtio_mmio_cap, CAP_RIGHTS_VIRTIO_MMIO_CONFIG); + if (compiler_unexpected(p.e != OK)) { + ret.error = p.e; + goto out; + } + virtio_mmio_t *virtio_mmio = p.r; + + spinlock_acquire(&virtio_mmio->lock); + ret.vqs_bitmap = atomic_exchange_explicit(&virtio_mmio->vqs_bitmap, 0U, + memory_order_relaxed); + ret.reason = atomic_load_relaxed(&virtio_mmio->reason); + atomic_store_relaxed(&virtio_mmio->reason, + virtio_mmio_notify_reason_default()); + spinlock_release(&virtio_mmio->lock); + + ret.error = OK; + + object_put_virtio_mmio(virtio_mmio); +out: + return ret; +} + +error_t +hypercall_virtio_mmio_backend_acknowledge_reset(cap_id_t virtio_mmio_cap) +{ + error_t ret = OK; + cspace_t *cspace = cspace_get_self(); + + virtio_mmio_ptr_result_t p = cspace_lookup_virtio_mmio( + cspace, virtio_mmio_cap, CAP_RIGHTS_VIRTIO_MMIO_CONFIG); + if (compiler_unexpected(p.e != OK)) { + ret = p.e; + goto out; + } + virtio_mmio_t *virtio_mmio = p.r; + + spinlock_acquire(&virtio_mmio->lock); + atomic_store_relaxed(&virtio_mmio->regs->status, + virtio_mmio_status_reg_default()); + spinlock_release(&virtio_mmio->lock); + + object_put_virtio_mmio(virtio_mmio); +out: + return ret; +} + +error_t +hypercall_virtio_mmio_backend_update_status(cap_id_t virtio_mmio_cap, + uint32_t val) +{ + error_t ret = OK; + cspace_t *cspace = cspace_get_self(); + + virtio_mmio_ptr_result_t p = cspace_lookup_virtio_mmio( + cspace, virtio_mmio_cap, CAP_RIGHTS_VIRTIO_MMIO_CONFIG); + if (compiler_unexpected(p.e != OK)) { + ret = p.e; + goto out; + } + virtio_mmio_t *virtio_mmio = p.r; + + spinlock_acquire(&virtio_mmio->lock); + uint32_t status = virtio_mmio_status_reg_raw( + atomic_load_relaxed(&virtio_mmio->regs->status)); + status |= val; + atomic_store_relaxed(&virtio_mmio->regs->status, + virtio_mmio_status_reg_cast(status)); + spinlock_release(&virtio_mmio->lock); + + object_put_virtio_mmio(virtio_mmio); +out: + return ret; +} diff --git a/hyp/vm/virtio_mmio/src/vdevice.c b/hyp/vm/virtio_mmio/src/vdevice.c new file mode 100644 index 0000000..fe6bbfb --- /dev/null +++ b/hyp/vm/virtio_mmio/src/vdevice.c @@ -0,0 +1,371 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include "event_handlers.h" +#include "virtio_mmio.h" + +static bool +virtio_mmio_access_allowed(size_t size, size_t offset) +{ + bool ret; + + // First check if the access is size-aligned + if ((offset & (size - 1U)) != 0UL) { + ret = false; + } else if (size == sizeof(uint32_t)) { + // Word accesses, always allowed + ret = true; + } else if (size == sizeof(uint8_t)) { + // Byte accesses only allowed for config + ret = ((offset >= OFS_VIRTIO_MMIO_REGS_CONFIG(0U)) && + (offset <= OFS_VIRTIO_MMIO_REGS_CONFIG(( + VIRTIO_MMIO_REG_CONFIG_BYTES - 1U)))); + } else { + // Invalid access size + ret = false; + } + + return ret; +} + +static bool +virtio_mmio_default_write(const virtio_mmio_t *virtio_mmio, size_t offset, + size_t access_size, uint32_t val) +{ + bool ret = true; + + if ((offset >= OFS_VIRTIO_MMIO_REGS_CONFIG(0U)) && + (offset <= OFS_VIRTIO_MMIO_REGS_CONFIG( + (VIRTIO_MMIO_REG_CONFIG_BYTES - 1U)))) { + index_t n = (index_t)(offset - OFS_VIRTIO_MMIO_REGS_CONFIG(0U)); + // Loop through every byte + uint32_t shifted_val = val; + for (index_t i = 0U; i < access_size; i++) { + atomic_store_relaxed(&virtio_mmio->regs->config[n + i], + (uint8_t)shifted_val); + shifted_val >>= 8U; + } + } else { + ret = false; + } + + return ret; +} + +static bool +virtio_mmio_write_queue_sel(virtio_mmio_t *virtio_mmio, uint32_t val) +{ + bool ret = true; + index_result_t res = nospec_range_check(val, virtio_mmio->vqs_num); + if (res.e == OK) { + virtio_mmio->queue_sel = res.r; + + // Update corresponding banked registers with read + // permission + spinlock_acquire(&virtio_mmio->lock); + atomic_store_relaxed( + &virtio_mmio->regs->queue_num_max, + virtio_mmio->banked_queue_regs[res.r].num_max); + atomic_store_relaxed( + &virtio_mmio->regs->queue_ready, + virtio_mmio->banked_queue_regs[res.r].ready); + spinlock_release(&virtio_mmio->lock); + } else { + ret = false; + } + + return ret; +} + +static bool +virtio_mmio_write_status_reg(virtio_mmio_t *virtio_mmio, uint32_t val) +{ + bool ret = true; + virtio_mmio_notify_reason_t reason; + + if (val != 0U) { + bool assert_virq = false; + + spinlock_acquire(&virtio_mmio->lock); + + virtio_mmio_status_reg_t old_val = + atomic_load_relaxed(&virtio_mmio->regs->status); + virtio_mmio_status_reg_t new_val = + virtio_mmio_status_reg_cast(val); + atomic_store_relaxed(&virtio_mmio->regs->status, new_val); + + reason = atomic_load_relaxed(&virtio_mmio->reason); + + if (!virtio_mmio_status_reg_get_driver_ok(&old_val) && + virtio_mmio_status_reg_get_driver_ok(&new_val)) { + virtio_mmio_notify_reason_set_driver_ok(&reason, true); + assert_virq = true; + + } else if (!virtio_mmio_status_reg_get_failed(&old_val) && + virtio_mmio_status_reg_get_failed(&new_val)) { + virtio_mmio_notify_reason_set_failed(&reason, true); + assert_virq = true; + } else { + // Nothing to do + } + + atomic_store_relaxed(&virtio_mmio->reason, reason); + + spinlock_release(&virtio_mmio->lock); + + if (assert_virq) { + atomic_thread_fence(memory_order_release); + (void)virq_assert(&virtio_mmio->frontend_source, false); + } + } else if (virtio_mmio_status_reg_raw(atomic_load_relaxed( + &virtio_mmio->regs->status)) == 0U) { + // We do not request a reset the first time the frontend + // tries to write a zero to the status register + } else { + // Assert backend's IRQ to let the + // backend know that a device reset has been requested. + spinlock_acquire(&virtio_mmio->lock); + virtio_mmio_status_reg_t status = + atomic_load_relaxed(&virtio_mmio->regs->status); + virtio_mmio_status_reg_set_device_needs_reset(&status, true); + atomic_store_relaxed(&virtio_mmio->regs->status, status); + + reason = atomic_load_relaxed(&virtio_mmio->reason); + virtio_mmio_notify_reason_set_reset_rqst(&reason, true); + atomic_store_relaxed(&virtio_mmio->reason, reason); + spinlock_release(&virtio_mmio->lock); + + // Clear all bits QueueReady for all queues in the + // device. + for (index_t i = 0; i < virtio_mmio->vqs_num; i++) { + virtio_mmio->banked_queue_regs[i].ready = 0U; + } + atomic_store_relaxed(&virtio_mmio->regs->queue_ready, 0U); + + atomic_thread_fence(memory_order_release); + ret = virq_assert(&virtio_mmio->frontend_source, false).r; + } + + return ret; +} + +static bool +virtio_mmio_write_dev_feat_sel(const virtio_mmio_t *virtio_mmio, uint32_t val) +{ + bool ret = true; + index_result_t res = nospec_range_check(val, VIRTIO_MMIO_DEV_FEAT_NUM); + if (res.e == OK) { + // Update corresponding banked register + atomic_store_relaxed(&virtio_mmio->regs->dev_feat, + virtio_mmio->banked_dev_feat[res.r]); + } else { + ret = false; + } + + return ret; +} + +static bool +virtio_mmio_write_drv_feat_sel(virtio_mmio_t *virtio_mmio, uint32_t val) +{ + bool ret = true; + index_result_t res = nospec_range_check(val, VIRTIO_MMIO_DRV_FEAT_NUM); + if (res.e == OK) { + virtio_mmio->drv_feat_sel = res.r; + } else { + ret = false; + } + + return ret; +} + +static void +virtio_mmio_write_queue_notify(virtio_mmio_t *virtio_mmio, uint32_t val) +{ + virtio_mmio_notify_reason_t reason; + + spinlock_acquire(&virtio_mmio->lock); + + // Update bitmap of virtual queues to be notified + (void)atomic_fetch_or_explicit(&virtio_mmio->vqs_bitmap, util_bit(val), + memory_order_relaxed); + + reason = atomic_load_relaxed(&virtio_mmio->reason); + virtio_mmio_notify_reason_set_new_buffer(&reason, true); + atomic_store_relaxed(&virtio_mmio->reason, reason); + + spinlock_release(&virtio_mmio->lock); + + // Assert backend's IRQ to notify the backend that there are new + // buffers to process + atomic_thread_fence(memory_order_release); + (void)virq_assert(&virtio_mmio->frontend_source, false); +} + +static void +virtio_mmio_write_interrupt_ack(virtio_mmio_t *virtio_mmio, uint32_t val) +{ + virtio_mmio_notify_reason_t reason; + + atomic_store_relaxed(&virtio_mmio->regs->interrupt_ack, val); + + spinlock_acquire(&virtio_mmio->lock); + + uint32_t interrupt_status = + atomic_load_relaxed(&virtio_mmio->regs->interrupt_status); + interrupt_status &= ~val; + atomic_store_relaxed(&virtio_mmio->regs->interrupt_status, + interrupt_status); + + // Assert backend's IRQ so that the backend can continue raising + // interrupts + reason = atomic_load_relaxed(&virtio_mmio->reason); + virtio_mmio_notify_reason_set_interrupt_ack(&reason, true); + atomic_store_relaxed(&virtio_mmio->reason, reason); + + spinlock_release(&virtio_mmio->lock); + + atomic_thread_fence(memory_order_release); + (void)virq_assert(&virtio_mmio->frontend_source, false); +} + +static bool +virtio_mmio_vdevice_write(virtio_mmio_t *virtio_mmio, size_t offset, + uint32_t val, size_t access_size) +{ + bool ret = true; + + switch (offset) { + case OFS_VIRTIO_MMIO_REGS_DEV_FEAT_SEL: + ret = virtio_mmio_write_dev_feat_sel(virtio_mmio, val); + break; + + case OFS_VIRTIO_MMIO_REGS_DRV_FEAT: + virtio_mmio->banked_drv_feat[virtio_mmio->drv_feat_sel] = val; + break; + + case OFS_VIRTIO_MMIO_REGS_DRV_FEAT_SEL: + ret = virtio_mmio_write_drv_feat_sel(virtio_mmio, val); + break; + + case OFS_VIRTIO_MMIO_REGS_QUEUE_SEL: + ret = virtio_mmio_write_queue_sel(virtio_mmio, val); + break; + + case OFS_VIRTIO_MMIO_REGS_QUEUE_NUM: + virtio_mmio->banked_queue_regs[virtio_mmio->queue_sel].num = + val; + break; + + case OFS_VIRTIO_MMIO_REGS_QUEUE_READY: + atomic_store_relaxed(&virtio_mmio->regs->queue_ready, val); + + virtio_mmio->banked_queue_regs[virtio_mmio->queue_sel].ready = + val; + break; + + case OFS_VIRTIO_MMIO_REGS_QUEUE_NOTIFY: + virtio_mmio_write_queue_notify(virtio_mmio, val); + break; + + case OFS_VIRTIO_MMIO_REGS_INTERRUPT_ACK: + virtio_mmio_write_interrupt_ack(virtio_mmio, val); + break; + + case OFS_VIRTIO_MMIO_REGS_STATUS: + // We should not allow the frontend to write 0 to the device + // status since a 0 status means that the device reset is + // complete + ret = virtio_mmio_write_status_reg(virtio_mmio, val); + break; + + case OFS_VIRTIO_MMIO_REGS_QUEUE_DESC_LOW: + virtio_mmio->banked_queue_regs[virtio_mmio->queue_sel].desc_low = + val; + break; + + case OFS_VIRTIO_MMIO_REGS_QUEUE_DESC_HIGH: + virtio_mmio->banked_queue_regs[virtio_mmio->queue_sel] + .desc_high = val; + break; + + case OFS_VIRTIO_MMIO_REGS_QUEUE_DRV_LOW: + virtio_mmio->banked_queue_regs[virtio_mmio->queue_sel].drv_low = + val; + break; + + case OFS_VIRTIO_MMIO_REGS_QUEUE_DRV_HIGH: + virtio_mmio->banked_queue_regs[virtio_mmio->queue_sel].drv_high = + val; + break; + + case OFS_VIRTIO_MMIO_REGS_QUEUE_DEV_LOW: + virtio_mmio->banked_queue_regs[virtio_mmio->queue_sel].dev_low = + val; + break; + + case OFS_VIRTIO_MMIO_REGS_QUEUE_DEV_HIGH: + virtio_mmio->banked_queue_regs[virtio_mmio->queue_sel].dev_high = + val; + break; + + default: + ret = virtio_mmio_default_write(virtio_mmio, offset, + access_size, val); + break; + } + + return ret; +} + +vcpu_trap_result_t +virtio_mmio_handle_vdevice_access(vdevice_t *vdevice, size_t offset, + size_t access_size, register_t *value, + bool is_write) +{ + vcpu_trap_result_t ret; + + // Trap only writes from virtio's frontend + if (!is_write) { + ret = VCPU_TRAP_RESULT_UNHANDLED; + goto out; + } + + assert((vdevice != NULL) && + (vdevice->type == VDEVICE_TYPE_VIRTIO_MMIO)); + virtio_mmio_t *virtio_mmio = + virtio_mmio_container_of_frontend_device(vdevice); + if (virtio_mmio == NULL) { + ret = VCPU_TRAP_RESULT_UNHANDLED; + goto out; + } + + if (!virtio_mmio_access_allowed(access_size, offset)) { + ret = VCPU_TRAP_RESULT_FAULT; + goto out; + } + + ret = virtio_mmio_vdevice_write(virtio_mmio, offset, (uint32_t)*value, + access_size) + ? VCPU_TRAP_RESULT_EMULATED + : VCPU_TRAP_RESULT_FAULT; + +out: + return ret; +} diff --git a/hyp/vm/virtio_mmio/src/virtio_mmio.c b/hyp/vm/virtio_mmio/src/virtio_mmio.c new file mode 100644 index 0000000..4292a30 --- /dev/null +++ b/hyp/vm/virtio_mmio/src/virtio_mmio.c @@ -0,0 +1,271 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "event_handlers.h" +#include "panic.h" +#include "virtio_mmio.h" + +error_t +virtio_mmio_handle_object_create_virtio_mmio(virtio_mmio_create_t create) +{ + virtio_mmio_t *virtio_mmio = create.virtio_mmio; + spinlock_init(&virtio_mmio->lock); + + return OK; +} + +error_t +virtio_mmio_configure(virtio_mmio_t *virtio_mmio, memextent_t *memextent, + count_t vqs_num) +{ + error_t ret = OK; + + assert(virtio_mmio != NULL); + assert(memextent != NULL); + + // Memextent should only cover one contiguous virtio config page + if ((memextent->type != MEMEXTENT_TYPE_BASIC) || + (memextent->size != PGTABLE_VM_PAGE_SIZE) || + (vqs_num > VIRTIO_MMIO_MAX_VQS)) { + ret = ERROR_ARGUMENT_INVALID; + goto out; + } + + if (virtio_mmio->me != NULL) { + object_put_memextent(virtio_mmio->me); + } + + virtio_mmio->me = object_get_memextent_additional(memextent); + virtio_mmio->vqs_num = vqs_num; + +out: + return ret; +} + +error_t +virtio_mmio_handle_object_activate_virtio_mmio(virtio_mmio_t *virtio_mmio) +{ + error_t ret = OK; + + assert(virtio_mmio != NULL); + + partition_t *partition = virtio_mmio->header.partition; + + if (virtio_mmio->me == NULL) { + ret = ERROR_OBJECT_CONFIG; + goto error_no_me; + } + + virtio_mmio->frontend_device.type = VDEVICE_TYPE_VIRTIO_MMIO; + ret = vdevice_attach_phys(&virtio_mmio->frontend_device, + virtio_mmio->me); + if (ret != OK) { + goto error_vdevice; + } + + // Allocate banked registers based on how many virtual queues will be + // used + size_t alloc_size = virtio_mmio->vqs_num * + sizeof(virtio_mmio_banked_queue_registers_t); + + void_ptr_result_t alloc_ret = + partition_alloc(partition, alloc_size, alignof(uint32_t *)); + if (alloc_ret.e != OK) { + ret = ERROR_NOMEM; + goto out; + } + (void)memset_s(alloc_ret.r, alloc_size, 0, alloc_size); + + virtio_mmio->banked_queue_regs = + (virtio_mmio_banked_queue_registers_t *)alloc_ret.r; + + // Allocate virtio config page + size_t size = virtio_mmio->me->size; + if (size < sizeof(*virtio_mmio->regs)) { + ret = ERROR_ARGUMENT_SIZE; + goto out; + } + + virt_range_result_t range = hyp_aspace_allocate(size); + if (range.e != OK) { + ret = range.e; + goto out; + } + + ret = memextent_attach(partition, virtio_mmio->me, range.r.base, + sizeof(*virtio_mmio->regs)); + if (ret != OK) { + hyp_aspace_deallocate(partition, range.r); + goto out; + } + + virtio_mmio->regs = (virtio_mmio_regs_t *)range.r.base; + virtio_mmio->size = range.r.size; + + // Flush cache before using the uncached mapping + CACHE_CLEAN_OBJECT(*virtio_mmio->regs); + +out: + if (ret != OK) { + vdevice_detach_phys(&virtio_mmio->frontend_device, + virtio_mmio->me); + } +error_vdevice: +error_no_me: + return ret; +} + +void +virtio_mmio_handle_object_deactivate_virtio_mmio(virtio_mmio_t *virtio_mmio) +{ + assert(virtio_mmio != NULL); + + vic_unbind(&virtio_mmio->backend_source); + vic_unbind(&virtio_mmio->frontend_source); + + vdevice_detach_phys(&virtio_mmio->frontend_device, virtio_mmio->me); +} + +void +virtio_mmio_handle_object_cleanup_virtio_mmio(virtio_mmio_t *virtio_mmio) +{ + assert(virtio_mmio != NULL); + + partition_t *partition = virtio_mmio->header.partition; + + if (virtio_mmio->regs != NULL) { + memextent_detach(partition, virtio_mmio->me); + + virt_range_t range = { .base = (uintptr_t)virtio_mmio->regs, + .size = virtio_mmio->size }; + + hyp_aspace_deallocate(partition, range); + + virtio_mmio->regs = NULL; + virtio_mmio->size = 0U; + } + + if (virtio_mmio->banked_queue_regs != NULL) { + size_t alloc_size = + virtio_mmio->vqs_num * + sizeof(virtio_mmio_banked_queue_registers_t); + void *alloc_base = (void *)virtio_mmio->banked_queue_regs; + + error_t err = partition_free(partition, alloc_base, alloc_size); + assert(err == OK); + + virtio_mmio->banked_queue_regs = NULL; + virtio_mmio->vqs_num = 0U; + } + + if (virtio_mmio->me != NULL) { + object_put_memextent(virtio_mmio->me); + virtio_mmio->me = NULL; + } +} + +void +virtio_mmio_unwind_object_activate_virtio_mmio(virtio_mmio_t *virtio_mmio) +{ + virtio_mmio_handle_object_deactivate_virtio_mmio(virtio_mmio); + virtio_mmio_handle_object_cleanup_virtio_mmio(virtio_mmio); +} + +error_t +virtio_mmio_backend_bind_virq(virtio_mmio_t *virtio_mmio, vic_t *vic, + virq_t virq) +{ + error_t ret = OK; + + assert(virtio_mmio != NULL); + assert(vic != NULL); + + ret = vic_bind_shared(&virtio_mmio->backend_source, vic, virq, + VIRQ_TRIGGER_VIRTIO_MMIO_BACKEND); + + return ret; +} + +void +virtio_mmio_backend_unbind_virq(virtio_mmio_t *virtio_mmio) +{ + assert(virtio_mmio != NULL); + + vic_unbind_sync(&virtio_mmio->backend_source); +} + +error_t +virtio_mmio_frontend_bind_virq(virtio_mmio_t *virtio_mmio, vic_t *vic, + virq_t virq) +{ + error_t ret = OK; + + assert(virtio_mmio != NULL); + assert(vic != NULL); + + ret = vic_bind_shared(&virtio_mmio->frontend_source, vic, virq, + VIRQ_TRIGGER_VIRTIO_MMIO_FRONTEND); + + return ret; +} + +void +virtio_mmio_frontend_unbind_virq(virtio_mmio_t *virtio_mmio) +{ + assert(virtio_mmio != NULL); + + vic_unbind_sync(&virtio_mmio->frontend_source); +} + +bool +virtio_mmio_frontend_handle_virq_check_pending(virq_source_t *source) +{ + assert(source != NULL); + + // Deassert backend's IRQ when get_notification has been called + virtio_mmio_t *virtio_mmio = + virtio_mmio_container_of_frontend_source(source); + + virtio_mmio_notify_reason_t reason = + atomic_load_relaxed(&virtio_mmio->reason); + return !virtio_mmio_notify_reason_is_equal( + reason, virtio_mmio_notify_reason_default()); +} + +bool +virtio_mmio_backend_handle_virq_check_pending(virq_source_t *source) +{ + assert(source != NULL); + + // Deassert frontend's IRQ when interrupt_status is zero, meaning no + // interrupts are pending to be handled + virtio_mmio_t *virtio_mmio = + virtio_mmio_container_of_backend_source(source); + + return (atomic_load_relaxed(&virtio_mmio->regs->interrupt_status) != + 0U); +} diff --git a/hyp/vm/virtio_mmio/virtio_mmio.ev b/hyp/vm/virtio_mmio/virtio_mmio.ev new file mode 100644 index 0000000..0233078 --- /dev/null +++ b/hyp/vm/virtio_mmio/virtio_mmio.ev @@ -0,0 +1,22 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module virtio_mmio + +subscribe object_create_virtio_mmio + +subscribe object_activate_virtio_mmio + unwinder(virtio_mmio) + +subscribe object_deactivate_virtio_mmio + +subscribe object_cleanup_virtio_mmio(virtio_mmio) + +subscribe virq_check_pending[VIRQ_TRIGGER_VIRTIO_MMIO_FRONTEND] + handler virtio_mmio_frontend_handle_virq_check_pending(source) + +subscribe virq_check_pending[VIRQ_TRIGGER_VIRTIO_MMIO_BACKEND] + handler virtio_mmio_backend_handle_virq_check_pending(source) + +subscribe vdevice_access[VDEVICE_TYPE_VIRTIO_MMIO](vdevice, offset, access_size, value, is_write) diff --git a/hyp/vm/virtio_mmio/virtio_mmio.tc b/hyp/vm/virtio_mmio/virtio_mmio.tc new file mode 100644 index 0000000..26fadfb --- /dev/null +++ b/hyp/vm/virtio_mmio/virtio_mmio.tc @@ -0,0 +1,97 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +extend cap_rights_virtio_mmio bitfield { + 0 bind_backend_virq bool; + 1 bind_frontend_virq bool; + 2 assert_virq bool; + 3 config bool; +}; + +define VIRTIO_MMIO_MAX_VQS constant type count_t = 16; + +// Feature bits +define VIRTIO_F_VERSION_1 constant type index_t = 32; +define VIRTIO_F_ACCESS_PLATFORM constant type index_t = 33; +define VIRTIO_F_NOTIFICATION_DATA constant type index_t = 38; + +define VIRTIO_MMIO_REG_CONFIG_BYTES constant type count_t = (PGTABLE_HYP_PAGE_SIZE - 0x100); + +define VIRTIO_MMIO_DEV_FEAT_NUM constant type count_t = 2; +define VIRTIO_MMIO_DRV_FEAT_NUM constant type count_t = 2; + +// Device Status +define virtio_mmio_status_reg bitfield<32> { + 2 driver_ok bool; + 6 device_needs_reset bool; + 7 failed bool; + others unknown=0; +}; + +extend virtio_mmio object { + me pointer object memextent; + backend_source structure virq_source(contained); + frontend_source structure virq_source(contained); + frontend_device structure vdevice(contained); + regs pointer structure virtio_mmio_regs; + size size; + vqs_num type count_t; + lock structure spinlock; + vqs_bitmap type register_t(atomic); + drv_feat_sel uint32; + queue_sel uint32; + reason bitfield virtio_mmio_notify_reason(atomic); + banked_dev_feat array(VIRTIO_MMIO_DEV_FEAT_NUM) uint32; + banked_drv_feat array(VIRTIO_MMIO_DRV_FEAT_NUM) uint32; + banked_queue_regs pointer structure virtio_mmio_banked_queue_registers; + pending_rst bool; +}; + +define virtio_mmio_banked_queue_registers structure { + num_max uint32; + num uint32; + ready uint32; + desc_low uint32; + desc_high uint32; + drv_low uint32; + drv_high uint32; + dev_low uint32; + dev_high uint32; +}; + +define virtio_mmio_regs structure(aligned(PGTABLE_HYP_PAGE_SIZE)) { + magic_value @ 0x000 uint32(atomic); + version @ 0x004 uint32(atomic); + dev_id @ 0x008 uint32(atomic); + vendor_id @ 0x00c uint32(atomic); + dev_feat @ 0x010 uint32(atomic); + dev_feat_sel @ 0x014 uint32(atomic); + drv_feat @ 0x020 uint32(atomic); + drv_feat_sel @ 0x024 uint32(atomic); + queue_sel @ 0x030 uint32(atomic); + queue_num_max @ 0x034 uint32(atomic); + queue_num @ 0x038 uint32(atomic); + queue_ready @ 0x044 uint32(atomic); + queue_notify @ 0x050 uint32(atomic); + interrupt_status @ 0x060 uint32(atomic); + interrupt_ack @ 0x064 uint32(atomic); + status @ 0x070 bitfield virtio_mmio_status_reg(atomic); + queue_desc_low @ 0x080 uint32(atomic); + queue_desc_high @ 0x084 uint32(atomic); + queue_drv_low @ 0x090 uint32(atomic); + queue_drv_high @ 0x094 uint32(atomic); + queue_dev_low @ 0x0a0 uint32(atomic); + queue_dev_high @ 0x0a4 uint32(atomic); + config_gen @ 0x0fc uint32(atomic); + config @ 0x100 array(VIRTIO_MMIO_REG_CONFIG_BYTES) uint8(atomic); +}; + +extend virq_trigger enumeration { + virtio_mmio_backend; + virtio_mmio_frontend; +}; + +extend vdevice_type enumeration { + VIRTIO_MMIO; +}; diff --git a/hyp/vm/vpm_base/build.conf b/hyp/vm/vpm_base/build.conf new file mode 100644 index 0000000..6d5a436 --- /dev/null +++ b/hyp/vm/vpm_base/build.conf @@ -0,0 +1,7 @@ +# © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +interface vpm +types vpm_base.tc +source hypercalls.c diff --git a/hyp/vm/psci_pc/src/hypercalls.c b/hyp/vm/vpm_base/src/hypercalls.c similarity index 74% rename from hyp/vm/psci_pc/src/hypercalls.c rename to hyp/vm/vpm_base/src/hypercalls.c index fb06273..8867591 100644 --- a/hyp/vm/psci_pc/src/hypercalls.c +++ b/hyp/vm/vpm_base/src/hypercalls.c @@ -16,6 +16,49 @@ #include #include +error_t +hypercall_vpm_group_configure(cap_id_t vpm_group_cap, + vpm_group_option_flags_t flags) +{ + error_t err; + cspace_t *cspace = cspace_get_self(); + + if (!vpm_group_option_flags_is_clean(flags)) { + err = ERROR_UNIMPLEMENTED; + goto out; + } + + object_type_t type; + object_ptr_result_t o = cspace_lookup_object_any( + cspace, vpm_group_cap, CAP_RIGHTS_GENERIC_OBJECT_ACTIVATE, + &type); + if (compiler_unexpected(o.e != OK)) { + err = o.e; + goto out; + } + if (type != OBJECT_TYPE_VPM_GROUP) { + err = ERROR_CSPACE_WRONG_OBJECT_TYPE; + goto out_release_vpm_group; + } + vpm_group_t *vpm_group = o.r.vpm_group; + + spinlock_acquire(&vpm_group->header.lock); + + if (atomic_load_relaxed(&vpm_group->header.state) == + OBJECT_STATE_INIT) { + err = vpm_group_configure(vpm_group, flags); + } else { + err = ERROR_OBJECT_STATE; + } + + spinlock_release(&vpm_group->header.lock); + +out_release_vpm_group: + object_put(type, o.r); +out: + return err; +} + error_t hypercall_vpm_group_attach_vcpu(cap_id_t vpm_group_cap, cap_id_t vcpu_cap, index_t index) @@ -116,7 +159,7 @@ hypercall_vpm_group_get_state_result_t hypercall_vpm_group_get_state(cap_id_t vpm_group_cap) { hypercall_vpm_group_get_state_result_t ret = { 0 }; - cspace_t *cspace = cspace_get_self(); + cspace_t *cspace = cspace_get_self(); vpm_group_ptr_result_t p = cspace_lookup_vpm_group( cspace, vpm_group_cap, CAP_RIGHTS_VPM_GROUP_QUERY); @@ -129,7 +172,7 @@ hypercall_vpm_group_get_state(cap_id_t vpm_group_cap) vpm_state_t state = vpm_get_state(vpm_group); ret.error = OK; - ret.vpm_state = state; + ret.vpm_state = (uint64_t)state; object_put_vpm_group(vpm_group); out: diff --git a/hyp/vm/vpm_base/vpm_base.tc b/hyp/vm/vpm_base/vpm_base.tc new file mode 100644 index 0000000..cd86ffa --- /dev/null +++ b/hyp/vm/vpm_base/vpm_base.tc @@ -0,0 +1,16 @@ +// © 2022 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +extend vpm_group object { + options bitfield vpm_group_option_flags; +}; + +define vpm_mode enumeration { + NONE = 0; + IDLE; +}; + +extend thread object { + vpm_mode enumeration vpm_mode; +}; diff --git a/hyp/vm/vrtc_pl031/build.conf b/hyp/vm/vrtc_pl031/build.conf new file mode 100644 index 0000000..711a6db --- /dev/null +++ b/hyp/vm/vrtc_pl031/build.conf @@ -0,0 +1,8 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +interface vrtc +events vrtc_pl031.ev +types vrtc_pl031.tc +source vrtc_pl031.c hypercalls.c diff --git a/hyp/vm/vrtc_pl031/src/hypercalls.c b/hyp/vm/vrtc_pl031/src/hypercalls.c new file mode 100644 index 0000000..9d77fe1 --- /dev/null +++ b/hyp/vm/vrtc_pl031/src/hypercalls.c @@ -0,0 +1,147 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +error_t +hypercall_vrtc_configure(cap_id_t vrtc_cap, vmaddr_t ipa) +{ + error_t err = OK; + cspace_t *cspace = cspace_get_self(); + + if (!util_is_baligned(ipa, VRTC_DEV_SIZE) || + util_add_overflows(ipa, VRTC_DEV_SIZE)) { + err = ERROR_ADDR_INVALID; + goto out; + } + + vrtc_ptr_result_t vrtc_r = cspace_lookup_vrtc_any( + cspace, vrtc_cap, CAP_RIGHTS_VRTC_CONFIGURE); + if (compiler_unexpected(vrtc_r.e) != OK) { + err = vrtc_r.e; + goto out; + } + vrtc_t *vrtc = vrtc_r.r; + + spinlock_acquire(&vrtc->header.lock); + if (atomic_load_relaxed(&vrtc->header.state) == OBJECT_STATE_INIT) { + vrtc->ipa = ipa; + } else { + err = ERROR_OBJECT_STATE; + } + spinlock_release(&vrtc->header.lock); + +out: + return err; +} + +error_t +hypercall_vrtc_set_time_base(cap_id_t vrtc_cap, nanoseconds_t time_base, + ticks_t sys_timer_ref) +{ + error_t err = OK; + cspace_t *cspace = cspace_get_self(); + + vrtc_ptr_result_t vrtc_r = cspace_lookup_vrtc( + cspace, vrtc_cap, CAP_RIGHTS_VRTC_SET_TIME_BASE); + if (compiler_unexpected(vrtc_r.e) != OK) { + err = vrtc_r.e; + goto out; + } + vrtc_t *vrtc = vrtc_r.r; + + if (vrtc->time_base != 0U) { + // The time base has already been set once + err = ERROR_BUSY; + goto out; + } + + preempt_disable(); + ticks_t now = platform_timer_get_current_ticks(); + if (now < sys_timer_ref) { + // The snapshot was taken in the future?! + err = ERROR_ARGUMENT_INVALID; + goto out_preempt; + } + + ticks_t time_base_ticks = platform_convert_ns_to_ticks(time_base); + + // Set the time_base to the moment the device was turned on. By using + // "sys_timer_ref" instead of "now" we account for the time delta + // between the moment the snapshot was taken and the moment the + // hypercall is handled in the hypervisor. + vrtc->time_base = time_base_ticks - sys_timer_ref; + vrtc->lr = (rtc_seconds_t)(time_base / TIMER_NANOSECS_IN_SECOND); + +out_preempt: + preempt_enable(); +out: + return err; +} + +error_t +hypercall_vrtc_attach_addrspace(cap_id_t vrtc_cap, cap_id_t addrspace_cap) +{ + error_t err = OK; + cspace_t *cspace = cspace_get_self(); + + vrtc_ptr_result_t vrtc_r = cspace_lookup_vrtc( + cspace, vrtc_cap, CAP_RIGHTS_VRTC_ATTACH_ADDRSPACE); + if (compiler_unexpected(vrtc_r.e) != OK) { + err = vrtc_r.e; + goto out; + } + vrtc_t *vrtc = vrtc_r.r; + + addrspace_ptr_result_t addrspace_r = cspace_lookup_addrspace_any( + cspace, addrspace_cap, CAP_RIGHTS_ADDRSPACE_MAP); + if (compiler_unexpected(addrspace_r.e != OK)) { + err = addrspace_r.e; + goto out_release_vrtc; + } + addrspace_t *addrspace = addrspace_r.r; + + spinlock_acquire(&addrspace->header.lock); + if (atomic_load_relaxed(&addrspace->header.state) != + OBJECT_STATE_ACTIVE) { + err = ERROR_OBJECT_STATE; + goto out_release_addrspace; + } + + err = addrspace_check_range(addrspace, vrtc->ipa, VRTC_DEV_SIZE); + if (err != OK) { + goto out_release_addrspace; + } + + vrtc_t *old_vrtc = addrspace->vrtc; + if (old_vrtc != NULL) { + object_put_vrtc(old_vrtc); + } + + addrspace->vrtc = object_get_vrtc_additional(vrtc); + +out_release_addrspace: + spinlock_release(&addrspace->header.lock); + object_put_addrspace(addrspace); +out_release_vrtc: + object_put_vrtc(vrtc); +out: + return err; +} diff --git a/hyp/vm/vrtc_pl031/src/vrtc_pl031.c b/hyp/vm/vrtc_pl031/src/vrtc_pl031.c new file mode 100644 index 0000000..de9173f --- /dev/null +++ b/hyp/vm/vrtc_pl031/src/vrtc_pl031.c @@ -0,0 +1,150 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "event_handlers.h" + +error_t +vrtc_pl031_handle_object_create_vrtc(vrtc_create_t params) +{ + vrtc_t *vrtc = params.vrtc; + assert(vrtc != NULL); + + vrtc->ipa = VMADDR_INVALID; + vrtc->lr = 0; + vrtc->time_base = 0; + + return OK; +} + +error_t +vrtc_pl031_handle_object_activate_vrtc(vrtc_t *vrtc) +{ + error_t err = OK; + + assert(vrtc != NULL); + + if (vrtc->ipa == VMADDR_INVALID) { + // Not configured yet + err = ERROR_OBJECT_CONFIG; + } + + return err; +} + +void +vrtc_pl031_handle_object_deactivate_addrspace(addrspace_t *addrspace) +{ + assert(addrspace != NULL); + + vrtc_t *vrtc = addrspace->vrtc; + + if (vrtc != NULL) { + object_put_vrtc(vrtc); + addrspace->vrtc = NULL; + } +} + +static void +vrtc_pl031_reg_read(vrtc_t *vrtc, size_t offset, register_t *value) +{ + if (offset == offsetof(vrtc_pl031_t, RTCDR)) { + uint64_t now = platform_timer_get_current_ticks(); + *value = platform_convert_ticks_to_ns(vrtc->time_base + now) / + TIMER_NANOSECS_IN_SECOND; + } else if (offset == offsetof(vrtc_pl031_t, RTCLR)) { + *value = vrtc->lr; + } else if (offset == offsetof(vrtc_pl031_t, RTCCR)) { + // Always enabled + *value = 1U; + } else if ((offset >= offsetof(vrtc_pl031_t, RTCPeriphID0)) && + (offset <= offsetof(vrtc_pl031_t, RTCPeriphID3))) { + // Calculate which byte in the ID register they are after + uint8_t id = (uint8_t)((offset - + offsetof(vrtc_pl031_t, RTCPeriphID0)) >> + 2); + *value = ((register_t)VRTC_PL031_PERIPH_ID >> (id << 3)) & + 0xffU; + } else if ((offset >= offsetof(vrtc_pl031_t, RTCPCellID0)) && + (offset <= offsetof(vrtc_pl031_t, RTCPCellID3))) { + // Calculate which byte in the ID register they are after + uint8_t id = (uint8_t)((offset - + offsetof(vrtc_pl031_t, RTCPCellID0)) >> + 2); + *value = ((register_t)VRTC_PL031_PCELL_ID >> (id << 3)) & 0xffU; + } else { + // All other PL031 registers are treated as RAZ + *value = 0U; + } +} + +static void +vrtc_pl031_reg_write(vrtc_t *vrtc, size_t offset, register_t *value) +{ + if (offset == offsetof(vrtc_pl031_t, RTCLR)) { + ticks_t value_ticks = platform_convert_ns_to_ticks( + *value * TIMER_NANOSECS_IN_SECOND); + preempt_disable(); + ticks_t now = platform_timer_get_current_ticks(); + vrtc->time_base = value_ticks - now; + preempt_enable(); + vrtc->lr = (rtc_seconds_t)(*value); + } + // The rest of the registers are WI. +} + +vcpu_trap_result_t +vrtc_pl031_handle_vdevice_access_fixed_addr(vmaddr_t ipa, size_t access_size, + register_t *value, bool is_write) +{ + vcpu_trap_result_t ret = VCPU_TRAP_RESULT_UNHANDLED; + + thread_t *thread = thread_get_self(); + + vrtc_t *vrtc = thread->addrspace->vrtc; + if ((vrtc == NULL) || (vrtc->ipa == VMADDR_INVALID)) { + // vRTC not initialised + goto out; + } + + if ((ipa < vrtc->ipa) || util_add_overflows(ipa, access_size) || + ((ipa + access_size) > (vrtc->ipa + VRTC_DEV_SIZE))) { + // Not vRTC + goto out; + } + + // Only 32-bit registers of PL031 are emulated + if (access_size != sizeof(uint32_t) || + !util_is_baligned(ipa, sizeof(uint32_t))) { + ret = VCPU_TRAP_RESULT_FAULT; + goto out; + } + + size_t offset = (size_t)(ipa - vrtc->ipa); + + if (is_write) { + vrtc_pl031_reg_write(vrtc, offset, value); + } else { + vrtc_pl031_reg_read(vrtc, offset, value); + } + + ret = VCPU_TRAP_RESULT_EMULATED; + +out: + return ret; +} diff --git a/hyp/vm/vrtc_pl031/vrtc_pl031.ev b/hyp/vm/vrtc_pl031/vrtc_pl031.ev new file mode 100644 index 0000000..0801a15 --- /dev/null +++ b/hyp/vm/vrtc_pl031/vrtc_pl031.ev @@ -0,0 +1,13 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module vrtc_pl031 + +subscribe object_create_vrtc(vrtc_create) + +subscribe object_activate_vrtc + +subscribe object_deactivate_addrspace + +subscribe vdevice_access_fixed_addr diff --git a/hyp/vm/vrtc_pl031/vrtc_pl031.tc b/hyp/vm/vrtc_pl031/vrtc_pl031.tc new file mode 100644 index 0000000..7bd0ec0 --- /dev/null +++ b/hyp/vm/vrtc_pl031/vrtc_pl031.tc @@ -0,0 +1,50 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +define rtc_seconds_t newtype uint32; + +define VRTC_DEV_SIZE constant size = PGTABLE_HYP_PAGE_SIZE; + +define VRTC_PL031_PERIPH_ID constant uint32 = 0x00041031; +define VRTC_PL031_PCELL_ID constant uint32 = 0xb105f00d; + +extend cap_rights_vrtc bitfield +{ + 0 configure bool; + 1 attach_addrspace bool; + 2 set_time_base bool; +}; + +// One vRTC object per VM. All the vCPUs in the VM get the same vRTC. +extend vrtc object { + time_base type ticks_t; + lr type rtc_seconds_t; + ipa type vmaddr_t; +}; + +extend addrspace object +{ + vrtc pointer object vrtc; +}; + +define vrtc_pl031 structure +{ + RTCDR @0x000 uint32; + RTCMR @0x004 uint32; + RTCLR @0x008 uint32; + RTCCR @0x00C uint32; + RTCIMSC @0x010 uint32; + RTCRRS @0x014 uint32; + RTCMIS @0x018 uint32; + RTCICR @0x01C uint32; + + RTCPeriphID0 @0xfe0 uint32; + RTCPeriphID1 @0xfe4 uint32; + RTCPeriphID2 @0xfe8 uint32; + RTCPeriphID3 @0xfec uint32; + RTCPCellID0 @0xff0 uint32; + RTCPCellID1 @0xff4 uint32; + RTCPCellID2 @0xff8 uint32; + RTCPCellID3 @0xffc uint32; +}; diff --git a/hyp/vm/vtbre/build.conf b/hyp/vm/vtbre/build.conf new file mode 100644 index 0000000..fd20f96 --- /dev/null +++ b/hyp/vm/vtbre/build.conf @@ -0,0 +1,8 @@ +# © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +base_module hyp/platform/tbre +base_module hyp/misc/vet +events vtbre.ev +source vtbre.c diff --git a/hyp/vm/vtbre/src/vtbre.c b/hyp/vm/vtbre/src/vtbre.c new file mode 100644 index 0000000..a9b71d7 --- /dev/null +++ b/hyp/vm/vtbre/src/vtbre.c @@ -0,0 +1,186 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include "event_handlers.h" +#include "tbre.h" + +void +vtbre_handle_boot_cpu_cold_init(void) +{ + ID_AA64DFR0_EL1_t id_aa64dfr0 = register_ID_AA64DFR0_EL1_read(); + // NOTE: ID_AA64DFR0.TraceBuffer just indicates if trace buffer is + // implemented, so here we use equal for assertion. + assert(ID_AA64DFR0_EL1_get_TraceBuffer(&id_aa64dfr0) == 1U); +} + +error_t +vtbre_handle_object_create_thread(thread_create_t thread_create) +{ + thread_t *thread = thread_create.thread; + + // MDCR_EL2.E2TB == 0b10 to prohibit trace EL2 + MDCR_EL2_set_E2TB(&thread->vcpu_regs_el2.mdcr_el2, 0x2); + + return OK; +} + +void +vet_update_trace_buffer_status(thread_t *self) +{ + assert(self != NULL); + +#if !DISABLE_TBRE + // check/set by reading TBRLIMITR.EN == 1 + TRBLIMITR_EL1_t trb_limitr = + register_TRBLIMITR_EL1_read_ordered(&vet_ordering); + self->vet_trace_buffer_enabled = TRBLIMITR_EL1_get_E(&trb_limitr); +#endif +} + +void +vet_flush_buffer(thread_t *self) +{ + assert(self != NULL); + + if (compiler_unexpected(self->vet_trace_buffer_enabled)) { + __asm__ volatile("tsb csync" : "+m"(vet_ordering)); + } +} + +void +vet_disable_buffer(void) +{ + TRBLIMITR_EL1_t trb_limitr = + register_TRBLIMITR_EL1_read_ordered(&vet_ordering); + TRBLIMITR_EL1_set_E(&trb_limitr, false); + register_TRBLIMITR_EL1_write_ordered(trb_limitr, &vet_ordering); +} + +static void +vtbre_prohibit_registers_access(thread_t *self, bool prohibit) +{ + assert(self != NULL); + + // MDCR_EL2.E2TB == 0b11 to enable access to TBRE + // MDCR_EL2.E2TB == 0b10 to disable access to TBRE + uint8_t expect = prohibit ? 0x2U : 0x3U; + + MDCR_EL2_set_E2TB(&self->vcpu_regs_el2.mdcr_el2, expect); + register_MDCR_EL2_write_ordered(self->vcpu_regs_el2.mdcr_el2, + &vet_ordering); +} + +void +vet_save_buffer_thread_context(thread_t *self) +{ + (void)self; + vtbre_prohibit_registers_access(self, true); +} + +void +vet_restore_buffer_thread_context(thread_t *self) +{ + vtbre_prohibit_registers_access(self, false); +} + +void +vet_enable_buffer(void) +{ + TRBLIMITR_EL1_t trb_limitr = + register_TRBLIMITR_EL1_read_ordered(&vet_ordering); + TRBLIMITR_EL1_set_E(&trb_limitr, true); + register_TRBLIMITR_EL1_write_ordered(trb_limitr, &vet_ordering); +} + +void +vet_save_buffer_power_context(void) +{ + MDCR_EL2_t mdcr_el2; + + // Enable E2TB access + mdcr_el2 = register_MDCR_EL2_read_ordered(&vet_ordering); + MDCR_EL2_set_E2TB(&mdcr_el2, 3); + register_MDCR_EL2_write_ordered(mdcr_el2, &vet_ordering); + + asm_context_sync_ordered(&vet_ordering); + + tbre_save_context_percpu(cpulocal_get_index()); + + // Disable E2TB access + MDCR_EL2_set_E2TB(&mdcr_el2, 2); + register_MDCR_EL2_write_ordered(mdcr_el2, &vet_ordering); +} + +void +vet_restore_buffer_power_context(void) +{ + MDCR_EL2_t mdcr_el2; + + // Enable E2TB access + mdcr_el2 = register_MDCR_EL2_read_ordered(&vet_ordering); + MDCR_EL2_set_E2TB(&mdcr_el2, 3); + register_MDCR_EL2_write_ordered(mdcr_el2, &vet_ordering); + + asm_context_sync_ordered(&vet_ordering); + + tbre_restore_context_percpu(cpulocal_get_index()); + + // Disable E2TB access + MDCR_EL2_set_E2TB(&mdcr_el2, 2); + register_MDCR_EL2_write_ordered(mdcr_el2, &vet_ordering); +} + +vcpu_trap_result_t +vtbre_handle_vcpu_trap_sysreg(ESR_EL2_ISS_MSR_MRS_t iss) +{ + vcpu_trap_result_t ret; + +#if DISABLE_TBRE + (void)iss; + + ret = VCPU_TRAP_RESULT_UNHANDLED; +#else + thread_t *current = thread_get_self(); + + if (compiler_expected((ESR_EL2_ISS_MSR_MRS_get_Op0(&iss) != 3U) || + (ESR_EL2_ISS_MSR_MRS_get_Op1(&iss) != 0U) || + (ESR_EL2_ISS_MSR_MRS_get_CRn(&iss) != 9U) || + (ESR_EL2_ISS_MSR_MRS_get_CRm(&iss) != 11U))) { + // Not a TBRE register access. + ret = VCPU_TRAP_RESULT_UNHANDLED; + } else if (!vcpu_option_flags_get_trace_allowed( + &thread->vcpu_options)) { + // This VCPU isn't allowed to trace. Fault immediately. + ret = VCPU_TRAP_RESULT_FAULT; + } else if (!current->vet_trace_buffer_enabled) { + // Lazily enable trace buffer register access and restore + // context. + current->vet_trace_buffer_enabled = true; + + // only enable the register access + vtbre_prohibit_registers_access(false); + + ret = VCPU_TRAP_RESULT_RETRY; + } else { + // Probably an attempted OS lock; fall back to default RAZ/WI. + ret = VCPU_TRAP_RESULT_UNHANDLED; + } +#endif + + return ret; +} diff --git a/hyp/vm/vtbre/vtbre.ev b/hyp/vm/vtbre/vtbre.ev new file mode 100644 index 0000000..745d0f1 --- /dev/null +++ b/hyp/vm/vtbre/vtbre.ev @@ -0,0 +1,18 @@ +// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +module vtbre + +subscribe boot_cpu_cold_init() + +// Lower priority than vcpu_arch_handle_object_create_thread so it can set up +// MDCR_EL2 properly after it has been initialised to the default value. +subscribe object_create_thread + priority -10 + +subscribe vcpu_trap_sysreg_read + handler vtbre_handle_vcpu_trap_sysreg + +subscribe vcpu_trap_sysreg_write + handler vtbre_handle_vcpu_trap_sysreg diff --git a/tools/build/__main__.py b/tools/build/__main__.py index a3d08c1..50c7a72 100644 --- a/tools/build/__main__.py +++ b/tools/build/__main__.py @@ -51,6 +51,19 @@ def add(self, key, value): raise KeyError("Item duplicate {:s}".format(key)) +class module_set(set): + def __init__(self): + self.module_re = re.compile( + r'[A-Za-z][A-Za-z0-9_]*([/][A-Za-z][A-Za-z0-9_]*)*') + super(module_set, self).__init__() + + def add(self, value): + if not self.module_re.fullmatch(value): + print("invalid module name:", value) + sys.exit(1) + super(module_set, self).add(value) + + build_dir = graph.build_dir config_base = 'config' @@ -58,13 +71,17 @@ def add(self, key, value): arch_base = os.path.join(module_base, 'arch') interface_base = os.path.join(module_base, 'interfaces') -modules = {'arch'} +conf_includes = set() +modules = module_set() +modules.add('arch') + interfaces = set() objects = set() external_objects = set() guestapis = set() types = set() hypercalls = set() +registers = list() test_programs = set() sa_html = set() asts = set() @@ -98,6 +115,11 @@ def add(self, key, value): template_engines['hypercalls'] = \ TemplateEngine(hypercalls_templates, None) +registers_templates = list() + +template_engines['registers'] = \ + TemplateEngine(registers_templates, None) + shvars_re = re.compile(r'\$((\w+)\b|{(\w+)})') @@ -128,8 +150,11 @@ def process_variant_conf(variant_key, conf, basename): platform = variant_key == 'platform' featureset = variant_key == 'featureset' + allow_arch = variant_key and not platform if platform: + if basename in target_arch_names: + logger.error("existing arch: %s", basename) target_arch_names.append(basename) with open(conf, 'r', encoding='utf-8') as f: for s in f.readlines(): @@ -137,6 +162,12 @@ def process_variant_conf(variant_key, conf, basename): if not words or words[0].startswith('#'): # Skip comments or blank lines pass + elif words[0] == 'include': + include_conf = os.path.join(config_base, + words[1] + '.conf') + if include_conf not in conf_includes: + process_variant_conf(None, include_conf, None) + conf_includes.add(include_conf) elif featureset and words[0] == 'platforms': global featureset_platforms featureset_platforms = words[1:] @@ -144,18 +175,17 @@ def process_variant_conf(variant_key, conf, basename): arch_conf = os.path.join(config_base, 'arch', words[1] + '.conf') process_variant_conf(variant_key, arch_conf, words[1]) + elif platform and words[0] == 'alias_arch': + if words[1] in target_arch_names: + logger.error("Alias existing arch: %s", + words[1]) + target_arch_names.append(words[1]) elif platform and words[0] == 'is_abi': global abi_arch if abi_arch is not None: logger.warning("Duplicate abi definitions: %s and %s", abi_arch, basename) abi_arch = basename - elif platform and words[0] == 'defines_registers': - global registers_arch - if registers_arch is not None: - logger.warning("Duplicate register definitions: %s and %s", - registers_arch, basename) - registers_arch = basename elif platform and words[0] == 'defines_link': global link_arch if link_arch is not None: @@ -173,21 +203,24 @@ def process_variant_conf(variant_key, conf, basename): modules.add(words[1]) elif words[0] == 'flags': variant_cflags.extend(map(var_subst, words[1:])) + elif words[0] == 'ldflags': + variant_ldflags.extend(map(var_subst, words[1:])) elif words[0] == 'configs': for c in map(var_subst, words[1:]): - check_global_define(c) - variant_defines.append(c) - elif words[0] == 'arch_module': + add_global_define(c) + elif allow_arch and words[0] == 'arch_module': if arch_match(words[1]): modules.add(words[2]) - elif words[0] == 'arch_flags': + elif allow_arch and words[0] == 'arch_flags': if arch_match(words[1]): variant_cflags.extend(map(var_subst, words[2:])) - elif words[0] == 'arch_configs': + elif allow_arch and words[0] == 'arch_ldflags': + if arch_match(words[1]): + variant_ldflags.extend(map(var_subst, words[2:])) + elif allow_arch and words[0] == 'arch_configs': if arch_match(words[1]): for c in map(var_subst, words[2:]): - check_global_define(c) - variant_defines.append(c) + add_global_define(c) else: # TODO: dependencies, configuration variables, etc # Restructure this to use a proper parser first @@ -208,13 +241,13 @@ def process_variant_conf(variant_key, conf, basename): missing_variant = False abi_arch = None -registers_arch = None link_arch = None target_triple = None target_arch_names = [] variant_cflags = [] variant_cppflags = [] variant_defines = [] +variant_ldflags = [] featureset_platforms = ['*'] @@ -234,12 +267,17 @@ def check_global_define(d): if configs[define] == val: logger.warning("Duplicate configuration: %s", d) else: - logger.error("Conficting configuration: %s and %s", + logger.error("Conflicting configuration: %s and %s", '='.join([define, configs[define]]), d) sys.exit(-1) configs[define] = val +def add_global_define(d): + check_global_define(d) + variant_defines.append(d) + + for variant_key in ('platform', 'featureset', 'quality'): try: variant_value = graph.get_env('VARIANT_' + variant_key) @@ -362,7 +400,8 @@ def template_match(template_engine, d): try: llvm_root = graph.get_env('LLVM') except KeyError: - logger.error("Please set $LLVM to the root of the prebuilt LLVM") + logger.error( + "Please set $LLVM to the root of the prebuilt LLVM") sys.exit(1) # Use a QC prebuilt LLVM @@ -396,6 +435,8 @@ def template_match(template_engine, d): # No need for C++ compatibility graph.append_env('CFLAGS', '-Wno-c++98-compat') graph.append_env('CFLAGS', '-Wno-c++-compat') +# No need for pre-C99 compatibility; we always use C18 +graph.append_env('CFLAGS', '-Wno-declaration-after-statement') # No need for GCC compatibility graph.append_env('CFLAGS', '-Wno-gcc-compat') # Allow GCC's _Alignof(lvalue) as a project deviation from MISRA rule 1.2. @@ -416,6 +457,9 @@ def template_match(template_engine, d): # Ensure that there are no symbol clashes with externally linked objects. graph.append_env('CFLAGS', '-fvisibility=hidden') +# Generate DWARF compatible with older T32 releases +graph.append_env('CFLAGS', '-gdwarf-4') + # Catch undefined switches during type system preprocessing graph.append_env('CPPFLAGS', '-Wundef') graph.append_env('CPPFLAGS', '-Werror') @@ -426,6 +470,14 @@ def template_match(template_engine, d): if variant_cppflags: graph.append_env('CPPFLAGS', ' '.join(variant_cppflags)) graph.append_env('CODEGEN_CONFIGS', ' '.join(variant_cppflags)) +if variant_ldflags: + graph.append_env('TARGET_LDFLAGS', ' '.join(variant_ldflags)) + +# On scons builds, the abs path may be put into the commandline, strip it out +# of the __FILE__ macro. +root = os.path.abspath(os.curdir) + os.sep +graph.append_env('CFLAGS', + '-fmacro-prefix-map={:s}={:s}'.format(root, '')) graph.append_env('TARGET_CPPFLAGS', '-nostdlibinc') graph.append_env('TARGET_LDFLAGS', '-nostdlib') @@ -485,6 +537,8 @@ def template_match(template_engine, d): '-Xanalyzer unroll-loops=true ' '-Xanalyzer -analyzer-config ' '-Xanalyzer ctu-dir=$CTU_DIR ' + '-Xanalyzer -analyzer-disable-checker ' + '-Xanalyzer alpha.core.FixedAddr ' '-o ${out} ' '${in}') @@ -522,6 +576,7 @@ def parse_module_conf(d, f): ) objs = [] have_events = False + for s in f.readlines(): words = s.split() if not words or words[0].startswith('#'): @@ -540,6 +595,12 @@ def parse_module_conf(d, f): for w in map(var_subst, words[1:]): event_sources.add(add_event_dsl(d, w, local_env)) have_events = True + elif words[0] == 'registers': + for w in map(var_subst, words[1:]): + f = os.path.join(d, w) + if f in registers: + raise KeyError("duplicate {:s}".format(f)) + registers.append(f) elif words[0] == 'local_include': add_include(d, 'include', local_env) elif words[0] == 'source': @@ -573,6 +634,13 @@ def parse_module_conf(d, f): event_sources.add(add_event_dsl( os.path.join(d, words[1]), w, local_env)) have_events = True + elif words[0] == 'arch_registers': + if arch_match(words[1]): + for w in map(var_subst, words[2:]): + f = os.path.join(d, words[1], w) + if f in registers: + raise KeyError("duplicate {:s}".format(f)) + registers.append(f) elif words[0] == 'arch_local_include': if arch_match(words[1]): add_include(d, os.path.join(words[1], 'include'), local_env) @@ -644,6 +712,15 @@ def parse_module_conf(d, f): if add_template(ts, d, words[2], w, src_requires, local_env, module): have_events = True + elif words[0] == 'assert_config': + test = ' '.join(words[1:]) + + result = eval(test, {}, configs_as_ints) + if result is True: + continue + logger.error('assert_config failed "%s" in module conf for %s', + test, d) + sys.exit(1) else: # TODO: dependencies, configuration variables, etc # Restructure this to use a proper parser first @@ -676,6 +753,12 @@ def parse_interface_conf(d, f): for w in map(var_subst, words[1:]): event_sources.add(add_event_dsl(d, w, local_env)) have_events = True + elif words[0] == 'registers': + for w in map(var_subst, words[1:]): + f = os.path.join(d, w) + if f in registers: + raise KeyError("duplicate {:s}".format(f)) + registers.append(f) elif words[0] == 'macros': for w in map(var_subst, words[1:]): add_macro_include(d, 'include', w) @@ -743,19 +826,13 @@ def add_flags(flags, local_env): local_env['LOCAL_CFLAGS'] += ' '.join(flags) -def add_global_define(d): - check_global_define(d) - variant_defines.append(d) - - def add_source_file(src, obj, requires, local_env): file_env = local_env.copy() - file_define = '-D__BUILD_FILE__=\\"{:s}\\"'.format(src) if 'LOCAL_CPPFLAGS' not in file_env: file_env['LOCAL_CPPFLAGS'] = '' else: file_env['LOCAL_CPPFLAGS'] += ' ' - file_env['LOCAL_CPPFLAGS'] += file_define + graph.add_target([obj], 'cc', [src], requires=requires, **file_env) objects.add(obj) @@ -858,7 +935,8 @@ def add_simple_template(d, t, requires, local_env, local_headers=False, else: logger.error("Unsupported template output: %s", out_name) sys.exit(1) - graph.add_target([o], 'code_gen', [i]) + graph.add_target([o], 'code_gen_asm' if out_ext == '.S' else 'code_gen', + [i]) event_handler_modules = set() @@ -948,6 +1026,9 @@ def get_event_src_file(module): graph.add_rule('code_gen', '${CODEGEN} ${CODEGEN_ARCHS} ${CODEGEN_CONFIGS} ' '-f ${FORMATTER} -o ${out} -d ${out}.d ${in}', depfile='${out}.d') +graph.add_rule('code_gen_asm', '${CODEGEN} ${CODEGEN_ARCHS} ' + '${CODEGEN_CONFIGS} -o ${out} -d ${out}.d ${in}', + depfile='${out}.d') # @@ -956,6 +1037,21 @@ def get_event_src_file(module): defmap = os.path.join(ctu_dir, "externalDefMap.txt") ast_gen = graph.future_alias(os.path.join(build_dir, 'ast-gen')) +# Get all configs as Ints or strings +configs_as_ints = dict() + + +def configs_get_int(c): + try: + s = configs[c].strip('uU') + return int(s, 0) + except ValueError: + return configs[c] + + +for c in configs: + configs_as_ints[c] = configs_get_int(c) + # # Collect the lists of objects, modules and interfaces # @@ -1143,7 +1239,7 @@ def add_object_type_template(module, template, object_str, target): for module_dir, target, arch, src_requires, is_module, local_env in \ typed_guestapi_templates: - assert(is_module) + assert (is_module) ext = os.path.splitext(target)[1] template = os.path.join(module_dir, arch, 'templates', target + '.tmpl') if ext == '.h': @@ -1176,6 +1272,7 @@ def add_object_type_template(module, template, object_str, target): hypercalls_guest_templates = (('guest_interface.c', hypguest_interface_src), ('guest_interface.h', hypguest_interface_header)) +# FIXME: # FIXME: upgrade Lark and remove LANG env workaround. graph.add_rule('hypercalls_gen', 'LANG=C.UTF-8' ' ${HYPERCALLS} -a ${ABI} -f ${FORMATTER}' @@ -1255,8 +1352,8 @@ def event_template(name): event_out = get_event_src_file(module) graph.add_target([event_out], 'event_gen', events_pickle, MODULE=module, TEMPLATE=relpath(event_src_tmpl), - OPTIONS='-f ${FORMATTER}', depends=[event_src_tmpl]) +# OPTIONS='-f ${FORMATTER}', # An alias target is used to order header generation before source compliation graph.add_alias(event_headers_gen, event_headers) @@ -1281,18 +1378,22 @@ def event_template(name): graph.add_rule('registers_gen', '${REGISTERS} -t ${TEMPLATE} -f ${FORMATTER} ' '-o ${out} ${in}') -# Pre-process the register script -# FIXME: add build.conf support for more flexible location(s) specification -registers_base = os.path.join(arch_base, registers_arch) -registers_in = os.path.join(registers_base, 'registers.reg') -registers_template = os.path.join(registers_base, 'registers.tmpl') +registers_pp = list() + +# Pre-process the register scripts +for f in registers: + f_pp = os.path.join(build_dir, f + '.pp') + graph.add_target([f_pp], 'cpp-dsl', [f]) + registers_pp.append(f_pp) -registers_pp = os.path.join(build_dir, 'registers.reg.pp') -graph.add_target([registers_pp], 'cpp-dsl', [registers_in]) +for module_dir, target, arch, src_requires, is_module, local_env in \ + registers_templates: + template = os.path.join(module_dir, arch, 'templates', target + '.tmpl') -graph.add_target([registers_header], 'registers_gen', [registers_pp], - TEMPLATE=relpath(registers_template), - depends=[registers_template, registers_script]) + header = os.path.join(build_includes, target) + graph.add_target([header], 'registers_gen', registers_pp, + TEMPLATE=relpath(template), + depends=[template, registers_script]) # # Build version setup diff --git a/tools/build/gen_ver.py b/tools/build/gen_ver.py index ce0fd70..2d7b89b 100755 --- a/tools/build/gen_ver.py +++ b/tools/build/gen_ver.py @@ -56,7 +56,10 @@ def main(): time = ret.stdout.decode("utf-8").strip() time = time.replace('+0000', 'UTC') - out = '#define HYP_GIT_VERSION {:s}\n'.format(id) + out = '// This file is automatically generated.\n' + out += '// Do not manually resolve conflicts! Contact ' \ + 'hypervisor.team for assistance.\n' + out += '#define HYP_GIT_VERSION {:s}\n'.format(id) out += '#define HYP_BUILD_DATE \"{:s}\"\n'.format(time) if options.output: diff --git a/tools/build/gen_ver.sh b/tools/build/gen_ver.sh index 23248da..6de3930 100755 --- a/tools/build/gen_ver.sh +++ b/tools/build/gen_ver.sh @@ -5,6 +5,8 @@ status=`git diff HEAD --quiet || echo '-dirty'` +echo '// This file is automatically generated.' +echo '// Do not manually resolve conflicts! Contact hypervisor.team for assistance.' echo "#define HYP_GIT_VERSION `git rev-parse --short HEAD`$status" if [ -z "$status" ] diff --git a/tools/codegen/codegen.py b/tools/codegen/codegen.py index 49d2884..afd90ae 100755 --- a/tools/codegen/codegen.py +++ b/tools/codegen/codegen.py @@ -81,7 +81,7 @@ def main(): defines.update(options.defines) if options.imacros: - d = re.compile(r'#define (?P\w+)(\s+\"?(?P[\w0-9\ ]+)\"?)?') + d = re.compile(r'#define (?P\w+)(\s+\"?(?P[\w0-9,\ ]+)\"?)?') for line in options.imacros.readlines(): match = d.search(line) define = match.group('def') @@ -94,13 +94,14 @@ def main(): except AttributeError: pass except ValueError: - val = True + pass if define in defines: raise Exception("multiply defined: {}\n", define) defines[define] = val + sys.path.append(os.path.dirname(options.template.name)) output = str(Template(file=options.template, searchList=(defines, {'arch_list': options.archs}))) diff --git a/tools/cpptest/Checkers_Man_All_Req.properties b/tools/cpptest/Checkers_Man_All_Req.properties new file mode 100644 index 0000000..b16bba1 --- /dev/null +++ b/tools/cpptest/Checkers_Man_All_Req.properties @@ -0,0 +1,380 @@ +CERT_C-ENV33-a=true +CERT_C-STR02-a=true +CERT_C-STR02-b=true +CERT_C-STR02-c=true +CERT_C-STR07-a=true +com.parasoft.xtest.checkers.api.common.fullBuild=false +com.parasoft.xtest.checkers.api.common.incrementalBuild=false +com.parasoft.xtest.checkers.api.config.name=MISRA C 2012 +com.parasoft.xtest.checkers.api.config.path=Compliance Packs/Automotive Pack +com.parasoft.xtest.checkers.api.config.tool=3 +com.parasoft.xtest.checkers.api.config.version=9.5.0 +com.parasoft.xtest.checkers.api.config.xtestVersion=1.0.1 +com.parasoft.xtest.execution.api.enabled=false +#com.parasoft.xtest.standards.api.cap_errors_per_rule=false +com.parasoft.xtest.standards.api.enabled=true +com.parasoft.xtest.standards.api.max_errors_per_rule=10000 +com.parasoft.xtest.testgen.api.enabled=false +#Compliance Standard Misra C 2012 +MISRAC2012-DIR_4_10-a=true +MISRAC2012-DIR_4_11-a=true +MISRAC2012-DIR_4_12-a=false +MISRAC2012-DIR_4_13-a=true +MISRAC2012-DIR_4_13-b=true +MISRAC2012-DIR_4_13-c=true +MISRAC2012-DIR_4_13-d=true +MISRAC2012-DIR_4_13-e=true +MISRAC2012-DIR_4_14-a=true +MISRAC2012-DIR_4_14-b=true +MISRAC2012-DIR_4_14-c=true +MISRAC2012-DIR_4_14-d=true +MISRAC2012-DIR_4_14-e=true +MISRAC2012-DIR_4_14-f=true +MISRAC2012-DIR_4_14-g=true +MISRAC2012-DIR_4_14-h=true +MISRAC2012-DIR_4_14-i=true +MISRAC2012-DIR_4_14-j=true +MISRAC2012-DIR_4_14-k=true +MISRAC2012-DIR_4_14-l=true +MISRAC2012-DIR_4_15-a=true +MISRAC2012-DIR_4_1-a=true +MISRAC2012-DIR_4_1-b=true +MISRAC2012-DIR_4_1-c=true +MISRAC2012-DIR_4_1-d=true +MISRAC2012-DIR_4_1-e=true +MISRAC2012-DIR_4_1-f-arbitariesInInternalLinkage=false +MISRAC2012-DIR_4_1-f=true +MISRAC2012-DIR_4_1-f-visibilityThresholdEnabled=true +MISRAC2012-DIR_4_1-f-visibilityThreshold=PRIVATE +MISRAC2012-DIR_4_1-g=true +MISRAC2012-DIR_4_1-h=true +MISRAC2012-DIR_4_1-i=true +MISRAC2012-DIR_4_1-j=true +MISRAC2012-DIR_4_2-a=true +MISRAC2012-DIR_4_3-a=true +MISRAC2012-DIR_4_4-a=true +MISRAC2012-DIR_4_5-a=true +MISRAC2012-DIR_4_6-a=true +MISRAC2012-DIR_4_6-b=true +MISRAC2012-DIR_4_6-c=true +MISRAC2012-DIR_4_7-a=true +MISRAC2012-DIR_4_7-b=true +MISRAC2012-DIR_4_8-a=false +MISRAC2012-DIR_4_9-a=false +MISRAC2012-RULE_10_1-a=true +MISRAC2012-RULE_10_1-b=true +MISRAC2012-RULE_10_1-c=true +MISRAC2012-RULE_10_1-d=true +MISRAC2012-RULE_10_1-e=true +MISRAC2012-RULE_10_1-f=true +MISRAC2012-RULE_10_1-g=true +MISRAC2012-RULE_10_2-a=true +MISRAC2012-RULE_10_3-a=true +MISRAC2012-RULE_10_3-b=true +MISRAC2012-RULE_10_4-a=true +MISRAC2012-RULE_10_4-b=true +MISRAC2012-RULE_10_5-a=true +MISRAC2012-RULE_10_5-b=true +MISRAC2012-RULE_10_5-c=true +MISRAC2012-RULE_10_6-a=true +MISRAC2012-RULE_10_7-a=true +MISRAC2012-RULE_10_7-b=true +MISRAC2012-RULE_10_8-a=true +MISRAC2012-RULE_11_1-a=true +MISRAC2012-RULE_11_1-b=true +MISRAC2012-RULE_11_2-a=true +MISRAC2012-RULE_11_3-a=true +MISRAC2012-RULE_11_4-a=true +MISRAC2012-RULE_11_5-a=true +MISRAC2012-RULE_11_6-a=true +MISRAC2012-RULE_11_7-a=true +MISRAC2012-RULE_11_8-a=true +MISRAC2012-RULE_11_9-a=true +MISRAC2012-RULE_11_9-b=true +MISRAC2012-RULE_1_1-a=false +MISRAC2012-RULE_1_1-b=false +MISRAC2012-RULE_1_1-c=false +MISRAC2012-RULE_1_1-d=true +MISRAC2012-RULE_12_1-a=true +MISRAC2012-RULE_12_1-b=false +MISRAC2012-RULE_12_1-c=true +MISRAC2012-RULE_12_2-a=true +MISRAC2012-RULE_12_3-a=false +MISRAC2012-RULE_12_4-a=true +MISRAC2012-RULE_12_4-b=true +MISRAC2012-RULE_12_5-a=true +MISRAC2012-RULE_13_1-a=true +MISRAC2012-RULE_13_2-a=true +MISRAC2012-RULE_13_2-b=true +MISRAC2012-RULE_13_2-c=true +MISRAC2012-RULE_13_2-d=true +MISRAC2012-RULE_13_2-e=true +MISRAC2012-RULE_13_2-f=true +MISRAC2012-RULE_13_2-g=true +MISRAC2012-RULE_13_3-a=true +MISRAC2012-RULE_13_4-a=true +MISRAC2012-RULE_13_5-a=true +MISRAC2012-RULE_13_6-a=true +MISRAC2012-RULE_13_6-b=true +MISRAC2012-RULE_13_6-c=true +MISRAC2012-RULE_1_3-a=true +MISRAC2012-RULE_1_3-b=true +MISRAC2012-RULE_1_3-c=true +MISRAC2012-RULE_1_3-d=true +MISRAC2012-RULE_1_3-e=true +MISRAC2012-RULE_1_3-f=true +MISRAC2012-RULE_1_3-g=true +MISRAC2012-RULE_1_3-h=true +MISRAC2012-RULE_1_3-i=true +MISRAC2012-RULE_1_3-j=true +MISRAC2012-RULE_1_3-k=true +MISRAC2012-RULE_1_3-l=true +MISRAC2012-RULE_1_3-m=true +MISRAC2012-RULE_1_3-n=true +MISRAC2012-RULE_1_3-o=true +MISRAC2012-RULE_1_5-a=true +MISRAC2012-RULE_1_5-b=true +MISRAC2012-RULE_1_5-c=true +MISRAC2012-RULE_1_5-d=true +MISRAC2012-RULE_1_5-e=true +MISRAC2012-RULE_1_5-f=true +MISRAC2012-RULE_1_5-g=true +MISRAC2012-RULE_14_1-a=true +MISRAC2012-RULE_14_1-b=true +MISRAC2012-RULE_14_2-a=true +MISRAC2012-RULE_14_2-b=true +MISRAC2012-RULE_14_2-c=true +MISRAC2012-RULE_14_2-d=true +MISRAC2012-RULE_14_3-aa=true +MISRAC2012-RULE_14_3-ab=true +MISRAC2012-RULE_14_3-ac=true +MISRAC2012-RULE_14_3-ad=true +MISRAC2012-RULE_14_3-a=true +MISRAC2012-RULE_14_3-b=true +MISRAC2012-RULE_14_3-c=true +MISRAC2012-RULE_14_3-d=true +MISRAC2012-RULE_14_3-e=true +MISRAC2012-RULE_14_3-f=true +MISRAC2012-RULE_14_3-g=true +MISRAC2012-RULE_14_3-h=true +MISRAC2012-RULE_14_3-i=true +MISRAC2012-RULE_14_3-j=true +MISRAC2012-RULE_14_3-k=true +MISRAC2012-RULE_14_3-l=true +MISRAC2012-RULE_14_3-m=true +MISRAC2012-RULE_14_3-n=true +MISRAC2012-RULE_14_3-o=true +MISRAC2012-RULE_14_3-p=true +MISRAC2012-RULE_14_3-q=true +MISRAC2012-RULE_14_3-r=true +MISRAC2012-RULE_14_3-s=true +MISRAC2012-RULE_14_3-t=true +MISRAC2012-RULE_14_3-u=true +MISRAC2012-RULE_14_3-v=true +MISRAC2012-RULE_14_3-w=true +MISRAC2012-RULE_14_3-x=true +MISRAC2012-RULE_14_3-y=true +MISRAC2012-RULE_14_3-z=true +MISRAC2012-RULE_14_4-a=true +MISRAC2012-RULE_15_1-a=false +MISRAC2012-RULE_15_2-a=true +MISRAC2012-RULE_15_3-a=true +MISRAC2012-RULE_15_4-a=false +MISRAC2012-RULE_15_5-a=true +MISRAC2012-RULE_15_6-a=true +MISRAC2012-RULE_15_6-b=true +MISRAC2012-RULE_15_7-a=true +MISRAC2012-RULE_16_1-a=true +MISRAC2012-RULE_16_1-b=true +MISRAC2012-RULE_16_1-c=true +MISRAC2012-RULE_16_1-d=true +MISRAC2012-RULE_16_1-e=true +MISRAC2012-RULE_16_1-f=true +MISRAC2012-RULE_16_1-g=true +MISRAC2012-RULE_16_1-h=true +MISRAC2012-RULE_16_2-a=true +MISRAC2012-RULE_16_3-a=true +MISRAC2012-RULE_16_3-b=true +MISRAC2012-RULE_16_4-a=true +MISRAC2012-RULE_16_4-b=true +MISRAC2012-RULE_16_5-a=true +MISRAC2012-RULE_16_6-a=true +MISRAC2012-RULE_16_7-a=true +MISRAC2012-RULE_16_7-b=true +MISRAC2012-RULE_17_1-a=true +MISRAC2012-RULE_17_1-b=true +MISRAC2012-RULE_17_2-a=true +MISRAC2012-RULE_17_3-a=true +MISRAC2012-RULE_17_4-a=true +MISRAC2012-RULE_17_5-a=true +MISRAC2012-RULE_17_6-a=true +MISRAC2012-RULE_17_7-a=true +MISRAC2012-RULE_17_7-b=true +MISRAC2012-RULE_17_8-a=true +MISRAC2012-RULE_17_9-a=true +MISRAC2012-RULE_18_1-a=true +MISRAC2012-RULE_18_1-b=true +MISRAC2012-RULE_18_1-c=true +MISRAC2012-RULE_18_2-a=true +MISRAC2012-RULE_18_3-a=true +MISRAC2012-RULE_18_4-a=false +MISRAC2012-RULE_18_5-a=true +MISRAC2012-RULE_18_6-a=true +MISRAC2012-RULE_18_6-b=true +MISRAC2012-RULE_18_7-a=true +MISRAC2012-RULE_18_8-a=true +MISRAC2012-RULE_18_9-a=true +MISRAC2012-RULE_19_1-a=true +MISRAC2012-RULE_19_1-b=true +MISRAC2012-RULE_19_1-c=true +MISRAC2012-RULE_19_2-a=false +MISRAC2012-RULE_20_10-a=false +MISRAC2012-RULE_20_11-a=true +MISRAC2012-RULE_20_12-a=true +MISRAC2012-RULE_20_13-a=true +MISRAC2012-RULE_20_14-a=true +MISRAC2012-RULE_20_1-a=true +MISRAC2012-RULE_20_2-a=true +MISRAC2012-RULE_20_2-b=true +MISRAC2012-RULE_20_3-a=true +MISRAC2012-RULE_20_4-a=true +MISRAC2012-RULE_20_4-b=true +MISRAC2012-RULE_20_5-a=true +MISRAC2012-RULE_20_6-a=true +MISRAC2012-RULE_20_7-a=true +MISRAC2012-RULE_20_8-a=true +MISRAC2012-RULE_20_9-a=true +MISRAC2012-RULE_20_9-b=true +MISRAC2012-RULE_21_10-a=true +MISRAC2012-RULE_21_11-a=true +MISRAC2012-RULE_21_12-a=true +MISRAC2012-RULE_21_13-a=true +MISRAC2012-RULE_21_14-a=true +MISRAC2012-RULE_21_15-a=true +MISRAC2012-RULE_21_16-a=true +MISRAC2012-RULE_21_17-a=true +MISRAC2012-RULE_21_17-b=true +MISRAC2012-RULE_21_18-a=true +MISRAC2012-RULE_21_19-a=true +MISRAC2012-RULE_21_19-b=true +MISRAC2012-RULE_21_1-a=true +MISRAC2012-RULE_21_1-b=false +MISRAC2012-RULE_21_1-c=true +MISRAC2012-RULE_21_1-d=true +MISRAC2012-RULE_21_20-a=true +MISRAC2012-RULE_21_21-a=true +MISRAC2012-RULE_21_22-a=true +MISRAC2012-RULE_21_23-a=true +MISRAC2012-RULE_21_24-a=true +MISRAC2012-RULE_21_2-a=true +MISRAC2012-RULE_21_2-b=false +MISRAC2012-RULE_21_2-c=true +MISRAC2012-RULE_21_3-a=false +MISRAC2012-RULE_21_4-a=true +MISRAC2012-RULE_21_4-b=true +MISRAC2012-RULE_21_5-a=true +MISRAC2012-RULE_21_5-b=true +MISRAC2012-RULE_21_6-a=true +MISRAC2012-RULE_21_7-a=true +MISRAC2012-RULE_21_8-a=true +MISRAC2012-RULE_21_9-a=true +MISRAC2012-RULE_2_1-a=true +MISRAC2012-RULE_2_1-b=true +MISRAC2012-RULE_2_1-c=true +MISRAC2012-RULE_2_1-d=true +MISRAC2012-RULE_2_1-e=true +MISRAC2012-RULE_2_1-f=true +MISRAC2012-RULE_2_1-g=true +MISRAC2012-RULE-22_10-a-reportOnMissingErrnoCheck=false +MISRAC2012-RULE_22_10-a-reportWhenErrnoIsNotZero=false +MISRAC2012-RULE_22_10-a=true +MISRAC2012-RULE_22_1-a-fieldsStoreResources=false +MISRAC2012-RULE_22_1-a-nonMemberMethodsStoreResource=false +MISRAC2012-RULE_22_1-a-patternName=^malloc|calloc|realloc|fopen$ +MISRAC2012-RULE_22_1-a-patternNameMethodsStore=true +MISRAC2012-RULE_22_1-a-reportUnvalidatedViolations=false +MISRAC2012-RULE_22_1-a-storeByTPMethods=false +MISRAC2012-RULE_22_1-a=true +MISRAC2012-RULE_22_2-a=true +MISRAC2012-RULE_22_2-b=true +MISRAC2012-RULE_22_3-a=true +MISRAC2012-RULE_22_4-a=true +MISRAC2012-RULE_22_5-a=true +MISRAC2012-RULE_22_5-b=true +MISRAC2012-RULE_22_6-a=true +MISRAC2012-RULE_22_7-a=true +MISRAC2012-RULE_22_8-a-reportOnMissingErrnoCheck=false +MISRAC2012-RULE_22_8-a-reportOnUnnecessaryErrnoCheck=false +MISRAC2012-RULE_22_8-a=true +MISRAC2012-RULE_22_9-a-reportOnUnnecessaryErrnoCheck=false +MISRAC2012-RULE_22_9-a-reportWhenErrnoIsNotZero=false +MISRAC2012-RULE_22_9-a=true +MISRAC2012-RULE_2_2-a=true +MISRAC2012-RULE_2_3-a=false +MISRAC2012-RULE_2_3-b=false +MISRAC2012-RULE_2_4-a=false +MISRAC2012-RULE_2_4-b=false +MISRAC2012-RULE_2_5-a=false +MISRAC2012-RULE_2_6-a=false +MISRAC2012-RULE_2_7-a=false +MISRAC2012-RULE_3_1-a=true +MISRAC2012-RULE_3_1-b=true +MISRAC2012-RULE_3_1-c=true +MISRAC2012-RULE_3_2-a=true +MISRAC2012-RULE_4_1-a=true +MISRAC2012-RULE_4_2-a=true +MISRAC2012-RULE_5_1-a=false +MISRAC2012-RULE_5_2-a=false +MISRAC2012-RULE_5_2-b=true +MISRAC2012-RULE_5_2-c=false +MISRAC2012-RULE_5_2-d=true +MISRAC2012-RULE_5_3-a=true +MISRAC2012-RULE_5_3-b=true +MISRAC2012-RULE_5_4-a=false +MISRAC2012-RULE_5_4-b=true +MISRAC2012-RULE_5_4-c=false +MISRAC2012-RULE_5_4-d=true +MISRAC2012-RULE_5_5-a=false +MISRAC2012-RULE_5_5-b=true +MISRAC2012-RULE_5_6-a=true +MISRAC2012-RULE_5_6-b=true +MISRAC2012-RULE_5_7-a=true +MISRAC2012-RULE_5_7-b=true +MISRAC2012-RULE_5_8-a=true +MISRAC2012-RULE_5_9-a=true +MISRAC2012-RULE_5_9-b=true +MISRAC2012-RULE_6_1-a=true +MISRAC2012-RULE_6_2-a=true +MISRAC2012-RULE_7_1-a=true +MISRAC2012-RULE_7_2-a=true +MISRAC2012-RULE_7_3-a=true +MISRAC2012-RULE_7_4-a=true +MISRAC2012-RULE_8_10-a=true +MISRAC2012-RULE_8_11-a=true +MISRAC2012-RULE_8_12-a=true +MISRAC2012-RULE_8_13-a=true +MISRAC2012-RULE_8_13-b=true +MISRAC2012-RULE_8_14-a=true +MISRAC2012-RULE_8_15-a=true +MISRAC2012-RULE_8_15-b=true +MISRAC2012-RULE_8_16-a=true +MISRAC2012-RULE_8_17-a=true +MISRAC2012-RULE_8_1-a=true +MISRAC2012-RULE_8_1-b=true +MISRAC2012-RULE_8_2-a=true +MISRAC2012-RULE_8_2-b=true +MISRAC2012-RULE_8_2-c=true +MISRAC2012-RULE_8_3-a=true +MISRAC2012-RULE_8_3-b=true +MISRAC2012-RULE_8_4-a=true +MISRAC2012-RULE_8_4-b=true +MISRAC2012-RULE_8_5-a=true +MISRAC2012-RULE_8_6-a=false +MISRAC2012-RULE_8_7-a=true +MISRAC2012-RULE_8_8-a=true +MISRAC2012-RULE_8_9-a=true +MISRAC2012-RULE_9_1-a=true +MISRAC2012-RULE_9_2-a=true +MISRAC2012-RULE_9_3-a=true +MISRAC2012-RULE_9_4-a=true +MISRAC2012-RULE_9_5-a=true diff --git a/tools/cpptest/misra_xml_to_json.py b/tools/cpptest/misra_xml_to_json.py new file mode 100755 index 0000000..596f7dc --- /dev/null +++ b/tools/cpptest/misra_xml_to_json.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# coding: utf-8 +# +# © 2023 Qualcomm Innovation Center, Inc. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Run as a part of gitlab CI, after Parasoft reports have been generated. + +This script converts the Parasoft XML-format report to a Code Climate +compatible json file, that gitlab code quality can interpret. +""" + +import xml.etree.ElementTree as ET +import json +import argparse +import sys +import os +import re + +argparser = argparse.ArgumentParser( + description="Convert Parasoft XML to Code Climate JSON") +argparser.add_argument('input', type=argparse.FileType('r'), nargs='?', + default=sys.stdin, help="the Parasoft XML input") +argparser.add_argument('--output', '-o', type=argparse.FileType('w'), + default=sys.stdout, help="the Code Climate JSON output") +args = argparser.parse_args() + +tree = ET.parse(args.input) + +parasoft_viols = tree.findall(".//StdViol") + tree.findall(".//FlowViol") + +cc_viols = [] + +severity_map = { + 1: "blocker", + 2: "critical", + 3: "major", + 4: "minor", + 5: "info", +} + +deviation_map = { + 'MISRAC2012-RULE_20_12-a': [ + (None, re.compile(r"parameter of potential macro 'assert'")), + ], + # False positives due to __c11 builtins taking int memory order arguments + # instead of enum + 'MISRAC2012-RULE_10_3-b': [ + (None, re.compile(r"number '2'.*'essentially Enum'.*" + r"'__c11_atomic_load'.*'essentially signed'")), + (None, re.compile(r"number '3'.*'essentially Enum'.*" + r"'__c11_atomic_(store'|fetch_).*" + r"'essentially signed'")), + (None, re.compile(r"number '[45]'.*'essentially Enum'.*" + r"'__c11_atomic_compare_exchange_(strong|weak)'.*" + r"'essentially signed'")), + ], + # False positives with unknown cause: the return value of assert_if_const() + # is always used, to determine whether to call assert_failed() + 'MISRAC2012-RULE_17_7-b': [ + (None, re.compile(r'"assert_if_const"')), + ], + # Advisory rule which is impractical to enforce for generated accessors, + # since the type system has no information about which accessors are used. + 'MISRAC2012-RULE_8_7-a': [ + (re.compile(r'^build/.*/accessors\.c$'), None), + ], +} + + +def matches_deviation(v): + rule = v.attrib['rule'] + if rule not in deviation_map: + return False + + msg = v.attrib['msg'] + path = v.attrib['locFile'].split(os.sep, 2)[2] + + def check_constraint(constraint, value): + if constraint is None: + return True + try: + return constraint.search(value) + except AttributeError: + return constraint == value + + for d_path, d_msg in deviation_map[rule]: + if check_constraint(d_path, path) and check_constraint(d_msg, msg): + return True + + return False + + +cc_viols = [ + ({ + "type": "issue", + "categories": ["Bug Risk"], + "severity": ('info' if matches_deviation(v) + else severity_map[int(v.attrib['sev'])]), + "check_name": v.attrib['rule'], + "description": (v.attrib['msg'] + '. ' + + v.attrib['rule.header'] + '. (' + + v.attrib['rule'] + ')'), + "fingerprint": v.attrib['unbViolId'], + "location": { + "path": v.attrib['locFile'].split(os.sep, 2)[2], + "lines": { + "begin": int(v.attrib['locStartln']), + "end": int(v.attrib['locEndLn']) + } + } + }) + for v in parasoft_viols] + +args.output.write(json.dumps(cc_viols)) +args.output.close() diff --git a/tools/debug/tracebuf.py b/tools/debug/tracebuf.py index 4c61af7..b5b9ec9 100755 --- a/tools/debug/tracebuf.py +++ b/tools/debug/tracebuf.py @@ -39,6 +39,8 @@ 38: "VGIC_SGI", 39: "VGIC_ITS_COMMAND", 40: "VGIC_ROUTE", + 41: "VGIC_ICC_WRITE", + 42: "VGIC_ASYNC_EVENT", 48: "PSCI_PSTATE_VALIDATION", 49: "PSCI_VPM_STATE_CHANGED", 50: "PSCI_VPM_SYSTEM_SUSPEND", @@ -245,6 +247,23 @@ def read_entries(args): cpu_mask = struct.unpack(endian + 'QQQQ', header[8:40]) + cpu_mask = cpu_mask[0] | (cpu_mask[1] << 64) | (cpu_mask[2] << 128) | \ + (cpu_mask[2] << 192) + global_buffer = cpu_mask == 0 + + cpus = '' + while cpu_mask != 0: + msb = cpu_mask.bit_length() - 1 + cpus += '{:d}'.format(msb) + cpu_mask &= ~(1 << msb) + if cpu_mask != 0: + cpus += ',' + + if global_buffer: + print("Processing global buffer...") + else: + print("Processing CPU {:s} buffer...".format(cpus)) + entries_max = struct.unpack(endian + 'L', header[4:8])[0] head_index = struct.unpack(endian + 'L', header[40:44])[0] @@ -258,31 +277,30 @@ def read_entries(args): if entry_count == 0: # Empty buffer, skip over the unused bytes - print("Empty buffer") + print(" Empty buffer") args.input.seek(entries_max * 64, 1) return iter(()) + else: + print(" Found {:d} entries. Wrapped: {}".format(entry_count, wrapped)) + warn = True entries = [] for i in range(entry_count): trace = args.input.read(64) + if len(trace) < 64: + print(" Warning, log truncated. Read {:d} of {:d} entries".format( + i, entry_count)) + break try: entries.append(Event(args, *struct.unpack(endian + "QQQQQQQQ", trace))) except ValueError: + if warn: + print(" Warning, bad input. Read {:d} of {:d} entries".format( + i, entry_count)) + warn = False pass - cpu_mask = cpu_mask[0] | (cpu_mask[1] << 64) | (cpu_mask[2] << 128) | \ - (cpu_mask[2] << 192) - global_buffer = cpu_mask == 0 - - cpus = '' - while cpu_mask != 0: - msb = cpu_mask.bit_length() - 1 - cpus += '{:d}'.format(msb) - cpu_mask &= ~(1 << msb) - if cpu_mask != 0: - cpus += ',' - if args.sort == 'm': if global_buffer: header_string = "=== GLOBAL TRACES START ===\n" @@ -294,7 +312,7 @@ def read_entries(args): else: header_string = "=== CPU {:s} TRACE ===\n".format(cpus) - if not wrapped: + if not wrapped or (head_index == entries_max): first_index = 0 else: first_index = head_index @@ -311,7 +329,10 @@ def read_entries(args): if not wrapped: # Skip over the unused bytes - args.input.seek((entries_max - head_index) * 64, 1) + if args.input.seekable(): + args.input.seek((entries_max - head_index) * 64, 1) + else: + args.input.read((entries_max - head_index) * 64) return entry_iter diff --git a/tools/elf/package_apps.py b/tools/elf/package_apps.py index a394a14..d058258 100755 --- a/tools/elf/package_apps.py +++ b/tools/elf/package_apps.py @@ -165,7 +165,7 @@ def insert_segment(self, newseg, phys): idx += 1 # print(seg, hex(seg.header.p_paddr)) last = seg.header.p_offset + seg.header.p_filesz - assert(last >= p_adj) + assert (last >= p_adj) p_adj = last p_prev = p_adj @@ -187,7 +187,7 @@ def insert_segment(self, newseg, phys): # Update file offsets of remaining segments for seg in self.segments[idx+1:]: last = seg.header.p_offset + seg.header.p_filesz - assert(last >= p_prev) + assert (last >= p_prev) p_next = seg.header.p_offset + p_off p_adj = (p_next + (seg.header.p_align - 1)) & \ (0xffffffffffffffff ^ (seg.header.p_align - 1)) diff --git a/tools/events/ir.py b/tools/events/ir.py index ff30074..6e9ad6f 100644 --- a/tools/events/ir.py +++ b/tools/events/ir.py @@ -139,9 +139,16 @@ class Priority(float, IRObject): class Result(IRObject): - def __init__(self, children): - self.type = _first_of_type(children, Type) - self.default = _first_of_type(children, ConstExpr) + def __init__(self, children, void=False): + try: + self.type = _first_of_type(children, Type) + self.default = _first_of_type(children, ConstExpr) + except StopIteration: + if void: + self.type = Type('void') + self.default = None + else: + raise StopIteration class ExpectedArgs(list, IRObject): @@ -173,7 +180,8 @@ def __init__(self, children): (c.name, c) for c in children if isinstance(c, Param)) def set_owner(self, module): - self.module = module + self.module_name = module.name + self.module_includes = module.includes @abc.abstractmethod def subscribe(self, subscription): @@ -194,6 +202,10 @@ def lock_opts(self): def return_type(self): raise NotImplementedError + @abc.abstractproperty + def noreturn(self): + raise NotImplementedError + def param(self, name): return self._param_dict[name] @@ -272,16 +284,41 @@ def finalise(self): for lock_opt in s.lock_opts: lock_opt.apply(acquires, releases, requires, excludes, kinds) lock_opts = [] - for lock in acquires: + for lock in sorted(acquires): lock_opts.append(LockAnnotation('acquire', kinds[lock], lock)) - for lock in releases: + for lock in sorted(releases): lock_opts.append(LockAnnotation('release', kinds[lock], lock)) - for lock in requires: + for lock in sorted(requires): lock_opts.append(LockAnnotation('require', kinds[lock], lock)) - for lock in excludes: + for lock in sorted(excludes): lock_opts.append(LockAnnotation('exclude', kinds[lock], lock)) self._lock_opts = tuple(lock_opts) + noreturn = (self._subscribers and self._subscribers[-1].handler and + self._subscribers[-1].handler.noreturn) + if noreturn and self.return_type != 'void': + s = self._subscribers[-1] + n = s.handler.noreturn + logger.error("%s:%d:%d: error: last handler %s for event %s must " + "return, but is declared as noreturn", + n.meta.filename, n.meta.line, n.meta.column, + s.handler, self.name) + raise DSLError() + for s in self._subscribers[:-1]: + if s.handler is not None and s.handler.noreturn: + n = s.handler.noreturn + logger.error("%s:%d:%d: error: handler %s for event %s does " + "not return, but is not the last handler (%s)", + n.meta.filename, n.meta.line, n.meta.column, + s.handler, self.name, + self._subscribers[-1].handler) + raise DSLError() + self._noreturn = noreturn + + @property + def noreturn(self): + return self._noreturn + @property def subscribers(self): return self._subscribers @@ -430,6 +467,12 @@ def lock_opts(self): def return_type(self): return self.result.type + @property + def noreturn(self): + # Note: this could be true if the selector is a enum type that is + # covered by noreturn handlers. We're not likely to ever do that. + return False + @property def unused_param_names(self): return super().unused_param_names - {self.selector.name} @@ -443,6 +486,10 @@ class Public(IRObject): pass +class NoReturn(IRObject): + pass + + class Subscription(IRObject): def __init__(self, children): self.event_name = _first_of_type(children, Symbol) @@ -451,13 +498,14 @@ def __init__(self, children): self.handler = _first_of_type_opt(children, Handler) self.constant = _first_of_type_opt(children, Constant) if self.handler is None and self.constant is None: - self.handler = Handler(_first_of_type_opt(children, ExpectedArgs)) + self.handler = Handler(_first_of_type_opt(children, ExpectedArgs), + _first_of_type_opt(children, NoReturn)) self.unwinder = _first_of_type_opt(children, Unwinder) self.priority = _first_of_type_opt(children, Priority) self.lock_opts = _all_of_type(children, LockAnnotation) def set_owner(self, module): - self.module = module + self.module_name = module.name def resolve(self, events): try: @@ -490,10 +538,11 @@ def __init__(self, *children): self.name = _first_of_type_opt(children, Symbol) self.args = _first_of_type_opt(children, ExpectedArgs) self.public = any(c for c in children if isinstance(c, Public)) + self._noreturn = _first_of_type_opt(children, NoReturn) def resolve(self, subscription): self.subscription = subscription - self.module = subscription.module + self.module_name = subscription.module_name self.event = subscription.event if self.name is None: @@ -518,9 +567,13 @@ def _default_name(self): def _available_params(self): return self.event.param_names + @property + def noreturn(self): + return self._noreturn + @property def return_type(self): - return self.event.return_type + return self.event.return_type if not self.noreturn else 'void' @property def params(self): @@ -538,17 +591,22 @@ def __lt__(self, other): def __str__(self): return self.name + def __hash__(self): + """Generate a unique hash for the function.""" + return hash((self.name, self.return_type) + + tuple((p.name, p.type) for p in self.params)) + class Handler(AbstractFunction): @property def _default_name(self): - return "{:s}_handle_{:s}".format(self.module.name, self.event.name) + return "{:s}_handle_{:s}".format(self.module_name, self.event.name) class Unwinder(AbstractFunction): @property def _default_name(self): - return "{:s}_unwind_{:s}".format(self.module.name, self.event.name) + return "{:s}_unwind_{:s}".format(self.module_name, self.event.name) @property def _available_params(self): @@ -617,30 +675,34 @@ def handlers(self): # Each of these may be used by multiple subscriptions, either to # different events, or to the same selector event with different # selections, or even repeatedly for one event. - # - # Currently we do not check that the arguments are consistent, since - # this means determining whether C types are compatible — we leave - # that to the C compiler. - seen_handlers = set() + seen_handlers = dict() for s in self.subscriptions: for h in s.all_handlers: - if h in seen_handlers: + if h.name in seen_handlers: + if seen_handlers[h.name] != hash(h): + logger.error("handler decl mismatch: %s", + h.name) + raise DSLError() continue - seen_handlers.add(h) + seen_handlers[h.name] = hash(h) yield h @property - def called_handlers(self): - # Unique event handlers called by this module's events. - seen_handlers = set() + def declared_handlers(self): + # Unique event handlers declared by this module's events. + seen_handlers = dict() for e in self.events: for s in e.subscribers: for h in s.all_handlers: - if h in seen_handlers: + if h.name in seen_handlers: + if seen_handlers[h.name] != hash(h): + logger.error("handler decl mismatch: %s", + h.name) + raise DSLError() continue if h.public: continue - seen_handlers.add(h) + seen_handlers[h.name] = hash(h) yield h @property @@ -652,12 +714,12 @@ def handler_includes(self): if e is NotImplemented: continue - m = e.module + m = e.module_name if m in seen_modules: continue seen_modules.add(m) - for i in m.includes: + for i in e.module_includes: if i in seen_includes: continue seen_includes.add(i) diff --git a/tools/events/parser.py b/tools/events/parser.py index 836f7f7..28c671d 100644 --- a/tools/events/parser.py +++ b/tools/events/parser.py @@ -9,7 +9,7 @@ Include, Symbol, Type, ConstExpr, Priority, Result, ExpectedArgs, Param, Selectors, SelectorParam, CountParam, Module, Event, HandledEvent, MultiEvent, SetupEvent, SelectorEvent, Subscription, Optional, Public, - Handler, Constant, Unwinder, Success, LockAnnotation, LockName) + Handler, Constant, Unwinder, Success, LockAnnotation, LockName, NoReturn) import collections import logging @@ -133,12 +133,24 @@ def result(self, children, meta): r.meta = meta return r + @v_args(meta=True) + def void_result(self, children, meta): + r = Result(children, void=True) + r.meta = meta + return r + @v_args(meta=True) def type_decl(self, children, meta): t = Type(' '.join(str(c) for c in children)) t.meta = meta return t + @v_args(meta=True) + def selector_const(self, children, meta): + t = Type(' '.join(str(c) for c in children)) + t.meta = meta + return t + @v_args(meta=True) def subscribe(self, children, meta): s = Subscription(children) @@ -197,6 +209,12 @@ def priority(self, children, meta): c.meta = meta return c + @v_args(meta=True) + def noreturn(self, children, meta): + c = NoReturn() + c.meta = meta + return c + @v_args(meta=True) def constexpr(self, children, meta): c = ConstExpr(' '.join(children)) diff --git a/tools/events/templates/c.tmpl b/tools/events/templates/c.tmpl index 4c04985..6110f5a 100644 --- a/tools/events/templates/c.tmpl +++ b/tools/events/templates/c.tmpl @@ -19,10 +19,11 @@ #end for \#include -#for h in $called_handlers +#for h in $declared_handlers #set empty=False -$h.return_type +#set noreturn = 'noreturn ' if $h.noreturn else '' +$noreturn$h.return_type ${h.name}(#slurp #set sep = '' #set params = list($h.params) @@ -35,14 +36,15 @@ ${sep}${p.type}${space}${p.name}#slurp #else void#slurp #end if -) +)#slurp #for l in $h.lock_opts ${l.action.upper()}_${l.kind.upper()}(${l.lock})#slurp #end for ; #end for #def prototype(e) -$e.return_type +#set noreturn = 'noreturn ' if $e.noreturn else '' +$noreturn$e.return_type trigger_${e.name}_event(#slurp #set sep = '' #if $e.params @@ -56,11 +58,20 @@ void#slurp #end if )#slurp #end def -#def call(s) +#def call(s, r) +#if r is not None +#set ret_assign = r + ' = ' +#else +#set ret_assign = '' +#end if #if s.handler is not None +#if s.handler.noreturn ${s.handler.name}(${', '.join(s.handler.args)})#slurp +#else +$ret_assign${s.handler.name}(${', '.join(s.handler.args)})#slurp +#end if #elif s.constant is not None -${s.constant}#slurp +$ret_assign${s.constant}#slurp #else #raise Exception("No handler or constant for subscription " + str($s)) #end if @@ -73,7 +84,7 @@ $prototype(e) (void)$p; #end for #for s in e.subscribers - $call(s); + $call(s, None); #end for } #end for @@ -88,7 +99,7 @@ $prototype(e) $e.return_type ret; #for s in e.subscribers - ret = $call(s); + $call(s, 'ret'); if (ret != $e.default) { goto out; } @@ -112,7 +123,7 @@ $prototype(e) $e.count.type _ret; #end if #for s in e.subscribers - _ret = $call(s); + $call(s, '_ret'); if (_ret >= $e.count.name) { return ($e.count.type)0; } @@ -130,29 +141,33 @@ $prototype(e) #end for $e.result.type $e.result.name = $e.result.default; +#set indent = ' '*4 #set first = True #for s in e.subscribers #if first #set first = False #else - if (compiler_expected($e.result.name == $e.success)) { +${indent}if (compiler_expected($e.result.name == $e.success)) { +#set indent = indent + ' '*4 #end if - $e.result.name = $call(s); +${indent}$call(s, $e.result.name); #end for #for s in tuple(reversed(e.subscribers))[1:] #if s.unwinder is not None - if (compiler_unexpected($e.result.name != $e.success)) { - ${s.unwinder.name}(${', '.join(s.unwinder.args)}); - } +${indent}if (compiler_unexpected($e.result.name != $e.success)) { +${indent} ${s.unwinder.name}(${', '.join(s.unwinder.args)}); +${indent}} #end if - } +#set indent = indent[:-4] +${indent}} #end for return $e.result.name; } #end for #for e in $selector_events +#set void=$e.return_type=='void' \#pragma clang diagnostic push \#pragma clang diagnostic ignored "-Wswitch-enum" @@ -162,27 +177,42 @@ $prototype(e) #for p in sorted(e.unused_param_names) (void)$p; #end for +#if not $void $e.return_type ret; - +#set ret='ret' +#else +#set ret=None +#end if switch ($e.selector.name) { #for s in e.subscribers #for selector in $s.selectors case $selector: #end for - ret = $call(s); + $call(s, ret); +#if s.handler and s.handler.noreturn + // ${s.handler}() does not return + compiler_unreachable_break; +#else break; +#end if #end for default: - ret = $e.result.default; +#if $void + // Nothing to do +#else + $ret = $e.result.default; +#end if break; } +#if not $void return ret; +#end if } \#pragma clang diagnostic pop #end for #if empty -inline void -events_dummy_function(void); +void +${name}_events_dummy_function(void); #end if diff --git a/tools/events/templates/handlers.h.tmpl b/tools/events/templates/handlers.h.tmpl index 3778bac..19b45a0 100644 --- a/tools/events/templates/handlers.h.tmpl +++ b/tools/events/templates/handlers.h.tmpl @@ -21,7 +21,8 @@ #continue #end if -$h.return_type +#set noreturn = 'noreturn ' if $h.noreturn else '' +$noreturn$h.return_type ${h.name}(#slurp #set sep = '' #for p in $h.params diff --git a/tools/events/templates/triggers.h.tmpl b/tools/events/templates/triggers.h.tmpl index 18e510b..42e142b 100644 --- a/tools/events/templates/triggers.h.tmpl +++ b/tools/events/templates/triggers.h.tmpl @@ -7,7 +7,8 @@ #for e in $events -$e.return_type +#set noreturn = 'noreturn ' if $e.noreturn else '' +$noreturn$e.return_type trigger_${e.name}_event(#slurp #set sep = '' #if $e.params diff --git a/tools/grammars/events_dsl.lark b/tools/grammars/events_dsl.lark index 0c15708..52e3044 100644 --- a/tools/grammars/events_dsl.lark +++ b/tools/grammars/events_dsl.lark @@ -31,7 +31,7 @@ publish_multi_event : "multi_event" symbol count_param _event_params? _END_DECL publish_setup_event : "setup_event" symbol _event_params? result success _END_DECL -publish_selector_event : "selector_event" symbol selector_param _event_params? result? _END_DECL +publish_selector_event : "selector_event" symbol selector_param _event_params? void_result? _END_DECL symbol : IDENTIFIER @@ -46,25 +46,30 @@ _param : symbol ":" type_decl _RESULT : _CONT_DECL "return" WS_INLINE? ":" result : _RESULT type_decl "=" constexpr +void_result : _RESULT type_decl "=" constexpr | _RESULT "void" _SUCCESS : _CONT_DECL "success" WS_INLINE? ":" success : _SUCCESS constexpr type_decl: _type_decl -_type_decl: (IDENTIFIER POINTER?)+ +_type_decl: (QUALIFIER* IDENTIFIER (POINTER QUALIFIER*)*) | LPAREN _type_decl RPAREN | _type_decl LPAREN POINTER RPAREN LPAREN _arg_type_list? RPAREN _arg_type_list: _arg_type_list COMMA _type_decl | _type_decl +_type_cast: LPAREN _type_decl RPAREN subscribe : "subscribe" optional? symbol selector? _subscriber? _subscribe_opts _END_DECL subscribe_public : "subscribe" optional? symbol selector? _subscriber_public _subscribe_opts _END_DECL optional : "optional" selector : "[" _selector_list "]" -_subscriber : expected_args unwinder? | handler unwinder? | unwinder | constant +// Note, the syntax, "subscribe event_name(args)" without an explicit 'handler' +// is a short-hand for "subscribe event_name handler default_name().. With a +// handler, the braces for arguments should go with the handler_name. +_subscriber : expected_args noreturn? unwinder? | handler unwinder? | unwinder | constant _subscriber_public : handler_public unwinder_public? | constant _HANDLER : ":" | _CONT_DECL "handler" -handler : _HANDLER symbol expected_args? public? -handler_public : _HANDLER symbol expected_args? public +handler : _HANDLER symbol expected_args? public? noreturn? +handler_public : _HANDLER symbol expected_args? public noreturn? _UNWINDER : _CONT_DECL "unwinder" unwinder : _UNWINDER symbol? expected_args? public? unwinder_public : _UNWINDER symbol? expected_args? public @@ -73,6 +78,7 @@ constant : _CONSTANT constexpr public : _CONT_DECL? "public" expected_args : "(" _NL_INDENT? _expected_arg_list? ")" _expected_arg_list : symbol ("," _NL_INDENT? symbol)* +noreturn : _CONT_DECL? "noreturn" %import common.SIGNED_NUMBER -> NUMBER _PRIORITY : _CONT_DECL "priority" priority : _PRIORITY (NUMBER | /min|max|first|last|default/) @@ -84,6 +90,7 @@ lock_opt : _lock_opt_kind lock_name _subscribe_opts : priority? lock_opt* IDENTIFIER : /[a-zA-Z_][a-zA-Z0-9_]*/ +QUALIFIER.2 : "const" | "volatile" | "_Atomic" POINTER : "*" ARROW : "->" constexpr : _constexpr @@ -93,7 +100,8 @@ RPAREN : ")" COMMA : "," _args : LPAREN _arg_list? RPAREN _arg_list : _NL_INDENT? _constexpr [COMMA _arg_list] -_selector_list : _NL_INDENT? _constexpr ["," _selector_list] +selector_const: _type_cast? _constexpr +_selector_list : _NL_INDENT? selector_const ["," _selector_list] %import common.SIGNED_INT %import common.SIGNED_FLOAT %import common.INT diff --git a/tools/grammars/hypercalls_dsl.lark b/tools/grammars/hypercalls_dsl.lark index 0320f62..649c32d 100644 --- a/tools/grammars/hypercalls_dsl.lark +++ b/tools/grammars/hypercalls_dsl.lark @@ -6,15 +6,15 @@ start : _top_level_member* _top_level_member : _definition -%import .typed_dsl (_customized_type, IDENTIFIER, constant_value, LINE_COMMENT, WS) +%import .typed_dsl (_customized_type, _identifier, constant_value, LINE_COMMENT, WS) hypercall_declaration : (_hypercall_property | _hypercall_params) ";" _hypercall_property : declaration_sensitive | declaration_call_num | declaration_vendor_hyp_call -_hypercall_params : (IDENTIFIER | RESERVED) (declaration_input | declaration_output) +_hypercall_params : (_identifier | RESERVED) (declaration_input | declaration_output) -RESERVED : "_res0" | "_res1" +RESERVED : "res0" | "res1" declaration_input : "input" _customized_type declaration_output : "output" _customized_type @@ -25,7 +25,7 @@ declaration_vendor_hyp_call : "vendor_hyp_call" _definition : "define" _type_definition _type_definition : hypercall_definition -hypercall_definition : IDENTIFIER "hypercall" "{" hypercall_declaration* "}" ";" +hypercall_definition : _identifier "hypercall" "{" hypercall_declaration* "}" ";" %ignore WS %ignore LINE_COMMENT diff --git a/tools/grammars/typed_dsl.lark b/tools/grammars/typed_dsl.lark index 8e8b2b2..19377b2 100644 --- a/tools/grammars/typed_dsl.lark +++ b/tools/grammars/typed_dsl.lark @@ -21,7 +21,7 @@ INTEGER : DIGIT_NONZERO DIGIT* UNSIGNED_SUFFIX? | ZERO UNSIGNED_SUFFIX? constant_value : INTEGER -constant_reference : IDENTIFIER +constant_reference : _identifier _primary_expr : constant_value | constant_reference @@ -94,7 +94,7 @@ _indirect_type : array array_size : constant_expression -array : "array" "(" array_size ")" _type +array : "array" "(" array_size ")" _qualifiers? _type pointer : "pointer" _qualifiers? _type @@ -103,18 +103,18 @@ direct_type : _customized_type _qualifiers? _qualifiers : "(" qualifier_list? ")" qualifier_list : _qualifier ("," _qualifier)* -declaration : IDENTIFIER declaration_offset? _type ";" +declaration : _identifier declaration_offset? _type ";" declaration_offset : "@" bracketed_constant_expression enumeration_noprefix : "noprefix" enumeration_explicit : "explicit" enumeration_expr : "=" constant_expression -enumeration_constant : IDENTIFIER enumeration_expr? enumeration_noprefix? ";" +enumeration_constant : _identifier enumeration_expr? enumeration_noprefix? ";" bitfield_declaration: (_bitfield_normal | bitfield_delete) ";" _bitfield_normal: bitfield_specifier _bitfield_type_specifier _bitfield_type_specifier: bitfield_member | bitfield_unknown | bitfield_const -bitfield_member: IDENTIFIER _type _bitfield_modifiers bitfield_default? +bitfield_member: _identifier _type _bitfield_modifiers bitfield_default? _definition : "define" _type_definition _type_definition : bitfield_definition @@ -128,13 +128,13 @@ _type_definition : bitfield_definition public : "public" -alternative_definition : IDENTIFIER public? "newtype" _type ";" +alternative_definition : _identifier public? "newtype" _type ";" // Constants aka 'C' defines -constant_definition : IDENTIFIER public? "constant" direct_type? "=" constant_expression ";" +constant_definition : _identifier public? "constant" direct_type? "=" constant_expression ";" // Global variables (never public; declaration only) -global_definition : IDENTIFIER "global" direct_type ";" +global_definition : _identifier "global" direct_type ";" bitfield_size : constant_expression _bitfield_size : "<" bitfield_size ">" @@ -142,18 +142,18 @@ bitfield_const_decl : "const" bitfield_set_ops_decl : "set_ops" _bitfield_param : bitfield_const_decl | bitfield_set_ops_decl _bitfield_params : "(" _bitfield_param ("," _bitfield_param)* ")" -bitfield_definition : IDENTIFIER public? "bitfield" _bitfield_size? _bitfield_params? "{" bitfield_declaration* "}" ";" +bitfield_definition : _identifier public? "bitfield" _bitfield_size? _bitfield_params? "{" bitfield_declaration* "}" ";" // NOTE: restrict object declaration only in object when handling AST -object_definition : IDENTIFIER public? "object" _qualifiers? "{" declaration* "}" ";" +object_definition : _identifier public? "object" _qualifiers? "{" declaration* "}" ";" -structure_definition : IDENTIFIER public? "structure" _qualifiers? "{" declaration* "}" ";" +structure_definition : _identifier public? "structure" _qualifiers? "{" declaration* "}" ";" -union_definition : IDENTIFIER public? "union" _qualifiers? "{" declaration* "}" ";" +union_definition : _identifier public? "union" _qualifiers? "{" declaration* "}" ";" _enumeration_param : enumeration_noprefix | enumeration_explicit _enumeration_params : "(" _enumeration_param? ("," _enumeration_param)* ")" -enumeration_definition : IDENTIFIER public? "enumeration" _enumeration_params? ("{" enumeration_constant* "}")? ";" +enumeration_definition : _identifier public? "enumeration" _enumeration_params? ("{" enumeration_constant* "}")? ";" _extension : "extend" _type_extension _type_extension : bitfield_extension @@ -162,18 +162,18 @@ _type_extension : bitfield_extension | union_extension | enumeration_extension -module_name : "module" IDENTIFIER +module_name : "module" _identifier -bitfield_extension : IDENTIFIER "bitfield" module_name? "{" bitfield_declaration+ "}" ";" +bitfield_extension : _identifier "bitfield" module_name? "{" bitfield_declaration+ "}" ";" -object_extension : IDENTIFIER "object" module_name? "{" declaration+ "}" ";" +object_extension : _identifier "object" module_name? "{" declaration+ "}" ";" -structure_extension : IDENTIFIER "structure" module_name? "{" declaration+ "}" ";" +structure_extension : _identifier "structure" module_name? "{" declaration+ "}" ";" -union_extension : IDENTIFIER "union" module_name? "{" declaration+ "}" ";" +union_extension : _identifier "union" module_name? "{" declaration+ "}" ";" // TODO: enumeration_extension to support module_name ? -enumeration_extension : IDENTIFIER "enumeration" "{" enumeration_constant+ "}" ";" +enumeration_extension : _identifier "enumeration" "{" enumeration_constant+ "}" ";" _customized_type : bitfield_type | structure_type @@ -184,17 +184,16 @@ _customized_type : bitfield_type | primitive_type -bitfield_type : "bitfield" IDENTIFIER -structure_type : "structure" IDENTIFIER +bitfield_type : "bitfield" _identifier +structure_type : "structure" _identifier object_noprefix : "noprefix" -object_type : "object" ("(" object_noprefix? ")")? IDENTIFIER -union_type : "union" IDENTIFIER -enumeration_type : "enumeration" IDENTIFIER -alternative_type : "type" IDENTIFIER - -PRIMITIVE_TYPE.2 : "sint8" | "sint16" | "sint32" | "sint64" - | "bool" | "uint8" | "uint16" | "uint32" | "uint64" | "char" - | "sintptr" | "uintptr" | "sregister" | "uregister" | "size" +object_type : "object" ("(" object_noprefix? ")")? _identifier +union_type : "union" _identifier +enumeration_type : "enumeration" _identifier +_TYPE : "type" +alternative_type : _TYPE _identifier + +PRIMITIVE_TYPE.2 : /([su]int(8|16|32|64|ptr)|bool|char|size|[su]register)\b/ primitive_type : PRIMITIVE_TYPE bitfield_specifier: bitfield_others | bitfield_auto | _bitfield_bit_list @@ -206,7 +205,7 @@ _bitfield_bit: bracketed_constant_expression bitfield_width: "<" constant_expression ">" bitfield_auto: AUTO bitfield_width? bitfield_others: "others" -bitfield_delete: "delete" IDENTIFIER +bitfield_delete: "delete" _identifier bitfield_default: "=" constant_expression _bitfield_modifiers: bitfield_shift? bitfield_shift: "lsl" "(" constant_expression ")" @@ -214,17 +213,19 @@ UNKNOWN : "unknown" bitfield_unknown : UNKNOWN bitfield_default? bitfield_const: bracketed_constant_expression -_qualifier : basic_qualifier | aligned_qualifier | atomic_qualifier - | packed_qualifier | contained_qualifier | writeonly_qualifier - | lockable_qualifier +_qualifier : basic_qualifier | aligned_qualifier | group_qualifier + | atomic_qualifier | packed_qualifier | contained_qualifier + | writeonly_qualifier | lockable_qualifier | optimize_qualifier BASIC_QUALIFIER : "const" | "restrict" | "volatile" basic_qualifier : BASIC_QUALIFIER aligned_qualifier : "aligned" "(" constant_expression ")" +group_qualifier : "group" "(" GROUPNAME ("," GROUPNAME )*")" atomic_qualifier : "atomic" packed_qualifier : "packed" contained_qualifier : "contained" writeonly_qualifier : "writeonly" lockable_qualifier : "lockable" +optimize_qualifier : "optimize" BIT : "bit" BITS : "bits" @@ -233,11 +234,9 @@ ONES : "ones" SELF : "self" AUTO : "auto" -LCASE_LETTER : "a".."z" -UCASE_LETTER : "A".."Z" -LETTER : UCASE_LETTER | LCASE_LETTER - -IDENTIFIER : LETTER ("_"|LETTER|DIGIT)* +!_identifier : IDENTIFIER | _TYPE | PRIMITIVE_TYPE +IDENTIFIER : /\b(?!\d)\w+\b/ +GROUPNAME : /~?\b\w+\b/ WS: /[ \t\f\r\n]/+ diff --git a/tools/hypercalls/hypercall.py b/tools/hypercalls/hypercall.py index cdb5a75..b50f6e8 100755 --- a/tools/hypercalls/hypercall.py +++ b/tools/hypercalls/hypercall.py @@ -57,7 +57,10 @@ def __init__(self, ctype, name, type_definition): self.name = name self.size = type_definition.size self.category = type_definition.category - self.ignore = name.startswith('_') + if type_definition.category == "bitfield": + self.type_name = type_definition.type_name + self.ignore = name.startswith('res0') or name.startswith( + 'res1') or name.endswith('_') self.pointer = False try: from ir import PointerType @@ -67,10 +70,13 @@ def __init__(self, ctype, name, type_definition): except AttributeError: pass if self.ignore: - if name.startswith('_res0'): + if name.startswith('res0'): self.default = 0 - elif name.startswith('_res1'): + elif name.startswith('res1'): self.default = 0xffffffffffffffff + elif name.endswith('_'): + raise Exception( + "Invalid name ending with underscore: {:s}".format(name)) else: raise Exception("Invalid ignored name {:s}".format(name)) diff --git a/tools/hypercalls/hypercall_gen.py b/tools/hypercalls/hypercall_gen.py index 59f243e..c12365b 100755 --- a/tools/hypercalls/hypercall_gen.py +++ b/tools/hypercalls/hypercall_gen.py @@ -63,7 +63,7 @@ def get_constant(c): type_parent = c.data c = c.children[0] - assert(type_parent == 'constant_value') + assert (type_parent == 'constant_value') return c @@ -206,7 +206,8 @@ def main(): grammar_file = os.path.join(__loc__, '..', 'grammars', 'hypercalls_dsl.lark') - parser = Lark.open(grammar_file, 'start', propagate_positions=True) + parser = Lark.open(grammar_file, 'start', parser='lalr', + lexer='contextual', propagate_positions=True) from ir import PrimitiveType # Load typed pickle to get the types used for the inputs and output of the diff --git a/tools/hypercalls/templates/guest_interface.c.tmpl b/tools/hypercalls/templates/guest_interface.c.tmpl index 9e18509..b82084c 100644 --- a/tools/hypercalls/templates/guest_interface.c.tmpl +++ b/tools/hypercalls/templates/guest_interface.c.tmpl @@ -26,7 +26,7 @@ #end if #slurp register uint${input.size * 8}_t#slurp - _in_$in_reg __asm__("$in_reg") = #slurp + in_${in_reg}_ __asm__("$in_reg") = #slurp #if $input.ignore ${hex(input.default)}U; #else @@ -38,7 +38,7 @@ #end for #for $out_reg, $output in $hypcall.outputs register uint${output.size * 8}_t#slurp - _out_$out_reg __asm__("$out_reg"); + out_${out_reg}_ __asm__("$out_reg"); #if $output.pointer #set $has_pointer = True #end if @@ -48,12 +48,12 @@ "hvc $hypcall.hvc_num" : #slurp #set sep = '' #for $out_reg, _ in $hypcall.outputs - $sep"=r"(_out_$out_reg) #slurp + $sep"=r"(out_${out_reg}_) #slurp #set sep = ', ' #end for #for $in_reg, _ in $hypcall.inputs #if in_reg not in outregs - ${sep}"+r"(_in_$in_reg) #slurp + ${sep}"+r"(in_${in_reg}_) #slurp #set sep = ', ' #end if #end for @@ -61,7 +61,7 @@ #set sep = '' #for $in_reg, _ in $hypcall.inputs #if in_reg in outregs - ${sep}"r"(_in_$in_reg) #slurp + ${sep}"r"(in_${in_reg}_) #slurp #set sep = ', ' #end if #end for @@ -74,11 +74,11 @@ #if $hypcall.outputs return #slurp #if len($hypcall.outputs) < 2 - $register_init($output, '_out_' + $out_reg); + $register_init($output, 'out_' + $out_reg + '_'); #else ($return_type($hypcall)){ #for $out_reg, $output in $hypcall.outputs - .${output.name} = $register_init($output, '_out_' + $out_reg), + .${output.name} = $register_init($output, 'out_' + $out_reg + '_'), #end for }; #end if diff --git a/tools/misc/get_genfiles.py b/tools/misc/get_genfiles.py index e812612..06d965f 100755 --- a/tools/misc/get_genfiles.py +++ b/tools/misc/get_genfiles.py @@ -19,7 +19,8 @@ commands_file = 'compile_commands.json' compile_commands = [] -build_preferences = [("gunyah-rm", 120)] +conf_default_weight = 60 +build_preferences = {} files = set() incdirs = set() @@ -27,6 +28,19 @@ include_regex = re.compile('(-iquote|-I) (\\w+[-\\{:s}\\w]+)'.format(os.sep)) imacros_regex = re.compile('(-imacros) (\\w+[-\\{:s}\\w.]+)'.format(os.sep)) +for dir, dir_dirs, dir_files in os.walk('config/featureset'): + regex = re.compile('^# indexer-weight: (\\d+)') + for file in dir_files: + file_base = os.path.splitext(file)[0] + if file.endswith('.conf'): + build_preferences[file_base] = conf_default_weight + infile = os.path.join(dir, file) + with open(infile, 'r') as f: + for i, line in enumerate(f): + weights = regex.findall(line) + if weights: + build_preferences[file_base] = int(weights[0]) + for dir, dir_dirs, dir_files in os.walk(build_dir): if commands_file in dir_files: x = os.stat(dir) @@ -42,10 +56,10 @@ for time, f in compile_commands: x = os.stat(f) time = max(x.st_atime, x.st_mtime, x.st_ctime, time) - for p, weight in build_preferences: + for p in build_preferences.keys(): if p in f: # Boost these to preference them - time += weight + time += build_preferences[p] if time > newest: newest = time diff --git a/tools/registers/register_gen.py b/tools/registers/register_gen.py index 19d4f54..9ae42bc 100755 --- a/tools/registers/register_gen.py +++ b/tools/registers/register_gen.py @@ -138,7 +138,7 @@ def main(): default=sys.stdout, help="Write output to file") args.add_argument("-f", "--formatter", help="specify clang-format to format the code") - args.add_argument("input", metavar='INPUT', nargs=1, + args.add_argument("input", metavar='INPUT', nargs='*', help="Input type register file to process", type=argparse.FileType('r', encoding="utf-8")) options = args.parse_args() diff --git a/tools/requirements.txt b/tools/requirements.txt index c64a7a6..d405c74 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -1,3 +1,3 @@ -lark_parser==0.6.6 -Cheetah3==3.1.0 +lark_parser==0.8.9 +Cheetah3==3.2.6 pyelftools==0.26 diff --git a/tools/typed/abi.py b/tools/typed/abi.py index d778f43..0831d2b 100644 --- a/tools/typed/abi.py +++ b/tools/typed/abi.py @@ -114,7 +114,7 @@ def layout_struct_member(self, current_offset, current_alignment, alignment = next_alignment else: alignment = current_alignment - assert(self.is_power2(alignment)) + assert (self.is_power2(alignment)) align_mask = alignment - 1 return (current_offset + align_mask) & ~align_mask @@ -153,7 +153,7 @@ def signed_char(self): def get_c_type_name(self, abi_type_name): """Return the c name for the abi type name.""" - assert(abi_type_name in self.abi_type_map) + assert (abi_type_name in self.abi_type_map) return self.abi_type_map[abi_type_name] def get_enum_properties(self, e_min, e_max): diff --git a/tools/typed/ast_nodes.py b/tools/typed/ast_nodes.py index 1db2f50..d25ed31 100644 --- a/tools/typed/ast_nodes.py +++ b/tools/typed/ast_nodes.py @@ -13,12 +13,13 @@ AlternativeDefinition, BitFieldDefinition, StructureDefinition, EnumerationDefinition, EnumerationConstant, EnumerationExtension, ObjectType, ObjectDeclaration, ObjectDefinition, BitFieldExtension, - ObjectExtension, Qualifier, AlignedQualifier, AtomicQualifier, - PackedQualifier, ConstantExpression, ConstantReference, UnaryOperation, - SizeofOperation, AlignofOperation, BinaryOperation, ConditionalOperation, - UnionType, UnionDefinition, UnionExtension, StructureExtension, - MinofOperation, MaxofOperation, ContainedQualifier, WriteonlyQualifier, - PureFunctionCall, LockableQualifier, GlobalDefinition, + ObjectExtension, Qualifier, AlignedQualifier, GroupQualifier, + AtomicQualifier, PackedQualifier, ConstantExpression, ConstantReference, + UnaryOperation, SizeofOperation, AlignofOperation, BinaryOperation, + ConditionalOperation, UnionType, UnionDefinition, UnionExtension, + StructureExtension, MinofOperation, MaxofOperation, ContainedQualifier, + WriteonlyQualifier, PureFunctionCall, LockableQualifier, GlobalDefinition, + OptimizeQualifier ) from exceptions import DSLError @@ -547,9 +548,11 @@ def init_interfaces(self): "atomic_qualifier", "packed_qualifier", "aligned_qualifier", + "group_qualifier", "contained_qualifier", "writeonly_qualifier", "lockable_qualifier", + "optimize_qualifier", ] self.action_list.take_all(self, al) self.actions = [Action(self.set_qualifiers, "qualifier_list")] @@ -595,6 +598,15 @@ def add_qualifier(self, obj): obj.qualifiers.add(self.qualifier) +class group_qualifier(CommonListener): + def init_interfaces(self): + self.qualifier = GroupQualifier(self, self.children) + self.actions = [Action(self.add_qualifier, "group_qualifier")] + + def add_qualifier(self, obj): + obj.qualifiers.add(self.qualifier) + + class contained_qualifier(CommonListener): def init_interfaces(self): self.qualifier = ContainedQualifier(self) @@ -628,6 +640,21 @@ def set_name(self, obj): self.qualifier.resource_name = ' '.join((obj.type_name, obj.category)) +class optimize_qualifier(CommonListener): + def init_interfaces(self): + self.qualifier = OptimizeQualifier(self) + self.actions = [ + Action(self.add_qualifier, "optimize_qualifier"), + Action(self.set_category, "describe_optimized_type"), + ] + + def add_qualifier(self, obj): + obj.qualifiers.add(self.qualifier) + + def set_category(self, obj): + self.qualifier.category = obj.category + + class array_size(CommonListener): def init_interfaces(self): self.value = self.children[0].value @@ -641,14 +668,11 @@ def set_size(self, obj): class array(CommonListener): def init_interfaces(self): - self.length = NotImplemented self.compound_type = ArrayType(self) - al = ["array_size"] - self.action_list.take_all(self, al) - self.compound_type.length = self.length - al = ["object_type_set_complex"] + al = ["array_size", "qualifier_list", "object_type_set_complex"] self.action_list.take_all(self.compound_type, accept_list=al) + self.actions = [Action(self.set_type, "array")] def set_type(self, declaration): @@ -789,7 +813,7 @@ def init_interfaces(self): self.actions = [ Action(self.add_range, "bitfield_bit_range"), ] - assert(len(self.children) <= 2) + assert (len(self.children) <= 2) def add_range(self, specifier): specifier.add_bit_range(self) @@ -1119,6 +1143,7 @@ def init_interfaces(self): self.action_list.take(d, "qualifier_list") self.action_list.take(d, "public") self.action_list.take(d, "describe_lockable_type") + self.action_list.take(d, "describe_optimized_type") class union_definition(ITypeDefinition): @@ -1131,6 +1156,7 @@ def init_interfaces(self): self.action_list.take(d, "declaration", single=False) self.action_list.take(d, "qualifier_list") self.action_list.take(d, "public") + self.action_list.take(d, "describe_lockable_type") class enumeration_definition(IABISpecific, ITypeDefinition): @@ -1158,6 +1184,7 @@ def init_interfaces(self): self.action_list.take(d, "qualifier_list") self.action_list.take(d, "public") self.action_list.take(d, "describe_lockable_type") + self.action_list.take(d, "describe_optimized_type") rl = ["object_type_has_object"] self.action_list.remove_all(rl) diff --git a/tools/typed/ir.py b/tools/typed/ir.py index 7e29304..479428a 100644 --- a/tools/typed/ir.py +++ b/tools/typed/ir.py @@ -16,6 +16,12 @@ from lark import Transformer, Tree, Token, Discard +import tempfile +import Cheetah.ImportHooks + +td = tempfile.TemporaryDirectory() +Cheetah.ImportHooks.setCacheDir(td.name) + """ The classes in the module represent the features of the DSL language. They form an intermediate representation and are used to generate the output. An @@ -115,6 +121,17 @@ def visit(d): return output + def get_output_templates(self): + templates = set() + + # Sort, ensuring that dependencies come before dependent definitions + for d in self.definitions: + deps = d.get_template_deps() + for t in deps: + templates.add(t) + + return list(templates) + def apply_template(self, template_file, public_only=False): ns = [{ 'declarations': tuple(d for d in self.declarations @@ -601,6 +618,14 @@ def is_contained(self): """ return any(q.is_contained for q in self.qualifiers) + @property + def is_ordered(self): + """ + Return True if the type should be generated exactly as ordered in its + definition. + """ + raise NotImplementedError + @property def is_signed(self): """ @@ -765,6 +790,9 @@ class IGenCode(metaclass=abc.ABCMeta): def gen_code(self): return ([], []) + def get_template_deps(self): + return [] + class IAggregation: """ @@ -875,7 +903,7 @@ def __init__(self): self.owner = None def set_ignored(self): - assert(not self.is_ignore) + assert (not self.is_ignore) self.is_ignore = True def set_const(self): @@ -1101,7 +1129,7 @@ def set_signed(self, signed=True): self.field_signed = signed def add_bit_range(self, field_bit, mapped_bit, length): - assert field_bit >= 0 + assert int(field_bit) >= 0 fmap = FieldMap(field_bit, mapped_bit, length) self.field_maps.append(fmap) @@ -1151,21 +1179,21 @@ def __init__(self): self.mapping = None def add_bit_range(self, bit_range): - assert(self.specifier_type in (self.NONE, self.RANGE)) + assert (self.specifier_type in (self.NONE, self.RANGE)) self.specifier_type = self.RANGE self._bit_ranges.append(bit_range) def set_type_shift(self, shift): - assert(self.specifier_type in (self.RANGE, self.AUTO)) + assert (self.specifier_type in (self.RANGE, self.AUTO)) self.shift = shift def set_type_auto(self, width=None): - assert(self.specifier_type is self.NONE) + assert (self.specifier_type is self.NONE) self.specifier_type = self.AUTO self.auto_width = width def set_type_others(self): - assert(self.specifier_type is self.NONE) + assert (self.specifier_type is self.NONE) self.specifier_type = self.OTHERS @property @@ -1449,9 +1477,9 @@ def get_members(self, prefix=None): templates = {} -class BitFieldDeclaration(IDeclaration): +class BitFieldDeclaration(IGenCode, IDeclaration): """ - Declaration of a field in a BitfieldDefinition. + Declaration of a field in a BitFieldDefinition. """ ACCESSOR_TEMPLATE = "templates/bitfield-generic-accessor.tmpl" @@ -1461,12 +1489,12 @@ def __init__(self): self.category = self.BITFIELD self.prefix = None self.bitfield_specifier = None - self.template = None self.bf_type_name = None self.unit_type = None self.unit_size = -1 self.ranges = None self.default = None + self._template = None def gen_code(self): # validate parameters @@ -1482,12 +1510,12 @@ def gen_code(self): # generate code to extra # FIXME: should be a list of templates (header, c, etc.) ? - assert self.template is not None + assert self._template is not None if 'bitfield' in templates: template = templates['bitfield'] else: - template = Template.compile(file=open(self.template, 'r', + template = Template.compile(file=open(self._template, 'r', encoding='utf-8')) templates['bitfield'] = template @@ -1505,6 +1533,9 @@ def get_members(self, prefix=None): """ raise NotImplementedError + def get_template_deps(self): + return [os.path.join(__loc__, self.ACCESSOR_TEMPLATE)] + def update(self): super().update() @@ -1534,7 +1565,7 @@ def update(self): "unable to allocate {:d} bits from {:s}".format( width, repr(self.ranges)), self.member_name) - assert(r[1] == width) + assert (r[1] == width) b.bit_length = width range_shift = b.shift @@ -1550,7 +1581,7 @@ def update(self): range_shift += split_width b.mapping.add_bit_range(range_shift, bit, width) - assert(b.bit_length is not None) + assert (b.bit_length is not None) b.set_signed(self.compound_type.is_signed) @@ -1576,7 +1607,7 @@ def update(self): b.bit_length, member_bitsize), self.member_name) - self.template = os.path.join(__loc__, self.ACCESSOR_TEMPLATE) + self._template = os.path.join(__loc__, self.ACCESSOR_TEMPLATE) @property def indicator(self): @@ -1666,6 +1697,7 @@ def __init__(self, type_name): self._size = None self._alignment = None self._layout = None + self._ordered = True def set_abi(self, abi): """ @@ -1673,12 +1705,47 @@ def set_abi(self, abi): """ self._abi = abi + def set_ordered(self, ordered): + """ + Set the structure layout rules. + """ + self._ordered = ordered + def _update_layout(self): """ Determine the layout of the structure. """ + for q in self.qualifiers: + if q.is_optimized: + self.set_ordered(False) + + if self.is_ordered: + member_list = iter(self._members()) + else: + # Sort members by group, module and alignment + member_list = list(self._members()) + + def member_key(member): + member_type = member[1] + member_decl = member[2] + + default_group = chr(0xff) + if hasattr(member_decl, "module_name"): + if member_decl.module_name: + default_group = '~' + member_decl.module_name + + key = (default_group, -member_type.alignment) + for q in member_type.qualifiers: + if q.is_group: + key = tuple(['/'.join(q.group), + -member_type.alignment]) + break + return key + + # list of lists, sort by group and alignment + member_list = sorted(self._members(), key=member_key) + packed = self.is_packed - member_list = iter(self._members()) layout = [] abi = self._abi @@ -1688,7 +1755,12 @@ def _update_layout(self): offset = 0 max_alignment = 1 + members = set() for member_name, member_type, member_decl in member_list: + if member_name in members: + raise DSLError("structure {}: duplicated members".format( + self.type_name), member_decl.member_name) + members.add(member_name) if member_decl.offset is not None: member_pos = int(member_decl.offset) if member_pos < offset: @@ -1723,24 +1795,23 @@ def _update_layout(self): offset += member_type.size max_alignment = max(max_alignment, member_type.alignment) - if offset == 0: - raise DSLError("Structure has no members", self.type_name) - - # update max_alignment for end padding - for q in self.qualifiers: - if q.is_aligned: - max_alignment = max(max_alignment, q.align_bytes) + if offset != 0: + # update max_alignment for end padding + for q in self.qualifiers: + if q.is_aligned: + max_alignment = max(max_alignment, q.align_bytes) - if not packed: - # Pad the structure at the end - end = abi.layout_struct_member(offset, max_alignment, None, None) - if end > offset: - layout.append(('pad_end_', StructurePadding(end - offset), - offset)) - offset = end + if not packed: + # Pad the structure at the end + end = abi.layout_struct_member( + offset, max_alignment, None, None) + if end > offset: + layout.append(('pad_end_', StructurePadding(end - offset), + offset)) + offset = end + self._alignment = max_alignment self._size = offset - self._alignment = max_alignment self._layout = tuple(layout) def gen_forward_decl(self): @@ -1755,8 +1826,8 @@ def gen_forward_decl(self): pass elif q.is_atomic or q.is_const: code.extend(q.gen_qualifier()) - code.append("struct " + self.type_name + ' ' + self.type_name + '_t' - ";\n") + code.append("struct " + self.type_name + '_s' + ' ' + + self.type_name + '_t'";\n") return (code) @@ -1767,27 +1838,28 @@ def gen_code(self): code = [] extra = [] - code.append("struct ") - for q in self.qualifiers: - if q.is_aligned or q.is_atomic or q.is_const: - pass - elif q.is_packed or q.is_lockable: - code.extend(q.gen_qualifier()) - else: - raise DSLError("Invalid qualifier for structure", q.name) + if self._size != 0: + code.append("struct ") + for q in self.qualifiers: + if q.is_aligned or q.is_atomic or q.is_const or q.is_optimized: + pass + elif q.is_packed or q.is_lockable: + code.extend(q.gen_qualifier()) + else: + raise DSLError("Invalid qualifier for structure", q.name) - code.append(" " + self.type_name + " {\n") + code.append(" " + self.type_name + '_s' " {\n") - for q in self.qualifiers: - if q.is_aligned: - code.extend(q.gen_qualifier()) + for q in self.qualifiers: + if q.is_aligned: + code.extend(q.gen_qualifier()) - for member_name, member_type, member_offset in self._layout: - code.append(member_type.gen_declaration(member_name) + ';\n') + for member_name, member_type, member_offset in self._layout: + code.append(member_type.gen_declaration(member_name) + ';\n') - code.append("} ") + code.append("} ") - code.append(';\n\n') + code.append(';\n\n') return (code, extra) @@ -1813,6 +1885,10 @@ def layout(self): def is_container(self): return True + @property + def is_ordered(self): + return self._ordered + @property def default_alignment(self): if self._alignment is None: @@ -1825,6 +1901,18 @@ def _members(self, prefix=None): for e in self.extensions: yield from e._members(prefix=prefix) + def update(self): + """ + Update internal data, prepare to generate code + """ + used_names = set() + for member_name, member_type, _ in self._members(): + if member_name in used_names: + raise DSLError("'structure {:s}': each member needs to have a" + " unique name".format(self.type_name), + member_type.type_name) + used_names.add(member_name) + @property def dependencies(self): """ @@ -1845,6 +1933,7 @@ def __init__(self, type_name): # embedded in any other object. # FIXME: this should be explicit in the language self.need_export = True + self.set_ordered(False) def gen_code(self): if self.need_export: @@ -1890,13 +1979,14 @@ def gen_forward_decl(self): code.append("typedef") for q in self.qualifiers: - if q.is_aligned: + if q.is_aligned or q.is_lockable: pass elif q.is_atomic or q.is_const: code.extend(q.gen_qualifier()) else: raise DSLError("Invalid qualifier for union", q.name) - code.append("union " + self.type_name + ' ' + self.type_name + '_t;\n') + code.append("union " + self.type_name + '_u' + + ' ' + self.type_name + '_t;\n') return code @@ -1907,10 +1997,12 @@ def gen_code(self): for q in self.qualifiers: if q.is_aligned or q.is_atomic or q.is_const: pass + elif q.is_lockable: + code.extend(q.gen_qualifier()) else: raise DSLError("Invalid qualifier for union", q.name) - code.append(" " + self.type_name + " {\n") + code.append(" " + self.type_name + '_u' + " {\n") align_qualifiers = tuple(q for q in self.qualifiers if q.is_aligned) @@ -1954,6 +2046,18 @@ def _members(self, prefix=None): for e in self.extensions: yield from e._members(prefix=prefix) + def update(self): + """ + Update internal data, prepare to generate code + """ + used_names = set() + for member_name, member_type, _ in self._members(): + if member_name in used_names: + raise DSLError("'union {:s}': each member needs to have a" + " unique name".format(self.type_name), + member_type.type_name) + used_names.add(member_name) + @property def dependencies(self): """ @@ -1997,8 +2101,8 @@ def __init__(self, type_name): def __getstate__(self): """ - Temporary workaround to ensure types are updated before pickling. Auto - update should be removed entirely. + Temporary workaround to ensure types are updated before pickling. Auto + update should be removed entirely when issue is resolved. """ if not self._updated: self._autoupdate() @@ -2034,12 +2138,19 @@ def _autoupdate(self): Update internal data, prepare to generate code """ def _check_enumerator(e): + if e.name in used_names: + raise DSLError("'enumeration {:s}': each enumerator needs to" + " have a unique name".format(self.type_name), + e.name) if e.value in used_values: - raise DSLError("each enumerator needs to have a unique value", + raise DSLError("'enumeration {:s}': each enumerator needs to" + " have a unique value".format(self.type_name), e.name) + used_names.add(e.name) used_values.add(e.value) # Ensure constant values are resolved and not duplicates + used_names = set() used_values = set() for e in self._enumerators: if e.value is not None: @@ -2126,7 +2237,7 @@ def gen_code(self): extra = [] # generate code now - code = ['typedef', 'enum', self.type_name, '{\n'] + code = ['typedef', 'enum', self.type_name + '_e', '{\n'] sorted_enumerators = sorted(self._enumerators, key=lambda x: x.value) @@ -2140,13 +2251,16 @@ def gen_code(self): code.append(';\n\n') - suffix = '' if self.is_signed else 'U' - code.append('#define {:s}__MAX ({:s})({:d}{:s})\n'.format( - self.type_name.upper(), self.type_name + '_t', self.maximum_value, - suffix)) - code.append('#define {:s}__MIN ({:s})({:d}{:s})\n'.format( - self.type_name.upper(), self.type_name + '_t', self.minimum_value, - suffix)) + for e in self._enumerators: + if e.value == self.minimum_value: + e_min = e.prefix + self.get_enum_name(e) + if e.value == self.maximum_value: + e_max = e.prefix + self.get_enum_name(e) + + code.append('#define {:s}__MAX {:s}\n'.format( + self.type_name.upper(), e_max)) + code.append('#define {:s}__MIN {:s}\n'.format( + self.type_name.upper(), e_min)) code.append('\n') return (code, extra) @@ -2278,6 +2392,7 @@ def __init__(self, type_name): self.const = False self._signed = False self._has_set_ops = None + self._template = None def update_unit_info(self): if self.length <= 8: @@ -2328,9 +2443,12 @@ def _all_declarations(self): @property def fields(self): + items = [] for d in self._all_declarations: if d.compound_type is not None: - yield d + items.append(d) + items = sorted(items, key=lambda x: x.field_maps[0].mapped_bit) + return tuple(items) @property def all_fields_boolean(self): @@ -2359,6 +2477,7 @@ def _gen_definition_code(self): ns = { "type_name": self.type_name, "unit_type": self.unit_type, + "unit_size": self.unit_size, "unit_cnt": self.unit_count, "declarations": self._all_declarations, "init_values": self.init_values, @@ -2367,8 +2486,15 @@ def _gen_definition_code(self): "all_fields_boolean": self.all_fields_boolean, "has_set_ops": self.has_set_ops, } - fn = os.path.join(__loc__, self.TYPE_TEMPLATE) - t = Template(file=open(fn, 'r', encoding='utf-8'), searchList=ns) + + if 'bitfield-type' in templates: + template = templates['bitfield-type'] + else: + template = Template.compile(file=open(self._template, 'r', + encoding='utf-8')) + templates['bitfield-type'] = template + + t = template(namespaces=(ns)) return str(t) @property @@ -2445,6 +2571,8 @@ def gen_code(self): code = [] extra = [] + assert self._template is not None + code.append(self._gen_definition_code()) # generate getters and setters for all declarations @@ -2461,11 +2589,22 @@ def gen_code(self): return (code, extra) + def get_template_deps(self): + templates = [] + for d in self._all_declarations: + if d.compound_type is None: + continue + else: + templates += d.get_template_deps() + return templates + [os.path.join(__loc__, self.TYPE_TEMPLATE)] + def update(self): super().update() if self._ranges is None: self._update_layout() + self._template = os.path.join(__loc__, self.TYPE_TEMPLATE) + def _update_layout(self): """ Determine the layout of the bitfield. @@ -2476,19 +2615,28 @@ def _update_layout(self): found = False for i, d in enumerate(self.declarations): if str(d.member_name) == name: - del(self.declarations[i]) + del (self.declarations[i]) found = True break if not found: raise DSLError("can't delete unknown member", name) self._ranges = BitFieldRangeCollector(self.length) + for d in self._all_declarations: d.update_ranges(self._ranges, self.unit_size) + used_names = set() for d in self._all_declarations: if d.is_ignore: continue + + if d.member_name in used_names: + raise DSLError("'bitfield {:s}': each member needs to" + " have a unique name".format(self.type_name), + d.member_name) + used_names.add(d.member_name) + # if bitfield is constant, update all members if self.const: d.set_const() @@ -2525,6 +2673,9 @@ def link(self, definition): self.prefix = self.module_name definition.extensions.append(self) + for d in self.declarations: + d.module_name = self.prefix + def _members(self, prefix=None): if prefix is not None and self.prefix is not None: p = prefix + "_" + self.prefix @@ -2674,6 +2825,14 @@ def is_contained(self): def is_lockable(self): return False + @property + def is_optimized(self): + return False + + @property + def is_group(self): + return False + def gen_qualifier(self): return [self.name] @@ -2717,6 +2876,19 @@ def __hash__(self): return hash(id(self)) +class GroupQualifier(Qualifier): + def __init__(self, name, group): + super().__init__(name) + self.group = group + + def gen_qualifier(self): + return [""] + + @property + def is_group(self): + return True + + class AtomicQualifier(Qualifier): def gen_qualifier(self): return ["_Atomic"] @@ -2761,7 +2933,7 @@ def __init__(self, name): def gen_qualifier(self): if self.resource_name is None: raise DSLError( - "Only structure and object definitions may be lockable", + "Only structure, object and union definitions may be lockable", self.name) return ['__attribute__((capability("{:s}")))' .format(self.resource_name)] @@ -2771,6 +2943,23 @@ def is_lockable(self): return True +class OptimizeQualifier(Qualifier): + def __init__(self, name): + super().__init__(name) + self.category = None + + def gen_qualifier(self): + if self.category is None: + raise DSLError( + "Only structure and object definitions may be optimized", + self.name) + return [""] + + @property + def is_optimized(self): + return True + + class BitFieldRangeCollector: """ BitFieldRangeCollector manages the range defined by length [0, length). diff --git a/tools/typed/templates/bitfield-type.tmpl b/tools/typed/templates/bitfield-type.tmpl index ed952dc..67de619 100644 --- a/tools/typed/templates/bitfield-type.tmpl +++ b/tools/typed/templates/bitfield-type.tmpl @@ -11,7 +11,7 @@ #set unit_postfix=' x{:d}'.format($unit_cnt) #end if // Bitfield: $type_name <${unit_type}${unit_postfix}> -typedef struct ${type_name} { +typedef struct ${type_name}_b { #set decs = [] #for $d in $declarations: #if $d.field_maps @@ -42,7 +42,7 @@ typedef struct ${type_name} { ## in contexts that require constant expressions, e.g. in case labels. #define ${type_name}_default() \ (${type_name}_t){ .bf = { #slurp -${', '.join(str(v) for v in $init_values)} } } +${', '.join(hex(v) for v in $init_values)} } } #define ${type_name}_cast(#slurp #for i in range($unit_cnt) @@ -71,12 +71,23 @@ ${type_name}_atomic_ptr_raw(_Atomic ${type_name}_t *ptr); void ${type_name}_init(${type_name}_t *bit_field); +// Set all unknown/unnamed fields to their expected default values. +// Note, this does NOT clean const named fields to default values. ${type_name}_t -${type_name}_clean(${type_name}_t val); +${type_name}_clean(${type_name}_t bit_field); bool ${type_name}_is_equal(${type_name}_t b1, ${type_name}_t b2); +bool +${type_name}_is_empty(${type_name}_t bit_field); + +// Check all unknown/unnamed fields have expected default values. +// Note, this does NOT check whether const named fields have their default +// values. +bool +${type_name}_is_clean(${type_name}_t bit_field); + #if $has_set_ops // Union of boolean fields of two ${type_name}_t values #if not $all_fields_boolean diff --git a/tools/typed/test/sample_inputs/constexpr.tc b/tools/typed/test/sample_inputs/constexpr.tc index 0e8d286..ceb546e 100644 --- a/tools/typed/test/sample_inputs/constexpr.tc +++ b/tools/typed/test/sample_inputs/constexpr.tc @@ -1,6 +1,3 @@ -// © 2021 Qualcomm Innovation Center, Inc. All rights reserved. -// SPDX-License-Identifier: BSD-3-Clause - define PAGE_SIZE constant size = 1 << 12; define sizeof_test structure { diff --git a/tools/typed/type_gen.py b/tools/typed/type_gen.py index 262d45f..fa16d83 100755 --- a/tools/typed/type_gen.py +++ b/tools/typed/type_gen.py @@ -5,7 +5,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -from lark import Lark +from lark import Lark, ParseError from exceptions import RangeError, DSLError from ir import TransformTypes from abi import AArch64ABI @@ -30,10 +30,13 @@ def parse_dsl(parser, inputs, abi): trees = [] for p in inputs: text = p.read() - parse_tree = parser.parse(text) - cur_tree = TransformTypes(text).transform(parse_tree) - - trees.append(cur_tree.get_intermediate_tree()) + try: + parse_tree = parser.parse(text) + cur_tree = TransformTypes(text).transform(parse_tree) + trees.append(cur_tree.get_intermediate_tree()) + except ParseError as e: + raise Exception("Parse error in {:s}: {:s}".format(p.name, + str(e))) final_tree = trees.pop(0) for t in trees: @@ -103,7 +106,8 @@ def main(): grammar_file = os.path.join(__loc__, 'grammars', 'typed_dsl.lark') - parser = Lark.open(grammar_file, 'start', propagate_positions=True) + parser = Lark.open(grammar_file, 'start', parser='lalr', + lexer='contextual', propagate_positions=True) if options.input: try: @@ -143,7 +147,7 @@ def main(): if options.deps is not None: deps = set() - deps.add(grammar_file) + deps.add(os.path.relpath(grammar_file)) if options.template is not None: deps.add(options.template.name) for m in sys.modules.values(): @@ -157,7 +161,10 @@ def main(): if f.startswith('../'): continue deps.add(f) - # TODO: include Cheetah templates + if options.template is None: + templates = ir.get_output_templates() + for t in templates: + deps.add(os.path.relpath(t)) if options.dump_pickle: out_name = options.dump_pickle.name else: