diff options
Diffstat (limited to 'arch')
501 files changed, 12049 insertions, 6149 deletions
diff --git a/arch/arc/configs/axs103_smp_defconfig b/arch/arc/configs/axs103_smp_defconfig index 2f336d99a8cf..07c89281c2e3 100644 --- a/arch/arc/configs/axs103_smp_defconfig +++ b/arch/arc/configs/axs103_smp_defconfig @@ -13,7 +13,6 @@ CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y # CONFIG_VM_EVENT_COUNTERS is not set # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_FORCE_LOAD=y CONFIG_MODULE_UNLOAD=y diff --git a/arch/arc/configs/haps_hs_defconfig b/arch/arc/configs/haps_hs_defconfig index 899b2fd5c71d..8c3ed5d6e6c3 100644 --- a/arch/arc/configs/haps_hs_defconfig +++ b/arch/arc/configs/haps_hs_defconfig @@ -14,7 +14,6 @@ CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y CONFIG_PERF_EVENTS=y # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_ARC_BUILTIN_DTB_NAME="haps_hs" CONFIG_MODULES=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/arc/configs/haps_hs_smp_defconfig b/arch/arc/configs/haps_hs_smp_defconfig index 0d32aac8069f..61107e8bac33 100644 --- a/arch/arc/configs/haps_hs_smp_defconfig +++ b/arch/arc/configs/haps_hs_smp_defconfig @@ -15,7 +15,6 @@ CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y # CONFIG_VM_EVENT_COUNTERS is not set # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_SMP=y CONFIG_ARC_BUILTIN_DTB_NAME="haps_hs_idu" CONFIG_KPROBES=y diff --git a/arch/arc/configs/hsdk_defconfig b/arch/arc/configs/hsdk_defconfig index d18378d2c2a6..4ee2a1507b57 100644 --- a/arch/arc/configs/hsdk_defconfig +++ b/arch/arc/configs/hsdk_defconfig @@ -13,7 +13,6 @@ CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y # CONFIG_VM_EVENT_COUNTERS is not set # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_ARC_SOC_HSDK=y diff --git a/arch/arc/configs/tb10x_defconfig b/arch/arc/configs/tb10x_defconfig index 6f0d2be9d926..941bbadd6bf2 100644 --- a/arch/arc/configs/tb10x_defconfig +++ b/arch/arc/configs/tb10x_defconfig @@ -19,7 +19,6 @@ CONFIG_KALLSYMS_ALL=y CONFIG_EMBEDDED=y # CONFIG_COMPAT_BRK is not set CONFIG_ISA_ARCOMPACT=y -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_FORCE_LOAD=y CONFIG_MODULE_UNLOAD=y diff --git a/arch/arc/include/asm/linkage.h b/arch/arc/include/asm/linkage.h index c9434ff3aa4c..8a3fb71e9cfa 100644 --- a/arch/arc/include/asm/linkage.h +++ b/arch/arc/include/asm/linkage.h @@ -8,6 +8,10 @@ #include <asm/dwarf.h> +#define ASM_NL ` /* use '`' to mark new line in macro */ +#define __ALIGN .align 4 +#define __ALIGN_STR __stringify(__ALIGN) + #ifdef __ASSEMBLY__ .macro ST2 e, o, off @@ -28,10 +32,6 @@ #endif .endm -#define ASM_NL ` /* use '`' to mark new line in macro */ -#define __ALIGN .align 4 -#define __ALIGN_STR __stringify(__ALIGN) - /* annotation for data we want in DCCM - if enabled in .config */ .macro ARCFP_DATA nm #ifdef CONFIG_ARC_HAS_DCCM diff --git a/arch/arm/configs/am200epdkit_defconfig b/arch/arm/configs/am200epdkit_defconfig index 3f633eaf9770..a25834e4c901 100644 --- a/arch/arm/configs/am200epdkit_defconfig +++ b/arch/arm/configs/am200epdkit_defconfig @@ -15,7 +15,6 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_PARTITION_ADVANCED=y # CONFIG_SWAP is not set -CONFIG_SLAB=y # CONFIG_VM_EVENT_COUNTERS is not set CONFIG_NET=y CONFIG_PACKET=m diff --git a/arch/arm/configs/at91_dt_defconfig b/arch/arm/configs/at91_dt_defconfig index 82bcf4dc7f54..8cc602853cc5 100644 --- a/arch/arm/configs/at91_dt_defconfig +++ b/arch/arm/configs/at91_dt_defconfig @@ -27,7 +27,6 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set # CONFIG_SWAP is not set -CONFIG_SLAB=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/arm/configs/dove_defconfig b/arch/arm/configs/dove_defconfig index c8d559f38f48..46859e6fee5b 100644 --- a/arch/arm/configs/dove_defconfig +++ b/arch/arm/configs/dove_defconfig @@ -17,7 +17,6 @@ CONFIG_VFP=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_PARTITION_ADVANCED=y -CONFIG_SLAB=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/arm/configs/ep93xx_defconfig b/arch/arm/configs/ep93xx_defconfig index 3154125321c5..44e89a980d29 100644 --- a/arch/arm/configs/ep93xx_defconfig +++ b/arch/arm/configs/ep93xx_defconfig @@ -24,7 +24,6 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y CONFIG_PARTITION_ADVANCED=y -CONFIG_SLAB=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/arm/configs/imx_v4_v5_defconfig b/arch/arm/configs/imx_v4_v5_defconfig index c9a602aee715..ec45e6225225 100644 --- a/arch/arm/configs/imx_v4_v5_defconfig +++ b/arch/arm/configs/imx_v4_v5_defconfig @@ -26,7 +26,6 @@ CONFIG_KPROBES=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_SWAP is not set -CONFIG_SLAB=y # CONFIG_COMPAT_BRK is not set CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/arm/configs/lpc32xx_defconfig b/arch/arm/configs/lpc32xx_defconfig index fabb66a53350..d7df0486850f 100644 --- a/arch/arm/configs/lpc32xx_defconfig +++ b/arch/arm/configs/lpc32xx_defconfig @@ -23,7 +23,6 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_PARTITION_ADVANCED=y # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set -CONFIG_SLAB=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/arm/configs/mmp2_defconfig b/arch/arm/configs/mmp2_defconfig index 7984640e994e..3d1d6f3b592a 100644 --- a/arch/arm/configs/mmp2_defconfig +++ b/arch/arm/configs/mmp2_defconfig @@ -12,7 +12,6 @@ CONFIG_VFP=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y -CONFIG_SLAB=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/arm/configs/mvebu_v7_defconfig b/arch/arm/configs/mvebu_v7_defconfig index 68a18264f31b..2d2a4dc8f379 100644 --- a/arch/arm/configs/mvebu_v7_defconfig +++ b/arch/arm/configs/mvebu_v7_defconfig @@ -23,7 +23,6 @@ CONFIG_VFP=y CONFIG_NEON=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y -CONFIG_SLAB=y # CONFIG_COMPACTION is not set CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/arm/configs/nhk8815_defconfig b/arch/arm/configs/nhk8815_defconfig index 4171dafddc0a..ea28ed8991b4 100644 --- a/arch/arm/configs/nhk8815_defconfig +++ b/arch/arm/configs/nhk8815_defconfig @@ -16,7 +16,6 @@ CONFIG_AEABI=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_SWAP is not set -CONFIG_SLAB=y CONFIG_CMA=y CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/arm/configs/omap2plus_defconfig b/arch/arm/configs/omap2plus_defconfig index c4216c552100..9bd36dd39bd0 100644 --- a/arch/arm/configs/omap2plus_defconfig +++ b/arch/arm/configs/omap2plus_defconfig @@ -62,7 +62,6 @@ CONFIG_MODVERSIONS=y CONFIG_MODULE_SRCVERSION_ALL=y CONFIG_PARTITION_ADVANCED=y CONFIG_BINFMT_MISC=y -CONFIG_SLAB=y CONFIG_CMA=y CONFIG_ZSMALLOC=m CONFIG_NET=y diff --git a/arch/arm/configs/pxa168_defconfig b/arch/arm/configs/pxa168_defconfig index 8422ddc9bab2..ec3a43f9c85e 100644 --- a/arch/arm/configs/pxa168_defconfig +++ b/arch/arm/configs/pxa168_defconfig @@ -12,7 +12,6 @@ CONFIG_FPE_NWFPE=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y -CONFIG_SLAB=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/arm/configs/pxa3xx_defconfig b/arch/arm/configs/pxa3xx_defconfig index d1e83b52e03a..7f95fa273a7d 100644 --- a/arch/arm/configs/pxa3xx_defconfig +++ b/arch/arm/configs/pxa3xx_defconfig @@ -12,7 +12,6 @@ CONFIG_AEABI=y CONFIG_CMDLINE="root=/dev/nfs rootfstype=nfs nfsroot=192.168.1.100:/nfsroot/ ip=192.168.1.101:192.168.1.100::255.255.255.0::eth0:on console=ttyS0,115200 mem=64M debug" CONFIG_FPE_NWFPE=y CONFIG_MODULES=y -CONFIG_SLAB=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/arm/configs/pxa910_defconfig b/arch/arm/configs/pxa910_defconfig index 48e41ca582af..958d958377dc 100644 --- a/arch/arm/configs/pxa910_defconfig +++ b/arch/arm/configs/pxa910_defconfig @@ -13,7 +13,6 @@ CONFIG_MODULE_FORCE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set CONFIG_AEABI=y CONFIG_FPE_NWFPE=y -CONFIG_SLAB=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/arm/configs/realview_defconfig b/arch/arm/configs/realview_defconfig index 92f803c2805c..a221a99f6472 100644 --- a/arch/arm/configs/realview_defconfig +++ b/arch/arm/configs/realview_defconfig @@ -20,7 +20,6 @@ CONFIG_VFP=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_SWAP is not set -CONFIG_SLAB=y CONFIG_CMA=y CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/arm/configs/rpc_defconfig b/arch/arm/configs/rpc_defconfig index 210974364d61..65a3fded55be 100644 --- a/arch/arm/configs/rpc_defconfig +++ b/arch/arm/configs/rpc_defconfig @@ -11,7 +11,6 @@ CONFIG_CPU_SA110=y CONFIG_FPE_NWFPE=y CONFIG_PARTITION_ADVANCED=y CONFIG_BSD_DISKLABEL=y -CONFIG_SLAB=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/arm/configs/sama5_defconfig b/arch/arm/configs/sama5_defconfig index c6aff6fb084d..56ee511210de 100644 --- a/arch/arm/configs/sama5_defconfig +++ b/arch/arm/configs/sama5_defconfig @@ -26,7 +26,6 @@ CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set # CONFIG_SWAP is not set -CONFIG_SLAB=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/arm/configs/sama7_defconfig b/arch/arm/configs/sama7_defconfig index 954112041403..a89767c89d17 100644 --- a/arch/arm/configs/sama7_defconfig +++ b/arch/arm/configs/sama7_defconfig @@ -41,7 +41,6 @@ CONFIG_PARTITION_ADVANCED=y # CONFIG_EFI_PARTITION is not set # CONFIG_COREDUMP is not set # CONFIG_SWAP is not set -CONFIG_SLAB=y # CONFIG_COMPACTION is not set CONFIG_CMA=y # CONFIG_VM_EVENT_COUNTERS is not set diff --git a/arch/arm/configs/shmobile_defconfig b/arch/arm/configs/shmobile_defconfig index 99748cff1ca4..22205ef0f376 100644 --- a/arch/arm/configs/shmobile_defconfig +++ b/arch/arm/configs/shmobile_defconfig @@ -24,7 +24,6 @@ CONFIG_CPUFREQ_DT=y CONFIG_VFP=y CONFIG_NEON=y # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set -CONFIG_SLAB=y CONFIG_CMA=y CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/arm/configs/sp7021_defconfig b/arch/arm/configs/sp7021_defconfig index c6448ac860b6..ec723401b440 100644 --- a/arch/arm/configs/sp7021_defconfig +++ b/arch/arm/configs/sp7021_defconfig @@ -24,7 +24,6 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODVERSIONS=y # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set -CONFIG_SLAB=y CONFIG_UEVENT_HELPER=y CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_DEVTMPFS=y diff --git a/arch/arm/configs/tegra_defconfig b/arch/arm/configs/tegra_defconfig index f32047e24b63..3c6af935e932 100644 --- a/arch/arm/configs/tegra_defconfig +++ b/arch/arm/configs/tegra_defconfig @@ -32,7 +32,6 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y CONFIG_PARTITION_ADVANCED=y -CONFIG_SLAB=y CONFIG_CMA=y CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/arm/configs/versatile_defconfig b/arch/arm/configs/versatile_defconfig index 67e3f9138306..849118cbbb44 100644 --- a/arch/arm/configs/versatile_defconfig +++ b/arch/arm/configs/versatile_defconfig @@ -14,7 +14,6 @@ CONFIG_VFP=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_PARTITION_ADVANCED=y -CONFIG_SLAB=y CONFIG_CMA=y CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/arm/crypto/sha1_neon_glue.c b/arch/arm/crypto/sha1_neon_glue.c index cfe36ae0f3f5..9c70b87e69f7 100644 --- a/arch/arm/crypto/sha1_neon_glue.c +++ b/arch/arm/crypto/sha1_neon_glue.c @@ -26,8 +26,8 @@ #include "sha1.h" -asmlinkage void sha1_transform_neon(void *state_h, const char *data, - unsigned int rounds); +asmlinkage void sha1_transform_neon(struct sha1_state *state_h, + const u8 *data, int rounds); static int sha1_neon_update(struct shash_desc *desc, const u8 *data, unsigned int len) @@ -39,8 +39,7 @@ static int sha1_neon_update(struct shash_desc *desc, const u8 *data, return sha1_update_arm(desc, data, len); kernel_neon_begin(); - sha1_base_do_update(desc, data, len, - (sha1_block_fn *)sha1_transform_neon); + sha1_base_do_update(desc, data, len, sha1_transform_neon); kernel_neon_end(); return 0; @@ -54,9 +53,8 @@ static int sha1_neon_finup(struct shash_desc *desc, const u8 *data, kernel_neon_begin(); if (len) - sha1_base_do_update(desc, data, len, - (sha1_block_fn *)sha1_transform_neon); - sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_transform_neon); + sha1_base_do_update(desc, data, len, sha1_transform_neon); + sha1_base_do_finalize(desc, sha1_transform_neon); kernel_neon_end(); return sha1_base_finish(desc, out); diff --git a/arch/arm/crypto/sha256_neon_glue.c b/arch/arm/crypto/sha256_neon_glue.c index 701706262ef3..ccdcfff71910 100644 --- a/arch/arm/crypto/sha256_neon_glue.c +++ b/arch/arm/crypto/sha256_neon_glue.c @@ -21,8 +21,8 @@ #include "sha256_glue.h" -asmlinkage void sha256_block_data_order_neon(u32 *digest, const void *data, - unsigned int num_blks); +asmlinkage void sha256_block_data_order_neon(struct sha256_state *digest, + const u8 *data, int num_blks); static int crypto_sha256_neon_update(struct shash_desc *desc, const u8 *data, unsigned int len) @@ -34,8 +34,7 @@ static int crypto_sha256_neon_update(struct shash_desc *desc, const u8 *data, return crypto_sha256_arm_update(desc, data, len); kernel_neon_begin(); - sha256_base_do_update(desc, data, len, - (sha256_block_fn *)sha256_block_data_order_neon); + sha256_base_do_update(desc, data, len, sha256_block_data_order_neon); kernel_neon_end(); return 0; @@ -50,9 +49,8 @@ static int crypto_sha256_neon_finup(struct shash_desc *desc, const u8 *data, kernel_neon_begin(); if (len) sha256_base_do_update(desc, data, len, - (sha256_block_fn *)sha256_block_data_order_neon); - sha256_base_do_finalize(desc, - (sha256_block_fn *)sha256_block_data_order_neon); + sha256_block_data_order_neon); + sha256_base_do_finalize(desc, sha256_block_data_order_neon); kernel_neon_end(); return sha256_base_finish(desc, out); diff --git a/arch/arm/crypto/sha512-neon-glue.c b/arch/arm/crypto/sha512-neon-glue.c index c879ad32db51..c6e58fe475ac 100644 --- a/arch/arm/crypto/sha512-neon-glue.c +++ b/arch/arm/crypto/sha512-neon-glue.c @@ -20,8 +20,8 @@ MODULE_ALIAS_CRYPTO("sha384-neon"); MODULE_ALIAS_CRYPTO("sha512-neon"); -asmlinkage void sha512_block_data_order_neon(u64 *state, u8 const *src, - int blocks); +asmlinkage void sha512_block_data_order_neon(struct sha512_state *state, + const u8 *src, int blocks); static int sha512_neon_update(struct shash_desc *desc, const u8 *data, unsigned int len) @@ -33,8 +33,7 @@ static int sha512_neon_update(struct shash_desc *desc, const u8 *data, return sha512_arm_update(desc, data, len); kernel_neon_begin(); - sha512_base_do_update(desc, data, len, - (sha512_block_fn *)sha512_block_data_order_neon); + sha512_base_do_update(desc, data, len, sha512_block_data_order_neon); kernel_neon_end(); return 0; @@ -49,9 +48,8 @@ static int sha512_neon_finup(struct shash_desc *desc, const u8 *data, kernel_neon_begin(); if (len) sha512_base_do_update(desc, data, len, - (sha512_block_fn *)sha512_block_data_order_neon); - sha512_base_do_finalize(desc, - (sha512_block_fn *)sha512_block_data_order_neon); + sha512_block_data_order_neon); + sha512_base_do_finalize(desc, sha512_block_data_order_neon); kernel_neon_end(); return sha512_base_finish(desc, out); diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 595028bd9160..7856c3a3e35a 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -202,6 +202,7 @@ config ARM64 select HAVE_FTRACE_MCOUNT_RECORD select HAVE_FUNCTION_TRACER select HAVE_FUNCTION_ERROR_INJECTION + select HAVE_FUNCTION_GRAPH_RETVAL if HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_GRAPH_TRACER select HAVE_GCC_PLUGINS select HAVE_HARDLOCKUP_DETECTOR_PERF if PERF_EVENTS && \ @@ -413,6 +414,25 @@ menu "Kernel Features" menu "ARM errata workarounds via the alternatives framework" +config AMPERE_ERRATUM_AC03_CPU_38 + bool "AmpereOne: AC03_CPU_38: Certain bits in the Virtualization Translation Control Register and Translation Control Registers do not follow RES0 semantics" + default y + help + This option adds an alternative code sequence to work around Ampere + erratum AC03_CPU_38 on AmpereOne. + + The affected design reports FEAT_HAFDBS as not implemented in + ID_AA64MMFR1_EL1.HAFDBS, but (V)TCR_ELx.{HA,HD} are not RES0 + as required by the architecture. The unadvertised HAFDBS + implementation suffers from an additional erratum where hardware + A/D updates can occur after a PTE has been marked invalid. + + The workaround forces KVM to explicitly set VTCR_EL2.HA to 0, + which avoids enabling unadvertised hardware Access Flag management + at stage-2. + + If unsure, say Y. + config ARM64_WORKAROUND_CLEAN_CACHE bool diff --git a/arch/arm64/crypto/sha256-glue.c b/arch/arm64/crypto/sha256-glue.c index 9462f6088b3f..9b5c86e07a9a 100644 --- a/arch/arm64/crypto/sha256-glue.c +++ b/arch/arm64/crypto/sha256-glue.c @@ -12,8 +12,9 @@ #include <crypto/internal/simd.h> #include <crypto/sha2.h> #include <crypto/sha256_base.h> -#include <linux/types.h> +#include <linux/module.h> #include <linux/string.h> +#include <linux/types.h> MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash for arm64"); MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>"); diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 7a95c324e52a..96e50227f940 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -15,6 +15,9 @@ #define MAX_CPU_FEATURES 128 #define cpu_feature(x) KERNEL_HWCAP_ ## x +#define ARM64_SW_FEATURE_OVERRIDE_NOKASLR 0 +#define ARM64_SW_FEATURE_OVERRIDE_HVHE 4 + #ifndef __ASSEMBLY__ #include <linux/bug.h> @@ -905,6 +908,7 @@ static inline unsigned int get_vmid_bits(u64 mmfr1) return 8; } +s64 arm64_ftr_safe_value(const struct arm64_ftr_bits *ftrp, s64 new, s64 cur); struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id); extern struct arm64_ftr_override id_aa64mmfr1_override; @@ -915,6 +919,8 @@ extern struct arm64_ftr_override id_aa64smfr0_override; extern struct arm64_ftr_override id_aa64isar1_override; extern struct arm64_ftr_override id_aa64isar2_override; +extern struct arm64_ftr_override arm64_sw_feature_override; + u32 get_kvm_ipa_limit(void); void dump_cpu_features(void); diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index f4c3d30bf746..8e5ffb58f83e 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -43,6 +43,11 @@ */ .macro __init_el2_timers mov x0, #3 // Enable EL1 physical timers + mrs x1, hcr_el2 + and x1, x1, #HCR_E2H + cbz x1, .LnVHE_\@ + lsl x0, x0, #10 +.LnVHE_\@: msr cnthctl_el2, x0 msr cntvoff_el2, xzr // Clear virtual offset .endm @@ -133,8 +138,15 @@ .endm /* Coprocessor traps */ -.macro __init_el2_nvhe_cptr +.macro __init_el2_cptr + mrs x1, hcr_el2 + and x1, x1, #HCR_E2H + cbz x1, .LnVHE_\@ + mov x0, #(CPACR_EL1_FPEN_EL1EN | CPACR_EL1_FPEN_EL0EN) + b .Lset_cptr_\@ +.LnVHE_\@: mov x0, #0x33ff +.Lset_cptr_\@: msr cptr_el2, x0 // Disable copro. traps to EL2 .endm @@ -210,9 +222,8 @@ __init_el2_gicv3 __init_el2_hstr __init_el2_nvhe_idregs - __init_el2_nvhe_cptr + __init_el2_cptr __init_el2_fgt - __init_el2_nvhe_prepare_eret .endm #ifndef __KVM_NVHE_HYPERVISOR__ @@ -258,7 +269,17 @@ .Linit_sve_\@: /* SVE register access */ mrs x0, cptr_el2 // Disable SVE traps + mrs x1, hcr_el2 + and x1, x1, #HCR_E2H + cbz x1, .Lcptr_nvhe_\@ + + // VHE case + orr x0, x0, #(CPACR_EL1_ZEN_EL1EN | CPACR_EL1_ZEN_EL0EN) + b .Lset_cptr_\@ + +.Lcptr_nvhe_\@: // nVHE case bic x0, x0, #CPTR_EL2_TZ +.Lset_cptr_\@: msr cptr_el2, x0 isb mov x1, #ZCR_ELx_LEN_MASK // SVE: Enable full vector diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h index cd03819a3b68..cdf6a35e3994 100644 --- a/arch/arm64/include/asm/fpsimdmacros.h +++ b/arch/arm64/include/asm/fpsimdmacros.h @@ -316,12 +316,12 @@ _for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16 cbz \save_ffr, 921f _sve_rdffr 0 - _sve_str_p 0, \nxbase - _sve_ldr_p 0, \nxbase, -16 b 922f 921: - str xzr, [x\nxbase] // Zero out FFR + _sve_pfalse 0 // Zero out FFR 922: + _sve_str_p 0, \nxbase + _sve_ldr_p 0, \nxbase, -16 mrs x\nxtmp, fpsr str w\nxtmp, [\xpfpsr] mrs x\nxtmp, fpcr diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h index b87d70b693c6..21ac1c5c71d3 100644 --- a/arch/arm64/include/asm/ftrace.h +++ b/arch/arm64/include/asm/ftrace.h @@ -192,4 +192,26 @@ static inline bool arch_syscall_match_sym_name(const char *sym, } #endif /* ifndef __ASSEMBLY__ */ +#ifndef __ASSEMBLY__ +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +struct fgraph_ret_regs { + /* x0 - x7 */ + unsigned long regs[8]; + + unsigned long fp; + unsigned long __unused; +}; + +static inline unsigned long fgraph_ret_regs_return_value(struct fgraph_ret_regs *ret_regs) +{ + return ret_regs->regs[0]; +} + +static inline unsigned long fgraph_ret_regs_frame_pointer(struct fgraph_ret_regs *ret_regs) +{ + return ret_regs->fp; +} +#endif /* ifdef CONFIG_FUNCTION_GRAPH_TRACER */ +#endif + #endif /* __ASM_FTRACE_H */ diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index c6e12e8f2751..58e5eb27da68 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -19,6 +19,7 @@ #define HCR_ATA_SHIFT 56 #define HCR_ATA (UL(1) << HCR_ATA_SHIFT) #define HCR_AMVOFFEN (UL(1) << 51) +#define HCR_TID4 (UL(1) << 49) #define HCR_FIEN (UL(1) << 47) #define HCR_FWB (UL(1) << 46) #define HCR_API (UL(1) << 41) @@ -87,7 +88,7 @@ #define HCR_GUEST_FLAGS (HCR_TSC | HCR_TSW | HCR_TWE | HCR_TWI | HCR_VM | \ HCR_BSU_IS | HCR_FB | HCR_TACR | \ HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW | HCR_TLOR | \ - HCR_FMO | HCR_IMO | HCR_PTW | HCR_TID3 | HCR_TID2) + HCR_FMO | HCR_IMO | HCR_PTW | HCR_TID3) #define HCR_VIRT_EXCP_MASK (HCR_VSE | HCR_VI | HCR_VF) #define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK | HCR_ATA) #define HCR_HOST_NVHE_PROTECTED_FLAGS (HCR_HOST_NVHE_FLAGS | HCR_TSC) @@ -289,7 +290,6 @@ #define CPTR_EL2_TFP (1 << CPTR_EL2_TFP_SHIFT) #define CPTR_EL2_TZ (1 << 8) #define CPTR_NVHE_EL2_RES1 0x000032ff /* known RES1 bits in CPTR_EL2 (nVHE) */ -#define CPTR_EL2_DEFAULT CPTR_NVHE_EL2_RES1 #define CPTR_NVHE_EL2_RES0 (GENMASK(63, 32) | \ GENMASK(29, 21) | \ GENMASK(19, 14) | \ @@ -351,8 +351,7 @@ ECN(SOFTSTP_CUR), ECN(WATCHPT_LOW), ECN(WATCHPT_CUR), \ ECN(BKPT32), ECN(VECTOR32), ECN(BRK64), ECN(ERET) -#define CPACR_EL1_DEFAULT (CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN |\ - CPACR_EL1_ZEN_EL1EN) +#define CPACR_EL1_TTA (1 << 28) #define kvm_mode_names \ { PSR_MODE_EL0t, "EL0t" }, \ diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 86042afa86c3..7d170aaa2db4 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -68,6 +68,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___kvm_vcpu_run, __KVM_HOST_SMCCC_FUNC___kvm_flush_vm_context, __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa, + __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa_nsh, __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid, __KVM_HOST_SMCCC_FUNC___kvm_flush_cpu_context, __KVM_HOST_SMCCC_FUNC___kvm_timer_set_cntvoff, @@ -225,6 +226,9 @@ extern void __kvm_flush_vm_context(void); extern void __kvm_flush_cpu_context(struct kvm_s2_mmu *mmu); extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa, int level); +extern void __kvm_tlb_flush_vmid_ipa_nsh(struct kvm_s2_mmu *mmu, + phys_addr_t ipa, + int level); extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu); extern void __kvm_timer_set_cntvoff(u64 cntvoff); diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index b31b32ecbe2d..efc0b45d79c3 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -62,19 +62,14 @@ static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu) #else static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu) { - struct kvm *kvm = vcpu->kvm; - - WARN_ON_ONCE(!test_bit(KVM_ARCH_FLAG_REG_WIDTH_CONFIGURED, - &kvm->arch.flags)); - - return test_bit(KVM_ARCH_FLAG_EL1_32BIT, &kvm->arch.flags); + return test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features); } #endif static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) { vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS; - if (is_kernel_in_hyp_mode()) + if (has_vhe() || has_hvhe()) vcpu->arch.hcr_el2 |= HCR_E2H; if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN)) { /* route synchronous external abort exceptions to EL2 */ @@ -95,6 +90,12 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) vcpu->arch.hcr_el2 |= HCR_TVM; } + if (cpus_have_final_cap(ARM64_HAS_EVT) && + !cpus_have_final_cap(ARM64_MISMATCHED_CACHE_TYPE)) + vcpu->arch.hcr_el2 |= HCR_TID4; + else + vcpu->arch.hcr_el2 |= HCR_TID2; + if (vcpu_el1_is_32bit(vcpu)) vcpu->arch.hcr_el2 &= ~HCR_RW; @@ -570,4 +571,35 @@ static inline bool vcpu_has_feature(struct kvm_vcpu *vcpu, int feature) return test_bit(feature, vcpu->arch.features); } +static __always_inline u64 kvm_get_reset_cptr_el2(struct kvm_vcpu *vcpu) +{ + u64 val; + + if (has_vhe()) { + val = (CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN | + CPACR_EL1_ZEN_EL1EN); + } else if (has_hvhe()) { + val = (CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN); + } else { + val = CPTR_NVHE_EL2_RES1; + + if (vcpu_has_sve(vcpu) && + (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED)) + val |= CPTR_EL2_TZ; + if (cpus_have_final_cap(ARM64_SME)) + val &= ~CPTR_EL2_TSM; + } + + return val; +} + +static __always_inline void kvm_reset_cptr_el2(struct kvm_vcpu *vcpu) +{ + u64 val = kvm_get_reset_cptr_el2(vcpu); + + if (has_vhe() || has_hvhe()) + write_sysreg(val, cpacr_el1); + else + write_sysreg(val, cptr_el2); +} #endif /* __ARM64_KVM_EMULATE_H__ */ diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index d48609d95423..8b6096753740 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -39,6 +39,7 @@ #define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS #define KVM_VCPU_MAX_FEATURES 7 +#define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1) #define KVM_REQ_SLEEP \ KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) @@ -159,6 +160,21 @@ struct kvm_s2_mmu { /* The last vcpu id that ran on each physical CPU */ int __percpu *last_vcpu_ran; +#define KVM_ARM_EAGER_SPLIT_CHUNK_SIZE_DEFAULT 0 + /* + * Memory cache used to split + * KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE worth of huge pages. It + * is used to allocate stage2 page tables while splitting huge + * pages. The choice of KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE + * influences both the capacity of the split page cache, and + * how often KVM reschedules. Be wary of raising CHUNK_SIZE + * too high. + * + * Protected by kvm->slots_lock. + */ + struct kvm_mmu_memory_cache split_page_cache; + uint64_t split_page_chunk_size; + struct kvm_arch *arch; }; @@ -214,25 +230,23 @@ struct kvm_arch { #define KVM_ARCH_FLAG_MTE_ENABLED 1 /* At least one vCPU has ran in the VM */ #define KVM_ARCH_FLAG_HAS_RAN_ONCE 2 - /* - * The following two bits are used to indicate the guest's EL1 - * register width configuration. A value of KVM_ARCH_FLAG_EL1_32BIT - * bit is valid only when KVM_ARCH_FLAG_REG_WIDTH_CONFIGURED is set. - * Otherwise, the guest's EL1 register width has not yet been - * determined yet. - */ -#define KVM_ARCH_FLAG_REG_WIDTH_CONFIGURED 3 -#define KVM_ARCH_FLAG_EL1_32BIT 4 + /* The vCPU feature set for the VM is configured */ +#define KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED 3 /* PSCI SYSTEM_SUSPEND enabled for the guest */ -#define KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED 5 +#define KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED 4 /* VM counter offset */ -#define KVM_ARCH_FLAG_VM_COUNTER_OFFSET 6 +#define KVM_ARCH_FLAG_VM_COUNTER_OFFSET 5 /* Timer PPIs made immutable */ -#define KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE 7 +#define KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE 6 /* SMCCC filter initialized for the VM */ -#define KVM_ARCH_FLAG_SMCCC_FILTER_CONFIGURED 8 +#define KVM_ARCH_FLAG_SMCCC_FILTER_CONFIGURED 7 + /* Initial ID reg values loaded */ +#define KVM_ARCH_FLAG_ID_REGS_INITIALIZED 8 unsigned long flags; + /* VM-wide vCPU feature set */ + DECLARE_BITMAP(vcpu_features, KVM_VCPU_MAX_FEATURES); + /* * VM-wide PMU filter, implemented as a bitmap and big enough for * up to 2^10 events (ARMv8.0) or 2^16 events (ARMv8.1+). @@ -242,18 +256,24 @@ struct kvm_arch { cpumask_var_t supported_cpus; - u8 pfr0_csv2; - u8 pfr0_csv3; - struct { - u8 imp:4; - u8 unimp:4; - } dfr0_pmuver; - /* Hypercall features firmware registers' descriptor */ struct kvm_smccc_features smccc_feat; struct maple_tree smccc_filter; /* + * Emulated CPU ID registers per VM + * (Op0, Op1, CRn, CRm, Op2) of the ID registers to be saved in it + * is (3, 0, 0, crm, op2), where 1<=crm<8, 0<=op2<8. + * + * These emulated idregs are VM-wide, but accessed from the context of a vCPU. + * Atomic access to multiple idregs are guarded by kvm_arch.config_lock. + */ +#define IDREG_IDX(id) (((sys_reg_CRm(id) - 1) << 3) | sys_reg_Op2(id)) +#define IDREG(kvm, id) ((kvm)->arch.id_regs[IDREG_IDX(id)]) +#define KVM_ARM_ID_REG_NUM (IDREG_IDX(sys_reg(3, 0, 0, 7, 7)) + 1) + u64 id_regs[KVM_ARM_ID_REG_NUM]; + + /* * For an untrusted host VM, 'pkvm.handle' is used to lookup * the associated pKVM instance in the hypervisor. */ @@ -410,6 +430,7 @@ struct kvm_host_data { struct kvm_host_psci_config { /* PSCI version used by host. */ u32 version; + u32 smccc_version; /* Function IDs used by host if version is v0.1. */ struct psci_0_1_function_ids function_ids_0_1; diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index bdd9cf546d95..b7238c72a04c 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -16,12 +16,35 @@ DECLARE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt); DECLARE_PER_CPU(unsigned long, kvm_hyp_vector); DECLARE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); +/* + * Unified accessors for registers that have a different encoding + * between VHE and non-VHE. They must be specified without their "ELx" + * encoding, but with the SYS_ prefix, as defined in asm/sysreg.h. + */ + +#if defined(__KVM_VHE_HYPERVISOR__) + +#define read_sysreg_el0(r) read_sysreg_s(r##_EL02) +#define write_sysreg_el0(v,r) write_sysreg_s(v, r##_EL02) +#define read_sysreg_el1(r) read_sysreg_s(r##_EL12) +#define write_sysreg_el1(v,r) write_sysreg_s(v, r##_EL12) +#define read_sysreg_el2(r) read_sysreg_s(r##_EL1) +#define write_sysreg_el2(v,r) write_sysreg_s(v, r##_EL1) + +#else // !__KVM_VHE_HYPERVISOR__ + +#if defined(__KVM_NVHE_HYPERVISOR__) +#define VHE_ALT_KEY ARM64_KVM_HVHE +#else +#define VHE_ALT_KEY ARM64_HAS_VIRT_HOST_EXTN +#endif + #define read_sysreg_elx(r,nvh,vh) \ ({ \ u64 reg; \ - asm volatile(ALTERNATIVE(__mrs_s("%0", r##nvh), \ + asm volatile(ALTERNATIVE(__mrs_s("%0", r##nvh), \ __mrs_s("%0", r##vh), \ - ARM64_HAS_VIRT_HOST_EXTN) \ + VHE_ALT_KEY) \ : "=r" (reg)); \ reg; \ }) @@ -31,16 +54,10 @@ DECLARE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); u64 __val = (u64)(v); \ asm volatile(ALTERNATIVE(__msr_s(r##nvh, "%x0"), \ __msr_s(r##vh, "%x0"), \ - ARM64_HAS_VIRT_HOST_EXTN) \ + VHE_ALT_KEY) \ : : "rZ" (__val)); \ } while (0) -/* - * Unified accessors for registers that have a different encoding - * between VHE and non-VHE. They must be specified without their "ELx" - * encoding, but with the SYS_ prefix, as defined in asm/sysreg.h. - */ - #define read_sysreg_el0(r) read_sysreg_elx(r, _EL0, _EL02) #define write_sysreg_el0(v,r) write_sysreg_elx(v, r, _EL0, _EL02) #define read_sysreg_el1(r) read_sysreg_elx(r, _EL1, _EL12) @@ -48,6 +65,8 @@ DECLARE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); #define read_sysreg_el2(r) read_sysreg_elx(r, _EL2, _EL1) #define write_sysreg_el2(v,r) write_sysreg_elx(v, r, _EL2, _EL1) +#endif // __KVM_VHE_HYPERVISOR__ + /* * Without an __arch_swab32(), we fall back to ___constant_swab32(), but the * static inline can allow the compiler to out-of-line this. KVM always wants diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 27e63c111f78..0e1e1ab17b4d 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -172,6 +172,7 @@ void __init free_hyp_pgds(void); void stage2_unmap_vm(struct kvm *kvm); int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long type); +void kvm_uninit_stage2_mmu(struct kvm *kvm); void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu); int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, phys_addr_t pa, unsigned long size, bool writable); @@ -227,7 +228,8 @@ static inline void __invalidate_icache_guest_page(void *va, size_t size) if (icache_is_aliasing()) { /* any kind of VIPT cache */ icache_inval_all_pou(); - } else if (is_kernel_in_hyp_mode() || !icache_is_vpipt()) { + } else if (read_sysreg(CurrentEL) != CurrentEL_EL1 || + !icache_is_vpipt()) { /* PIPT or VPIPT at EL2 (see comment in __kvm_tlb_flush_vmid_ipa) */ icache_inval_pou((unsigned long)va, (unsigned long)va + size); } diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index 93bd0975b15f..8294a9a7e566 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -92,6 +92,24 @@ static inline bool kvm_level_supports_block_mapping(u32 level) return level >= KVM_PGTABLE_MIN_BLOCK_LEVEL; } +static inline u32 kvm_supported_block_sizes(void) +{ + u32 level = KVM_PGTABLE_MIN_BLOCK_LEVEL; + u32 r = 0; + + for (; level < KVM_PGTABLE_MAX_LEVELS; level++) + r |= BIT(kvm_granule_shift(level)); + + return r; +} + +static inline bool kvm_is_block_size_supported(u64 size) +{ + bool is_power_of_two = IS_ALIGNED(size, size); + + return is_power_of_two && (size & kvm_supported_block_sizes()); +} + /** * struct kvm_pgtable_mm_ops - Memory management callbacks. * @zalloc_page: Allocate a single zeroed memory page. @@ -104,7 +122,7 @@ static inline bool kvm_level_supports_block_mapping(u32 level) * allocation is physically contiguous. * @free_pages_exact: Free an exact number of memory pages previously * allocated by zalloc_pages_exact. - * @free_removed_table: Free a removed paging structure by unlinking and + * @free_unlinked_table: Free an unlinked paging structure by unlinking and * dropping references. * @get_page: Increment the refcount on a page. * @put_page: Decrement the refcount on a page. When the @@ -124,7 +142,7 @@ struct kvm_pgtable_mm_ops { void* (*zalloc_page)(void *arg); void* (*zalloc_pages_exact)(size_t size); void (*free_pages_exact)(void *addr, size_t size); - void (*free_removed_table)(void *addr, u32 level); + void (*free_unlinked_table)(void *addr, u32 level); void (*get_page)(void *addr); void (*put_page)(void *addr); int (*page_count)(void *addr); @@ -195,6 +213,12 @@ typedef bool (*kvm_pgtable_force_pte_cb_t)(u64 addr, u64 end, * with other software walkers. * @KVM_PGTABLE_WALK_HANDLE_FAULT: Indicates the page-table walk was * invoked from a fault handler. + * @KVM_PGTABLE_WALK_SKIP_BBM_TLBI: Visit and update table entries + * without Break-before-make's + * TLB invalidation. + * @KVM_PGTABLE_WALK_SKIP_CMO: Visit and update table entries + * without Cache maintenance + * operations required. */ enum kvm_pgtable_walk_flags { KVM_PGTABLE_WALK_LEAF = BIT(0), @@ -202,6 +226,8 @@ enum kvm_pgtable_walk_flags { KVM_PGTABLE_WALK_TABLE_POST = BIT(2), KVM_PGTABLE_WALK_SHARED = BIT(3), KVM_PGTABLE_WALK_HANDLE_FAULT = BIT(4), + KVM_PGTABLE_WALK_SKIP_BBM_TLBI = BIT(5), + KVM_PGTABLE_WALK_SKIP_CMO = BIT(6), }; struct kvm_pgtable_visit_ctx { @@ -441,7 +467,7 @@ int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu, void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt); /** - * kvm_pgtable_stage2_free_removed() - Free a removed stage-2 paging structure. + * kvm_pgtable_stage2_free_unlinked() - Free an unlinked stage-2 paging structure. * @mm_ops: Memory management callbacks. * @pgtable: Unlinked stage-2 paging structure to be freed. * @level: Level of the stage-2 paging structure to be freed. @@ -449,7 +475,33 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt); * The page-table is assumed to be unreachable by any hardware walkers prior to * freeing and therefore no TLB invalidation is performed. */ -void kvm_pgtable_stage2_free_removed(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, u32 level); +void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, u32 level); + +/** + * kvm_pgtable_stage2_create_unlinked() - Create an unlinked stage-2 paging structure. + * @pgt: Page-table structure initialised by kvm_pgtable_stage2_init*(). + * @phys: Physical address of the memory to map. + * @level: Starting level of the stage-2 paging structure to be created. + * @prot: Permissions and attributes for the mapping. + * @mc: Cache of pre-allocated and zeroed memory from which to allocate + * page-table pages. + * @force_pte: Force mappings to PAGE_SIZE granularity. + * + * Returns an unlinked page-table tree. This new page-table tree is + * not reachable (i.e., it is unlinked) from the root pgd and it's + * therefore unreachableby the hardware page-table walker. No TLB + * invalidation or CMOs are performed. + * + * If device attributes are not explicitly requested in @prot, then the + * mapping will be normal, cacheable. + * + * Return: The fully populated (unlinked) stage-2 paging structure, or + * an ERR_PTR(error) on failure. + */ +kvm_pte_t *kvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt, + u64 phys, u32 level, + enum kvm_pgtable_prot prot, + void *mc, bool force_pte); /** * kvm_pgtable_stage2_map() - Install a mapping in a guest stage-2 page-table. @@ -621,6 +673,25 @@ bool kvm_pgtable_stage2_is_young(struct kvm_pgtable *pgt, u64 addr); int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size); /** + * kvm_pgtable_stage2_split() - Split a range of huge pages into leaf PTEs pointing + * to PAGE_SIZE guest pages. + * @pgt: Page-table structure initialised by kvm_pgtable_stage2_init(). + * @addr: Intermediate physical address from which to split. + * @size: Size of the range. + * @mc: Cache of pre-allocated and zeroed memory from which to allocate + * page-table pages. + * + * The function tries to split any level 1 or 2 entry that overlaps + * with the input range (given by @addr and @size). + * + * Return: 0 on success, negative error code on failure. Note that + * kvm_pgtable_stage2_split() is best effort: it tries to break as many + * blocks in the input range as allowed by @mc_capacity. + */ +int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size, + struct kvm_mmu_memory_cache *mc); + +/** * kvm_pgtable_walk() - Walk a page-table. * @pgt: Page-table structure initialised by kvm_pgtable_*_init(). * @addr: Input address for the start of the walk. diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index 01129b0d4c68..e46250a02017 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -6,7 +6,9 @@ #ifndef __ARM64_KVM_PKVM_H__ #define __ARM64_KVM_PKVM_H__ +#include <linux/arm_ffa.h> #include <linux/memblock.h> +#include <linux/scatterlist.h> #include <asm/kvm_pgtable.h> /* Maximum number of VMs that can co-exist under pKVM. */ @@ -106,4 +108,23 @@ static inline unsigned long host_s2_pgtable_pages(void) return res; } +#define KVM_FFA_MBOX_NR_PAGES 1 + +static inline unsigned long hyp_ffa_proxy_pages(void) +{ + size_t desc_max; + + /* + * The hypervisor FFA proxy needs enough memory to buffer a fragmented + * descriptor returned from EL3 in response to a RETRIEVE_REQ call. + */ + desc_max = sizeof(struct ffa_mem_region) + + sizeof(struct ffa_mem_region_attributes) + + sizeof(struct ffa_composite_mem_region) + + SG_MAX_SEGMENTS * sizeof(struct ffa_mem_region_addr_range); + + /* Plus a page each for the hypervisor's RX and TX mailboxes. */ + return (2 * KVM_FFA_MBOX_NR_PAGES) + DIV_ROUND_UP(desc_max, PAGE_SIZE); +} + #endif /* __ARM64_KVM_PKVM_H__ */ diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 7a1e62631814..b481935e9314 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -510,6 +510,7 @@ (BIT(18)) | (BIT(22)) | (BIT(23)) | (BIT(28)) | \ (BIT(29))) +#define SCTLR_EL2_BT (BIT(36)) #ifdef CONFIG_CPU_BIG_ENDIAN #define ENDIAN_SET_EL2 SCTLR_ELx_EE #else diff --git a/arch/arm64/include/asm/virt.h b/arch/arm64/include/asm/virt.h index 4eb601e7de50..5227db7640c8 100644 --- a/arch/arm64/include/asm/virt.h +++ b/arch/arm64/include/asm/virt.h @@ -110,8 +110,10 @@ static inline bool is_hyp_mode_mismatched(void) return __boot_cpu_mode[0] != __boot_cpu_mode[1]; } -static inline bool is_kernel_in_hyp_mode(void) +static __always_inline bool is_kernel_in_hyp_mode(void) { + BUILD_BUG_ON(__is_defined(__KVM_NVHE_HYPERVISOR__) || + __is_defined(__KVM_VHE_HYPERVISOR__)); return read_sysreg(CurrentEL) == CurrentEL_EL2; } @@ -140,6 +142,14 @@ static __always_inline bool is_protected_kvm_enabled(void) return cpus_have_final_cap(ARM64_KVM_PROTECTED_MODE); } +static __always_inline bool has_hvhe(void) +{ + if (is_vhe_hyp_code()) + return false; + + return cpus_have_final_cap(ARM64_KVM_HVHE); +} + static inline bool is_hyp_nvhe(void) { return is_hyp_mode_available() && !is_kernel_in_hyp_mode(); diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index 0996094b0d22..757d01a68ffd 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -200,6 +200,19 @@ int main(void) #endif #ifdef CONFIG_FUNCTION_TRACER DEFINE(FTRACE_OPS_FUNC, offsetof(struct ftrace_ops, func)); +#endif + BLANK(); +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + DEFINE(FGRET_REGS_X0, offsetof(struct fgraph_ret_regs, regs[0])); + DEFINE(FGRET_REGS_X1, offsetof(struct fgraph_ret_regs, regs[1])); + DEFINE(FGRET_REGS_X2, offsetof(struct fgraph_ret_regs, regs[2])); + DEFINE(FGRET_REGS_X3, offsetof(struct fgraph_ret_regs, regs[3])); + DEFINE(FGRET_REGS_X4, offsetof(struct fgraph_ret_regs, regs[4])); + DEFINE(FGRET_REGS_X5, offsetof(struct fgraph_ret_regs, regs[5])); + DEFINE(FGRET_REGS_X6, offsetof(struct fgraph_ret_regs, regs[6])); + DEFINE(FGRET_REGS_X7, offsetof(struct fgraph_ret_regs, regs[7])); + DEFINE(FGRET_REGS_FP, offsetof(struct fgraph_ret_regs, fp)); + DEFINE(FGRET_REGS_SIZE, sizeof(struct fgraph_ret_regs)); #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS DEFINE(FTRACE_OPS_DIRECT_CALL, offsetof(struct ftrace_ops, direct_call)); #endif diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 307faa2b4395..be66e94a21bd 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -730,6 +730,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .cpu_enable = cpu_clear_bf16_from_user_emulation, }, #endif +#ifdef CONFIG_AMPERE_ERRATUM_AC03_CPU_38 + { + .desc = "AmpereOne erratum AC03_CPU_38", + .capability = ARM64_WORKAROUND_AMPERE_AC03_CPU_38, + ERRATA_MIDR_ALL_VERSIONS(MIDR_AMPERE1), + }, +#endif { } }; diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 6ea7f23b1287..f9d456fe132d 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -672,6 +672,8 @@ struct arm64_ftr_override __ro_after_init id_aa64smfr0_override; struct arm64_ftr_override __ro_after_init id_aa64isar1_override; struct arm64_ftr_override __ro_after_init id_aa64isar2_override; +struct arm64_ftr_override arm64_sw_feature_override; + static const struct __ftr_reg_entry { u32 sys_id; struct arm64_ftr_reg *reg; @@ -807,7 +809,7 @@ static u64 arm64_ftr_set_value(const struct arm64_ftr_bits *ftrp, s64 reg, return reg; } -static s64 arm64_ftr_safe_value(const struct arm64_ftr_bits *ftrp, s64 new, +s64 arm64_ftr_safe_value(const struct arm64_ftr_bits *ftrp, s64 new, s64 cur) { s64 ret = 0; @@ -2009,6 +2011,19 @@ static bool has_nested_virt_support(const struct arm64_cpu_capabilities *cap, return true; } +static bool hvhe_possible(const struct arm64_cpu_capabilities *entry, + int __unused) +{ + u64 val; + + val = read_sysreg(id_aa64mmfr1_el1); + if (!cpuid_feature_extract_unsigned_field(val, ID_AA64MMFR1_EL1_VH_SHIFT)) + return false; + + val = arm64_sw_feature_override.val & arm64_sw_feature_override.mask; + return cpuid_feature_extract_unsigned_field(val, ARM64_SW_FEATURE_OVERRIDE_HVHE); +} + #ifdef CONFIG_ARM64_PAN static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused) { @@ -2683,6 +2698,23 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, ARM64_CPUID_FIELDS(ID_AA64MMFR3_EL1, S1PIE, IMP) }, + { + .desc = "VHE for hypervisor only", + .capability = ARM64_KVM_HVHE, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = hvhe_possible, + }, + { + .desc = "Enhanced Virtualization Traps", + .capability = ARM64_HAS_EVT, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .sys_reg = SYS_ID_AA64MMFR2_EL1, + .sign = FTR_UNSIGNED, + .field_pos = ID_AA64MMFR2_EL1_EVT_SHIFT, + .field_width = 4, + .min_field_value = ID_AA64MMFR2_EL1_EVT_IMP, + .matches = has_cpuid_feature, + }, {}, }; diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S index 1c38a60575aa..f0c16640ef21 100644 --- a/arch/arm64/kernel/entry-ftrace.S +++ b/arch/arm64/kernel/entry-ftrace.S @@ -330,22 +330,23 @@ SYM_FUNC_END(ftrace_stub_graph) */ SYM_CODE_START(return_to_handler) /* save return value regs */ - sub sp, sp, #64 - stp x0, x1, [sp] - stp x2, x3, [sp, #16] - stp x4, x5, [sp, #32] - stp x6, x7, [sp, #48] + sub sp, sp, #FGRET_REGS_SIZE + stp x0, x1, [sp, #FGRET_REGS_X0] + stp x2, x3, [sp, #FGRET_REGS_X2] + stp x4, x5, [sp, #FGRET_REGS_X4] + stp x6, x7, [sp, #FGRET_REGS_X6] + str x29, [sp, #FGRET_REGS_FP] // parent's fp - mov x0, x29 // parent's fp - bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp); - mov x30, x0 // restore the original return address + mov x0, sp + bl ftrace_return_to_handler // addr = ftrace_return_to_hander(regs); + mov x30, x0 // restore the original return address /* restore return value regs */ - ldp x0, x1, [sp] - ldp x2, x3, [sp, #16] - ldp x4, x5, [sp, #32] - ldp x6, x7, [sp, #48] - add sp, sp, #64 + ldp x0, x1, [sp, #FGRET_REGS_X0] + ldp x2, x3, [sp, #FGRET_REGS_X2] + ldp x4, x5, [sp, #FGRET_REGS_X4] + ldp x6, x7, [sp, #FGRET_REGS_X6] + add sp, sp, #FGRET_REGS_SIZE ret SYM_CODE_END(return_to_handler) diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 0f5a30f109d9..757a0de07f91 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -603,6 +603,8 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL) msr sctlr_el1, x1 mov x2, xzr 2: + __init_el2_nvhe_prepare_eret + mov w0, #BOOT_CPU_MODE_EL2 orr x0, x0, x2 eret diff --git a/arch/arm64/kernel/hyp-stub.S b/arch/arm64/kernel/hyp-stub.S index d63de1973ddb..65f76064c86b 100644 --- a/arch/arm64/kernel/hyp-stub.S +++ b/arch/arm64/kernel/hyp-stub.S @@ -82,7 +82,15 @@ SYM_CODE_START_LOCAL(__finalise_el2) tbnz x1, #0, 1f // Needs to be VHE capable, obviously - check_override id_aa64mmfr1 ID_AA64MMFR1_EL1_VH_SHIFT 2f 1f x1 x2 + check_override id_aa64mmfr1 ID_AA64MMFR1_EL1_VH_SHIFT 0f 1f x1 x2 + +0: // Check whether we only want the hypervisor to run VHE, not the kernel + adr_l x1, arm64_sw_feature_override + ldr x2, [x1, FTR_OVR_VAL_OFFSET] + ldr x1, [x1, FTR_OVR_MASK_OFFSET] + and x2, x2, x1 + ubfx x2, x2, #ARM64_SW_FEATURE_OVERRIDE_HVHE, #4 + cbz x2, 2f 1: mov_q x0, HVC_STUB_ERR eret diff --git a/arch/arm64/kernel/idreg-override.c b/arch/arm64/kernel/idreg-override.c index 8439248c21d3..2fe2491b692c 100644 --- a/arch/arm64/kernel/idreg-override.c +++ b/arch/arm64/kernel/idreg-override.c @@ -139,15 +139,22 @@ static const struct ftr_set_desc smfr0 __initconst = { }, }; -extern struct arm64_ftr_override kaslr_feature_override; +static bool __init hvhe_filter(u64 val) +{ + u64 mmfr1 = read_sysreg(id_aa64mmfr1_el1); + + return (val == 1 && + lower_32_bits(__boot_status) == BOOT_CPU_MODE_EL2 && + cpuid_feature_extract_unsigned_field(mmfr1, + ID_AA64MMFR1_EL1_VH_SHIFT)); +} -static const struct ftr_set_desc kaslr __initconst = { - .name = "kaslr", -#ifdef CONFIG_RANDOMIZE_BASE - .override = &kaslr_feature_override, -#endif +static const struct ftr_set_desc sw_features __initconst = { + .name = "arm64_sw", + .override = &arm64_sw_feature_override, .fields = { - FIELD("disabled", 0, NULL), + FIELD("nokaslr", ARM64_SW_FEATURE_OVERRIDE_NOKASLR, NULL), + FIELD("hvhe", ARM64_SW_FEATURE_OVERRIDE_HVHE, hvhe_filter), {} }, }; @@ -159,7 +166,7 @@ static const struct ftr_set_desc * const regs[] __initconst = { &isar1, &isar2, &smfr0, - &kaslr, + &sw_features, }; static const struct { @@ -177,7 +184,7 @@ static const struct { "id_aa64isar2.gpa3=0 id_aa64isar2.apa3=0" }, { "arm64.nomops", "id_aa64isar2.mops=0" }, { "arm64.nomte", "id_aa64pfr1.mte=0" }, - { "nokaslr", "kaslr.disabled=1" }, + { "nokaslr", "arm64_sw.nokaslr=1" }, }; static int __init parse_nokaslr(char *unused) diff --git a/arch/arm64/kernel/kaslr.c b/arch/arm64/kernel/kaslr.c index 17f96a19781d..94a269cd1f07 100644 --- a/arch/arm64/kernel/kaslr.c +++ b/arch/arm64/kernel/kaslr.c @@ -12,13 +12,13 @@ u16 __initdata memstart_offset_seed; -struct arm64_ftr_override kaslr_feature_override __initdata; - bool __ro_after_init __kaslr_is_enabled = false; void __init kaslr_init(void) { - if (kaslr_feature_override.val & kaslr_feature_override.mask & 0xf) { + if (cpuid_feature_extract_unsigned_field(arm64_sw_feature_override.val & + arm64_sw_feature_override.mask, + ARM64_SW_FEATURE_OVERRIDE_NOKASLR)) { pr_info("KASLR disabled on command line\n"); return; } diff --git a/arch/arm64/kernel/vdso32/Makefile b/arch/arm64/kernel/vdso32/Makefile index d014162c5c71..2f73e5bca213 100644 --- a/arch/arm64/kernel/vdso32/Makefile +++ b/arch/arm64/kernel/vdso32/Makefile @@ -65,11 +65,9 @@ VDSO_CFLAGS += -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs \ -fno-strict-aliasing -fno-common \ -Werror-implicit-function-declaration \ -Wno-format-security \ - -Wdeclaration-after-statement \ -std=gnu11 VDSO_CFLAGS += -O2 # Some useful compiler-dependent flags from top-level Makefile -VDSO_CFLAGS += $(call cc32-option,-Wdeclaration-after-statement,) VDSO_CFLAGS += $(call cc32-option,-Wno-pointer-sign) VDSO_CFLAGS += -fno-strict-overflow VDSO_CFLAGS += $(call cc32-option,-Werror=strict-prototypes) diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index 05b022be885b..0696732fa38c 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -1406,7 +1406,7 @@ int __init kvm_timer_hyp_init(bool has_gic) kvm_get_running_vcpus()); if (err) { kvm_err("kvm_arch_timer: error setting vcpu affinity\n"); - goto out_free_irq; + goto out_free_vtimer_irq; } static_branch_enable(&has_gic_active_state); @@ -1422,7 +1422,7 @@ int __init kvm_timer_hyp_init(bool has_gic) if (err) { kvm_err("kvm_arch_timer: can't request ptimer interrupt %d (%d)\n", host_ptimer_irq, err); - return err; + goto out_free_vtimer_irq; } if (has_gic) { @@ -1430,7 +1430,7 @@ int __init kvm_timer_hyp_init(bool has_gic) kvm_get_running_vcpus()); if (err) { kvm_err("kvm_arch_timer: error setting vcpu affinity\n"); - goto out_free_irq; + goto out_free_ptimer_irq; } } @@ -1439,11 +1439,15 @@ int __init kvm_timer_hyp_init(bool has_gic) kvm_err("kvm_arch_timer: invalid physical timer IRQ: %d\n", info->physical_irq); err = -ENODEV; - goto out_free_irq; + goto out_free_vtimer_irq; } return 0; -out_free_irq: + +out_free_ptimer_irq: + if (info->physical_irq > 0) + free_percpu_irq(host_ptimer_irq, kvm_get_running_vcpus()); +out_free_vtimer_irq: free_percpu_irq(host_vtimer_irq, kvm_get_running_vcpus()); return err; } diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 14391826241c..c2c14059f6a8 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -51,6 +51,8 @@ DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector); DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page); DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); +DECLARE_KVM_NVHE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt); + static bool vgic_present; static DEFINE_PER_CPU(unsigned char, kvm_arm_hardware_enabled); @@ -65,6 +67,7 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) { int r; + u64 new_cap; if (cap->flags) return -EINVAL; @@ -89,6 +92,24 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, r = 0; set_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags); break; + case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE: + new_cap = cap->args[0]; + + mutex_lock(&kvm->slots_lock); + /* + * To keep things simple, allow changing the chunk + * size only when no memory slots have been created. + */ + if (!kvm_are_all_memslots_empty(kvm)) { + r = -EINVAL; + } else if (new_cap && !kvm_is_block_size_supported(new_cap)) { + r = -EINVAL; + } else { + r = 0; + kvm->arch.mmu.split_page_chunk_size = new_cap; + } + mutex_unlock(&kvm->slots_lock); + break; default: r = -EINVAL; break; @@ -102,22 +123,6 @@ static int kvm_arm_default_max_vcpus(void) return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS; } -static void set_default_spectre(struct kvm *kvm) -{ - /* - * The default is to expose CSV2 == 1 if the HW isn't affected. - * Although this is a per-CPU feature, we make it global because - * asymmetric systems are just a nuisance. - * - * Userspace can override this as long as it doesn't promise - * the impossible. - */ - if (arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED) - kvm->arch.pfr0_csv2 = 1; - if (arm64_get_meltdown_state() == SPECTRE_UNAFFECTED) - kvm->arch.pfr0_csv3 = 1; -} - /** * kvm_arch_init_vm - initializes a VM data structure * @kvm: pointer to the KVM struct @@ -161,14 +166,9 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) /* The maximum number of VCPUs is limited by the host's GIC model */ kvm->max_vcpus = kvm_arm_default_max_vcpus(); - set_default_spectre(kvm); kvm_arm_init_hypercalls(kvm); - /* - * Initialise the default PMUver before there is a chance to - * create an actual PMU. - */ - kvm->arch.dfr0_pmuver.imp = kvm_arm_pmu_get_pmuver_limit(); + bitmap_zero(kvm->arch.vcpu_features, KVM_VCPU_MAX_FEATURES); return 0; @@ -302,6 +302,15 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_ARM_PTRAUTH_GENERIC: r = system_has_full_ptr_auth(); break; + case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE: + if (kvm) + r = kvm->arch.mmu.split_page_chunk_size; + else + r = KVM_ARM_EAGER_SPLIT_CHUNK_SIZE_DEFAULT; + break; + case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES: + r = kvm_supported_block_sizes(); + break; default: r = 0; } @@ -1167,58 +1176,115 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, return -EINVAL; } -static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu, - const struct kvm_vcpu_init *init) +static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu, + const struct kvm_vcpu_init *init) { - unsigned int i, ret; - u32 phys_target = kvm_target_cpu(); + unsigned long features = init->features[0]; + int i; + + if (features & ~KVM_VCPU_VALID_FEATURES) + return -ENOENT; + + for (i = 1; i < ARRAY_SIZE(init->features); i++) { + if (init->features[i]) + return -ENOENT; + } + + if (!test_bit(KVM_ARM_VCPU_EL1_32BIT, &features)) + return 0; - if (init->target != phys_target) + if (!cpus_have_const_cap(ARM64_HAS_32BIT_EL1)) return -EINVAL; - /* - * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must - * use the same target. - */ - if (vcpu->arch.target != -1 && vcpu->arch.target != init->target) + /* MTE is incompatible with AArch32 */ + if (kvm_has_mte(vcpu->kvm)) return -EINVAL; - /* -ENOENT for unknown features, -EINVAL for invalid combinations. */ - for (i = 0; i < sizeof(init->features) * 8; i++) { - bool set = (init->features[i / 32] & (1 << (i % 32))); + /* NV is incompatible with AArch32 */ + if (test_bit(KVM_ARM_VCPU_HAS_EL2, &features)) + return -EINVAL; - if (set && i >= KVM_VCPU_MAX_FEATURES) - return -ENOENT; + return 0; +} - /* - * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must - * use the same feature set. - */ - if (vcpu->arch.target != -1 && i < KVM_VCPU_MAX_FEATURES && - test_bit(i, vcpu->arch.features) != set) - return -EINVAL; +static bool kvm_vcpu_init_changed(struct kvm_vcpu *vcpu, + const struct kvm_vcpu_init *init) +{ + unsigned long features = init->features[0]; - if (set) - set_bit(i, vcpu->arch.features); - } + return !bitmap_equal(vcpu->arch.features, &features, KVM_VCPU_MAX_FEATURES) || + vcpu->arch.target != init->target; +} + +static int __kvm_vcpu_set_target(struct kvm_vcpu *vcpu, + const struct kvm_vcpu_init *init) +{ + unsigned long features = init->features[0]; + struct kvm *kvm = vcpu->kvm; + int ret = -EINVAL; + + mutex_lock(&kvm->arch.config_lock); - vcpu->arch.target = phys_target; + if (test_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags) && + !bitmap_equal(kvm->arch.vcpu_features, &features, KVM_VCPU_MAX_FEATURES)) + goto out_unlock; + + vcpu->arch.target = init->target; + bitmap_copy(vcpu->arch.features, &features, KVM_VCPU_MAX_FEATURES); /* Now we know what it is, we can reset it. */ ret = kvm_reset_vcpu(vcpu); if (ret) { vcpu->arch.target = -1; bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES); + goto out_unlock; } + bitmap_copy(kvm->arch.vcpu_features, &features, KVM_VCPU_MAX_FEATURES); + set_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags); + +out_unlock: + mutex_unlock(&kvm->arch.config_lock); return ret; } +static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu, + const struct kvm_vcpu_init *init) +{ + int ret; + + if (init->target != kvm_target_cpu()) + return -EINVAL; + + ret = kvm_vcpu_init_check_features(vcpu, init); + if (ret) + return ret; + + if (vcpu->arch.target == -1) + return __kvm_vcpu_set_target(vcpu, init); + + if (kvm_vcpu_init_changed(vcpu, init)) + return -EINVAL; + + return kvm_reset_vcpu(vcpu); +} + static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init) { + bool power_off = false; int ret; + /* + * Treat the power-off vCPU feature as ephemeral. Clear the bit to avoid + * reflecting it in the finalized feature set, thus limiting its scope + * to a single KVM_ARM_VCPU_INIT call. + */ + if (init->features[0] & BIT(KVM_ARM_VCPU_POWER_OFF)) { + init->features[0] &= ~BIT(KVM_ARM_VCPU_POWER_OFF); + power_off = true; + } + ret = kvm_vcpu_set_target(vcpu, init); if (ret) return ret; @@ -1240,14 +1306,14 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, } vcpu_reset_hcr(vcpu); - vcpu->arch.cptr_el2 = CPTR_EL2_DEFAULT; + vcpu->arch.cptr_el2 = kvm_get_reset_cptr_el2(vcpu); /* * Handle the "start in power-off" case. */ spin_lock(&vcpu->arch.mp_state_lock); - if (test_bit(KVM_ARM_VCPU_POWER_OFF, vcpu->arch.features)) + if (power_off) __kvm_arm_vcpu_power_off(vcpu); else WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE); @@ -1666,7 +1732,13 @@ static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits) params->mair_el2 = read_sysreg(mair_el1); - tcr = (read_sysreg(tcr_el1) & TCR_EL2_MASK) | TCR_EL2_RES1; + tcr = read_sysreg(tcr_el1); + if (cpus_have_final_cap(ARM64_KVM_HVHE)) { + tcr |= TCR_EPD1_MASK; + } else { + tcr &= TCR_EL2_MASK; + tcr |= TCR_EL2_RES1; + } tcr &= ~TCR_T0SZ_MASK; tcr |= TCR_T0SZ(hyp_va_bits); params->tcr_el2 = tcr; @@ -1676,6 +1748,8 @@ static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits) params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS; else params->hcr_el2 = HCR_HOST_NVHE_FLAGS; + if (cpus_have_final_cap(ARM64_KVM_HVHE)) + params->hcr_el2 |= HCR_E2H; params->vttbr = params->vtcr = 0; /* @@ -1910,6 +1984,7 @@ static bool __init init_psci_relay(void) } kvm_host_psci_config.version = psci_ops.get_version(); + kvm_host_psci_config.smccc_version = arm_smccc_get_version(); if (kvm_host_psci_config.version == PSCI_VERSION(0, 1)) { kvm_host_psci_config.function_ids_0_1 = get_psci_0_1_function_ids(); @@ -2067,6 +2142,26 @@ static int __init kvm_hyp_init_protection(u32 hyp_va_bits) return 0; } +static void pkvm_hyp_init_ptrauth(void) +{ + struct kvm_cpu_context *hyp_ctxt; + int cpu; + + for_each_possible_cpu(cpu) { + hyp_ctxt = per_cpu_ptr_nvhe_sym(kvm_hyp_ctxt, cpu); + hyp_ctxt->sys_regs[APIAKEYLO_EL1] = get_random_long(); + hyp_ctxt->sys_regs[APIAKEYHI_EL1] = get_random_long(); + hyp_ctxt->sys_regs[APIBKEYLO_EL1] = get_random_long(); + hyp_ctxt->sys_regs[APIBKEYHI_EL1] = get_random_long(); + hyp_ctxt->sys_regs[APDAKEYLO_EL1] = get_random_long(); + hyp_ctxt->sys_regs[APDAKEYHI_EL1] = get_random_long(); + hyp_ctxt->sys_regs[APDBKEYLO_EL1] = get_random_long(); + hyp_ctxt->sys_regs[APDBKEYHI_EL1] = get_random_long(); + hyp_ctxt->sys_regs[APGAKEYLO_EL1] = get_random_long(); + hyp_ctxt->sys_regs[APGAKEYHI_EL1] = get_random_long(); + } +} + /* Inits Hyp-mode on all online CPUs */ static int __init init_hyp_mode(void) { @@ -2228,6 +2323,10 @@ static int __init init_hyp_mode(void) kvm_hyp_init_symbols(); if (is_protected_kvm_enabled()) { + if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL) && + cpus_have_const_cap(ARM64_HAS_ADDRESS_AUTH)) + pkvm_hyp_init_ptrauth(); + init_cpu_logical_map(); if (!init_psci_relay()) { diff --git a/arch/arm64/kvm/fpsimd.c b/arch/arm64/kvm/fpsimd.c index 4c9dcd8fc939..8c1d0d4853df 100644 --- a/arch/arm64/kvm/fpsimd.c +++ b/arch/arm64/kvm/fpsimd.c @@ -180,7 +180,7 @@ void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu) /* * If we have VHE then the Hyp code will reset CPACR_EL1 to - * CPACR_EL1_DEFAULT and we need to reenable SME. + * the default value and we need to reenable SME. */ if (has_vhe() && system_supports_sme()) { /* Also restore EL0 state seen on entry */ @@ -210,7 +210,7 @@ void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu) /* * The FPSIMD/SVE state in the CPU has not been touched, and we * have SVE (and VHE): CPACR_EL1 (alias CPTR_EL2) has been - * reset to CPACR_EL1_DEFAULT by the Hyp code, disabling SVE + * reset by kvm_reset_cptr_el2() in the Hyp code, disabling SVE * for EL0. To avoid spurious traps, restore the trap state * seen by kvm_arch_vcpu_load_fp(): */ diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 2f6e0b3e4a75..4bddb8541bec 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -70,6 +70,56 @@ static inline void __activate_traps_fpsimd32(struct kvm_vcpu *vcpu) } } +static inline bool __hfgxtr_traps_required(void) +{ + if (cpus_have_final_cap(ARM64_SME)) + return true; + + if (cpus_have_final_cap(ARM64_WORKAROUND_AMPERE_AC03_CPU_38)) + return true; + + return false; +} + +static inline void __activate_traps_hfgxtr(void) +{ + u64 r_clr = 0, w_clr = 0, r_set = 0, w_set = 0, tmp; + + if (cpus_have_final_cap(ARM64_SME)) { + tmp = HFGxTR_EL2_nSMPRI_EL1_MASK | HFGxTR_EL2_nTPIDR2_EL0_MASK; + + r_clr |= tmp; + w_clr |= tmp; + } + + /* + * Trap guest writes to TCR_EL1 to prevent it from enabling HA or HD. + */ + if (cpus_have_final_cap(ARM64_WORKAROUND_AMPERE_AC03_CPU_38)) + w_set |= HFGxTR_EL2_TCR_EL1_MASK; + + sysreg_clear_set_s(SYS_HFGRTR_EL2, r_clr, r_set); + sysreg_clear_set_s(SYS_HFGWTR_EL2, w_clr, w_set); +} + +static inline void __deactivate_traps_hfgxtr(void) +{ + u64 r_clr = 0, w_clr = 0, r_set = 0, w_set = 0, tmp; + + if (cpus_have_final_cap(ARM64_SME)) { + tmp = HFGxTR_EL2_nSMPRI_EL1_MASK | HFGxTR_EL2_nTPIDR2_EL0_MASK; + + r_set |= tmp; + w_set |= tmp; + } + + if (cpus_have_final_cap(ARM64_WORKAROUND_AMPERE_AC03_CPU_38)) + w_clr |= HFGxTR_EL2_TCR_EL1_MASK; + + sysreg_clear_set_s(SYS_HFGRTR_EL2, r_clr, r_set); + sysreg_clear_set_s(SYS_HFGWTR_EL2, w_clr, w_set); +} + static inline void __activate_traps_common(struct kvm_vcpu *vcpu) { /* Trap on AArch32 cp15 c15 (impdef sysregs) accesses (EL1 or EL0) */ @@ -95,16 +145,8 @@ static inline void __activate_traps_common(struct kvm_vcpu *vcpu) vcpu->arch.mdcr_el2_host = read_sysreg(mdcr_el2); write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2); - if (cpus_have_final_cap(ARM64_SME)) { - sysreg_clear_set_s(SYS_HFGRTR_EL2, - HFGxTR_EL2_nSMPRI_EL1_MASK | - HFGxTR_EL2_nTPIDR2_EL0_MASK, - 0); - sysreg_clear_set_s(SYS_HFGWTR_EL2, - HFGxTR_EL2_nSMPRI_EL1_MASK | - HFGxTR_EL2_nTPIDR2_EL0_MASK, - 0); - } + if (__hfgxtr_traps_required()) + __activate_traps_hfgxtr(); } static inline void __deactivate_traps_common(struct kvm_vcpu *vcpu) @@ -120,14 +162,8 @@ static inline void __deactivate_traps_common(struct kvm_vcpu *vcpu) vcpu_clear_flag(vcpu, PMUSERENR_ON_CPU); } - if (cpus_have_final_cap(ARM64_SME)) { - sysreg_clear_set_s(SYS_HFGRTR_EL2, 0, - HFGxTR_EL2_nSMPRI_EL1_MASK | - HFGxTR_EL2_nTPIDR2_EL0_MASK); - sysreg_clear_set_s(SYS_HFGWTR_EL2, 0, - HFGxTR_EL2_nSMPRI_EL1_MASK | - HFGxTR_EL2_nTPIDR2_EL0_MASK); - } + if (__hfgxtr_traps_required()) + __deactivate_traps_hfgxtr(); } static inline void ___activate_traps(struct kvm_vcpu *vcpu) @@ -209,7 +245,7 @@ static bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) /* Valid trap. Switch the context: */ /* First disable enough traps to allow us to update the registers */ - if (has_vhe()) { + if (has_vhe() || has_hvhe()) { reg = CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN; if (sve_guest) reg |= CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN; @@ -401,12 +437,39 @@ static bool kvm_hyp_handle_cntpct(struct kvm_vcpu *vcpu) return true; } +static bool handle_ampere1_tcr(struct kvm_vcpu *vcpu) +{ + u32 sysreg = esr_sys64_to_sysreg(kvm_vcpu_get_esr(vcpu)); + int rt = kvm_vcpu_sys_get_rt(vcpu); + u64 val = vcpu_get_reg(vcpu, rt); + + if (sysreg != SYS_TCR_EL1) + return false; + + /* + * Affected parts do not advertise support for hardware Access Flag / + * Dirty state management in ID_AA64MMFR1_EL1.HAFDBS, but the underlying + * control bits are still functional. The architecture requires these be + * RES0 on systems that do not implement FEAT_HAFDBS. + * + * Uphold the requirements of the architecture by masking guest writes + * to TCR_EL1.{HA,HD} here. + */ + val &= ~(TCR_HD | TCR_HA); + write_sysreg_el1(val, SYS_TCR); + return true; +} + static bool kvm_hyp_handle_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code) { if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM) && handle_tx2_tvm(vcpu)) return true; + if (cpus_have_final_cap(ARM64_WORKAROUND_AMPERE_AC03_CPU_38) && + handle_ampere1_tcr(vcpu)) + return true; + if (static_branch_unlikely(&vgic_v3_cpuif_trap) && __vgic_v3_perform_cpuif_access(vcpu) == 1) return true; diff --git a/arch/arm64/kvm/hyp/include/nvhe/ffa.h b/arch/arm64/kvm/hyp/include/nvhe/ffa.h new file mode 100644 index 000000000000..1becb10ecd80 --- /dev/null +++ b/arch/arm64/kvm/hyp/include/nvhe/ffa.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2022 - Google LLC + * Author: Andrew Walbran <qwandor@google.com> + */ +#ifndef __KVM_HYP_FFA_H +#define __KVM_HYP_FFA_H + +#include <asm/kvm_host.h> + +#define FFA_MIN_FUNC_NUM 0x60 +#define FFA_MAX_FUNC_NUM 0x7F + +int hyp_ffa_init(void *pages); +bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt); + +#endif /* __KVM_HYP_FFA_H */ diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h index b7bdbe63deed..0972faccc2af 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -57,6 +57,7 @@ extern struct host_mmu host_mmu; enum pkvm_component_id { PKVM_ID_HOST, PKVM_ID_HYP, + PKVM_ID_FFA, }; extern unsigned long hyp_nr_cpus; @@ -66,6 +67,8 @@ int __pkvm_host_share_hyp(u64 pfn); int __pkvm_host_unshare_hyp(u64 pfn); int __pkvm_host_donate_hyp(u64 pfn, u64 nr_pages); int __pkvm_hyp_donate_host(u64 pfn, u64 nr_pages); +int __pkvm_host_share_ffa(u64 pfn, u64 nr_pages); +int __pkvm_host_unshare_ffa(u64 pfn, u64 nr_pages); bool addr_is_memory(phys_addr_t phys); int host_stage2_idmap_locked(phys_addr_t addr, u64 size, enum kvm_pgtable_prot prot); diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile index 530347cdebe3..9ddc025e4b86 100644 --- a/arch/arm64/kvm/hyp/nvhe/Makefile +++ b/arch/arm64/kvm/hyp/nvhe/Makefile @@ -22,7 +22,7 @@ lib-objs := $(addprefix ../../../lib/, $(lib-objs)) hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \ hyp-main.o hyp-smp.o psci-relay.o early_alloc.o page_alloc.o \ - cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o + cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ ../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o hyp-obj-$(CONFIG_DEBUG_LIST) += list_debug.o diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c new file mode 100644 index 000000000000..58dcd92bf346 --- /dev/null +++ b/arch/arm64/kvm/hyp/nvhe/ffa.c @@ -0,0 +1,762 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * FF-A v1.0 proxy to filter out invalid memory-sharing SMC calls issued by + * the host. FF-A is a slightly more palatable abbreviation of "Arm Firmware + * Framework for Arm A-profile", which is specified by Arm in document + * number DEN0077. + * + * Copyright (C) 2022 - Google LLC + * Author: Andrew Walbran <qwandor@google.com> + * + * This driver hooks into the SMC trapping logic for the host and intercepts + * all calls falling within the FF-A range. Each call is either: + * + * - Forwarded on unmodified to the SPMD at EL3 + * - Rejected as "unsupported" + * - Accompanied by a host stage-2 page-table check/update and reissued + * + * Consequently, any attempts by the host to make guest memory pages + * accessible to the secure world using FF-A will be detected either here + * (in the case that the memory is already owned by the guest) or during + * donation to the guest (in the case that the memory was previously shared + * with the secure world). + * + * To allow the rolling-back of page-table updates and FF-A calls in the + * event of failure, operations involving the RXTX buffers are locked for + * the duration and are therefore serialised. + */ + +#include <linux/arm-smccc.h> +#include <linux/arm_ffa.h> +#include <asm/kvm_pkvm.h> + +#include <nvhe/ffa.h> +#include <nvhe/mem_protect.h> +#include <nvhe/memory.h> +#include <nvhe/trap_handler.h> +#include <nvhe/spinlock.h> + +/* + * "ID value 0 must be returned at the Non-secure physical FF-A instance" + * We share this ID with the host. + */ +#define HOST_FFA_ID 0 + +/* + * A buffer to hold the maximum descriptor size we can see from the host, + * which is required when the SPMD returns a fragmented FFA_MEM_RETRIEVE_RESP + * when resolving the handle on the reclaim path. + */ +struct kvm_ffa_descriptor_buffer { + void *buf; + size_t len; +}; + +static struct kvm_ffa_descriptor_buffer ffa_desc_buf; + +struct kvm_ffa_buffers { + hyp_spinlock_t lock; + void *tx; + void *rx; +}; + +/* + * Note that we don't currently lock these buffers explicitly, instead + * relying on the locking of the host FFA buffers as we only have one + * client. + */ +static struct kvm_ffa_buffers hyp_buffers; +static struct kvm_ffa_buffers host_buffers; + +static void ffa_to_smccc_error(struct arm_smccc_res *res, u64 ffa_errno) +{ + *res = (struct arm_smccc_res) { + .a0 = FFA_ERROR, + .a2 = ffa_errno, + }; +} + +static void ffa_to_smccc_res_prop(struct arm_smccc_res *res, int ret, u64 prop) +{ + if (ret == FFA_RET_SUCCESS) { + *res = (struct arm_smccc_res) { .a0 = FFA_SUCCESS, + .a2 = prop }; + } else { + ffa_to_smccc_error(res, ret); + } +} + +static void ffa_to_smccc_res(struct arm_smccc_res *res, int ret) +{ + ffa_to_smccc_res_prop(res, ret, 0); +} + +static void ffa_set_retval(struct kvm_cpu_context *ctxt, + struct arm_smccc_res *res) +{ + cpu_reg(ctxt, 0) = res->a0; + cpu_reg(ctxt, 1) = res->a1; + cpu_reg(ctxt, 2) = res->a2; + cpu_reg(ctxt, 3) = res->a3; +} + +static bool is_ffa_call(u64 func_id) +{ + return ARM_SMCCC_IS_FAST_CALL(func_id) && + ARM_SMCCC_OWNER_NUM(func_id) == ARM_SMCCC_OWNER_STANDARD && + ARM_SMCCC_FUNC_NUM(func_id) >= FFA_MIN_FUNC_NUM && + ARM_SMCCC_FUNC_NUM(func_id) <= FFA_MAX_FUNC_NUM; +} + +static int ffa_map_hyp_buffers(u64 ffa_page_count) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(FFA_FN64_RXTX_MAP, + hyp_virt_to_phys(hyp_buffers.tx), + hyp_virt_to_phys(hyp_buffers.rx), + ffa_page_count, + 0, 0, 0, 0, + &res); + + return res.a0 == FFA_SUCCESS ? FFA_RET_SUCCESS : res.a2; +} + +static int ffa_unmap_hyp_buffers(void) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(FFA_RXTX_UNMAP, + HOST_FFA_ID, + 0, 0, 0, 0, 0, 0, + &res); + + return res.a0 == FFA_SUCCESS ? FFA_RET_SUCCESS : res.a2; +} + +static void ffa_mem_frag_tx(struct arm_smccc_res *res, u32 handle_lo, + u32 handle_hi, u32 fraglen, u32 endpoint_id) +{ + arm_smccc_1_1_smc(FFA_MEM_FRAG_TX, + handle_lo, handle_hi, fraglen, endpoint_id, + 0, 0, 0, + res); +} + +static void ffa_mem_frag_rx(struct arm_smccc_res *res, u32 handle_lo, + u32 handle_hi, u32 fragoff) +{ + arm_smccc_1_1_smc(FFA_MEM_FRAG_RX, + handle_lo, handle_hi, fragoff, HOST_FFA_ID, + 0, 0, 0, + res); +} + +static void ffa_mem_xfer(struct arm_smccc_res *res, u64 func_id, u32 len, + u32 fraglen) +{ + arm_smccc_1_1_smc(func_id, len, fraglen, + 0, 0, 0, 0, 0, + res); +} + +static void ffa_mem_reclaim(struct arm_smccc_res *res, u32 handle_lo, + u32 handle_hi, u32 flags) +{ + arm_smccc_1_1_smc(FFA_MEM_RECLAIM, + handle_lo, handle_hi, flags, + 0, 0, 0, 0, + res); +} + +static void ffa_retrieve_req(struct arm_smccc_res *res, u32 len) +{ + arm_smccc_1_1_smc(FFA_FN64_MEM_RETRIEVE_REQ, + len, len, + 0, 0, 0, 0, 0, + res); +} + +static void do_ffa_rxtx_map(struct arm_smccc_res *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(phys_addr_t, tx, ctxt, 1); + DECLARE_REG(phys_addr_t, rx, ctxt, 2); + DECLARE_REG(u32, npages, ctxt, 3); + int ret = 0; + void *rx_virt, *tx_virt; + + if (npages != (KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) / FFA_PAGE_SIZE) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out; + } + + if (!PAGE_ALIGNED(tx) || !PAGE_ALIGNED(rx)) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out; + } + + hyp_spin_lock(&host_buffers.lock); + if (host_buffers.tx) { + ret = FFA_RET_DENIED; + goto out_unlock; + } + + /* + * Map our hypervisor buffers into the SPMD before mapping and + * pinning the host buffers in our own address space. + */ + ret = ffa_map_hyp_buffers(npages); + if (ret) + goto out_unlock; + + ret = __pkvm_host_share_hyp(hyp_phys_to_pfn(tx)); + if (ret) { + ret = FFA_RET_INVALID_PARAMETERS; + goto err_unmap; + } + + ret = __pkvm_host_share_hyp(hyp_phys_to_pfn(rx)); + if (ret) { + ret = FFA_RET_INVALID_PARAMETERS; + goto err_unshare_tx; + } + + tx_virt = hyp_phys_to_virt(tx); + ret = hyp_pin_shared_mem(tx_virt, tx_virt + 1); + if (ret) { + ret = FFA_RET_INVALID_PARAMETERS; + goto err_unshare_rx; + } + + rx_virt = hyp_phys_to_virt(rx); + ret = hyp_pin_shared_mem(rx_virt, rx_virt + 1); + if (ret) { + ret = FFA_RET_INVALID_PARAMETERS; + goto err_unpin_tx; + } + + host_buffers.tx = tx_virt; + host_buffers.rx = rx_virt; + +out_unlock: + hyp_spin_unlock(&host_buffers.lock); +out: + ffa_to_smccc_res(res, ret); + return; + +err_unpin_tx: + hyp_unpin_shared_mem(tx_virt, tx_virt + 1); +err_unshare_rx: + __pkvm_host_unshare_hyp(hyp_phys_to_pfn(rx)); +err_unshare_tx: + __pkvm_host_unshare_hyp(hyp_phys_to_pfn(tx)); +err_unmap: + ffa_unmap_hyp_buffers(); + goto out_unlock; +} + +static void do_ffa_rxtx_unmap(struct arm_smccc_res *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, id, ctxt, 1); + int ret = 0; + + if (id != HOST_FFA_ID) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out; + } + + hyp_spin_lock(&host_buffers.lock); + if (!host_buffers.tx) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out_unlock; + } + + hyp_unpin_shared_mem(host_buffers.tx, host_buffers.tx + 1); + WARN_ON(__pkvm_host_unshare_hyp(hyp_virt_to_pfn(host_buffers.tx))); + host_buffers.tx = NULL; + + hyp_unpin_shared_mem(host_buffers.rx, host_buffers.rx + 1); + WARN_ON(__pkvm_host_unshare_hyp(hyp_virt_to_pfn(host_buffers.rx))); + host_buffers.rx = NULL; + + ffa_unmap_hyp_buffers(); + +out_unlock: + hyp_spin_unlock(&host_buffers.lock); +out: + ffa_to_smccc_res(res, ret); +} + +static u32 __ffa_host_share_ranges(struct ffa_mem_region_addr_range *ranges, + u32 nranges) +{ + u32 i; + + for (i = 0; i < nranges; ++i) { + struct ffa_mem_region_addr_range *range = &ranges[i]; + u64 sz = (u64)range->pg_cnt * FFA_PAGE_SIZE; + u64 pfn = hyp_phys_to_pfn(range->address); + + if (!PAGE_ALIGNED(sz)) + break; + + if (__pkvm_host_share_ffa(pfn, sz / PAGE_SIZE)) + break; + } + + return i; +} + +static u32 __ffa_host_unshare_ranges(struct ffa_mem_region_addr_range *ranges, + u32 nranges) +{ + u32 i; + + for (i = 0; i < nranges; ++i) { + struct ffa_mem_region_addr_range *range = &ranges[i]; + u64 sz = (u64)range->pg_cnt * FFA_PAGE_SIZE; + u64 pfn = hyp_phys_to_pfn(range->address); + + if (!PAGE_ALIGNED(sz)) + break; + + if (__pkvm_host_unshare_ffa(pfn, sz / PAGE_SIZE)) + break; + } + + return i; +} + +static int ffa_host_share_ranges(struct ffa_mem_region_addr_range *ranges, + u32 nranges) +{ + u32 nshared = __ffa_host_share_ranges(ranges, nranges); + int ret = 0; + + if (nshared != nranges) { + WARN_ON(__ffa_host_unshare_ranges(ranges, nshared) != nshared); + ret = FFA_RET_DENIED; + } + + return ret; +} + +static int ffa_host_unshare_ranges(struct ffa_mem_region_addr_range *ranges, + u32 nranges) +{ + u32 nunshared = __ffa_host_unshare_ranges(ranges, nranges); + int ret = 0; + + if (nunshared != nranges) { + WARN_ON(__ffa_host_share_ranges(ranges, nunshared) != nunshared); + ret = FFA_RET_DENIED; + } + + return ret; +} + +static void do_ffa_mem_frag_tx(struct arm_smccc_res *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, handle_lo, ctxt, 1); + DECLARE_REG(u32, handle_hi, ctxt, 2); + DECLARE_REG(u32, fraglen, ctxt, 3); + DECLARE_REG(u32, endpoint_id, ctxt, 4); + struct ffa_mem_region_addr_range *buf; + int ret = FFA_RET_INVALID_PARAMETERS; + u32 nr_ranges; + + if (fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) + goto out; + + if (fraglen % sizeof(*buf)) + goto out; + + hyp_spin_lock(&host_buffers.lock); + if (!host_buffers.tx) + goto out_unlock; + + buf = hyp_buffers.tx; + memcpy(buf, host_buffers.tx, fraglen); + nr_ranges = fraglen / sizeof(*buf); + + ret = ffa_host_share_ranges(buf, nr_ranges); + if (ret) { + /* + * We're effectively aborting the transaction, so we need + * to restore the global state back to what it was prior to + * transmission of the first fragment. + */ + ffa_mem_reclaim(res, handle_lo, handle_hi, 0); + WARN_ON(res->a0 != FFA_SUCCESS); + goto out_unlock; + } + + ffa_mem_frag_tx(res, handle_lo, handle_hi, fraglen, endpoint_id); + if (res->a0 != FFA_SUCCESS && res->a0 != FFA_MEM_FRAG_RX) + WARN_ON(ffa_host_unshare_ranges(buf, nr_ranges)); + +out_unlock: + hyp_spin_unlock(&host_buffers.lock); +out: + if (ret) + ffa_to_smccc_res(res, ret); + + /* + * If for any reason this did not succeed, we're in trouble as we have + * now lost the content of the previous fragments and we can't rollback + * the host stage-2 changes. The pages previously marked as shared will + * remain stuck in that state forever, hence preventing the host from + * sharing/donating them again and may possibly lead to subsequent + * failures, but this will not compromise confidentiality. + */ + return; +} + +static __always_inline void do_ffa_mem_xfer(const u64 func_id, + struct arm_smccc_res *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, len, ctxt, 1); + DECLARE_REG(u32, fraglen, ctxt, 2); + DECLARE_REG(u64, addr_mbz, ctxt, 3); + DECLARE_REG(u32, npages_mbz, ctxt, 4); + struct ffa_composite_mem_region *reg; + struct ffa_mem_region *buf; + u32 offset, nr_ranges; + int ret = 0; + + BUILD_BUG_ON(func_id != FFA_FN64_MEM_SHARE && + func_id != FFA_FN64_MEM_LEND); + + if (addr_mbz || npages_mbz || fraglen > len || + fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out; + } + + if (fraglen < sizeof(struct ffa_mem_region) + + sizeof(struct ffa_mem_region_attributes)) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out; + } + + hyp_spin_lock(&host_buffers.lock); + if (!host_buffers.tx) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out_unlock; + } + + buf = hyp_buffers.tx; + memcpy(buf, host_buffers.tx, fraglen); + + offset = buf->ep_mem_access[0].composite_off; + if (!offset || buf->ep_count != 1 || buf->sender_id != HOST_FFA_ID) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out_unlock; + } + + if (fraglen < offset + sizeof(struct ffa_composite_mem_region)) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out_unlock; + } + + reg = (void *)buf + offset; + nr_ranges = ((void *)buf + fraglen) - (void *)reg->constituents; + if (nr_ranges % sizeof(reg->constituents[0])) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out_unlock; + } + + nr_ranges /= sizeof(reg->constituents[0]); + ret = ffa_host_share_ranges(reg->constituents, nr_ranges); + if (ret) + goto out_unlock; + + ffa_mem_xfer(res, func_id, len, fraglen); + if (fraglen != len) { + if (res->a0 != FFA_MEM_FRAG_RX) + goto err_unshare; + + if (res->a3 != fraglen) + goto err_unshare; + } else if (res->a0 != FFA_SUCCESS) { + goto err_unshare; + } + +out_unlock: + hyp_spin_unlock(&host_buffers.lock); +out: + if (ret) + ffa_to_smccc_res(res, ret); + return; + +err_unshare: + WARN_ON(ffa_host_unshare_ranges(reg->constituents, nr_ranges)); + goto out_unlock; +} + +static void do_ffa_mem_reclaim(struct arm_smccc_res *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, handle_lo, ctxt, 1); + DECLARE_REG(u32, handle_hi, ctxt, 2); + DECLARE_REG(u32, flags, ctxt, 3); + struct ffa_composite_mem_region *reg; + u32 offset, len, fraglen, fragoff; + struct ffa_mem_region *buf; + int ret = 0; + u64 handle; + + handle = PACK_HANDLE(handle_lo, handle_hi); + + hyp_spin_lock(&host_buffers.lock); + + buf = hyp_buffers.tx; + *buf = (struct ffa_mem_region) { + .sender_id = HOST_FFA_ID, + .handle = handle, + }; + + ffa_retrieve_req(res, sizeof(*buf)); + buf = hyp_buffers.rx; + if (res->a0 != FFA_MEM_RETRIEVE_RESP) + goto out_unlock; + + len = res->a1; + fraglen = res->a2; + + offset = buf->ep_mem_access[0].composite_off; + /* + * We can trust the SPMD to get this right, but let's at least + * check that we end up with something that doesn't look _completely_ + * bogus. + */ + if (WARN_ON(offset > len || + fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE)) { + ret = FFA_RET_ABORTED; + goto out_unlock; + } + + if (len > ffa_desc_buf.len) { + ret = FFA_RET_NO_MEMORY; + goto out_unlock; + } + + buf = ffa_desc_buf.buf; + memcpy(buf, hyp_buffers.rx, fraglen); + + for (fragoff = fraglen; fragoff < len; fragoff += fraglen) { + ffa_mem_frag_rx(res, handle_lo, handle_hi, fragoff); + if (res->a0 != FFA_MEM_FRAG_TX) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out_unlock; + } + + fraglen = res->a3; + memcpy((void *)buf + fragoff, hyp_buffers.rx, fraglen); + } + + ffa_mem_reclaim(res, handle_lo, handle_hi, flags); + if (res->a0 != FFA_SUCCESS) + goto out_unlock; + + reg = (void *)buf + offset; + /* If the SPMD was happy, then we should be too. */ + WARN_ON(ffa_host_unshare_ranges(reg->constituents, + reg->addr_range_cnt)); +out_unlock: + hyp_spin_unlock(&host_buffers.lock); + + if (ret) + ffa_to_smccc_res(res, ret); +} + +/* + * Is a given FFA function supported, either by forwarding on directly + * or by handling at EL2? + */ +static bool ffa_call_supported(u64 func_id) +{ + switch (func_id) { + /* Unsupported memory management calls */ + case FFA_FN64_MEM_RETRIEVE_REQ: + case FFA_MEM_RETRIEVE_RESP: + case FFA_MEM_RELINQUISH: + case FFA_MEM_OP_PAUSE: + case FFA_MEM_OP_RESUME: + case FFA_MEM_FRAG_RX: + case FFA_FN64_MEM_DONATE: + /* Indirect message passing via RX/TX buffers */ + case FFA_MSG_SEND: + case FFA_MSG_POLL: + case FFA_MSG_WAIT: + /* 32-bit variants of 64-bit calls */ + case FFA_MSG_SEND_DIRECT_REQ: + case FFA_MSG_SEND_DIRECT_RESP: + case FFA_RXTX_MAP: + case FFA_MEM_DONATE: + case FFA_MEM_RETRIEVE_REQ: + return false; + } + + return true; +} + +static bool do_ffa_features(struct arm_smccc_res *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, id, ctxt, 1); + u64 prop = 0; + int ret = 0; + + if (!ffa_call_supported(id)) { + ret = FFA_RET_NOT_SUPPORTED; + goto out_handled; + } + + switch (id) { + case FFA_MEM_SHARE: + case FFA_FN64_MEM_SHARE: + case FFA_MEM_LEND: + case FFA_FN64_MEM_LEND: + ret = FFA_RET_SUCCESS; + prop = 0; /* No support for dynamic buffers */ + goto out_handled; + default: + return false; + } + +out_handled: + ffa_to_smccc_res_prop(res, ret, prop); + return true; +} + +bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(u64, func_id, host_ctxt, 0); + struct arm_smccc_res res; + + /* + * There's no way we can tell what a non-standard SMC call might + * be up to. Ideally, we would terminate these here and return + * an error to the host, but sadly devices make use of custom + * firmware calls for things like power management, debugging, + * RNG access and crash reporting. + * + * Given that the architecture requires us to trust EL3 anyway, + * we forward unrecognised calls on under the assumption that + * the firmware doesn't expose a mechanism to access arbitrary + * non-secure memory. Short of a per-device table of SMCs, this + * is the best we can do. + */ + if (!is_ffa_call(func_id)) + return false; + + switch (func_id) { + case FFA_FEATURES: + if (!do_ffa_features(&res, host_ctxt)) + return false; + goto out_handled; + /* Memory management */ + case FFA_FN64_RXTX_MAP: + do_ffa_rxtx_map(&res, host_ctxt); + goto out_handled; + case FFA_RXTX_UNMAP: + do_ffa_rxtx_unmap(&res, host_ctxt); + goto out_handled; + case FFA_MEM_SHARE: + case FFA_FN64_MEM_SHARE: + do_ffa_mem_xfer(FFA_FN64_MEM_SHARE, &res, host_ctxt); + goto out_handled; + case FFA_MEM_RECLAIM: + do_ffa_mem_reclaim(&res, host_ctxt); + goto out_handled; + case FFA_MEM_LEND: + case FFA_FN64_MEM_LEND: + do_ffa_mem_xfer(FFA_FN64_MEM_LEND, &res, host_ctxt); + goto out_handled; + case FFA_MEM_FRAG_TX: + do_ffa_mem_frag_tx(&res, host_ctxt); + goto out_handled; + } + + if (ffa_call_supported(func_id)) + return false; /* Pass through */ + + ffa_to_smccc_error(&res, FFA_RET_NOT_SUPPORTED); +out_handled: + ffa_set_retval(host_ctxt, &res); + return true; +} + +int hyp_ffa_init(void *pages) +{ + struct arm_smccc_res res; + size_t min_rxtx_sz; + void *tx, *rx; + + if (kvm_host_psci_config.smccc_version < ARM_SMCCC_VERSION_1_2) + return 0; + + arm_smccc_1_1_smc(FFA_VERSION, FFA_VERSION_1_0, 0, 0, 0, 0, 0, 0, &res); + if (res.a0 == FFA_RET_NOT_SUPPORTED) + return 0; + + if (res.a0 != FFA_VERSION_1_0) + return -EOPNOTSUPP; + + arm_smccc_1_1_smc(FFA_ID_GET, 0, 0, 0, 0, 0, 0, 0, &res); + if (res.a0 != FFA_SUCCESS) + return -EOPNOTSUPP; + + if (res.a2 != HOST_FFA_ID) + return -EINVAL; + + arm_smccc_1_1_smc(FFA_FEATURES, FFA_FN64_RXTX_MAP, + 0, 0, 0, 0, 0, 0, &res); + if (res.a0 != FFA_SUCCESS) + return -EOPNOTSUPP; + + switch (res.a2) { + case FFA_FEAT_RXTX_MIN_SZ_4K: + min_rxtx_sz = SZ_4K; + break; + case FFA_FEAT_RXTX_MIN_SZ_16K: + min_rxtx_sz = SZ_16K; + break; + case FFA_FEAT_RXTX_MIN_SZ_64K: + min_rxtx_sz = SZ_64K; + break; + default: + return -EINVAL; + } + + if (min_rxtx_sz > PAGE_SIZE) + return -EOPNOTSUPP; + + tx = pages; + pages += KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE; + rx = pages; + pages += KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE; + + ffa_desc_buf = (struct kvm_ffa_descriptor_buffer) { + .buf = pages, + .len = PAGE_SIZE * + (hyp_ffa_proxy_pages() - (2 * KVM_FFA_MBOX_NR_PAGES)), + }; + + hyp_buffers = (struct kvm_ffa_buffers) { + .lock = __HYP_SPIN_LOCK_UNLOCKED, + .tx = tx, + .rx = rx, + }; + + host_buffers = (struct kvm_ffa_buffers) { + .lock = __HYP_SPIN_LOCK_UNLOCKED, + }; + + return 0; +} diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S index b6c0188c4b35..c87c63133e10 100644 --- a/arch/arm64/kvm/hyp/nvhe/host.S +++ b/arch/arm64/kvm/hyp/nvhe/host.S @@ -10,6 +10,7 @@ #include <asm/kvm_arm.h> #include <asm/kvm_asm.h> #include <asm/kvm_mmu.h> +#include <asm/kvm_ptrauth.h> .text @@ -37,10 +38,43 @@ SYM_FUNC_START(__host_exit) /* Save the host context pointer in x29 across the function call */ mov x29, x0 + +#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL +alternative_if_not ARM64_HAS_ADDRESS_AUTH +b __skip_pauth_save +alternative_else_nop_endif + +alternative_if ARM64_KVM_PROTECTED_MODE + /* Save kernel ptrauth keys. */ + add x18, x29, #CPU_APIAKEYLO_EL1 + ptrauth_save_state x18, x19, x20 + + /* Use hyp keys. */ + adr_this_cpu x18, kvm_hyp_ctxt, x19 + add x18, x18, #CPU_APIAKEYLO_EL1 + ptrauth_restore_state x18, x19, x20 + isb +alternative_else_nop_endif +__skip_pauth_save: +#endif /* CONFIG_ARM64_PTR_AUTH_KERNEL */ + bl handle_trap - /* Restore host regs x0-x17 */ __host_enter_restore_full: + /* Restore kernel keys. */ +#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL +alternative_if_not ARM64_HAS_ADDRESS_AUTH +b __skip_pauth_restore +alternative_else_nop_endif + +alternative_if ARM64_KVM_PROTECTED_MODE + add x18, x29, #CPU_APIAKEYLO_EL1 + ptrauth_restore_state x18, x19, x20 +alternative_else_nop_endif +__skip_pauth_restore: +#endif /* CONFIG_ARM64_PTR_AUTH_KERNEL */ + + /* Restore host regs x0-x17 */ ldp x0, x1, [x29, #CPU_XREG_OFFSET(0)] ldp x2, x3, [x29, #CPU_XREG_OFFSET(2)] ldp x4, x5, [x29, #CPU_XREG_OFFSET(4)] diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index a6d67c2bb5ae..90fade1b032e 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -83,9 +83,6 @@ SYM_CODE_END(__kvm_hyp_init) * x0: struct kvm_nvhe_init_params PA */ SYM_CODE_START_LOCAL(___kvm_hyp_init) - ldr x1, [x0, #NVHE_INIT_TPIDR_EL2] - msr tpidr_el2, x1 - ldr x1, [x0, #NVHE_INIT_STACK_HYP_VA] mov sp, x1 @@ -95,6 +92,22 @@ SYM_CODE_START_LOCAL(___kvm_hyp_init) ldr x1, [x0, #NVHE_INIT_HCR_EL2] msr hcr_el2, x1 + mov x2, #HCR_E2H + and x2, x1, x2 + cbz x2, 1f + + // hVHE: Replay the EL2 setup to account for the E2H bit + // TPIDR_EL2 is used to preserve x0 across the macro maze... + isb + msr tpidr_el2, x0 + init_el2_state + finalise_el2_state + mrs x0, tpidr_el2 + +1: + ldr x1, [x0, #NVHE_INIT_TPIDR_EL2] + msr tpidr_el2, x1 + ldr x1, [x0, #NVHE_INIT_VTTBR] msr vttbr_el2, x1 @@ -128,6 +141,13 @@ alternative_if ARM64_HAS_ADDRESS_AUTH SCTLR_ELx_ENDA | SCTLR_ELx_ENDB) orr x0, x0, x1 alternative_else_nop_endif + +#ifdef CONFIG_ARM64_BTI_KERNEL +alternative_if ARM64_BTI + orr x0, x0, #SCTLR_EL2_BT +alternative_else_nop_endif +#endif /* CONFIG_ARM64_BTI_KERNEL */ + msr sctlr_el2, x0 isb @@ -184,6 +204,7 @@ SYM_CODE_START_LOCAL(__kvm_hyp_init_cpu) /* Initialize EL2 CPU state to sane values. */ init_el2_state // Clobbers x0..x2 finalise_el2_state + __init_el2_nvhe_prepare_eret /* Enable MMU, set vectors and stack. */ mov x0, x28 @@ -196,6 +217,11 @@ SYM_CODE_START_LOCAL(__kvm_hyp_init_cpu) SYM_CODE_END(__kvm_hyp_init_cpu) SYM_CODE_START(__kvm_handle_stub_hvc) + /* + * __kvm_handle_stub_hvc called from __host_hvc through branch instruction(br) so + * we need bti j at beginning. + */ + bti j cmp x0, #HVC_SOFT_RESTART b.ne 1f diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 728e01d4536b..a169c619db60 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -13,6 +13,7 @@ #include <asm/kvm_hyp.h> #include <asm/kvm_mmu.h> +#include <nvhe/ffa.h> #include <nvhe/mem_protect.h> #include <nvhe/mm.h> #include <nvhe/pkvm.h> @@ -125,6 +126,15 @@ static void handle___kvm_tlb_flush_vmid_ipa(struct kvm_cpu_context *host_ctxt) __kvm_tlb_flush_vmid_ipa(kern_hyp_va(mmu), ipa, level); } +static void handle___kvm_tlb_flush_vmid_ipa_nsh(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); + DECLARE_REG(phys_addr_t, ipa, host_ctxt, 2); + DECLARE_REG(int, level, host_ctxt, 3); + + __kvm_tlb_flush_vmid_ipa_nsh(kern_hyp_va(mmu), ipa, level); +} + static void handle___kvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); @@ -315,6 +325,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__kvm_vcpu_run), HANDLE_FUNC(__kvm_flush_vm_context), HANDLE_FUNC(__kvm_tlb_flush_vmid_ipa), + HANDLE_FUNC(__kvm_tlb_flush_vmid_ipa_nsh), HANDLE_FUNC(__kvm_tlb_flush_vmid), HANDLE_FUNC(__kvm_flush_cpu_context), HANDLE_FUNC(__kvm_timer_set_cntvoff), @@ -374,6 +385,8 @@ static void handle_host_smc(struct kvm_cpu_context *host_ctxt) handled = kvm_host_psci_handler(host_ctxt); if (!handled) + handled = kvm_host_ffa_handler(host_ctxt); + if (!handled) default_host_smc_handler(host_ctxt); /* SMC was trapped, move ELR past the current PC. */ @@ -392,7 +405,11 @@ void handle_trap(struct kvm_cpu_context *host_ctxt) handle_host_smc(host_ctxt); break; case ESR_ELx_EC_SVE: - sysreg_clear_set(cptr_el2, CPTR_EL2_TZ, 0); + if (has_hvhe()) + sysreg_clear_set(cpacr_el1, 0, (CPACR_EL1_ZEN_EL1EN | + CPACR_EL1_ZEN_EL0EN)); + else + sysreg_clear_set(cptr_el2, CPTR_EL2_TZ, 0); isb(); sve_cond_update_zcr_vq(ZCR_ELx_LEN_MASK, SYS_ZCR_EL2); break; diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index a8813b212996..9d703441278b 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -91,9 +91,9 @@ static void host_s2_put_page(void *addr) hyp_put_page(&host_s2_pool, addr); } -static void host_s2_free_removed_table(void *addr, u32 level) +static void host_s2_free_unlinked_table(void *addr, u32 level) { - kvm_pgtable_stage2_free_removed(&host_mmu.mm_ops, addr, level); + kvm_pgtable_stage2_free_unlinked(&host_mmu.mm_ops, addr, level); } static int prepare_s2_pool(void *pgt_pool_base) @@ -110,7 +110,7 @@ static int prepare_s2_pool(void *pgt_pool_base) host_mmu.mm_ops = (struct kvm_pgtable_mm_ops) { .zalloc_pages_exact = host_s2_zalloc_pages_exact, .zalloc_page = host_s2_zalloc_page, - .free_removed_table = host_s2_free_removed_table, + .free_unlinked_table = host_s2_free_unlinked_table, .phys_to_virt = hyp_phys_to_virt, .virt_to_phys = hyp_virt_to_phys, .page_count = hyp_page_count, @@ -842,6 +842,13 @@ static int check_share(struct pkvm_mem_share *share) case PKVM_ID_HYP: ret = hyp_ack_share(completer_addr, tx, share->completer_prot); break; + case PKVM_ID_FFA: + /* + * We only check the host; the secure side will check the other + * end when we forward the FFA call. + */ + ret = 0; + break; default: ret = -EINVAL; } @@ -870,6 +877,13 @@ static int __do_share(struct pkvm_mem_share *share) case PKVM_ID_HYP: ret = hyp_complete_share(completer_addr, tx, share->completer_prot); break; + case PKVM_ID_FFA: + /* + * We're not responsible for any secure page-tables, so there's + * nothing to do here. + */ + ret = 0; + break; default: ret = -EINVAL; } @@ -918,6 +932,10 @@ static int check_unshare(struct pkvm_mem_share *share) case PKVM_ID_HYP: ret = hyp_ack_unshare(completer_addr, tx); break; + case PKVM_ID_FFA: + /* See check_share() */ + ret = 0; + break; default: ret = -EINVAL; } @@ -946,6 +964,10 @@ static int __do_unshare(struct pkvm_mem_share *share) case PKVM_ID_HYP: ret = hyp_complete_unshare(completer_addr, tx); break; + case PKVM_ID_FFA: + /* See __do_share() */ + ret = 0; + break; default: ret = -EINVAL; } @@ -1235,3 +1257,49 @@ void hyp_unpin_shared_mem(void *from, void *to) hyp_unlock_component(); host_unlock_component(); } + +int __pkvm_host_share_ffa(u64 pfn, u64 nr_pages) +{ + int ret; + struct pkvm_mem_share share = { + .tx = { + .nr_pages = nr_pages, + .initiator = { + .id = PKVM_ID_HOST, + .addr = hyp_pfn_to_phys(pfn), + }, + .completer = { + .id = PKVM_ID_FFA, + }, + }, + }; + + host_lock_component(); + ret = do_share(&share); + host_unlock_component(); + + return ret; +} + +int __pkvm_host_unshare_ffa(u64 pfn, u64 nr_pages) +{ + int ret; + struct pkvm_mem_share share = { + .tx = { + .nr_pages = nr_pages, + .initiator = { + .id = PKVM_ID_HOST, + .addr = hyp_pfn_to_phys(pfn), + }, + .completer = { + .id = PKVM_ID_FFA, + }, + }, + }; + + host_lock_component(); + ret = do_unshare(&share); + host_unlock_component(); + + return ret; +} diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index a06ece14a6d8..8033ef353a5d 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -27,6 +27,7 @@ static void pvm_init_traps_aa64pfr0(struct kvm_vcpu *vcpu) u64 hcr_set = HCR_RW; u64 hcr_clear = 0; u64 cptr_set = 0; + u64 cptr_clear = 0; /* Protected KVM does not support AArch32 guests. */ BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL0), @@ -43,6 +44,9 @@ static void pvm_init_traps_aa64pfr0(struct kvm_vcpu *vcpu) BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AdvSIMD), PVM_ID_AA64PFR0_ALLOW)); + if (has_hvhe()) + hcr_set |= HCR_E2H; + /* Trap RAS unless all current versions are supported */ if (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_RAS), feature_ids) < ID_AA64PFR0_EL1_RAS_V1P1) { @@ -57,12 +61,17 @@ static void pvm_init_traps_aa64pfr0(struct kvm_vcpu *vcpu) } /* Trap SVE */ - if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_SVE), feature_ids)) - cptr_set |= CPTR_EL2_TZ; + if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_SVE), feature_ids)) { + if (has_hvhe()) + cptr_clear |= CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN; + else + cptr_set |= CPTR_EL2_TZ; + } vcpu->arch.hcr_el2 |= hcr_set; vcpu->arch.hcr_el2 &= ~hcr_clear; vcpu->arch.cptr_el2 |= cptr_set; + vcpu->arch.cptr_el2 &= ~cptr_clear; } /* @@ -120,8 +129,12 @@ static void pvm_init_traps_aa64dfr0(struct kvm_vcpu *vcpu) mdcr_set |= MDCR_EL2_TTRF; /* Trap Trace */ - if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_TraceVer), feature_ids)) - cptr_set |= CPTR_EL2_TTA; + if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_TraceVer), feature_ids)) { + if (has_hvhe()) + cptr_set |= CPACR_EL1_TTA; + else + cptr_set |= CPTR_EL2_TTA; + } vcpu->arch.mdcr_el2 |= mdcr_set; vcpu->arch.mdcr_el2 &= ~mdcr_clear; @@ -176,8 +189,10 @@ static void pvm_init_trap_regs(struct kvm_vcpu *vcpu) /* Clear res0 and set res1 bits to trap potential new features. */ vcpu->arch.hcr_el2 &= ~(HCR_RES0); vcpu->arch.mdcr_el2 &= ~(MDCR_EL2_RES0); - vcpu->arch.cptr_el2 |= CPTR_NVHE_EL2_RES1; - vcpu->arch.cptr_el2 &= ~(CPTR_NVHE_EL2_RES0); + if (!has_hvhe()) { + vcpu->arch.cptr_el2 |= CPTR_NVHE_EL2_RES1; + vcpu->arch.cptr_el2 &= ~(CPTR_NVHE_EL2_RES0); + } } /* diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index 110f04627785..bb98630dfeaf 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -11,6 +11,7 @@ #include <asm/kvm_pkvm.h> #include <nvhe/early_alloc.h> +#include <nvhe/ffa.h> #include <nvhe/fixed_config.h> #include <nvhe/gfp.h> #include <nvhe/memory.h> @@ -28,6 +29,7 @@ static void *vmemmap_base; static void *vm_table_base; static void *hyp_pgt_base; static void *host_s2_pgt_base; +static void *ffa_proxy_pages; static struct kvm_pgtable_mm_ops pkvm_pgtable_mm_ops; static struct hyp_pool hpool; @@ -57,6 +59,11 @@ static int divide_memory_pool(void *virt, unsigned long size) if (!host_s2_pgt_base) return -ENOMEM; + nr_pages = hyp_ffa_proxy_pages(); + ffa_proxy_pages = hyp_early_alloc_contig(nr_pages); + if (!ffa_proxy_pages) + return -ENOMEM; + return 0; } @@ -314,6 +321,10 @@ void __noreturn __pkvm_init_finalise(void) if (ret) goto out; + ret = hyp_ffa_init(ffa_proxy_pages); + if (ret) + goto out; + pkvm_hyp_vm_table_init(vm_table_base); out: /* diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 77791495c995..0a6271052def 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -44,13 +44,24 @@ static void __activate_traps(struct kvm_vcpu *vcpu) __activate_traps_common(vcpu); val = vcpu->arch.cptr_el2; - val |= CPTR_EL2_TTA | CPTR_EL2_TAM; + val |= CPTR_EL2_TAM; /* Same bit irrespective of E2H */ + val |= has_hvhe() ? CPACR_EL1_TTA : CPTR_EL2_TTA; + if (cpus_have_final_cap(ARM64_SME)) { + if (has_hvhe()) + val &= ~(CPACR_EL1_SMEN_EL1EN | CPACR_EL1_SMEN_EL0EN); + else + val |= CPTR_EL2_TSM; + } + if (!guest_owns_fp_regs(vcpu)) { - val |= CPTR_EL2_TFP | CPTR_EL2_TZ; + if (has_hvhe()) + val &= ~(CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN | + CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN); + else + val |= CPTR_EL2_TFP | CPTR_EL2_TZ; + __activate_traps_fpsimd32(vcpu); } - if (cpus_have_final_cap(ARM64_SME)) - val |= CPTR_EL2_TSM; write_sysreg(val, cptr_el2); write_sysreg(__this_cpu_read(kvm_hyp_vector), vbar_el2); @@ -73,7 +84,6 @@ static void __activate_traps(struct kvm_vcpu *vcpu) static void __deactivate_traps(struct kvm_vcpu *vcpu) { extern char __kvm_hyp_host_vector[]; - u64 cptr; ___deactivate_traps(vcpu); @@ -98,13 +108,7 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu) write_sysreg(this_cpu_ptr(&kvm_init_params)->hcr_el2, hcr_el2); - cptr = CPTR_EL2_DEFAULT; - if (vcpu_has_sve(vcpu) && (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED)) - cptr |= CPTR_EL2_TZ; - if (cpus_have_final_cap(ARM64_SME)) - cptr &= ~CPTR_EL2_TSM; - - write_sysreg(cptr, cptr_el2); + kvm_reset_cptr_el2(vcpu); write_sysreg(__kvm_hyp_host_vector, vbar_el2); } diff --git a/arch/arm64/kvm/hyp/nvhe/timer-sr.c b/arch/arm64/kvm/hyp/nvhe/timer-sr.c index b185ac0dbd47..3aaab20ae5b4 100644 --- a/arch/arm64/kvm/hyp/nvhe/timer-sr.c +++ b/arch/arm64/kvm/hyp/nvhe/timer-sr.c @@ -17,21 +17,24 @@ void __kvm_timer_set_cntvoff(u64 cntvoff) } /* - * Should only be called on non-VHE systems. + * Should only be called on non-VHE or hVHE setups. * VHE systems use EL2 timers and configure EL1 timers in kvm_timer_init_vhe(). */ void __timer_disable_traps(struct kvm_vcpu *vcpu) { - u64 val; + u64 val, shift = 0; + + if (has_hvhe()) + shift = 10; /* Allow physical timer/counter access for the host */ val = read_sysreg(cnthctl_el2); - val |= CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN; + val |= (CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN) << shift; write_sysreg(val, cnthctl_el2); } /* - * Should only be called on non-VHE systems. + * Should only be called on non-VHE or hVHE setups. * VHE systems use EL2 timers and configure EL1 timers in kvm_timer_init_vhe(). */ void __timer_enable_traps(struct kvm_vcpu *vcpu) @@ -50,5 +53,10 @@ void __timer_enable_traps(struct kvm_vcpu *vcpu) else clr |= CNTHCTL_EL1PCTEN; + if (has_hvhe()) { + clr <<= 10; + set <<= 10; + } + sysreg_clear_set(cnthctl_el2, clr, set); } diff --git a/arch/arm64/kvm/hyp/nvhe/tlb.c b/arch/arm64/kvm/hyp/nvhe/tlb.c index 978179133f4b..b9991bbd8e3f 100644 --- a/arch/arm64/kvm/hyp/nvhe/tlb.c +++ b/arch/arm64/kvm/hyp/nvhe/tlb.c @@ -130,6 +130,58 @@ void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, __tlb_switch_to_host(&cxt); } +void __kvm_tlb_flush_vmid_ipa_nsh(struct kvm_s2_mmu *mmu, + phys_addr_t ipa, int level) +{ + struct tlb_inv_context cxt; + + /* Switch to requested VMID */ + __tlb_switch_to_guest(mmu, &cxt, true); + + /* + * We could do so much better if we had the VA as well. + * Instead, we invalidate Stage-2 for this IPA, and the + * whole of Stage-1. Weep... + */ + ipa >>= 12; + __tlbi_level(ipas2e1, ipa, level); + + /* + * We have to ensure completion of the invalidation at Stage-2, + * since a table walk on another CPU could refill a TLB with a + * complete (S1 + S2) walk based on the old Stage-2 mapping if + * the Stage-1 invalidation happened first. + */ + dsb(nsh); + __tlbi(vmalle1); + dsb(nsh); + isb(); + + /* + * If the host is running at EL1 and we have a VPIPT I-cache, + * then we must perform I-cache maintenance at EL2 in order for + * it to have an effect on the guest. Since the guest cannot hit + * I-cache lines allocated with a different VMID, we don't need + * to worry about junk out of guest reset (we nuke the I-cache on + * VMID rollover), but we do need to be careful when remapping + * executable pages for the same guest. This can happen when KSM + * takes a CoW fault on an executable page, copies the page into + * a page that was previously mapped in the guest and then needs + * to invalidate the guest view of the I-cache for that page + * from EL1. To solve this, we invalidate the entire I-cache when + * unmapping a page from a guest if we have a VPIPT I-cache but + * the host is running at EL1. As above, we could do better if + * we had the VA. + * + * The moral of this story is: if you have a VPIPT I-cache, then + * you should be running with VHE enabled. + */ + if (icache_is_vpipt()) + icache_inval_all_pou(); + + __tlb_switch_to_host(&cxt); +} + void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu) { struct tlb_inv_context cxt; diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 95dae02ccc2e..aa740a974e02 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -21,8 +21,10 @@ #define KVM_PTE_LEAF_ATTR_LO_S1_ATTRIDX GENMASK(4, 2) #define KVM_PTE_LEAF_ATTR_LO_S1_AP GENMASK(7, 6) -#define KVM_PTE_LEAF_ATTR_LO_S1_AP_RO 3 -#define KVM_PTE_LEAF_ATTR_LO_S1_AP_RW 1 +#define KVM_PTE_LEAF_ATTR_LO_S1_AP_RO \ + ({ cpus_have_final_cap(ARM64_KVM_HVHE) ? 2 : 3; }) +#define KVM_PTE_LEAF_ATTR_LO_S1_AP_RW \ + ({ cpus_have_final_cap(ARM64_KVM_HVHE) ? 0 : 1; }) #define KVM_PTE_LEAF_ATTR_LO_S1_SH GENMASK(9, 8) #define KVM_PTE_LEAF_ATTR_LO_S1_SH_IS 3 #define KVM_PTE_LEAF_ATTR_LO_S1_AF BIT(10) @@ -34,7 +36,7 @@ #define KVM_PTE_LEAF_ATTR_LO_S2_SH_IS 3 #define KVM_PTE_LEAF_ATTR_LO_S2_AF BIT(10) -#define KVM_PTE_LEAF_ATTR_HI GENMASK(63, 51) +#define KVM_PTE_LEAF_ATTR_HI GENMASK(63, 50) #define KVM_PTE_LEAF_ATTR_HI_SW GENMASK(58, 55) @@ -42,6 +44,8 @@ #define KVM_PTE_LEAF_ATTR_HI_S2_XN BIT(54) +#define KVM_PTE_LEAF_ATTR_HI_S1_GP BIT(50) + #define KVM_PTE_LEAF_ATTR_S2_PERMS (KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R | \ KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W | \ KVM_PTE_LEAF_ATTR_HI_S2_XN) @@ -63,6 +67,16 @@ struct kvm_pgtable_walk_data { const u64 end; }; +static bool kvm_pgtable_walk_skip_bbm_tlbi(const struct kvm_pgtable_visit_ctx *ctx) +{ + return unlikely(ctx->flags & KVM_PGTABLE_WALK_SKIP_BBM_TLBI); +} + +static bool kvm_pgtable_walk_skip_cmo(const struct kvm_pgtable_visit_ctx *ctx) +{ + return unlikely(ctx->flags & KVM_PGTABLE_WALK_SKIP_CMO); +} + static bool kvm_phys_is_valid(u64 phys) { return phys < BIT(id_aa64mmfr0_parange_to_phys_shift(ID_AA64MMFR0_EL1_PARANGE_MAX)); @@ -386,6 +400,9 @@ static int hyp_set_prot_attr(enum kvm_pgtable_prot prot, kvm_pte_t *ptep) if (device) return -EINVAL; + + if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) && system_supports_bti()) + attr |= KVM_PTE_LEAF_ATTR_HI_S1_GP; } else { attr |= KVM_PTE_LEAF_ATTR_HI_S1_XN; } @@ -623,10 +640,18 @@ u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift) #ifdef CONFIG_ARM64_HW_AFDBM /* * Enable the Hardware Access Flag management, unconditionally - * on all CPUs. The features is RES0 on CPUs without the support - * and must be ignored by the CPUs. + * on all CPUs. In systems that have asymmetric support for the feature + * this allows KVM to leverage hardware support on the subset of cores + * that implement the feature. + * + * The architecture requires VTCR_EL2.HA to be RES0 (thus ignored by + * hardware) on implementations that do not advertise support for the + * feature. As such, setting HA unconditionally is safe, unless you + * happen to be running on a design that has unadvertised support for + * HAFDBS. Here be dragons. */ - vtcr |= VTCR_EL2_HA; + if (!cpus_have_final_cap(ARM64_WORKAROUND_AMPERE_AC03_CPU_38)) + vtcr |= VTCR_EL2_HA; #endif /* CONFIG_ARM64_HW_AFDBM */ /* Set the vmid bits */ @@ -755,14 +780,17 @@ static bool stage2_try_break_pte(const struct kvm_pgtable_visit_ctx *ctx, if (!stage2_try_set_pte(ctx, KVM_INVALID_PTE_LOCKED)) return false; - /* - * Perform the appropriate TLB invalidation based on the evicted pte - * value (if any). - */ - if (kvm_pte_table(ctx->old, ctx->level)) - kvm_call_hyp(__kvm_tlb_flush_vmid, mmu); - else if (kvm_pte_valid(ctx->old)) - kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ctx->addr, ctx->level); + if (!kvm_pgtable_walk_skip_bbm_tlbi(ctx)) { + /* + * Perform the appropriate TLB invalidation based on the + * evicted pte value (if any). + */ + if (kvm_pte_table(ctx->old, ctx->level)) + kvm_call_hyp(__kvm_tlb_flush_vmid, mmu); + else if (kvm_pte_valid(ctx->old)) + kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, + ctx->addr, ctx->level); + } if (stage2_pte_is_counted(ctx->old)) mm_ops->put_page(ctx->ptep); @@ -869,11 +897,13 @@ static int stage2_map_walker_try_leaf(const struct kvm_pgtable_visit_ctx *ctx, return -EAGAIN; /* Perform CMOs before installation of the guest stage-2 PTE */ - if (mm_ops->dcache_clean_inval_poc && stage2_pte_cacheable(pgt, new)) + if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->dcache_clean_inval_poc && + stage2_pte_cacheable(pgt, new)) mm_ops->dcache_clean_inval_poc(kvm_pte_follow(new, mm_ops), - granule); + granule); - if (mm_ops->icache_inval_pou && stage2_pte_executable(new)) + if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->icache_inval_pou && + stage2_pte_executable(new)) mm_ops->icache_inval_pou(kvm_pte_follow(new, mm_ops), granule); stage2_make_pte(ctx, new); @@ -895,7 +925,7 @@ static int stage2_map_walk_table_pre(const struct kvm_pgtable_visit_ctx *ctx, if (ret) return ret; - mm_ops->free_removed_table(childp, ctx->level); + mm_ops->free_unlinked_table(childp, ctx->level); return 0; } @@ -940,7 +970,7 @@ static int stage2_map_walk_leaf(const struct kvm_pgtable_visit_ctx *ctx, * The TABLE_PRE callback runs for table entries on the way down, looking * for table entries which we could conceivably replace with a block entry * for this mapping. If it finds one it replaces the entry and calls - * kvm_pgtable_mm_ops::free_removed_table() to tear down the detached table. + * kvm_pgtable_mm_ops::free_unlinked_table() to tear down the detached table. * * Otherwise, the LEAF callback performs the mapping at the existing leaves * instead. @@ -1209,7 +1239,7 @@ int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, KVM_PGTABLE_WALK_HANDLE_FAULT | KVM_PGTABLE_WALK_SHARED); if (!ret) - kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, pgt->mmu, addr, level); + kvm_call_hyp(__kvm_tlb_flush_vmid_ipa_nsh, pgt->mmu, addr, level); return ret; } @@ -1242,6 +1272,162 @@ int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size) return kvm_pgtable_walk(pgt, addr, size, &walker); } +kvm_pte_t *kvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt, + u64 phys, u32 level, + enum kvm_pgtable_prot prot, + void *mc, bool force_pte) +{ + struct stage2_map_data map_data = { + .phys = phys, + .mmu = pgt->mmu, + .memcache = mc, + .force_pte = force_pte, + }; + struct kvm_pgtable_walker walker = { + .cb = stage2_map_walker, + .flags = KVM_PGTABLE_WALK_LEAF | + KVM_PGTABLE_WALK_SKIP_BBM_TLBI | + KVM_PGTABLE_WALK_SKIP_CMO, + .arg = &map_data, + }; + /* + * The input address (.addr) is irrelevant for walking an + * unlinked table. Construct an ambiguous IA range to map + * kvm_granule_size(level) worth of memory. + */ + struct kvm_pgtable_walk_data data = { + .walker = &walker, + .addr = 0, + .end = kvm_granule_size(level), + }; + struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops; + kvm_pte_t *pgtable; + int ret; + + if (!IS_ALIGNED(phys, kvm_granule_size(level))) + return ERR_PTR(-EINVAL); + + ret = stage2_set_prot_attr(pgt, prot, &map_data.attr); + if (ret) + return ERR_PTR(ret); + + pgtable = mm_ops->zalloc_page(mc); + if (!pgtable) + return ERR_PTR(-ENOMEM); + + ret = __kvm_pgtable_walk(&data, mm_ops, (kvm_pteref_t)pgtable, + level + 1); + if (ret) { + kvm_pgtable_stage2_free_unlinked(mm_ops, pgtable, level); + mm_ops->put_page(pgtable); + return ERR_PTR(ret); + } + + return pgtable; +} + +/* + * Get the number of page-tables needed to replace a block with a + * fully populated tree up to the PTE entries. Note that @level is + * interpreted as in "level @level entry". + */ +static int stage2_block_get_nr_page_tables(u32 level) +{ + switch (level) { + case 1: + return PTRS_PER_PTE + 1; + case 2: + return 1; + case 3: + return 0; + default: + WARN_ON_ONCE(level < KVM_PGTABLE_MIN_BLOCK_LEVEL || + level >= KVM_PGTABLE_MAX_LEVELS); + return -EINVAL; + }; +} + +static int stage2_split_walker(const struct kvm_pgtable_visit_ctx *ctx, + enum kvm_pgtable_walk_flags visit) +{ + struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops; + struct kvm_mmu_memory_cache *mc = ctx->arg; + struct kvm_s2_mmu *mmu; + kvm_pte_t pte = ctx->old, new, *childp; + enum kvm_pgtable_prot prot; + u32 level = ctx->level; + bool force_pte; + int nr_pages; + u64 phys; + + /* No huge-pages exist at the last level */ + if (level == KVM_PGTABLE_MAX_LEVELS - 1) + return 0; + + /* We only split valid block mappings */ + if (!kvm_pte_valid(pte)) + return 0; + + nr_pages = stage2_block_get_nr_page_tables(level); + if (nr_pages < 0) + return nr_pages; + + if (mc->nobjs >= nr_pages) { + /* Build a tree mapped down to the PTE granularity. */ + force_pte = true; + } else { + /* + * Don't force PTEs, so create_unlinked() below does + * not populate the tree up to the PTE level. The + * consequence is that the call will require a single + * page of level 2 entries at level 1, or a single + * page of PTEs at level 2. If we are at level 1, the + * PTEs will be created recursively. + */ + force_pte = false; + nr_pages = 1; + } + + if (mc->nobjs < nr_pages) + return -ENOMEM; + + mmu = container_of(mc, struct kvm_s2_mmu, split_page_cache); + phys = kvm_pte_to_phys(pte); + prot = kvm_pgtable_stage2_pte_prot(pte); + + childp = kvm_pgtable_stage2_create_unlinked(mmu->pgt, phys, + level, prot, mc, force_pte); + if (IS_ERR(childp)) + return PTR_ERR(childp); + + if (!stage2_try_break_pte(ctx, mmu)) { + kvm_pgtable_stage2_free_unlinked(mm_ops, childp, level); + mm_ops->put_page(childp); + return -EAGAIN; + } + + /* + * Note, the contents of the page table are guaranteed to be made + * visible before the new PTE is assigned because stage2_make_pte() + * writes the PTE using smp_store_release(). + */ + new = kvm_init_table_pte(childp, mm_ops); + stage2_make_pte(ctx, new); + dsb(ishst); + return 0; +} + +int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size, + struct kvm_mmu_memory_cache *mc) +{ + struct kvm_pgtable_walker walker = { + .cb = stage2_split_walker, + .flags = KVM_PGTABLE_WALK_LEAF, + .arg = mc, + }; + + return kvm_pgtable_walk(pgt, addr, size, &walker); +} int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu, struct kvm_pgtable_mm_ops *mm_ops, @@ -1311,7 +1497,7 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt) pgt->pgd = NULL; } -void kvm_pgtable_stage2_free_removed(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, u32 level) +void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, u32 level) { kvm_pteref_t ptep = (kvm_pteref_t)pgtable; struct kvm_pgtable_walker walker = { diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index b37e7c96efea..6537f58b1a8c 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -84,7 +84,7 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu) */ asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT)); - write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1); + kvm_reset_cptr_el2(vcpu); if (!arm64_kernel_unmapped_at_el0()) host_vectors = __this_cpu_read(this_cpu_vector); diff --git a/arch/arm64/kvm/hyp/vhe/tlb.c b/arch/arm64/kvm/hyp/vhe/tlb.c index 24cef9b87f9e..e69da550cdc5 100644 --- a/arch/arm64/kvm/hyp/vhe/tlb.c +++ b/arch/arm64/kvm/hyp/vhe/tlb.c @@ -111,6 +111,38 @@ void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, __tlb_switch_to_host(&cxt); } +void __kvm_tlb_flush_vmid_ipa_nsh(struct kvm_s2_mmu *mmu, + phys_addr_t ipa, int level) +{ + struct tlb_inv_context cxt; + + dsb(nshst); + + /* Switch to requested VMID */ + __tlb_switch_to_guest(mmu, &cxt); + + /* + * We could do so much better if we had the VA as well. + * Instead, we invalidate Stage-2 for this IPA, and the + * whole of Stage-1. Weep... + */ + ipa >>= 12; + __tlbi_level(ipas2e1, ipa, level); + + /* + * We have to ensure completion of the invalidation at Stage-2, + * since a table walk on another CPU could refill a TLB with a + * complete (S1 + S2) walk based on the old Stage-2 mapping if + * the Stage-1 invalidation happened first. + */ + dsb(nsh); + __tlbi(vmalle1); + dsb(nsh); + isb(); + + __tlb_switch_to_host(&cxt); +} + void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu) { struct tlb_inv_context cxt; diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 3b9d4d24c361..6db9ef288ec3 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -31,14 +31,21 @@ static phys_addr_t __ro_after_init hyp_idmap_vector; static unsigned long __ro_after_init io_map_base; -static phys_addr_t stage2_range_addr_end(phys_addr_t addr, phys_addr_t end) +static phys_addr_t __stage2_range_addr_end(phys_addr_t addr, phys_addr_t end, + phys_addr_t size) { - phys_addr_t size = kvm_granule_size(KVM_PGTABLE_MIN_BLOCK_LEVEL); phys_addr_t boundary = ALIGN_DOWN(addr + size, size); return (boundary - 1 < end - 1) ? boundary : end; } +static phys_addr_t stage2_range_addr_end(phys_addr_t addr, phys_addr_t end) +{ + phys_addr_t size = kvm_granule_size(KVM_PGTABLE_MIN_BLOCK_LEVEL); + + return __stage2_range_addr_end(addr, end, size); +} + /* * Release kvm_mmu_lock periodically if the memory region is large. Otherwise, * we may see kernel panics with CONFIG_DETECT_HUNG_TASK, @@ -75,6 +82,79 @@ static int stage2_apply_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, #define stage2_apply_range_resched(mmu, addr, end, fn) \ stage2_apply_range(mmu, addr, end, fn, true) +/* + * Get the maximum number of page-tables pages needed to split a range + * of blocks into PAGE_SIZE PTEs. It assumes the range is already + * mapped at level 2, or at level 1 if allowed. + */ +static int kvm_mmu_split_nr_page_tables(u64 range) +{ + int n = 0; + + if (KVM_PGTABLE_MIN_BLOCK_LEVEL < 2) + n += DIV_ROUND_UP(range, PUD_SIZE); + n += DIV_ROUND_UP(range, PMD_SIZE); + return n; +} + +static bool need_split_memcache_topup_or_resched(struct kvm *kvm) +{ + struct kvm_mmu_memory_cache *cache; + u64 chunk_size, min; + + if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) + return true; + + chunk_size = kvm->arch.mmu.split_page_chunk_size; + min = kvm_mmu_split_nr_page_tables(chunk_size); + cache = &kvm->arch.mmu.split_page_cache; + return kvm_mmu_memory_cache_nr_free_objects(cache) < min; +} + +static int kvm_mmu_split_huge_pages(struct kvm *kvm, phys_addr_t addr, + phys_addr_t end) +{ + struct kvm_mmu_memory_cache *cache; + struct kvm_pgtable *pgt; + int ret, cache_capacity; + u64 next, chunk_size; + + lockdep_assert_held_write(&kvm->mmu_lock); + + chunk_size = kvm->arch.mmu.split_page_chunk_size; + cache_capacity = kvm_mmu_split_nr_page_tables(chunk_size); + + if (chunk_size == 0) + return 0; + + cache = &kvm->arch.mmu.split_page_cache; + + do { + if (need_split_memcache_topup_or_resched(kvm)) { + write_unlock(&kvm->mmu_lock); + cond_resched(); + /* Eager page splitting is best-effort. */ + ret = __kvm_mmu_topup_memory_cache(cache, + cache_capacity, + cache_capacity); + write_lock(&kvm->mmu_lock); + if (ret) + break; + } + + pgt = kvm->arch.mmu.pgt; + if (!pgt) + return -EINVAL; + + next = __stage2_range_addr_end(addr, end, chunk_size); + ret = kvm_pgtable_stage2_split(pgt, addr, next - addr, cache); + if (ret) + break; + } while (addr = next, addr != end); + + return ret; +} + static bool memslot_is_logging(struct kvm_memory_slot *memslot) { return memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY); @@ -131,21 +211,21 @@ static void kvm_s2_free_pages_exact(void *virt, size_t size) static struct kvm_pgtable_mm_ops kvm_s2_mm_ops; -static void stage2_free_removed_table_rcu_cb(struct rcu_head *head) +static void stage2_free_unlinked_table_rcu_cb(struct rcu_head *head) { struct page *page = container_of(head, struct page, rcu_head); void *pgtable = page_to_virt(page); u32 level = page_private(page); - kvm_pgtable_stage2_free_removed(&kvm_s2_mm_ops, pgtable, level); + kvm_pgtable_stage2_free_unlinked(&kvm_s2_mm_ops, pgtable, level); } -static void stage2_free_removed_table(void *addr, u32 level) +static void stage2_free_unlinked_table(void *addr, u32 level) { struct page *page = virt_to_page(addr); set_page_private(page, (unsigned long)level); - call_rcu(&page->rcu_head, stage2_free_removed_table_rcu_cb); + call_rcu(&page->rcu_head, stage2_free_unlinked_table_rcu_cb); } static void kvm_host_get_page(void *addr) @@ -701,7 +781,7 @@ static struct kvm_pgtable_mm_ops kvm_s2_mm_ops = { .zalloc_page = stage2_memcache_zalloc_page, .zalloc_pages_exact = kvm_s2_zalloc_pages_exact, .free_pages_exact = kvm_s2_free_pages_exact, - .free_removed_table = stage2_free_removed_table, + .free_unlinked_table = stage2_free_unlinked_table, .get_page = kvm_host_get_page, .put_page = kvm_s2_put_page, .page_count = kvm_host_page_count, @@ -775,6 +855,10 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long t for_each_possible_cpu(cpu) *per_cpu_ptr(mmu->last_vcpu_ran, cpu) = -1; + /* The eager page splitting is disabled by default */ + mmu->split_page_chunk_size = KVM_ARM_EAGER_SPLIT_CHUNK_SIZE_DEFAULT; + mmu->split_page_cache.gfp_zero = __GFP_ZERO; + mmu->pgt = pgt; mmu->pgd_phys = __pa(pgt->pgd); return 0; @@ -786,6 +870,12 @@ out_free_pgtable: return err; } +void kvm_uninit_stage2_mmu(struct kvm *kvm) +{ + kvm_free_stage2_pgd(&kvm->arch.mmu); + kvm_mmu_free_memory_cache(&kvm->arch.mmu.split_page_cache); +} + static void stage2_unmap_memslot(struct kvm *kvm, struct kvm_memory_slot *memslot) { @@ -989,39 +1079,66 @@ static void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot) } /** - * kvm_mmu_write_protect_pt_masked() - write protect dirty pages + * kvm_mmu_split_memory_region() - split the stage 2 blocks into PAGE_SIZE + * pages for memory slot * @kvm: The KVM pointer - * @slot: The memory slot associated with mask - * @gfn_offset: The gfn offset in memory slot - * @mask: The mask of dirty pages at offset 'gfn_offset' in this memory - * slot to be write protected + * @slot: The memory slot to split * - * Walks bits set in mask write protects the associated pte's. Caller must - * acquire kvm_mmu_lock. + * Acquires kvm->mmu_lock. Called with kvm->slots_lock mutex acquired, + * serializing operations for VM memory regions. */ -static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, - struct kvm_memory_slot *slot, - gfn_t gfn_offset, unsigned long mask) +static void kvm_mmu_split_memory_region(struct kvm *kvm, int slot) { - phys_addr_t base_gfn = slot->base_gfn + gfn_offset; - phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT; - phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT; + struct kvm_memslots *slots; + struct kvm_memory_slot *memslot; + phys_addr_t start, end; - stage2_wp_range(&kvm->arch.mmu, start, end); + lockdep_assert_held(&kvm->slots_lock); + + slots = kvm_memslots(kvm); + memslot = id_to_memslot(slots, slot); + + start = memslot->base_gfn << PAGE_SHIFT; + end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT; + + write_lock(&kvm->mmu_lock); + kvm_mmu_split_huge_pages(kvm, start, end); + write_unlock(&kvm->mmu_lock); } /* - * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected - * dirty pages. + * kvm_arch_mmu_enable_log_dirty_pt_masked() - enable dirty logging for selected pages. + * @kvm: The KVM pointer + * @slot: The memory slot associated with mask + * @gfn_offset: The gfn offset in memory slot + * @mask: The mask of pages at offset 'gfn_offset' in this memory + * slot to enable dirty logging on * - * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to - * enable dirty logging for them. + * Writes protect selected pages to enable dirty logging, and then + * splits them to PAGE_SIZE. Caller must acquire kvm->mmu_lock. */ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn_offset, unsigned long mask) { - kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask); + phys_addr_t base_gfn = slot->base_gfn + gfn_offset; + phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT; + phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT; + + lockdep_assert_held_write(&kvm->mmu_lock); + + stage2_wp_range(&kvm->arch.mmu, start, end); + + /* + * Eager-splitting is done when manual-protect is set. We + * also check for initially-all-set because we can avoid + * eager-splitting if initially-all-set is false. + * Initially-all-set equal false implies that huge-pages were + * already split when enabling dirty logging: no need to do it + * again. + */ + if (kvm_dirty_log_manual_protect_and_init_set(kvm)) + kvm_mmu_split_huge_pages(kvm, start, end); } static void kvm_send_hwpoison_signal(unsigned long address, short lsb) @@ -1790,20 +1907,42 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, const struct kvm_memory_slot *new, enum kvm_mr_change change) { + bool log_dirty_pages = new && new->flags & KVM_MEM_LOG_DIRTY_PAGES; + /* * At this point memslot has been committed and there is an * allocated dirty_bitmap[], dirty pages will be tracked while the * memory slot is write protected. */ - if (change != KVM_MR_DELETE && new->flags & KVM_MEM_LOG_DIRTY_PAGES) { + if (log_dirty_pages) { + + if (change == KVM_MR_DELETE) + return; + /* - * If we're with initial-all-set, we don't need to write - * protect any pages because they're all reported as dirty. - * Huge pages and normal pages will be write protect gradually. + * Huge and normal pages are write-protected and split + * on either of these two cases: + * + * 1. with initial-all-set: gradually with CLEAR ioctls, */ - if (!kvm_dirty_log_manual_protect_and_init_set(kvm)) { - kvm_mmu_wp_memory_region(kvm, new->id); - } + if (kvm_dirty_log_manual_protect_and_init_set(kvm)) + return; + /* + * or + * 2. without initial-all-set: all in one shot when + * enabling dirty logging. + */ + kvm_mmu_wp_memory_region(kvm, new->id); + kvm_mmu_split_memory_region(kvm, new->id); + } else { + /* + * Free any leftovers from the eager page splitting cache. Do + * this when deleting, moving, disabling dirty logging, or + * creating the memslot (a nop). Doing it for deletes makes + * sure we don't leak memory, and there's no need to keep the + * cache around for any of the other cases. + */ + kvm_mmu_free_memory_cache(&kvm->arch.mmu.split_page_cache); } } @@ -1877,7 +2016,7 @@ void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) void kvm_arch_flush_shadow_all(struct kvm *kvm) { - kvm_free_stage2_pgd(&kvm->arch.mmu); + kvm_uninit_stage2_mmu(kvm); } void kvm_arch_flush_shadow_memslot(struct kvm *kvm, diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 6e9ece1ebbe7..994a494703c3 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -78,6 +78,7 @@ void __init kvm_hyp_reserve(void) hyp_mem_pages += host_s2_pgtable_pages(); hyp_mem_pages += hyp_vm_table_pages(); hyp_mem_pages += hyp_vmemmap_pages(STRUCT_HYP_PAGE_SIZE); + hyp_mem_pages += hyp_ffa_proxy_pages(); /* * Try to allocate a PMD-aligned region to reduce TLB pressure once diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c index b5dee8e57e77..bc8556b6f459 100644 --- a/arch/arm64/kvm/reset.c +++ b/arch/arm64/kvm/reset.c @@ -187,57 +187,6 @@ static int kvm_vcpu_enable_ptrauth(struct kvm_vcpu *vcpu) } /** - * kvm_set_vm_width() - set the register width for the guest - * @vcpu: Pointer to the vcpu being configured - * - * Set both KVM_ARCH_FLAG_EL1_32BIT and KVM_ARCH_FLAG_REG_WIDTH_CONFIGURED - * in the VM flags based on the vcpu's requested register width, the HW - * capabilities and other options (such as MTE). - * When REG_WIDTH_CONFIGURED is already set, the vcpu settings must be - * consistent with the value of the FLAG_EL1_32BIT bit in the flags. - * - * Return: 0 on success, negative error code on failure. - */ -static int kvm_set_vm_width(struct kvm_vcpu *vcpu) -{ - struct kvm *kvm = vcpu->kvm; - bool is32bit; - - is32bit = vcpu_has_feature(vcpu, KVM_ARM_VCPU_EL1_32BIT); - - lockdep_assert_held(&kvm->arch.config_lock); - - if (test_bit(KVM_ARCH_FLAG_REG_WIDTH_CONFIGURED, &kvm->arch.flags)) { - /* - * The guest's register width is already configured. - * Make sure that the vcpu is consistent with it. - */ - if (is32bit == test_bit(KVM_ARCH_FLAG_EL1_32BIT, &kvm->arch.flags)) - return 0; - - return -EINVAL; - } - - if (!cpus_have_const_cap(ARM64_HAS_32BIT_EL1) && is32bit) - return -EINVAL; - - /* MTE is incompatible with AArch32 */ - if (kvm_has_mte(kvm) && is32bit) - return -EINVAL; - - /* NV is incompatible with AArch32 */ - if (vcpu_has_nv(vcpu) && is32bit) - return -EINVAL; - - if (is32bit) - set_bit(KVM_ARCH_FLAG_EL1_32BIT, &kvm->arch.flags); - - set_bit(KVM_ARCH_FLAG_REG_WIDTH_CONFIGURED, &kvm->arch.flags); - - return 0; -} - -/** * kvm_reset_vcpu - sets core registers and sys_regs to reset value * @vcpu: The VCPU pointer * @@ -262,13 +211,6 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu) bool loaded; u32 pstate; - mutex_lock(&vcpu->kvm->arch.config_lock); - ret = kvm_set_vm_width(vcpu); - mutex_unlock(&vcpu->kvm->arch.config_lock); - - if (ret) - return ret; - spin_lock(&vcpu->arch.mp_state_lock); reset_state = vcpu->arch.reset_state; vcpu->arch.reset_state.reset = false; diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 5b5d5e5449dc..bd3431823ec5 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -42,6 +42,8 @@ */ static u64 sys_reg_to_index(const struct sys_reg_desc *reg); +static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, + u64 val); static bool read_from_write_only(struct kvm_vcpu *vcpu, struct sys_reg_params *params, @@ -553,10 +555,11 @@ static int get_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, return 0; } -static void reset_bvr(struct kvm_vcpu *vcpu, +static u64 reset_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd) { vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm] = rd->val; + return rd->val; } static bool trap_bcr(struct kvm_vcpu *vcpu, @@ -589,10 +592,11 @@ static int get_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, return 0; } -static void reset_bcr(struct kvm_vcpu *vcpu, +static u64 reset_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd) { vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm] = rd->val; + return rd->val; } static bool trap_wvr(struct kvm_vcpu *vcpu, @@ -626,10 +630,11 @@ static int get_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, return 0; } -static void reset_wvr(struct kvm_vcpu *vcpu, +static u64 reset_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd) { vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm] = rd->val; + return rd->val; } static bool trap_wcr(struct kvm_vcpu *vcpu, @@ -662,25 +667,28 @@ static int get_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, return 0; } -static void reset_wcr(struct kvm_vcpu *vcpu, +static u64 reset_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd) { vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm] = rd->val; + return rd->val; } -static void reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) +static u64 reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { u64 amair = read_sysreg(amair_el1); vcpu_write_sys_reg(vcpu, amair, AMAIR_EL1); + return amair; } -static void reset_actlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) +static u64 reset_actlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { u64 actlr = read_sysreg(actlr_el1); vcpu_write_sys_reg(vcpu, actlr, ACTLR_EL1); + return actlr; } -static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) +static u64 reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { u64 mpidr; @@ -694,7 +702,10 @@ static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0); mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1); mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2); - vcpu_write_sys_reg(vcpu, (1ULL << 31) | mpidr, MPIDR_EL1); + mpidr |= (1ULL << 31); + vcpu_write_sys_reg(vcpu, mpidr, MPIDR_EL1); + + return mpidr; } static unsigned int pmu_visibility(const struct kvm_vcpu *vcpu, @@ -706,13 +717,13 @@ static unsigned int pmu_visibility(const struct kvm_vcpu *vcpu, return REG_HIDDEN; } -static void reset_pmu_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) +static u64 reset_pmu_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { u64 n, mask = BIT(ARMV8_PMU_CYCLE_IDX); /* No PMU available, any PMU reg may UNDEF... */ if (!kvm_arm_support_pmu_v3()) - return; + return 0; n = read_sysreg(pmcr_el0) >> ARMV8_PMU_PMCR_N_SHIFT; n &= ARMV8_PMU_PMCR_N_MASK; @@ -721,33 +732,41 @@ static void reset_pmu_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) reset_unknown(vcpu, r); __vcpu_sys_reg(vcpu, r->reg) &= mask; + + return __vcpu_sys_reg(vcpu, r->reg); } -static void reset_pmevcntr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) +static u64 reset_pmevcntr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { reset_unknown(vcpu, r); __vcpu_sys_reg(vcpu, r->reg) &= GENMASK(31, 0); + + return __vcpu_sys_reg(vcpu, r->reg); } -static void reset_pmevtyper(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) +static u64 reset_pmevtyper(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { reset_unknown(vcpu, r); __vcpu_sys_reg(vcpu, r->reg) &= ARMV8_PMU_EVTYPE_MASK; + + return __vcpu_sys_reg(vcpu, r->reg); } -static void reset_pmselr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) +static u64 reset_pmselr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { reset_unknown(vcpu, r); __vcpu_sys_reg(vcpu, r->reg) &= ARMV8_PMU_COUNTER_MASK; + + return __vcpu_sys_reg(vcpu, r->reg); } -static void reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) +static u64 reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { u64 pmcr; /* No PMU available, PMCR_EL0 may UNDEF... */ if (!kvm_arm_support_pmu_v3()) - return; + return 0; /* Only preserve PMCR_EL0.N, and reset the rest to 0 */ pmcr = read_sysreg(pmcr_el0) & (ARMV8_PMU_PMCR_N_MASK << ARMV8_PMU_PMCR_N_SHIFT); @@ -755,6 +774,8 @@ static void reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) pmcr |= ARMV8_PMU_PMCR_LC; __vcpu_sys_reg(vcpu, r->reg) = pmcr; + + return __vcpu_sys_reg(vcpu, r->reg); } static bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags) @@ -1187,25 +1208,89 @@ static bool access_arch_timer(struct kvm_vcpu *vcpu, return true; } -static u8 vcpu_pmuver(const struct kvm_vcpu *vcpu) +static s64 kvm_arm64_ftr_safe_value(u32 id, const struct arm64_ftr_bits *ftrp, + s64 new, s64 cur) { - if (kvm_vcpu_has_pmu(vcpu)) - return vcpu->kvm->arch.dfr0_pmuver.imp; + struct arm64_ftr_bits kvm_ftr = *ftrp; + + /* Some features have different safe value type in KVM than host features */ + switch (id) { + case SYS_ID_AA64DFR0_EL1: + if (kvm_ftr.shift == ID_AA64DFR0_EL1_PMUVer_SHIFT) + kvm_ftr.type = FTR_LOWER_SAFE; + break; + case SYS_ID_DFR0_EL1: + if (kvm_ftr.shift == ID_DFR0_EL1_PerfMon_SHIFT) + kvm_ftr.type = FTR_LOWER_SAFE; + break; + } - return vcpu->kvm->arch.dfr0_pmuver.unimp; + return arm64_ftr_safe_value(&kvm_ftr, new, cur); } -static u8 perfmon_to_pmuver(u8 perfmon) +/** + * arm64_check_features() - Check if a feature register value constitutes + * a subset of features indicated by the idreg's KVM sanitised limit. + * + * This function will check if each feature field of @val is the "safe" value + * against idreg's KVM sanitised limit return from reset() callback. + * If a field value in @val is the same as the one in limit, it is always + * considered the safe value regardless For register fields that are not in + * writable, only the value in limit is considered the safe value. + * + * Return: 0 if all the fields are safe. Otherwise, return negative errno. + */ +static int arm64_check_features(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *rd, + u64 val) { - switch (perfmon) { - case ID_DFR0_EL1_PerfMon_PMUv3: - return ID_AA64DFR0_EL1_PMUVer_IMP; - case ID_DFR0_EL1_PerfMon_IMPDEF: - return ID_AA64DFR0_EL1_PMUVer_IMP_DEF; - default: - /* Anything ARMv8.1+ and NI have the same value. For now. */ - return perfmon; + const struct arm64_ftr_reg *ftr_reg; + const struct arm64_ftr_bits *ftrp = NULL; + u32 id = reg_to_encoding(rd); + u64 writable_mask = rd->val; + u64 limit = rd->reset(vcpu, rd); + u64 mask = 0; + + /* + * Hidden and unallocated ID registers may not have a corresponding + * struct arm64_ftr_reg. Of course, if the register is RAZ we know the + * only safe value is 0. + */ + if (sysreg_visible_as_raz(vcpu, rd)) + return val ? -E2BIG : 0; + + ftr_reg = get_arm64_ftr_reg(id); + if (!ftr_reg) + return -EINVAL; + + ftrp = ftr_reg->ftr_bits; + + for (; ftrp && ftrp->width; ftrp++) { + s64 f_val, f_lim, safe_val; + u64 ftr_mask; + + ftr_mask = arm64_ftr_mask(ftrp); + if ((ftr_mask & writable_mask) != ftr_mask) + continue; + + f_val = arm64_ftr_value(ftrp, val); + f_lim = arm64_ftr_value(ftrp, limit); + mask |= ftr_mask; + + if (f_val == f_lim) + safe_val = f_val; + else + safe_val = kvm_arm64_ftr_safe_value(id, ftrp, f_val, f_lim); + + if (safe_val != f_val) + return -E2BIG; } + + /* For fields that are not writable, values in limit are the safe values. */ + if ((val & ~mask) != (limit & ~mask)) + return -E2BIG; + + return 0; } static u8 pmuver_to_perfmon(u8 pmuver) @@ -1222,7 +1307,8 @@ static u8 pmuver_to_perfmon(u8 pmuver) } /* Read a sanitised cpufeature ID register by sys_reg_desc */ -static u64 read_id_reg(const struct kvm_vcpu *vcpu, struct sys_reg_desc const *r) +static u64 __kvm_read_sanitised_id_reg(const struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r) { u32 id = reg_to_encoding(r); u64 val; @@ -1233,19 +1319,6 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, struct sys_reg_desc const *r val = read_sanitised_ftr_reg(id); switch (id) { - case SYS_ID_AA64PFR0_EL1: - if (!vcpu_has_sve(vcpu)) - val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_SVE); - val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AMU); - val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2); - val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2), (u64)vcpu->kvm->arch.pfr0_csv2); - val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3); - val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3), (u64)vcpu->kvm->arch.pfr0_csv3); - if (kvm_vgic_global_state.type == VGIC_V3) { - val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_GIC); - val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_GIC), 1); - } - break; case SYS_ID_AA64PFR1_EL1: if (!kvm_has_mte(vcpu->kvm)) val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTE); @@ -1267,22 +1340,6 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, struct sys_reg_desc const *r val &= ~ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_WFxT); val &= ~ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_MOPS); break; - case SYS_ID_AA64DFR0_EL1: - /* Limit debug to ARMv8.0 */ - val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DebugVer); - val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DebugVer), 6); - /* Set PMUver to the required version */ - val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer); - val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer), - vcpu_pmuver(vcpu)); - /* Hide SPE from guests */ - val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMSVer); - break; - case SYS_ID_DFR0_EL1: - val &= ~ARM64_FEATURE_MASK(ID_DFR0_EL1_PerfMon); - val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_DFR0_EL1_PerfMon), - pmuver_to_perfmon(vcpu_pmuver(vcpu))); - break; case SYS_ID_AA64MMFR2_EL1: val &= ~ID_AA64MMFR2_EL1_CCIDX_MASK; break; @@ -1294,6 +1351,28 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, struct sys_reg_desc const *r return val; } +static u64 kvm_read_sanitised_id_reg(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r) +{ + return __kvm_read_sanitised_id_reg(vcpu, r); +} + +static u64 read_id_reg(const struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) +{ + return IDREG(vcpu->kvm, reg_to_encoding(r)); +} + +/* + * Return true if the register's (Op0, Op1, CRn, CRm, Op2) is + * (3, 0, 0, crm, op2), where 1<=crm<8, 0<=op2<8. + */ +static inline bool is_id_reg(u32 id) +{ + return (sys_reg_Op0(id) == 3 && sys_reg_Op1(id) == 0 && + sys_reg_CRn(id) == 0 && sys_reg_CRm(id) >= 1 && + sys_reg_CRm(id) < 8); +} + static unsigned int id_visibility(const struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { @@ -1355,88 +1434,113 @@ static unsigned int sve_visibility(const struct kvm_vcpu *vcpu, return REG_HIDDEN; } -static int set_id_aa64pfr0_el1(struct kvm_vcpu *vcpu, - const struct sys_reg_desc *rd, - u64 val) +static u64 read_sanitised_id_aa64pfr0_el1(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *rd) { - u8 csv2, csv3; + u64 val = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); + + if (!vcpu_has_sve(vcpu)) + val &= ~ID_AA64PFR0_EL1_SVE_MASK; /* - * Allow AA64PFR0_EL1.CSV2 to be set from userspace as long as - * it doesn't promise more than what is actually provided (the - * guest could otherwise be covered in ectoplasmic residue). + * The default is to expose CSV2 == 1 if the HW isn't affected. + * Although this is a per-CPU feature, we make it global because + * asymmetric systems are just a nuisance. + * + * Userspace can override this as long as it doesn't promise + * the impossible. */ - csv2 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_EL1_CSV2_SHIFT); - if (csv2 > 1 || - (csv2 && arm64_get_spectre_v2_state() != SPECTRE_UNAFFECTED)) - return -EINVAL; + if (arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED) { + val &= ~ID_AA64PFR0_EL1_CSV2_MASK; + val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, CSV2, IMP); + } + if (arm64_get_meltdown_state() == SPECTRE_UNAFFECTED) { + val &= ~ID_AA64PFR0_EL1_CSV3_MASK; + val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, CSV3, IMP); + } - /* Same thing for CSV3 */ - csv3 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_EL1_CSV3_SHIFT); - if (csv3 > 1 || - (csv3 && arm64_get_meltdown_state() != SPECTRE_UNAFFECTED)) - return -EINVAL; + if (kvm_vgic_global_state.type == VGIC_V3) { + val &= ~ID_AA64PFR0_EL1_GIC_MASK; + val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, GIC, IMP); + } - /* We can only differ with CSV[23], and anything else is an error */ - val ^= read_id_reg(vcpu, rd); - val &= ~(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2) | - ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3)); - if (val) - return -EINVAL; + val &= ~ID_AA64PFR0_EL1_AMU_MASK; - vcpu->kvm->arch.pfr0_csv2 = csv2; - vcpu->kvm->arch.pfr0_csv3 = csv3; + return val; +} - return 0; +static u64 read_sanitised_id_aa64dfr0_el1(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *rd) +{ + u64 val = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1); + + /* Limit debug to ARMv8.0 */ + val &= ~ID_AA64DFR0_EL1_DebugVer_MASK; + val |= SYS_FIELD_PREP_ENUM(ID_AA64DFR0_EL1, DebugVer, IMP); + + /* + * Only initialize the PMU version if the vCPU was configured with one. + */ + val &= ~ID_AA64DFR0_EL1_PMUVer_MASK; + if (kvm_vcpu_has_pmu(vcpu)) + val |= SYS_FIELD_PREP(ID_AA64DFR0_EL1, PMUVer, + kvm_arm_pmu_get_pmuver_limit()); + + /* Hide SPE from guests */ + val &= ~ID_AA64DFR0_EL1_PMSVer_MASK; + + return val; } static int set_id_aa64dfr0_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, u64 val) { - u8 pmuver, host_pmuver; - bool valid_pmu; - - host_pmuver = kvm_arm_pmu_get_pmuver_limit(); + u8 pmuver = SYS_FIELD_GET(ID_AA64DFR0_EL1, PMUVer, val); /* - * Allow AA64DFR0_EL1.PMUver to be set from userspace as long - * as it doesn't promise more than what the HW gives us. We - * allow an IMPDEF PMU though, only if no PMU is supported - * (KVM backward compatibility handling). + * Prior to commit 3d0dba5764b9 ("KVM: arm64: PMU: Move the + * ID_AA64DFR0_EL1.PMUver limit to VM creation"), KVM erroneously + * exposed an IMP_DEF PMU to userspace and the guest on systems w/ + * non-architectural PMUs. Of course, PMUv3 is the only game in town for + * PMU virtualization, so the IMP_DEF value was rather user-hostile. + * + * At minimum, we're on the hook to allow values that were given to + * userspace by KVM. Cover our tracks here and replace the IMP_DEF value + * with a more sensible NI. The value of an ID register changing under + * the nose of the guest is unfortunate, but is certainly no more + * surprising than an ill-guided PMU driver poking at impdef system + * registers that end in an UNDEF... */ - pmuver = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer), val); - if ((pmuver != ID_AA64DFR0_EL1_PMUVer_IMP_DEF && pmuver > host_pmuver)) - return -EINVAL; + if (pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF) + val &= ~ID_AA64DFR0_EL1_PMUVer_MASK; - valid_pmu = (pmuver != 0 && pmuver != ID_AA64DFR0_EL1_PMUVer_IMP_DEF); - - /* Make sure view register and PMU support do match */ - if (kvm_vcpu_has_pmu(vcpu) != valid_pmu) - return -EINVAL; + return set_id_reg(vcpu, rd, val); +} - /* We can only differ with PMUver, and anything else is an error */ - val ^= read_id_reg(vcpu, rd); - val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer); - if (val) - return -EINVAL; +static u64 read_sanitised_id_dfr0_el1(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *rd) +{ + u8 perfmon = pmuver_to_perfmon(kvm_arm_pmu_get_pmuver_limit()); + u64 val = read_sanitised_ftr_reg(SYS_ID_DFR0_EL1); - if (valid_pmu) - vcpu->kvm->arch.dfr0_pmuver.imp = pmuver; - else - vcpu->kvm->arch.dfr0_pmuver.unimp = pmuver; + val &= ~ID_DFR0_EL1_PerfMon_MASK; + if (kvm_vcpu_has_pmu(vcpu)) + val |= SYS_FIELD_PREP(ID_DFR0_EL1, PerfMon, perfmon); - return 0; + return val; } static int set_id_dfr0_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, u64 val) { - u8 perfmon, host_perfmon; - bool valid_pmu; + u8 perfmon = SYS_FIELD_GET(ID_DFR0_EL1, PerfMon, val); - host_perfmon = pmuver_to_perfmon(kvm_arm_pmu_get_pmuver_limit()); + if (perfmon == ID_DFR0_EL1_PerfMon_IMPDEF) { + val &= ~ID_DFR0_EL1_PerfMon_MASK; + perfmon = 0; + } /* * Allow DFR0_EL1.PerfMon to be set from userspace as long as @@ -1444,29 +1548,10 @@ static int set_id_dfr0_el1(struct kvm_vcpu *vcpu, * AArch64 side (as everything is emulated with that), and * that this is a PMUv3. */ - perfmon = FIELD_GET(ARM64_FEATURE_MASK(ID_DFR0_EL1_PerfMon), val); - if ((perfmon != ID_DFR0_EL1_PerfMon_IMPDEF && perfmon > host_perfmon) || - (perfmon != 0 && perfmon < ID_DFR0_EL1_PerfMon_PMUv3)) + if (perfmon != 0 && perfmon < ID_DFR0_EL1_PerfMon_PMUv3) return -EINVAL; - valid_pmu = (perfmon != 0 && perfmon != ID_DFR0_EL1_PerfMon_IMPDEF); - - /* Make sure view register and PMU support do match */ - if (kvm_vcpu_has_pmu(vcpu) != valid_pmu) - return -EINVAL; - - /* We can only differ with PerfMon, and anything else is an error */ - val ^= read_id_reg(vcpu, rd); - val &= ~ARM64_FEATURE_MASK(ID_DFR0_EL1_PerfMon); - if (val) - return -EINVAL; - - if (valid_pmu) - vcpu->kvm->arch.dfr0_pmuver.imp = perfmon_to_pmuver(perfmon); - else - vcpu->kvm->arch.dfr0_pmuver.unimp = perfmon_to_pmuver(perfmon); - - return 0; + return set_id_reg(vcpu, rd, val); } /* @@ -1479,18 +1564,60 @@ static int set_id_dfr0_el1(struct kvm_vcpu *vcpu, static int get_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, u64 *val) { + /* + * Avoid locking if the VM has already started, as the ID registers are + * guaranteed to be invariant at that point. + */ + if (kvm_vm_has_ran_once(vcpu->kvm)) { + *val = read_id_reg(vcpu, rd); + return 0; + } + + mutex_lock(&vcpu->kvm->arch.config_lock); *val = read_id_reg(vcpu, rd); + mutex_unlock(&vcpu->kvm->arch.config_lock); + return 0; } static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, u64 val) { - /* This is what we mean by invariant: you can't change it. */ - if (val != read_id_reg(vcpu, rd)) - return -EINVAL; + u32 id = reg_to_encoding(rd); + int ret; - return 0; + mutex_lock(&vcpu->kvm->arch.config_lock); + + /* + * Once the VM has started the ID registers are immutable. Reject any + * write that does not match the final register value. + */ + if (kvm_vm_has_ran_once(vcpu->kvm)) { + if (val != read_id_reg(vcpu, rd)) + ret = -EBUSY; + else + ret = 0; + + mutex_unlock(&vcpu->kvm->arch.config_lock); + return ret; + } + + ret = arm64_check_features(vcpu, rd, val); + if (!ret) + IDREG(vcpu->kvm, id) = val; + + mutex_unlock(&vcpu->kvm->arch.config_lock); + + /* + * arm64_check_features() returns -E2BIG to indicate the register's + * feature set is a superset of the maximally-allowed register value. + * While it would be nice to precisely describe this to userspace, the + * existing UAPI for KVM_SET_ONE_REG has it that invalid register + * writes return -EINVAL. + */ + if (ret == -E2BIG) + ret = -EINVAL; + return ret; } static int get_raz_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, @@ -1530,7 +1657,7 @@ static bool access_clidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p, * Fabricate a CLIDR_EL1 value instead of using the real value, which can vary * by the physical CPU which the vcpu currently resides in. */ -static void reset_clidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) +static u64 reset_clidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0); u64 clidr; @@ -1578,6 +1705,8 @@ static void reset_clidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) clidr |= 2 << CLIDR_TTYPE_SHIFT(loc); __vcpu_sys_reg(vcpu, r->reg) = clidr; + + return __vcpu_sys_reg(vcpu, r->reg); } static int set_clidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, @@ -1677,6 +1806,17 @@ static unsigned int elx2_visibility(const struct kvm_vcpu *vcpu, .visibility = elx2_visibility, \ } +/* + * Since reset() callback and field val are not used for idregs, they will be + * used for specific purposes for idregs. + * The reset() would return KVM sanitised register value. The value would be the + * same as the host kernel sanitised value if there is no KVM sanitisation. + * The val would be used as a mask indicating writable fields for the idreg. + * Only bits with 1 are writable from userspace. This mask might not be + * necessary in the future whenever all ID registers are enabled as writable + * from userspace. + */ + /* sys_reg_desc initialiser for known cpufeature ID registers */ #define ID_SANITISED(name) { \ SYS_DESC(SYS_##name), \ @@ -1684,6 +1824,8 @@ static unsigned int elx2_visibility(const struct kvm_vcpu *vcpu, .get_user = get_id_reg, \ .set_user = set_id_reg, \ .visibility = id_visibility, \ + .reset = kvm_read_sanitised_id_reg, \ + .val = 0, \ } /* sys_reg_desc initialiser for known cpufeature ID registers */ @@ -1693,6 +1835,8 @@ static unsigned int elx2_visibility(const struct kvm_vcpu *vcpu, .get_user = get_id_reg, \ .set_user = set_id_reg, \ .visibility = aa32_id_visibility, \ + .reset = kvm_read_sanitised_id_reg, \ + .val = 0, \ } /* @@ -1705,7 +1849,9 @@ static unsigned int elx2_visibility(const struct kvm_vcpu *vcpu, .access = access_id_reg, \ .get_user = get_id_reg, \ .set_user = set_id_reg, \ - .visibility = raz_visibility \ + .visibility = raz_visibility, \ + .reset = kvm_read_sanitised_id_reg, \ + .val = 0, \ } /* @@ -1719,6 +1865,8 @@ static unsigned int elx2_visibility(const struct kvm_vcpu *vcpu, .get_user = get_id_reg, \ .set_user = set_id_reg, \ .visibility = raz_visibility, \ + .reset = kvm_read_sanitised_id_reg, \ + .val = 0, \ } static bool access_sp_el1(struct kvm_vcpu *vcpu, @@ -1826,9 +1974,13 @@ static const struct sys_reg_desc sys_reg_descs[] = { /* CRm=1 */ AA32_ID_SANITISED(ID_PFR0_EL1), AA32_ID_SANITISED(ID_PFR1_EL1), - { SYS_DESC(SYS_ID_DFR0_EL1), .access = access_id_reg, - .get_user = get_id_reg, .set_user = set_id_dfr0_el1, - .visibility = aa32_id_visibility, }, + { SYS_DESC(SYS_ID_DFR0_EL1), + .access = access_id_reg, + .get_user = get_id_reg, + .set_user = set_id_dfr0_el1, + .visibility = aa32_id_visibility, + .reset = read_sanitised_id_dfr0_el1, + .val = ID_DFR0_EL1_PerfMon_MASK, }, ID_HIDDEN(ID_AFR0_EL1), AA32_ID_SANITISED(ID_MMFR0_EL1), AA32_ID_SANITISED(ID_MMFR1_EL1), @@ -1857,8 +2009,12 @@ static const struct sys_reg_desc sys_reg_descs[] = { /* AArch64 ID registers */ /* CRm=4 */ - { SYS_DESC(SYS_ID_AA64PFR0_EL1), .access = access_id_reg, - .get_user = get_id_reg, .set_user = set_id_aa64pfr0_el1, }, + { SYS_DESC(SYS_ID_AA64PFR0_EL1), + .access = access_id_reg, + .get_user = get_id_reg, + .set_user = set_id_reg, + .reset = read_sanitised_id_aa64pfr0_el1, + .val = ID_AA64PFR0_EL1_CSV2_MASK | ID_AA64PFR0_EL1_CSV3_MASK, }, ID_SANITISED(ID_AA64PFR1_EL1), ID_UNALLOCATED(4,2), ID_UNALLOCATED(4,3), @@ -1868,8 +2024,12 @@ static const struct sys_reg_desc sys_reg_descs[] = { ID_UNALLOCATED(4,7), /* CRm=5 */ - { SYS_DESC(SYS_ID_AA64DFR0_EL1), .access = access_id_reg, - .get_user = get_id_reg, .set_user = set_id_aa64dfr0_el1, }, + { SYS_DESC(SYS_ID_AA64DFR0_EL1), + .access = access_id_reg, + .get_user = get_id_reg, + .set_user = set_id_aa64dfr0_el1, + .reset = read_sanitised_id_aa64dfr0_el1, + .val = ID_AA64DFR0_EL1_PMUVer_MASK, }, ID_SANITISED(ID_AA64DFR1_EL1), ID_UNALLOCATED(5,2), ID_UNALLOCATED(5,3), @@ -2203,7 +2363,7 @@ static const struct sys_reg_desc sys_reg_descs[] = { EL2_REG(ACTLR_EL2, access_rw, reset_val, 0), EL2_REG(HCR_EL2, access_rw, reset_val, 0), EL2_REG(MDCR_EL2, access_rw, reset_val, 0), - EL2_REG(CPTR_EL2, access_rw, reset_val, CPTR_EL2_DEFAULT ), + EL2_REG(CPTR_EL2, access_rw, reset_val, CPTR_NVHE_EL2_RES1), EL2_REG(HSTR_EL2, access_rw, reset_val, 0), EL2_REG(HACR_EL2, access_rw, reset_val, 0), @@ -2260,6 +2420,8 @@ static const struct sys_reg_desc sys_reg_descs[] = { EL2_REG(SP_EL2, NULL, reset_unknown, 0), }; +static const struct sys_reg_desc *first_idreg; + static bool trap_dbgdidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p, const struct sys_reg_desc *r) @@ -2950,6 +3112,28 @@ static bool emulate_sys_reg(struct kvm_vcpu *vcpu, return false; } +static void kvm_reset_id_regs(struct kvm_vcpu *vcpu) +{ + const struct sys_reg_desc *idreg = first_idreg; + u32 id = reg_to_encoding(idreg); + struct kvm *kvm = vcpu->kvm; + + if (test_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags)) + return; + + lockdep_assert_held(&kvm->arch.config_lock); + + /* Initialize all idregs */ + while (is_id_reg(id)) { + IDREG(kvm, id) = idreg->reset(vcpu, idreg); + + idreg++; + id = reg_to_encoding(idreg); + } + + set_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags); +} + /** * kvm_reset_sys_regs - sets system registers to reset value * @vcpu: The VCPU pointer @@ -2961,9 +3145,17 @@ void kvm_reset_sys_regs(struct kvm_vcpu *vcpu) { unsigned long i; - for (i = 0; i < ARRAY_SIZE(sys_reg_descs); i++) - if (sys_reg_descs[i].reset) - sys_reg_descs[i].reset(vcpu, &sys_reg_descs[i]); + kvm_reset_id_regs(vcpu); + + for (i = 0; i < ARRAY_SIZE(sys_reg_descs); i++) { + const struct sys_reg_desc *r = &sys_reg_descs[i]; + + if (is_id_reg(reg_to_encoding(r))) + continue; + + if (r->reset) + r->reset(vcpu, r); + } } /** @@ -3064,19 +3256,21 @@ id_to_sys_reg_desc(struct kvm_vcpu *vcpu, u64 id, */ #define FUNCTION_INVARIANT(reg) \ - static void get_##reg(struct kvm_vcpu *v, \ + static u64 get_##reg(struct kvm_vcpu *v, \ const struct sys_reg_desc *r) \ { \ ((struct sys_reg_desc *)r)->val = read_sysreg(reg); \ + return ((struct sys_reg_desc *)r)->val; \ } FUNCTION_INVARIANT(midr_el1) FUNCTION_INVARIANT(revidr_el1) FUNCTION_INVARIANT(aidr_el1) -static void get_ctr_el0(struct kvm_vcpu *v, const struct sys_reg_desc *r) +static u64 get_ctr_el0(struct kvm_vcpu *v, const struct sys_reg_desc *r) { ((struct sys_reg_desc *)r)->val = read_sanitised_ftr_reg(SYS_CTR_EL0); + return ((struct sys_reg_desc *)r)->val; } /* ->val is filled in by kvm_sys_reg_table_init() */ @@ -3368,6 +3562,7 @@ int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) int __init kvm_sys_reg_table_init(void) { + struct sys_reg_params params; bool valid = true; unsigned int i; @@ -3386,5 +3581,11 @@ int __init kvm_sys_reg_table_init(void) for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++) invariant_sys_regs[i].reset(NULL, &invariant_sys_regs[i]); + /* Find the first idreg (SYS_ID_PFR0_EL1) in sys_reg_descs. */ + params = encoding_to_params(SYS_ID_PFR0_EL1); + first_idreg = find_reg(¶ms, sys_reg_descs, ARRAY_SIZE(sys_reg_descs)); + if (!first_idreg) + return -EINVAL; + return 0; } diff --git a/arch/arm64/kvm/sys_regs.h b/arch/arm64/kvm/sys_regs.h index 6b11f2cc7146..c65c129b3500 100644 --- a/arch/arm64/kvm/sys_regs.h +++ b/arch/arm64/kvm/sys_regs.h @@ -27,6 +27,13 @@ struct sys_reg_params { bool is_write; }; +#define encoding_to_params(reg) \ + ((struct sys_reg_params){ .Op0 = sys_reg_Op0(reg), \ + .Op1 = sys_reg_Op1(reg), \ + .CRn = sys_reg_CRn(reg), \ + .CRm = sys_reg_CRm(reg), \ + .Op2 = sys_reg_Op2(reg) }) + #define esr_sys64_to_params(esr) \ ((struct sys_reg_params){ .Op0 = ((esr) >> 20) & 3, \ .Op1 = ((esr) >> 14) & 0x7, \ @@ -64,13 +71,16 @@ struct sys_reg_desc { struct sys_reg_params *, const struct sys_reg_desc *); - /* Initialization for vcpu. */ - void (*reset)(struct kvm_vcpu *, const struct sys_reg_desc *); + /* + * Initialization for vcpu. Return initialized value, or KVM + * sanitized value for ID registers. + */ + u64 (*reset)(struct kvm_vcpu *, const struct sys_reg_desc *); /* Index into sys_reg[], or 0 if we don't need to save it. */ int reg; - /* Value (usually reset value) */ + /* Value (usually reset value), or write mask for idregs */ u64 val; /* Custom get/set_user functions, fallback to generic if NULL */ @@ -123,19 +133,21 @@ static inline bool read_zero(struct kvm_vcpu *vcpu, } /* Reset functions */ -static inline void reset_unknown(struct kvm_vcpu *vcpu, +static inline u64 reset_unknown(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { BUG_ON(!r->reg); BUG_ON(r->reg >= NR_SYS_REGS); __vcpu_sys_reg(vcpu, r->reg) = 0x1de7ec7edbadc0deULL; + return __vcpu_sys_reg(vcpu, r->reg); } -static inline void reset_val(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) +static inline u64 reset_val(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { BUG_ON(!r->reg); BUG_ON(r->reg >= NR_SYS_REGS); __vcpu_sys_reg(vcpu, r->reg) = r->val; + return __vcpu_sys_reg(vcpu, r->reg); } static inline unsigned int sysreg_visibility(const struct kvm_vcpu *vcpu, diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 935f0a8911f9..3fe516b32577 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -536,9 +536,7 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, unsigned long vm_flags; unsigned int mm_flags = FAULT_FLAG_DEFAULT; unsigned long addr = untagged_addr(far); -#ifdef CONFIG_PER_VMA_LOCK struct vm_area_struct *vma; -#endif if (kprobe_page_fault(regs, esr)) return 0; diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 19c23c4fa2da..c80ed4f3cbce 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -25,6 +25,7 @@ HAS_E0PD HAS_ECV HAS_ECV_CNTPOFF HAS_EPAN +HAS_EVT HAS_GENERIC_AUTH HAS_GENERIC_AUTH_ARCH_QARMA3 HAS_GENERIC_AUTH_ARCH_QARMA5 @@ -51,6 +52,7 @@ HAS_TLB_RANGE HAS_VIRT_HOST_EXTN HAS_WFXT HW_DBM +KVM_HVHE KVM_PROTECTED_MODE MISMATCHED_CACHE_TYPE MTE @@ -81,6 +83,7 @@ WORKAROUND_2077057 WORKAROUND_2457168 WORKAROUND_2645198 WORKAROUND_2658417 +WORKAROUND_AMPERE_AC03_CPU_38 WORKAROUND_TRBE_OVERWRITE_FILL_MODE WORKAROUND_TSB_FLUSH_FAILURE WORKAROUND_TRBE_WRITE_OUT_OF_RANGE diff --git a/arch/csky/kernel/probes/uprobes.c b/arch/csky/kernel/probes/uprobes.c index 2d31a12e46cf..936bea6fd32d 100644 --- a/arch/csky/kernel/probes/uprobes.c +++ b/arch/csky/kernel/probes/uprobes.c @@ -64,6 +64,7 @@ int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) struct uprobe_task *utask = current->utask; WARN_ON_ONCE(current->thread.trap_no != UPROBE_TRAP_NR); + current->thread.trap_no = utask->autask.saved_trap_no; instruction_pointer_set(regs, utask->vaddr + auprobe->insn_size); @@ -101,6 +102,8 @@ void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) { struct uprobe_task *utask = current->utask; + current->thread.trap_no = utask->autask.saved_trap_no; + /* * Task has received a fatal signal, so reset back to probed * address. diff --git a/arch/csky/mm/fault.c b/arch/csky/mm/fault.c index ae9781b7d92e..a885518ce1dd 100644 --- a/arch/csky/mm/fault.c +++ b/arch/csky/mm/fault.c @@ -237,7 +237,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs) if (is_write(regs)) flags |= FAULT_FLAG_WRITE; retry: - vma = lock_mm_and_find_vma(mm, address, regs); + vma = lock_mm_and_find_vma(mm, addr, regs); if (unlikely(!vma)) { bad_area_nosemaphore(regs, mm, code, addr); return; diff --git a/arch/ia64/include/asm/Kbuild b/arch/ia64/include/asm/Kbuild index aefae2efde9f..33733245f42b 100644 --- a/arch/ia64/include/asm/Kbuild +++ b/arch/ia64/include/asm/Kbuild @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 generated-y += syscall_table.h generic-y += agp.h +generic-y += export.h generic-y += kvm_para.h generic-y += mcs_spinlock.h generic-y += vtime.h diff --git a/arch/ia64/include/asm/export.h b/arch/ia64/include/asm/export.h deleted file mode 100644 index ad18c6583252..000000000000 --- a/arch/ia64/include/asm/export.h +++ /dev/null @@ -1,3 +0,0 @@ -/* EXPORT_DATA_SYMBOL != EXPORT_SYMBOL here */ -#define KSYM_FUNC(name) @fptr(name) -#include <asm-generic/export.h> diff --git a/arch/ia64/kernel/head.S b/arch/ia64/kernel/head.S index f22469f1c1fc..c096500590e9 100644 --- a/arch/ia64/kernel/head.S +++ b/arch/ia64/kernel/head.S @@ -170,7 +170,7 @@ RestRR: \ __PAGE_ALIGNED_DATA .global empty_zero_page -EXPORT_DATA_SYMBOL_GPL(empty_zero_page) +EXPORT_SYMBOL_GPL(empty_zero_page) empty_zero_page: .skip PAGE_SIZE diff --git a/arch/ia64/kernel/ivt.S b/arch/ia64/kernel/ivt.S index d6d4229b28db..7a418e324d30 100644 --- a/arch/ia64/kernel/ivt.S +++ b/arch/ia64/kernel/ivt.S @@ -87,7 +87,7 @@ .align 32768 // align on 32KB boundary .global ia64_ivt - EXPORT_DATA_SYMBOL(ia64_ivt) + EXPORT_SYMBOL(ia64_ivt) ia64_ivt: ///////////////////////////////////////////////////////////////////////////////////////// // 0x0000 Entry 0 (size 64 bundles) VHPT Translation (8,20,47) diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 32b9da4622ce..e55511af4c77 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -5,6 +5,7 @@ config LOONGARCH select ACPI select ACPI_GENERIC_GSI if ACPI select ACPI_MCFG if ACPI + select ACPI_PPTT if ACPI select ACPI_SYSTEM_POWER_STATES_SUPPORT if ACPI select ARCH_BINFMT_ELF_STATE select ARCH_ENABLE_MEMORY_HOTPLUG @@ -49,6 +50,8 @@ config LOONGARCH select ARCH_SUPPORTS_ACPI select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_HUGETLBFS + select ARCH_SUPPORTS_LTO_CLANG + select ARCH_SUPPORTS_LTO_CLANG_THIN select ARCH_SUPPORTS_NUMA_BALANCING select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF @@ -81,9 +84,12 @@ config LOONGARCH select GENERIC_SCHED_CLOCK select GENERIC_SMP_IDLE_THREAD select GENERIC_TIME_VSYSCALL + select GENERIC_VDSO_TIME_NS select GPIOLIB select HAS_IOPORT select HAVE_ARCH_AUDITSYSCALL + select HAVE_ARCH_JUMP_LABEL + select HAVE_ARCH_JUMP_LABEL_RELATIVE select HAVE_ARCH_MMAP_RND_BITS if MMU select HAVE_ARCH_SECCOMP_FILTER select HAVE_ARCH_TRACEHOOK @@ -91,6 +97,7 @@ config LOONGARCH select HAVE_ASM_MODVERSIONS select HAVE_CONTEXT_TRACKING_USER select HAVE_C_RECORDMCOUNT + select HAVE_DEBUG_KMEMLEAK select HAVE_DEBUG_STACKOVERFLOW select HAVE_DMA_CONTIGUOUS select HAVE_DYNAMIC_FTRACE @@ -104,6 +111,7 @@ config LOONGARCH select HAVE_FTRACE_MCOUNT_RECORD select HAVE_FUNCTION_ARG_ACCESS_API select HAVE_FUNCTION_ERROR_INJECTION + select HAVE_FUNCTION_GRAPH_RETVAL if HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_TRACER select HAVE_GENERIC_VDSO @@ -121,6 +129,7 @@ config LOONGARCH select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select HAVE_REGS_AND_STACK_ACCESS_API + select HAVE_RETHOOK select HAVE_RSEQ select HAVE_SAMPLE_FTRACE_DIRECT select HAVE_SAMPLE_FTRACE_DIRECT_MULTI @@ -163,14 +172,6 @@ config 32BIT config 64BIT def_bool y -config CPU_HAS_FPU - bool - default y - -config CPU_HAS_PREFETCH - bool - default y - config GENERIC_BUG def_bool y depends on BUG @@ -243,6 +244,15 @@ config SCHED_OMIT_FRAME_POINTER config AS_HAS_EXPLICIT_RELOCS def_bool $(as-instr,x:pcalau12i \$t0$(comma)%pc_hi20(x)) +config AS_HAS_FCSR_CLASS + def_bool $(as-instr,movfcsr2gr \$t0$(comma)\$fcsr0) + +config AS_HAS_LSX_EXTENSION + def_bool $(as-instr,vld \$vr0$(comma)\$a0$(comma)0) + +config AS_HAS_LASX_EXTENSION + def_bool $(as-instr,xvld \$xr0$(comma)\$a0$(comma)0) + menu "Kernel type and options" source "kernel/Kconfig.hz" @@ -374,6 +384,13 @@ config EFI_STUB This kernel feature allows the kernel to be loaded directly by EFI firmware without the use of a bootloader. +config SCHED_SMT + bool "SMT scheduler support" + default y + help + Improves scheduler's performance when there are multiple + threads in one physical core. + config SMP bool "Multi-Processing support" help @@ -483,6 +500,43 @@ config ARCH_STRICT_ALIGN to run kernel only on systems with h/w unaligned access support in order to optimise for performance. +config CPU_HAS_FPU + bool + default y + +config CPU_HAS_LSX + bool "Support for the Loongson SIMD Extension" + depends on AS_HAS_LSX_EXTENSION + help + Loongson SIMD Extension (LSX) introduces 128 bit wide vector registers + and a set of SIMD instructions to operate on them. When this option + is enabled the kernel will support allocating & switching LSX + vector register contexts. If you know that your kernel will only be + running on CPUs which do not support LSX or that your userland will + not be making use of it then you may wish to say N here to reduce + the size & complexity of your kernel. + + If unsure, say Y. + +config CPU_HAS_LASX + bool "Support for the Loongson Advanced SIMD Extension" + depends on CPU_HAS_LSX + depends on AS_HAS_LASX_EXTENSION + help + Loongson Advanced SIMD Extension (LASX) introduces 256 bit wide vector + registers and a set of SIMD instructions to operate on them. When this + option is enabled the kernel will support allocating & switching LASX + vector register contexts. If you know that your kernel will only be + running on CPUs which do not support LASX or that your userland will + not be making use of it then you may wish to say N here to reduce + the size & complexity of your kernel. + + If unsure, say Y. + +config CPU_HAS_PREFETCH + bool + default y + config KEXEC bool "Kexec system call" select KEXEC_CORE @@ -592,6 +646,9 @@ config ARCH_MMAP_RND_BITS_MIN config ARCH_MMAP_RND_BITS_MAX default 18 +config ARCH_SUPPORTS_UPROBES + def_bool y + menu "Power management options" config ARCH_SUSPEND_POSSIBLE diff --git a/arch/loongarch/Makefile b/arch/loongarch/Makefile index a27e264bdaa5..09ba338a64de 100644 --- a/arch/loongarch/Makefile +++ b/arch/loongarch/Makefile @@ -46,8 +46,8 @@ ld-emul = $(64bit-emul) cflags-y += -mabi=lp64s endif -cflags-y += -G0 -pipe -msoft-float -LDFLAGS_vmlinux += -G0 -static -n -nostdlib +cflags-y += -pipe -msoft-float +LDFLAGS_vmlinux += -static -n -nostdlib # When the assembler supports explicit relocation hint, we must use it. # GCC may have -mexplicit-relocs off by default if it was built with an old @@ -56,13 +56,18 @@ LDFLAGS_vmlinux += -G0 -static -n -nostdlib # When the assembler does not supports explicit relocation hint, we can't use # it. Disable it if the compiler supports it. # -# If you've seen "unknown reloc hint" message building the kernel and you are -# now wondering why "-mexplicit-relocs" is not wrapped with cc-option: the -# combination of a "new" assembler and "old" compiler is not supported. Either -# upgrade the compiler or downgrade the assembler. +# The combination of a "new" assembler and "old" GCC is not supported, given +# the rarity of this combo and the extra complexity needed to make it work. +# Either upgrade the compiler or downgrade the assembler; the build will error +# out if it is the case (by probing for the model attribute; all supported +# compilers in this case would have support). +# +# Also, -mdirect-extern-access is useful in case of building with explicit +# relocs, for avoiding unnecessary GOT accesses. It is harmless to not have +# support though. ifdef CONFIG_AS_HAS_EXPLICIT_RELOCS -cflags-y += -mexplicit-relocs -KBUILD_CFLAGS_KERNEL += -mdirect-extern-access +cflags-y += $(call cc-option,-mexplicit-relocs) +KBUILD_CFLAGS_KERNEL += $(call cc-option,-mdirect-extern-access) else cflags-y += $(call cc-option,-mno-explicit-relocs) KBUILD_AFLAGS_KERNEL += -Wa,-mla-global-with-pcrel @@ -107,7 +112,7 @@ KBUILD_CFLAGS += -isystem $(shell $(CC) -print-file-name=include) KBUILD_LDFLAGS += -m $(ld-emul) ifdef CONFIG_LOONGARCH -CHECKFLAGS += $(shell $(CC) $(KBUILD_CFLAGS) -dM -E -x c /dev/null | \ +CHECKFLAGS += $(shell $(CC) $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS) -dM -E -x c /dev/null | \ grep -E -vw '__GNUC_(MINOR_|PATCHLEVEL_)?_' | \ sed -e "s/^\#define /-D'/" -e "s/ /'='/" -e "s/$$/'/" -e 's/\$$/&&/g') endif diff --git a/arch/loongarch/include/asm/Kbuild b/arch/loongarch/include/asm/Kbuild index 77ad8e6f0906..6b222f227342 100644 --- a/arch/loongarch/include/asm/Kbuild +++ b/arch/loongarch/include/asm/Kbuild @@ -5,7 +5,6 @@ generic-y += mcs_spinlock.h generic-y += parport.h generic-y += early_ioremap.h generic-y += qrwlock.h -generic-y += qspinlock.h generic-y += rwsem.h generic-y += segment.h generic-y += user.h diff --git a/arch/loongarch/include/asm/acpi.h b/arch/loongarch/include/asm/acpi.h index 976a810352c6..8de6c4b83a61 100644 --- a/arch/loongarch/include/asm/acpi.h +++ b/arch/loongarch/include/asm/acpi.h @@ -8,11 +8,14 @@ #ifndef _ASM_LOONGARCH_ACPI_H #define _ASM_LOONGARCH_ACPI_H +#include <asm/suspend.h> + #ifdef CONFIG_ACPI extern int acpi_strict; extern int acpi_disabled; extern int acpi_pci_disabled; extern int acpi_noirq; +extern int pptt_enabled; #define acpi_os_ioremap acpi_os_ioremap void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size); @@ -30,6 +33,14 @@ static inline bool acpi_has_cpu_in_madt(void) } extern struct list_head acpi_wakeup_device_list; +extern struct acpi_madt_core_pic acpi_core_pic[NR_CPUS]; + +extern int __init parse_acpi_topology(void); + +static inline u32 get_acpi_id_for_cpu(unsigned int cpu) +{ + return acpi_core_pic[cpu_logical_map(cpu)].processor_id; +} #endif /* !CONFIG_ACPI */ @@ -37,12 +48,10 @@ extern struct list_head acpi_wakeup_device_list; extern int loongarch_acpi_suspend(void); extern int (*acpi_suspend_lowlevel)(void); -extern void loongarch_suspend_enter(void); static inline unsigned long acpi_get_wakeup_address(void) { #ifdef CONFIG_SUSPEND - extern void loongarch_wakeup_start(void); return (unsigned long)loongarch_wakeup_start; #endif return 0UL; diff --git a/arch/loongarch/include/asm/asmmacro.h b/arch/loongarch/include/asm/asmmacro.h index c51a1b43acb4..79e1d53fea89 100644 --- a/arch/loongarch/include/asm/asmmacro.h +++ b/arch/loongarch/include/asm/asmmacro.h @@ -270,6 +270,399 @@ fld.d $f31, \tmp, THREAD_FPR31 - THREAD_FPR0 .endm + .macro lsx_save_data thread tmp + li.w \tmp, THREAD_FPR0 + PTR_ADD \tmp, \thread, \tmp + vst $vr0, \tmp, THREAD_FPR0 - THREAD_FPR0 + vst $vr1, \tmp, THREAD_FPR1 - THREAD_FPR0 + vst $vr2, \tmp, THREAD_FPR2 - THREAD_FPR0 + vst $vr3, \tmp, THREAD_FPR3 - THREAD_FPR0 + vst $vr4, \tmp, THREAD_FPR4 - THREAD_FPR0 + vst $vr5, \tmp, THREAD_FPR5 - THREAD_FPR0 + vst $vr6, \tmp, THREAD_FPR6 - THREAD_FPR0 + vst $vr7, \tmp, THREAD_FPR7 - THREAD_FPR0 + vst $vr8, \tmp, THREAD_FPR8 - THREAD_FPR0 + vst $vr9, \tmp, THREAD_FPR9 - THREAD_FPR0 + vst $vr10, \tmp, THREAD_FPR10 - THREAD_FPR0 + vst $vr11, \tmp, THREAD_FPR11 - THREAD_FPR0 + vst $vr12, \tmp, THREAD_FPR12 - THREAD_FPR0 + vst $vr13, \tmp, THREAD_FPR13 - THREAD_FPR0 + vst $vr14, \tmp, THREAD_FPR14 - THREAD_FPR0 + vst $vr15, \tmp, THREAD_FPR15 - THREAD_FPR0 + vst $vr16, \tmp, THREAD_FPR16 - THREAD_FPR0 + vst $vr17, \tmp, THREAD_FPR17 - THREAD_FPR0 + vst $vr18, \tmp, THREAD_FPR18 - THREAD_FPR0 + vst $vr19, \tmp, THREAD_FPR19 - THREAD_FPR0 + vst $vr20, \tmp, THREAD_FPR20 - THREAD_FPR0 + vst $vr21, \tmp, THREAD_FPR21 - THREAD_FPR0 + vst $vr22, \tmp, THREAD_FPR22 - THREAD_FPR0 + vst $vr23, \tmp, THREAD_FPR23 - THREAD_FPR0 + vst $vr24, \tmp, THREAD_FPR24 - THREAD_FPR0 + vst $vr25, \tmp, THREAD_FPR25 - THREAD_FPR0 + vst $vr26, \tmp, THREAD_FPR26 - THREAD_FPR0 + vst $vr27, \tmp, THREAD_FPR27 - THREAD_FPR0 + vst $vr28, \tmp, THREAD_FPR28 - THREAD_FPR0 + vst $vr29, \tmp, THREAD_FPR29 - THREAD_FPR0 + vst $vr30, \tmp, THREAD_FPR30 - THREAD_FPR0 + vst $vr31, \tmp, THREAD_FPR31 - THREAD_FPR0 + .endm + + .macro lsx_restore_data thread tmp + li.w \tmp, THREAD_FPR0 + PTR_ADD \tmp, \thread, \tmp + vld $vr0, \tmp, THREAD_FPR0 - THREAD_FPR0 + vld $vr1, \tmp, THREAD_FPR1 - THREAD_FPR0 + vld $vr2, \tmp, THREAD_FPR2 - THREAD_FPR0 + vld $vr3, \tmp, THREAD_FPR3 - THREAD_FPR0 + vld $vr4, \tmp, THREAD_FPR4 - THREAD_FPR0 + vld $vr5, \tmp, THREAD_FPR5 - THREAD_FPR0 + vld $vr6, \tmp, THREAD_FPR6 - THREAD_FPR0 + vld $vr7, \tmp, THREAD_FPR7 - THREAD_FPR0 + vld $vr8, \tmp, THREAD_FPR8 - THREAD_FPR0 + vld $vr9, \tmp, THREAD_FPR9 - THREAD_FPR0 + vld $vr10, \tmp, THREAD_FPR10 - THREAD_FPR0 + vld $vr11, \tmp, THREAD_FPR11 - THREAD_FPR0 + vld $vr12, \tmp, THREAD_FPR12 - THREAD_FPR0 + vld $vr13, \tmp, THREAD_FPR13 - THREAD_FPR0 + vld $vr14, \tmp, THREAD_FPR14 - THREAD_FPR0 + vld $vr15, \tmp, THREAD_FPR15 - THREAD_FPR0 + vld $vr16, \tmp, THREAD_FPR16 - THREAD_FPR0 + vld $vr17, \tmp, THREAD_FPR17 - THREAD_FPR0 + vld $vr18, \tmp, THREAD_FPR18 - THREAD_FPR0 + vld $vr19, \tmp, THREAD_FPR19 - THREAD_FPR0 + vld $vr20, \tmp, THREAD_FPR20 - THREAD_FPR0 + vld $vr21, \tmp, THREAD_FPR21 - THREAD_FPR0 + vld $vr22, \tmp, THREAD_FPR22 - THREAD_FPR0 + vld $vr23, \tmp, THREAD_FPR23 - THREAD_FPR0 + vld $vr24, \tmp, THREAD_FPR24 - THREAD_FPR0 + vld $vr25, \tmp, THREAD_FPR25 - THREAD_FPR0 + vld $vr26, \tmp, THREAD_FPR26 - THREAD_FPR0 + vld $vr27, \tmp, THREAD_FPR27 - THREAD_FPR0 + vld $vr28, \tmp, THREAD_FPR28 - THREAD_FPR0 + vld $vr29, \tmp, THREAD_FPR29 - THREAD_FPR0 + vld $vr30, \tmp, THREAD_FPR30 - THREAD_FPR0 + vld $vr31, \tmp, THREAD_FPR31 - THREAD_FPR0 + .endm + + .macro lsx_save_all thread tmp0 tmp1 + fpu_save_cc \thread, \tmp0, \tmp1 + fpu_save_csr \thread, \tmp0 + lsx_save_data \thread, \tmp0 + .endm + + .macro lsx_restore_all thread tmp0 tmp1 + lsx_restore_data \thread, \tmp0 + fpu_restore_cc \thread, \tmp0, \tmp1 + fpu_restore_csr \thread, \tmp0 + .endm + + .macro lsx_save_upper vd base tmp off + vpickve2gr.d \tmp, \vd, 1 + st.d \tmp, \base, (\off+8) + .endm + + .macro lsx_save_all_upper thread base tmp + li.w \tmp, THREAD_FPR0 + PTR_ADD \base, \thread, \tmp + lsx_save_upper $vr0, \base, \tmp, (THREAD_FPR0-THREAD_FPR0) + lsx_save_upper $vr1, \base, \tmp, (THREAD_FPR1-THREAD_FPR0) + lsx_save_upper $vr2, \base, \tmp, (THREAD_FPR2-THREAD_FPR0) + lsx_save_upper $vr3, \base, \tmp, (THREAD_FPR3-THREAD_FPR0) + lsx_save_upper $vr4, \base, \tmp, (THREAD_FPR4-THREAD_FPR0) + lsx_save_upper $vr5, \base, \tmp, (THREAD_FPR5-THREAD_FPR0) + lsx_save_upper $vr6, \base, \tmp, (THREAD_FPR6-THREAD_FPR0) + lsx_save_upper $vr7, \base, \tmp, (THREAD_FPR7-THREAD_FPR0) + lsx_save_upper $vr8, \base, \tmp, (THREAD_FPR8-THREAD_FPR0) + lsx_save_upper $vr9, \base, \tmp, (THREAD_FPR9-THREAD_FPR0) + lsx_save_upper $vr10, \base, \tmp, (THREAD_FPR10-THREAD_FPR0) + lsx_save_upper $vr11, \base, \tmp, (THREAD_FPR11-THREAD_FPR0) + lsx_save_upper $vr12, \base, \tmp, (THREAD_FPR12-THREAD_FPR0) + lsx_save_upper $vr13, \base, \tmp, (THREAD_FPR13-THREAD_FPR0) + lsx_save_upper $vr14, \base, \tmp, (THREAD_FPR14-THREAD_FPR0) + lsx_save_upper $vr15, \base, \tmp, (THREAD_FPR15-THREAD_FPR0) + lsx_save_upper $vr16, \base, \tmp, (THREAD_FPR16-THREAD_FPR0) + lsx_save_upper $vr17, \base, \tmp, (THREAD_FPR17-THREAD_FPR0) + lsx_save_upper $vr18, \base, \tmp, (THREAD_FPR18-THREAD_FPR0) + lsx_save_upper $vr19, \base, \tmp, (THREAD_FPR19-THREAD_FPR0) + lsx_save_upper $vr20, \base, \tmp, (THREAD_FPR20-THREAD_FPR0) + lsx_save_upper $vr21, \base, \tmp, (THREAD_FPR21-THREAD_FPR0) + lsx_save_upper $vr22, \base, \tmp, (THREAD_FPR22-THREAD_FPR0) + lsx_save_upper $vr23, \base, \tmp, (THREAD_FPR23-THREAD_FPR0) + lsx_save_upper $vr24, \base, \tmp, (THREAD_FPR24-THREAD_FPR0) + lsx_save_upper $vr25, \base, \tmp, (THREAD_FPR25-THREAD_FPR0) + lsx_save_upper $vr26, \base, \tmp, (THREAD_FPR26-THREAD_FPR0) + lsx_save_upper $vr27, \base, \tmp, (THREAD_FPR27-THREAD_FPR0) + lsx_save_upper $vr28, \base, \tmp, (THREAD_FPR28-THREAD_FPR0) + lsx_save_upper $vr29, \base, \tmp, (THREAD_FPR29-THREAD_FPR0) + lsx_save_upper $vr30, \base, \tmp, (THREAD_FPR30-THREAD_FPR0) + lsx_save_upper $vr31, \base, \tmp, (THREAD_FPR31-THREAD_FPR0) + .endm + + .macro lsx_restore_upper vd base tmp off + ld.d \tmp, \base, (\off+8) + vinsgr2vr.d \vd, \tmp, 1 + .endm + + .macro lsx_restore_all_upper thread base tmp + li.w \tmp, THREAD_FPR0 + PTR_ADD \base, \thread, \tmp + lsx_restore_upper $vr0, \base, \tmp, (THREAD_FPR0-THREAD_FPR0) + lsx_restore_upper $vr1, \base, \tmp, (THREAD_FPR1-THREAD_FPR0) + lsx_restore_upper $vr2, \base, \tmp, (THREAD_FPR2-THREAD_FPR0) + lsx_restore_upper $vr3, \base, \tmp, (THREAD_FPR3-THREAD_FPR0) + lsx_restore_upper $vr4, \base, \tmp, (THREAD_FPR4-THREAD_FPR0) + lsx_restore_upper $vr5, \base, \tmp, (THREAD_FPR5-THREAD_FPR0) + lsx_restore_upper $vr6, \base, \tmp, (THREAD_FPR6-THREAD_FPR0) + lsx_restore_upper $vr7, \base, \tmp, (THREAD_FPR7-THREAD_FPR0) + lsx_restore_upper $vr8, \base, \tmp, (THREAD_FPR8-THREAD_FPR0) + lsx_restore_upper $vr9, \base, \tmp, (THREAD_FPR9-THREAD_FPR0) + lsx_restore_upper $vr10, \base, \tmp, (THREAD_FPR10-THREAD_FPR0) + lsx_restore_upper $vr11, \base, \tmp, (THREAD_FPR11-THREAD_FPR0) + lsx_restore_upper $vr12, \base, \tmp, (THREAD_FPR12-THREAD_FPR0) + lsx_restore_upper $vr13, \base, \tmp, (THREAD_FPR13-THREAD_FPR0) + lsx_restore_upper $vr14, \base, \tmp, (THREAD_FPR14-THREAD_FPR0) + lsx_restore_upper $vr15, \base, \tmp, (THREAD_FPR15-THREAD_FPR0) + lsx_restore_upper $vr16, \base, \tmp, (THREAD_FPR16-THREAD_FPR0) + lsx_restore_upper $vr17, \base, \tmp, (THREAD_FPR17-THREAD_FPR0) + lsx_restore_upper $vr18, \base, \tmp, (THREAD_FPR18-THREAD_FPR0) + lsx_restore_upper $vr19, \base, \tmp, (THREAD_FPR19-THREAD_FPR0) + lsx_restore_upper $vr20, \base, \tmp, (THREAD_FPR20-THREAD_FPR0) + lsx_restore_upper $vr21, \base, \tmp, (THREAD_FPR21-THREAD_FPR0) + lsx_restore_upper $vr22, \base, \tmp, (THREAD_FPR22-THREAD_FPR0) + lsx_restore_upper $vr23, \base, \tmp, (THREAD_FPR23-THREAD_FPR0) + lsx_restore_upper $vr24, \base, \tmp, (THREAD_FPR24-THREAD_FPR0) + lsx_restore_upper $vr25, \base, \tmp, (THREAD_FPR25-THREAD_FPR0) + lsx_restore_upper $vr26, \base, \tmp, (THREAD_FPR26-THREAD_FPR0) + lsx_restore_upper $vr27, \base, \tmp, (THREAD_FPR27-THREAD_FPR0) + lsx_restore_upper $vr28, \base, \tmp, (THREAD_FPR28-THREAD_FPR0) + lsx_restore_upper $vr29, \base, \tmp, (THREAD_FPR29-THREAD_FPR0) + lsx_restore_upper $vr30, \base, \tmp, (THREAD_FPR30-THREAD_FPR0) + lsx_restore_upper $vr31, \base, \tmp, (THREAD_FPR31-THREAD_FPR0) + .endm + + .macro lsx_init_upper vd tmp + vinsgr2vr.d \vd, \tmp, 1 + .endm + + .macro lsx_init_all_upper tmp + not \tmp, zero + lsx_init_upper $vr0 \tmp + lsx_init_upper $vr1 \tmp + lsx_init_upper $vr2 \tmp + lsx_init_upper $vr3 \tmp + lsx_init_upper $vr4 \tmp + lsx_init_upper $vr5 \tmp + lsx_init_upper $vr6 \tmp + lsx_init_upper $vr7 \tmp + lsx_init_upper $vr8 \tmp + lsx_init_upper $vr9 \tmp + lsx_init_upper $vr10 \tmp + lsx_init_upper $vr11 \tmp + lsx_init_upper $vr12 \tmp + lsx_init_upper $vr13 \tmp + lsx_init_upper $vr14 \tmp + lsx_init_upper $vr15 \tmp + lsx_init_upper $vr16 \tmp + lsx_init_upper $vr17 \tmp + lsx_init_upper $vr18 \tmp + lsx_init_upper $vr19 \tmp + lsx_init_upper $vr20 \tmp + lsx_init_upper $vr21 \tmp + lsx_init_upper $vr22 \tmp + lsx_init_upper $vr23 \tmp + lsx_init_upper $vr24 \tmp + lsx_init_upper $vr25 \tmp + lsx_init_upper $vr26 \tmp + lsx_init_upper $vr27 \tmp + lsx_init_upper $vr28 \tmp + lsx_init_upper $vr29 \tmp + lsx_init_upper $vr30 \tmp + lsx_init_upper $vr31 \tmp + .endm + + .macro lasx_save_data thread tmp + li.w \tmp, THREAD_FPR0 + PTR_ADD \tmp, \thread, \tmp + xvst $xr0, \tmp, THREAD_FPR0 - THREAD_FPR0 + xvst $xr1, \tmp, THREAD_FPR1 - THREAD_FPR0 + xvst $xr2, \tmp, THREAD_FPR2 - THREAD_FPR0 + xvst $xr3, \tmp, THREAD_FPR3 - THREAD_FPR0 + xvst $xr4, \tmp, THREAD_FPR4 - THREAD_FPR0 + xvst $xr5, \tmp, THREAD_FPR5 - THREAD_FPR0 + xvst $xr6, \tmp, THREAD_FPR6 - THREAD_FPR0 + xvst $xr7, \tmp, THREAD_FPR7 - THREAD_FPR0 + xvst $xr8, \tmp, THREAD_FPR8 - THREAD_FPR0 + xvst $xr9, \tmp, THREAD_FPR9 - THREAD_FPR0 + xvst $xr10, \tmp, THREAD_FPR10 - THREAD_FPR0 + xvst $xr11, \tmp, THREAD_FPR11 - THREAD_FPR0 + xvst $xr12, \tmp, THREAD_FPR12 - THREAD_FPR0 + xvst $xr13, \tmp, THREAD_FPR13 - THREAD_FPR0 + xvst $xr14, \tmp, THREAD_FPR14 - THREAD_FPR0 + xvst $xr15, \tmp, THREAD_FPR15 - THREAD_FPR0 + xvst $xr16, \tmp, THREAD_FPR16 - THREAD_FPR0 + xvst $xr17, \tmp, THREAD_FPR17 - THREAD_FPR0 + xvst $xr18, \tmp, THREAD_FPR18 - THREAD_FPR0 + xvst $xr19, \tmp, THREAD_FPR19 - THREAD_FPR0 + xvst $xr20, \tmp, THREAD_FPR20 - THREAD_FPR0 + xvst $xr21, \tmp, THREAD_FPR21 - THREAD_FPR0 + xvst $xr22, \tmp, THREAD_FPR22 - THREAD_FPR0 + xvst $xr23, \tmp, THREAD_FPR23 - THREAD_FPR0 + xvst $xr24, \tmp, THREAD_FPR24 - THREAD_FPR0 + xvst $xr25, \tmp, THREAD_FPR25 - THREAD_FPR0 + xvst $xr26, \tmp, THREAD_FPR26 - THREAD_FPR0 + xvst $xr27, \tmp, THREAD_FPR27 - THREAD_FPR0 + xvst $xr28, \tmp, THREAD_FPR28 - THREAD_FPR0 + xvst $xr29, \tmp, THREAD_FPR29 - THREAD_FPR0 + xvst $xr30, \tmp, THREAD_FPR30 - THREAD_FPR0 + xvst $xr31, \tmp, THREAD_FPR31 - THREAD_FPR0 + .endm + + .macro lasx_restore_data thread tmp + li.w \tmp, THREAD_FPR0 + PTR_ADD \tmp, \thread, \tmp + xvld $xr0, \tmp, THREAD_FPR0 - THREAD_FPR0 + xvld $xr1, \tmp, THREAD_FPR1 - THREAD_FPR0 + xvld $xr2, \tmp, THREAD_FPR2 - THREAD_FPR0 + xvld $xr3, \tmp, THREAD_FPR3 - THREAD_FPR0 + xvld $xr4, \tmp, THREAD_FPR4 - THREAD_FPR0 + xvld $xr5, \tmp, THREAD_FPR5 - THREAD_FPR0 + xvld $xr6, \tmp, THREAD_FPR6 - THREAD_FPR0 + xvld $xr7, \tmp, THREAD_FPR7 - THREAD_FPR0 + xvld $xr8, \tmp, THREAD_FPR8 - THREAD_FPR0 + xvld $xr9, \tmp, THREAD_FPR9 - THREAD_FPR0 + xvld $xr10, \tmp, THREAD_FPR10 - THREAD_FPR0 + xvld $xr11, \tmp, THREAD_FPR11 - THREAD_FPR0 + xvld $xr12, \tmp, THREAD_FPR12 - THREAD_FPR0 + xvld $xr13, \tmp, THREAD_FPR13 - THREAD_FPR0 + xvld $xr14, \tmp, THREAD_FPR14 - THREAD_FPR0 + xvld $xr15, \tmp, THREAD_FPR15 - THREAD_FPR0 + xvld $xr16, \tmp, THREAD_FPR16 - THREAD_FPR0 + xvld $xr17, \tmp, THREAD_FPR17 - THREAD_FPR0 + xvld $xr18, \tmp, THREAD_FPR18 - THREAD_FPR0 + xvld $xr19, \tmp, THREAD_FPR19 - THREAD_FPR0 + xvld $xr20, \tmp, THREAD_FPR20 - THREAD_FPR0 + xvld $xr21, \tmp, THREAD_FPR21 - THREAD_FPR0 + xvld $xr22, \tmp, THREAD_FPR22 - THREAD_FPR0 + xvld $xr23, \tmp, THREAD_FPR23 - THREAD_FPR0 + xvld $xr24, \tmp, THREAD_FPR24 - THREAD_FPR0 + xvld $xr25, \tmp, THREAD_FPR25 - THREAD_FPR0 + xvld $xr26, \tmp, THREAD_FPR26 - THREAD_FPR0 + xvld $xr27, \tmp, THREAD_FPR27 - THREAD_FPR0 + xvld $xr28, \tmp, THREAD_FPR28 - THREAD_FPR0 + xvld $xr29, \tmp, THREAD_FPR29 - THREAD_FPR0 + xvld $xr30, \tmp, THREAD_FPR30 - THREAD_FPR0 + xvld $xr31, \tmp, THREAD_FPR31 - THREAD_FPR0 + .endm + + .macro lasx_save_all thread tmp0 tmp1 + fpu_save_cc \thread, \tmp0, \tmp1 + fpu_save_csr \thread, \tmp0 + lasx_save_data \thread, \tmp0 + .endm + + .macro lasx_restore_all thread tmp0 tmp1 + lasx_restore_data \thread, \tmp0 + fpu_restore_cc \thread, \tmp0, \tmp1 + fpu_restore_csr \thread, \tmp0 + .endm + + .macro lasx_save_upper xd base tmp off + /* Nothing */ + .endm + + .macro lasx_save_all_upper thread base tmp + /* Nothing */ + .endm + + .macro lasx_restore_upper xd base tmp0 tmp1 off + vld \tmp0, \base, (\off+16) + xvpermi.q \xd, \tmp1, 0x2 + .endm + + .macro lasx_restore_all_upper thread base tmp + li.w \tmp, THREAD_FPR0 + PTR_ADD \base, \thread, \tmp + /* Save $vr31 ($xr31 lower bits) with xvpickve2gr */ + xvpickve2gr.d $r17, $xr31, 0 + xvpickve2gr.d $r18, $xr31, 1 + lasx_restore_upper $xr0, \base, $vr31, $xr31, (THREAD_FPR0-THREAD_FPR0) + lasx_restore_upper $xr1, \base, $vr31, $xr31, (THREAD_FPR1-THREAD_FPR0) + lasx_restore_upper $xr2, \base, $vr31, $xr31, (THREAD_FPR2-THREAD_FPR0) + lasx_restore_upper $xr3, \base, $vr31, $xr31, (THREAD_FPR3-THREAD_FPR0) + lasx_restore_upper $xr4, \base, $vr31, $xr31, (THREAD_FPR4-THREAD_FPR0) + lasx_restore_upper $xr5, \base, $vr31, $xr31, (THREAD_FPR5-THREAD_FPR0) + lasx_restore_upper $xr6, \base, $vr31, $xr31, (THREAD_FPR6-THREAD_FPR0) + lasx_restore_upper $xr7, \base, $vr31, $xr31, (THREAD_FPR7-THREAD_FPR0) + lasx_restore_upper $xr8, \base, $vr31, $xr31, (THREAD_FPR8-THREAD_FPR0) + lasx_restore_upper $xr9, \base, $vr31, $xr31, (THREAD_FPR9-THREAD_FPR0) + lasx_restore_upper $xr10, \base, $vr31, $xr31, (THREAD_FPR10-THREAD_FPR0) + lasx_restore_upper $xr11, \base, $vr31, $xr31, (THREAD_FPR11-THREAD_FPR0) + lasx_restore_upper $xr12, \base, $vr31, $xr31, (THREAD_FPR12-THREAD_FPR0) + lasx_restore_upper $xr13, \base, $vr31, $xr31, (THREAD_FPR13-THREAD_FPR0) + lasx_restore_upper $xr14, \base, $vr31, $xr31, (THREAD_FPR14-THREAD_FPR0) + lasx_restore_upper $xr15, \base, $vr31, $xr31, (THREAD_FPR15-THREAD_FPR0) + lasx_restore_upper $xr16, \base, $vr31, $xr31, (THREAD_FPR16-THREAD_FPR0) + lasx_restore_upper $xr17, \base, $vr31, $xr31, (THREAD_FPR17-THREAD_FPR0) + lasx_restore_upper $xr18, \base, $vr31, $xr31, (THREAD_FPR18-THREAD_FPR0) + lasx_restore_upper $xr19, \base, $vr31, $xr31, (THREAD_FPR19-THREAD_FPR0) + lasx_restore_upper $xr20, \base, $vr31, $xr31, (THREAD_FPR20-THREAD_FPR0) + lasx_restore_upper $xr21, \base, $vr31, $xr31, (THREAD_FPR21-THREAD_FPR0) + lasx_restore_upper $xr22, \base, $vr31, $xr31, (THREAD_FPR22-THREAD_FPR0) + lasx_restore_upper $xr23, \base, $vr31, $xr31, (THREAD_FPR23-THREAD_FPR0) + lasx_restore_upper $xr24, \base, $vr31, $xr31, (THREAD_FPR24-THREAD_FPR0) + lasx_restore_upper $xr25, \base, $vr31, $xr31, (THREAD_FPR25-THREAD_FPR0) + lasx_restore_upper $xr26, \base, $vr31, $xr31, (THREAD_FPR26-THREAD_FPR0) + lasx_restore_upper $xr27, \base, $vr31, $xr31, (THREAD_FPR27-THREAD_FPR0) + lasx_restore_upper $xr28, \base, $vr31, $xr31, (THREAD_FPR28-THREAD_FPR0) + lasx_restore_upper $xr29, \base, $vr31, $xr31, (THREAD_FPR29-THREAD_FPR0) + lasx_restore_upper $xr30, \base, $vr31, $xr31, (THREAD_FPR30-THREAD_FPR0) + lasx_restore_upper $xr31, \base, $vr31, $xr31, (THREAD_FPR31-THREAD_FPR0) + /* Restore $vr31 ($xr31 lower bits) with xvinsgr2vr */ + xvinsgr2vr.d $xr31, $r17, 0 + xvinsgr2vr.d $xr31, $r18, 1 + .endm + + .macro lasx_init_upper xd tmp + xvinsgr2vr.d \xd, \tmp, 2 + xvinsgr2vr.d \xd, \tmp, 3 + .endm + + .macro lasx_init_all_upper tmp + not \tmp, zero + lasx_init_upper $xr0 \tmp + lasx_init_upper $xr1 \tmp + lasx_init_upper $xr2 \tmp + lasx_init_upper $xr3 \tmp + lasx_init_upper $xr4 \tmp + lasx_init_upper $xr5 \tmp + lasx_init_upper $xr6 \tmp + lasx_init_upper $xr7 \tmp + lasx_init_upper $xr8 \tmp + lasx_init_upper $xr9 \tmp + lasx_init_upper $xr10 \tmp + lasx_init_upper $xr11 \tmp + lasx_init_upper $xr12 \tmp + lasx_init_upper $xr13 \tmp + lasx_init_upper $xr14 \tmp + lasx_init_upper $xr15 \tmp + lasx_init_upper $xr16 \tmp + lasx_init_upper $xr17 \tmp + lasx_init_upper $xr18 \tmp + lasx_init_upper $xr19 \tmp + lasx_init_upper $xr20 \tmp + lasx_init_upper $xr21 \tmp + lasx_init_upper $xr22 \tmp + lasx_init_upper $xr23 \tmp + lasx_init_upper $xr24 \tmp + lasx_init_upper $xr25 \tmp + lasx_init_upper $xr26 \tmp + lasx_init_upper $xr27 \tmp + lasx_init_upper $xr28 \tmp + lasx_init_upper $xr29 \tmp + lasx_init_upper $xr30 \tmp + lasx_init_upper $xr31 \tmp + .endm + .macro not dst src nor \dst, \src, zero .endm diff --git a/arch/loongarch/include/asm/barrier.h b/arch/loongarch/include/asm/barrier.h index cda977675854..4b663f197706 100644 --- a/arch/loongarch/include/asm/barrier.h +++ b/arch/loongarch/include/asm/barrier.h @@ -5,27 +5,56 @@ #ifndef __ASM_BARRIER_H #define __ASM_BARRIER_H -#define __sync() __asm__ __volatile__("dbar 0" : : : "memory") +/* + * Hint encoding: + * + * Bit4: ordering or completion (0: completion, 1: ordering) + * Bit3: barrier for previous read (0: true, 1: false) + * Bit2: barrier for previous write (0: true, 1: false) + * Bit1: barrier for succeeding read (0: true, 1: false) + * Bit0: barrier for succeeding write (0: true, 1: false) + * + * Hint 0x700: barrier for "read after read" from the same address + */ + +#define DBAR(hint) __asm__ __volatile__("dbar %0 " : : "I"(hint) : "memory") + +#define crwrw 0b00000 +#define cr_r_ 0b00101 +#define c_w_w 0b01010 -#define fast_wmb() __sync() -#define fast_rmb() __sync() -#define fast_mb() __sync() -#define fast_iob() __sync() -#define wbflush() __sync() +#define orwrw 0b10000 +#define or_r_ 0b10101 +#define o_w_w 0b11010 -#define wmb() fast_wmb() -#define rmb() fast_rmb() -#define mb() fast_mb() -#define iob() fast_iob() +#define orw_w 0b10010 +#define or_rw 0b10100 -#define __smp_mb() __asm__ __volatile__("dbar 0" : : : "memory") -#define __smp_rmb() __asm__ __volatile__("dbar 0" : : : "memory") -#define __smp_wmb() __asm__ __volatile__("dbar 0" : : : "memory") +#define c_sync() DBAR(crwrw) +#define c_rsync() DBAR(cr_r_) +#define c_wsync() DBAR(c_w_w) + +#define o_sync() DBAR(orwrw) +#define o_rsync() DBAR(or_r_) +#define o_wsync() DBAR(o_w_w) + +#define ldacq_mb() DBAR(or_rw) +#define strel_mb() DBAR(orw_w) + +#define mb() c_sync() +#define rmb() c_rsync() +#define wmb() c_wsync() +#define iob() c_sync() +#define wbflush() c_sync() + +#define __smp_mb() o_sync() +#define __smp_rmb() o_rsync() +#define __smp_wmb() o_wsync() #ifdef CONFIG_SMP -#define __WEAK_LLSC_MB " dbar 0 \n" +#define __WEAK_LLSC_MB " dbar 0x700 \n" #else -#define __WEAK_LLSC_MB " \n" +#define __WEAK_LLSC_MB " \n" #endif #define __smp_mb__before_atomic() barrier() @@ -59,68 +88,19 @@ static inline unsigned long array_index_mask_nospec(unsigned long index, return mask; } -#define __smp_load_acquire(p) \ -({ \ - union { typeof(*p) __val; char __c[1]; } __u; \ - unsigned long __tmp = 0; \ - compiletime_assert_atomic_type(*p); \ - switch (sizeof(*p)) { \ - case 1: \ - *(__u8 *)__u.__c = *(volatile __u8 *)p; \ - __smp_mb(); \ - break; \ - case 2: \ - *(__u16 *)__u.__c = *(volatile __u16 *)p; \ - __smp_mb(); \ - break; \ - case 4: \ - __asm__ __volatile__( \ - "amor_db.w %[val], %[tmp], %[mem] \n" \ - : [val] "=&r" (*(__u32 *)__u.__c) \ - : [mem] "ZB" (*(u32 *) p), [tmp] "r" (__tmp) \ - : "memory"); \ - break; \ - case 8: \ - __asm__ __volatile__( \ - "amor_db.d %[val], %[tmp], %[mem] \n" \ - : [val] "=&r" (*(__u64 *)__u.__c) \ - : [mem] "ZB" (*(u64 *) p), [tmp] "r" (__tmp) \ - : "memory"); \ - break; \ - } \ - (typeof(*p))__u.__val; \ +#define __smp_load_acquire(p) \ +({ \ + typeof(*p) ___p1 = READ_ONCE(*p); \ + compiletime_assert_atomic_type(*p); \ + ldacq_mb(); \ + ___p1; \ }) -#define __smp_store_release(p, v) \ -do { \ - union { typeof(*p) __val; char __c[1]; } __u = \ - { .__val = (__force typeof(*p)) (v) }; \ - unsigned long __tmp; \ - compiletime_assert_atomic_type(*p); \ - switch (sizeof(*p)) { \ - case 1: \ - __smp_mb(); \ - *(volatile __u8 *)p = *(__u8 *)__u.__c; \ - break; \ - case 2: \ - __smp_mb(); \ - *(volatile __u16 *)p = *(__u16 *)__u.__c; \ - break; \ - case 4: \ - __asm__ __volatile__( \ - "amswap_db.w %[tmp], %[val], %[mem] \n" \ - : [mem] "+ZB" (*(u32 *)p), [tmp] "=&r" (__tmp) \ - : [val] "r" (*(__u32 *)__u.__c) \ - : ); \ - break; \ - case 8: \ - __asm__ __volatile__( \ - "amswap_db.d %[tmp], %[val], %[mem] \n" \ - : [mem] "+ZB" (*(u64 *)p), [tmp] "=&r" (__tmp) \ - : [val] "r" (*(__u64 *)__u.__c) \ - : ); \ - break; \ - } \ +#define __smp_store_release(p, v) \ +do { \ + compiletime_assert_atomic_type(*p); \ + strel_mb(); \ + WRITE_ONCE(*p, v); \ } while (0) #define __smp_store_mb(p, v) \ diff --git a/arch/loongarch/include/asm/cpu-features.h b/arch/loongarch/include/asm/cpu-features.h index f6177f133477..2eafe6a6aca8 100644 --- a/arch/loongarch/include/asm/cpu-features.h +++ b/arch/loongarch/include/asm/cpu-features.h @@ -64,6 +64,6 @@ #define cpu_has_eiodecode cpu_opt(LOONGARCH_CPU_EIODECODE) #define cpu_has_guestid cpu_opt(LOONGARCH_CPU_GUESTID) #define cpu_has_hypervisor cpu_opt(LOONGARCH_CPU_HYPERVISOR) - +#define cpu_has_ptw cpu_opt(LOONGARCH_CPU_PTW) #endif /* __ASM_CPU_FEATURES_H */ diff --git a/arch/loongarch/include/asm/cpu-info.h b/arch/loongarch/include/asm/cpu-info.h index cd73a6f57fe3..900589cb159d 100644 --- a/arch/loongarch/include/asm/cpu-info.h +++ b/arch/loongarch/include/asm/cpu-info.h @@ -54,6 +54,7 @@ struct cpuinfo_loongarch { struct cache_desc cache_leaves[CACHE_LEAVES_MAX]; int core; /* physical core number in package */ int package;/* physical package number */ + int global_id; /* physical global thread number */ int vabits; /* Virtual Address size in bits */ int pabits; /* Physical Address size in bits */ unsigned int ksave_mask; /* Usable KSave mask. */ diff --git a/arch/loongarch/include/asm/cpu.h b/arch/loongarch/include/asm/cpu.h index 88773d849e33..48b9f7168bcc 100644 --- a/arch/loongarch/include/asm/cpu.h +++ b/arch/loongarch/include/asm/cpu.h @@ -98,6 +98,7 @@ enum cpu_type_enum { #define CPU_FEATURE_EIODECODE 23 /* CPU has EXTIOI interrupt pin decode mode */ #define CPU_FEATURE_GUESTID 24 /* CPU has GuestID feature */ #define CPU_FEATURE_HYPERVISOR 25 /* CPU has hypervisor (running in VM) */ +#define CPU_FEATURE_PTW 26 /* CPU has hardware page table walker */ #define LOONGARCH_CPU_CPUCFG BIT_ULL(CPU_FEATURE_CPUCFG) #define LOONGARCH_CPU_LAM BIT_ULL(CPU_FEATURE_LAM) @@ -125,5 +126,6 @@ enum cpu_type_enum { #define LOONGARCH_CPU_EIODECODE BIT_ULL(CPU_FEATURE_EIODECODE) #define LOONGARCH_CPU_GUESTID BIT_ULL(CPU_FEATURE_GUESTID) #define LOONGARCH_CPU_HYPERVISOR BIT_ULL(CPU_FEATURE_HYPERVISOR) +#define LOONGARCH_CPU_PTW BIT_ULL(CPU_FEATURE_PTW) #endif /* _ASM_CPU_H */ diff --git a/arch/loongarch/include/asm/fpregdef.h b/arch/loongarch/include/asm/fpregdef.h index b6be527831dd..e56610ae8592 100644 --- a/arch/loongarch/include/asm/fpregdef.h +++ b/arch/loongarch/include/asm/fpregdef.h @@ -40,6 +40,7 @@ #define fs6 $f30 #define fs7 $f31 +#ifndef CONFIG_AS_HAS_FCSR_CLASS /* * Current binutils expects *GPRs* at FCSR position for the FCSR * operation instructions, so define aliases for those used. @@ -48,5 +49,11 @@ #define fcsr1 $r1 #define fcsr2 $r2 #define fcsr3 $r3 +#else +#define fcsr0 $fcsr0 +#define fcsr1 $fcsr1 +#define fcsr2 $fcsr2 +#define fcsr3 $fcsr3 +#endif #endif /* _ASM_FPREGDEF_H */ diff --git a/arch/loongarch/include/asm/fpu.h b/arch/loongarch/include/asm/fpu.h index 192f8e35d912..e4193d637f66 100644 --- a/arch/loongarch/include/asm/fpu.h +++ b/arch/loongarch/include/asm/fpu.h @@ -28,6 +28,26 @@ extern void _init_fpu(unsigned int); extern void _save_fp(struct loongarch_fpu *); extern void _restore_fp(struct loongarch_fpu *); +extern void _save_lsx(struct loongarch_fpu *fpu); +extern void _restore_lsx(struct loongarch_fpu *fpu); +extern void _init_lsx_upper(void); +extern void _restore_lsx_upper(struct loongarch_fpu *fpu); + +extern void _save_lasx(struct loongarch_fpu *fpu); +extern void _restore_lasx(struct loongarch_fpu *fpu); +extern void _init_lasx_upper(void); +extern void _restore_lasx_upper(struct loongarch_fpu *fpu); + +static inline void enable_lsx(void); +static inline void disable_lsx(void); +static inline void save_lsx(struct task_struct *t); +static inline void restore_lsx(struct task_struct *t); + +static inline void enable_lasx(void); +static inline void disable_lasx(void); +static inline void save_lasx(struct task_struct *t); +static inline void restore_lasx(struct task_struct *t); + /* * Mask the FCSR Cause bits according to the Enable bits, observing * that Unimplemented is always enabled. @@ -44,6 +64,29 @@ static inline int is_fp_enabled(void) 1 : 0; } +static inline int is_lsx_enabled(void) +{ + if (!cpu_has_lsx) + return 0; + + return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_LSXEN) ? + 1 : 0; +} + +static inline int is_lasx_enabled(void) +{ + if (!cpu_has_lasx) + return 0; + + return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_LASXEN) ? + 1 : 0; +} + +static inline int is_simd_enabled(void) +{ + return is_lsx_enabled() | is_lasx_enabled(); +} + #define enable_fpu() set_csr_euen(CSR_EUEN_FPEN) #define disable_fpu() clear_csr_euen(CSR_EUEN_FPEN) @@ -81,9 +124,22 @@ static inline void own_fpu(int restore) static inline void lose_fpu_inatomic(int save, struct task_struct *tsk) { if (is_fpu_owner()) { - if (save) - _save_fp(&tsk->thread.fpu); - disable_fpu(); + if (!is_simd_enabled()) { + if (save) + _save_fp(&tsk->thread.fpu); + disable_fpu(); + } else { + if (save) { + if (!is_lasx_enabled()) + save_lsx(tsk); + else + save_lasx(tsk); + } + disable_fpu(); + disable_lsx(); + disable_lasx(); + clear_tsk_thread_flag(tsk, TIF_USEDSIMD); + } clear_tsk_thread_flag(tsk, TIF_USEDFPU); } KSTK_EUEN(tsk) &= ~(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN); @@ -129,4 +185,127 @@ static inline union fpureg *get_fpu_regs(struct task_struct *tsk) return tsk->thread.fpu.fpr; } +static inline int is_simd_owner(void) +{ + return test_thread_flag(TIF_USEDSIMD); +} + +#ifdef CONFIG_CPU_HAS_LSX + +static inline void enable_lsx(void) +{ + if (cpu_has_lsx) + csr_xchg32(CSR_EUEN_LSXEN, CSR_EUEN_LSXEN, LOONGARCH_CSR_EUEN); +} + +static inline void disable_lsx(void) +{ + if (cpu_has_lsx) + csr_xchg32(0, CSR_EUEN_LSXEN, LOONGARCH_CSR_EUEN); +} + +static inline void save_lsx(struct task_struct *t) +{ + if (cpu_has_lsx) + _save_lsx(&t->thread.fpu); +} + +static inline void restore_lsx(struct task_struct *t) +{ + if (cpu_has_lsx) + _restore_lsx(&t->thread.fpu); +} + +static inline void init_lsx_upper(void) +{ + /* + * Check cpu_has_lsx only if it's a constant. This will allow the + * compiler to optimise out code for CPUs without LSX without adding + * an extra redundant check for CPUs with LSX. + */ + if (__builtin_constant_p(cpu_has_lsx) && !cpu_has_lsx) + return; + + _init_lsx_upper(); +} + +static inline void restore_lsx_upper(struct task_struct *t) +{ + if (cpu_has_lsx) + _restore_lsx_upper(&t->thread.fpu); +} + +#else +static inline void enable_lsx(void) {} +static inline void disable_lsx(void) {} +static inline void save_lsx(struct task_struct *t) {} +static inline void restore_lsx(struct task_struct *t) {} +static inline void init_lsx_upper(void) {} +static inline void restore_lsx_upper(struct task_struct *t) {} +#endif + +#ifdef CONFIG_CPU_HAS_LASX + +static inline void enable_lasx(void) +{ + + if (cpu_has_lasx) + csr_xchg32(CSR_EUEN_LASXEN, CSR_EUEN_LASXEN, LOONGARCH_CSR_EUEN); +} + +static inline void disable_lasx(void) +{ + if (cpu_has_lasx) + csr_xchg32(0, CSR_EUEN_LASXEN, LOONGARCH_CSR_EUEN); +} + +static inline void save_lasx(struct task_struct *t) +{ + if (cpu_has_lasx) + _save_lasx(&t->thread.fpu); +} + +static inline void restore_lasx(struct task_struct *t) +{ + if (cpu_has_lasx) + _restore_lasx(&t->thread.fpu); +} + +static inline void init_lasx_upper(void) +{ + if (cpu_has_lasx) + _init_lasx_upper(); +} + +static inline void restore_lasx_upper(struct task_struct *t) +{ + if (cpu_has_lasx) + _restore_lasx_upper(&t->thread.fpu); +} + +#else +static inline void enable_lasx(void) {} +static inline void disable_lasx(void) {} +static inline void save_lasx(struct task_struct *t) {} +static inline void restore_lasx(struct task_struct *t) {} +static inline void init_lasx_upper(void) {} +static inline void restore_lasx_upper(struct task_struct *t) {} +#endif + +static inline int thread_lsx_context_live(void) +{ + if (__builtin_constant_p(cpu_has_lsx) && !cpu_has_lsx) + return 0; + + return test_thread_flag(TIF_LSX_CTX_LIVE); +} + +static inline int thread_lasx_context_live(void) +{ + if (__builtin_constant_p(cpu_has_lasx) && !cpu_has_lasx) + return 0; + + return test_thread_flag(TIF_LASX_CTX_LIVE); +} + #endif /* _ASM_FPU_H */ diff --git a/arch/loongarch/include/asm/ftrace.h b/arch/loongarch/include/asm/ftrace.h index 23e2ba78dcb0..a11996eb5892 100644 --- a/arch/loongarch/include/asm/ftrace.h +++ b/arch/loongarch/include/asm/ftrace.h @@ -100,4 +100,26 @@ __arch_ftrace_set_direct_caller(struct pt_regs *regs, unsigned long addr) #endif /* CONFIG_FUNCTION_TRACER */ +#ifndef __ASSEMBLY__ +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +struct fgraph_ret_regs { + /* a0 - a1 */ + unsigned long regs[2]; + + unsigned long fp; + unsigned long __unused; +}; + +static inline unsigned long fgraph_ret_regs_return_value(struct fgraph_ret_regs *ret_regs) +{ + return ret_regs->regs[0]; +} + +static inline unsigned long fgraph_ret_regs_frame_pointer(struct fgraph_ret_regs *ret_regs) +{ + return ret_regs->fp; +} +#endif /* ifdef CONFIG_FUNCTION_GRAPH_TRACER */ +#endif + #endif /* _ASM_LOONGARCH_FTRACE_H */ diff --git a/arch/loongarch/include/asm/gpr-num.h b/arch/loongarch/include/asm/gpr-num.h index e0941af20c7e..996038da806d 100644 --- a/arch/loongarch/include/asm/gpr-num.h +++ b/arch/loongarch/include/asm/gpr-num.h @@ -9,6 +9,22 @@ .equ .L__gpr_num_$r\num, \num .endr + /* ABI names of registers */ + .equ .L__gpr_num_$ra, 1 + .equ .L__gpr_num_$tp, 2 + .equ .L__gpr_num_$sp, 3 + .irp num,0,1,2,3,4,5,6,7 + .equ .L__gpr_num_$a\num, 4 + \num + .endr + .irp num,0,1,2,3,4,5,6,7,8 + .equ .L__gpr_num_$t\num, 12 + \num + .endr + .equ .L__gpr_num_$s9, 22 + .equ .L__gpr_num_$fp, 22 + .irp num,0,1,2,3,4,5,6,7,8 + .equ .L__gpr_num_$s\num, 23 + \num + .endr + #else /* __ASSEMBLY__ */ #define __DEFINE_ASM_GPR_NUMS \ @@ -16,6 +32,20 @@ " .irp num,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31\n" \ " .equ .L__gpr_num_$r\\num, \\num\n" \ " .endr\n" \ +" .equ .L__gpr_num_$ra, 1\n" \ +" .equ .L__gpr_num_$tp, 2\n" \ +" .equ .L__gpr_num_$sp, 3\n" \ +" .irp num,0,1,2,3,4,5,6,7\n" \ +" .equ .L__gpr_num_$a\\num, 4 + \\num\n" \ +" .endr\n" \ +" .irp num,0,1,2,3,4,5,6,7,8\n" \ +" .equ .L__gpr_num_$t\\num, 12 + \\num\n" \ +" .endr\n" \ +" .equ .L__gpr_num_$s9, 22\n" \ +" .equ .L__gpr_num_$fp, 22\n" \ +" .irp num,0,1,2,3,4,5,6,7,8\n" \ +" .equ .L__gpr_num_$s\\num, 23 + \\num\n" \ +" .endr\n" \ #endif /* __ASSEMBLY__ */ diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h index b09887ffcd15..71e1ed4165c8 100644 --- a/arch/loongarch/include/asm/inst.h +++ b/arch/loongarch/include/asm/inst.h @@ -5,6 +5,7 @@ #ifndef _ASM_INST_H #define _ASM_INST_H +#include <linux/bitops.h> #include <linux/types.h> #include <asm/asm.h> #include <asm/ptrace.h> @@ -15,14 +16,22 @@ #define ADDR_IMMMASK_LU52ID 0xFFF0000000000000 #define ADDR_IMMMASK_LU32ID 0x000FFFFF00000000 #define ADDR_IMMMASK_LU12IW 0x00000000FFFFF000 +#define ADDR_IMMMASK_ORI 0x0000000000000FFF #define ADDR_IMMMASK_ADDU16ID 0x00000000FFFF0000 #define ADDR_IMMSHIFT_LU52ID 52 +#define ADDR_IMMSBIDX_LU52ID 11 #define ADDR_IMMSHIFT_LU32ID 32 +#define ADDR_IMMSBIDX_LU32ID 19 #define ADDR_IMMSHIFT_LU12IW 12 +#define ADDR_IMMSBIDX_LU12IW 19 +#define ADDR_IMMSHIFT_ORI 0 +#define ADDR_IMMSBIDX_ORI 63 #define ADDR_IMMSHIFT_ADDU16ID 16 +#define ADDR_IMMSBIDX_ADDU16ID 15 -#define ADDR_IMM(addr, INSN) ((addr & ADDR_IMMMASK_##INSN) >> ADDR_IMMSHIFT_##INSN) +#define ADDR_IMM(addr, INSN) \ + (sign_extend64(((addr & ADDR_IMMMASK_##INSN) >> ADDR_IMMSHIFT_##INSN), ADDR_IMMSBIDX_##INSN)) enum reg0i15_op { break_op = 0x54, @@ -178,6 +187,32 @@ enum reg3_op { amord_op = 0x70c7, amxorw_op = 0x70c8, amxord_op = 0x70c9, + ammaxw_op = 0x70ca, + ammaxd_op = 0x70cb, + amminw_op = 0x70cc, + ammind_op = 0x70cd, + ammaxwu_op = 0x70ce, + ammaxdu_op = 0x70cf, + amminwu_op = 0x70d0, + ammindu_op = 0x70d1, + amswapdbw_op = 0x70d2, + amswapdbd_op = 0x70d3, + amadddbw_op = 0x70d4, + amadddbd_op = 0x70d5, + amanddbw_op = 0x70d6, + amanddbd_op = 0x70d7, + amordbw_op = 0x70d8, + amordbd_op = 0x70d9, + amxordbw_op = 0x70da, + amxordbd_op = 0x70db, + ammaxdbw_op = 0x70dc, + ammaxdbd_op = 0x70dd, + ammindbw_op = 0x70de, + ammindbd_op = 0x70df, + ammaxdbwu_op = 0x70e0, + ammaxdbdu_op = 0x70e1, + ammindbwu_op = 0x70e2, + ammindbdu_op = 0x70e3, fldgts_op = 0x70e8, fldgtd_op = 0x70e9, fldles_op = 0x70ea, @@ -435,6 +470,10 @@ static inline bool is_self_loop_ins(union loongarch_instruction *ip, struct pt_r void simu_pc(struct pt_regs *regs, union loongarch_instruction insn); void simu_branch(struct pt_regs *regs, union loongarch_instruction insn); +bool insns_not_supported(union loongarch_instruction insn); +bool insns_need_simulation(union loongarch_instruction insn); +void arch_simulate_insn(union loongarch_instruction insn, struct pt_regs *regs); + int larch_insn_read(void *addr, u32 *insnp); int larch_insn_write(void *addr, u32 insn); int larch_insn_patch_text(void *addr, u32 insn); @@ -443,13 +482,15 @@ u32 larch_insn_gen_nop(void); u32 larch_insn_gen_b(unsigned long pc, unsigned long dest); u32 larch_insn_gen_bl(unsigned long pc, unsigned long dest); +u32 larch_insn_gen_break(int imm); + u32 larch_insn_gen_or(enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk); u32 larch_insn_gen_move(enum loongarch_gpr rd, enum loongarch_gpr rj); u32 larch_insn_gen_lu12iw(enum loongarch_gpr rd, int imm); u32 larch_insn_gen_lu32id(enum loongarch_gpr rd, int imm); u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm); -u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, unsigned long pc, unsigned long dest); +u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm); static inline bool signed_imm_check(long val, unsigned int bit) { @@ -461,6 +502,16 @@ static inline bool unsigned_imm_check(unsigned long val, unsigned int bit) return val < (1UL << bit); } +#define DEF_EMIT_REG0I15_FORMAT(NAME, OP) \ +static inline void emit_##NAME(union loongarch_instruction *insn, \ + int imm) \ +{ \ + insn->reg0i15_format.opcode = OP; \ + insn->reg0i15_format.immediate = imm; \ +} + +DEF_EMIT_REG0I15_FORMAT(break, break_op) + #define DEF_EMIT_REG0I26_FORMAT(NAME, OP) \ static inline void emit_##NAME(union loongarch_instruction *insn, \ int offset) \ diff --git a/arch/loongarch/include/asm/io.h b/arch/loongarch/include/asm/io.h index 545e2708fbf7..1c9410220040 100644 --- a/arch/loongarch/include/asm/io.h +++ b/arch/loongarch/include/asm/io.h @@ -62,7 +62,7 @@ extern pgprot_t pgprot_wc; #define ioremap_cache(offset, size) \ ioremap_prot((offset), (size), pgprot_val(PAGE_KERNEL)) -#define mmiowb() asm volatile ("dbar 0" ::: "memory") +#define mmiowb() wmb() /* * String version of I/O memory access operations. diff --git a/arch/loongarch/include/asm/jump_label.h b/arch/loongarch/include/asm/jump_label.h new file mode 100644 index 000000000000..3cea299a5ef5 --- /dev/null +++ b/arch/loongarch/include/asm/jump_label.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2023 Loongson Technology Corporation Limited + * + * Based on arch/arm64/include/asm/jump_label.h + */ +#ifndef __ASM_JUMP_LABEL_H +#define __ASM_JUMP_LABEL_H + +#ifndef __ASSEMBLY__ + +#include <linux/types.h> + +#define JUMP_LABEL_NOP_SIZE 4 + +#define JUMP_TABLE_ENTRY \ + ".pushsection __jump_table, \"aw\" \n\t" \ + ".align 3 \n\t" \ + ".long 1b - ., %l[l_yes] - . \n\t" \ + ".quad %0 - . \n\t" \ + ".popsection \n\t" + +static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch) +{ + asm_volatile_goto( + "1: nop \n\t" + JUMP_TABLE_ENTRY + : : "i"(&((char *)key)[branch]) : : l_yes); + + return false; + +l_yes: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch) +{ + asm_volatile_goto( + "1: b %l[l_yes] \n\t" + JUMP_TABLE_ENTRY + : : "i"(&((char *)key)[branch]) : : l_yes); + + return false; + +l_yes: + return true; +} + +#endif /* __ASSEMBLY__ */ +#endif /* __ASM_JUMP_LABEL_H */ diff --git a/arch/loongarch/include/asm/kdebug.h b/arch/loongarch/include/asm/kdebug.h index d721b4b82fae..c00ed874bf06 100644 --- a/arch/loongarch/include/asm/kdebug.h +++ b/arch/loongarch/include/asm/kdebug.h @@ -13,11 +13,6 @@ enum die_val { DIE_FP, DIE_SIMD, DIE_TRAP, - DIE_PAGE_FAULT, - DIE_BREAK, - DIE_SSTEPBP, - DIE_UPROBE, - DIE_UPROBE_XOL, }; #endif /* _ASM_LOONGARCH_KDEBUG_H */ diff --git a/arch/loongarch/include/asm/kprobes.h b/arch/loongarch/include/asm/kprobes.h index 798020ae02c6..60fa753a010d 100644 --- a/arch/loongarch/include/asm/kprobes.h +++ b/arch/loongarch/include/asm/kprobes.h @@ -22,7 +22,7 @@ do { \ #define kretprobe_blacklist_size 0 -typedef union loongarch_instruction kprobe_opcode_t; +typedef u32 kprobe_opcode_t; /* Architecture specific copy of original instruction */ struct arch_specific_insn { @@ -49,9 +49,6 @@ bool kprobe_fault_handler(struct pt_regs *regs, int trapnr); bool kprobe_breakpoint_handler(struct pt_regs *regs); bool kprobe_singlestep_handler(struct pt_regs *regs); -void __kretprobe_trampoline(void); -void *trampoline_probe_handler(struct pt_regs *regs); - #else /* !CONFIG_KPROBES */ static inline bool kprobe_breakpoint_handler(struct pt_regs *regs) { return false; } diff --git a/arch/loongarch/include/asm/loongarch.h b/arch/loongarch/include/asm/loongarch.h index 1c2a0a2c8830..10748a20a2ab 100644 --- a/arch/loongarch/include/asm/loongarch.h +++ b/arch/loongarch/include/asm/loongarch.h @@ -56,10 +56,7 @@ __asm__(".macro parse_r var r\n\t" #undef _IFC_REG /* CPUCFG */ -static inline u32 read_cpucfg(u32 reg) -{ - return __cpucfg(reg); -} +#define read_cpucfg(reg) __cpucfg(reg) #endif /* !__ASSEMBLY__ */ @@ -138,6 +135,7 @@ static inline u32 read_cpucfg(u32 reg) #define CPUCFG2_MIPSBT BIT(20) #define CPUCFG2_LSPW BIT(21) #define CPUCFG2_LAM BIT(22) +#define CPUCFG2_PTW BIT(24) #define LOONGARCH_CPUCFG3 0x3 #define CPUCFG3_CCDMA BIT(0) @@ -206,56 +204,18 @@ static inline u32 read_cpucfg(u32 reg) #ifndef __ASSEMBLY__ /* CSR */ -static __always_inline u32 csr_read32(u32 reg) -{ - return __csrrd_w(reg); -} - -static __always_inline u64 csr_read64(u32 reg) -{ - return __csrrd_d(reg); -} - -static __always_inline void csr_write32(u32 val, u32 reg) -{ - __csrwr_w(val, reg); -} - -static __always_inline void csr_write64(u64 val, u32 reg) -{ - __csrwr_d(val, reg); -} - -static __always_inline u32 csr_xchg32(u32 val, u32 mask, u32 reg) -{ - return __csrxchg_w(val, mask, reg); -} - -static __always_inline u64 csr_xchg64(u64 val, u64 mask, u32 reg) -{ - return __csrxchg_d(val, mask, reg); -} +#define csr_read32(reg) __csrrd_w(reg) +#define csr_read64(reg) __csrrd_d(reg) +#define csr_write32(val, reg) __csrwr_w(val, reg) +#define csr_write64(val, reg) __csrwr_d(val, reg) +#define csr_xchg32(val, mask, reg) __csrxchg_w(val, mask, reg) +#define csr_xchg64(val, mask, reg) __csrxchg_d(val, mask, reg) /* IOCSR */ -static __always_inline u32 iocsr_read32(u32 reg) -{ - return __iocsrrd_w(reg); -} - -static __always_inline u64 iocsr_read64(u32 reg) -{ - return __iocsrrd_d(reg); -} - -static __always_inline void iocsr_write32(u32 val, u32 reg) -{ - __iocsrwr_w(val, reg); -} - -static __always_inline void iocsr_write64(u64 val, u32 reg) -{ - __iocsrwr_d(val, reg); -} +#define iocsr_read32(reg) __iocsrrd_w(reg) +#define iocsr_read64(reg) __iocsrrd_d(reg) +#define iocsr_write32(val, reg) __iocsrwr_w(val, reg) +#define iocsr_write64(val, reg) __iocsrwr_d(val, reg) #endif /* !__ASSEMBLY__ */ @@ -453,6 +413,9 @@ static __always_inline void iocsr_write64(u64 val, u32 reg) #define CSR_PWCTL0_PTBASE (_ULCAST_(0x1f) << CSR_PWCTL0_PTBASE_SHIFT) #define LOONGARCH_CSR_PWCTL1 0x1d /* PWCtl1 */ +#define CSR_PWCTL1_PTW_SHIFT 24 +#define CSR_PWCTL1_PTW_WIDTH 1 +#define CSR_PWCTL1_PTW (_ULCAST_(0x1) << CSR_PWCTL1_PTW_SHIFT) #define CSR_PWCTL1_DIR3WIDTH_SHIFT 18 #define CSR_PWCTL1_DIR3WIDTH_WIDTH 5 #define CSR_PWCTL1_DIR3WIDTH (_ULCAST_(0x1f) << CSR_PWCTL1_DIR3WIDTH_SHIFT) @@ -1441,11 +1404,18 @@ __BUILD_CSR_OP(tlbidx) #define EXCCODE_INT_START 64 #define EXCCODE_INT_END (EXCCODE_INT_START + EXCCODE_INT_NUM - 1) -/* FPU register names */ +/* FPU Status Register Names */ +#ifndef CONFIG_AS_HAS_FCSR_CLASS #define LOONGARCH_FCSR0 $r0 #define LOONGARCH_FCSR1 $r1 #define LOONGARCH_FCSR2 $r2 #define LOONGARCH_FCSR3 $r3 +#else +#define LOONGARCH_FCSR0 $fcsr0 +#define LOONGARCH_FCSR1 $fcsr1 +#define LOONGARCH_FCSR2 $fcsr2 +#define LOONGARCH_FCSR3 $fcsr3 +#endif /* FPU Status Register Values */ #define FPU_CSR_RSVD 0xe0e0fce0 diff --git a/arch/loongarch/include/asm/module.h b/arch/loongarch/include/asm/module.h index 12a0f1e66916..2ecd82bb64e1 100644 --- a/arch/loongarch/include/asm/module.h +++ b/arch/loongarch/include/asm/module.h @@ -55,7 +55,7 @@ static inline struct plt_entry emit_plt_entry(unsigned long val) lu12iw = larch_insn_gen_lu12iw(LOONGARCH_GPR_T1, ADDR_IMM(val, LU12IW)); lu32id = larch_insn_gen_lu32id(LOONGARCH_GPR_T1, ADDR_IMM(val, LU32ID)); lu52id = larch_insn_gen_lu52id(LOONGARCH_GPR_T1, LOONGARCH_GPR_T1, ADDR_IMM(val, LU52ID)); - jirl = larch_insn_gen_jirl(0, LOONGARCH_GPR_T1, 0, (val & 0xfff)); + jirl = larch_insn_gen_jirl(0, LOONGARCH_GPR_T1, ADDR_IMM(val, ORI)); return (struct plt_entry) { lu12iw, lu32id, lu52id, jirl }; } diff --git a/arch/loongarch/include/asm/page.h b/arch/loongarch/include/asm/page.h index fb5338b352e6..26e8dccb6619 100644 --- a/arch/loongarch/include/asm/page.h +++ b/arch/loongarch/include/asm/page.h @@ -81,6 +81,7 @@ typedef struct { unsigned long pgprot; } pgprot_t; #define __va(x) ((void *)((unsigned long)(x) + PAGE_OFFSET - PHYS_OFFSET)) #define pfn_to_kaddr(pfn) __va((pfn) << PAGE_SHIFT) +#define sym_to_pfn(x) __phys_to_pfn(__pa_symbol(x)) #define virt_to_pfn(kaddr) PFN_DOWN(PHYSADDR(kaddr)) #define virt_to_page(kaddr) pfn_to_page(virt_to_pfn(kaddr)) diff --git a/arch/loongarch/include/asm/percpu.h b/arch/loongarch/include/asm/percpu.h index ad8d88494554..b9f567e66016 100644 --- a/arch/loongarch/include/asm/percpu.h +++ b/arch/loongarch/include/asm/percpu.h @@ -14,7 +14,11 @@ * loaded. Tell the compiler this fact when using explicit relocs. */ #if defined(MODULE) && defined(CONFIG_AS_HAS_EXPLICIT_RELOCS) -#define PER_CPU_ATTRIBUTES __attribute__((model("extreme"))) +# if __has_attribute(model) +# define PER_CPU_ATTRIBUTES __attribute__((model("extreme"))) +# else +# error compiler support for the model attribute is necessary when a recent assembler is used +# endif #endif /* Use r21 for fast access */ diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index 9a9f9ff9b709..38afeb7dd58b 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -362,7 +362,7 @@ extern pgd_t invalid_pg_dir[]; */ static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_WRITE; } static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; } -static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_MODIFIED; } +static inline int pte_dirty(pte_t pte) { return pte_val(pte) & (_PAGE_DIRTY | _PAGE_MODIFIED); } static inline pte_t pte_mkold(pte_t pte) { @@ -506,7 +506,7 @@ static inline pmd_t pmd_wrprotect(pmd_t pmd) static inline int pmd_dirty(pmd_t pmd) { - return !!(pmd_val(pmd) & _PAGE_MODIFIED); + return !!(pmd_val(pmd) & (_PAGE_DIRTY | _PAGE_MODIFIED)); } static inline pmd_t pmd_mkclean(pmd_t pmd) diff --git a/arch/loongarch/include/asm/qspinlock.h b/arch/loongarch/include/asm/qspinlock.h new file mode 100644 index 000000000000..34f43f8ad591 --- /dev/null +++ b/arch/loongarch/include/asm/qspinlock.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_QSPINLOCK_H +#define _ASM_QSPINLOCK_H + +#include <asm-generic/qspinlock_types.h> + +#define queued_spin_unlock queued_spin_unlock + +static inline void queued_spin_unlock(struct qspinlock *lock) +{ + compiletime_assert_atomic_type(lock->locked); + c_sync(); + WRITE_ONCE(lock->locked, 0); +} + +#include <asm-generic/qspinlock.h> + +#endif /* _ASM_QSPINLOCK_H */ diff --git a/arch/loongarch/include/asm/suspend.h b/arch/loongarch/include/asm/suspend.h new file mode 100644 index 000000000000..4025c9d5d7cf --- /dev/null +++ b/arch/loongarch/include/asm/suspend.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_SUSPEND_H +#define __ASM_SUSPEND_H + +void loongarch_common_suspend(void); +void loongarch_common_resume(void); +void loongarch_suspend_enter(void); +void loongarch_wakeup_start(void); + +#endif diff --git a/arch/loongarch/include/asm/tlb.h b/arch/loongarch/include/asm/tlb.h index f5e4deb97402..da7a3b5b9374 100644 --- a/arch/loongarch/include/asm/tlb.h +++ b/arch/loongarch/include/asm/tlb.h @@ -88,52 +88,47 @@ enum invtlb_ops { INVTLB_GID_ADDR = 0x16, }; -/* - * invtlb op info addr - * (0x1 << 26) | (0x24 << 20) | (0x13 << 15) | - * (addr << 10) | (info << 5) | op - */ -static inline void invtlb(u32 op, u32 info, u64 addr) +static __always_inline void invtlb(u32 op, u32 info, u64 addr) { __asm__ __volatile__( - "parse_r addr,%0\n\t" - "parse_r info,%1\n\t" - ".word ((0x6498000) | (addr << 10) | (info << 5) | %2)\n\t" - : - : "r"(addr), "r"(info), "i"(op) + "invtlb %0, %1, %2\n\t" : + : "i"(op), "r"(info), "r"(addr) + : "memory" ); } -static inline void invtlb_addr(u32 op, u32 info, u64 addr) +static __always_inline void invtlb_addr(u32 op, u32 info, u64 addr) { + BUILD_BUG_ON(!__builtin_constant_p(info) || info != 0); __asm__ __volatile__( - "parse_r addr,%0\n\t" - ".word ((0x6498000) | (addr << 10) | (0 << 5) | %1)\n\t" - : - : "r"(addr), "i"(op) + "invtlb %0, $zero, %1\n\t" : + : "i"(op), "r"(addr) + : "memory" ); } -static inline void invtlb_info(u32 op, u32 info, u64 addr) +static __always_inline void invtlb_info(u32 op, u32 info, u64 addr) { + BUILD_BUG_ON(!__builtin_constant_p(addr) || addr != 0); __asm__ __volatile__( - "parse_r info,%0\n\t" - ".word ((0x6498000) | (0 << 10) | (info << 5) | %1)\n\t" - : - : "r"(info), "i"(op) + "invtlb %0, %1, $zero\n\t" : + : "i"(op), "r"(info) + : "memory" ); } -static inline void invtlb_all(u32 op, u32 info, u64 addr) +static __always_inline void invtlb_all(u32 op, u32 info, u64 addr) { + BUILD_BUG_ON(!__builtin_constant_p(info) || info != 0); + BUILD_BUG_ON(!__builtin_constant_p(addr) || addr != 0); __asm__ __volatile__( - ".word ((0x6498000) | (0 << 10) | (0 << 5) | %0)\n\t" + "invtlb %0, $zero, $zero\n\t" : : "i"(op) - : + : "memory" ); } @@ -163,6 +158,9 @@ extern void handle_tlb_store(void); extern void handle_tlb_modify(void); extern void handle_tlb_refill(void); extern void handle_tlb_protect(void); +extern void handle_tlb_load_ptw(void); +extern void handle_tlb_store_ptw(void); +extern void handle_tlb_modify_ptw(void); extern void dump_tlb_all(void); extern void dump_tlb_regs(void); diff --git a/arch/loongarch/include/asm/uprobes.h b/arch/loongarch/include/asm/uprobes.h new file mode 100644 index 000000000000..c8f59983f702 --- /dev/null +++ b/arch/loongarch/include/asm/uprobes.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_LOONGARCH_UPROBES_H +#define __ASM_LOONGARCH_UPROBES_H + +#include <asm/inst.h> + +typedef u32 uprobe_opcode_t; + +#define MAX_UINSN_BYTES 8 +#define UPROBE_XOL_SLOT_BYTES MAX_UINSN_BYTES + +#define UPROBE_SWBP_INSN larch_insn_gen_break(BRK_UPROBE_BP) +#define UPROBE_SWBP_INSN_SIZE LOONGARCH_INSN_SIZE + +#define UPROBE_XOLBP_INSN larch_insn_gen_break(BRK_UPROBE_XOLBP) + +struct arch_uprobe { + unsigned long resume_era; + u32 insn[2]; + u32 ixol[2]; + bool simulate; +}; + +struct arch_uprobe_task { + unsigned long saved_trap_nr; +}; + +#ifdef CONFIG_UPROBES +bool uprobe_breakpoint_handler(struct pt_regs *regs); +bool uprobe_singlestep_handler(struct pt_regs *regs); +#else /* !CONFIG_UPROBES */ +static inline bool uprobe_breakpoint_handler(struct pt_regs *regs) { return false; } +static inline bool uprobe_singlestep_handler(struct pt_regs *regs) { return false; } +#endif /* CONFIG_UPROBES */ + +#endif /* __ASM_LOONGARCH_UPROBES_H */ diff --git a/arch/loongarch/include/asm/vdso/gettimeofday.h b/arch/loongarch/include/asm/vdso/gettimeofday.h index 7b2cd37641e2..89e6b222c2f2 100644 --- a/arch/loongarch/include/asm/vdso/gettimeofday.h +++ b/arch/loongarch/include/asm/vdso/gettimeofday.h @@ -91,9 +91,16 @@ static inline bool loongarch_vdso_hres_capable(void) static __always_inline const struct vdso_data *__arch_get_vdso_data(void) { - return get_vdso_data(); + return (const struct vdso_data *)get_vdso_data(); } +#ifdef CONFIG_TIME_NS +static __always_inline +const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) +{ + return (const struct vdso_data *)(get_vdso_data() + VVAR_TIMENS_PAGE_OFFSET * PAGE_SIZE); +} +#endif #endif /* !__ASSEMBLY__ */ #endif /* __ASM_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/loongarch/include/asm/vdso/vdso.h b/arch/loongarch/include/asm/vdso/vdso.h index 3b55d32a0619..5a12309d9fb5 100644 --- a/arch/loongarch/include/asm/vdso/vdso.h +++ b/arch/loongarch/include/asm/vdso/vdso.h @@ -16,10 +16,33 @@ struct vdso_pcpu_data { struct loongarch_vdso_data { struct vdso_pcpu_data pdata[NR_CPUS]; - struct vdso_data data[CS_BASES]; /* Arch-independent data */ }; -#define VDSO_DATA_SIZE PAGE_ALIGN(sizeof(struct loongarch_vdso_data)) +/* + * The layout of vvar: + * + * high + * +---------------------+--------------------------+ + * | loongarch vdso data | LOONGARCH_VDSO_DATA_SIZE | + * +---------------------+--------------------------+ + * | time-ns vdso data | PAGE_SIZE | + * +---------------------+--------------------------+ + * | generic vdso data | PAGE_SIZE | + * +---------------------+--------------------------+ + * low + */ +#define LOONGARCH_VDSO_DATA_SIZE PAGE_ALIGN(sizeof(struct loongarch_vdso_data)) +#define LOONGARCH_VDSO_DATA_PAGES (LOONGARCH_VDSO_DATA_SIZE >> PAGE_SHIFT) + +enum vvar_pages { + VVAR_GENERIC_PAGE_OFFSET, + VVAR_TIMENS_PAGE_OFFSET, + VVAR_LOONGARCH_PAGES_START, + VVAR_LOONGARCH_PAGES_END = VVAR_LOONGARCH_PAGES_START + LOONGARCH_VDSO_DATA_PAGES - 1, + VVAR_NR_PAGES, +}; + +#define VVAR_SIZE (VVAR_NR_PAGES << PAGE_SHIFT) static inline unsigned long get_vdso_base(void) { @@ -34,10 +57,9 @@ static inline unsigned long get_vdso_base(void) return addr; } -static inline const struct vdso_data *get_vdso_data(void) +static inline unsigned long get_vdso_data(void) { - return (const struct vdso_data *)(get_vdso_base() - - VDSO_DATA_SIZE + SMP_CACHE_BYTES * NR_CPUS); + return get_vdso_base() - VVAR_SIZE; } #endif /* __ASSEMBLY__ */ diff --git a/arch/loongarch/include/uapi/asm/hwcap.h b/arch/loongarch/include/uapi/asm/hwcap.h index 8840b72fa8e8..6955a7cb2c65 100644 --- a/arch/loongarch/include/uapi/asm/hwcap.h +++ b/arch/loongarch/include/uapi/asm/hwcap.h @@ -16,5 +16,6 @@ #define HWCAP_LOONGARCH_LBT_X86 (1 << 10) #define HWCAP_LOONGARCH_LBT_ARM (1 << 11) #define HWCAP_LOONGARCH_LBT_MIPS (1 << 12) +#define HWCAP_LOONGARCH_PTW (1 << 13) #endif /* _UAPI_ASM_HWCAP_H */ diff --git a/arch/loongarch/include/uapi/asm/ptrace.h b/arch/loongarch/include/uapi/asm/ptrace.h index 82d811b5c6e9..06e3be52cb04 100644 --- a/arch/loongarch/include/uapi/asm/ptrace.h +++ b/arch/loongarch/include/uapi/asm/ptrace.h @@ -41,9 +41,19 @@ struct user_pt_regs { } __attribute__((aligned(8))); struct user_fp_state { - uint64_t fpr[32]; - uint64_t fcc; - uint32_t fcsr; + uint64_t fpr[32]; + uint64_t fcc; + uint32_t fcsr; +}; + +struct user_lsx_state { + /* 32 registers, 128 bits width per register. */ + uint64_t vregs[32*2]; +}; + +struct user_lasx_state { + /* 32 registers, 256 bits width per register. */ + uint64_t vregs[32*4]; }; struct user_watch_state { diff --git a/arch/loongarch/include/uapi/asm/sigcontext.h b/arch/loongarch/include/uapi/asm/sigcontext.h index 52e49b8bf4be..4cd7d16f7037 100644 --- a/arch/loongarch/include/uapi/asm/sigcontext.h +++ b/arch/loongarch/include/uapi/asm/sigcontext.h @@ -41,4 +41,22 @@ struct fpu_context { __u32 fcsr; }; +/* LSX context */ +#define LSX_CTX_MAGIC 0x53580001 +#define LSX_CTX_ALIGN 16 +struct lsx_context { + __u64 regs[2*32]; + __u64 fcc; + __u32 fcsr; +}; + +/* LASX context */ +#define LASX_CTX_MAGIC 0x41535801 +#define LASX_CTX_ALIGN 32 +struct lasx_context { + __u64 regs[4*32]; + __u64 fcc; + __u32 fcsr; +}; + #endif /* _UAPI_ASM_SIGCONTEXT_H */ diff --git a/arch/loongarch/kernel/Makefile b/arch/loongarch/kernel/Makefile index 9a72d91cd104..8e279f04f9e7 100644 --- a/arch/loongarch/kernel/Makefile +++ b/arch/loongarch/kernel/Makefile @@ -28,6 +28,8 @@ ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_inst.o = $(CC_FLAGS_FTRACE) CFLAGS_REMOVE_time.o = $(CC_FLAGS_FTRACE) CFLAGS_REMOVE_perf_event.o = $(CC_FLAGS_FTRACE) + CFLAGS_REMOVE_rethook.o = $(CC_FLAGS_FTRACE) + CFLAGS_REMOVE_rethook_trampoline.o = $(CC_FLAGS_FTRACE) endif obj-$(CONFIG_MODULES) += module.o module-sections.o @@ -52,6 +54,10 @@ obj-$(CONFIG_UNWINDER_PROLOGUE) += unwind_prologue.o obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_regs.o obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o -obj-$(CONFIG_KPROBES) += kprobes.o kprobes_trampoline.o +obj-$(CONFIG_KPROBES) += kprobes.o +obj-$(CONFIG_RETHOOK) += rethook.o rethook_trampoline.o +obj-$(CONFIG_UPROBES) += uprobes.o + +obj-$(CONFIG_JUMP_LABEL) += jump_label.o CPPFLAGS_vmlinux.lds := $(KBUILD_CFLAGS) diff --git a/arch/loongarch/kernel/acpi.c b/arch/loongarch/kernel/acpi.c index 98f431157e4c..9450e09073eb 100644 --- a/arch/loongarch/kernel/acpi.c +++ b/arch/loongarch/kernel/acpi.c @@ -33,6 +33,8 @@ u64 acpi_saved_sp; #define PREFIX "ACPI: " +struct acpi_madt_core_pic acpi_core_pic[NR_CPUS]; + void __init __iomem * __acpi_map_table(unsigned long phys, unsigned long size) { @@ -99,6 +101,7 @@ acpi_parse_processor(union acpi_subtable_headers *header, const unsigned long en acpi_table_print_madt_entry(&header->common); #ifdef CONFIG_SMP + acpi_core_pic[processor->core_id] = *processor; set_processor_mask(processor->core_id, processor->flags); #endif @@ -140,6 +143,35 @@ static void __init acpi_process_madt(void) loongson_sysconf.nr_cpus = num_processors; } +int pptt_enabled; + +int __init parse_acpi_topology(void) +{ + int cpu, topology_id; + + for_each_possible_cpu(cpu) { + topology_id = find_acpi_cpu_topology(cpu, 0); + if (topology_id < 0) { + pr_warn("Invalid BIOS PPTT\n"); + return -ENOENT; + } + + if (acpi_pptt_cpu_is_thread(cpu) <= 0) + cpu_data[cpu].core = topology_id; + else { + topology_id = find_acpi_cpu_topology(cpu, 1); + if (topology_id < 0) + return -ENOENT; + + cpu_data[cpu].core = topology_id; + } + } + + pptt_enabled = 1; + + return 0; +} + #ifndef CONFIG_SUSPEND int (*acpi_suspend_lowlevel)(void); #else diff --git a/arch/loongarch/kernel/asm-offsets.c b/arch/loongarch/kernel/asm-offsets.c index 4bdb203fc66e..505e4bf59603 100644 --- a/arch/loongarch/kernel/asm-offsets.c +++ b/arch/loongarch/kernel/asm-offsets.c @@ -12,6 +12,7 @@ #include <asm/cpu-info.h> #include <asm/ptrace.h> #include <asm/processor.h> +#include <asm/ftrace.h> void output_ptreg_defines(void) { @@ -264,7 +265,7 @@ void output_smpboot_defines(void) #ifdef CONFIG_HIBERNATION void output_pbe_defines(void) { - COMMENT(" Linux struct pbe offsets. "); + COMMENT("Linux struct pbe offsets."); OFFSET(PBE_ADDRESS, pbe, address); OFFSET(PBE_ORIG_ADDRESS, pbe, orig_address); OFFSET(PBE_NEXT, pbe, next); @@ -272,3 +273,15 @@ void output_pbe_defines(void) BLANK(); } #endif + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +void output_fgraph_ret_regs_defines(void) +{ + COMMENT("LoongArch fgraph_ret_regs offsets."); + OFFSET(FGRET_REGS_A0, fgraph_ret_regs, regs[0]); + OFFSET(FGRET_REGS_A1, fgraph_ret_regs, regs[1]); + OFFSET(FGRET_REGS_FP, fgraph_ret_regs, fp); + DEFINE(FGRET_REGS_SIZE, sizeof(struct fgraph_ret_regs)); + BLANK(); +} +#endif diff --git a/arch/loongarch/kernel/cpu-probe.c b/arch/loongarch/kernel/cpu-probe.c index 5adf0f736c6d..e925579c7a71 100644 --- a/arch/loongarch/kernel/cpu-probe.c +++ b/arch/loongarch/kernel/cpu-probe.c @@ -116,6 +116,18 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c) c->options |= LOONGARCH_CPU_FPU; elf_hwcap |= HWCAP_LOONGARCH_FPU; } +#ifdef CONFIG_CPU_HAS_LSX + if (config & CPUCFG2_LSX) { + c->options |= LOONGARCH_CPU_LSX; + elf_hwcap |= HWCAP_LOONGARCH_LSX; + } +#endif +#ifdef CONFIG_CPU_HAS_LASX + if (config & CPUCFG2_LASX) { + c->options |= LOONGARCH_CPU_LASX; + elf_hwcap |= HWCAP_LOONGARCH_LASX; + } +#endif if (config & CPUCFG2_COMPLEX) { c->options |= LOONGARCH_CPU_COMPLEX; elf_hwcap |= HWCAP_LOONGARCH_COMPLEX; @@ -124,6 +136,10 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c) c->options |= LOONGARCH_CPU_CRYPTO; elf_hwcap |= HWCAP_LOONGARCH_CRYPTO; } + if (config & CPUCFG2_PTW) { + c->options |= LOONGARCH_CPU_PTW; + elf_hwcap |= HWCAP_LOONGARCH_PTW; + } if (config & CPUCFG2_LVZP) { c->options |= LOONGARCH_CPU_LVZ; elf_hwcap |= HWCAP_LOONGARCH_LVZ; diff --git a/arch/loongarch/kernel/efi-header.S b/arch/loongarch/kernel/efi-header.S index 8c1d229a2afa..5f23b85d78ca 100644 --- a/arch/loongarch/kernel/efi-header.S +++ b/arch/loongarch/kernel/efi-header.S @@ -24,7 +24,7 @@ .byte 0x02 /* MajorLinkerVersion */ .byte 0x14 /* MinorLinkerVersion */ .long __inittext_end - .Lefi_header_end /* SizeOfCode */ - .long _end - __initdata_begin /* SizeOfInitializedData */ + .long _kernel_vsize /* SizeOfInitializedData */ .long 0 /* SizeOfUninitializedData */ .long __efistub_efi_pe_entry - _head /* AddressOfEntryPoint */ .long .Lefi_header_end - _head /* BaseOfCode */ @@ -79,9 +79,9 @@ IMAGE_SCN_MEM_EXECUTE /* Characteristics */ .ascii ".data\0\0\0" - .long _end - __initdata_begin /* VirtualSize */ + .long _kernel_vsize /* VirtualSize */ .long __initdata_begin - _head /* VirtualAddress */ - .long _edata - __initdata_begin /* SizeOfRawData */ + .long _kernel_rsize /* SizeOfRawData */ .long __initdata_begin - _head /* PointerToRawData */ .long 0 /* PointerToRelocations */ diff --git a/arch/loongarch/kernel/fpu.S b/arch/loongarch/kernel/fpu.S index ccde94140c89..f3df5f0a4509 100644 --- a/arch/loongarch/kernel/fpu.S +++ b/arch/loongarch/kernel/fpu.S @@ -145,6 +145,154 @@ movgr2fcsr fcsr0, \tmp0 .endm + .macro sc_save_lsx base +#ifdef CONFIG_CPU_HAS_LSX + EX vst $vr0, \base, (0 * LSX_REG_WIDTH) + EX vst $vr1, \base, (1 * LSX_REG_WIDTH) + EX vst $vr2, \base, (2 * LSX_REG_WIDTH) + EX vst $vr3, \base, (3 * LSX_REG_WIDTH) + EX vst $vr4, \base, (4 * LSX_REG_WIDTH) + EX vst $vr5, \base, (5 * LSX_REG_WIDTH) + EX vst $vr6, \base, (6 * LSX_REG_WIDTH) + EX vst $vr7, \base, (7 * LSX_REG_WIDTH) + EX vst $vr8, \base, (8 * LSX_REG_WIDTH) + EX vst $vr9, \base, (9 * LSX_REG_WIDTH) + EX vst $vr10, \base, (10 * LSX_REG_WIDTH) + EX vst $vr11, \base, (11 * LSX_REG_WIDTH) + EX vst $vr12, \base, (12 * LSX_REG_WIDTH) + EX vst $vr13, \base, (13 * LSX_REG_WIDTH) + EX vst $vr14, \base, (14 * LSX_REG_WIDTH) + EX vst $vr15, \base, (15 * LSX_REG_WIDTH) + EX vst $vr16, \base, (16 * LSX_REG_WIDTH) + EX vst $vr17, \base, (17 * LSX_REG_WIDTH) + EX vst $vr18, \base, (18 * LSX_REG_WIDTH) + EX vst $vr19, \base, (19 * LSX_REG_WIDTH) + EX vst $vr20, \base, (20 * LSX_REG_WIDTH) + EX vst $vr21, \base, (21 * LSX_REG_WIDTH) + EX vst $vr22, \base, (22 * LSX_REG_WIDTH) + EX vst $vr23, \base, (23 * LSX_REG_WIDTH) + EX vst $vr24, \base, (24 * LSX_REG_WIDTH) + EX vst $vr25, \base, (25 * LSX_REG_WIDTH) + EX vst $vr26, \base, (26 * LSX_REG_WIDTH) + EX vst $vr27, \base, (27 * LSX_REG_WIDTH) + EX vst $vr28, \base, (28 * LSX_REG_WIDTH) + EX vst $vr29, \base, (29 * LSX_REG_WIDTH) + EX vst $vr30, \base, (30 * LSX_REG_WIDTH) + EX vst $vr31, \base, (31 * LSX_REG_WIDTH) +#endif + .endm + + .macro sc_restore_lsx base +#ifdef CONFIG_CPU_HAS_LSX + EX vld $vr0, \base, (0 * LSX_REG_WIDTH) + EX vld $vr1, \base, (1 * LSX_REG_WIDTH) + EX vld $vr2, \base, (2 * LSX_REG_WIDTH) + EX vld $vr3, \base, (3 * LSX_REG_WIDTH) + EX vld $vr4, \base, (4 * LSX_REG_WIDTH) + EX vld $vr5, \base, (5 * LSX_REG_WIDTH) + EX vld $vr6, \base, (6 * LSX_REG_WIDTH) + EX vld $vr7, \base, (7 * LSX_REG_WIDTH) + EX vld $vr8, \base, (8 * LSX_REG_WIDTH) + EX vld $vr9, \base, (9 * LSX_REG_WIDTH) + EX vld $vr10, \base, (10 * LSX_REG_WIDTH) + EX vld $vr11, \base, (11 * LSX_REG_WIDTH) + EX vld $vr12, \base, (12 * LSX_REG_WIDTH) + EX vld $vr13, \base, (13 * LSX_REG_WIDTH) + EX vld $vr14, \base, (14 * LSX_REG_WIDTH) + EX vld $vr15, \base, (15 * LSX_REG_WIDTH) + EX vld $vr16, \base, (16 * LSX_REG_WIDTH) + EX vld $vr17, \base, (17 * LSX_REG_WIDTH) + EX vld $vr18, \base, (18 * LSX_REG_WIDTH) + EX vld $vr19, \base, (19 * LSX_REG_WIDTH) + EX vld $vr20, \base, (20 * LSX_REG_WIDTH) + EX vld $vr21, \base, (21 * LSX_REG_WIDTH) + EX vld $vr22, \base, (22 * LSX_REG_WIDTH) + EX vld $vr23, \base, (23 * LSX_REG_WIDTH) + EX vld $vr24, \base, (24 * LSX_REG_WIDTH) + EX vld $vr25, \base, (25 * LSX_REG_WIDTH) + EX vld $vr26, \base, (26 * LSX_REG_WIDTH) + EX vld $vr27, \base, (27 * LSX_REG_WIDTH) + EX vld $vr28, \base, (28 * LSX_REG_WIDTH) + EX vld $vr29, \base, (29 * LSX_REG_WIDTH) + EX vld $vr30, \base, (30 * LSX_REG_WIDTH) + EX vld $vr31, \base, (31 * LSX_REG_WIDTH) +#endif + .endm + + .macro sc_save_lasx base +#ifdef CONFIG_CPU_HAS_LASX + EX xvst $xr0, \base, (0 * LASX_REG_WIDTH) + EX xvst $xr1, \base, (1 * LASX_REG_WIDTH) + EX xvst $xr2, \base, (2 * LASX_REG_WIDTH) + EX xvst $xr3, \base, (3 * LASX_REG_WIDTH) + EX xvst $xr4, \base, (4 * LASX_REG_WIDTH) + EX xvst $xr5, \base, (5 * LASX_REG_WIDTH) + EX xvst $xr6, \base, (6 * LASX_REG_WIDTH) + EX xvst $xr7, \base, (7 * LASX_REG_WIDTH) + EX xvst $xr8, \base, (8 * LASX_REG_WIDTH) + EX xvst $xr9, \base, (9 * LASX_REG_WIDTH) + EX xvst $xr10, \base, (10 * LASX_REG_WIDTH) + EX xvst $xr11, \base, (11 * LASX_REG_WIDTH) + EX xvst $xr12, \base, (12 * LASX_REG_WIDTH) + EX xvst $xr13, \base, (13 * LASX_REG_WIDTH) + EX xvst $xr14, \base, (14 * LASX_REG_WIDTH) + EX xvst $xr15, \base, (15 * LASX_REG_WIDTH) + EX xvst $xr16, \base, (16 * LASX_REG_WIDTH) + EX xvst $xr17, \base, (17 * LASX_REG_WIDTH) + EX xvst $xr18, \base, (18 * LASX_REG_WIDTH) + EX xvst $xr19, \base, (19 * LASX_REG_WIDTH) + EX xvst $xr20, \base, (20 * LASX_REG_WIDTH) + EX xvst $xr21, \base, (21 * LASX_REG_WIDTH) + EX xvst $xr22, \base, (22 * LASX_REG_WIDTH) + EX xvst $xr23, \base, (23 * LASX_REG_WIDTH) + EX xvst $xr24, \base, (24 * LASX_REG_WIDTH) + EX xvst $xr25, \base, (25 * LASX_REG_WIDTH) + EX xvst $xr26, \base, (26 * LASX_REG_WIDTH) + EX xvst $xr27, \base, (27 * LASX_REG_WIDTH) + EX xvst $xr28, \base, (28 * LASX_REG_WIDTH) + EX xvst $xr29, \base, (29 * LASX_REG_WIDTH) + EX xvst $xr30, \base, (30 * LASX_REG_WIDTH) + EX xvst $xr31, \base, (31 * LASX_REG_WIDTH) +#endif + .endm + + .macro sc_restore_lasx base +#ifdef CONFIG_CPU_HAS_LASX + EX xvld $xr0, \base, (0 * LASX_REG_WIDTH) + EX xvld $xr1, \base, (1 * LASX_REG_WIDTH) + EX xvld $xr2, \base, (2 * LASX_REG_WIDTH) + EX xvld $xr3, \base, (3 * LASX_REG_WIDTH) + EX xvld $xr4, \base, (4 * LASX_REG_WIDTH) + EX xvld $xr5, \base, (5 * LASX_REG_WIDTH) + EX xvld $xr6, \base, (6 * LASX_REG_WIDTH) + EX xvld $xr7, \base, (7 * LASX_REG_WIDTH) + EX xvld $xr8, \base, (8 * LASX_REG_WIDTH) + EX xvld $xr9, \base, (9 * LASX_REG_WIDTH) + EX xvld $xr10, \base, (10 * LASX_REG_WIDTH) + EX xvld $xr11, \base, (11 * LASX_REG_WIDTH) + EX xvld $xr12, \base, (12 * LASX_REG_WIDTH) + EX xvld $xr13, \base, (13 * LASX_REG_WIDTH) + EX xvld $xr14, \base, (14 * LASX_REG_WIDTH) + EX xvld $xr15, \base, (15 * LASX_REG_WIDTH) + EX xvld $xr16, \base, (16 * LASX_REG_WIDTH) + EX xvld $xr17, \base, (17 * LASX_REG_WIDTH) + EX xvld $xr18, \base, (18 * LASX_REG_WIDTH) + EX xvld $xr19, \base, (19 * LASX_REG_WIDTH) + EX xvld $xr20, \base, (20 * LASX_REG_WIDTH) + EX xvld $xr21, \base, (21 * LASX_REG_WIDTH) + EX xvld $xr22, \base, (22 * LASX_REG_WIDTH) + EX xvld $xr23, \base, (23 * LASX_REG_WIDTH) + EX xvld $xr24, \base, (24 * LASX_REG_WIDTH) + EX xvld $xr25, \base, (25 * LASX_REG_WIDTH) + EX xvld $xr26, \base, (26 * LASX_REG_WIDTH) + EX xvld $xr27, \base, (27 * LASX_REG_WIDTH) + EX xvld $xr28, \base, (28 * LASX_REG_WIDTH) + EX xvld $xr29, \base, (29 * LASX_REG_WIDTH) + EX xvld $xr30, \base, (30 * LASX_REG_WIDTH) + EX xvld $xr31, \base, (31 * LASX_REG_WIDTH) +#endif + .endm + /* * Save a thread's fp context. */ @@ -166,6 +314,76 @@ SYM_FUNC_START(_restore_fp) jr ra SYM_FUNC_END(_restore_fp) +#ifdef CONFIG_CPU_HAS_LSX + +/* + * Save a thread's LSX vector context. + */ +SYM_FUNC_START(_save_lsx) + lsx_save_all a0 t1 t2 + jr ra +SYM_FUNC_END(_save_lsx) +EXPORT_SYMBOL(_save_lsx) + +/* + * Restore a thread's LSX vector context. + */ +SYM_FUNC_START(_restore_lsx) + lsx_restore_all a0 t1 t2 + jr ra +SYM_FUNC_END(_restore_lsx) + +SYM_FUNC_START(_save_lsx_upper) + lsx_save_all_upper a0 t0 t1 + jr ra +SYM_FUNC_END(_save_lsx_upper) + +SYM_FUNC_START(_restore_lsx_upper) + lsx_restore_all_upper a0 t0 t1 + jr ra +SYM_FUNC_END(_restore_lsx_upper) + +SYM_FUNC_START(_init_lsx_upper) + lsx_init_all_upper t1 + jr ra +SYM_FUNC_END(_init_lsx_upper) +#endif + +#ifdef CONFIG_CPU_HAS_LASX + +/* + * Save a thread's LASX vector context. + */ +SYM_FUNC_START(_save_lasx) + lasx_save_all a0 t1 t2 + jr ra +SYM_FUNC_END(_save_lasx) +EXPORT_SYMBOL(_save_lasx) + +/* + * Restore a thread's LASX vector context. + */ +SYM_FUNC_START(_restore_lasx) + lasx_restore_all a0 t1 t2 + jr ra +SYM_FUNC_END(_restore_lasx) + +SYM_FUNC_START(_save_lasx_upper) + lasx_save_all_upper a0 t0 t1 + jr ra +SYM_FUNC_END(_save_lasx_upper) + +SYM_FUNC_START(_restore_lasx_upper) + lasx_restore_all_upper a0 t0 t1 + jr ra +SYM_FUNC_END(_restore_lasx_upper) + +SYM_FUNC_START(_init_lasx_upper) + lasx_init_all_upper t1 + jr ra +SYM_FUNC_END(_init_lasx_upper) +#endif + /* * Load the FPU with signalling NANS. This bit pattern we're using has * the property that no matter whether considered as single or as double @@ -244,6 +462,58 @@ SYM_FUNC_START(_restore_fp_context) jr ra SYM_FUNC_END(_restore_fp_context) +/* + * a0: fpregs + * a1: fcc + * a2: fcsr + */ +SYM_FUNC_START(_save_lsx_context) + sc_save_fcc a1, t0, t1 + sc_save_fcsr a2, t0 + sc_save_lsx a0 + li.w a0, 0 # success + jr ra +SYM_FUNC_END(_save_lsx_context) + +/* + * a0: fpregs + * a1: fcc + * a2: fcsr + */ +SYM_FUNC_START(_restore_lsx_context) + sc_restore_lsx a0 + sc_restore_fcc a1, t1, t2 + sc_restore_fcsr a2, t1 + li.w a0, 0 # success + jr ra +SYM_FUNC_END(_restore_lsx_context) + +/* + * a0: fpregs + * a1: fcc + * a2: fcsr + */ +SYM_FUNC_START(_save_lasx_context) + sc_save_fcc a1, t0, t1 + sc_save_fcsr a2, t0 + sc_save_lasx a0 + li.w a0, 0 # success + jr ra +SYM_FUNC_END(_save_lasx_context) + +/* + * a0: fpregs + * a1: fcc + * a2: fcsr + */ +SYM_FUNC_START(_restore_lasx_context) + sc_restore_lasx a0 + sc_restore_fcc a1, t1, t2 + sc_restore_fcsr a2, t1 + li.w a0, 0 # success + jr ra +SYM_FUNC_END(_restore_lasx_context) + SYM_FUNC_START(fault) li.w a0, -EFAULT # failure jr ra diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S index aa64b179744f..5e828a8bc0a0 100644 --- a/arch/loongarch/kernel/head.S +++ b/arch/loongarch/kernel/head.S @@ -23,7 +23,7 @@ _head: .word MZ_MAGIC /* "MZ", MS-DOS header */ .org 0x8 .dword kernel_entry /* Kernel entry point */ - .dword _end - _text /* Kernel image effective size */ + .dword _kernel_asize /* Kernel image effective size */ .quad PHYS_LINK_KADDR /* Kernel image load offset from start of RAM */ .org 0x38 /* 0x20 ~ 0x37 reserved */ .long LINUX_PE_MAGIC @@ -32,9 +32,9 @@ _head: pe_header: __EFI_PE_HEADER -SYM_DATA(kernel_asize, .long _end - _text); -SYM_DATA(kernel_fsize, .long _edata - _text); -SYM_DATA(kernel_offset, .long kernel_offset - _text); +SYM_DATA(kernel_asize, .long _kernel_asize); +SYM_DATA(kernel_fsize, .long _kernel_fsize); +SYM_DATA(kernel_offset, .long _kernel_offset); #endif diff --git a/arch/loongarch/kernel/inst.c b/arch/loongarch/kernel/inst.c index 258ef267cd30..3050329556d1 100644 --- a/arch/loongarch/kernel/inst.c +++ b/arch/loongarch/kernel/inst.c @@ -133,6 +133,51 @@ void simu_branch(struct pt_regs *regs, union loongarch_instruction insn) } } +bool insns_not_supported(union loongarch_instruction insn) +{ + switch (insn.reg3_format.opcode) { + case amswapw_op ... ammindbdu_op: + pr_notice("atomic memory access instructions are not supported\n"); + return true; + } + + switch (insn.reg2i14_format.opcode) { + case llw_op: + case lld_op: + case scw_op: + case scd_op: + pr_notice("ll and sc instructions are not supported\n"); + return true; + } + + switch (insn.reg1i21_format.opcode) { + case bceqz_op: + pr_notice("bceqz and bcnez instructions are not supported\n"); + return true; + } + + return false; +} + +bool insns_need_simulation(union loongarch_instruction insn) +{ + if (is_pc_ins(&insn)) + return true; + + if (is_branch_ins(&insn)) + return true; + + return false; +} + +void arch_simulate_insn(union loongarch_instruction insn, struct pt_regs *regs) +{ + if (is_pc_ins(&insn)) + simu_pc(regs, insn); + else if (is_branch_ins(&insn)) + simu_branch(regs, insn); +} + int larch_insn_read(void *addr, u32 *insnp) { int ret; @@ -208,6 +253,20 @@ u32 larch_insn_gen_bl(unsigned long pc, unsigned long dest) return insn.word; } +u32 larch_insn_gen_break(int imm) +{ + union loongarch_instruction insn; + + if (imm < 0 || imm >= SZ_32K) { + pr_warn("The generated break instruction is out of range.\n"); + return INSN_BREAK; + } + + emit_break(&insn, imm); + + return insn.word; +} + u32 larch_insn_gen_or(enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk) { union loongarch_instruction insn; @@ -226,6 +285,11 @@ u32 larch_insn_gen_lu12iw(enum loongarch_gpr rd, int imm) { union loongarch_instruction insn; + if (imm < -SZ_512K || imm >= SZ_512K) { + pr_warn("The generated lu12i.w instruction is out of range.\n"); + return INSN_BREAK; + } + emit_lu12iw(&insn, rd, imm); return insn.word; @@ -235,6 +299,11 @@ u32 larch_insn_gen_lu32id(enum loongarch_gpr rd, int imm) { union loongarch_instruction insn; + if (imm < -SZ_512K || imm >= SZ_512K) { + pr_warn("The generated lu32i.d instruction is out of range.\n"); + return INSN_BREAK; + } + emit_lu32id(&insn, rd, imm); return insn.word; @@ -244,16 +313,26 @@ u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm) { union loongarch_instruction insn; + if (imm < -SZ_2K || imm >= SZ_2K) { + pr_warn("The generated lu52i.d instruction is out of range.\n"); + return INSN_BREAK; + } + emit_lu52id(&insn, rd, rj, imm); return insn.word; } -u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, unsigned long pc, unsigned long dest) +u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm) { union loongarch_instruction insn; - emit_jirl(&insn, rj, rd, (dest - pc) >> 2); + if ((imm & 3) || imm < -SZ_128K || imm >= SZ_128K) { + pr_warn("The generated jirl instruction is out of range.\n"); + return INSN_BREAK; + } + + emit_jirl(&insn, rj, rd, imm >> 2); return insn.word; } diff --git a/arch/loongarch/kernel/jump_label.c b/arch/loongarch/kernel/jump_label.c new file mode 100644 index 000000000000..31891214b767 --- /dev/null +++ b/arch/loongarch/kernel/jump_label.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2023 Loongson Technology Corporation Limited + * + * Based on arch/arm64/kernel/jump_label.c + */ +#include <linux/kernel.h> +#include <linux/jump_label.h> +#include <asm/inst.h> + +void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type) +{ + u32 insn; + void *addr = (void *)jump_entry_code(entry); + + if (type == JUMP_LABEL_JMP) + insn = larch_insn_gen_b(jump_entry_code(entry), jump_entry_target(entry)); + else + insn = larch_insn_gen_nop(); + + larch_insn_patch_text(addr, insn); +} diff --git a/arch/loongarch/kernel/kprobes.c b/arch/loongarch/kernel/kprobes.c index 56c8c4b09a42..17b040bd6067 100644 --- a/arch/loongarch/kernel/kprobes.c +++ b/arch/loongarch/kernel/kprobes.c @@ -4,69 +4,16 @@ #include <linux/preempt.h> #include <asm/break.h> -static const union loongarch_instruction breakpoint_insn = { - .reg0i15_format = { - .opcode = break_op, - .immediate = BRK_KPROBE_BP, - } -}; - -static const union loongarch_instruction singlestep_insn = { - .reg0i15_format = { - .opcode = break_op, - .immediate = BRK_KPROBE_SSTEPBP, - } -}; +#define KPROBE_BP_INSN larch_insn_gen_break(BRK_KPROBE_BP) +#define KPROBE_SSTEPBP_INSN larch_insn_gen_break(BRK_KPROBE_SSTEPBP) DEFINE_PER_CPU(struct kprobe *, current_kprobe); DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); -static bool insns_not_supported(union loongarch_instruction insn) -{ - switch (insn.reg2i14_format.opcode) { - case llw_op: - case lld_op: - case scw_op: - case scd_op: - pr_notice("kprobe: ll and sc instructions are not supported\n"); - return true; - } - - switch (insn.reg1i21_format.opcode) { - case bceqz_op: - pr_notice("kprobe: bceqz and bcnez instructions are not supported\n"); - return true; - } - - return false; -} -NOKPROBE_SYMBOL(insns_not_supported); - -static bool insns_need_simulation(struct kprobe *p) -{ - if (is_pc_ins(&p->opcode)) - return true; - - if (is_branch_ins(&p->opcode)) - return true; - - return false; -} -NOKPROBE_SYMBOL(insns_need_simulation); - -static void arch_simulate_insn(struct kprobe *p, struct pt_regs *regs) -{ - if (is_pc_ins(&p->opcode)) - simu_pc(regs, p->opcode); - else if (is_branch_ins(&p->opcode)) - simu_branch(regs, p->opcode); -} -NOKPROBE_SYMBOL(arch_simulate_insn); - static void arch_prepare_ss_slot(struct kprobe *p) { p->ainsn.insn[0] = *p->addr; - p->ainsn.insn[1] = singlestep_insn; + p->ainsn.insn[1] = KPROBE_SSTEPBP_INSN; p->ainsn.restore = (unsigned long)p->addr + LOONGARCH_INSN_SIZE; } NOKPROBE_SYMBOL(arch_prepare_ss_slot); @@ -79,17 +26,20 @@ NOKPROBE_SYMBOL(arch_prepare_simulate); int arch_prepare_kprobe(struct kprobe *p) { + union loongarch_instruction insn; + if ((unsigned long)p->addr & 0x3) return -EILSEQ; /* copy instruction */ p->opcode = *p->addr; + insn.word = p->opcode; /* decode instruction */ - if (insns_not_supported(p->opcode)) + if (insns_not_supported(insn)) return -EINVAL; - if (insns_need_simulation(p)) { + if (insns_need_simulation(insn)) { p->ainsn.insn = NULL; } else { p->ainsn.insn = get_insn_slot(); @@ -110,7 +60,7 @@ NOKPROBE_SYMBOL(arch_prepare_kprobe); /* Install breakpoint in text */ void arch_arm_kprobe(struct kprobe *p) { - *p->addr = breakpoint_insn; + *p->addr = KPROBE_BP_INSN; flush_insn_slot(p); } NOKPROBE_SYMBOL(arch_arm_kprobe); @@ -205,6 +155,8 @@ NOKPROBE_SYMBOL(post_kprobe_handler); static void setup_singlestep(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb, int reenter) { + union loongarch_instruction insn; + if (reenter) { save_previous_kprobe(kcb); set_current_kprobe(p); @@ -220,7 +172,8 @@ static void setup_singlestep(struct kprobe *p, struct pt_regs *regs, regs->csr_era = (unsigned long)p->ainsn.insn; } else { /* simulate single steping */ - arch_simulate_insn(p, regs); + insn.word = p->opcode; + arch_simulate_insn(insn, regs); /* now go for post processing */ post_kprobe_handler(p, kcb, regs); } @@ -295,7 +248,7 @@ bool kprobe_breakpoint_handler(struct pt_regs *regs) } } - if (addr->word != breakpoint_insn.word) { + if (*addr != KPROBE_BP_INSN) { /* * The breakpoint instruction was removed right * after we hit it. Another cpu has removed @@ -378,27 +331,6 @@ int __init arch_init_kprobes(void) return 0; } -/* ASM function that handles the kretprobes must not be probed */ -NOKPROBE_SYMBOL(__kretprobe_trampoline); - -/* Called from __kretprobe_trampoline */ -void __used *trampoline_probe_handler(struct pt_regs *regs) -{ - return (void *)kretprobe_trampoline_handler(regs, NULL); -} -NOKPROBE_SYMBOL(trampoline_probe_handler); - -void arch_prepare_kretprobe(struct kretprobe_instance *ri, - struct pt_regs *regs) -{ - ri->ret_addr = (kprobe_opcode_t *)regs->regs[1]; - ri->fp = NULL; - - /* Replace the return addr with trampoline addr */ - regs->regs[1] = (unsigned long)&__kretprobe_trampoline; -} -NOKPROBE_SYMBOL(arch_prepare_kretprobe); - int arch_trampoline_kprobe(struct kprobe *p) { return 0; diff --git a/arch/loongarch/kernel/mcount.S b/arch/loongarch/kernel/mcount.S index 8cdc1563cd33..cb8e5803de4b 100644 --- a/arch/loongarch/kernel/mcount.S +++ b/arch/loongarch/kernel/mcount.S @@ -79,18 +79,20 @@ SYM_FUNC_START(ftrace_graph_caller) SYM_FUNC_END(ftrace_graph_caller) SYM_FUNC_START(return_to_handler) - PTR_ADDI sp, sp, -2 * SZREG - PTR_S a0, sp, 0 - PTR_S a1, sp, SZREG + PTR_ADDI sp, sp, -FGRET_REGS_SIZE + PTR_S a0, sp, FGRET_REGS_A0 + PTR_S a1, sp, FGRET_REGS_A1 + PTR_S zero, sp, FGRET_REGS_FP + move a0, sp bl ftrace_return_to_handler /* Restore the real parent address: a0 -> ra */ move ra, a0 - PTR_L a0, sp, 0 - PTR_L a1, sp, SZREG - PTR_ADDI sp, sp, 2 * SZREG + PTR_L a0, sp, FGRET_REGS_A0 + PTR_L a1, sp, FGRET_REGS_A1 + PTR_ADDI sp, sp, FGRET_REGS_SIZE jr ra SYM_FUNC_END(return_to_handler) #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/loongarch/kernel/mcount_dyn.S b/arch/loongarch/kernel/mcount_dyn.S index c7d961fc72c2..e16ab0b98e5a 100644 --- a/arch/loongarch/kernel/mcount_dyn.S +++ b/arch/loongarch/kernel/mcount_dyn.S @@ -136,18 +136,19 @@ SYM_CODE_END(ftrace_graph_caller) SYM_CODE_START(return_to_handler) /* Save return value regs */ - PTR_ADDI sp, sp, -2 * SZREG - PTR_S a0, sp, 0 - PTR_S a1, sp, SZREG + PTR_ADDI sp, sp, -FGRET_REGS_SIZE + PTR_S a0, sp, FGRET_REGS_A0 + PTR_S a1, sp, FGRET_REGS_A1 + PTR_S zero, sp, FGRET_REGS_FP - move a0, zero + move a0, sp bl ftrace_return_to_handler move ra, a0 /* Restore return value regs */ - PTR_L a0, sp, 0 - PTR_L a1, sp, SZREG - PTR_ADDI sp, sp, 2 * SZREG + PTR_L a0, sp, FGRET_REGS_A0 + PTR_L a1, sp, FGRET_REGS_A1 + PTR_ADDI sp, sp, FGRET_REGS_SIZE jr ra SYM_CODE_END(return_to_handler) diff --git a/arch/loongarch/kernel/proc.c b/arch/loongarch/kernel/proc.c index 0d82907b5404..0d33cbc47e51 100644 --- a/arch/loongarch/kernel/proc.c +++ b/arch/loongarch/kernel/proc.c @@ -49,6 +49,7 @@ static int show_cpuinfo(struct seq_file *m, void *v) seq_printf(m, "processor\t\t: %ld\n", n); seq_printf(m, "package\t\t\t: %d\n", cpu_data[n].package); seq_printf(m, "core\t\t\t: %d\n", cpu_data[n].core); + seq_printf(m, "global_id\t\t: %d\n", cpu_data[n].global_id); seq_printf(m, "CPU Family\t\t: %s\n", __cpu_family[n]); seq_printf(m, "Model Name\t\t: %s\n", __cpu_full_name[n]); seq_printf(m, "CPU Revision\t\t: 0x%02x\n", version); @@ -79,6 +80,7 @@ static int show_cpuinfo(struct seq_file *m, void *v) if (cpu_has_crc32) seq_printf(m, " crc32"); if (cpu_has_complex) seq_printf(m, " complex"); if (cpu_has_crypto) seq_printf(m, " crypto"); + if (cpu_has_ptw) seq_printf(m, " ptw"); if (cpu_has_lvz) seq_printf(m, " lvz"); if (cpu_has_lbt_x86) seq_printf(m, " lbt_x86"); if (cpu_has_lbt_arm) seq_printf(m, " lbt_arm"); diff --git a/arch/loongarch/kernel/process.c b/arch/loongarch/kernel/process.c index b71e17c1cc0c..2e04eb07abb6 100644 --- a/arch/loongarch/kernel/process.c +++ b/arch/loongarch/kernel/process.c @@ -117,8 +117,14 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) */ preempt_disable(); - if (is_fpu_owner()) - save_fp(current); + if (is_fpu_owner()) { + if (is_lasx_enabled()) + save_lasx(current); + else if (is_lsx_enabled()) + save_lsx(current); + else + save_fp(current); + } preempt_enable(); @@ -285,7 +291,7 @@ unsigned long stack_top(void) /* Space for the VDSO & data page */ top -= PAGE_ALIGN(current->thread.vdso->size); - top -= PAGE_SIZE; + top -= VVAR_SIZE; /* Space to randomize the VDSO base */ if (current->flags & PF_RANDOMIZE) diff --git a/arch/loongarch/kernel/ptrace.c b/arch/loongarch/kernel/ptrace.c index 5fcffb452367..a0767c3a0f0a 100644 --- a/arch/loongarch/kernel/ptrace.c +++ b/arch/loongarch/kernel/ptrace.c @@ -250,6 +250,90 @@ static int cfg_set(struct task_struct *target, return 0; } +#ifdef CONFIG_CPU_HAS_LSX + +static void copy_pad_fprs(struct task_struct *target, + const struct user_regset *regset, + struct membuf *to, unsigned int live_sz) +{ + int i, j; + unsigned long long fill = ~0ull; + unsigned int cp_sz, pad_sz; + + cp_sz = min(regset->size, live_sz); + pad_sz = regset->size - cp_sz; + WARN_ON(pad_sz % sizeof(fill)); + + for (i = 0; i < NUM_FPU_REGS; i++) { + membuf_write(to, &target->thread.fpu.fpr[i], cp_sz); + for (j = 0; j < (pad_sz / sizeof(fill)); j++) { + membuf_store(to, fill); + } + } +} + +static int simd_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + const unsigned int wr_size = NUM_FPU_REGS * regset->size; + + if (!tsk_used_math(target)) { + /* The task hasn't used FP or LSX, fill with 0xff */ + copy_pad_fprs(target, regset, &to, 0); + } else if (!test_tsk_thread_flag(target, TIF_LSX_CTX_LIVE)) { + /* Copy scalar FP context, fill the rest with 0xff */ + copy_pad_fprs(target, regset, &to, 8); +#ifdef CONFIG_CPU_HAS_LASX + } else if (!test_tsk_thread_flag(target, TIF_LASX_CTX_LIVE)) { + /* Copy LSX 128 Bit context, fill the rest with 0xff */ + copy_pad_fprs(target, regset, &to, 16); +#endif + } else if (sizeof(target->thread.fpu.fpr[0]) == regset->size) { + /* Trivially copy the vector registers */ + membuf_write(&to, &target->thread.fpu.fpr, wr_size); + } else { + /* Copy as much context as possible, fill the rest with 0xff */ + copy_pad_fprs(target, regset, &to, sizeof(target->thread.fpu.fpr[0])); + } + + return 0; +} + +static int simd_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + const unsigned int wr_size = NUM_FPU_REGS * regset->size; + unsigned int cp_sz; + int i, err, start; + + init_fp_ctx(target); + + if (sizeof(target->thread.fpu.fpr[0]) == regset->size) { + /* Trivially copy the vector registers */ + err = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &target->thread.fpu.fpr, + 0, wr_size); + } else { + /* Copy as much context as possible */ + cp_sz = min_t(unsigned int, regset->size, + sizeof(target->thread.fpu.fpr[0])); + + i = start = err = 0; + for (; i < NUM_FPU_REGS; i++, start += regset->size) { + err |= user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &target->thread.fpu.fpr[i], + start, start + cp_sz); + } + } + + return err; +} + +#endif /* CONFIG_CPU_HAS_LSX */ + #ifdef CONFIG_HAVE_HW_BREAKPOINT /* @@ -708,6 +792,12 @@ enum loongarch_regset { REGSET_GPR, REGSET_FPR, REGSET_CPUCFG, +#ifdef CONFIG_CPU_HAS_LSX + REGSET_LSX, +#endif +#ifdef CONFIG_CPU_HAS_LASX + REGSET_LASX, +#endif #ifdef CONFIG_HAVE_HW_BREAKPOINT REGSET_HW_BREAK, REGSET_HW_WATCH, @@ -739,6 +829,26 @@ static const struct user_regset loongarch64_regsets[] = { .regset_get = cfg_get, .set = cfg_set, }, +#ifdef CONFIG_CPU_HAS_LSX + [REGSET_LSX] = { + .core_note_type = NT_LOONGARCH_LSX, + .n = NUM_FPU_REGS, + .size = 16, + .align = 16, + .regset_get = simd_get, + .set = simd_set, + }, +#endif +#ifdef CONFIG_CPU_HAS_LASX + [REGSET_LASX] = { + .core_note_type = NT_LOONGARCH_LASX, + .n = NUM_FPU_REGS, + .size = 32, + .align = 32, + .regset_get = simd_get, + .set = simd_set, + }, +#endif #ifdef CONFIG_HAVE_HW_BREAKPOINT [REGSET_HW_BREAK] = { .core_note_type = NT_LOONGARCH_HW_BREAK, diff --git a/arch/loongarch/kernel/rethook.c b/arch/loongarch/kernel/rethook.c new file mode 100644 index 000000000000..db1c5f5024fd --- /dev/null +++ b/arch/loongarch/kernel/rethook.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Generic return hook for LoongArch. + */ + +#include <linux/kprobes.h> +#include <linux/rethook.h> +#include "rethook.h" + +/* This is called from arch_rethook_trampoline() */ +unsigned long __used arch_rethook_trampoline_callback(struct pt_regs *regs) +{ + return rethook_trampoline_handler(regs, 0); +} +NOKPROBE_SYMBOL(arch_rethook_trampoline_callback); + +void arch_rethook_prepare(struct rethook_node *rhn, struct pt_regs *regs, bool mcount) +{ + rhn->frame = 0; + rhn->ret_addr = regs->regs[1]; + + /* replace return addr with trampoline */ + regs->regs[1] = (unsigned long)arch_rethook_trampoline; +} +NOKPROBE_SYMBOL(arch_rethook_prepare); + +/* ASM function that handles the rethook must not be probed itself */ +NOKPROBE_SYMBOL(arch_rethook_trampoline); diff --git a/arch/loongarch/kernel/rethook.h b/arch/loongarch/kernel/rethook.h new file mode 100644 index 000000000000..3f1c1edf0d0b --- /dev/null +++ b/arch/loongarch/kernel/rethook.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LOONGARCH_RETHOOK_H +#define __LOONGARCH_RETHOOK_H + +unsigned long arch_rethook_trampoline_callback(struct pt_regs *regs); +void arch_rethook_prepare(struct rethook_node *rhn, struct pt_regs *regs, bool mcount); + +#endif diff --git a/arch/loongarch/kernel/kprobes_trampoline.S b/arch/loongarch/kernel/rethook_trampoline.S index af94b0d213fa..bd5772c96338 100644 --- a/arch/loongarch/kernel/kprobes_trampoline.S +++ b/arch/loongarch/kernel/rethook_trampoline.S @@ -75,7 +75,7 @@ csrxchg t0, t1, LOONGARCH_CSR_CRMD .endm -SYM_CODE_START(__kretprobe_trampoline) +SYM_CODE_START(arch_rethook_trampoline) addi.d sp, sp, -PT_SIZE save_all_base_regs @@ -84,7 +84,7 @@ SYM_CODE_START(__kretprobe_trampoline) move a0, sp /* pt_regs */ - bl trampoline_probe_handler + bl arch_rethook_trampoline_callback /* use the result as the return-address */ move ra, a0 @@ -93,4 +93,4 @@ SYM_CODE_START(__kretprobe_trampoline) addi.d sp, sp, PT_SIZE jr ra -SYM_CODE_END(__kretprobe_trampoline) +SYM_CODE_END(arch_rethook_trampoline) diff --git a/arch/loongarch/kernel/signal.c b/arch/loongarch/kernel/signal.c index 8f5b7986374b..ceb899366c0a 100644 --- a/arch/loongarch/kernel/signal.c +++ b/arch/loongarch/kernel/signal.c @@ -50,6 +50,14 @@ extern asmlinkage int _save_fp_context(void __user *fpregs, void __user *fcc, void __user *csr); extern asmlinkage int _restore_fp_context(void __user *fpregs, void __user *fcc, void __user *csr); +extern asmlinkage int +_save_lsx_context(void __user *fpregs, void __user *fcc, void __user *fcsr); +extern asmlinkage int +_restore_lsx_context(void __user *fpregs, void __user *fcc, void __user *fcsr); +extern asmlinkage int +_save_lasx_context(void __user *fpregs, void __user *fcc, void __user *fcsr); +extern asmlinkage int +_restore_lasx_context(void __user *fpregs, void __user *fcc, void __user *fcsr); struct rt_sigframe { struct siginfo rs_info; @@ -65,6 +73,8 @@ struct extctx_layout { unsigned long size; unsigned int flags; struct _ctx_layout fpu; + struct _ctx_layout lsx; + struct _ctx_layout lasx; struct _ctx_layout end; }; @@ -115,6 +125,96 @@ static int copy_fpu_from_sigcontext(struct fpu_context __user *ctx) return err; } +static int copy_lsx_to_sigcontext(struct lsx_context __user *ctx) +{ + int i; + int err = 0; + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + for (i = 0; i < NUM_FPU_REGS; i++) { + err |= __put_user(get_fpr64(¤t->thread.fpu.fpr[i], 0), + ®s[2*i]); + err |= __put_user(get_fpr64(¤t->thread.fpu.fpr[i], 1), + ®s[2*i+1]); + } + err |= __put_user(current->thread.fpu.fcc, fcc); + err |= __put_user(current->thread.fpu.fcsr, fcsr); + + return err; +} + +static int copy_lsx_from_sigcontext(struct lsx_context __user *ctx) +{ + int i; + int err = 0; + u64 fpr_val; + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + for (i = 0; i < NUM_FPU_REGS; i++) { + err |= __get_user(fpr_val, ®s[2*i]); + set_fpr64(¤t->thread.fpu.fpr[i], 0, fpr_val); + err |= __get_user(fpr_val, ®s[2*i+1]); + set_fpr64(¤t->thread.fpu.fpr[i], 1, fpr_val); + } + err |= __get_user(current->thread.fpu.fcc, fcc); + err |= __get_user(current->thread.fpu.fcsr, fcsr); + + return err; +} + +static int copy_lasx_to_sigcontext(struct lasx_context __user *ctx) +{ + int i; + int err = 0; + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + for (i = 0; i < NUM_FPU_REGS; i++) { + err |= __put_user(get_fpr64(¤t->thread.fpu.fpr[i], 0), + ®s[4*i]); + err |= __put_user(get_fpr64(¤t->thread.fpu.fpr[i], 1), + ®s[4*i+1]); + err |= __put_user(get_fpr64(¤t->thread.fpu.fpr[i], 2), + ®s[4*i+2]); + err |= __put_user(get_fpr64(¤t->thread.fpu.fpr[i], 3), + ®s[4*i+3]); + } + err |= __put_user(current->thread.fpu.fcc, fcc); + err |= __put_user(current->thread.fpu.fcsr, fcsr); + + return err; +} + +static int copy_lasx_from_sigcontext(struct lasx_context __user *ctx) +{ + int i; + int err = 0; + u64 fpr_val; + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + for (i = 0; i < NUM_FPU_REGS; i++) { + err |= __get_user(fpr_val, ®s[4*i]); + set_fpr64(¤t->thread.fpu.fpr[i], 0, fpr_val); + err |= __get_user(fpr_val, ®s[4*i+1]); + set_fpr64(¤t->thread.fpu.fpr[i], 1, fpr_val); + err |= __get_user(fpr_val, ®s[4*i+2]); + set_fpr64(¤t->thread.fpu.fpr[i], 2, fpr_val); + err |= __get_user(fpr_val, ®s[4*i+3]); + set_fpr64(¤t->thread.fpu.fpr[i], 3, fpr_val); + } + err |= __get_user(current->thread.fpu.fcc, fcc); + err |= __get_user(current->thread.fpu.fcsr, fcsr); + + return err; +} + /* * Wrappers for the assembly _{save,restore}_fp_context functions. */ @@ -136,6 +236,42 @@ static int restore_hw_fpu_context(struct fpu_context __user *ctx) return _restore_fp_context(regs, fcc, fcsr); } +static int save_hw_lsx_context(struct lsx_context __user *ctx) +{ + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + return _save_lsx_context(regs, fcc, fcsr); +} + +static int restore_hw_lsx_context(struct lsx_context __user *ctx) +{ + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + return _restore_lsx_context(regs, fcc, fcsr); +} + +static int save_hw_lasx_context(struct lasx_context __user *ctx) +{ + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + return _save_lasx_context(regs, fcc, fcsr); +} + +static int restore_hw_lasx_context(struct lasx_context __user *ctx) +{ + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + return _restore_lasx_context(regs, fcc, fcsr); +} + static int fcsr_pending(unsigned int __user *fcsr) { int err, sig = 0; @@ -227,6 +363,162 @@ static int protected_restore_fpu_context(struct extctx_layout *extctx) return err ?: sig; } +static int protected_save_lsx_context(struct extctx_layout *extctx) +{ + int err = 0; + struct sctx_info __user *info = extctx->lsx.addr; + struct lsx_context __user *lsx_ctx = (struct lsx_context *)get_ctx_through_ctxinfo(info); + uint64_t __user *regs = (uint64_t *)&lsx_ctx->regs; + uint64_t __user *fcc = &lsx_ctx->fcc; + uint32_t __user *fcsr = &lsx_ctx->fcsr; + + while (1) { + lock_fpu_owner(); + if (is_lsx_enabled()) + err = save_hw_lsx_context(lsx_ctx); + else { + if (is_fpu_owner()) + save_fp(current); + err = copy_lsx_to_sigcontext(lsx_ctx); + } + unlock_fpu_owner(); + + err |= __put_user(LSX_CTX_MAGIC, &info->magic); + err |= __put_user(extctx->lsx.size, &info->size); + + if (likely(!err)) + break; + /* Touch the LSX context and try again */ + err = __put_user(0, ®s[0]) | + __put_user(0, ®s[32*2-1]) | + __put_user(0, fcc) | + __put_user(0, fcsr); + if (err) + return err; /* really bad sigcontext */ + } + + return err; +} + +static int protected_restore_lsx_context(struct extctx_layout *extctx) +{ + int err = 0, sig = 0, tmp __maybe_unused; + struct sctx_info __user *info = extctx->lsx.addr; + struct lsx_context __user *lsx_ctx = (struct lsx_context *)get_ctx_through_ctxinfo(info); + uint64_t __user *regs = (uint64_t *)&lsx_ctx->regs; + uint64_t __user *fcc = &lsx_ctx->fcc; + uint32_t __user *fcsr = &lsx_ctx->fcsr; + + err = sig = fcsr_pending(fcsr); + if (err < 0) + return err; + + while (1) { + lock_fpu_owner(); + if (is_lsx_enabled()) + err = restore_hw_lsx_context(lsx_ctx); + else { + err = copy_lsx_from_sigcontext(lsx_ctx); + if (is_fpu_owner()) + restore_fp(current); + } + unlock_fpu_owner(); + + if (likely(!err)) + break; + /* Touch the LSX context and try again */ + err = __get_user(tmp, ®s[0]) | + __get_user(tmp, ®s[32*2-1]) | + __get_user(tmp, fcc) | + __get_user(tmp, fcsr); + if (err) + break; /* really bad sigcontext */ + } + + return err ?: sig; +} + +static int protected_save_lasx_context(struct extctx_layout *extctx) +{ + int err = 0; + struct sctx_info __user *info = extctx->lasx.addr; + struct lasx_context __user *lasx_ctx = + (struct lasx_context *)get_ctx_through_ctxinfo(info); + uint64_t __user *regs = (uint64_t *)&lasx_ctx->regs; + uint64_t __user *fcc = &lasx_ctx->fcc; + uint32_t __user *fcsr = &lasx_ctx->fcsr; + + while (1) { + lock_fpu_owner(); + if (is_lasx_enabled()) + err = save_hw_lasx_context(lasx_ctx); + else { + if (is_lsx_enabled()) + save_lsx(current); + else if (is_fpu_owner()) + save_fp(current); + err = copy_lasx_to_sigcontext(lasx_ctx); + } + unlock_fpu_owner(); + + err |= __put_user(LASX_CTX_MAGIC, &info->magic); + err |= __put_user(extctx->lasx.size, &info->size); + + if (likely(!err)) + break; + /* Touch the LASX context and try again */ + err = __put_user(0, ®s[0]) | + __put_user(0, ®s[32*4-1]) | + __put_user(0, fcc) | + __put_user(0, fcsr); + if (err) + return err; /* really bad sigcontext */ + } + + return err; +} + +static int protected_restore_lasx_context(struct extctx_layout *extctx) +{ + int err = 0, sig = 0, tmp __maybe_unused; + struct sctx_info __user *info = extctx->lasx.addr; + struct lasx_context __user *lasx_ctx = + (struct lasx_context *)get_ctx_through_ctxinfo(info); + uint64_t __user *regs = (uint64_t *)&lasx_ctx->regs; + uint64_t __user *fcc = &lasx_ctx->fcc; + uint32_t __user *fcsr = &lasx_ctx->fcsr; + + err = sig = fcsr_pending(fcsr); + if (err < 0) + return err; + + while (1) { + lock_fpu_owner(); + if (is_lasx_enabled()) + err = restore_hw_lasx_context(lasx_ctx); + else { + err = copy_lasx_from_sigcontext(lasx_ctx); + if (is_lsx_enabled()) + restore_lsx(current); + else if (is_fpu_owner()) + restore_fp(current); + } + unlock_fpu_owner(); + + if (likely(!err)) + break; + /* Touch the LASX context and try again */ + err = __get_user(tmp, ®s[0]) | + __get_user(tmp, ®s[32*4-1]) | + __get_user(tmp, fcc) | + __get_user(tmp, fcsr); + if (err) + break; /* really bad sigcontext */ + } + + return err ?: sig; +} + static int setup_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc, struct extctx_layout *extctx) { @@ -240,7 +532,11 @@ static int setup_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc, for (i = 1; i < 32; i++) err |= __put_user(regs->regs[i], &sc->sc_regs[i]); - if (extctx->fpu.addr) + if (extctx->lasx.addr) + err |= protected_save_lasx_context(extctx); + else if (extctx->lsx.addr) + err |= protected_save_lsx_context(extctx); + else if (extctx->fpu.addr) err |= protected_save_fpu_context(extctx); /* Set the "end" magic */ @@ -274,6 +570,20 @@ static int parse_extcontext(struct sigcontext __user *sc, struct extctx_layout * extctx->fpu.addr = info; break; + case LSX_CTX_MAGIC: + if (size < (sizeof(struct sctx_info) + + sizeof(struct lsx_context))) + goto invalid; + extctx->lsx.addr = info; + break; + + case LASX_CTX_MAGIC: + if (size < (sizeof(struct sctx_info) + + sizeof(struct lasx_context))) + goto invalid; + extctx->lasx.addr = info; + break; + default: goto invalid; } @@ -319,7 +629,11 @@ static int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc for (i = 1; i < 32; i++) err |= __get_user(regs->regs[i], &sc->sc_regs[i]); - if (extctx.fpu.addr) + if (extctx.lasx.addr) + err |= protected_restore_lasx_context(&extctx); + else if (extctx.lsx.addr) + err |= protected_restore_lsx_context(&extctx); + else if (extctx.fpu.addr) err |= protected_restore_fpu_context(&extctx); bad: @@ -375,7 +689,13 @@ static unsigned long setup_extcontext(struct extctx_layout *extctx, unsigned lon extctx->size += extctx->end.size; if (extctx->flags & SC_USED_FP) { - if (cpu_has_fpu) + if (cpu_has_lasx && thread_lasx_context_live()) + new_sp = extframe_alloc(extctx, &extctx->lasx, + sizeof(struct lasx_context), LASX_CTX_ALIGN, new_sp); + else if (cpu_has_lsx && thread_lsx_context_live()) + new_sp = extframe_alloc(extctx, &extctx->lsx, + sizeof(struct lsx_context), LSX_CTX_ALIGN, new_sp); + else if (cpu_has_fpu) new_sp = extframe_alloc(extctx, &extctx->fpu, sizeof(struct fpu_context), FPU_CTX_ALIGN, new_sp); } diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c index ed167e244cda..8ea1bbcf13a7 100644 --- a/arch/loongarch/kernel/smp.c +++ b/arch/loongarch/kernel/smp.c @@ -8,6 +8,7 @@ * Copyright (C) 2000, 2001 Silicon Graphics, Inc. * Copyright (C) 2000, 2001, 2003 Broadcom Corporation */ +#include <linux/acpi.h> #include <linux/cpu.h> #include <linux/cpumask.h> #include <linux/init.h> @@ -37,10 +38,6 @@ EXPORT_SYMBOL(__cpu_number_map); int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */ EXPORT_SYMBOL(__cpu_logical_map); -/* Number of threads (siblings) per CPU core */ -int smp_num_siblings = 1; -EXPORT_SYMBOL(smp_num_siblings); - /* Representing the threads (siblings) of each logical CPU */ cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly; EXPORT_SYMBOL(cpu_sibling_map); @@ -118,7 +115,7 @@ static u32 ipi_read_clear(int cpu) action = iocsr_read32(LOONGARCH_IOCSR_IPI_STATUS); /* Clear the ipi register to clear the interrupt */ iocsr_write32(action, LOONGARCH_IOCSR_IPI_CLEAR); - smp_mb(); + wbflush(); return action; } @@ -210,6 +207,7 @@ static void __init fdt_smp_setup(void) } loongson_sysconf.nr_cpus = num_processors; + set_bit(0, &(loongson_sysconf.cores_io_master)); #endif } @@ -228,9 +226,12 @@ void __init loongson_prepare_cpus(unsigned int max_cpus) { int i = 0; + parse_acpi_topology(); + for (i = 0; i < loongson_sysconf.nr_cpus; i++) { set_cpu_present(i, true); csr_mail_send(0, __cpu_logical_map[i], 0); + cpu_data[i].global_id = __cpu_logical_map[i]; } per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; @@ -271,10 +272,10 @@ void loongson_init_secondary(void) numa_add_cpu(cpu); #endif per_cpu(cpu_state, cpu) = CPU_ONLINE; - cpu_data[cpu].core = - cpu_logical_map(cpu) % loongson_sysconf.cores_per_package; cpu_data[cpu].package = cpu_logical_map(cpu) / loongson_sysconf.cores_per_package; + cpu_data[cpu].core = pptt_enabled ? cpu_data[cpu].core : + cpu_logical_map(cpu) % loongson_sysconf.cores_per_package; } void loongson_smp_finish(void) @@ -380,14 +381,10 @@ static inline void set_cpu_sibling_map(int cpu) cpumask_set_cpu(cpu, &cpu_sibling_setup_map); - if (smp_num_siblings <= 1) - cpumask_set_cpu(cpu, &cpu_sibling_map[cpu]); - else { - for_each_cpu(i, &cpu_sibling_setup_map) { - if (cpus_are_siblings(cpu, i)) { - cpumask_set_cpu(i, &cpu_sibling_map[cpu]); - cpumask_set_cpu(cpu, &cpu_sibling_map[i]); - } + for_each_cpu(i, &cpu_sibling_setup_map) { + if (cpus_are_siblings(cpu, i)) { + cpumask_set_cpu(i, &cpu_sibling_map[cpu]); + cpumask_set_cpu(cpu, &cpu_sibling_map[i]); } } } diff --git a/arch/loongarch/kernel/traps.c b/arch/loongarch/kernel/traps.c index 8db26e4ca447..8fb5e7a77145 100644 --- a/arch/loongarch/kernel/traps.c +++ b/arch/loongarch/kernel/traps.c @@ -47,6 +47,7 @@ #include <asm/tlb.h> #include <asm/types.h> #include <asm/unwind.h> +#include <asm/uprobes.h> #include "access-helper.h" @@ -689,7 +690,6 @@ asmlinkage void noinstr do_bp(struct pt_regs *regs) if (regs->csr_prmd & CSR_PRMD_PIE) local_irq_enable(); - current->thread.trap_nr = read_csr_excode(); if (__get_inst(&opcode, (u32 *)era, user)) goto out_sigsegv; @@ -711,18 +711,17 @@ asmlinkage void noinstr do_bp(struct pt_regs *regs) else break; case BRK_UPROBE_BP: - if (notify_die(DIE_UPROBE, "Uprobe", regs, bcode, - current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) + if (uprobe_breakpoint_handler(regs)) goto out; else break; case BRK_UPROBE_XOLBP: - if (notify_die(DIE_UPROBE_XOL, "Uprobe_XOL", regs, bcode, - current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) + if (uprobe_singlestep_handler(regs)) goto out; else break; default: + current->thread.trap_nr = read_csr_excode(); if (notify_die(DIE_TRAP, "Break", regs, bcode, current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) goto out; @@ -852,12 +851,67 @@ static void init_restore_fp(void) BUG_ON(!is_fp_enabled()); } +static void init_restore_lsx(void) +{ + enable_lsx(); + + if (!thread_lsx_context_live()) { + /* First time LSX context user */ + init_restore_fp(); + init_lsx_upper(); + set_thread_flag(TIF_LSX_CTX_LIVE); + } else { + if (!is_simd_owner()) { + if (is_fpu_owner()) { + restore_lsx_upper(current); + } else { + __own_fpu(); + restore_lsx(current); + } + } + } + + set_thread_flag(TIF_USEDSIMD); + + BUG_ON(!is_fp_enabled()); + BUG_ON(!is_lsx_enabled()); +} + +static void init_restore_lasx(void) +{ + enable_lasx(); + + if (!thread_lasx_context_live()) { + /* First time LASX context user */ + init_restore_lsx(); + init_lasx_upper(); + set_thread_flag(TIF_LASX_CTX_LIVE); + } else { + if (is_fpu_owner() || is_simd_owner()) { + init_restore_lsx(); + restore_lasx_upper(current); + } else { + __own_fpu(); + enable_lsx(); + restore_lasx(current); + } + } + + set_thread_flag(TIF_USEDSIMD); + + BUG_ON(!is_fp_enabled()); + BUG_ON(!is_lsx_enabled()); + BUG_ON(!is_lasx_enabled()); +} + asmlinkage void noinstr do_fpu(struct pt_regs *regs) { irqentry_state_t state = irqentry_enter(regs); local_irq_enable(); die_if_kernel("do_fpu invoked from kernel context!", regs); + BUG_ON(is_lsx_enabled()); + BUG_ON(is_lasx_enabled()); preempt_disable(); init_restore_fp(); @@ -872,9 +926,20 @@ asmlinkage void noinstr do_lsx(struct pt_regs *regs) irqentry_state_t state = irqentry_enter(regs); local_irq_enable(); - force_sig(SIGILL); - local_irq_disable(); + if (!cpu_has_lsx) { + force_sig(SIGILL); + goto out; + } + die_if_kernel("do_lsx invoked from kernel context!", regs); + BUG_ON(is_lasx_enabled()); + + preempt_disable(); + init_restore_lsx(); + preempt_enable(); + +out: + local_irq_disable(); irqentry_exit(regs, state); } @@ -883,9 +948,19 @@ asmlinkage void noinstr do_lasx(struct pt_regs *regs) irqentry_state_t state = irqentry_enter(regs); local_irq_enable(); - force_sig(SIGILL); - local_irq_disable(); + if (!cpu_has_lasx) { + force_sig(SIGILL); + goto out; + } + + die_if_kernel("do_lasx invoked from kernel context!", regs); + preempt_disable(); + init_restore_lasx(); + preempt_enable(); + +out: + local_irq_disable(); irqentry_exit(regs, state); } @@ -924,7 +999,7 @@ asmlinkage void cache_parity_error(void) /* For the moment, report the problem and hang. */ pr_err("Cache error exception:\n"); pr_err("csr_merrctl == %08x\n", csr_read32(LOONGARCH_CSR_MERRCTL)); - pr_err("csr_merrera == %016llx\n", csr_read64(LOONGARCH_CSR_MERRERA)); + pr_err("csr_merrera == %016lx\n", csr_read64(LOONGARCH_CSR_MERRERA)); panic("Can't handle the cache error!"); } diff --git a/arch/loongarch/kernel/unaligned.c b/arch/loongarch/kernel/unaligned.c index 85fae3d2d71a..3abf163dda05 100644 --- a/arch/loongarch/kernel/unaligned.c +++ b/arch/loongarch/kernel/unaligned.c @@ -485,8 +485,6 @@ static int __init debugfs_unaligned(void) struct dentry *d; d = debugfs_create_dir("loongarch", NULL); - if (IS_ERR_OR_NULL(d)) - return -ENOMEM; debugfs_create_u32("unaligned_instructions_user", S_IRUGO, d, &unaligned_instructions_user); diff --git a/arch/loongarch/kernel/uprobes.c b/arch/loongarch/kernel/uprobes.c new file mode 100644 index 000000000000..87abc7137b73 --- /dev/null +++ b/arch/loongarch/kernel/uprobes.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include <linux/highmem.h> +#include <linux/ptrace.h> +#include <linux/sched.h> +#include <linux/uprobes.h> +#include <asm/cacheflush.h> + +#define UPROBE_TRAP_NR UINT_MAX + +int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, + struct mm_struct *mm, unsigned long addr) +{ + int idx; + union loongarch_instruction insn; + + if (addr & 0x3) + return -EILSEQ; + + for (idx = ARRAY_SIZE(auprobe->insn) - 1; idx >= 0; idx--) { + insn.word = auprobe->insn[idx]; + if (insns_not_supported(insn)) + return -EINVAL; + } + + if (insns_need_simulation(insn)) { + auprobe->ixol[0] = larch_insn_gen_nop(); + auprobe->simulate = true; + } else { + auprobe->ixol[0] = auprobe->insn[0]; + auprobe->simulate = false; + } + + auprobe->ixol[1] = UPROBE_XOLBP_INSN; + + return 0; +} + +int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + utask->autask.saved_trap_nr = current->thread.trap_nr; + current->thread.trap_nr = UPROBE_TRAP_NR; + instruction_pointer_set(regs, utask->xol_vaddr); + user_enable_single_step(current); + + return 0; +} + +int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + WARN_ON_ONCE(current->thread.trap_nr != UPROBE_TRAP_NR); + current->thread.trap_nr = utask->autask.saved_trap_nr; + + if (auprobe->simulate) + instruction_pointer_set(regs, auprobe->resume_era); + else + instruction_pointer_set(regs, utask->vaddr + LOONGARCH_INSN_SIZE); + + user_disable_single_step(current); + + return 0; +} + +void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + current->thread.trap_nr = utask->autask.saved_trap_nr; + instruction_pointer_set(regs, utask->vaddr); + user_disable_single_step(current); +} + +bool arch_uprobe_xol_was_trapped(struct task_struct *t) +{ + if (t->thread.trap_nr != UPROBE_TRAP_NR) + return true; + + return false; +} + +bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + union loongarch_instruction insn; + + if (!auprobe->simulate) + return false; + + insn.word = auprobe->insn[0]; + arch_simulate_insn(insn, regs); + auprobe->resume_era = regs->csr_era; + + return true; +} + +unsigned long arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, + struct pt_regs *regs) +{ + unsigned long ra = regs->regs[1]; + + regs->regs[1] = trampoline_vaddr; + + return ra; +} + +bool arch_uretprobe_is_alive(struct return_instance *ret, + enum rp_check ctx, struct pt_regs *regs) +{ + if (ctx == RP_CHECK_CHAIN_CALL) + return regs->regs[3] <= ret->stack; + else + return regs->regs[3] < ret->stack; +} + +int arch_uprobe_exception_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + return NOTIFY_DONE; +} + +bool uprobe_breakpoint_handler(struct pt_regs *regs) +{ + if (uprobe_pre_sstep_notifier(regs)) + return true; + + return false; +} + +bool uprobe_singlestep_handler(struct pt_regs *regs) +{ + if (uprobe_post_sstep_notifier(regs)) + return true; + + return false; +} + +unsigned long uprobe_get_swbp_addr(struct pt_regs *regs) +{ + return instruction_pointer(regs); +} + +void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, + void *src, unsigned long len) +{ + void *kaddr = kmap_local_page(page); + void *dst = kaddr + (vaddr & ~PAGE_MASK); + + memcpy(dst, src, len); + flush_icache_range((unsigned long)dst, (unsigned long)dst + len); + kunmap_local(kaddr); +} diff --git a/arch/loongarch/kernel/vdso.c b/arch/loongarch/kernel/vdso.c index eaebd2e0f725..14941e4be66d 100644 --- a/arch/loongarch/kernel/vdso.c +++ b/arch/loongarch/kernel/vdso.c @@ -14,6 +14,7 @@ #include <linux/random.h> #include <linux/sched.h> #include <linux/slab.h> +#include <linux/time_namespace.h> #include <linux/timekeeper_internal.h> #include <asm/page.h> @@ -26,12 +27,17 @@ extern char vdso_start[], vdso_end[]; /* Kernel-provided data used by the VDSO. */ static union { - u8 page[VDSO_DATA_SIZE]; + u8 page[PAGE_SIZE]; + struct vdso_data data[CS_BASES]; +} generic_vdso_data __page_aligned_data; + +static union { + u8 page[LOONGARCH_VDSO_DATA_SIZE]; struct loongarch_vdso_data vdata; } loongarch_vdso_data __page_aligned_data; static struct page *vdso_pages[] = { NULL }; -struct vdso_data *vdso_data = loongarch_vdso_data.vdata.data; +struct vdso_data *vdso_data = generic_vdso_data.data; struct vdso_pcpu_data *vdso_pdata = loongarch_vdso_data.vdata.pdata; static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma) @@ -41,6 +47,43 @@ static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struc return 0; } +static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, + struct vm_area_struct *vma, struct vm_fault *vmf) +{ + unsigned long pfn; + struct page *timens_page = find_timens_vvar_page(vma); + + switch (vmf->pgoff) { + case VVAR_GENERIC_PAGE_OFFSET: + if (!timens_page) + pfn = sym_to_pfn(vdso_data); + else + pfn = page_to_pfn(timens_page); + break; +#ifdef CONFIG_TIME_NS + case VVAR_TIMENS_PAGE_OFFSET: + /* + * If a task belongs to a time namespace then a namespace specific + * VVAR is mapped with the VVAR_GENERIC_PAGE_OFFSET and the real + * VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET offset. + * See also the comment near timens_setup_vdso_data(). + */ + if (!timens_page) + return VM_FAULT_SIGBUS; + else + pfn = sym_to_pfn(vdso_data); + break; +#endif /* CONFIG_TIME_NS */ + case VVAR_LOONGARCH_PAGES_START ... VVAR_LOONGARCH_PAGES_END: + pfn = sym_to_pfn(&loongarch_vdso_data) + vmf->pgoff - VVAR_LOONGARCH_PAGES_START; + break; + default: + return VM_FAULT_SIGBUS; + } + + return vmf_insert_pfn(vma, vmf->address, pfn); +} + struct loongarch_vdso_info vdso_info = { .vdso = vdso_start, .size = PAGE_SIZE, @@ -51,6 +94,7 @@ struct loongarch_vdso_info vdso_info = { }, .data_mapping = { .name = "[vvar]", + .fault = vvar_fault, }, .offset_sigreturn = vdso_offset_sigreturn, }; @@ -73,6 +117,37 @@ static int __init init_vdso(void) } subsys_initcall(init_vdso); +#ifdef CONFIG_TIME_NS +struct vdso_data *arch_get_vdso_data(void *vvar_page) +{ + return (struct vdso_data *)(vvar_page); +} + +/* + * The vvar mapping contains data for a specific time namespace, so when a + * task changes namespace we must unmap its vvar data for the old namespace. + * Subsequent faults will map in data for the new namespace. + * + * For more details see timens_setup_vdso_data(). + */ +int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) +{ + struct mm_struct *mm = task->mm; + struct vm_area_struct *vma; + + VMA_ITERATOR(vmi, mm, 0); + + mmap_read_lock(mm); + for_each_vma(vmi, vma) { + if (vma_is_special_mapping(vma, &vdso_info.data_mapping)) + zap_vma_pages(vma); + } + mmap_read_unlock(mm); + + return 0; +} +#endif + static unsigned long vdso_base(void) { unsigned long base = STACK_TOP; @@ -88,7 +163,7 @@ static unsigned long vdso_base(void) int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) { int ret; - unsigned long vvar_size, size, data_addr, vdso_addr; + unsigned long size, data_addr, vdso_addr; struct mm_struct *mm = current->mm; struct vm_area_struct *vma; struct loongarch_vdso_info *info = current->thread.vdso; @@ -100,32 +175,23 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) * Determine total area size. This includes the VDSO data itself * and the data pages. */ - vvar_size = VDSO_DATA_SIZE; - size = vvar_size + info->size; + size = VVAR_SIZE + info->size; data_addr = get_unmapped_area(NULL, vdso_base(), size, 0, 0); if (IS_ERR_VALUE(data_addr)) { ret = data_addr; goto out; } - vdso_addr = data_addr + VDSO_DATA_SIZE; - vma = _install_special_mapping(mm, data_addr, vvar_size, - VM_READ | VM_MAYREAD, + vma = _install_special_mapping(mm, data_addr, VVAR_SIZE, + VM_READ | VM_MAYREAD | VM_PFNMAP, &info->data_mapping); if (IS_ERR(vma)) { ret = PTR_ERR(vma); goto out; } - /* Map VDSO data page. */ - ret = remap_pfn_range(vma, data_addr, - virt_to_phys(&loongarch_vdso_data) >> PAGE_SHIFT, - vvar_size, PAGE_READONLY); - if (ret) - goto out; - - /* Map VDSO code page. */ + vdso_addr = data_addr + VVAR_SIZE; vma = _install_special_mapping(mm, vdso_addr, info->size, VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC, &info->code_mapping); diff --git a/arch/loongarch/kernel/vmlinux.lds.S b/arch/loongarch/kernel/vmlinux.lds.S index 0c7b041be9d8..b1686afcf876 100644 --- a/arch/loongarch/kernel/vmlinux.lds.S +++ b/arch/loongarch/kernel/vmlinux.lds.S @@ -136,6 +136,15 @@ SECTIONS DWARF_DEBUG ELF_DETAILS +#ifdef CONFIG_EFI_STUB + /* header symbols */ + _kernel_asize = _end - _text; + _kernel_fsize = _edata - _text; + _kernel_vsize = _end - __initdata_begin; + _kernel_rsize = _edata - __initdata_begin; + _kernel_offset = kernel_offset - _text; +#endif + .gptab.sdata : { *(.gptab.data) *(.gptab.sdata) diff --git a/arch/loongarch/lib/dump_tlb.c b/arch/loongarch/lib/dump_tlb.c index c2cc7ce343c9..0b886a6e260f 100644 --- a/arch/loongarch/lib/dump_tlb.c +++ b/arch/loongarch/lib/dump_tlb.c @@ -20,9 +20,9 @@ void dump_tlb_regs(void) pr_info("Index : 0x%0x\n", read_csr_tlbidx()); pr_info("PageSize : 0x%0x\n", read_csr_pagesize()); - pr_info("EntryHi : 0x%0*llx\n", field, read_csr_entryhi()); - pr_info("EntryLo0 : 0x%0*llx\n", field, read_csr_entrylo0()); - pr_info("EntryLo1 : 0x%0*llx\n", field, read_csr_entrylo1()); + pr_info("EntryHi : 0x%0*lx\n", field, read_csr_entryhi()); + pr_info("EntryLo0 : 0x%0*lx\n", field, read_csr_entrylo0()); + pr_info("EntryLo1 : 0x%0*lx\n", field, read_csr_entrylo1()); } static void dump_tlb(int first, int last) diff --git a/arch/loongarch/mm/tlb.c b/arch/loongarch/mm/tlb.c index 8bad6b0cff59..00bb563e3c89 100644 --- a/arch/loongarch/mm/tlb.c +++ b/arch/loongarch/mm/tlb.c @@ -167,6 +167,9 @@ void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep int idx; unsigned long flags; + if (cpu_has_ptw) + return; + /* * Handle debugger faulting in for debugee. */ @@ -222,6 +225,9 @@ static void setup_ptwalker(void) pwctl0 = pte_i | pte_w << 5 | pmd_i << 10 | pmd_w << 15 | pud_i << 20 | pud_w << 25; pwctl1 = pgd_i | pgd_w << 6; + if (cpu_has_ptw) + pwctl1 |= CSR_PWCTL1_PTW; + csr_write64(pwctl0, LOONGARCH_CSR_PWCTL0); csr_write64(pwctl1, LOONGARCH_CSR_PWCTL1); csr_write64((long)swapper_pg_dir, LOONGARCH_CSR_PGDH); @@ -264,10 +270,17 @@ void setup_tlb_handler(int cpu) if (cpu == 0) { memcpy((void *)tlbrentry, handle_tlb_refill, 0x80); local_flush_icache_range(tlbrentry, tlbrentry + 0x80); - set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE); - set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE); - set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE); - set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE); + if (!cpu_has_ptw) { + set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE); + set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE); + set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE); + set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE); + } else { + set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load_ptw, VECSIZE); + set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load_ptw, VECSIZE); + set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store_ptw, VECSIZE); + set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify_ptw, VECSIZE); + } set_handler(EXCCODE_TLBNR * VECSIZE, handle_tlb_protect, VECSIZE); set_handler(EXCCODE_TLBNX * VECSIZE, handle_tlb_protect, VECSIZE); set_handler(EXCCODE_TLBPE * VECSIZE, handle_tlb_protect, VECSIZE); diff --git a/arch/loongarch/mm/tlbex.S b/arch/loongarch/mm/tlbex.S index 244e2f5aeee5..4ad78703de6f 100644 --- a/arch/loongarch/mm/tlbex.S +++ b/arch/loongarch/mm/tlbex.S @@ -184,12 +184,19 @@ tlb_huge_update_load: ertn nopage_tlb_load: - dbar 0 + dbar 0x700 csrrd ra, EXCEPTION_KS2 la_abs t0, tlb_do_page_fault_0 jr t0 SYM_FUNC_END(handle_tlb_load) +SYM_FUNC_START(handle_tlb_load_ptw) + csrwr t0, LOONGARCH_CSR_KS0 + csrwr t1, LOONGARCH_CSR_KS1 + la_abs t0, tlb_do_page_fault_0 + jr t0 +SYM_FUNC_END(handle_tlb_load_ptw) + SYM_FUNC_START(handle_tlb_store) csrwr t0, EXCEPTION_KS0 csrwr t1, EXCEPTION_KS1 @@ -333,12 +340,19 @@ tlb_huge_update_store: ertn nopage_tlb_store: - dbar 0 + dbar 0x700 csrrd ra, EXCEPTION_KS2 la_abs t0, tlb_do_page_fault_1 jr t0 SYM_FUNC_END(handle_tlb_store) +SYM_FUNC_START(handle_tlb_store_ptw) + csrwr t0, LOONGARCH_CSR_KS0 + csrwr t1, LOONGARCH_CSR_KS1 + la_abs t0, tlb_do_page_fault_1 + jr t0 +SYM_FUNC_END(handle_tlb_store_ptw) + SYM_FUNC_START(handle_tlb_modify) csrwr t0, EXCEPTION_KS0 csrwr t1, EXCEPTION_KS1 @@ -480,12 +494,19 @@ tlb_huge_update_modify: ertn nopage_tlb_modify: - dbar 0 + dbar 0x700 csrrd ra, EXCEPTION_KS2 la_abs t0, tlb_do_page_fault_1 jr t0 SYM_FUNC_END(handle_tlb_modify) +SYM_FUNC_START(handle_tlb_modify_ptw) + csrwr t0, LOONGARCH_CSR_KS0 + csrwr t1, LOONGARCH_CSR_KS1 + la_abs t0, tlb_do_page_fault_1 + jr t0 +SYM_FUNC_END(handle_tlb_modify_ptw) + SYM_FUNC_START(handle_tlb_refill) csrwr t0, LOONGARCH_CSR_TLBRSAVE csrrd t0, LOONGARCH_CSR_PGD diff --git a/arch/loongarch/power/suspend.c b/arch/loongarch/power/suspend.c index 5e19733e5e05..166d9e06a64b 100644 --- a/arch/loongarch/power/suspend.c +++ b/arch/loongarch/power/suspend.c @@ -27,7 +27,7 @@ struct saved_registers { }; static struct saved_registers saved_regs; -static void arch_common_suspend(void) +void loongarch_common_suspend(void) { save_counter(); saved_regs.pgd = csr_read64(LOONGARCH_CSR_PGDL); @@ -40,7 +40,7 @@ static void arch_common_suspend(void) loongarch_suspend_addr = loongson_sysconf.suspend_addr; } -static void arch_common_resume(void) +void loongarch_common_resume(void) { sync_counter(); local_flush_tlb_all(); @@ -62,12 +62,12 @@ int loongarch_acpi_suspend(void) enable_gpe_wakeup(); enable_pci_wakeup(); - arch_common_suspend(); + loongarch_common_suspend(); /* processor specific suspend */ loongarch_suspend_enter(); - arch_common_resume(); + loongarch_common_resume(); return 0; } diff --git a/arch/loongarch/vdso/Makefile b/arch/loongarch/vdso/Makefile index 461240ab4436..a50308b6fc25 100644 --- a/arch/loongarch/vdso/Makefile +++ b/arch/loongarch/vdso/Makefile @@ -12,16 +12,13 @@ ccflags-vdso := \ $(filter -E%,$(KBUILD_CFLAGS)) \ $(filter -march=%,$(KBUILD_CFLAGS)) \ $(filter -m%-float,$(KBUILD_CFLAGS)) \ + $(CLANG_FLAGS) \ -D__VDSO__ -ifeq ($(cc-name),clang) -ccflags-vdso += $(filter --target=%,$(KBUILD_CFLAGS)) -endif - cflags-vdso := $(ccflags-vdso) \ -isystem $(shell $(CC) -print-file-name=include) \ $(filter -W%,$(filter-out -Wa$(comma)%,$(KBUILD_CFLAGS))) \ - -O2 -g -fno-strict-aliasing -fno-common -fno-builtin -G0 \ + -O2 -g -fno-strict-aliasing -fno-common -fno-builtin \ -fno-stack-protector -fno-jump-tables -DDISABLE_BRANCH_PROFILING \ $(call cc-option, -fno-asynchronous-unwind-tables) \ $(call cc-option, -fno-stack-protector) diff --git a/arch/loongarch/vdso/vgetcpu.c b/arch/loongarch/vdso/vgetcpu.c index e02e775f5360..9e445be39763 100644 --- a/arch/loongarch/vdso/vgetcpu.c +++ b/arch/loongarch/vdso/vgetcpu.c @@ -21,7 +21,7 @@ static __always_inline int read_cpu_id(void) static __always_inline const struct vdso_pcpu_data *get_pcpu_data(void) { - return (struct vdso_pcpu_data *)(get_vdso_base() - VDSO_DATA_SIZE); + return (struct vdso_pcpu_data *)(get_vdso_data() + VVAR_LOONGARCH_PAGES_START * PAGE_SIZE); } extern diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index 62fdca7efce4..4383ed851063 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -41,7 +41,6 @@ CONFIG_MQ_IOSCHED_KYBER=m CONFIG_IOSCHED_BFQ=m # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=m -CONFIG_SLAB=y # CONFIG_COMPACTION is not set CONFIG_DMAPOOL_TEST=m CONFIG_USERFAULTFD=y diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index 5bfbd0444bb5..ec0f9c9f9562 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -37,7 +37,6 @@ CONFIG_MQ_IOSCHED_KYBER=m CONFIG_IOSCHED_BFQ=m # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=m -CONFIG_SLAB=y # CONFIG_COMPACTION is not set CONFIG_DMAPOOL_TEST=m CONFIG_USERFAULTFD=y diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index 44302f11c9ea..8656ae1f239e 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -44,7 +44,6 @@ CONFIG_MQ_IOSCHED_KYBER=m CONFIG_IOSCHED_BFQ=m # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=m -CONFIG_SLAB=y # CONFIG_COMPACTION is not set CONFIG_DMAPOOL_TEST=m CONFIG_USERFAULTFD=y diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index f3336f1774ec..496fb6a415ea 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -34,7 +34,6 @@ CONFIG_MQ_IOSCHED_KYBER=m CONFIG_IOSCHED_BFQ=m # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=m -CONFIG_SLAB=y # CONFIG_COMPACTION is not set CONFIG_DMAPOOL_TEST=m CONFIG_USERFAULTFD=y diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index 2d1bbac68066..4add7ab9973b 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -36,7 +36,6 @@ CONFIG_MQ_IOSCHED_KYBER=m CONFIG_IOSCHED_BFQ=m # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=m -CONFIG_SLAB=y # CONFIG_COMPACTION is not set CONFIG_DMAPOOL_TEST=m CONFIG_USERFAULTFD=y diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index b4428dc36102..5845f1f71fd1 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -35,7 +35,6 @@ CONFIG_MQ_IOSCHED_KYBER=m CONFIG_IOSCHED_BFQ=m # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=m -CONFIG_SLAB=y # CONFIG_COMPACTION is not set CONFIG_DMAPOOL_TEST=m CONFIG_USERFAULTFD=y diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index 4cd9fa4cb10c..bbb251bab81a 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -55,7 +55,6 @@ CONFIG_MQ_IOSCHED_KYBER=m CONFIG_IOSCHED_BFQ=m # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=m -CONFIG_SLAB=y # CONFIG_COMPACTION is not set CONFIG_DMAPOOL_TEST=m CONFIG_USERFAULTFD=y diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index 7ee9ad50f0ad..4f9cfc70c66d 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -33,7 +33,6 @@ CONFIG_MQ_IOSCHED_KYBER=m CONFIG_IOSCHED_BFQ=m # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=m -CONFIG_SLAB=y # CONFIG_COMPACTION is not set CONFIG_DMAPOOL_TEST=m CONFIG_USERFAULTFD=y diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index 2488893616dc..67c42b4822f0 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -34,7 +34,6 @@ CONFIG_MQ_IOSCHED_KYBER=m CONFIG_IOSCHED_BFQ=m # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=m -CONFIG_SLAB=y # CONFIG_COMPACTION is not set CONFIG_DMAPOOL_TEST=m CONFIG_USERFAULTFD=y diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index ffc676289f87..85f19515200b 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -35,7 +35,6 @@ CONFIG_MQ_IOSCHED_KYBER=m CONFIG_IOSCHED_BFQ=m # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=m -CONFIG_SLAB=y # CONFIG_COMPACTION is not set CONFIG_DMAPOOL_TEST=m CONFIG_USERFAULTFD=y diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index 198179657ce0..b1b15acb5d5f 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -31,7 +31,6 @@ CONFIG_MQ_IOSCHED_KYBER=m CONFIG_IOSCHED_BFQ=m # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=m -CONFIG_SLAB=y # CONFIG_COMPACTION is not set CONFIG_USERFAULTFD=y CONFIG_NET=y diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index 85364f6178d4..91d66c0f5ab6 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -31,7 +31,6 @@ CONFIG_MQ_IOSCHED_KYBER=m CONFIG_IOSCHED_BFQ=m # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=m -CONFIG_SLAB=y # CONFIG_COMPACTION is not set CONFIG_DMAPOOL_TEST=m CONFIG_USERFAULTFD=y diff --git a/arch/microblaze/configs/mmu_defconfig b/arch/microblaze/configs/mmu_defconfig index 756feb9369a6..38dcdce367ec 100644 --- a/arch/microblaze/configs/mmu_defconfig +++ b/arch/microblaze/configs/mmu_defconfig @@ -8,7 +8,6 @@ CONFIG_SYSFS_DEPRECATED_V2=y # CONFIG_BASE_FULL is not set CONFIG_KALLSYMS_ALL=y CONFIG_EMBEDDED=y -CONFIG_SLAB=y CONFIG_XILINX_MICROBLAZE0_USE_MSR_INSTR=1 CONFIG_XILINX_MICROBLAZE0_USE_PCMP_INSTR=1 CONFIG_XILINX_MICROBLAZE0_USE_BARREL=1 diff --git a/arch/mips/Makefile b/arch/mips/Makefile index a7a4ee66a9d3..ef7b05ae92ce 100644 --- a/arch/mips/Makefile +++ b/arch/mips/Makefile @@ -346,7 +346,7 @@ KBUILD_CFLAGS += -fno-asynchronous-unwind-tables KBUILD_LDFLAGS += -m $(ld-emul) ifdef CONFIG_MIPS -CHECKFLAGS += $(shell $(CC) $(KBUILD_CFLAGS) -dM -E -x c /dev/null | \ +CHECKFLAGS += $(shell $(CC) $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS) -dM -E -x c /dev/null | \ grep -E -vw '__GNUC_(MINOR_|PATCHLEVEL_)?_' | \ sed -e "s/^\#define /-D'/" -e "s/ /'='/" -e "s/$$/'/" -e 's/\$$/&&/g') endif diff --git a/arch/mips/alchemy/common/platform.c b/arch/mips/alchemy/common/platform.c index b8f3397c59c9..d4ab34b3b404 100644 --- a/arch/mips/alchemy/common/platform.c +++ b/arch/mips/alchemy/common/platform.c @@ -51,9 +51,9 @@ static void alchemy_8250_pm(struct uart_port *port, unsigned int state, #define PORT(_base, _irq) \ { \ .mapbase = _base, \ + .mapsize = 0x1000, \ .irq = _irq, \ .regshift = 2, \ - .iotype = UPIO_AU, \ .flags = UPF_SKIP_TEST | UPF_IOREMAP | \ UPF_FIXED_TYPE, \ .type = PORT_16550A, \ @@ -124,8 +124,14 @@ static void __init alchemy_setup_uarts(int ctype) au1xx0_uart_device.dev.platform_data = ports; /* Fill up uartclk. */ - for (s = 0; s < c; s++) + for (s = 0; s < c; s++) { ports[s].uartclk = uartclk; + if (au_platform_setup(&ports[s]) < 0) { + kfree(ports); + printk(KERN_INFO "Alchemy: missing support for UARTs\n"); + return; + } + } if (platform_device_register(&au1xx0_uart_device)) printk(KERN_INFO "Alchemy: failed to register UARTs\n"); } diff --git a/arch/mips/configs/ar7_defconfig b/arch/mips/configs/ar7_defconfig index ed4a6388791e..329c60aa570a 100644 --- a/arch/mips/configs/ar7_defconfig +++ b/arch/mips/configs/ar7_defconfig @@ -11,7 +11,6 @@ CONFIG_EXPERT=y # CONFIG_KALLSYMS is not set # CONFIG_VM_EVENT_COUNTERS is not set # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_AR7=y CONFIG_HZ_100=y CONFIG_KEXEC=y diff --git a/arch/mips/configs/bcm47xx_defconfig b/arch/mips/configs/bcm47xx_defconfig index 22ffde722bb9..62c462a23edc 100644 --- a/arch/mips/configs/bcm47xx_defconfig +++ b/arch/mips/configs/bcm47xx_defconfig @@ -3,7 +3,6 @@ CONFIG_HIGH_RES_TIMERS=y CONFIG_BLK_DEV_INITRD=y CONFIG_CC_OPTIMIZE_FOR_SIZE=y CONFIG_EMBEDDED=y -CONFIG_SLAB=y CONFIG_BCM47XX=y CONFIG_PCI=y # CONFIG_SUSPEND is not set diff --git a/arch/mips/configs/bigsur_defconfig b/arch/mips/configs/bigsur_defconfig index d15961f00ece..b0cbfa781ffc 100644 --- a/arch/mips/configs/bigsur_defconfig +++ b/arch/mips/configs/bigsur_defconfig @@ -17,7 +17,6 @@ CONFIG_USER_NS=y CONFIG_RELAY=y CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y -CONFIG_SLAB=y CONFIG_SIBYTE_BIGSUR=y CONFIG_64BIT=y CONFIG_SMP=y diff --git a/arch/mips/configs/cavium_octeon_defconfig b/arch/mips/configs/cavium_octeon_defconfig index 0bc2e3cc573b..f523ee6f25bf 100644 --- a/arch/mips/configs/cavium_octeon_defconfig +++ b/arch/mips/configs/cavium_octeon_defconfig @@ -9,7 +9,6 @@ CONFIG_LOG_BUF_SHIFT=14 CONFIG_RELAY=y CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y -CONFIG_SLAB=y CONFIG_CAVIUM_OCTEON_SOC=y CONFIG_CAVIUM_CN63XXP1=y CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE=2 diff --git a/arch/mips/configs/ci20_defconfig b/arch/mips/configs/ci20_defconfig index 920b27977dac..812287a5b4fd 100644 --- a/arch/mips/configs/ci20_defconfig +++ b/arch/mips/configs/ci20_defconfig @@ -27,7 +27,6 @@ CONFIG_HZ_100=y # CONFIG_SECCOMP is not set CONFIG_MODULES=y # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set -CONFIG_SLAB=y # CONFIG_COMPAT_BRK is not set # CONFIG_COMPACTION is not set CONFIG_CMA=y diff --git a/arch/mips/configs/cu1000-neo_defconfig b/arch/mips/configs/cu1000-neo_defconfig index 1cbc9302e1d1..afe39ce7568e 100644 --- a/arch/mips/configs/cu1000-neo_defconfig +++ b/arch/mips/configs/cu1000-neo_defconfig @@ -18,7 +18,6 @@ CONFIG_KALLSYMS_ALL=y CONFIG_EMBEDDED=y # CONFIG_VM_EVENT_COUNTERS is not set # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_MACH_INGENIC_SOC=y CONFIG_X1000_CU1000_NEO=y CONFIG_HIGHMEM=y diff --git a/arch/mips/configs/cu1830-neo_defconfig b/arch/mips/configs/cu1830-neo_defconfig index a0f73f3cd6ce..347c9fd1c21d 100644 --- a/arch/mips/configs/cu1830-neo_defconfig +++ b/arch/mips/configs/cu1830-neo_defconfig @@ -18,7 +18,6 @@ CONFIG_KALLSYMS_ALL=y CONFIG_EMBEDDED=y # CONFIG_VM_EVENT_COUNTERS is not set # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_MACH_INGENIC_SOC=y CONFIG_X1830_CU1830_NEO=y CONFIG_HIGHMEM=y diff --git a/arch/mips/configs/db1xxx_defconfig b/arch/mips/configs/db1xxx_defconfig index af070be1b583..08d29b56e175 100644 --- a/arch/mips/configs/db1xxx_defconfig +++ b/arch/mips/configs/db1xxx_defconfig @@ -18,7 +18,6 @@ CONFIG_CGROUP_DEVICE=y CONFIG_CGROUP_CPUACCT=y CONFIG_KALLSYMS_ALL=y CONFIG_EMBEDDED=y -CONFIG_SLAB=y CONFIG_MIPS_ALCHEMY=y CONFIG_HZ_100=y CONFIG_PCI=y diff --git a/arch/mips/configs/decstation_64_defconfig b/arch/mips/configs/decstation_64_defconfig index 49ec1575234e..92a1d0aea38c 100644 --- a/arch/mips/configs/decstation_64_defconfig +++ b/arch/mips/configs/decstation_64_defconfig @@ -9,7 +9,6 @@ CONFIG_EXPERT=y # CONFIG_SYSFS_SYSCALL is not set CONFIG_BPF_SYSCALL=y # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_MACH_DECSTATION=y CONFIG_64BIT=y CONFIG_PAGE_SIZE_16KB=y diff --git a/arch/mips/configs/decstation_defconfig b/arch/mips/configs/decstation_defconfig index 5cec4c096e2c..db214fcebcbe 100644 --- a/arch/mips/configs/decstation_defconfig +++ b/arch/mips/configs/decstation_defconfig @@ -9,7 +9,6 @@ CONFIG_EXPERT=y # CONFIG_SYSFS_SYSCALL is not set CONFIG_BPF_SYSCALL=y # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_MACH_DECSTATION=y CONFIG_CPU_R3000=y CONFIG_TC=y diff --git a/arch/mips/configs/decstation_r4k_defconfig b/arch/mips/configs/decstation_r4k_defconfig index af37e26d9b5b..15b769e96d5b 100644 --- a/arch/mips/configs/decstation_r4k_defconfig +++ b/arch/mips/configs/decstation_r4k_defconfig @@ -9,7 +9,6 @@ CONFIG_EXPERT=y # CONFIG_SYSFS_SYSCALL is not set CONFIG_BPF_SYSCALL=y # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_MACH_DECSTATION=y CONFIG_TC=y # CONFIG_SUSPEND is not set diff --git a/arch/mips/configs/fuloong2e_defconfig b/arch/mips/configs/fuloong2e_defconfig index 843d6a5a4f61..5458573286a2 100644 --- a/arch/mips/configs/fuloong2e_defconfig +++ b/arch/mips/configs/fuloong2e_defconfig @@ -13,7 +13,6 @@ CONFIG_NAMESPACES=y CONFIG_USER_NS=y CONFIG_EXPERT=y # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_MACH_LOONGSON2EF=y CONFIG_PCI=y diff --git a/arch/mips/configs/gpr_defconfig b/arch/mips/configs/gpr_defconfig index eb755650f821..92fc0edbac47 100644 --- a/arch/mips/configs/gpr_defconfig +++ b/arch/mips/configs/gpr_defconfig @@ -8,7 +8,6 @@ CONFIG_BSD_PROCESS_ACCT_V3=y CONFIG_RELAY=y CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_MIPS_ALCHEMY=y CONFIG_MIPS_GPR=y diff --git a/arch/mips/configs/ip22_defconfig b/arch/mips/configs/ip22_defconfig index 13df29901237..897e55579af0 100644 --- a/arch/mips/configs/ip22_defconfig +++ b/arch/mips/configs/ip22_defconfig @@ -10,7 +10,6 @@ CONFIG_USER_NS=y CONFIG_RELAY=y CONFIG_EXPERT=y # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_SGI_IP22=y CONFIG_ARC_CONSOLE=y CONFIG_CPU_R5000=y diff --git a/arch/mips/configs/ip27_defconfig b/arch/mips/configs/ip27_defconfig index 3e86f8106ba0..b51f738a39a0 100644 --- a/arch/mips/configs/ip27_defconfig +++ b/arch/mips/configs/ip27_defconfig @@ -9,7 +9,6 @@ CONFIG_CGROUPS=y CONFIG_CPUSETS=y CONFIG_RELAY=y CONFIG_EXPERT=y -CONFIG_SLAB=y CONFIG_SGI_IP27=y CONFIG_NUMA=y CONFIG_SMP=y diff --git a/arch/mips/configs/ip28_defconfig b/arch/mips/configs/ip28_defconfig index ba13eea0509f..e0040110a3ee 100644 --- a/arch/mips/configs/ip28_defconfig +++ b/arch/mips/configs/ip28_defconfig @@ -5,7 +5,6 @@ CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_RELAY=y CONFIG_EXPERT=y -CONFIG_SLAB=y CONFIG_SGI_IP28=y CONFIG_ARC_CONSOLE=y CONFIG_EISA=y diff --git a/arch/mips/configs/ip32_defconfig b/arch/mips/configs/ip32_defconfig index 8ced2224c328..418a4181e2fe 100644 --- a/arch/mips/configs/ip32_defconfig +++ b/arch/mips/configs/ip32_defconfig @@ -7,7 +7,6 @@ CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_RELAY=y CONFIG_EXPERT=y -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_SGI_IP32=y # CONFIG_SECCOMP is not set diff --git a/arch/mips/configs/jazz_defconfig b/arch/mips/configs/jazz_defconfig index 106b21cb677f..6b5d7e963ed9 100644 --- a/arch/mips/configs/jazz_defconfig +++ b/arch/mips/configs/jazz_defconfig @@ -7,7 +7,6 @@ CONFIG_BSD_PROCESS_ACCT=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_RELAY=y CONFIG_EXPERT=y -CONFIG_SLAB=y CONFIG_MACH_JAZZ=y CONFIG_OLIVETTI_M700=y CONFIG_MIPS_MAGNUM_4000=y diff --git a/arch/mips/configs/malta_defconfig b/arch/mips/configs/malta_defconfig index 265d38dffbf6..743209047792 100644 --- a/arch/mips/configs/malta_defconfig +++ b/arch/mips/configs/malta_defconfig @@ -7,7 +7,6 @@ CONFIG_RELAY=y CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_MIPS_MALTA=y CONFIG_CPU_LITTLE_ENDIAN=y CONFIG_CPU_MIPS32_R2=y diff --git a/arch/mips/configs/malta_kvm_defconfig b/arch/mips/configs/malta_kvm_defconfig index 1d2b248c7cd3..dd2b9c181f32 100644 --- a/arch/mips/configs/malta_kvm_defconfig +++ b/arch/mips/configs/malta_kvm_defconfig @@ -7,7 +7,6 @@ CONFIG_RELAY=y CONFIG_EXPERT=y CONFIG_PERF_EVENTS=y # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_MIPS_MALTA=y CONFIG_CPU_LITTLE_ENDIAN=y CONFIG_CPU_MIPS32_R2=y diff --git a/arch/mips/configs/malta_qemu_32r6_defconfig b/arch/mips/configs/malta_qemu_32r6_defconfig index fd63a2b152f6..82183ec6bc31 100644 --- a/arch/mips/configs/malta_qemu_32r6_defconfig +++ b/arch/mips/configs/malta_qemu_32r6_defconfig @@ -6,7 +6,6 @@ CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=15 CONFIG_EMBEDDED=y -CONFIG_SLAB=y CONFIG_MIPS_MALTA=y CONFIG_CPU_LITTLE_ENDIAN=y CONFIG_CPU_MIPS32_R6=y diff --git a/arch/mips/configs/maltaaprp_defconfig b/arch/mips/configs/maltaaprp_defconfig index 1f07e354c954..9a199867a5e7 100644 --- a/arch/mips/configs/maltaaprp_defconfig +++ b/arch/mips/configs/maltaaprp_defconfig @@ -6,7 +6,6 @@ CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=15 CONFIG_EMBEDDED=y -CONFIG_SLAB=y CONFIG_MIPS_MALTA=y CONFIG_CPU_LITTLE_ENDIAN=y CONFIG_CPU_MIPS32_R2=y diff --git a/arch/mips/configs/maltasmvp_defconfig b/arch/mips/configs/maltasmvp_defconfig index 5cd3eca236de..e5502d66a474 100644 --- a/arch/mips/configs/maltasmvp_defconfig +++ b/arch/mips/configs/maltasmvp_defconfig @@ -6,7 +6,6 @@ CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=15 CONFIG_EMBEDDED=y -CONFIG_SLAB=y CONFIG_MIPS_MALTA=y CONFIG_CPU_LITTLE_ENDIAN=y CONFIG_CPU_MIPS32_R2=y diff --git a/arch/mips/configs/maltasmvp_eva_defconfig b/arch/mips/configs/maltasmvp_eva_defconfig index 45688e742a15..a378aad97138 100644 --- a/arch/mips/configs/maltasmvp_eva_defconfig +++ b/arch/mips/configs/maltasmvp_eva_defconfig @@ -6,7 +6,6 @@ CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=15 CONFIG_EMBEDDED=y -CONFIG_SLAB=y CONFIG_MIPS_MALTA=y CONFIG_CPU_LITTLE_ENDIAN=y CONFIG_CPU_MIPS32_R2=y diff --git a/arch/mips/configs/maltaup_defconfig b/arch/mips/configs/maltaup_defconfig index 136f965784db..fc6f88cae7be 100644 --- a/arch/mips/configs/maltaup_defconfig +++ b/arch/mips/configs/maltaup_defconfig @@ -7,7 +7,6 @@ CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=15 CONFIG_EMBEDDED=y -CONFIG_SLAB=y CONFIG_MIPS_MALTA=y CONFIG_CPU_LITTLE_ENDIAN=y CONFIG_CPU_MIPS32_R2=y diff --git a/arch/mips/configs/maltaup_xpa_defconfig b/arch/mips/configs/maltaup_xpa_defconfig index 75b8da8d9927..97c2d7f530b3 100644 --- a/arch/mips/configs/maltaup_xpa_defconfig +++ b/arch/mips/configs/maltaup_xpa_defconfig @@ -8,7 +8,6 @@ CONFIG_NAMESPACES=y CONFIG_RELAY=y CONFIG_EXPERT=y # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_MIPS_MALTA=y CONFIG_CPU_LITTLE_ENDIAN=y CONFIG_CPU_MIPS32_R2=y diff --git a/arch/mips/configs/mtx1_defconfig b/arch/mips/configs/mtx1_defconfig index a5c68e9b0482..b64172179160 100644 --- a/arch/mips/configs/mtx1_defconfig +++ b/arch/mips/configs/mtx1_defconfig @@ -8,7 +8,6 @@ CONFIG_BSD_PROCESS_ACCT_V3=y CONFIG_RELAY=y CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_MIPS_ALCHEMY=y CONFIG_MIPS_MTX1=y diff --git a/arch/mips/configs/pic32mzda_defconfig b/arch/mips/configs/pic32mzda_defconfig index 48dd02d01ac1..8397f28355d5 100644 --- a/arch/mips/configs/pic32mzda_defconfig +++ b/arch/mips/configs/pic32mzda_defconfig @@ -9,7 +9,6 @@ CONFIG_RELAY=y CONFIG_CC_OPTIMIZE_FOR_SIZE=y CONFIG_EMBEDDED=y # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_MACH_PIC32=y CONFIG_DTB_PIC32_MZDA_SK=y CONFIG_HZ_100=y diff --git a/arch/mips/configs/qi_lb60_defconfig b/arch/mips/configs/qi_lb60_defconfig index 7e5d9741bd5d..c27c8c7151a1 100644 --- a/arch/mips/configs/qi_lb60_defconfig +++ b/arch/mips/configs/qi_lb60_defconfig @@ -6,7 +6,6 @@ CONFIG_KALLSYMS_ALL=y CONFIG_EMBEDDED=y # CONFIG_VM_EVENT_COUNTERS is not set # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_MACH_INGENIC_SOC=y CONFIG_JZ4740_QI_LB60=y CONFIG_HZ_100=y diff --git a/arch/mips/configs/rb532_defconfig b/arch/mips/configs/rb532_defconfig index 93306f5e045b..02ec6c1a5116 100644 --- a/arch/mips/configs/rb532_defconfig +++ b/arch/mips/configs/rb532_defconfig @@ -11,7 +11,6 @@ CONFIG_EXPERT=y # CONFIG_ELF_CORE is not set # CONFIG_KALLSYMS is not set # CONFIG_VM_EVENT_COUNTERS is not set -CONFIG_SLAB=y CONFIG_MIKROTIK_RB532=y CONFIG_HZ_100=y # CONFIG_SECCOMP is not set diff --git a/arch/mips/configs/rbtx49xx_defconfig b/arch/mips/configs/rbtx49xx_defconfig index 30c195f28278..03a7bbe28a53 100644 --- a/arch/mips/configs/rbtx49xx_defconfig +++ b/arch/mips/configs/rbtx49xx_defconfig @@ -7,7 +7,6 @@ CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y # CONFIG_EPOLL is not set -CONFIG_SLAB=y CONFIG_MACH_TX49XX=y CONFIG_TOSHIBA_RBTX4927=y # CONFIG_SECCOMP is not set diff --git a/arch/mips/configs/rm200_defconfig b/arch/mips/configs/rm200_defconfig index 9932a593e3c3..7475c2cbea89 100644 --- a/arch/mips/configs/rm200_defconfig +++ b/arch/mips/configs/rm200_defconfig @@ -7,7 +7,6 @@ CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_RELAY=y CONFIG_EXPERT=y -CONFIG_SLAB=y CONFIG_SNI_RM=y CONFIG_CPU_LITTLE_ENDIAN=y CONFIG_ARC_CONSOLE=y diff --git a/arch/mips/configs/rs90_defconfig b/arch/mips/configs/rs90_defconfig index 7ce3b814fdc8..85ea2a6775f5 100644 --- a/arch/mips/configs/rs90_defconfig +++ b/arch/mips/configs/rs90_defconfig @@ -17,7 +17,6 @@ CONFIG_LD_DEAD_CODE_DATA_ELIMINATION=y # CONFIG_KALLSYMS is not set CONFIG_EMBEDDED=y # CONFIG_PERF_EVENTS is not set -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_MACH_INGENIC_SOC=y CONFIG_JZ4740_RS90=y diff --git a/arch/mips/configs/sb1250_swarm_defconfig b/arch/mips/configs/sb1250_swarm_defconfig index 030186f89501..ce855b644bb0 100644 --- a/arch/mips/configs/sb1250_swarm_defconfig +++ b/arch/mips/configs/sb1250_swarm_defconfig @@ -10,7 +10,6 @@ CONFIG_RELAY=y CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_SIBYTE_SWARM=y CONFIG_CPU_SB1_PASS_2_2=y CONFIG_64BIT=y diff --git a/arch/nios2/configs/10m50_defconfig b/arch/nios2/configs/10m50_defconfig index 91c3fce4dc7f..63151ebf1470 100644 --- a/arch/nios2/configs/10m50_defconfig +++ b/arch/nios2/configs/10m50_defconfig @@ -10,7 +10,6 @@ CONFIG_LOG_BUF_SHIFT=14 # CONFIG_SHMEM is not set # CONFIG_AIO is not set CONFIG_EMBEDDED=y -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_NIOS2_MEM_BASE=0x8000000 diff --git a/arch/nios2/configs/3c120_defconfig b/arch/nios2/configs/3c120_defconfig index c42ad7e162a3..0daf3038d7aa 100644 --- a/arch/nios2/configs/3c120_defconfig +++ b/arch/nios2/configs/3c120_defconfig @@ -10,7 +10,6 @@ CONFIG_LOG_BUF_SHIFT=14 # CONFIG_SHMEM is not set # CONFIG_AIO is not set CONFIG_EMBEDDED=y -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_NIOS2_MEM_BASE=0x10000000 diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index c0b4b1c253d1..4cb46d5c64a2 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -304,16 +304,6 @@ config IRQSTACKS for handling hard and soft interrupts. This can help avoid overflowing the process kernel stacks. -config TLB_PTLOCK - bool "Use page table locks in TLB fault handler" - depends on SMP - default n - help - Select this option to enable page table locking in the TLB - fault handler. This ensures that page table entries are - updated consistently on SMP machines at the expense of some - loss in performance. - config HOTPLUG_CPU bool default y if SMP @@ -346,7 +336,7 @@ config NR_CPUS int "Maximum number of CPUs (2-32)" range 2 32 depends on SMP - default "4" if 64BIT + default "8" if 64BIT default "16" config KEXEC diff --git a/arch/parisc/Kconfig.debug b/arch/parisc/Kconfig.debug index 3a059cb5e112..1401e4c5fe5f 100644 --- a/arch/parisc/Kconfig.debug +++ b/arch/parisc/Kconfig.debug @@ -10,3 +10,14 @@ config LIGHTWEIGHT_SPINLOCK_CHECK spinlock debugging you should choose the DEBUG_SPINLOCK option which will detect unitialized spinlocks too. If unsure say Y here. + +config TLB_PTLOCK + bool "Use page table locks in TLB fault handler" + depends on SMP + default n + help + Select this option to enable page table locking in the TLB + fault handler. This ensures that page table entries are + updated consistently on SMP machines at the expense of some + loss in performance. + diff --git a/arch/parisc/configs/generic-32bit_defconfig b/arch/parisc/configs/generic-32bit_defconfig index 8ce0ae370680..9651f4390029 100644 --- a/arch/parisc/configs/generic-32bit_defconfig +++ b/arch/parisc/configs/generic-32bit_defconfig @@ -12,7 +12,6 @@ CONFIG_USER_NS=y CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y CONFIG_PERF_EVENTS=y -CONFIG_SLAB=y CONFIG_PA7100LC=y CONFIG_SMP=y CONFIG_HZ_100=y @@ -83,7 +82,6 @@ CONFIG_TUN=m # CONFIG_NET_VENDOR_AMD is not set # CONFIG_NET_VENDOR_ATHEROS is not set # CONFIG_NET_VENDOR_BROADCOM is not set -# CONFIG_NET_VENDOR_BROCADE is not set # CONFIG_NET_VENDOR_CHELSIO is not set # CONFIG_NET_VENDOR_CISCO is not set CONFIG_NET_TULIP=y @@ -98,6 +96,7 @@ CONFIG_LASI_82596=y # CONFIG_NET_VENDOR_NVIDIA is not set # CONFIG_NET_VENDOR_OKI is not set # CONFIG_NET_VENDOR_QLOGIC is not set +# CONFIG_NET_VENDOR_BROCADE is not set # CONFIG_NET_VENDOR_RDC is not set # CONFIG_NET_VENDOR_REALTEK is not set # CONFIG_NET_VENDOR_SEEQ is not set @@ -130,17 +129,53 @@ CONFIG_PRINTER=m CONFIG_PPDEV=m # CONFIG_HW_RANDOM is not set CONFIG_I2C=y -# CONFIG_HWMON is not set +CONFIG_HWMON=m +CONFIG_DRM=m +CONFIG_DRM_DP_CEC=y +# CONFIG_DRM_I2C_CH7006 is not set +# CONFIG_DRM_I2C_SIL164 is not set +CONFIG_DRM_RADEON=m +CONFIG_DRM_NOUVEAU=m +# CONFIG_DRM_NOUVEAU_BACKLIGHT is not set +CONFIG_DRM_VGEM=m +CONFIG_DRM_UDL=m +CONFIG_DRM_MGAG200=m CONFIG_FB=y CONFIG_FB_FOREIGN_ENDIAN=y -CONFIG_FB_MODE_HELPERS=y +CONFIG_FB_PM2=m +CONFIG_FB_PM2_FIFO_DISCONNECT=y +CONFIG_FB_NVIDIA=m +CONFIG_FB_NVIDIA_I2C=y +# CONFIG_FB_NVIDIA_BACKLIGHT is not set +CONFIG_FB_RIVA=m +CONFIG_FB_RIVA_I2C=y +# CONFIG_FB_RIVA_BACKLIGHT is not set CONFIG_FB_MATROX=m +CONFIG_FB_MATROX_MILLENIUM=y +CONFIG_FB_MATROX_MYSTIQUE=y CONFIG_FB_MATROX_G=y +CONFIG_FB_MATROX_I2C=m +CONFIG_FB_MATROX_MAVEN=m +CONFIG_FB_ATY128=m +# CONFIG_FB_ATY128_BACKLIGHT is not set +CONFIG_FB_ATY=m +CONFIG_FB_ATY_CT=y +CONFIG_FB_ATY_GX=y +# CONFIG_FB_ATY_BACKLIGHT is not set +CONFIG_FB_S3=m +CONFIG_FB_SAVAGE=m +CONFIG_FB_SAVAGE_I2C=y +CONFIG_FB_SAVAGE_ACCEL=y +CONFIG_FB_SIS=m +CONFIG_FB_SIS_300=y +CONFIG_FB_SIS_315=y CONFIG_FB_VOODOO1=m +CONFIG_FB_TRIDENT=m +CONFIG_FB_SMSCUFX=m +CONFIG_FB_UDL=m CONFIG_DUMMY_CONSOLE_COLUMNS=128 CONFIG_DUMMY_CONSOLE_ROWS=48 CONFIG_FRAMEBUFFER_CONSOLE=y -CONFIG_FRAMEBUFFER_CONSOLE_DETECT_PRIMARY=y CONFIG_LOGO=y # CONFIG_LOGO_LINUX_MONO is not set # CONFIG_LOGO_LINUX_VGA16 is not set @@ -217,17 +252,17 @@ CONFIG_CIFS_XATTR=y CONFIG_CIFS_POSIX=y # CONFIG_CIFS_DEBUG is not set CONFIG_CRYPTO_TEST=m -CONFIG_CRYPTO_HMAC=y -CONFIG_CRYPTO_MD5=y -CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_SHA1=y -CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_DES=y CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_TWOFISH=m +CONFIG_CRYPTO_HMAC=y +CONFIG_CRYPTO_MD5=y +CONFIG_CRYPTO_MICHAEL_MIC=m +CONFIG_CRYPTO_SHA1=y +CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_DEFLATE=y CONFIG_CRC_CCITT=m CONFIG_CRC_T10DIF=y diff --git a/arch/parisc/configs/generic-64bit_defconfig b/arch/parisc/configs/generic-64bit_defconfig index 57501b0aed92..6758c030b09d 100644 --- a/arch/parisc/configs/generic-64bit_defconfig +++ b/arch/parisc/configs/generic-64bit_defconfig @@ -20,9 +20,6 @@ CONFIG_USER_NS=y CONFIG_RELAY=y CONFIG_BLK_DEV_INITRD=y CONFIG_CC_OPTIMIZE_FOR_SIZE=y -# CONFIG_COMPAT_BRK is not set -CONFIG_PA8X00=y -CONFIG_64BIT=y CONFIG_SMP=y CONFIG_HPPB=y CONFIG_IOMMU_CCIO=y @@ -37,6 +34,7 @@ CONFIG_MODULE_FORCE_UNLOAD=y CONFIG_MODVERSIONS=y CONFIG_BLK_DEV_INTEGRITY=y CONFIG_BINFMT_MISC=m +# CONFIG_COMPAT_BRK is not set # CONFIG_COMPACTION is not set CONFIG_MEMORY_FAILURE=y CONFIG_NET=y @@ -103,7 +101,6 @@ CONFIG_TUN=y # CONFIG_NET_VENDOR_AMD is not set # CONFIG_NET_VENDOR_ATHEROS is not set # CONFIG_NET_VENDOR_BROADCOM is not set -# CONFIG_NET_VENDOR_BROCADE is not set # CONFIG_NET_VENDOR_CHELSIO is not set # CONFIG_NET_VENDOR_CISCO is not set CONFIG_NET_TULIP=y @@ -121,6 +118,7 @@ CONFIG_E1000=y # CONFIG_NET_VENDOR_OKI is not set CONFIG_QLA3XXX=m CONFIG_QLCNIC=m +# CONFIG_NET_VENDOR_BROCADE is not set # CONFIG_NET_VENDOR_RDC is not set # CONFIG_NET_VENDOR_REALTEK is not set # CONFIG_NET_VENDOR_SEEQ is not set @@ -186,7 +184,6 @@ CONFIG_WATCHDOG=y CONFIG_SOFT_WATCHDOG=m CONFIG_SSB=m CONFIG_SSB_DRIVER_PCICORE=y -CONFIG_HTC_PASIC3=m CONFIG_LPC_SCH=m CONFIG_MFD_SM501=m CONFIG_REGULATOR=y @@ -196,14 +193,46 @@ CONFIG_MEDIA_SUPPORT=m CONFIG_AGP=y CONFIG_AGP_PARISC=y CONFIG_DRM=y +# CONFIG_DRM_I2C_CH7006 is not set +# CONFIG_DRM_I2C_SIL164 is not set CONFIG_DRM_RADEON=y +CONFIG_DRM_NOUVEAU=m +# CONFIG_DRM_NOUVEAU_BACKLIGHT is not set +CONFIG_DRM_MGAG200=m CONFIG_FB=y -CONFIG_FB_MATROX=y +CONFIG_FB_PM2=m +CONFIG_FB_PM2_FIFO_DISCONNECT=y +CONFIG_FB_NVIDIA=m +CONFIG_FB_NVIDIA_I2C=y +# CONFIG_FB_NVIDIA_BACKLIGHT is not set +CONFIG_FB_RIVA=m +CONFIG_FB_RIVA_I2C=y +# CONFIG_FB_RIVA_BACKLIGHT is not set +CONFIG_FB_MATROX=m +CONFIG_FB_MATROX_MILLENIUM=y CONFIG_FB_MATROX_MYSTIQUE=y CONFIG_FB_MATROX_G=y -CONFIG_FB_MATROX_I2C=y -CONFIG_FB_MATROX_MAVEN=y +CONFIG_FB_MATROX_I2C=m +CONFIG_FB_MATROX_MAVEN=m CONFIG_FB_RADEON=y +# CONFIG_FB_RADEON_BACKLIGHT is not set +CONFIG_FB_ATY128=m +# CONFIG_FB_ATY128_BACKLIGHT is not set +CONFIG_FB_ATY=m +CONFIG_FB_ATY_CT=y +CONFIG_FB_ATY_GX=y +# CONFIG_FB_ATY_BACKLIGHT is not set +CONFIG_FB_S3=m +CONFIG_FB_SAVAGE=m +CONFIG_FB_SAVAGE_I2C=y +CONFIG_FB_SAVAGE_ACCEL=y +CONFIG_FB_SIS=m +CONFIG_FB_SIS_300=y +CONFIG_FB_SIS_315=y +CONFIG_FB_VOODOO1=m +CONFIG_FB_TRIDENT=m +CONFIG_FB_SMSCUFX=m +CONFIG_FB_UDL=m CONFIG_LOGO=y # CONFIG_LOGO_LINUX_CLUT224 is not set CONFIG_HIDRAW=y @@ -257,12 +286,12 @@ CONFIG_NLS_ISO8859_1=m CONFIG_NLS_ISO8859_2=m CONFIG_NLS_UTF8=m CONFIG_CRYPTO_MANAGER=y +CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_ECB=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MD5=y CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_DEFLATE=m # CONFIG_CRYPTO_HW is not set CONFIG_CRC_CCITT=m diff --git a/arch/parisc/include/asm/irqflags.h b/arch/parisc/include/asm/irqflags.h index 38a19c0bac3a..00fd87724588 100644 --- a/arch/parisc/include/asm/irqflags.h +++ b/arch/parisc/include/asm/irqflags.h @@ -31,6 +31,11 @@ static inline unsigned long arch_local_irq_save(void) static inline void arch_local_irq_restore(unsigned long flags) { + /* warn if IRQs are on although they should be off */ + if (IS_ENABLED(CONFIG_LIGHTWEIGHT_SPINLOCK_CHECK)) + if (arch_local_save_flags() & PSW_I) + asm volatile("break 6,6\n"); /* SPINLOCK_BREAK_INSN */ + asm volatile("mtsm %0" : : "r" (flags) : "memory"); } diff --git a/arch/parisc/include/asm/pdc.h b/arch/parisc/include/asm/pdc.h index 2b4fad8328e8..269b9a159f01 100644 --- a/arch/parisc/include/asm/pdc.h +++ b/arch/parisc/include/asm/pdc.h @@ -88,8 +88,8 @@ int pdc_iodc_print(const unsigned char *str, unsigned count); void pdc_emergency_unlock(void); int pdc_sti_call(unsigned long func, unsigned long flags, - unsigned long inptr, unsigned long outputr, - unsigned long glob_cfg); + unsigned long inptr, unsigned long outputr, + unsigned long glob_cfg, int do_call64); int __pdc_cpu_rendezvous(void); void pdc_cpu_rendezvous_lock(void); diff --git a/arch/parisc/include/asm/processor.h b/arch/parisc/include/asm/processor.h index a608970b249a..e132b2819fc9 100644 --- a/arch/parisc/include/asm/processor.h +++ b/arch/parisc/include/asm/processor.h @@ -11,6 +11,7 @@ #ifndef __ASSEMBLY__ #include <linux/threads.h> +#include <linux/irqreturn.h> #include <asm/assembly.h> #include <asm/prefetch.h> @@ -291,6 +292,42 @@ extern void __noreturn toc_intr(struct pt_regs *regs); extern void toc_handler(void); extern unsigned int toc_handler_size; extern unsigned int toc_handler_csum; +extern void do_cpu_irq_mask(struct pt_regs *); +extern irqreturn_t timer_interrupt(int, void *); +extern irqreturn_t ipi_interrupt(int, void *); +extern void start_cpu_itimer(void); +extern void handle_interruption(int, struct pt_regs *); + +/* called from assembly code: */ +extern void start_parisc(void); +extern void smp_callin(unsigned long); +extern void sys_rt_sigreturn(struct pt_regs *, int); +extern void do_notify_resume(struct pt_regs *, long); +extern long do_syscall_trace_enter(struct pt_regs *); +extern void do_syscall_trace_exit(struct pt_regs *); + +/* CPU startup and info */ +struct seq_file; +extern void early_trap_init(void); +extern void collect_boot_cpu_data(void); +extern int show_cpuinfo (struct seq_file *m, void *v); + +/* driver code in driver/parisc */ +extern void gsc_init(void); +extern void processor_init(void); +extern void ccio_init(void); +extern void hppb_init(void); +extern void dino_init(void); +extern void iosapic_init(void); +extern void lba_init(void); +extern void sba_init(void); +extern void parisc_eisa_init(void); +struct parisc_device; +struct resource; +extern void sba_distributed_lmmio(struct parisc_device *, struct resource *); +extern void sba_directed_lmmio(struct parisc_device *, struct resource *); +extern void lba_set_iregs(struct parisc_device *lba, u32 ibase, u32 imask); +extern void ccio_cujo20_fixup(struct parisc_device *dev, u32 iovp); #endif /* __ASSEMBLY__ */ diff --git a/arch/parisc/include/uapi/asm/cachectl.h b/arch/parisc/include/uapi/asm/cachectl.h new file mode 100644 index 000000000000..68d6b455498b --- /dev/null +++ b/arch/parisc/include/uapi/asm/cachectl.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _ASM_CACHECTL +#define _ASM_CACHECTL + +/* + * Options for cacheflush system call + */ +#define ICACHE (1<<0) /* flush instruction cache */ +#define DCACHE (1<<1) /* writeback and flush data cache */ +#define BCACHE (ICACHE|DCACHE) /* flush both caches */ + +#endif /* _ASM_CACHECTL */ diff --git a/arch/parisc/kernel/Makefile b/arch/parisc/kernel/Makefile index 3d138c9cf9ce..2d1478fc4aa5 100644 --- a/arch/parisc/kernel/Makefile +++ b/arch/parisc/kernel/Makefile @@ -21,6 +21,9 @@ CFLAGS_REMOVE_unwind.o = $(CC_FLAGS_FTRACE) CFLAGS_REMOVE_patch.o = $(CC_FLAGS_FTRACE) endif +CFLAGS_REMOVE_sys_parisc.o = -Wmissing-prototypes -Wmissing-declarations +CFLAGS_REMOVE_sys_parisc32.o = -Wmissing-prototypes -Wmissing-declarations + obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_PA11) += pci-dma.o obj-$(CONFIG_PCI) += pci.o diff --git a/arch/parisc/kernel/audit.c b/arch/parisc/kernel/audit.c index f420b5552140..375cd73b5281 100644 --- a/arch/parisc/kernel/audit.c +++ b/arch/parisc/kernel/audit.c @@ -40,11 +40,6 @@ int audit_classify_arch(int arch) int audit_classify_syscall(int abi, unsigned syscall) { -#ifdef CONFIG_COMPAT - extern int parisc32_classify_syscall(unsigned); - if (abi == AUDIT_ARCH_PARISC) - return parisc32_classify_syscall(syscall); -#endif switch (syscall) { case __NR_open: return AUDITSC_OPEN; @@ -55,6 +50,10 @@ int audit_classify_syscall(int abi, unsigned syscall) case __NR_openat2: return AUDITSC_OPENAT2; default: +#ifdef CONFIG_COMPAT + if (abi == AUDIT_ARCH_PARISC) + return AUDITSC_COMPAT; +#endif return AUDITSC_NATIVE; } } diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c index 501160250bb7..b55b35c89d6a 100644 --- a/arch/parisc/kernel/cache.c +++ b/arch/parisc/kernel/cache.c @@ -19,6 +19,7 @@ #include <linux/pagemap.h> #include <linux/sched.h> #include <linux/sched/mm.h> +#include <linux/syscalls.h> #include <asm/pdc.h> #include <asm/cache.h> #include <asm/cacheflush.h> @@ -28,6 +29,7 @@ #include <asm/sections.h> #include <asm/shmparam.h> #include <asm/mmu_context.h> +#include <asm/cachectl.h> int split_tlb __ro_after_init; int dcache_stride __ro_after_init; @@ -790,3 +792,50 @@ void invalidate_kernel_vmap_range(void *vaddr, int size) flush_tlb_kernel_range(start, end); } EXPORT_SYMBOL(invalidate_kernel_vmap_range); + + +SYSCALL_DEFINE3(cacheflush, unsigned long, addr, unsigned long, bytes, + unsigned int, cache) +{ + unsigned long start, end; + ASM_EXCEPTIONTABLE_VAR(error); + + if (bytes == 0) + return 0; + if (!access_ok((void __user *) addr, bytes)) + return -EFAULT; + + end = addr + bytes; + + if (cache & DCACHE) { + start = addr; + __asm__ __volatile__ ( +#ifdef CONFIG_64BIT + "1: cmpb,*<<,n %0,%2,1b\n" +#else + "1: cmpb,<<,n %0,%2,1b\n" +#endif + " fic,m %3(%4,%0)\n" + "2: sync\n" + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 2b) + : "+r" (start), "+r" (error) + : "r" (end), "r" (dcache_stride), "i" (SR_USER)); + } + + if (cache & ICACHE && error == 0) { + start = addr; + __asm__ __volatile__ ( +#ifdef CONFIG_64BIT + "1: cmpb,*<<,n %0,%2,1b\n" +#else + "1: cmpb,<<,n %0,%2,1b\n" +#endif + " fdc,m %3(%4,%0)\n" + "2: sync\n" + ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 2b) + : "+r" (start), "+r" (error) + : "r" (end), "r" (icache_stride), "i" (SR_USER)); + } + + return error; +} diff --git a/arch/parisc/kernel/compat_audit.c b/arch/parisc/kernel/compat_audit.c index 539b16891bdf..3ac53f1ab860 100644 --- a/arch/parisc/kernel/compat_audit.c +++ b/arch/parisc/kernel/compat_audit.c @@ -26,19 +26,3 @@ unsigned int parisc32_signal_class[] = { #include <asm-generic/audit_signal.h> ~0U }; - -int parisc32_classify_syscall(unsigned syscall) -{ - switch (syscall) { - case __NR_open: - return AUDITSC_OPEN; - case __NR_openat: - return AUDITSC_OPENAT; - case __NR_execve: - return AUDITSC_EXECVE; - case __NR_openat2: - return AUDITSC_OPENAT2; - default: - return AUDITSC_COMPAT; - } -} diff --git a/arch/parisc/kernel/drivers.c b/arch/parisc/kernel/drivers.c index 28f47285d448..8f4b77648491 100644 --- a/arch/parisc/kernel/drivers.c +++ b/arch/parisc/kernel/drivers.c @@ -4,7 +4,7 @@ * * Copyright (c) 1999 The Puffin Group * Copyright (c) 2001 Matthew Wilcox for Hewlett Packard - * Copyright (c) 2001 Helge Deller <deller@gmx.de> + * Copyright (c) 2001-2023 Helge Deller <deller@gmx.de> * Copyright (c) 2001,2002 Ryan Bradetich * Copyright (c) 2004-2005 Thibaut VARENE <varenet@parisc-linux.org> * @@ -74,13 +74,13 @@ static int descend_children(struct device * dev, void * data) } /** - * for_each_padev - Iterate over all devices in the tree - * @fn: Function to call for each device. - * @data: Data to pass to the called function. + * for_each_padev - Iterate over all devices in the tree + * @fn: Function to call for each device. + * @data: Data to pass to the called function. * - * This performs a depth-first traversal of the tree, calling the - * function passed for each node. It calls the function for parents - * before children. + * This performs a depth-first traversal of the tree, calling the + * function passed for each node. It calls the function for parents + * before children. */ static int for_each_padev(int (*fn)(struct device *, void *), void * data) @@ -280,7 +280,7 @@ int __init machine_has_merced_bus(void) /** * find_pa_parent_type - Find a parent of a specific type - * @dev: The device to start searching from + * @padev: The device to start searching from * @type: The device type to search for. * * Walks up the device tree looking for a device of the specified type. @@ -344,8 +344,8 @@ static char *print_hwpath(struct hardware_path *path, char *output) /** * print_pa_hwpath - Returns hardware path for PA devices - * dev: The device to return the path for - * output: Pointer to a previously-allocated array to place the path in. + * @dev: The device to return the path for + * @output: Pointer to a previously-allocated array to place the path in. * * This function fills in the output array with a human-readable path * to a PA device. This string is compatible with that used by PDC, and @@ -379,8 +379,8 @@ EXPORT_SYMBOL(get_pci_node_path); /** * print_pci_hwpath - Returns hardware path for PCI devices - * dev: The device to return the path for - * output: Pointer to a previously-allocated array to place the path in. + * @dev: The device to return the path for + * @output: Pointer to a previously-allocated array to place the path in. * * This function fills in the output array with a human-readable path * to a PCI device. This string is compatible with that used by PDC, and @@ -415,7 +415,8 @@ static void setup_bus_id(struct parisc_device *padev) dev_set_name(&padev->dev, name); } -struct parisc_device * __init create_tree_node(char id, struct device *parent) +static struct parisc_device * __init create_tree_node(char id, + struct device *parent) { struct parisc_device *dev = kzalloc(sizeof(*dev), GFP_KERNEL); if (!dev) @@ -741,7 +742,7 @@ parse_tree_node(struct device *parent, int index, struct hardware_path *modpath) }; if (device_for_each_child(parent, &recurse_data, descend_children)) - /* nothing */; + { /* nothing */ }; return d.dev; } @@ -771,8 +772,8 @@ EXPORT_SYMBOL(hwpath_to_device); /** * device_to_hwpath - Populates the hwpath corresponding to the given device. - * @param dev the target device - * @param path pointer to a previously allocated hwpath struct to be filled in + * @dev: the target device + * @path: pointer to a previously allocated hwpath struct to be filled in */ void device_to_hwpath(struct device *dev, struct hardware_path *path) { diff --git a/arch/parisc/kernel/firmware.c b/arch/parisc/kernel/firmware.c index cc124d9f1f7f..6d1c781eb1db 100644 --- a/arch/parisc/kernel/firmware.c +++ b/arch/parisc/kernel/firmware.c @@ -134,7 +134,7 @@ static unsigned long f_extend(unsigned long address) /** * convert_to_wide - Convert the return buffer addresses into kernel addresses. - * @address: The return buffer from PDC. + * @addr: The return buffer from PDC. * * This function is used to convert the return buffer addresses retrieved from PDC * into kernel addresses when the PDC address size and kernel address size are @@ -160,6 +160,8 @@ void set_firmware_width_unlocked(void) ret = mem_pdc_call(PDC_MODEL, PDC_MODEL_CAPABILITIES, __pa(pdc_result), 0); + if (ret < 0) + return; convert_to_wide(pdc_result); if (pdc_result[0] != NARROW_FIRMWARE) parisc_narrow_firmware = 0; @@ -255,8 +257,8 @@ int __init pdc_instr(unsigned int *instr) /** * pdc_chassis_info - Return chassis information. - * @result: The return buffer. * @chassis_info: The memory buffer address. + * @led_info: The size of the memory buffer address. * @len: The size of the memory buffer address. * * An HVERSION dependent call for returning the chassis information. @@ -280,7 +282,8 @@ int __init pdc_chassis_info(struct pdc_chassis_info *chassis_info, void *led_inf /** * pdc_pat_chassis_send_log - Sends a PDC PAT CHASSIS log message. - * @retval: -1 on error, 0 on success. Other value are PDC errors + * @state: state of the machine + * @data: value for that state * * Must be correctly formatted or expect system crash */ @@ -303,7 +306,7 @@ int pdc_pat_chassis_send_log(unsigned long state, unsigned long data) /** * pdc_chassis_disp - Updates chassis code - * @retval: -1 on error, 0 on success + * @disp: value to show on display */ int pdc_chassis_disp(unsigned long disp) { @@ -318,8 +321,7 @@ int pdc_chassis_disp(unsigned long disp) } /** - * pdc_cpu_rendenzvous - Stop currently executing CPU - * @retval: -1 on error, 0 on success + * __pdc_cpu_rendezvous - Stop currently executing CPU and do not return. */ int __pdc_cpu_rendezvous(void) { @@ -347,7 +349,7 @@ void pdc_cpu_rendezvous_unlock(void) /** * pdc_pat_get_PDC_entrypoint - Get PDC entry point for current CPU - * @retval: -1 on error, 0 on success + * @pdc_entry: pointer to where the PDC entry point should be stored */ int pdc_pat_get_PDC_entrypoint(unsigned long *pdc_entry) { @@ -369,7 +371,7 @@ int pdc_pat_get_PDC_entrypoint(unsigned long *pdc_entry) } /** * pdc_chassis_warn - Fetches chassis warnings - * @retval: -1 on error, 0 on success + * @warn: The warning value to be shown */ int pdc_chassis_warn(unsigned long *warn) { @@ -521,6 +523,7 @@ int pdc_model_info(struct pdc_model *model) /** * pdc_model_sysmodel - Get the system model name. + * @os_id: The operating system ID asked for (an OS_ID_* value) * @name: A char array of at least 81 characters. * * Get system model name from PDC ROM (e.g. 9000/715 or 9000/778/B160L). @@ -549,7 +552,7 @@ int pdc_model_sysmodel(unsigned int os_id, char *name) /** * pdc_model_versions - Identify the version number of each processor. - * @cpu_id: The return buffer. + * @versions: The return buffer. * @id: The id of the processor to check. * * Returns the version number for each processor component. @@ -996,8 +999,8 @@ int pdc_pci_irt(unsigned long num_entries, unsigned long hpa, void *tbl) /** * pdc_pci_config_read - read PCI config space. - * @hpa token from PDC to indicate which PCI device - * @pci_addr configuration space address to read from + * @hpa: Token from PDC to indicate which PCI device + * @cfg_addr: Configuration space address to read from * * Read PCI Configuration space *before* linux PCI subsystem is running. */ @@ -1019,9 +1022,9 @@ unsigned int pdc_pci_config_read(void *hpa, unsigned long cfg_addr) /** * pdc_pci_config_write - read PCI config space. - * @hpa token from PDC to indicate which PCI device - * @pci_addr configuration space address to write - * @val value we want in the 32-bit register + * @hpa: Token from PDC to indicate which PCI device + * @cfg_addr: Configuration space address to write + * @val: Value we want in the 32-bit register * * Write PCI Configuration space *before* linux PCI subsystem is running. */ @@ -1389,17 +1392,25 @@ int pdc_iodc_getc(void) } int pdc_sti_call(unsigned long func, unsigned long flags, - unsigned long inptr, unsigned long outputr, - unsigned long glob_cfg) + unsigned long inptr, unsigned long outputr, + unsigned long glob_cfg, int do_call64) { - int retval; + int retval = 0; unsigned long irqflags; - spin_lock_irqsave(&pdc_lock, irqflags); - retval = real32_call(func, flags, inptr, outputr, glob_cfg); - spin_unlock_irqrestore(&pdc_lock, irqflags); + spin_lock_irqsave(&pdc_lock, irqflags); + if (IS_ENABLED(CONFIG_64BIT) && do_call64) { +#ifdef CONFIG_64BIT + retval = real64_call(func, flags, inptr, outputr, glob_cfg); +#else + WARN_ON(1); +#endif + } else { + retval = real32_call(func, flags, inptr, outputr, glob_cfg); + } + spin_unlock_irqrestore(&pdc_lock, irqflags); - return retval; + return retval; } EXPORT_SYMBOL(pdc_sti_call); @@ -1549,7 +1560,7 @@ int pdc_pat_get_irt(void *r_addr, unsigned long cell_num) /** * pdc_pat_pd_get_addr_map - Retrieve information about memory address ranges. - * @actlen: The return buffer. + * @actual_len: The return buffer. * @mem_addr: Pointer to the memory buffer. * @count: The number of bytes to read from the buffer. * @offset: The offset with respect to the beginning of the buffer. @@ -1572,7 +1583,7 @@ int pdc_pat_pd_get_addr_map(unsigned long *actual_len, void *mem_addr, } /** - * pdc_pat_pd_get_PDC_interface_revisions - Retrieve PDC interface revisions. + * pdc_pat_pd_get_pdc_revisions - Retrieve PDC interface revisions. * @legacy_rev: The legacy revision. * @pat_rev: The PAT revision. * @pdc_cap: The PDC capabilities. @@ -1627,7 +1638,7 @@ int pdc_pat_io_pci_cfg_read(unsigned long pci_addr, int pci_size, u32 *mem_addr) * pdc_pat_io_pci_cfg_write - Retrieve information about memory address ranges. * @pci_addr: PCI configuration space address for which the write request is being made. * @pci_size: Size of write in bytes. Valid values are 1, 2, and 4. - * @value: Pointer to 1, 2, or 4 byte value in low order end of argument to be + * @val: Pointer to 1, 2, or 4 byte value in low order end of argument to be * written to PCI Config space. * */ @@ -1645,7 +1656,7 @@ int pdc_pat_io_pci_cfg_write(unsigned long pci_addr, int pci_size, u32 val) } /** - * pdc_pat_mem_pdc_info - Retrieve information about page deallocation table + * pdc_pat_mem_pdt_info - Retrieve information about page deallocation table * @rinfo: memory pdt information * */ diff --git a/arch/parisc/kernel/irq.c b/arch/parisc/kernel/irq.c index b05055f3ba4b..12c4d4104ade 100644 --- a/arch/parisc/kernel/irq.c +++ b/arch/parisc/kernel/irq.c @@ -24,9 +24,6 @@ #undef PARISC_IRQ_CR16_COUNTS -extern irqreturn_t timer_interrupt(int, void *); -extern irqreturn_t ipi_interrupt(int, void *); - #define EIEM_MASK(irq) (1UL<<(CPU_IRQ_MAX - irq)) /* Bits in EIEM correlate with cpu_irq_action[]. @@ -489,7 +486,7 @@ void do_softirq_own_stack(void) #endif /* CONFIG_IRQSTACKS */ /* ONLY called from entry.S:intr_extint() */ -void do_cpu_irq_mask(struct pt_regs *regs) +asmlinkage void do_cpu_irq_mask(struct pt_regs *regs) { struct pt_regs *old_regs; unsigned long eirr_val; diff --git a/arch/parisc/kernel/module.c b/arch/parisc/kernel/module.c index f6e38c4d3904..d214bbe3c2af 100644 --- a/arch/parisc/kernel/module.c +++ b/arch/parisc/kernel/module.c @@ -845,7 +845,7 @@ int module_finalize(const Elf_Ehdr *hdr, const char *strtab = NULL; const Elf_Shdr *s; char *secstrings; - int symindex = -1; + int symindex __maybe_unused = -1; Elf_Sym *newptr, *oldptr; Elf_Shdr *symhdr = NULL; #ifdef DEBUG diff --git a/arch/parisc/kernel/pci-dma.c b/arch/parisc/kernel/pci-dma.c index 415f12d5bab3..d818ece23b4a 100644 --- a/arch/parisc/kernel/pci-dma.c +++ b/arch/parisc/kernel/pci-dma.c @@ -245,7 +245,7 @@ static void unmap_uncached_pages(unsigned long vaddr, unsigned long size) PCXL_SEARCH_LOOP(idx, mask, size); \ } -unsigned long +static unsigned long pcxl_alloc_range(size_t size) { int res_idx; diff --git a/arch/parisc/kernel/pdc_chassis.c b/arch/parisc/kernel/pdc_chassis.c index da154406d368..0a9d7008ef2a 100644 --- a/arch/parisc/kernel/pdc_chassis.c +++ b/arch/parisc/kernel/pdc_chassis.c @@ -40,7 +40,7 @@ static unsigned int pdc_chassis_enabled __read_mostly = 1; /** * pdc_chassis_setup() - Enable/disable pdc_chassis code at boot time. - * @str configuration param: 0 to disable chassis log + * @str: configuration param: 0 to disable chassis log * @return 1 */ @@ -55,7 +55,6 @@ __setup("pdcchassis=", pdc_chassis_setup); /** * pdc_chassis_checkold() - Checks for old PDC_CHASSIS compatibility - * @pdc_chassis_old: 1 if old pdc chassis style * * Currently, only E class and A180 are known to work with this. * Inspired by Christoph Plattner @@ -80,6 +79,9 @@ static void __init pdc_chassis_checkold(void) /** * pdc_chassis_panic_event() - Called by the panic handler. + * @this: unused + * @event: unused + * @ptr: unused * * As soon as a panic occurs, we should inform the PDC. */ @@ -88,7 +90,7 @@ static int pdc_chassis_panic_event(struct notifier_block *this, unsigned long event, void *ptr) { pdc_chassis_send_status(PDC_CHASSIS_DIRECT_PANIC); - return NOTIFY_DONE; + return NOTIFY_DONE; } @@ -99,7 +101,10 @@ static struct notifier_block pdc_chassis_panic_block = { /** - * parisc_reboot_event() - Called by the reboot handler. + * pdc_chassis_reboot_event() - Called by the reboot handler. + * @this: unused + * @event: unused + * @ptr: unused * * As soon as a reboot occurs, we should inform the PDC. */ @@ -108,7 +113,7 @@ static int pdc_chassis_reboot_event(struct notifier_block *this, unsigned long event, void *ptr) { pdc_chassis_send_status(PDC_CHASSIS_DIRECT_SHUTDOWN); - return NOTIFY_DONE; + return NOTIFY_DONE; } @@ -148,7 +153,7 @@ void __init parisc_pdc_chassis_init(void) /** * pdc_chassis_send_status() - Sends a predefined message to the chassis, * and changes the front panel LEDs according to the new system state - * @retval: PDC call return value. + * @message: Type of message, one of PDC_CHASSIS_DIRECT_* values. * * Only machines with 64 bits PDC PAT and those reported in * pdc_chassis_checkold() are supported atm. diff --git a/arch/parisc/kernel/pdt.c b/arch/parisc/kernel/pdt.c index 80943a00e245..0d24735bd918 100644 --- a/arch/parisc/kernel/pdt.c +++ b/arch/parisc/kernel/pdt.c @@ -16,6 +16,7 @@ #include <linux/memblock.h> #include <linux/seq_file.h> #include <linux/kthread.h> +#include <linux/proc_fs.h> #include <linux/initrd.h> #include <linux/pgtable.h> #include <linux/mm.h> @@ -23,6 +24,7 @@ #include <asm/pdc.h> #include <asm/pdcpat.h> #include <asm/sections.h> +#include <asm/pgtable.h> enum pdt_access_type { PDT_NONE, diff --git a/arch/parisc/kernel/perf.c b/arch/parisc/kernel/perf.c index d46b6709ec56..90b04d8af212 100644 --- a/arch/parisc/kernel/perf.c +++ b/arch/parisc/kernel/perf.c @@ -288,7 +288,7 @@ static ssize_t perf_read(struct file *file, char __user *buf, size_t cnt, loff_t static ssize_t perf_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - size_t image_size; + size_t image_size __maybe_unused; uint32_t image_type; uint32_t interface_type; uint32_t test; diff --git a/arch/parisc/kernel/processor.c b/arch/parisc/kernel/processor.c index ba07e760d3c7..00b0df97afb1 100644 --- a/arch/parisc/kernel/processor.c +++ b/arch/parisc/kernel/processor.c @@ -58,7 +58,7 @@ DEFINE_PER_CPU(struct cpuinfo_parisc, cpu_data); */ /** - * init_cpu_profiler - enable/setup per cpu profiling hooks. + * init_percpu_prof - enable/setup per cpu profiling hooks. * @cpunum: The processor instance. * * FIXME: doesn't do much yet... diff --git a/arch/parisc/kernel/setup.c b/arch/parisc/kernel/setup.c index 0797db617962..573f8303e2b0 100644 --- a/arch/parisc/kernel/setup.c +++ b/arch/parisc/kernel/setup.c @@ -45,7 +45,7 @@ struct proc_dir_entry * proc_runway_root __read_mostly = NULL; struct proc_dir_entry * proc_gsc_root __read_mostly = NULL; struct proc_dir_entry * proc_mckinley_root __read_mostly = NULL; -void __init setup_cmdline(char **cmdline_p) +static void __init setup_cmdline(char **cmdline_p) { extern unsigned int boot_args[]; char *p; @@ -86,7 +86,7 @@ void __init setup_cmdline(char **cmdline_p) } #ifdef CONFIG_PA11 -void __init dma_ops_init(void) +static void __init dma_ops_init(void) { switch (boot_cpu_data.cpu_type) { case pcx: @@ -106,8 +106,6 @@ void __init dma_ops_init(void) } #endif -extern void collect_boot_cpu_data(void); - void __init setup_arch(char **cmdline_p) { #ifdef CONFIG_64BIT @@ -167,10 +165,7 @@ void __init setup_arch(char **cmdline_p) /* * Display CPU info for all CPUs. - * for parisc this is in processor.c */ -extern int show_cpuinfo (struct seq_file *m, void *v); - static void * c_start (struct seq_file *m, loff_t *pos) { @@ -295,16 +290,6 @@ static int __init parisc_init_resources(void) return 0; } -extern void gsc_init(void); -extern void processor_init(void); -extern void ccio_init(void); -extern void hppb_init(void); -extern void dino_init(void); -extern void iosapic_init(void); -extern void lba_init(void); -extern void sba_init(void); -extern void eisa_init(void); - static int __init parisc_init(void) { u32 osid = (OS_ID_LINUX << 16); @@ -370,7 +355,7 @@ static int __init parisc_init(void) gsc_init(); #endif #ifdef CONFIG_EISA - eisa_init(); + parisc_eisa_init(); #endif #if defined(CONFIG_HPPB) @@ -391,8 +376,6 @@ arch_initcall(parisc_init); void __init start_parisc(void) { - extern void early_trap_init(void); - int ret, cpunum; struct pdc_coproc_cfg coproc_cfg; diff --git a/arch/parisc/kernel/signal.c b/arch/parisc/kernel/signal.c index 8bc0ddaa6219..f886ff0c75df 100644 --- a/arch/parisc/kernel/signal.c +++ b/arch/parisc/kernel/signal.c @@ -73,7 +73,7 @@ restore_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs) return err; } -void +asmlinkage void sys_rt_sigreturn(struct pt_regs *regs, int in_syscall) { struct rt_sigframe __user *frame; @@ -176,7 +176,7 @@ get_sigframe(struct k_sigaction *ka, unsigned long sp, size_t frame_size) } static long -setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs, int in_syscall) +setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs, long in_syscall) { unsigned long flags = 0; @@ -211,7 +211,7 @@ setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs, int in_sysc static long setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs, - int in_syscall) + long in_syscall) { struct rt_sigframe __user *frame; unsigned long rp, usp; @@ -380,7 +380,7 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs, */ static void -handle_signal(struct ksignal *ksig, struct pt_regs *regs, int in_syscall) +handle_signal(struct ksignal *ksig, struct pt_regs *regs, long in_syscall) { int ret; sigset_t *oldset = sigmask_to_save(); @@ -578,7 +578,7 @@ static void do_signal(struct pt_regs *regs, long in_syscall) restore_saved_sigmask(); } -void do_notify_resume(struct pt_regs *regs, long in_syscall) +asmlinkage void do_notify_resume(struct pt_regs *regs, long in_syscall) { if (test_thread_flag(TIF_SIGPENDING) || test_thread_flag(TIF_NOTIFY_SIGNAL)) diff --git a/arch/parisc/kernel/smp.c b/arch/parisc/kernel/smp.c index d0eb1bd19a13..4098f9a0964b 100644 --- a/arch/parisc/kernel/smp.c +++ b/arch/parisc/kernel/smp.c @@ -271,8 +271,6 @@ void arch_send_call_function_single_ipi(int cpu) static void smp_cpu_init(int cpunum) { - extern void start_cpu_itimer(void); /* arch/parisc/kernel/time.c */ - /* Set modes and Enable floating point coprocessor */ init_per_cpu(cpunum); diff --git a/arch/parisc/kernel/sys_parisc.c b/arch/parisc/kernel/sys_parisc.c index 39acccabf2ed..9915062d5243 100644 --- a/arch/parisc/kernel/sys_parisc.c +++ b/arch/parisc/kernel/sys_parisc.c @@ -24,6 +24,7 @@ #include <linux/personality.h> #include <linux/random.h> #include <linux/compat.h> +#include <linux/elf-randomize.h> /* * Construct an artificial page offset for the mapping based on the physical @@ -339,7 +340,7 @@ asmlinkage long parisc_fallocate(int fd, int mode, u32 offhi, u32 offlo, ((u64)lenhi << 32) | lenlo); } -long parisc_personality(unsigned long personality) +asmlinkage long parisc_personality(unsigned long personality) { long err; diff --git a/arch/parisc/kernel/syscalls/syscall.tbl b/arch/parisc/kernel/syscalls/syscall.tbl index 3c71fad78318..a0a9145b6dd4 100644 --- a/arch/parisc/kernel/syscalls/syscall.tbl +++ b/arch/parisc/kernel/syscalls/syscall.tbl @@ -400,6 +400,7 @@ 353 common pkey_free sys_pkey_free 354 common rseq sys_rseq 355 common kexec_file_load sys_kexec_file_load sys_kexec_file_load +356 common cacheflush sys_cacheflush # up to 402 is unassigned and reserved for arch specific syscalls 403 32 clock_gettime64 sys_clock_gettime sys_clock_gettime 404 32 clock_settime64 sys_clock_settime sys_clock_settime diff --git a/arch/parisc/kernel/traps.c b/arch/parisc/kernel/traps.c index 304eebd1c83e..3b97944c7291 100644 --- a/arch/parisc/kernel/traps.c +++ b/arch/parisc/kernel/traps.c @@ -338,7 +338,7 @@ static void default_trap(int code, struct pt_regs *regs) void (*cpu_lpmc) (int code, struct pt_regs *regs) __read_mostly = default_trap; -void transfer_pim_to_trap_frame(struct pt_regs *regs) +static void transfer_pim_to_trap_frame(struct pt_regs *regs) { register int i; extern unsigned int hpmc_pim_data[]; @@ -800,14 +800,13 @@ void notrace handle_interruption(int code, struct pt_regs *regs) } -void __init initialize_ivt(const void *iva) +static void __init initialize_ivt(const void *iva) { extern const u32 os_hpmc[]; int i; u32 check = 0; u32 *ivap; - u32 *hpmcp; u32 instr; if (strcmp((const char *)iva, "cows can fly")) @@ -840,8 +839,6 @@ void __init initialize_ivt(const void *iva) /* Setup IVA and compute checksum for HPMC handler */ ivap[6] = (u32)__pa(os_hpmc); - hpmcp = (u32 *)os_hpmc; - for (i=0; i<8; i++) check += ivap[i]; diff --git a/arch/parisc/kernel/unaligned.c b/arch/parisc/kernel/unaligned.c index e8a4d77cff53..033b9e50b44a 100644 --- a/arch/parisc/kernel/unaligned.c +++ b/arch/parisc/kernel/unaligned.c @@ -11,6 +11,7 @@ #include <linux/signal.h> #include <linux/ratelimit.h> #include <linux/uaccess.h> +#include <asm/unaligned.h> #include <asm/hardirq.h> #include <asm/traps.h> diff --git a/arch/parisc/kernel/unwind.c b/arch/parisc/kernel/unwind.c index 42acc3b52017..27ae40a443b8 100644 --- a/arch/parisc/kernel/unwind.c +++ b/arch/parisc/kernel/unwind.c @@ -24,12 +24,13 @@ #include <asm/unwind.h> #include <asm/switch_to.h> #include <asm/sections.h> +#include <asm/ftrace.h> /* #define DEBUG 1 */ #ifdef DEBUG #define dbg(x...) pr_debug(x) #else -#define dbg(x...) +#define dbg(x...) do { } while (0) #endif #define KERNEL_START (KERNEL_BINARY_TEXT_START) @@ -179,7 +180,7 @@ void unwind_table_remove(struct unwind_table *table) /* Called from setup_arch to import the kernel unwind info */ int __init unwind_init(void) { - long start, stop; + long start __maybe_unused, stop __maybe_unused; register unsigned long gp __asm__ ("r27"); start = (long)&__start___unwind[0]; @@ -220,7 +221,6 @@ static int unwind_special(struct unwind_frame_info *info, unsigned long pc, int * Note: We could use dereference_kernel_function_descriptor() * instead but we want to keep it simple here. */ - extern void * const handle_interruption; extern void * const ret_from_kernel_thread; extern void * const syscall_exit; extern void * const intr_return; @@ -228,8 +228,10 @@ static int unwind_special(struct unwind_frame_info *info, unsigned long pc, int #ifdef CONFIG_IRQSTACKS extern void * const _call_on_stack; #endif /* CONFIG_IRQSTACKS */ + void *ptr; - if (pc_is_kernel_fn(pc, handle_interruption)) { + ptr = dereference_kernel_function_descriptor(&handle_interruption); + if (pc_is_kernel_fn(pc, ptr)) { struct pt_regs *regs = (struct pt_regs *)(info->sp - frame_size - PT_SZ_ALGN); dbg("Unwinding through handle_interruption()\n"); info->prev_sp = regs->gr[30]; diff --git a/arch/parisc/math-emu/Makefile b/arch/parisc/math-emu/Makefile index 3747a0cbd3b8..7b64740e150a 100644 --- a/arch/parisc/math-emu/Makefile +++ b/arch/parisc/math-emu/Makefile @@ -6,7 +6,8 @@ # See arch/parisc/math-emu/README ccflags-y := -Wno-parentheses -Wno-implicit-function-declaration \ -Wno-uninitialized -Wno-strict-prototypes -Wno-return-type \ - -Wno-implicit-int + -Wno-implicit-int -Wno-missing-prototypes -Wno-missing-declarations \ + -Wno-old-style-definition -Wno-unused-but-set-variable obj-y := frnd.o driver.o decode_exc.o fpudispatch.o denormal.o \ dfmpy.o sfmpy.o sfsqrt.o dfsqrt.o dfadd.o fmpyfadd.o \ diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c index 6e894afa4249..a4c7c7630f48 100644 --- a/arch/parisc/mm/fault.c +++ b/arch/parisc/mm/fault.c @@ -289,7 +289,7 @@ retry: mmap_read_lock(mm); vma = find_vma_prev(mm, address, &prev_vma); if (!vma || address < vma->vm_start) { - if (!prev || !(prev->vm_flags & VM_GROWSUP)) + if (!prev_vma || !(prev_vma->vm_flags & VM_GROWSUP)) goto bad_area; vma = expand_stack(mm, address); if (!vma) diff --git a/arch/parisc/mm/init.c b/arch/parisc/mm/init.c index b0c43f3b0a5f..406c52fe23d5 100644 --- a/arch/parisc/mm/init.c +++ b/arch/parisc/mm/init.c @@ -626,12 +626,10 @@ static void __init pagetable_init(void) for (range = 0; range < npmem_ranges; range++) { unsigned long start_paddr; - unsigned long end_paddr; unsigned long size; start_paddr = pmem_ranges[range].start_pfn << PAGE_SHIFT; size = pmem_ranges[range].pages << PAGE_SHIFT; - end_paddr = start_paddr + size; map_pages((unsigned long)__va(start_paddr), start_paddr, size, PAGE_KERNEL, 0); diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 395044b705a1..0b1172cbeccb 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -210,7 +210,7 @@ config PPC select HAVE_ARCH_KASAN if PPC_RADIX_MMU select HAVE_ARCH_KASAN if PPC_BOOK3E_64 select HAVE_ARCH_KASAN_VMALLOC if HAVE_ARCH_KASAN - select HAVE_ARCH_KCSAN if PPC_BOOK3S_64 + select HAVE_ARCH_KCSAN select HAVE_ARCH_KFENCE if ARCH_SUPPORTS_DEBUG_PAGEALLOC select HAVE_ARCH_RANDOMIZE_KSTACK_OFFSET select HAVE_ARCH_WITHIN_STACK_FRAMES @@ -235,7 +235,7 @@ config PPC select HAVE_FUNCTION_DESCRIPTORS if PPC64_ELF_ABI_V1 select HAVE_FUNCTION_ERROR_INJECTION select HAVE_FUNCTION_GRAPH_TRACER - select HAVE_FUNCTION_TRACER + select HAVE_FUNCTION_TRACER if PPC64 || (PPC32 && CC_IS_GCC) select HAVE_GCC_PLUGINS if GCC_VERSION >= 50200 # plugin support on gcc <= 5.1 is buggy on PPC select HAVE_GENERIC_VDSO select HAVE_HARDLOCKUP_DETECTOR_ARCH if PPC_BOOK3S_64 && SMP @@ -547,8 +547,9 @@ config LD_HEAD_STUB_CATCH If unsure, say "N". config MPROFILE_KERNEL - depends on PPC64 && CPU_LITTLE_ENDIAN && FUNCTION_TRACER - def_bool $(success,$(srctree)/arch/powerpc/tools/gcc-check-mprofile-kernel.sh $(CC) -I$(srctree)/include -D__KERNEL__) + depends on PPC64_ELF_ABI_V2 && FUNCTION_TRACER + def_bool $(success,$(srctree)/arch/powerpc/tools/gcc-check-mprofile-kernel.sh $(CC) -mlittle-endian) if CPU_LITTLE_ENDIAN + def_bool $(success,$(srctree)/arch/powerpc/tools/gcc-check-mprofile-kernel.sh $(CC) -mbig-endian) if CPU_BIG_ENDIAN config HOTPLUG_CPU bool "Support for enabling/disabling CPUs" @@ -624,10 +625,12 @@ config ARCH_HAS_KEXEC_PURGATORY def_bool KEXEC_FILE config PPC64_BIG_ENDIAN_ELF_ABI_V2 - bool "Build big-endian kernel using ELF ABI V2 (EXPERIMENTAL)" + # Option is available to BFD, but LLD does not support ELFv1 so this is + # always true there. + prompt "Build big-endian kernel using ELF ABI V2" if LD_IS_BFD && EXPERT + def_bool y depends on PPC64 && CPU_BIG_ENDIAN depends on CC_HAS_ELFV2 - depends on LD_VERSION >= 22400 || LLD_VERSION >= 150000 help This builds the kernel image using the "Power Architecture 64-Bit ELF V2 ABI Specification", which has a reduced stack overhead and faster @@ -638,8 +641,6 @@ config PPC64_BIG_ENDIAN_ELF_ABI_V2 it is less well tested by kernel and toolchain. However some distros build userspace this way, and it can produce a functioning kernel. - This requires GCC and binutils 2.24 or newer. - config RELOCATABLE bool "Build a relocatable kernel" depends on PPC64 || (FLATMEM && (44x || PPC_85xx)) diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile index dca73f673d70..dac7ca153886 100644 --- a/arch/powerpc/Makefile +++ b/arch/powerpc/Makefile @@ -42,18 +42,13 @@ machine-$(CONFIG_PPC64) += 64 machine-$(CONFIG_CPU_LITTLE_ENDIAN) += le UTS_MACHINE := $(subst $(space),,$(machine-y)) -# XXX This needs to be before we override LD below -ifdef CONFIG_PPC32 -KBUILD_LDFLAGS_MODULE += arch/powerpc/lib/crtsavres.o -else -ifeq ($(call ld-ifversion, -ge, 22500, y),y) +ifeq ($(CONFIG_PPC64)$(CONFIG_LD_IS_BFD),yy) # Have the linker provide sfpr if possible. # There is a corresponding test in arch/powerpc/lib/Makefile KBUILD_LDFLAGS_MODULE += --save-restore-funcs else KBUILD_LDFLAGS_MODULE += arch/powerpc/lib/crtsavres.o endif -endif ifdef CONFIG_CPU_LITTLE_ENDIAN KBUILD_CFLAGS += -mlittle-endian @@ -166,7 +161,7 @@ asinstr := $(call as-instr,lis 9$(comma)foo@high,-DHAVE_AS_ATHIGH=1) KBUILD_CPPFLAGS += -I $(srctree)/arch/$(ARCH) $(asinstr) KBUILD_AFLAGS += $(AFLAGS-y) KBUILD_CFLAGS += $(call cc-option,-msoft-float) -KBUILD_CFLAGS += -pipe $(CFLAGS-y) +KBUILD_CFLAGS += $(CFLAGS-y) CPP = $(CC) -E $(KBUILD_CFLAGS) CHECKFLAGS += -m$(BITS) -D__powerpc__ -D__powerpc$(BITS)__ @@ -398,14 +393,12 @@ endif endif PHONY += checkbin -# Check toolchain versions: -# - gcc-4.6 is the minimum kernel-wide version so nothing required. checkbin: - @if test "x${CONFIG_LD_IS_LLD}" != "xy" -a \ - "x$(call ld-ifversion, -le, 22400, y)" = "xy" ; then \ - echo -n '*** binutils 2.24 miscompiles weak symbols ' ; \ - echo 'in some circumstances.' ; \ - echo '*** binutils 2.23 do not define the TOC symbol ' ; \ - echo -n '*** Please use a different binutils version.' ; \ + @if test "x${CONFIG_FTRACE_MCOUNT_USE_RECORDMCOUNT}" = "xy" -a \ + "x${CONFIG_LD_IS_BFD}" = "xy" -a \ + "${CONFIG_LD_VERSION}" = "23700" ; then \ + echo -n '*** binutils 2.37 drops unused section symbols, which recordmcount ' ; \ + echo 'is unable to handle.' ; \ + echo '*** Please use a different binutils version.' ; \ false ; \ fi diff --git a/arch/powerpc/boot/Makefile b/arch/powerpc/boot/Makefile index 771b79423bbc..968aee2025b8 100644 --- a/arch/powerpc/boot/Makefile +++ b/arch/powerpc/boot/Makefile @@ -32,46 +32,58 @@ else BOOTAR := $(AR) endif -BOOTCFLAGS := -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs \ - -fno-strict-aliasing -O2 -msoft-float -mno-altivec -mno-vsx \ - $(call cc-option,-mno-spe) $(call cc-option,-mspe=no) \ - -pipe -fomit-frame-pointer -fno-builtin -fPIC -nostdinc \ - $(LINUXINCLUDE) - ifdef CONFIG_PPC64_BOOT_WRAPPER -BOOTCFLAGS += -m64 +BOOTTARGETFLAGS += -m64 +BOOTTARGETFLAGS += -mabi=elfv2 ifdef CONFIG_PPC64_ELF_ABI_V2 -BOOTCFLAGS += $(call cc-option,-mabi=elfv2) +BOOTTARGETFLAGS += $(call cc-option,-mabi=elfv2) endif else -BOOTCFLAGS += -m32 +BOOTTARGETFLAGS := -m32 endif ifdef CONFIG_TARGET_CPU_BOOL -BOOTCFLAGS += -mcpu=$(CONFIG_TARGET_CPU) +BOOTTARGETFLAGS += -mcpu=$(CONFIG_TARGET_CPU) else ifdef CONFIG_PPC64_BOOT_WRAPPER ifdef CONFIG_CPU_LITTLE_ENDIAN -BOOTCFLAGS += -mcpu=powerpc64le +BOOTTARGETFLAGS += -mcpu=powerpc64le else -BOOTCFLAGS += -mcpu=powerpc64 +BOOTTARGETFLAGS += -mcpu=powerpc64 endif endif -BOOTCFLAGS += -isystem $(shell $(BOOTCC) -print-file-name=include) +$(obj)/4xx.o: BOOTTARGETFLAGS += -mcpu=405 +$(obj)/ebony.o: BOOTTARGETFLAGS += -mcpu=440 +$(obj)/cuboot-hotfoot.o: BOOTTARGETFLAGS += -mcpu=405 +$(obj)/cuboot-taishan.o: BOOTTARGETFLAGS += -mcpu=440 +$(obj)/cuboot-katmai.o: BOOTTARGETFLAGS += -mcpu=440 +$(obj)/cuboot-acadia.o: BOOTTARGETFLAGS += -mcpu=405 +$(obj)/treeboot-iss4xx.o: BOOTTARGETFLAGS += -mcpu=405 +$(obj)/treeboot-currituck.o: BOOTTARGETFLAGS += -mcpu=405 +$(obj)/treeboot-akebono.o: BOOTTARGETFLAGS += -mcpu=405 ifdef CONFIG_CPU_BIG_ENDIAN -BOOTCFLAGS += -mbig-endian +BOOTTARGETFLAGS += -mbig-endian else -BOOTCFLAGS += -mlittle-endian +BOOTTARGETFLAGS += -mlittle-endian endif -BOOTAFLAGS := -D__ASSEMBLY__ $(BOOTCFLAGS) -nostdinc - -BOOTARFLAGS := -crD +BOOTCPPFLAGS := -nostdinc $(LINUXINCLUDE) +BOOTCPPFLAGS += -isystem $(shell $(BOOTCC) -print-file-name=include) -BOOTCFLAGS += $(call cc-option,-mno-prefixed) \ +BOOTCFLAGS := $(BOOTTARGETFLAGS) \ + -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs \ + -fno-strict-aliasing -O2 \ + -msoft-float -mno-altivec -mno-vsx \ + $(call cc-option,-mno-prefixed) \ $(call cc-option,-mno-pcrel) \ - $(call cc-option,-mno-mma) + $(call cc-option,-mno-mma) \ + $(call cc-option,-mno-spe) $(call cc-option,-mspe=no) \ + -fomit-frame-pointer -fno-builtin -fPIC + +BOOTAFLAGS := $(BOOTTARGETFLAGS) -D__ASSEMBLY__ + +BOOTARFLAGS := -crD ifdef CONFIG_CC_IS_CLANG BOOTCFLAGS += $(CLANG_FLAGS) @@ -91,16 +103,6 @@ BOOTCFLAGS += -I$(objtree)/$(obj) -I$(srctree)/$(obj) DTC_FLAGS ?= -p 1024 -$(obj)/4xx.o: BOOTCFLAGS += -mcpu=405 -$(obj)/ebony.o: BOOTCFLAGS += -mcpu=440 -$(obj)/cuboot-hotfoot.o: BOOTCFLAGS += -mcpu=405 -$(obj)/cuboot-taishan.o: BOOTCFLAGS += -mcpu=440 -$(obj)/cuboot-katmai.o: BOOTCFLAGS += -mcpu=440 -$(obj)/cuboot-acadia.o: BOOTCFLAGS += -mcpu=405 -$(obj)/treeboot-iss4xx.o: BOOTCFLAGS += -mcpu=405 -$(obj)/treeboot-currituck.o: BOOTCFLAGS += -mcpu=405 -$(obj)/treeboot-akebono.o: BOOTCFLAGS += -mcpu=405 - # The pre-boot decompressors pull in a lot of kernel headers and other source # files. This creates a bit of a dependency headache since we need to copy # these files into the build dir, fix up any includes and ensure that dependent @@ -224,10 +226,10 @@ clean-files := $(zlib-) $(zlibheader-) $(zliblinuxheader-) \ empty.c zImage.coff.lds zImage.ps3.lds zImage.lds quiet_cmd_bootcc = BOOTCC $@ - cmd_bootcc = $(BOOTCC) -Wp,-MD,$(depfile) $(BOOTCFLAGS) -c -o $@ $< + cmd_bootcc = $(BOOTCC) -Wp,-MD,$(depfile) $(BOOTCPPFLAGS) $(BOOTCFLAGS) -c -o $@ $< quiet_cmd_bootas = BOOTAS $@ - cmd_bootas = $(BOOTCC) -Wp,-MD,$(depfile) $(BOOTAFLAGS) -c -o $@ $< + cmd_bootas = $(BOOTCC) -Wp,-MD,$(depfile) $(BOOTCPPFLAGS) $(BOOTAFLAGS) -c -o $@ $< quiet_cmd_bootar = BOOTAR $@ cmd_bootar = $(BOOTAR) $(BOOTARFLAGS) $@.$$$$ $(real-prereqs); mv $@.$$$$ $@ @@ -340,11 +342,6 @@ image-$(CONFIG_MPC834x_ITX) += cuImage.mpc8349emitx \ image-$(CONFIG_ASP834x) += dtbImage.asp834x-redboot # Board ports in arch/powerpc/platform/85xx/Kconfig -image-$(CONFIG_MPC8540_ADS) += cuImage.mpc8540ads -image-$(CONFIG_MPC8560_ADS) += cuImage.mpc8560ads -image-$(CONFIG_MPC85xx_CDS) += cuImage.mpc8541cds \ - cuImage.mpc8548cds_32b \ - cuImage.mpc8555cds image-$(CONFIG_MPC85xx_MDS) += cuImage.mpc8568mds image-$(CONFIG_MPC85xx_DS) += cuImage.mpc8544ds \ cuImage.mpc8572ds diff --git a/arch/powerpc/boot/dts/fsl/mpc8540ads.dts b/arch/powerpc/boot/dts/fsl/mpc8540ads.dts deleted file mode 100644 index e03ae130162b..000000000000 --- a/arch/powerpc/boot/dts/fsl/mpc8540ads.dts +++ /dev/null @@ -1,355 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * MPC8540 ADS Device Tree Source - * - * Copyright 2006, 2008 Freescale Semiconductor Inc. - */ - -/dts-v1/; - -/include/ "e500v1_power_isa.dtsi" - -/ { - model = "MPC8540ADS"; - compatible = "MPC8540ADS", "MPC85xxADS"; - #address-cells = <1>; - #size-cells = <1>; - - aliases { - ethernet0 = &enet0; - ethernet1 = &enet1; - ethernet2 = &enet2; - serial0 = &serial0; - serial1 = &serial1; - pci0 = &pci0; - }; - - cpus { - #address-cells = <1>; - #size-cells = <0>; - - PowerPC,8540@0 { - device_type = "cpu"; - reg = <0x0>; - d-cache-line-size = <32>; // 32 bytes - i-cache-line-size = <32>; // 32 bytes - d-cache-size = <0x8000>; // L1, 32K - i-cache-size = <0x8000>; // L1, 32K - timebase-frequency = <0>; // 33 MHz, from uboot - bus-frequency = <0>; // 166 MHz - clock-frequency = <0>; // 825 MHz, from uboot - next-level-cache = <&L2>; - }; - }; - - memory { - device_type = "memory"; - reg = <0x0 0x8000000>; // 128M at 0x0 - }; - - soc8540@e0000000 { - #address-cells = <1>; - #size-cells = <1>; - device_type = "soc"; - compatible = "simple-bus"; - ranges = <0x0 0xe0000000 0x100000>; - bus-frequency = <0>; - - ecm-law@0 { - compatible = "fsl,ecm-law"; - reg = <0x0 0x1000>; - fsl,num-laws = <8>; - }; - - ecm@1000 { - compatible = "fsl,mpc8540-ecm", "fsl,ecm"; - reg = <0x1000 0x1000>; - interrupts = <17 2>; - interrupt-parent = <&mpic>; - }; - - memory-controller@2000 { - compatible = "fsl,mpc8540-memory-controller"; - reg = <0x2000 0x1000>; - interrupt-parent = <&mpic>; - interrupts = <18 2>; - }; - - L2: l2-cache-controller@20000 { - compatible = "fsl,mpc8540-l2-cache-controller"; - reg = <0x20000 0x1000>; - cache-line-size = <32>; // 32 bytes - cache-size = <0x40000>; // L2, 256K - interrupt-parent = <&mpic>; - interrupts = <16 2>; - }; - - i2c@3000 { - #address-cells = <1>; - #size-cells = <0>; - cell-index = <0>; - compatible = "fsl-i2c"; - reg = <0x3000 0x100>; - interrupts = <43 2>; - interrupt-parent = <&mpic>; - dfsrr; - }; - - dma@21300 { - #address-cells = <1>; - #size-cells = <1>; - compatible = "fsl,mpc8540-dma", "fsl,eloplus-dma"; - reg = <0x21300 0x4>; - ranges = <0x0 0x21100 0x200>; - cell-index = <0>; - dma-channel@0 { - compatible = "fsl,mpc8540-dma-channel", - "fsl,eloplus-dma-channel"; - reg = <0x0 0x80>; - cell-index = <0>; - interrupt-parent = <&mpic>; - interrupts = <20 2>; - }; - dma-channel@80 { - compatible = "fsl,mpc8540-dma-channel", - "fsl,eloplus-dma-channel"; - reg = <0x80 0x80>; - cell-index = <1>; - interrupt-parent = <&mpic>; - interrupts = <21 2>; - }; - dma-channel@100 { - compatible = "fsl,mpc8540-dma-channel", - "fsl,eloplus-dma-channel"; - reg = <0x100 0x80>; - cell-index = <2>; - interrupt-parent = <&mpic>; - interrupts = <22 2>; - }; - dma-channel@180 { - compatible = "fsl,mpc8540-dma-channel", - "fsl,eloplus-dma-channel"; - reg = <0x180 0x80>; - cell-index = <3>; - interrupt-parent = <&mpic>; - interrupts = <23 2>; - }; - }; - - enet0: ethernet@24000 { - #address-cells = <1>; - #size-cells = <1>; - cell-index = <0>; - device_type = "network"; - model = "TSEC"; - compatible = "gianfar"; - reg = <0x24000 0x1000>; - ranges = <0x0 0x24000 0x1000>; - local-mac-address = [ 00 00 00 00 00 00 ]; - interrupts = <29 2 30 2 34 2>; - interrupt-parent = <&mpic>; - tbi-handle = <&tbi0>; - phy-handle = <&phy0>; - - mdio@520 { - #address-cells = <1>; - #size-cells = <0>; - compatible = "fsl,gianfar-mdio"; - reg = <0x520 0x20>; - - phy0: ethernet-phy@0 { - interrupt-parent = <&mpic>; - interrupts = <5 1>; - reg = <0x0>; - }; - phy1: ethernet-phy@1 { - interrupt-parent = <&mpic>; - interrupts = <5 1>; - reg = <0x1>; - }; - phy3: ethernet-phy@3 { - interrupt-parent = <&mpic>; - interrupts = <7 1>; - reg = <0x3>; - }; - tbi0: tbi-phy@11 { - reg = <0x11>; - device_type = "tbi-phy"; - }; - }; - }; - - enet1: ethernet@25000 { - #address-cells = <1>; - #size-cells = <1>; - cell-index = <1>; - device_type = "network"; - model = "TSEC"; - compatible = "gianfar"; - reg = <0x25000 0x1000>; - ranges = <0x0 0x25000 0x1000>; - local-mac-address = [ 00 00 00 00 00 00 ]; - interrupts = <35 2 36 2 40 2>; - interrupt-parent = <&mpic>; - tbi-handle = <&tbi1>; - phy-handle = <&phy1>; - - mdio@520 { - #address-cells = <1>; - #size-cells = <0>; - compatible = "fsl,gianfar-tbi"; - reg = <0x520 0x20>; - - tbi1: tbi-phy@11 { - reg = <0x11>; - device_type = "tbi-phy"; - }; - }; - }; - - enet2: ethernet@26000 { - #address-cells = <1>; - #size-cells = <1>; - cell-index = <2>; - device_type = "network"; - model = "FEC"; - compatible = "gianfar"; - reg = <0x26000 0x1000>; - ranges = <0x0 0x26000 0x1000>; - local-mac-address = [ 00 00 00 00 00 00 ]; - interrupts = <41 2>; - interrupt-parent = <&mpic>; - tbi-handle = <&tbi2>; - phy-handle = <&phy3>; - - mdio@520 { - #address-cells = <1>; - #size-cells = <0>; - compatible = "fsl,gianfar-tbi"; - reg = <0x520 0x20>; - - tbi2: tbi-phy@11 { - reg = <0x11>; - device_type = "tbi-phy"; - }; - }; - }; - - serial0: serial@4500 { - cell-index = <0>; - device_type = "serial"; - compatible = "fsl,ns16550", "ns16550"; - reg = <0x4500 0x100>; // reg base, size - clock-frequency = <0>; // should we fill in in uboot? - interrupts = <42 2>; - interrupt-parent = <&mpic>; - }; - - serial1: serial@4600 { - cell-index = <1>; - device_type = "serial"; - compatible = "fsl,ns16550", "ns16550"; - reg = <0x4600 0x100>; // reg base, size - clock-frequency = <0>; // should we fill in in uboot? - interrupts = <42 2>; - interrupt-parent = <&mpic>; - }; - mpic: pic@40000 { - interrupt-controller; - #address-cells = <0>; - #interrupt-cells = <2>; - reg = <0x40000 0x40000>; - compatible = "chrp,open-pic"; - device_type = "open-pic"; - }; - }; - - pci0: pci@e0008000 { - interrupt-map-mask = <0xf800 0x0 0x0 0x7>; - interrupt-map = < - - /* IDSEL 0x02 */ - 0x1000 0x0 0x0 0x1 &mpic 0x1 0x1 - 0x1000 0x0 0x0 0x2 &mpic 0x2 0x1 - 0x1000 0x0 0x0 0x3 &mpic 0x3 0x1 - 0x1000 0x0 0x0 0x4 &mpic 0x4 0x1 - - /* IDSEL 0x03 */ - 0x1800 0x0 0x0 0x1 &mpic 0x4 0x1 - 0x1800 0x0 0x0 0x2 &mpic 0x1 0x1 - 0x1800 0x0 0x0 0x3 &mpic 0x2 0x1 - 0x1800 0x0 0x0 0x4 &mpic 0x3 0x1 - - /* IDSEL 0x04 */ - 0x2000 0x0 0x0 0x1 &mpic 0x3 0x1 - 0x2000 0x0 0x0 0x2 &mpic 0x4 0x1 - 0x2000 0x0 0x0 0x3 &mpic 0x1 0x1 - 0x2000 0x0 0x0 0x4 &mpic 0x2 0x1 - - /* IDSEL 0x05 */ - 0x2800 0x0 0x0 0x1 &mpic 0x2 0x1 - 0x2800 0x0 0x0 0x2 &mpic 0x3 0x1 - 0x2800 0x0 0x0 0x3 &mpic 0x4 0x1 - 0x2800 0x0 0x0 0x4 &mpic 0x1 0x1 - - /* IDSEL 0x0c */ - 0x6000 0x0 0x0 0x1 &mpic 0x1 0x1 - 0x6000 0x0 0x0 0x2 &mpic 0x2 0x1 - 0x6000 0x0 0x0 0x3 &mpic 0x3 0x1 - 0x6000 0x0 0x0 0x4 &mpic 0x4 0x1 - - /* IDSEL 0x0d */ - 0x6800 0x0 0x0 0x1 &mpic 0x4 0x1 - 0x6800 0x0 0x0 0x2 &mpic 0x1 0x1 - 0x6800 0x0 0x0 0x3 &mpic 0x2 0x1 - 0x6800 0x0 0x0 0x4 &mpic 0x3 0x1 - - /* IDSEL 0x0e */ - 0x7000 0x0 0x0 0x1 &mpic 0x3 0x1 - 0x7000 0x0 0x0 0x2 &mpic 0x4 0x1 - 0x7000 0x0 0x0 0x3 &mpic 0x1 0x1 - 0x7000 0x0 0x0 0x4 &mpic 0x2 0x1 - - /* IDSEL 0x0f */ - 0x7800 0x0 0x0 0x1 &mpic 0x2 0x1 - 0x7800 0x0 0x0 0x2 &mpic 0x3 0x1 - 0x7800 0x0 0x0 0x3 &mpic 0x4 0x1 - 0x7800 0x0 0x0 0x4 &mpic 0x1 0x1 - - /* IDSEL 0x12 */ - 0x9000 0x0 0x0 0x1 &mpic 0x1 0x1 - 0x9000 0x0 0x0 0x2 &mpic 0x2 0x1 - 0x9000 0x0 0x0 0x3 &mpic 0x3 0x1 - 0x9000 0x0 0x0 0x4 &mpic 0x4 0x1 - - /* IDSEL 0x13 */ - 0x9800 0x0 0x0 0x1 &mpic 0x4 0x1 - 0x9800 0x0 0x0 0x2 &mpic 0x1 0x1 - 0x9800 0x0 0x0 0x3 &mpic 0x2 0x1 - 0x9800 0x0 0x0 0x4 &mpic 0x3 0x1 - - /* IDSEL 0x14 */ - 0xa000 0x0 0x0 0x1 &mpic 0x3 0x1 - 0xa000 0x0 0x0 0x2 &mpic 0x4 0x1 - 0xa000 0x0 0x0 0x3 &mpic 0x1 0x1 - 0xa000 0x0 0x0 0x4 &mpic 0x2 0x1 - - /* IDSEL 0x15 */ - 0xa800 0x0 0x0 0x1 &mpic 0x2 0x1 - 0xa800 0x0 0x0 0x2 &mpic 0x3 0x1 - 0xa800 0x0 0x0 0x3 &mpic 0x4 0x1 - 0xa800 0x0 0x0 0x4 &mpic 0x1 0x1>; - interrupt-parent = <&mpic>; - interrupts = <24 2>; - bus-range = <0 0>; - ranges = <0x2000000 0x0 0x80000000 0x80000000 0x0 0x20000000 - 0x1000000 0x0 0x0 0xe2000000 0x0 0x100000>; - clock-frequency = <66666666>; - #interrupt-cells = <1>; - #size-cells = <2>; - #address-cells = <3>; - reg = <0xe0008000 0x1000>; - compatible = "fsl,mpc8540-pcix", "fsl,mpc8540-pci"; - device_type = "pci"; - }; -}; diff --git a/arch/powerpc/boot/dts/fsl/mpc8541cds.dts b/arch/powerpc/boot/dts/fsl/mpc8541cds.dts deleted file mode 100644 index a2a6c5cf852e..000000000000 --- a/arch/powerpc/boot/dts/fsl/mpc8541cds.dts +++ /dev/null @@ -1,375 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * MPC8541 CDS Device Tree Source - * - * Copyright 2006, 2008 Freescale Semiconductor Inc. - */ - -/dts-v1/; - -/include/ "e500v1_power_isa.dtsi" - -/ { - model = "MPC8541CDS"; - compatible = "MPC8541CDS", "MPC85xxCDS"; - #address-cells = <1>; - #size-cells = <1>; - - aliases { - ethernet0 = &enet0; - ethernet1 = &enet1; - serial0 = &serial0; - serial1 = &serial1; - pci0 = &pci0; - pci1 = &pci1; - }; - - cpus { - #address-cells = <1>; - #size-cells = <0>; - - PowerPC,8541@0 { - device_type = "cpu"; - reg = <0x0>; - d-cache-line-size = <32>; // 32 bytes - i-cache-line-size = <32>; // 32 bytes - d-cache-size = <0x8000>; // L1, 32K - i-cache-size = <0x8000>; // L1, 32K - timebase-frequency = <0>; // 33 MHz, from uboot - bus-frequency = <0>; // 166 MHz - clock-frequency = <0>; // 825 MHz, from uboot - next-level-cache = <&L2>; - }; - }; - - memory { - device_type = "memory"; - reg = <0x0 0x8000000>; // 128M at 0x0 - }; - - soc8541@e0000000 { - #address-cells = <1>; - #size-cells = <1>; - device_type = "soc"; - compatible = "simple-bus"; - ranges = <0x0 0xe0000000 0x100000>; - bus-frequency = <0>; - - ecm-law@0 { - compatible = "fsl,ecm-law"; - reg = <0x0 0x1000>; - fsl,num-laws = <8>; - }; - - ecm@1000 { - compatible = "fsl,mpc8541-ecm", "fsl,ecm"; - reg = <0x1000 0x1000>; - interrupts = <17 2>; - interrupt-parent = <&mpic>; - }; - - memory-controller@2000 { - compatible = "fsl,mpc8541-memory-controller"; - reg = <0x2000 0x1000>; - interrupt-parent = <&mpic>; - interrupts = <18 2>; - }; - - L2: l2-cache-controller@20000 { - compatible = "fsl,mpc8541-l2-cache-controller"; - reg = <0x20000 0x1000>; - cache-line-size = <32>; // 32 bytes - cache-size = <0x40000>; // L2, 256K - interrupt-parent = <&mpic>; - interrupts = <16 2>; - }; - - i2c@3000 { - #address-cells = <1>; - #size-cells = <0>; - cell-index = <0>; - compatible = "fsl-i2c"; - reg = <0x3000 0x100>; - interrupts = <43 2>; - interrupt-parent = <&mpic>; - dfsrr; - }; - - dma@21300 { - #address-cells = <1>; - #size-cells = <1>; - compatible = "fsl,mpc8541-dma", "fsl,eloplus-dma"; - reg = <0x21300 0x4>; - ranges = <0x0 0x21100 0x200>; - cell-index = <0>; - dma-channel@0 { - compatible = "fsl,mpc8541-dma-channel", - "fsl,eloplus-dma-channel"; - reg = <0x0 0x80>; - cell-index = <0>; - interrupt-parent = <&mpic>; - interrupts = <20 2>; - }; - dma-channel@80 { - compatible = "fsl,mpc8541-dma-channel", - "fsl,eloplus-dma-channel"; - reg = <0x80 0x80>; - cell-index = <1>; - interrupt-parent = <&mpic>; - interrupts = <21 2>; - }; - dma-channel@100 { - compatible = "fsl,mpc8541-dma-channel", - "fsl,eloplus-dma-channel"; - reg = <0x100 0x80>; - cell-index = <2>; - interrupt-parent = <&mpic>; - interrupts = <22 2>; - }; - dma-channel@180 { - compatible = "fsl,mpc8541-dma-channel", - "fsl,eloplus-dma-channel"; - reg = <0x180 0x80>; - cell-index = <3>; - interrupt-parent = <&mpic>; - interrupts = <23 2>; - }; - }; - - enet0: ethernet@24000 { - #address-cells = <1>; - #size-cells = <1>; - cell-index = <0>; - device_type = "network"; - model = "TSEC"; - compatible = "gianfar"; - reg = <0x24000 0x1000>; - ranges = <0x0 0x24000 0x1000>; - local-mac-address = [ 00 00 00 00 00 00 ]; - interrupts = <29 2 30 2 34 2>; - interrupt-parent = <&mpic>; - tbi-handle = <&tbi0>; - phy-handle = <&phy0>; - - mdio@520 { - #address-cells = <1>; - #size-cells = <0>; - compatible = "fsl,gianfar-mdio"; - reg = <0x520 0x20>; - - phy0: ethernet-phy@0 { - interrupt-parent = <&mpic>; - interrupts = <5 1>; - reg = <0x0>; - }; - phy1: ethernet-phy@1 { - interrupt-parent = <&mpic>; - interrupts = <5 1>; - reg = <0x1>; - }; - tbi0: tbi-phy@11 { - reg = <0x11>; - device_type = "tbi-phy"; - }; - }; - }; - - enet1: ethernet@25000 { - #address-cells = <1>; - #size-cells = <1>; - cell-index = <1>; - device_type = "network"; - model = "TSEC"; - compatible = "gianfar"; - reg = <0x25000 0x1000>; - ranges = <0x0 0x25000 0x1000>; - local-mac-address = [ 00 00 00 00 00 00 ]; - interrupts = <35 2 36 2 40 2>; - interrupt-parent = <&mpic>; - tbi-handle = <&tbi1>; - phy-handle = <&phy1>; - - mdio@520 { - #address-cells = <1>; - #size-cells = <0>; - compatible = "fsl,gianfar-tbi"; - reg = <0x520 0x20>; - - tbi1: tbi-phy@11 { - reg = <0x11>; - device_type = "tbi-phy"; - }; - }; - }; - - serial0: serial@4500 { - cell-index = <0>; - device_type = "serial"; - compatible = "fsl,ns16550", "ns16550"; - reg = <0x4500 0x100>; // reg base, size - clock-frequency = <0>; // should we fill in in uboot? - interrupts = <42 2>; - interrupt-parent = <&mpic>; - }; - - serial1: serial@4600 { - cell-index = <1>; - device_type = "serial"; - compatible = "fsl,ns16550", "ns16550"; - reg = <0x4600 0x100>; // reg base, size - clock-frequency = <0>; // should we fill in in uboot? - interrupts = <42 2>; - interrupt-parent = <&mpic>; - }; - - crypto@30000 { - compatible = "fsl,sec2.0"; - reg = <0x30000 0x10000>; - interrupts = <45 2>; - interrupt-parent = <&mpic>; - fsl,num-channels = <4>; - fsl,channel-fifo-len = <24>; - fsl,exec-units-mask = <0x7e>; - fsl,descriptor-types-mask = <0x01010ebf>; - }; - - mpic: pic@40000 { - interrupt-controller; - #address-cells = <0>; - #interrupt-cells = <2>; - reg = <0x40000 0x40000>; - compatible = "chrp,open-pic"; - device_type = "open-pic"; - }; - - cpm@919c0 { - #address-cells = <1>; - #size-cells = <1>; - compatible = "fsl,mpc8541-cpm", "fsl,cpm2"; - reg = <0x919c0 0x30>; - ranges; - - muram@80000 { - #address-cells = <1>; - #size-cells = <1>; - ranges = <0x0 0x80000 0x10000>; - - data@0 { - compatible = "fsl,cpm-muram-data"; - reg = <0x0 0x2000 0x9000 0x1000>; - }; - }; - - brg@919f0 { - compatible = "fsl,mpc8541-brg", - "fsl,cpm2-brg", - "fsl,cpm-brg"; - reg = <0x919f0 0x10 0x915f0 0x10>; - }; - - cpmpic: pic@90c00 { - interrupt-controller; - #address-cells = <0>; - #interrupt-cells = <2>; - interrupts = <46 2>; - interrupt-parent = <&mpic>; - reg = <0x90c00 0x80>; - compatible = "fsl,mpc8541-cpm-pic", "fsl,cpm2-pic"; - }; - }; - }; - - pci0: pci@e0008000 { - interrupt-map-mask = <0x1f800 0x0 0x0 0x7>; - interrupt-map = < - - /* IDSEL 0x10 */ - 0x8000 0x0 0x0 0x1 &mpic 0x0 0x1 - 0x8000 0x0 0x0 0x2 &mpic 0x1 0x1 - 0x8000 0x0 0x0 0x3 &mpic 0x2 0x1 - 0x8000 0x0 0x0 0x4 &mpic 0x3 0x1 - - /* IDSEL 0x11 */ - 0x8800 0x0 0x0 0x1 &mpic 0x0 0x1 - 0x8800 0x0 0x0 0x2 &mpic 0x1 0x1 - 0x8800 0x0 0x0 0x3 &mpic 0x2 0x1 - 0x8800 0x0 0x0 0x4 &mpic 0x3 0x1 - - /* IDSEL 0x12 (Slot 1) */ - 0x9000 0x0 0x0 0x1 &mpic 0x0 0x1 - 0x9000 0x0 0x0 0x2 &mpic 0x1 0x1 - 0x9000 0x0 0x0 0x3 &mpic 0x2 0x1 - 0x9000 0x0 0x0 0x4 &mpic 0x3 0x1 - - /* IDSEL 0x13 (Slot 2) */ - 0x9800 0x0 0x0 0x1 &mpic 0x1 0x1 - 0x9800 0x0 0x0 0x2 &mpic 0x2 0x1 - 0x9800 0x0 0x0 0x3 &mpic 0x3 0x1 - 0x9800 0x0 0x0 0x4 &mpic 0x0 0x1 - - /* IDSEL 0x14 (Slot 3) */ - 0xa000 0x0 0x0 0x1 &mpic 0x2 0x1 - 0xa000 0x0 0x0 0x2 &mpic 0x3 0x1 - 0xa000 0x0 0x0 0x3 &mpic 0x0 0x1 - 0xa000 0x0 0x0 0x4 &mpic 0x1 0x1 - - /* IDSEL 0x15 (Slot 4) */ - 0xa800 0x0 0x0 0x1 &mpic 0x3 0x1 - 0xa800 0x0 0x0 0x2 &mpic 0x0 0x1 - 0xa800 0x0 0x0 0x3 &mpic 0x1 0x1 - 0xa800 0x0 0x0 0x4 &mpic 0x2 0x1 - - /* Bus 1 (Tundra Bridge) */ - /* IDSEL 0x12 (ISA bridge) */ - 0x19000 0x0 0x0 0x1 &mpic 0x0 0x1 - 0x19000 0x0 0x0 0x2 &mpic 0x1 0x1 - 0x19000 0x0 0x0 0x3 &mpic 0x2 0x1 - 0x19000 0x0 0x0 0x4 &mpic 0x3 0x1>; - interrupt-parent = <&mpic>; - interrupts = <24 2>; - bus-range = <0 0>; - ranges = <0x2000000 0x0 0x80000000 0x80000000 0x0 0x20000000 - 0x1000000 0x0 0x0 0xe2000000 0x0 0x100000>; - clock-frequency = <66666666>; - #interrupt-cells = <1>; - #size-cells = <2>; - #address-cells = <3>; - reg = <0xe0008000 0x1000>; - compatible = "fsl,mpc8540-pci"; - device_type = "pci"; - - i8259@19000 { - interrupt-controller; - device_type = "interrupt-controller"; - reg = <0x19000 0x0 0x0 0x0 0x1>; - #address-cells = <0>; - #interrupt-cells = <2>; - compatible = "chrp,iic"; - interrupts = <1>; - interrupt-parent = <&pci0>; - }; - }; - - pci1: pci@e0009000 { - interrupt-map-mask = <0xf800 0x0 0x0 0x7>; - interrupt-map = < - - /* IDSEL 0x15 */ - 0xa800 0x0 0x0 0x1 &mpic 0xb 0x1 - 0xa800 0x0 0x0 0x2 &mpic 0xb 0x1 - 0xa800 0x0 0x0 0x3 &mpic 0xb 0x1 - 0xa800 0x0 0x0 0x4 &mpic 0xb 0x1>; - interrupt-parent = <&mpic>; - interrupts = <25 2>; - bus-range = <0 0>; - ranges = <0x2000000 0x0 0xa0000000 0xa0000000 0x0 0x20000000 - 0x1000000 0x0 0x0 0xe3000000 0x0 0x100000>; - clock-frequency = <66666666>; - #interrupt-cells = <1>; - #size-cells = <2>; - #address-cells = <3>; - reg = <0xe0009000 0x1000>; - compatible = "fsl,mpc8540-pci"; - device_type = "pci"; - }; -}; diff --git a/arch/powerpc/boot/dts/fsl/mpc8548cds.dtsi b/arch/powerpc/boot/dts/fsl/mpc8548cds.dtsi deleted file mode 100644 index 3bc7d4711220..000000000000 --- a/arch/powerpc/boot/dts/fsl/mpc8548cds.dtsi +++ /dev/null @@ -1,302 +0,0 @@ -/* - * MPC8548CDS Device Tree Source stub (no addresses or top-level ranges) - * - * Copyright 2012 Freescale Semiconductor Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -&board_lbc { - nor@0,0 { - #address-cells = <1>; - #size-cells = <1>; - compatible = "cfi-flash"; - reg = <0x0 0x0 0x01000000>; - bank-width = <2>; - device-width = <2>; - - partition@0 { - reg = <0x0 0x0b00000>; - label = "ramdisk-nor"; - }; - - partition@300000 { - reg = <0x0b00000 0x0400000>; - label = "kernel-nor"; - }; - - partition@700000 { - reg = <0x0f00000 0x060000>; - label = "dtb-nor"; - }; - - partition@760000 { - reg = <0x0f60000 0x020000>; - label = "env-nor"; - read-only; - }; - - partition@780000 { - reg = <0x0f80000 0x080000>; - label = "u-boot-nor"; - read-only; - }; - }; - - board-control@1,0 { - compatible = "fsl,mpc8548cds-fpga"; - reg = <0x1 0x0 0x1000>; - }; -}; - -&board_soc { - i2c@3000 { - eeprom@50 { - compatible = "atmel,24c64"; - reg = <0x50>; - }; - - eeprom@56 { - compatible = "atmel,24c64"; - reg = <0x56>; - }; - - eeprom@57 { - compatible = "atmel,24c64"; - reg = <0x57>; - }; - }; - - i2c@3100 { - eeprom@50 { - compatible = "atmel,24c64"; - reg = <0x50>; - }; - }; - - enet0: ethernet@24000 { - tbi-handle = <&tbi0>; - phy-handle = <&phy0>; - }; - - mdio@24520 { - phy0: ethernet-phy@0 { - interrupts = <5 1 0 0>; - reg = <0x0>; - }; - phy1: ethernet-phy@1 { - interrupts = <5 1 0 0>; - reg = <0x1>; - }; - phy2: ethernet-phy@2 { - interrupts = <5 1 0 0>; - reg = <0x2>; - }; - phy3: ethernet-phy@3 { - interrupts = <5 1 0 0>; - reg = <0x3>; - }; - tbi0: tbi-phy@11 { - reg = <0x11>; - device_type = "tbi-phy"; - }; - }; - - enet1: ethernet@25000 { - tbi-handle = <&tbi1>; - phy-handle = <&phy1>; - }; - - mdio@25520 { - tbi1: tbi-phy@11 { - reg = <0x11>; - device_type = "tbi-phy"; - }; - }; - - enet2: ethernet@26000 { - tbi-handle = <&tbi2>; - phy-handle = <&phy2>; - }; - - mdio@26520 { - tbi2: tbi-phy@11 { - reg = <0x11>; - device_type = "tbi-phy"; - }; - }; - - enet3: ethernet@27000 { - tbi-handle = <&tbi3>; - phy-handle = <&phy3>; - }; - - mdio@27520 { - tbi3: tbi-phy@11 { - reg = <0x11>; - device_type = "tbi-phy"; - }; - }; -}; - -&board_pci0 { - interrupt-map-mask = <0xf800 0x0 0x0 0x7>; - interrupt-map = < - /* IDSEL 0x4 (PCIX Slot 2) */ - 0x2000 0x0 0x0 0x1 &mpic 0x0 0x1 0 0 - 0x2000 0x0 0x0 0x2 &mpic 0x1 0x1 0 0 - 0x2000 0x0 0x0 0x3 &mpic 0x2 0x1 0 0 - 0x2000 0x0 0x0 0x4 &mpic 0x3 0x1 0 0 - - /* IDSEL 0x5 (PCIX Slot 3) */ - 0x2800 0x0 0x0 0x1 &mpic 0x1 0x1 0 0 - 0x2800 0x0 0x0 0x2 &mpic 0x2 0x1 0 0 - 0x2800 0x0 0x0 0x3 &mpic 0x3 0x1 0 0 - 0x2800 0x0 0x0 0x4 &mpic 0x0 0x1 0 0 - - /* IDSEL 0x6 (PCIX Slot 4) */ - 0x3000 0x0 0x0 0x1 &mpic 0x2 0x1 0 0 - 0x3000 0x0 0x0 0x2 &mpic 0x3 0x1 0 0 - 0x3000 0x0 0x0 0x3 &mpic 0x0 0x1 0 0 - 0x3000 0x0 0x0 0x4 &mpic 0x1 0x1 0 0 - - /* IDSEL 0x8 (PCIX Slot 5) */ - 0x4000 0x0 0x0 0x1 &mpic 0x0 0x1 0 0 - 0x4000 0x0 0x0 0x2 &mpic 0x1 0x1 0 0 - 0x4000 0x0 0x0 0x3 &mpic 0x2 0x1 0 0 - 0x4000 0x0 0x0 0x4 &mpic 0x3 0x1 0 0 - - /* IDSEL 0xC (Tsi310 bridge) */ - 0x6000 0x0 0x0 0x1 &mpic 0x0 0x1 0 0 - 0x6000 0x0 0x0 0x2 &mpic 0x1 0x1 0 0 - 0x6000 0x0 0x0 0x3 &mpic 0x2 0x1 0 0 - 0x6000 0x0 0x0 0x4 &mpic 0x3 0x1 0 0 - - /* IDSEL 0x14 (Slot 2) */ - 0xa000 0x0 0x0 0x1 &mpic 0x0 0x1 0 0 - 0xa000 0x0 0x0 0x2 &mpic 0x1 0x1 0 0 - 0xa000 0x0 0x0 0x3 &mpic 0x2 0x1 0 0 - 0xa000 0x0 0x0 0x4 &mpic 0x3 0x1 0 0 - - /* IDSEL 0x15 (Slot 3) */ - 0xa800 0x0 0x0 0x1 &mpic 0x1 0x1 0 0 - 0xa800 0x0 0x0 0x2 &mpic 0x2 0x1 0 0 - 0xa800 0x0 0x0 0x3 &mpic 0x3 0x1 0 0 - 0xa800 0x0 0x0 0x4 &mpic 0x0 0x1 0 0 - - /* IDSEL 0x16 (Slot 4) */ - 0xb000 0x0 0x0 0x1 &mpic 0x2 0x1 0 0 - 0xb000 0x0 0x0 0x2 &mpic 0x3 0x1 0 0 - 0xb000 0x0 0x0 0x3 &mpic 0x0 0x1 0 0 - 0xb000 0x0 0x0 0x4 &mpic 0x1 0x1 0 0 - - /* IDSEL 0x18 (Slot 5) */ - 0xc000 0x0 0x0 0x1 &mpic 0x0 0x1 0 0 - 0xc000 0x0 0x0 0x2 &mpic 0x1 0x1 0 0 - 0xc000 0x0 0x0 0x3 &mpic 0x2 0x1 0 0 - 0xc000 0x0 0x0 0x4 &mpic 0x3 0x1 0 0 - - /* IDSEL 0x1C (Tsi310 bridge PCI primary) */ - 0xe000 0x0 0x0 0x1 &mpic 0x0 0x1 0 0 - 0xe000 0x0 0x0 0x2 &mpic 0x1 0x1 0 0 - 0xe000 0x0 0x0 0x3 &mpic 0x2 0x1 0 0 - 0xe000 0x0 0x0 0x4 &mpic 0x3 0x1 0 0>; - - pci_bridge@1c { - interrupt-map-mask = <0xf800 0x0 0x0 0x7>; - interrupt-map = < - - /* IDSEL 0x00 (PrPMC Site) */ - 0000 0x0 0x0 0x1 &mpic 0x0 0x1 0 0 - 0000 0x0 0x0 0x2 &mpic 0x1 0x1 0 0 - 0000 0x0 0x0 0x3 &mpic 0x2 0x1 0 0 - 0000 0x0 0x0 0x4 &mpic 0x3 0x1 0 0 - - /* IDSEL 0x04 (VIA chip) */ - 0x2000 0x0 0x0 0x1 &mpic 0x0 0x1 0 0 - 0x2000 0x0 0x0 0x2 &mpic 0x1 0x1 0 0 - 0x2000 0x0 0x0 0x3 &mpic 0x2 0x1 0 0 - 0x2000 0x0 0x0 0x4 &mpic 0x3 0x1 0 0 - - /* IDSEL 0x05 (8139) */ - 0x2800 0x0 0x0 0x1 &mpic 0x1 0x1 0 0 - - /* IDSEL 0x06 (Slot 6) */ - 0x3000 0x0 0x0 0x1 &mpic 0x2 0x1 0 0 - 0x3000 0x0 0x0 0x2 &mpic 0x3 0x1 0 0 - 0x3000 0x0 0x0 0x3 &mpic 0x0 0x1 0 0 - 0x3000 0x0 0x0 0x4 &mpic 0x1 0x1 0 0 - - /* IDESL 0x07 (Slot 7) */ - 0x3800 0x0 0x0 0x1 &mpic 0x3 0x1 0 0 - 0x3800 0x0 0x0 0x2 &mpic 0x0 0x1 0 0 - 0x3800 0x0 0x0 0x3 &mpic 0x1 0x1 0 0 - 0x3800 0x0 0x0 0x4 &mpic 0x2 0x1 0 0>; - - reg = <0xe000 0x0 0x0 0x0 0x0>; - #interrupt-cells = <1>; - #size-cells = <2>; - #address-cells = <3>; - ranges = <0x2000000 0x0 0x80000000 - 0x2000000 0x0 0x80000000 - 0x0 0x20000000 - 0x1000000 0x0 0x0 - 0x1000000 0x0 0x0 - 0x0 0x80000>; - clock-frequency = <33333333>; - - isa@4 { - device_type = "isa"; - #interrupt-cells = <2>; - #size-cells = <1>; - #address-cells = <2>; - reg = <0x2000 0x0 0x0 0x0 0x0>; - ranges = <0x1 0x0 0x1000000 0x0 0x0 0x1000>; - interrupt-parent = <&i8259>; - - i8259: interrupt-controller@20 { - interrupt-controller; - device_type = "interrupt-controller"; - reg = <0x1 0x20 0x2 - 0x1 0xa0 0x2 - 0x1 0x4d0 0x2>; - #address-cells = <0>; - #interrupt-cells = <2>; - compatible = "chrp,iic"; - interrupts = <0 1 0 0>; - interrupt-parent = <&mpic>; - }; - - rtc@70 { - compatible = "pnpPNP,b00"; - reg = <0x1 0x70 0x2>; - }; - }; - }; -}; diff --git a/arch/powerpc/boot/dts/fsl/mpc8548cds_32b.dts b/arch/powerpc/boot/dts/fsl/mpc8548cds_32b.dts deleted file mode 100644 index f6ba4a982766..000000000000 --- a/arch/powerpc/boot/dts/fsl/mpc8548cds_32b.dts +++ /dev/null @@ -1,82 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * MPC8548 CDS Device Tree Source (32-bit address map) - * - * Copyright 2006, 2008, 2011-2012 Freescale Semiconductor Inc. - */ - -/include/ "mpc8548si-pre.dtsi" - -/ { - model = "MPC8548CDS"; - compatible = "MPC8548CDS", "MPC85xxCDS"; - - memory { - device_type = "memory"; - reg = <0 0 0x0 0x8000000>; // 128M at 0x0 - }; - - board_lbc: lbc: localbus@e0005000 { - reg = <0 0xe0005000 0 0x1000>; - - ranges = <0x0 0x0 0x0 0xff000000 0x01000000 - 0x1 0x0 0x0 0xf8004000 0x00001000>; - - }; - - board_soc: soc: soc8548@e0000000 { - ranges = <0 0x0 0xe0000000 0x100000>; - }; - - board_pci0: pci0: pci@e0008000 { - reg = <0 0xe0008000 0 0x1000>; - ranges = <0x2000000 0x0 0x80000000 0 0x80000000 0x0 0x10000000 - 0x1000000 0x0 0x00000000 0 0xe2000000 0x0 0x800000>; - clock-frequency = <66666666>; - }; - - pci1: pci@e0009000 { - reg = <0 0xe0009000 0 0x1000>; - ranges = <0x2000000 0x0 0x90000000 0 0x90000000 0x0 0x10000000 - 0x1000000 0x0 0x00000000 0 0xe2800000 0x0 0x800000>; - clock-frequency = <66666666>; - interrupt-map-mask = <0xf800 0x0 0x0 0x7>; - interrupt-map = < - - /* IDSEL 0x15 */ - 0xa800 0x0 0x0 0x1 &mpic 0xb 0x1 0 0 - 0xa800 0x0 0x0 0x2 &mpic 0x1 0x1 0 0 - 0xa800 0x0 0x0 0x3 &mpic 0x2 0x1 0 0 - 0xa800 0x0 0x0 0x4 &mpic 0x3 0x1 0 0>; - }; - - pci2: pcie@e000a000 { - reg = <0 0xe000a000 0 0x1000>; - ranges = <0x2000000 0x0 0xa0000000 0 0xa0000000 0x0 0x20000000 - 0x1000000 0x0 0x00000000 0 0xe3000000 0x0 0x100000>; - pcie@0 { - ranges = <0x2000000 0x0 0xa0000000 - 0x2000000 0x0 0xa0000000 - 0x0 0x20000000 - - 0x1000000 0x0 0x0 - 0x1000000 0x0 0x0 - 0x0 0x100000>; - }; - }; - - rio: rapidio@e00c0000 { - reg = <0x0 0xe00c0000 0x0 0x20000>; - port1 { - ranges = <0x0 0x0 0x0 0xc0000000 0x0 0x20000000>; - }; - }; -}; - -/* - * mpc8548cds.dtsi must be last to ensure board_pci0 overrides pci0 settings - * for interrupt-map & interrupt-map-mask. - */ - -/include/ "mpc8548si-post.dtsi" -/include/ "mpc8548cds.dtsi" diff --git a/arch/powerpc/boot/dts/fsl/mpc8548cds_36b.dts b/arch/powerpc/boot/dts/fsl/mpc8548cds_36b.dts deleted file mode 100644 index 32e9076375ae..000000000000 --- a/arch/powerpc/boot/dts/fsl/mpc8548cds_36b.dts +++ /dev/null @@ -1,82 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * MPC8548 CDS Device Tree Source (36-bit address map) - * - * Copyright 2012 Freescale Semiconductor Inc. - */ - -/include/ "mpc8548si-pre.dtsi" - -/ { - model = "MPC8548CDS"; - compatible = "MPC8548CDS", "MPC85xxCDS"; - - memory { - device_type = "memory"; - reg = <0 0 0x0 0x8000000>; // 128M at 0x0 - }; - - board_lbc: lbc: localbus@fe0005000 { - reg = <0xf 0xe0005000 0 0x1000>; - - ranges = <0x0 0x0 0xf 0xff000000 0x01000000 - 0x1 0x0 0xf 0xf8004000 0x00001000>; - - }; - - board_soc: soc: soc8548@fe0000000 { - ranges = <0 0xf 0xe0000000 0x100000>; - }; - - board_pci0: pci0: pci@fe0008000 { - reg = <0xf 0xe0008000 0 0x1000>; - ranges = <0x2000000 0x0 0xe0000000 0xc 0x00000000 0x0 0x10000000 - 0x1000000 0x0 0x00000000 0xf 0xe2000000 0x0 0x800000>; - clock-frequency = <66666666>; - }; - - pci1: pci@fe0009000 { - reg = <0xf 0xe0009000 0 0x1000>; - ranges = <0x2000000 0x0 0xe0000000 0xc 0x10000000 0x0 0x10000000 - 0x1000000 0x0 0x00000000 0xf 0xe2800000 0x0 0x800000>; - clock-frequency = <66666666>; - interrupt-map-mask = <0xf800 0x0 0x0 0x7>; - interrupt-map = < - - /* IDSEL 0x15 */ - 0xa800 0x0 0x0 0x1 &mpic 0xb 0x1 0 0 - 0xa800 0x0 0x0 0x2 &mpic 0x1 0x1 0 0 - 0xa800 0x0 0x0 0x3 &mpic 0x2 0x1 0 0 - 0xa800 0x0 0x0 0x4 &mpic 0x3 0x1 0 0>; - }; - - pci2: pcie@fe000a000 { - reg = <0xf 0xe000a000 0 0x1000>; - ranges = <0x2000000 0x0 0xe0000000 0xc 0x20000000 0x0 0x20000000 - 0x1000000 0x0 0x00000000 0xf 0xe3000000 0x0 0x100000>; - pcie@0 { - ranges = <0x2000000 0x0 0xa0000000 - 0x2000000 0x0 0xa0000000 - 0x0 0x20000000 - - 0x1000000 0x0 0x0 - 0x1000000 0x0 0x0 - 0x0 0x100000>; - }; - }; - - rio: rapidio@fe00c0000 { - reg = <0xf 0xe00c0000 0x0 0x20000>; - port1 { - ranges = <0x0 0x0 0xc 0x40000000 0x0 0x20000000>; - }; - }; -}; - -/* - * mpc8548cds.dtsi must be last to ensure board_pci0 overrides pci0 settings - * for interrupt-map & interrupt-map-mask. - */ - -/include/ "mpc8548si-post.dtsi" -/include/ "mpc8548cds.dtsi" diff --git a/arch/powerpc/boot/dts/fsl/mpc8555cds.dts b/arch/powerpc/boot/dts/fsl/mpc8555cds.dts deleted file mode 100644 index 901b6ff06dfb..000000000000 --- a/arch/powerpc/boot/dts/fsl/mpc8555cds.dts +++ /dev/null @@ -1,375 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * MPC8555 CDS Device Tree Source - * - * Copyright 2006, 2008 Freescale Semiconductor Inc. - */ - -/dts-v1/; - -/include/ "e500v1_power_isa.dtsi" - -/ { - model = "MPC8555CDS"; - compatible = "MPC8555CDS", "MPC85xxCDS"; - #address-cells = <1>; - #size-cells = <1>; - - aliases { - ethernet0 = &enet0; - ethernet1 = &enet1; - serial0 = &serial0; - serial1 = &serial1; - pci0 = &pci0; - pci1 = &pci1; - }; - - cpus { - #address-cells = <1>; - #size-cells = <0>; - - PowerPC,8555@0 { - device_type = "cpu"; - reg = <0x0>; - d-cache-line-size = <32>; // 32 bytes - i-cache-line-size = <32>; // 32 bytes - d-cache-size = <0x8000>; // L1, 32K - i-cache-size = <0x8000>; // L1, 32K - timebase-frequency = <0>; // 33 MHz, from uboot - bus-frequency = <0>; // 166 MHz - clock-frequency = <0>; // 825 MHz, from uboot - next-level-cache = <&L2>; - }; - }; - - memory { - device_type = "memory"; - reg = <0x0 0x8000000>; // 128M at 0x0 - }; - - soc8555@e0000000 { - #address-cells = <1>; - #size-cells = <1>; - device_type = "soc"; - compatible = "simple-bus"; - ranges = <0x0 0xe0000000 0x100000>; - bus-frequency = <0>; - - ecm-law@0 { - compatible = "fsl,ecm-law"; - reg = <0x0 0x1000>; - fsl,num-laws = <8>; - }; - - ecm@1000 { - compatible = "fsl,mpc8555-ecm", "fsl,ecm"; - reg = <0x1000 0x1000>; - interrupts = <17 2>; - interrupt-parent = <&mpic>; - }; - - memory-controller@2000 { - compatible = "fsl,mpc8555-memory-controller"; - reg = <0x2000 0x1000>; - interrupt-parent = <&mpic>; - interrupts = <18 2>; - }; - - L2: l2-cache-controller@20000 { - compatible = "fsl,mpc8555-l2-cache-controller"; - reg = <0x20000 0x1000>; - cache-line-size = <32>; // 32 bytes - cache-size = <0x40000>; // L2, 256K - interrupt-parent = <&mpic>; - interrupts = <16 2>; - }; - - i2c@3000 { - #address-cells = <1>; - #size-cells = <0>; - cell-index = <0>; - compatible = "fsl-i2c"; - reg = <0x3000 0x100>; - interrupts = <43 2>; - interrupt-parent = <&mpic>; - dfsrr; - }; - - dma@21300 { - #address-cells = <1>; - #size-cells = <1>; - compatible = "fsl,mpc8555-dma", "fsl,eloplus-dma"; - reg = <0x21300 0x4>; - ranges = <0x0 0x21100 0x200>; - cell-index = <0>; - dma-channel@0 { - compatible = "fsl,mpc8555-dma-channel", - "fsl,eloplus-dma-channel"; - reg = <0x0 0x80>; - cell-index = <0>; - interrupt-parent = <&mpic>; - interrupts = <20 2>; - }; - dma-channel@80 { - compatible = "fsl,mpc8555-dma-channel", - "fsl,eloplus-dma-channel"; - reg = <0x80 0x80>; - cell-index = <1>; - interrupt-parent = <&mpic>; - interrupts = <21 2>; - }; - dma-channel@100 { - compatible = "fsl,mpc8555-dma-channel", - "fsl,eloplus-dma-channel"; - reg = <0x100 0x80>; - cell-index = <2>; - interrupt-parent = <&mpic>; - interrupts = <22 2>; - }; - dma-channel@180 { - compatible = "fsl,mpc8555-dma-channel", - "fsl,eloplus-dma-channel"; - reg = <0x180 0x80>; - cell-index = <3>; - interrupt-parent = <&mpic>; - interrupts = <23 2>; - }; - }; - - enet0: ethernet@24000 { - #address-cells = <1>; - #size-cells = <1>; - cell-index = <0>; - device_type = "network"; - model = "TSEC"; - compatible = "gianfar"; - reg = <0x24000 0x1000>; - ranges = <0x0 0x24000 0x1000>; - local-mac-address = [ 00 00 00 00 00 00 ]; - interrupts = <29 2 30 2 34 2>; - interrupt-parent = <&mpic>; - tbi-handle = <&tbi0>; - phy-handle = <&phy0>; - - mdio@520 { - #address-cells = <1>; - #size-cells = <0>; - compatible = "fsl,gianfar-mdio"; - reg = <0x520 0x20>; - - phy0: ethernet-phy@0 { - interrupt-parent = <&mpic>; - interrupts = <5 1>; - reg = <0x0>; - }; - phy1: ethernet-phy@1 { - interrupt-parent = <&mpic>; - interrupts = <5 1>; - reg = <0x1>; - }; - tbi0: tbi-phy@11 { - reg = <0x11>; - device_type = "tbi-phy"; - }; - }; - }; - - enet1: ethernet@25000 { - #address-cells = <1>; - #size-cells = <1>; - cell-index = <1>; - device_type = "network"; - model = "TSEC"; - compatible = "gianfar"; - reg = <0x25000 0x1000>; - ranges = <0x0 0x25000 0x1000>; - local-mac-address = [ 00 00 00 00 00 00 ]; - interrupts = <35 2 36 2 40 2>; - interrupt-parent = <&mpic>; - tbi-handle = <&tbi1>; - phy-handle = <&phy1>; - - mdio@520 { - #address-cells = <1>; - #size-cells = <0>; - compatible = "fsl,gianfar-tbi"; - reg = <0x520 0x20>; - - tbi1: tbi-phy@11 { - reg = <0x11>; - device_type = "tbi-phy"; - }; - }; - }; - - serial0: serial@4500 { - cell-index = <0>; - device_type = "serial"; - compatible = "fsl,ns16550", "ns16550"; - reg = <0x4500 0x100>; // reg base, size - clock-frequency = <0>; // should we fill in in uboot? - interrupts = <42 2>; - interrupt-parent = <&mpic>; - }; - - serial1: serial@4600 { - cell-index = <1>; - device_type = "serial"; - compatible = "fsl,ns16550", "ns16550"; - reg = <0x4600 0x100>; // reg base, size - clock-frequency = <0>; // should we fill in in uboot? - interrupts = <42 2>; - interrupt-parent = <&mpic>; - }; - - crypto@30000 { - compatible = "fsl,sec2.0"; - reg = <0x30000 0x10000>; - interrupts = <45 2>; - interrupt-parent = <&mpic>; - fsl,num-channels = <4>; - fsl,channel-fifo-len = <24>; - fsl,exec-units-mask = <0x7e>; - fsl,descriptor-types-mask = <0x01010ebf>; - }; - - mpic: pic@40000 { - interrupt-controller; - #address-cells = <0>; - #interrupt-cells = <2>; - reg = <0x40000 0x40000>; - compatible = "chrp,open-pic"; - device_type = "open-pic"; - }; - - cpm@919c0 { - #address-cells = <1>; - #size-cells = <1>; - compatible = "fsl,mpc8555-cpm", "fsl,cpm2"; - reg = <0x919c0 0x30>; - ranges; - - muram@80000 { - #address-cells = <1>; - #size-cells = <1>; - ranges = <0x0 0x80000 0x10000>; - - data@0 { - compatible = "fsl,cpm-muram-data"; - reg = <0x0 0x2000 0x9000 0x1000>; - }; - }; - - brg@919f0 { - compatible = "fsl,mpc8555-brg", - "fsl,cpm2-brg", - "fsl,cpm-brg"; - reg = <0x919f0 0x10 0x915f0 0x10>; - }; - - cpmpic: pic@90c00 { - interrupt-controller; - #address-cells = <0>; - #interrupt-cells = <2>; - interrupts = <46 2>; - interrupt-parent = <&mpic>; - reg = <0x90c00 0x80>; - compatible = "fsl,mpc8555-cpm-pic", "fsl,cpm2-pic"; - }; - }; - }; - - pci0: pci@e0008000 { - interrupt-map-mask = <0x1f800 0x0 0x0 0x7>; - interrupt-map = < - - /* IDSEL 0x10 */ - 0x8000 0x0 0x0 0x1 &mpic 0x0 0x1 - 0x8000 0x0 0x0 0x2 &mpic 0x1 0x1 - 0x8000 0x0 0x0 0x3 &mpic 0x2 0x1 - 0x8000 0x0 0x0 0x4 &mpic 0x3 0x1 - - /* IDSEL 0x11 */ - 0x8800 0x0 0x0 0x1 &mpic 0x0 0x1 - 0x8800 0x0 0x0 0x2 &mpic 0x1 0x1 - 0x8800 0x0 0x0 0x3 &mpic 0x2 0x1 - 0x8800 0x0 0x0 0x4 &mpic 0x3 0x1 - - /* IDSEL 0x12 (Slot 1) */ - 0x9000 0x0 0x0 0x1 &mpic 0x0 0x1 - 0x9000 0x0 0x0 0x2 &mpic 0x1 0x1 - 0x9000 0x0 0x0 0x3 &mpic 0x2 0x1 - 0x9000 0x0 0x0 0x4 &mpic 0x3 0x1 - - /* IDSEL 0x13 (Slot 2) */ - 0x9800 0x0 0x0 0x1 &mpic 0x1 0x1 - 0x9800 0x0 0x0 0x2 &mpic 0x2 0x1 - 0x9800 0x0 0x0 0x3 &mpic 0x3 0x1 - 0x9800 0x0 0x0 0x4 &mpic 0x0 0x1 - - /* IDSEL 0x14 (Slot 3) */ - 0xa000 0x0 0x0 0x1 &mpic 0x2 0x1 - 0xa000 0x0 0x0 0x2 &mpic 0x3 0x1 - 0xa000 0x0 0x0 0x3 &mpic 0x0 0x1 - 0xa000 0x0 0x0 0x4 &mpic 0x1 0x1 - - /* IDSEL 0x15 (Slot 4) */ - 0xa800 0x0 0x0 0x1 &mpic 0x3 0x1 - 0xa800 0x0 0x0 0x2 &mpic 0x0 0x1 - 0xa800 0x0 0x0 0x3 &mpic 0x1 0x1 - 0xa800 0x0 0x0 0x4 &mpic 0x2 0x1 - - /* Bus 1 (Tundra Bridge) */ - /* IDSEL 0x12 (ISA bridge) */ - 0x19000 0x0 0x0 0x1 &mpic 0x0 0x1 - 0x19000 0x0 0x0 0x2 &mpic 0x1 0x1 - 0x19000 0x0 0x0 0x3 &mpic 0x2 0x1 - 0x19000 0x0 0x0 0x4 &mpic 0x3 0x1>; - interrupt-parent = <&mpic>; - interrupts = <24 2>; - bus-range = <0 0>; - ranges = <0x2000000 0x0 0x80000000 0x80000000 0x0 0x20000000 - 0x1000000 0x0 0x0 0xe2000000 0x0 0x100000>; - clock-frequency = <66666666>; - #interrupt-cells = <1>; - #size-cells = <2>; - #address-cells = <3>; - reg = <0xe0008000 0x1000>; - compatible = "fsl,mpc8540-pci"; - device_type = "pci"; - - i8259@19000 { - interrupt-controller; - device_type = "interrupt-controller"; - reg = <0x19000 0x0 0x0 0x0 0x1>; - #address-cells = <0>; - #interrupt-cells = <2>; - compatible = "chrp,iic"; - interrupts = <1>; - interrupt-parent = <&pci0>; - }; - }; - - pci1: pci@e0009000 { - interrupt-map-mask = <0xf800 0x0 0x0 0x7>; - interrupt-map = < - - /* IDSEL 0x15 */ - 0xa800 0x0 0x0 0x1 &mpic 0xb 0x1 - 0xa800 0x0 0x0 0x2 &mpic 0xb 0x1 - 0xa800 0x0 0x0 0x3 &mpic 0xb 0x1 - 0xa800 0x0 0x0 0x4 &mpic 0xb 0x1>; - interrupt-parent = <&mpic>; - interrupts = <25 2>; - bus-range = <0 0>; - ranges = <0x2000000 0x0 0xa0000000 0xa0000000 0x0 0x20000000 - 0x1000000 0x0 0x0 0xe3000000 0x0 0x100000>; - clock-frequency = <66666666>; - #interrupt-cells = <1>; - #size-cells = <2>; - #address-cells = <3>; - reg = <0xe0009000 0x1000>; - compatible = "fsl,mpc8540-pci"; - device_type = "pci"; - }; -}; diff --git a/arch/powerpc/boot/dts/fsl/mpc8560ads.dts b/arch/powerpc/boot/dts/fsl/mpc8560ads.dts deleted file mode 100644 index c2f9aea78b29..000000000000 --- a/arch/powerpc/boot/dts/fsl/mpc8560ads.dts +++ /dev/null @@ -1,388 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * MPC8560 ADS Device Tree Source - * - * Copyright 2006, 2008 Freescale Semiconductor Inc. - */ - -/dts-v1/; - -/include/ "e500v1_power_isa.dtsi" - -/ { - model = "MPC8560ADS"; - compatible = "MPC8560ADS", "MPC85xxADS"; - #address-cells = <1>; - #size-cells = <1>; - - aliases { - ethernet0 = &enet0; - ethernet1 = &enet1; - ethernet2 = &enet2; - ethernet3 = &enet3; - serial0 = &serial0; - serial1 = &serial1; - pci0 = &pci0; - }; - - cpus { - #address-cells = <1>; - #size-cells = <0>; - - PowerPC,8560@0 { - device_type = "cpu"; - reg = <0x0>; - d-cache-line-size = <32>; // 32 bytes - i-cache-line-size = <32>; // 32 bytes - d-cache-size = <0x8000>; // L1, 32K - i-cache-size = <0x8000>; // L1, 32K - timebase-frequency = <82500000>; - bus-frequency = <330000000>; - clock-frequency = <825000000>; - }; - }; - - memory { - device_type = "memory"; - reg = <0x0 0x10000000>; - }; - - soc8560@e0000000 { - #address-cells = <1>; - #size-cells = <1>; - device_type = "soc"; - compatible = "simple-bus"; - ranges = <0x0 0xe0000000 0x100000>; - bus-frequency = <330000000>; - - ecm-law@0 { - compatible = "fsl,ecm-law"; - reg = <0x0 0x1000>; - fsl,num-laws = <8>; - }; - - ecm@1000 { - compatible = "fsl,mpc8560-ecm", "fsl,ecm"; - reg = <0x1000 0x1000>; - interrupts = <17 2>; - interrupt-parent = <&mpic>; - }; - - memory-controller@2000 { - compatible = "fsl,mpc8540-memory-controller"; - reg = <0x2000 0x1000>; - interrupt-parent = <&mpic>; - interrupts = <18 2>; - }; - - L2: l2-cache-controller@20000 { - compatible = "fsl,mpc8540-l2-cache-controller"; - reg = <0x20000 0x1000>; - cache-line-size = <32>; // 32 bytes - cache-size = <0x40000>; // L2, 256K - interrupt-parent = <&mpic>; - interrupts = <16 2>; - }; - - dma@21300 { - #address-cells = <1>; - #size-cells = <1>; - compatible = "fsl,mpc8560-dma", "fsl,eloplus-dma"; - reg = <0x21300 0x4>; - ranges = <0x0 0x21100 0x200>; - cell-index = <0>; - dma-channel@0 { - compatible = "fsl,mpc8560-dma-channel", - "fsl,eloplus-dma-channel"; - reg = <0x0 0x80>; - cell-index = <0>; - interrupt-parent = <&mpic>; - interrupts = <20 2>; - }; - dma-channel@80 { - compatible = "fsl,mpc8560-dma-channel", - "fsl,eloplus-dma-channel"; - reg = <0x80 0x80>; - cell-index = <1>; - interrupt-parent = <&mpic>; - interrupts = <21 2>; - }; - dma-channel@100 { - compatible = "fsl,mpc8560-dma-channel", - "fsl,eloplus-dma-channel"; - reg = <0x100 0x80>; - cell-index = <2>; - interrupt-parent = <&mpic>; - interrupts = <22 2>; - }; - dma-channel@180 { - compatible = "fsl,mpc8560-dma-channel", - "fsl,eloplus-dma-channel"; - reg = <0x180 0x80>; - cell-index = <3>; - interrupt-parent = <&mpic>; - interrupts = <23 2>; - }; - }; - - enet0: ethernet@24000 { - #address-cells = <1>; - #size-cells = <1>; - cell-index = <0>; - device_type = "network"; - model = "TSEC"; - compatible = "gianfar"; - reg = <0x24000 0x1000>; - ranges = <0x0 0x24000 0x1000>; - local-mac-address = [ 00 00 00 00 00 00 ]; - interrupts = <29 2 30 2 34 2>; - interrupt-parent = <&mpic>; - tbi-handle = <&tbi0>; - phy-handle = <&phy0>; - - mdio@520 { - #address-cells = <1>; - #size-cells = <0>; - compatible = "fsl,gianfar-mdio"; - reg = <0x520 0x20>; - - phy0: ethernet-phy@0 { - interrupt-parent = <&mpic>; - interrupts = <5 1>; - reg = <0x0>; - }; - phy1: ethernet-phy@1 { - interrupt-parent = <&mpic>; - interrupts = <5 1>; - reg = <0x1>; - }; - phy2: ethernet-phy@2 { - interrupt-parent = <&mpic>; - interrupts = <7 1>; - reg = <0x2>; - }; - phy3: ethernet-phy@3 { - interrupt-parent = <&mpic>; - interrupts = <7 1>; - reg = <0x3>; - }; - tbi0: tbi-phy@11 { - reg = <0x11>; - device_type = "tbi-phy"; - }; - }; - }; - - enet1: ethernet@25000 { - #address-cells = <1>; - #size-cells = <1>; - cell-index = <1>; - device_type = "network"; - model = "TSEC"; - compatible = "gianfar"; - reg = <0x25000 0x1000>; - ranges = <0x0 0x25000 0x1000>; - local-mac-address = [ 00 00 00 00 00 00 ]; - interrupts = <35 2 36 2 40 2>; - interrupt-parent = <&mpic>; - tbi-handle = <&tbi1>; - phy-handle = <&phy1>; - - mdio@520 { - #address-cells = <1>; - #size-cells = <0>; - compatible = "fsl,gianfar-tbi"; - reg = <0x520 0x20>; - - tbi1: tbi-phy@11 { - reg = <0x11>; - device_type = "tbi-phy"; - }; - }; - }; - - mpic: pic@40000 { - interrupt-controller; - #address-cells = <0>; - #interrupt-cells = <2>; - reg = <0x40000 0x40000>; - compatible = "chrp,open-pic"; - device_type = "open-pic"; - }; - - cpm@919c0 { - #address-cells = <1>; - #size-cells = <1>; - compatible = "fsl,mpc8560-cpm", "fsl,cpm2"; - reg = <0x919c0 0x30>; - ranges; - - muram@80000 { - #address-cells = <1>; - #size-cells = <1>; - ranges = <0x0 0x80000 0x10000>; - - data@0 { - compatible = "fsl,cpm-muram-data"; - reg = <0x0 0x4000 0x9000 0x2000>; - }; - }; - - brg@919f0 { - compatible = "fsl,mpc8560-brg", - "fsl,cpm2-brg", - "fsl,cpm-brg"; - reg = <0x919f0 0x10 0x915f0 0x10>; - clock-frequency = <165000000>; - }; - - cpmpic: pic@90c00 { - interrupt-controller; - #address-cells = <0>; - #interrupt-cells = <2>; - interrupts = <46 2>; - interrupt-parent = <&mpic>; - reg = <0x90c00 0x80>; - compatible = "fsl,mpc8560-cpm-pic", "fsl,cpm2-pic"; - }; - - serial0: serial@91a00 { - device_type = "serial"; - compatible = "fsl,mpc8560-scc-uart", - "fsl,cpm2-scc-uart"; - reg = <0x91a00 0x20 0x88000 0x100>; - fsl,cpm-brg = <1>; - fsl,cpm-command = <0x800000>; - current-speed = <115200>; - interrupts = <40 8>; - interrupt-parent = <&cpmpic>; - }; - - serial1: serial@91a20 { - device_type = "serial"; - compatible = "fsl,mpc8560-scc-uart", - "fsl,cpm2-scc-uart"; - reg = <0x91a20 0x20 0x88100 0x100>; - fsl,cpm-brg = <2>; - fsl,cpm-command = <0x4a00000>; - current-speed = <115200>; - interrupts = <41 8>; - interrupt-parent = <&cpmpic>; - }; - - enet2: ethernet@91320 { - device_type = "network"; - compatible = "fsl,mpc8560-fcc-enet", - "fsl,cpm2-fcc-enet"; - reg = <0x91320 0x20 0x88500 0x100 0x913b0 0x1>; - local-mac-address = [ 00 00 00 00 00 00 ]; - fsl,cpm-command = <0x16200300>; - interrupts = <33 8>; - interrupt-parent = <&cpmpic>; - phy-handle = <&phy2>; - }; - - enet3: ethernet@91340 { - device_type = "network"; - compatible = "fsl,mpc8560-fcc-enet", - "fsl,cpm2-fcc-enet"; - reg = <0x91340 0x20 0x88600 0x100 0x913d0 0x1>; - local-mac-address = [ 00 00 00 00 00 00 ]; - fsl,cpm-command = <0x1a400300>; - interrupts = <34 8>; - interrupt-parent = <&cpmpic>; - phy-handle = <&phy3>; - }; - }; - }; - - pci0: pci@e0008000 { - #interrupt-cells = <1>; - #size-cells = <2>; - #address-cells = <3>; - compatible = "fsl,mpc8540-pcix", "fsl,mpc8540-pci"; - device_type = "pci"; - reg = <0xe0008000 0x1000>; - clock-frequency = <66666666>; - interrupt-map-mask = <0xf800 0x0 0x0 0x7>; - interrupt-map = < - - /* IDSEL 0x2 */ - 0x1000 0x0 0x0 0x1 &mpic 0x1 0x1 - 0x1000 0x0 0x0 0x2 &mpic 0x2 0x1 - 0x1000 0x0 0x0 0x3 &mpic 0x3 0x1 - 0x1000 0x0 0x0 0x4 &mpic 0x4 0x1 - - /* IDSEL 0x3 */ - 0x1800 0x0 0x0 0x1 &mpic 0x4 0x1 - 0x1800 0x0 0x0 0x2 &mpic 0x1 0x1 - 0x1800 0x0 0x0 0x3 &mpic 0x2 0x1 - 0x1800 0x0 0x0 0x4 &mpic 0x3 0x1 - - /* IDSEL 0x4 */ - 0x2000 0x0 0x0 0x1 &mpic 0x3 0x1 - 0x2000 0x0 0x0 0x2 &mpic 0x4 0x1 - 0x2000 0x0 0x0 0x3 &mpic 0x1 0x1 - 0x2000 0x0 0x0 0x4 &mpic 0x2 0x1 - - /* IDSEL 0x5 */ - 0x2800 0x0 0x0 0x1 &mpic 0x2 0x1 - 0x2800 0x0 0x0 0x2 &mpic 0x3 0x1 - 0x2800 0x0 0x0 0x3 &mpic 0x4 0x1 - 0x2800 0x0 0x0 0x4 &mpic 0x1 0x1 - - /* IDSEL 12 */ - 0x6000 0x0 0x0 0x1 &mpic 0x1 0x1 - 0x6000 0x0 0x0 0x2 &mpic 0x2 0x1 - 0x6000 0x0 0x0 0x3 &mpic 0x3 0x1 - 0x6000 0x0 0x0 0x4 &mpic 0x4 0x1 - - /* IDSEL 13 */ - 0x6800 0x0 0x0 0x1 &mpic 0x4 0x1 - 0x6800 0x0 0x0 0x2 &mpic 0x1 0x1 - 0x6800 0x0 0x0 0x3 &mpic 0x2 0x1 - 0x6800 0x0 0x0 0x4 &mpic 0x3 0x1 - - /* IDSEL 14*/ - 0x7000 0x0 0x0 0x1 &mpic 0x3 0x1 - 0x7000 0x0 0x0 0x2 &mpic 0x4 0x1 - 0x7000 0x0 0x0 0x3 &mpic 0x1 0x1 - 0x7000 0x0 0x0 0x4 &mpic 0x2 0x1 - - /* IDSEL 15 */ - 0x7800 0x0 0x0 0x1 &mpic 0x2 0x1 - 0x7800 0x0 0x0 0x2 &mpic 0x3 0x1 - 0x7800 0x0 0x0 0x3 &mpic 0x4 0x1 - 0x7800 0x0 0x0 0x4 &mpic 0x1 0x1 - - /* IDSEL 18 */ - 0x9000 0x0 0x0 0x1 &mpic 0x1 0x1 - 0x9000 0x0 0x0 0x2 &mpic 0x2 0x1 - 0x9000 0x0 0x0 0x3 &mpic 0x3 0x1 - 0x9000 0x0 0x0 0x4 &mpic 0x4 0x1 - - /* IDSEL 19 */ - 0x9800 0x0 0x0 0x1 &mpic 0x4 0x1 - 0x9800 0x0 0x0 0x2 &mpic 0x1 0x1 - 0x9800 0x0 0x0 0x3 &mpic 0x2 0x1 - 0x9800 0x0 0x0 0x4 &mpic 0x3 0x1 - - /* IDSEL 20 */ - 0xa000 0x0 0x0 0x1 &mpic 0x3 0x1 - 0xa000 0x0 0x0 0x2 &mpic 0x4 0x1 - 0xa000 0x0 0x0 0x3 &mpic 0x1 0x1 - 0xa000 0x0 0x0 0x4 &mpic 0x2 0x1 - - /* IDSEL 21 */ - 0xa800 0x0 0x0 0x1 &mpic 0x2 0x1 - 0xa800 0x0 0x0 0x2 &mpic 0x3 0x1 - 0xa800 0x0 0x0 0x3 &mpic 0x4 0x1 - 0xa800 0x0 0x0 0x4 &mpic 0x1 0x1>; - - interrupt-parent = <&mpic>; - interrupts = <24 2>; - bus-range = <0 0>; - ranges = <0x2000000 0x0 0x80000000 0x80000000 0x0 0x20000000 - 0x1000000 0x0 0x0 0xe2000000 0x0 0x1000000>; - }; -}; diff --git a/arch/powerpc/configs/40x/klondike_defconfig b/arch/powerpc/configs/40x/klondike_defconfig index 6a735ee75715..acafbb8f6808 100644 --- a/arch/powerpc/configs/40x/klondike_defconfig +++ b/arch/powerpc/configs/40x/klondike_defconfig @@ -5,7 +5,6 @@ CONFIG_SYSFS_DEPRECATED=y CONFIG_SYSFS_DEPRECATED_V2=y CONFIG_BLK_DEV_INITRD=y CONFIG_EMBEDDED=y -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_APM8018X=y diff --git a/arch/powerpc/configs/52xx/pcm030_defconfig b/arch/powerpc/configs/52xx/pcm030_defconfig index 789622ffd844..88fbe0d42e11 100644 --- a/arch/powerpc/configs/52xx/pcm030_defconfig +++ b/arch/powerpc/configs/52xx/pcm030_defconfig @@ -10,7 +10,6 @@ CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_EXPERT=y # CONFIG_VM_EVENT_COUNTERS is not set -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/powerpc/configs/83xx/kmeter1_defconfig b/arch/powerpc/configs/83xx/kmeter1_defconfig index 24bf1bde1bb4..487e5e1bbf4c 100644 --- a/arch/powerpc/configs/83xx/kmeter1_defconfig +++ b/arch/powerpc/configs/83xx/kmeter1_defconfig @@ -5,7 +5,6 @@ CONFIG_NO_HZ=y CONFIG_HIGH_RES_TIMERS=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_EXPERT=y -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/powerpc/configs/83xx/mpc837x_rdb_defconfig b/arch/powerpc/configs/83xx/mpc837x_rdb_defconfig index 4e3373381ab6..58fae5131fa7 100644 --- a/arch/powerpc/configs/83xx/mpc837x_rdb_defconfig +++ b/arch/powerpc/configs/83xx/mpc837x_rdb_defconfig @@ -2,7 +2,6 @@ CONFIG_SYSVIPC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/powerpc/configs/85xx/ge_imp3a_defconfig b/arch/powerpc/configs/85xx/ge_imp3a_defconfig index 6cb7e90d52c1..da6fc203e2dc 100644 --- a/arch/powerpc/configs/85xx/ge_imp3a_defconfig +++ b/arch/powerpc/configs/85xx/ge_imp3a_defconfig @@ -17,7 +17,6 @@ CONFIG_SYSFS_DEPRECATED_V2=y CONFIG_RELAY=y CONFIG_BLK_DEV_INITRD=y CONFIG_PERF_EVENTS=y -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/powerpc/configs/85xx/mpc8540_ads_defconfig b/arch/powerpc/configs/85xx/mpc8540_ads_defconfig deleted file mode 100644 index 618e03e0706d..000000000000 --- a/arch/powerpc/configs/85xx/mpc8540_ads_defconfig +++ /dev/null @@ -1,47 +0,0 @@ -CONFIG_PPC_85xx=y -CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y -CONFIG_HIGH_RES_TIMERS=y -CONFIG_LOG_BUF_SHIFT=14 -CONFIG_BLK_DEV_INITRD=y -CONFIG_EXPERT=y -# CONFIG_BLK_DEV_BSG is not set -CONFIG_PARTITION_ADVANCED=y -# CONFIG_MSDOS_PARTITION is not set -CONFIG_MPC8540_ADS=y -CONFIG_GEN_RTC=y -CONFIG_BINFMT_MISC=y -CONFIG_MATH_EMULATION=y -# CONFIG_SECCOMP is not set -CONFIG_NET=y -CONFIG_PACKET=y -CONFIG_UNIX=y -CONFIG_XFRM_USER=y -CONFIG_INET=y -CONFIG_IP_MULTICAST=y -CONFIG_IP_PNP=y -CONFIG_IP_PNP_DHCP=y -CONFIG_IP_PNP_BOOTP=y -CONFIG_SYN_COOKIES=y -# CONFIG_IPV6 is not set -# CONFIG_FW_LOADER is not set -CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_RAM=y -CONFIG_BLK_DEV_RAM_SIZE=32768 -CONFIG_NETDEVICES=y -CONFIG_GIANFAR=y -# CONFIG_INPUT_KEYBOARD is not set -# CONFIG_INPUT_MOUSE is not set -# CONFIG_SERIO is not set -# CONFIG_VT is not set -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -# CONFIG_HW_RANDOM is not set -CONFIG_EXT2_FS=y -CONFIG_EXT4_FS=y -CONFIG_PROC_KCORE=y -CONFIG_TMPFS=y -CONFIG_NFS_FS=y -CONFIG_ROOT_NFS=y -CONFIG_DETECT_HUNG_TASK=y -CONFIG_DEBUG_MUTEXES=y diff --git a/arch/powerpc/configs/85xx/mpc8560_ads_defconfig b/arch/powerpc/configs/85xx/mpc8560_ads_defconfig deleted file mode 100644 index 9bc6283f2fb2..000000000000 --- a/arch/powerpc/configs/85xx/mpc8560_ads_defconfig +++ /dev/null @@ -1,50 +0,0 @@ -CONFIG_PPC_85xx=y -CONFIG_SYSVIPC=y -CONFIG_LOG_BUF_SHIFT=14 -CONFIG_BLK_DEV_INITRD=y -CONFIG_EXPERT=y -# CONFIG_BLK_DEV_BSG is not set -CONFIG_PARTITION_ADVANCED=y -# CONFIG_MSDOS_PARTITION is not set -CONFIG_MPC8560_ADS=y -CONFIG_GEN_RTC=y -CONFIG_BINFMT_MISC=y -CONFIG_MATH_EMULATION=y -# CONFIG_SECCOMP is not set -CONFIG_PCI=y -CONFIG_PCI_DEBUG=y -CONFIG_NET=y -CONFIG_PACKET=y -CONFIG_UNIX=y -CONFIG_INET=y -CONFIG_IP_MULTICAST=y -CONFIG_IP_PNP=y -CONFIG_IP_PNP_DHCP=y -CONFIG_IP_PNP_BOOTP=y -CONFIG_SYN_COOKIES=y -# CONFIG_IPV6 is not set -# CONFIG_FW_LOADER is not set -CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_RAM=y -CONFIG_BLK_DEV_RAM_SIZE=32768 -CONFIG_NETDEVICES=y -CONFIG_FS_ENET=y -# CONFIG_FS_ENET_HAS_SCC is not set -CONFIG_GIANFAR=y -CONFIG_E1000=y -CONFIG_DAVICOM_PHY=y -CONFIG_MARVELL_PHY=y -# CONFIG_INPUT_KEYBOARD is not set -# CONFIG_INPUT_MOUSE is not set -# CONFIG_SERIO is not set -# CONFIG_VT is not set -CONFIG_SERIAL_CPM=y -CONFIG_SERIAL_CPM_CONSOLE=y -CONFIG_EXT2_FS=y -CONFIG_EXT4_FS=y -CONFIG_PROC_KCORE=y -CONFIG_TMPFS=y -CONFIG_NFS_FS=y -CONFIG_ROOT_NFS=y -CONFIG_DETECT_HUNG_TASK=y -CONFIG_DEBUG_MUTEXES=y diff --git a/arch/powerpc/configs/85xx/mpc85xx_cds_defconfig b/arch/powerpc/configs/85xx/mpc85xx_cds_defconfig deleted file mode 100644 index cea72e85ed26..000000000000 --- a/arch/powerpc/configs/85xx/mpc85xx_cds_defconfig +++ /dev/null @@ -1,52 +0,0 @@ -CONFIG_PPC_85xx=y -CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y -CONFIG_HIGH_RES_TIMERS=y -CONFIG_LOG_BUF_SHIFT=14 -CONFIG_BLK_DEV_INITRD=y -CONFIG_EXPERT=y -# CONFIG_BLK_DEV_BSG is not set -CONFIG_PARTITION_ADVANCED=y -# CONFIG_MSDOS_PARTITION is not set -CONFIG_MPC85xx_CDS=y -CONFIG_GEN_RTC=y -CONFIG_BINFMT_MISC=y -CONFIG_MATH_EMULATION=y -# CONFIG_SECCOMP is not set -CONFIG_PCI=y -CONFIG_NET=y -CONFIG_PACKET=y -CONFIG_UNIX=y -CONFIG_XFRM_USER=y -CONFIG_INET=y -CONFIG_IP_MULTICAST=y -CONFIG_IP_PNP=y -CONFIG_IP_PNP_DHCP=y -CONFIG_IP_PNP_BOOTP=y -CONFIG_SYN_COOKIES=y -# CONFIG_IPV6 is not set -# CONFIG_FW_LOADER is not set -CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_RAM=y -CONFIG_BLK_DEV_RAM_SIZE=32768 -CONFIG_ATA=y -CONFIG_ATA_GENERIC=y -CONFIG_PATA_VIA=y -CONFIG_NETDEVICES=y -CONFIG_GIANFAR=y -CONFIG_E1000=y -# CONFIG_INPUT_KEYBOARD is not set -# CONFIG_INPUT_MOUSE is not set -# CONFIG_SERIO is not set -# CONFIG_VT is not set -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -# CONFIG_HW_RANDOM is not set -CONFIG_EXT2_FS=y -CONFIG_EXT4_FS=y -CONFIG_PROC_KCORE=y -CONFIG_TMPFS=y -CONFIG_NFS_FS=y -CONFIG_ROOT_NFS=y -CONFIG_DETECT_HUNG_TASK=y -CONFIG_DEBUG_MUTEXES=y diff --git a/arch/powerpc/configs/ep8248e_defconfig b/arch/powerpc/configs/ep8248e_defconfig index 8df6d3a293e3..b9dfa3a0713b 100644 --- a/arch/powerpc/configs/ep8248e_defconfig +++ b/arch/powerpc/configs/ep8248e_defconfig @@ -4,7 +4,6 @@ CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_EXPERT=y CONFIG_KALLSYMS_ALL=y -CONFIG_SLAB=y CONFIG_PARTITION_ADVANCED=y # CONFIG_PPC_CHRP is not set # CONFIG_PPC_PMAC is not set diff --git a/arch/powerpc/configs/gamecube_defconfig b/arch/powerpc/configs/gamecube_defconfig index 91a1b99f4e8f..d77eeb525366 100644 --- a/arch/powerpc/configs/gamecube_defconfig +++ b/arch/powerpc/configs/gamecube_defconfig @@ -8,7 +8,6 @@ CONFIG_EXPERT=y # CONFIG_ELF_CORE is not set CONFIG_PERF_EVENTS=y # CONFIG_VM_EVENT_COUNTERS is not set -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/powerpc/configs/mgcoge_defconfig b/arch/powerpc/configs/mgcoge_defconfig index 498d35db7833..ce220ceeb91c 100644 --- a/arch/powerpc/configs/mgcoge_defconfig +++ b/arch/powerpc/configs/mgcoge_defconfig @@ -10,7 +10,6 @@ CONFIG_BLK_DEV_INITRD=y CONFIG_KALLSYMS_ALL=y # CONFIG_PCSPKR_PLATFORM is not set CONFIG_EMBEDDED=y -CONFIG_SLAB=y CONFIG_PARTITION_ADVANCED=y # CONFIG_PPC_PMAC is not set CONFIG_PPC_82xx=y diff --git a/arch/powerpc/configs/mpc512x_defconfig b/arch/powerpc/configs/mpc512x_defconfig index 10fe061c5e6d..d24457bc5791 100644 --- a/arch/powerpc/configs/mpc512x_defconfig +++ b/arch/powerpc/configs/mpc512x_defconfig @@ -4,7 +4,6 @@ CONFIG_NO_HZ=y CONFIG_LOG_BUF_SHIFT=16 CONFIG_BLK_DEV_INITRD=y # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/powerpc/configs/mpc83xx_defconfig b/arch/powerpc/configs/mpc83xx_defconfig index 8779f03bced1..83c4710017e9 100644 --- a/arch/powerpc/configs/mpc83xx_defconfig +++ b/arch/powerpc/configs/mpc83xx_defconfig @@ -2,7 +2,6 @@ CONFIG_SYSVIPC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/powerpc/configs/mpc85xx_base.config b/arch/powerpc/configs/mpc85xx_base.config index 85907b776908..a1e4d72ed39d 100644 --- a/arch/powerpc/configs/mpc85xx_base.config +++ b/arch/powerpc/configs/mpc85xx_base.config @@ -1,8 +1,5 @@ CONFIG_MATH_EMULATION=y CONFIG_MPC8536_DS=y -CONFIG_MPC8540_ADS=y -CONFIG_MPC8560_ADS=y -CONFIG_MPC85xx_CDS=y CONFIG_MPC85xx_DS=y CONFIG_MPC85xx_MDS=y CONFIG_MPC85xx_RDB=y diff --git a/arch/powerpc/configs/ps3_defconfig b/arch/powerpc/configs/ps3_defconfig index 52a8c5450ecb..ea3ee0719bb2 100644 --- a/arch/powerpc/configs/ps3_defconfig +++ b/arch/powerpc/configs/ps3_defconfig @@ -33,7 +33,6 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=y -CONFIG_SLAB=y # CONFIG_COMPAT_BRK is not set # CONFIG_SPARSEMEM_VMEMMAP is not set # CONFIG_COMPACTION is not set diff --git a/arch/powerpc/configs/wii_defconfig b/arch/powerpc/configs/wii_defconfig index 0ab78c51455d..5017a697b67b 100644 --- a/arch/powerpc/configs/wii_defconfig +++ b/arch/powerpc/configs/wii_defconfig @@ -9,7 +9,6 @@ CONFIG_EXPERT=y # CONFIG_ELF_CORE is not set CONFIG_PERF_EVENTS=y # CONFIG_VM_EVENT_COUNTERS is not set -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_PPC_CHRP is not set diff --git a/arch/powerpc/include/asm/book3s/64/kexec.h b/arch/powerpc/include/asm/book3s/64/kexec.h index d4b9d476ecba..df37a76c1e9f 100644 --- a/arch/powerpc/include/asm/book3s/64/kexec.h +++ b/arch/powerpc/include/asm/book3s/64/kexec.h @@ -21,6 +21,11 @@ static inline void reset_sprs(void) plpar_set_ciabr(0); } + if (cpu_has_feature(CPU_FTR_ARCH_31)) { + mtspr(SPRN_DEXCR, 0); + mtspr(SPRN_HASHKEYR, 0); + } + /* Do we need isync()? We are going via a kexec reset */ isync(); } diff --git a/arch/powerpc/include/asm/book3s/64/kup.h b/arch/powerpc/include/asm/book3s/64/kup.h index 54cf46808157..84c09e546115 100644 --- a/arch/powerpc/include/asm/book3s/64/kup.h +++ b/arch/powerpc/include/asm/book3s/64/kup.h @@ -194,6 +194,7 @@ #else /* !__ASSEMBLY__ */ #include <linux/jump_label.h> +#include <linux/sched.h> DECLARE_STATIC_KEY_FALSE(uaccess_flush_key); diff --git a/arch/powerpc/include/asm/cputable.h b/arch/powerpc/include/asm/cputable.h index 757dbded11dc..443a9d482b15 100644 --- a/arch/powerpc/include/asm/cputable.h +++ b/arch/powerpc/include/asm/cputable.h @@ -192,6 +192,7 @@ static inline void cpu_feature_keys_init(void) { } #define CPU_FTR_P9_RADIX_PREFETCH_BUG LONG_ASM_CONST(0x0002000000000000) #define CPU_FTR_ARCH_31 LONG_ASM_CONST(0x0004000000000000) #define CPU_FTR_DAWR1 LONG_ASM_CONST(0x0008000000000000) +#define CPU_FTR_DEXCR_NPHIE LONG_ASM_CONST(0x0010000000000000) #ifndef __ASSEMBLY__ @@ -451,7 +452,8 @@ static inline void cpu_feature_keys_init(void) { } CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \ CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_ARCH_207S | \ CPU_FTR_ARCH_300 | CPU_FTR_ARCH_31 | \ - CPU_FTR_DAWR | CPU_FTR_DAWR1) + CPU_FTR_DAWR | CPU_FTR_DAWR1 | \ + CPU_FTR_DEXCR_NPHIE) #define CPU_FTRS_CELL (CPU_FTR_LWSYNC | \ CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ CPU_FTR_ALTIVEC_COMP | CPU_FTR_MMCRA | CPU_FTR_SMT | \ diff --git a/arch/powerpc/include/asm/elf.h b/arch/powerpc/include/asm/elf.h index 79f1c480b5eb..a26ca097d032 100644 --- a/arch/powerpc/include/asm/elf.h +++ b/arch/powerpc/include/asm/elf.h @@ -12,8 +12,14 @@ /* * This is used to ensure we don't load something for the wrong architecture. + * 64le only supports ELFv2 64-bit binaries (64be supports v1 and v2). */ +#if defined(CONFIG_PPC64) && defined(CONFIG_CPU_LITTLE_ENDIAN) +#define elf_check_arch(x) (((x)->e_machine == ELF_ARCH) && \ + (((x)->e_flags & 0x3) == 0x2)) +#else #define elf_check_arch(x) ((x)->e_machine == ELF_ARCH) +#endif #define compat_elf_check_arch(x) ((x)->e_machine == EM_PPC) #define CORE_DUMP_USE_REGSET diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h index 79a9c0bb8bba..d16d80ad2ae4 100644 --- a/arch/powerpc/include/asm/kvm_ppc.h +++ b/arch/powerpc/include/asm/kvm_ppc.h @@ -548,12 +548,12 @@ static inline void kvmppc_set_host_ipi(int cpu) * pairs with the barrier in kvmppc_clear_host_ipi() */ smp_mb(); - paca_ptrs[cpu]->kvm_hstate.host_ipi = 1; + WRITE_ONCE(paca_ptrs[cpu]->kvm_hstate.host_ipi, 1); } static inline void kvmppc_clear_host_ipi(int cpu) { - paca_ptrs[cpu]->kvm_hstate.host_ipi = 0; + WRITE_ONCE(paca_ptrs[cpu]->kvm_hstate.host_ipi, 0); /* * order clearing of host_ipi flag vs. processing of IPI messages * diff --git a/arch/powerpc/include/asm/mpc52xx.h b/arch/powerpc/include/asm/mpc52xx.h index 5ea16a71c2f0..01ae6c351e50 100644 --- a/arch/powerpc/include/asm/mpc52xx.h +++ b/arch/powerpc/include/asm/mpc52xx.h @@ -285,47 +285,6 @@ extern int mpc52xx_gpt_start_timer(struct mpc52xx_gpt_priv *gpt, u64 period, extern u64 mpc52xx_gpt_timer_period(struct mpc52xx_gpt_priv *gpt); extern int mpc52xx_gpt_stop_timer(struct mpc52xx_gpt_priv *gpt); -/* mpc52xx_lpbfifo.c */ -#define MPC52XX_LPBFIFO_FLAG_READ (0) -#define MPC52XX_LPBFIFO_FLAG_WRITE (1<<0) -#define MPC52XX_LPBFIFO_FLAG_NO_INCREMENT (1<<1) -#define MPC52XX_LPBFIFO_FLAG_NO_DMA (1<<2) -#define MPC52XX_LPBFIFO_FLAG_POLL_DMA (1<<3) - -struct mpc52xx_lpbfifo_request { - struct list_head list; - - /* localplus bus address */ - unsigned int cs; - size_t offset; - - /* Memory address */ - void *data; - phys_addr_t data_phys; - - /* Details of transfer */ - size_t size; - size_t pos; /* current position of transfer */ - int flags; - int defer_xfer_start; - - /* What to do when finished */ - void (*callback)(struct mpc52xx_lpbfifo_request *); - - void *priv; /* Driver private data */ - - /* statistics */ - int irq_count; - int irq_ticks; - u8 last_byte; - int buffer_not_done_cnt; -}; - -extern int mpc52xx_lpbfifo_submit(struct mpc52xx_lpbfifo_request *req); -extern void mpc52xx_lpbfifo_abort(struct mpc52xx_lpbfifo_request *req); -extern void mpc52xx_lpbfifo_poll(void); -extern int mpc52xx_lpbfifo_start_xfer(struct mpc52xx_lpbfifo_request *req); - /* mpc52xx_pic.c */ extern void mpc52xx_init_irq(void); extern unsigned int mpc52xx_get_irq(void); diff --git a/arch/powerpc/include/asm/opal.h b/arch/powerpc/include/asm/opal.h index 726125a534de..a9b31cc258fc 100644 --- a/arch/powerpc/include/asm/opal.h +++ b/arch/powerpc/include/asm/opal.h @@ -112,9 +112,6 @@ int64_t opal_pci_set_pe(uint64_t phb_id, uint64_t pe_number, uint64_t bus_dev_fu uint8_t pe_action); int64_t opal_pci_set_peltv(uint64_t phb_id, uint32_t parent_pe, uint32_t child_pe, uint8_t state); -int64_t opal_pci_set_mve(uint64_t phb_id, uint32_t mve_number, uint32_t pe_number); -int64_t opal_pci_set_mve_enable(uint64_t phb_id, uint32_t mve_number, - uint32_t state); int64_t opal_pci_get_xive_reissue(uint64_t phb_id, uint32_t xive_number, uint8_t *p_bit, uint8_t *q_bit); int64_t opal_pci_set_xive_reissue(uint64_t phb_id, uint32_t xive_number, diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h index da0377f46597..cb325938766a 100644 --- a/arch/powerpc/include/asm/paca.h +++ b/arch/powerpc/include/asm/paca.h @@ -191,6 +191,7 @@ struct paca_struct { #ifdef CONFIG_PPC_POWERNV /* PowerNV idle fields */ /* PNV_CORE_IDLE_* bits, all siblings work on thread 0 paca */ + unsigned long idle_lock; /* A value of 1 means acquired */ unsigned long idle_state; union { /* P7/P8 specific fields */ diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h index ca5a0da7df4e..ef6972aa33b9 100644 --- a/arch/powerpc/include/asm/ppc-opcode.h +++ b/arch/powerpc/include/asm/ppc-opcode.h @@ -222,6 +222,7 @@ #define OP_31_XOP_STFSX 663 #define OP_31_XOP_STFSUX 695 #define OP_31_XOP_STFDX 727 +#define OP_31_XOP_HASHCHK 754 #define OP_31_XOP_STFDUX 759 #define OP_31_XOP_LHBRX 790 #define OP_31_XOP_LFIWAX 855 diff --git a/arch/powerpc/include/asm/ppc_asm.h b/arch/powerpc/include/asm/ppc_asm.h index 5f05a984b103..e7792aa13510 100644 --- a/arch/powerpc/include/asm/ppc_asm.h +++ b/arch/powerpc/include/asm/ppc_asm.h @@ -406,6 +406,15 @@ n: /* offsets for stack frame layout */ #define LRSAVE 16 +/* + * GCC stack frames follow a different pattern on 32 vs 64. This can be used + * to make asm frames be consistent with C. + */ +#define PPC_CREATE_STACK_FRAME(size) \ + mflr r0; \ + std r0,16(r1); \ + stdu r1,-(size)(r1) + #else /* 32-bit */ #define LOAD_REG_IMMEDIATE(reg, expr) __LOAD_REG_IMMEDIATE_32 reg, expr @@ -422,6 +431,11 @@ n: /* offsets for stack frame layout */ #define LRSAVE 4 +#define PPC_CREATE_STACK_FRAME(size) \ + stwu r1,-(size)(r1); \ + mflr r0; \ + stw r0,(size+4)(r1) + #endif /* various errata or part fixups */ diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h index e96c9b8c2a60..8a6754ffdc7e 100644 --- a/arch/powerpc/include/asm/processor.h +++ b/arch/powerpc/include/asm/processor.h @@ -264,6 +264,7 @@ struct thread_struct { unsigned long mmcr3; unsigned long sier2; unsigned long sier3; + unsigned long hashkeyr; #endif }; diff --git a/arch/powerpc/include/asm/ptrace.h b/arch/powerpc/include/asm/ptrace.h index 0eb90a013346..9db8b16567e2 100644 --- a/arch/powerpc/include/asm/ptrace.h +++ b/arch/powerpc/include/asm/ptrace.h @@ -180,8 +180,8 @@ void do_syscall_trace_leave(struct pt_regs *regs); static inline void set_return_regs_changed(void) { #ifdef CONFIG_PPC_BOOK3S_64 - local_paca->hsrr_valid = 0; - local_paca->srr_valid = 0; + WRITE_ONCE(local_paca->hsrr_valid, 0); + WRITE_ONCE(local_paca->srr_valid, 0); #endif } diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h index 6372e5f55ef0..bb0121222ee3 100644 --- a/arch/powerpc/include/asm/reg.h +++ b/arch/powerpc/include/asm/reg.h @@ -382,7 +382,17 @@ #define SPRN_HIOR 0x137 /* 970 Hypervisor interrupt offset */ #define SPRN_RMOR 0x138 /* Real mode offset register */ #define SPRN_HRMOR 0x139 /* Real mode offset register */ +#define SPRN_HDEXCR_RO 0x1C7 /* Hypervisor DEXCR (non-privileged, readonly) */ +#define SPRN_HASHKEYR 0x1D4 /* Non-privileged hashst/hashchk key register */ +#define SPRN_HDEXCR 0x1D7 /* Hypervisor dynamic execution control register */ +#define SPRN_DEXCR_RO 0x32C /* DEXCR (non-privileged, readonly) */ #define SPRN_ASDR 0x330 /* Access segment descriptor register */ +#define SPRN_DEXCR 0x33C /* Dynamic execution control register */ +#define DEXCR_PR_SBHE 0x80000000UL /* 0: Speculative Branch Hint Enable */ +#define DEXCR_PR_IBRTPD 0x10000000UL /* 3: Indirect Branch Recurrent Target Prediction Disable */ +#define DEXCR_PR_SRAPD 0x08000000UL /* 4: Subroutine Return Address Prediction Disable */ +#define DEXCR_PR_NPHIE 0x04000000UL /* 5: Non-Privileged Hash Instruction Enable */ +#define DEXCR_INIT DEXCR_PR_NPHIE /* Fixed DEXCR value to initialise all CPUs with */ #define SPRN_IC 0x350 /* Virtual Instruction Count */ #define SPRN_VTB 0x351 /* Virtual Time Base */ #define SPRN_LDBAR 0x352 /* LD Base Address Register */ diff --git a/arch/powerpc/include/asm/simple_spinlock.h b/arch/powerpc/include/asm/simple_spinlock.h index 9dcc7e9993b9..4dd12dcb9ef8 100644 --- a/arch/powerpc/include/asm/simple_spinlock.h +++ b/arch/powerpc/include/asm/simple_spinlock.h @@ -15,6 +15,7 @@ * (the type definitions are in asm/simple_spinlock_types.h) */ #include <linux/irqflags.h> +#include <linux/kcsan-checks.h> #include <asm/paravirt.h> #include <asm/paca.h> #include <asm/synch.h> @@ -126,6 +127,7 @@ static inline void arch_spin_lock(arch_spinlock_t *lock) static inline void arch_spin_unlock(arch_spinlock_t *lock) { + kcsan_mb(); __asm__ __volatile__("# arch_spin_unlock\n\t" PPC_RELEASE_BARRIER: : :"memory"); lock->slock = 0; diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h index bf5dde1a4114..bc5d39a835fe 100644 --- a/arch/powerpc/include/asm/thread_info.h +++ b/arch/powerpc/include/asm/thread_info.h @@ -183,9 +183,13 @@ static inline bool test_thread_local_flags(unsigned int flags) #define clear_tsk_compat_task(tsk) do { } while (0) #endif -#if defined(CONFIG_PPC64) +#ifdef CONFIG_PPC64 +#ifdef CONFIG_CPU_BIG_ENDIAN #define is_elf2_task() (test_thread_flag(TIF_ELF2ABI)) #else +#define is_elf2_task() (1) +#endif +#else #define is_elf2_task() (0) #endif diff --git a/arch/powerpc/include/uapi/asm/elf.h b/arch/powerpc/include/uapi/asm/elf.h index dbc4a5b8d02d..a5377f494fa3 100644 --- a/arch/powerpc/include/uapi/asm/elf.h +++ b/arch/powerpc/include/uapi/asm/elf.h @@ -98,6 +98,8 @@ #define ELF_NEBB 3 /* includes ebbrr, ebbhr, bescr */ #define ELF_NPMU 5 /* includes siar, sdar, sier, mmcr2, mmcr0 */ #define ELF_NPKEY 3 /* includes amr, iamr, uamor */ +#define ELF_NDEXCR 2 /* includes dexcr, hdexcr */ +#define ELF_NHASHKEYR 1 /* includes hashkeyr */ typedef unsigned long elf_greg_t64; typedef elf_greg_t64 elf_gregset_t64[ELF_NGREG]; diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile index 9bf2be123093..2919433be355 100644 --- a/arch/powerpc/kernel/Makefile +++ b/arch/powerpc/kernel/Makefile @@ -68,7 +68,7 @@ CFLAGS_REMOVE_syscall.o = -fstack-protector -fstack-protector-strong CFLAGS_syscall.o += -fno-stack-protector #endif -obj-y := cputable.o syscalls.o \ +obj-y := cputable.o syscalls.o switch.o \ irq.o align.o signal_$(BITS).o pmc.o vdso.o \ process.o systbl.o idle.o \ signal.o sysfs.o cacheinfo.o time.o \ @@ -165,9 +165,6 @@ endif obj64-$(CONFIG_PPC_TRANSACTIONAL_MEM) += tm.o -obj-$(CONFIG_PPC64) += $(obj64-y) -obj-$(CONFIG_PPC32) += $(obj32-y) - ifneq ($(CONFIG_XMON)$(CONFIG_KEXEC_CORE)(CONFIG_PPC_BOOK3S),) obj-y += ppc_save_regs.o endif @@ -209,10 +206,13 @@ CFLAGS_paca.o += -fno-stack-protector obj-$(CONFIG_PPC_FPU) += fpu.o obj-$(CONFIG_ALTIVEC) += vector.o -obj-$(CONFIG_PPC64) += entry_64.o -obj-$(CONFIG_PPC_OF_BOOT_TRAMPOLINE) += prom_init.o -extra-$(CONFIG_PPC_OF_BOOT_TRAMPOLINE) += prom_init_check +obj-$(CONFIG_PPC_OF_BOOT_TRAMPOLINE) += prom_init.o +obj64-$(CONFIG_PPC_OF_BOOT_TRAMPOLINE) += prom_entry_64.o +extra-$(CONFIG_PPC_OF_BOOT_TRAMPOLINE) += prom_init_check + +obj-$(CONFIG_PPC64) += $(obj64-y) +obj-$(CONFIG_PPC32) += $(obj32-y) quiet_cmd_prom_init_check = PROMCHK $@ cmd_prom_init_check = $(CONFIG_SHELL) $< "$(NM)" $(obj)/prom_init.o; touch $@ diff --git a/arch/powerpc/kernel/cpu_setup_power.c b/arch/powerpc/kernel/cpu_setup_power.c index 097c033668f0..98bd4e6c1770 100644 --- a/arch/powerpc/kernel/cpu_setup_power.c +++ b/arch/powerpc/kernel/cpu_setup_power.c @@ -126,6 +126,12 @@ static void init_PMU_ISA31(void) mtspr(SPRN_MMCR0, MMCR0_FC | MMCR0_PMCCEXT); } +static void init_DEXCR(void) +{ + mtspr(SPRN_DEXCR, DEXCR_INIT); + mtspr(SPRN_HASHKEYR, 0); +} + /* * Note that we can be called twice of pseudo-PVRs. * The parameter offset is not used. @@ -241,6 +247,7 @@ void __setup_cpu_power10(unsigned long offset, struct cpu_spec *t) init_FSCR_power10(); init_PMU(); init_PMU_ISA31(); + init_DEXCR(); if (!init_hvmode_206(t)) return; @@ -263,6 +270,7 @@ void __restore_cpu_power10(void) init_FSCR_power10(); init_PMU(); init_PMU_ISA31(); + init_DEXCR(); msr = mfmsr(); if (!(msr & MSR_HV)) diff --git a/arch/powerpc/kernel/eeh_pe.c b/arch/powerpc/kernel/eeh_pe.c index d2873d17d2b1..e0ce81279624 100644 --- a/arch/powerpc/kernel/eeh_pe.c +++ b/arch/powerpc/kernel/eeh_pe.c @@ -671,9 +671,8 @@ static void eeh_bridge_check_link(struct eeh_dev *edev) eeh_ops->write_config(edev, cap + PCI_EXP_LNKCTL, 2, val); /* Check link */ - eeh_ops->read_config(edev, cap + PCI_EXP_LNKCAP, 4, &val); - if (!(val & PCI_EXP_LNKCAP_DLLLARC)) { - eeh_edev_dbg(edev, "No link reporting capability (0x%08x) \n", val); + if (!edev->pdev->link_active_reporting) { + eeh_edev_dbg(edev, "No link reporting capability\n"); msleep(1000); return; } diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S index 47f0dd9a45ad..fe27d41f9a3d 100644 --- a/arch/powerpc/kernel/entry_32.S +++ b/arch/powerpc/kernel/entry_32.S @@ -211,64 +211,6 @@ start_kernel_thread: 100: trap EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0 - -/* - * This routine switches between two different tasks. The process - * state of one is saved on its kernel stack. Then the state - * of the other is restored from its kernel stack. The memory - * management hardware is updated to the second process's state. - * Finally, we can return to the second process. - * On entry, r3 points to the THREAD for the current task, r4 - * points to the THREAD for the new task. - * - * This routine is always called with interrupts disabled. - * - * Note: there are two ways to get to the "going out" portion - * of this code; either by coming in via the entry (_switch) - * or via "fork" which must set up an environment equivalent - * to the "_switch" path. If you change this , you'll have to - * change the fork code also. - * - * The code which creates the new task context is in 'copy_thread' - * in arch/ppc/kernel/process.c - */ -_GLOBAL(_switch) - stwu r1,-SWITCH_FRAME_SIZE(r1) - mflr r0 - stw r0,SWITCH_FRAME_SIZE+4(r1) - /* r3-r12 are caller saved -- Cort */ - SAVE_NVGPRS(r1) - stw r0,_NIP(r1) /* Return to switch caller */ - mfcr r10 - stw r10,_CCR(r1) - stw r1,KSP(r3) /* Set old stack pointer */ - -#ifdef CONFIG_SMP - /* We need a sync somewhere here to make sure that if the - * previous task gets rescheduled on another CPU, it sees all - * stores it has performed on this one. - */ - sync -#endif /* CONFIG_SMP */ - - tophys(r0,r4) - mtspr SPRN_SPRG_THREAD,r0 /* Update current THREAD phys addr */ - lwz r1,KSP(r4) /* Load new stack pointer */ - - /* save the old current 'last' for return value */ - mr r3,r2 - addi r2,r4,-THREAD /* Update current */ - - lwz r0,_CCR(r1) - mtcrf 0xFF,r0 - /* r3-r12 are destroyed -- Cort */ - REST_NVGPRS(r1) - - lwz r4,_NIP(r1) /* Return to _switch caller in new task */ - mtlr r4 - addi r1,r1,SWITCH_FRAME_SIZE - blr - .globl fast_exception_return fast_exception_return: #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE)) diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c index e34c72285b4e..c4f6d3c69ba9 100644 --- a/arch/powerpc/kernel/interrupt.c +++ b/arch/powerpc/kernel/interrupt.c @@ -125,7 +125,7 @@ static notrace void check_return_regs_valid(struct pt_regs *regs) case 0x1600: case 0x1800: validp = &local_paca->hsrr_valid; - if (!*validp) + if (!READ_ONCE(*validp)) return; srr0 = mfspr(SPRN_HSRR0); @@ -135,7 +135,7 @@ static notrace void check_return_regs_valid(struct pt_regs *regs) break; default: validp = &local_paca->srr_valid; - if (!*validp) + if (!READ_ONCE(*validp)) return; srr0 = mfspr(SPRN_SRR0); @@ -161,19 +161,17 @@ static notrace void check_return_regs_valid(struct pt_regs *regs) * such things will get caught most of the time, statistically * enough to be able to get a warning out. */ - barrier(); - - if (!*validp) + if (!READ_ONCE(*validp)) return; - if (!warned) { - warned = true; + if (!data_race(warned)) { + data_race(warned = true); printk("%sSRR0 was: %lx should be: %lx\n", h, srr0, regs->nip); printk("%sSRR1 was: %lx should be: %lx\n", h, srr1, regs->msr); show_regs(regs); } - *validp = 0; /* fixup */ + WRITE_ONCE(*validp, 0); /* fixup */ #endif } @@ -368,7 +366,6 @@ void preempt_schedule_irq(void); notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs) { - unsigned long flags; unsigned long ret = 0; unsigned long kuap; bool stack_store = read_thread_flags() & _TIF_EMULATE_STACK_STORE; @@ -392,7 +389,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs) kuap = kuap_get_and_assert_locked(); - local_irq_save(flags); + local_irq_disable(); if (!arch_irq_disabled_regs(regs)) { /* Returning to a kernel context with local irqs enabled. */ diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c index 67f0b01e6ff5..c52449ae6936 100644 --- a/arch/powerpc/kernel/iommu.c +++ b/arch/powerpc/kernel/iommu.c @@ -1090,6 +1090,7 @@ void iommu_tce_kill(struct iommu_table *tbl, } EXPORT_SYMBOL_GPL(iommu_tce_kill); +#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) static int iommu_take_ownership(struct iommu_table *tbl) { unsigned long flags, i, sz = (tbl->it_size + 7) >> 3; @@ -1140,6 +1141,7 @@ static void iommu_release_ownership(struct iommu_table *tbl) spin_unlock(&tbl->pools[i].lock); spin_unlock_irqrestore(&tbl->large_pool.lock, flags); } +#endif int iommu_add_device(struct iommu_table_group *table_group, struct device *dev) { @@ -1171,6 +1173,7 @@ int iommu_add_device(struct iommu_table_group *table_group, struct device *dev) } EXPORT_SYMBOL_GPL(iommu_add_device); +#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) /* * A simple iommu_table_group_ops which only allows reusing the existing * iommu_table. This handles VFIO for POWER7 or the nested KVM. @@ -1398,5 +1401,6 @@ static int __init spapr_tce_setup_phb_iommus_initcall(void) return 0; } postcore_initcall_sync(spapr_tce_setup_phb_iommus_initcall); +#endif #endif /* CONFIG_IOMMU_API */ diff --git a/arch/powerpc/kernel/legacy_serial.c b/arch/powerpc/kernel/legacy_serial.c index c9ad12461d44..6ee65741dbd5 100644 --- a/arch/powerpc/kernel/legacy_serial.c +++ b/arch/powerpc/kernel/legacy_serial.c @@ -508,12 +508,16 @@ static void __init fixup_port_irq(int index, port->irq = virq; -#ifdef CONFIG_SERIAL_8250_FSL - if (of_device_is_compatible(np, "fsl,ns16550")) { - port->handle_irq = fsl8250_handle_irq; - port->has_sysrq = IS_ENABLED(CONFIG_SERIAL_8250_CONSOLE); + if (IS_ENABLED(CONFIG_SERIAL_8250) && + of_device_is_compatible(np, "fsl,ns16550")) { + if (IS_REACHABLE(CONFIG_SERIAL_8250_FSL)) { + port->handle_irq = fsl8250_handle_irq; + port->has_sysrq = IS_ENABLED(CONFIG_SERIAL_8250_CONSOLE); + } else { + pr_warn_once("Not activating Freescale specific workaround for device %pOFP\n", + np); + } } -#endif } static void __init fixup_port_pio(int index, diff --git a/arch/powerpc/kernel/ppc_save_regs.S b/arch/powerpc/kernel/ppc_save_regs.S index 49813f982468..a9b9c32d0c1f 100644 --- a/arch/powerpc/kernel/ppc_save_regs.S +++ b/arch/powerpc/kernel/ppc_save_regs.S @@ -31,10 +31,10 @@ _GLOBAL(ppc_save_regs) lbz r0,PACAIRQSOFTMASK(r13) PPC_STL r0,SOFTE(r3) #endif - /* go up one stack frame for SP */ - PPC_LL r4,0(r1) - PPC_STL r4,GPR1(r3) + /* store current SP */ + PPC_STL r1,GPR1(r3) /* get caller's LR */ + PPC_LL r4,0(r1) PPC_LL r0,LRSAVE(r4) PPC_STL r0,_LINK(r3) mflr r0 diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index 1fefafb2b29b..b68898ac07e1 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -1182,6 +1182,9 @@ static inline void save_sprs(struct thread_struct *t) */ t->tar = mfspr(SPRN_TAR); } + + if (cpu_has_feature(CPU_FTR_DEXCR_NPHIE)) + t->hashkeyr = mfspr(SPRN_HASHKEYR); #endif } @@ -1260,6 +1263,10 @@ static inline void restore_sprs(struct thread_struct *old_thread, if (cpu_has_feature(CPU_FTR_P9_TIDR) && old_thread->tidr != new_thread->tidr) mtspr(SPRN_TIDR, new_thread->tidr); + + if (cpu_has_feature(CPU_FTR_DEXCR_NPHIE) && + old_thread->hashkeyr != new_thread->hashkeyr) + mtspr(SPRN_HASHKEYR, new_thread->hashkeyr); #endif } @@ -1868,6 +1875,10 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) p->thread.tidr = 0; #endif +#ifdef CONFIG_PPC_BOOK3S_64 + if (cpu_has_feature(CPU_FTR_DEXCR_NPHIE)) + p->thread.hashkeyr = current->thread.hashkeyr; +#endif return 0; } @@ -1984,6 +1995,12 @@ void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp) current->thread.tm_tfiar = 0; current->thread.load_tm = 0; #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ +#ifdef CONFIG_PPC_BOOK3S_64 + if (cpu_has_feature(CPU_FTR_DEXCR_NPHIE)) { + current->thread.hashkeyr = get_random_long(); + mtspr(SPRN_HASHKEYR, current->thread.hashkeyr); + } +#endif /* CONFIG_PPC_BOOK3S_64 */ } EXPORT_SYMBOL(start_thread); diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c index 9d9ee4e9e1a1..0b5878c3125b 100644 --- a/arch/powerpc/kernel/prom.c +++ b/arch/powerpc/kernel/prom.c @@ -182,6 +182,7 @@ static struct ibm_feature ibm_pa_features[] __initdata = { .cpu_user_ftrs2 = PPC_FEATURE2_HTM_COMP | PPC_FEATURE2_HTM_NOSC_COMP }, { .pabyte = 64, .pabit = 0, .cpu_features = CPU_FTR_DAWR1 }, + { .pabyte = 68, .pabit = 5, .cpu_features = CPU_FTR_DEXCR_NPHIE }, }; /* diff --git a/arch/powerpc/kernel/prom_entry_64.S b/arch/powerpc/kernel/prom_entry_64.S new file mode 100644 index 000000000000..f1b8793d28c6 --- /dev/null +++ b/arch/powerpc/kernel/prom_entry_64.S @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * PowerPC version + * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) + * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP + * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu> + * Adapted for Power Macintosh by Paul Mackerras. + * Low-level exception handlers and MMU support + * rewritten by Paul Mackerras. + * Copyright (C) 1996 Paul Mackerras. + * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net). + * + * This file contains the 64-bit prom entry code. + */ +#include <asm/asm-offsets.h> +#ifdef CONFIG_PPC_BOOK3S +#include <asm/exception-64s.h> +#else +#include <asm/exception-64e.h> +#endif +#include <asm/ppc_asm.h> + +.section ".text","ax",@progbits + +_GLOBAL(enter_prom) + mflr r0 + std r0,16(r1) + stdu r1,-SWITCH_FRAME_SIZE(r1) /* Save SP and create stack space */ + + /* Because PROM is running in 32b mode, it clobbers the high order half + * of all registers that it saves. We therefore save those registers + * PROM might touch to the stack. (r0, r3-r13 are caller saved) + */ + SAVE_GPR(2, r1) + SAVE_GPR(13, r1) + SAVE_NVGPRS(r1) + mfcr r10 + mfmsr r11 + std r10,_CCR(r1) + std r11,_MSR(r1) + + /* Put PROM address in SRR0 */ + mtsrr0 r4 + + /* Setup our trampoline return addr in LR */ + bcl 20,31,$+4 +0: mflr r4 + addi r4,r4,(1f - 0b) + mtlr r4 + + /* Prepare a 32-bit mode big endian MSR + */ +#ifdef CONFIG_PPC_BOOK3E_64 + rlwinm r11,r11,0,1,31 + mtsrr1 r11 + rfi +#else /* CONFIG_PPC_BOOK3E_64 */ + LOAD_REG_IMMEDIATE(r12, MSR_SF | MSR_LE) + andc r11,r11,r12 + mtsrr1 r11 + RFI_TO_KERNEL +#endif /* CONFIG_PPC_BOOK3E_64 */ + +1: /* Return from OF */ + FIXUP_ENDIAN + + /* Just make sure that r1 top 32 bits didn't get + * corrupt by OF + */ + rldicl r1,r1,0,32 + + /* Restore the MSR (back to 64 bits) */ + ld r0,_MSR(r1) + MTMSRD(r0) + isync + + /* Restore other registers */ + REST_GPR(2, r1) + REST_GPR(13, r1) + REST_NVGPRS(r1) + ld r4,_CCR(r1) + mtcr r4 + + addi r1,r1,SWITCH_FRAME_SIZE + ld r0,16(r1) + mtlr r0 + blr diff --git a/arch/powerpc/kernel/ptrace/ptrace-decl.h b/arch/powerpc/kernel/ptrace/ptrace-decl.h index eafe5f0f6289..4171a5727197 100644 --- a/arch/powerpc/kernel/ptrace/ptrace-decl.h +++ b/arch/powerpc/kernel/ptrace/ptrace-decl.h @@ -1,5 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ +#include <linux/regset.h> + /* * Set of msr bits that gdb can change on behalf of a process. */ @@ -55,6 +57,10 @@ enum powerpc_regset { REGSET_TAR, /* TAR register */ REGSET_EBB, /* EBB registers */ REGSET_PMR, /* Performance Monitor Registers */ + REGSET_DEXCR, /* DEXCR registers */ +#ifdef CONFIG_CHECKPOINT_RESTORE + REGSET_HASHKEYR, /* HASHKEYR register */ +#endif #endif #ifdef CONFIG_PPC_MEM_KEYS REGSET_PKEY, /* AMR register */ diff --git a/arch/powerpc/kernel/ptrace/ptrace-view.c b/arch/powerpc/kernel/ptrace/ptrace-view.c index 5fff0d04b23f..3910cd7bb2d9 100644 --- a/arch/powerpc/kernel/ptrace/ptrace-view.c +++ b/arch/powerpc/kernel/ptrace/ptrace-view.c @@ -454,7 +454,65 @@ static int pmu_set(struct task_struct *target, const struct user_regset *regset, 5 * sizeof(unsigned long)); return ret; } -#endif + +static int dexcr_active(struct task_struct *target, const struct user_regset *regset) +{ + if (!cpu_has_feature(CPU_FTR_ARCH_31)) + return -ENODEV; + + return regset->n; +} + +static int dexcr_get(struct task_struct *target, const struct user_regset *regset, + struct membuf to) +{ + if (!cpu_has_feature(CPU_FTR_ARCH_31)) + return -ENODEV; + + /* + * The DEXCR is currently static across all CPUs, so we don't + * store the target's value anywhere, but the static value + * will also be correct. + */ + membuf_store(&to, (u64)lower_32_bits(DEXCR_INIT)); + + /* + * Technically the HDEXCR is per-cpu, but a hypervisor can't reasonably + * change it between CPUs of the same guest. + */ + return membuf_store(&to, (u64)lower_32_bits(mfspr(SPRN_HDEXCR_RO))); +} + +#ifdef CONFIG_CHECKPOINT_RESTORE +static int hashkeyr_active(struct task_struct *target, const struct user_regset *regset) +{ + if (!cpu_has_feature(CPU_FTR_ARCH_31)) + return -ENODEV; + + return regset->n; +} + +static int hashkeyr_get(struct task_struct *target, const struct user_regset *regset, + struct membuf to) +{ + if (!cpu_has_feature(CPU_FTR_ARCH_31)) + return -ENODEV; + + return membuf_store(&to, target->thread.hashkeyr); +} + +static int hashkeyr_set(struct task_struct *target, const struct user_regset *regset, + unsigned int pos, unsigned int count, const void *kbuf, + const void __user *ubuf) +{ + if (!cpu_has_feature(CPU_FTR_ARCH_31)) + return -ENODEV; + + return user_regset_copyin(&pos, &count, &kbuf, &ubuf, &target->thread.hashkeyr, + 0, sizeof(unsigned long)); +} +#endif /* CONFIG_CHECKPOINT_RESTORE */ +#endif /* CONFIG_PPC_BOOK3S_64 */ #ifdef CONFIG_PPC_MEM_KEYS static int pkey_active(struct task_struct *target, const struct user_regset *regset) @@ -615,6 +673,18 @@ static const struct user_regset native_regsets[] = { .size = sizeof(u64), .align = sizeof(u64), .active = pmu_active, .regset_get = pmu_get, .set = pmu_set }, + [REGSET_DEXCR] = { + .core_note_type = NT_PPC_DEXCR, .n = ELF_NDEXCR, + .size = sizeof(u64), .align = sizeof(u64), + .active = dexcr_active, .regset_get = dexcr_get + }, +#ifdef CONFIG_CHECKPOINT_RESTORE + [REGSET_HASHKEYR] = { + .core_note_type = NT_PPC_HASHKEYR, .n = ELF_NHASHKEYR, + .size = sizeof(u64), .align = sizeof(u64), + .active = hashkeyr_active, .regset_get = hashkeyr_get, .set = hashkeyr_set + }, +#endif #endif #ifdef CONFIG_PPC_MEM_KEYS [REGSET_PKEY] = { diff --git a/arch/powerpc/kernel/signal_32.c b/arch/powerpc/kernel/signal_32.c index c114c7f25645..7a718ed32b27 100644 --- a/arch/powerpc/kernel/signal_32.c +++ b/arch/powerpc/kernel/signal_32.c @@ -264,8 +264,9 @@ static void prepare_save_user_regs(int ctx_has_vsx_region) #endif } -static int __unsafe_save_user_regs(struct pt_regs *regs, struct mcontext __user *frame, - struct mcontext __user *tm_frame, int ctx_has_vsx_region) +static __always_inline int +__unsafe_save_user_regs(struct pt_regs *regs, struct mcontext __user *frame, + struct mcontext __user *tm_frame, int ctx_has_vsx_region) { unsigned long msr = regs->msr; @@ -364,8 +365,9 @@ static void prepare_save_tm_user_regs(void) current->thread.ckvrsave = mfspr(SPRN_VRSAVE); } -static int save_tm_user_regs_unsafe(struct pt_regs *regs, struct mcontext __user *frame, - struct mcontext __user *tm_frame, unsigned long msr) +static __always_inline int +save_tm_user_regs_unsafe(struct pt_regs *regs, struct mcontext __user *frame, + struct mcontext __user *tm_frame, unsigned long msr) { /* Save both sets of general registers */ unsafe_save_general_regs(¤t->thread.ckpt_regs, frame, failed); @@ -444,8 +446,9 @@ failed: #else static void prepare_save_tm_user_regs(void) { } -static int save_tm_user_regs_unsafe(struct pt_regs *regs, struct mcontext __user *frame, - struct mcontext __user *tm_frame, unsigned long msr) +static __always_inline int +save_tm_user_regs_unsafe(struct pt_regs *regs, struct mcontext __user *frame, + struct mcontext __user *tm_frame, unsigned long msr) { return 0; } diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index e95660e69414..fbbb695bae3d 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -291,7 +291,7 @@ void smp_muxed_ipi_set_message(int cpu, int msg) * Order previous accesses before accesses in the IPI handler. */ smp_mb(); - message[msg] = 1; + WRITE_ONCE(message[msg], 1); } void smp_muxed_ipi_message_pass(int cpu, int msg) @@ -350,7 +350,7 @@ irqreturn_t smp_ipi_demux_relaxed(void) if (all & IPI_MESSAGE(PPC_MSG_NMI_IPI)) nmi_ipi_action(0, NULL); #endif - } while (info->messages); + } while (READ_ONCE(info->messages)); return IRQ_HANDLED; } diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/switch.S index 1bf1121e17f1..608c0ce7cec6 100644 --- a/arch/powerpc/kernel/entry_64.S +++ b/arch/powerpc/kernel/switch.S @@ -1,56 +1,21 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * PowerPC version - * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) - * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP - * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu> - * Adapted for Power Macintosh by Paul Mackerras. - * Low-level exception handlers and MMU support - * rewritten by Paul Mackerras. - * Copyright (C) 1996 Paul Mackerras. - * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net). - * - * This file contains the system call entry code, context switch - * code, and exception/interrupt return code for PowerPC. - */ - #include <linux/objtool.h> -#include <linux/errno.h> -#include <linux/err.h> -#include <asm/cache.h> -#include <asm/unistd.h> -#include <asm/processor.h> -#include <asm/page.h> -#include <asm/mmu.h> -#include <asm/thread_info.h> +#include <asm/asm-offsets.h> #include <asm/code-patching-asm.h> +#include <asm/mmu.h> #include <asm/ppc_asm.h> -#include <asm/asm-offsets.h> -#include <asm/cputable.h> -#include <asm/firmware.h> -#include <asm/bug.h> -#include <asm/ptrace.h> -#include <asm/irqflags.h> -#include <asm/hw_irq.h> -#include <asm/context_tracking.h> -#include <asm/ppc-opcode.h> -#include <asm/barrier.h> -#include <asm/export.h> -#include <asm/asm-compat.h> -#ifdef CONFIG_PPC_BOOK3S -#include <asm/exception-64s.h> -#else -#include <asm/exception-64e.h> -#endif -#include <asm/feature-fixups.h> #include <asm/kup.h> +#include <asm/thread_info.h> -/* - * System calls. - */ - .section ".text" +.section ".text","ax",@progbits #ifdef CONFIG_PPC_BOOK3S_64 +/* + * Cancel all explict user streams as they will have no use after context + * switch and will stop the HW from creating streams itself + */ +#define STOP_STREAMS \ + DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r6) #define FLUSH_COUNT_CACHE \ 1: nop; \ @@ -105,87 +70,13 @@ flush_branch_caches: .endr blr -#else -#define FLUSH_COUNT_CACHE -#endif /* CONFIG_PPC_BOOK3S_64 */ +#ifdef CONFIG_PPC_64S_HASH_MMU +.balign 32 /* - * This routine switches between two different tasks. The process - * state of one is saved on its kernel stack. Then the state - * of the other is restored from its kernel stack. The memory - * management hardware is updated to the second process's state. - * Finally, we can return to the second process, via interrupt_return. - * On entry, r3 points to the THREAD for the current task, r4 - * points to the THREAD for the new task. - * - * Note: there are two ways to get to the "going out" portion - * of this code; either by coming in via the entry (_switch) - * or via "fork" which must set up an environment equivalent - * to the "_switch" path. If you change this you'll have to change - * the fork code also. - * - * The code which creates the new task context is in 'copy_thread' - * in arch/powerpc/kernel/process.c + * New stack pointer in r8, old stack pointer in r1, must not clobber r3 */ - .align 7 -_GLOBAL(_switch) - mflr r0 - std r0,16(r1) - stdu r1,-SWITCH_FRAME_SIZE(r1) - /* r3-r13 are caller saved -- Cort */ - SAVE_NVGPRS(r1) - std r0,_NIP(r1) /* Return to switch caller */ - mfcr r23 - std r23,_CCR(r1) - std r1,KSP(r3) /* Set old stack pointer */ - - kuap_check_amr r9, r10 - - FLUSH_COUNT_CACHE /* Clobbers r9, ctr */ - - /* - * On SMP kernels, care must be taken because a task may be - * scheduled off CPUx and on to CPUy. Memory ordering must be - * considered. - * - * Cacheable stores on CPUx will be visible when the task is - * scheduled on CPUy by virtue of the core scheduler barriers - * (see "Notes on Program-Order guarantees on SMP systems." in - * kernel/sched/core.c). - * - * Uncacheable stores in the case of involuntary preemption must - * be taken care of. The smp_mb__after_spinlock() in __schedule() - * is implemented as hwsync on powerpc, which orders MMIO too. So - * long as there is an hwsync in the context switch path, it will - * be executed on the source CPU after the task has performed - * all MMIO ops on that CPU, and on the destination CPU before the - * task performs any MMIO ops there. - */ - - /* - * The kernel context switch path must contain a spin_lock, - * which contains larx/stcx, which will clear any reservation - * of the task being switched. - */ -#ifdef CONFIG_PPC_BOOK3S -/* Cancel all explict user streams as they will have no use after context - * switch and will stop the HW from creating streams itself - */ - DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r6) -#endif - - addi r6,r4,-THREAD /* Convert THREAD to 'current' */ - std r6,PACACURRENT(r13) /* Set new 'current' */ -#if defined(CONFIG_STACKPROTECTOR) - ld r6, TASK_CANARY(r6) - std r6, PACA_CANARY(r13) -#endif - - ld r8,KSP(r4) /* new stack pointer */ -#ifdef CONFIG_PPC_64S_HASH_MMU -BEGIN_MMU_FTR_SECTION - b 2f -END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) +pin_stack_slb: BEGIN_FTR_SECTION clrrdi r6,r8,28 /* get its ESID */ clrrdi r9,r1,28 /* get current sp ESID */ @@ -233,15 +124,57 @@ BEGIN_FTR_SECTION END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) slbmte r7,r0 isync -2: +2: blr + .size pin_stack_slb,.-pin_stack_slb #endif /* CONFIG_PPC_64S_HASH_MMU */ - clrrdi r7, r8, THREAD_SHIFT /* base of new stack */ - /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE - because we don't need to leave the 288-byte ABI gap at the - top of the kernel stack. */ +#else +#define STOP_STREAMS +#define FLUSH_COUNT_CACHE +#endif /* CONFIG_PPC_BOOK3S_64 */ + +/* + * do_switch_32/64 have the same calling convention as _switch, i.e., r3,r4 + * are prev and next thread_struct *, and returns prev task_struct * in r3. + + * This switches the stack, current, and does other task switch housekeeping. + */ +.macro do_switch_32 + tophys(r0,r4) + mtspr SPRN_SPRG_THREAD,r0 /* Update current THREAD phys addr */ + lwz r1,KSP(r4) /* Load new stack pointer */ + + /* save the old current 'last' for return value */ + mr r3,r2 + addi r2,r4,-THREAD /* Update current */ +.endm + +.macro do_switch_64 + ld r8,KSP(r4) /* Load new stack pointer */ + + kuap_check_amr r9, r10 + + FLUSH_COUNT_CACHE /* Clobbers r9, ctr */ + + STOP_STREAMS /* Clobbers r6 */ + + addi r3,r3,-THREAD /* old thread -> task_struct for return value */ + addi r6,r4,-THREAD /* new thread -> task_struct */ + std r6,PACACURRENT(r13) /* Set new task_struct to 'current' */ +#if defined(CONFIG_STACKPROTECTOR) + ld r6, TASK_CANARY(r6) + std r6, PACA_CANARY(r13) +#endif + /* Set new PACAKSAVE */ + clrrdi r7,r8,THREAD_SHIFT /* base of new stack */ addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE + std r7,PACAKSAVE(r13) +#ifdef CONFIG_PPC_64S_HASH_MMU +BEGIN_MMU_FTR_SECTION + bl pin_stack_slb +END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX) +#endif /* * PMU interrupts in radix may come in here. They will use r1, not * PACAKSAVE, so this stack switch will not cause a problem. They @@ -251,81 +184,75 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) * active on the new CPU, will order those stores. */ mr r1,r8 /* start using new stack pointer */ - std r7,PACAKSAVE(r13) +.endm + +/* + * This routine switches between two different tasks. The process + * state of one is saved on its kernel stack. Then the state + * of the other is restored from its kernel stack. The memory + * management hardware is updated to the second process's state. + * Finally, we can return to the second process. + * On entry, r3 points to the THREAD for the current task, r4 + * points to the THREAD for the new task. + * + * This routine is always called with interrupts disabled. + * + * Note: there are two ways to get to the "going out" portion + * of this code; either by coming in via the entry (_switch) + * or via "fork" which must set up an environment equivalent + * to the "_switch" path. If you change this , you'll have to + * change the fork code also. + * + * The code which creates the new task context is in 'copy_thread' + * in arch/ppc/kernel/process.c + * + * Note: this uses SWITCH_FRAME_SIZE rather than USER_INT_FRAME_SIZE + * because we don't need to leave the redzone ABI gap at the top of + * the kernel stack. + */ +_GLOBAL(_switch) + PPC_CREATE_STACK_FRAME(SWITCH_FRAME_SIZE) + PPC_STL r1,KSP(r3) /* Set old stack pointer */ + SAVE_NVGPRS(r1) /* volatiles are caller-saved -- Cort */ + PPC_STL r0,_NIP(r1) /* Return to switch caller */ + mfcr r0 + stw r0,_CCR(r1) - ld r6,_CCR(r1) - mtcrf 0xFF,r6 + /* + * On SMP kernels, care must be taken because a task may be + * scheduled off CPUx and on to CPUy. Memory ordering must be + * considered. + * + * Cacheable stores on CPUx will be visible when the task is + * scheduled on CPUy by virtue of the core scheduler barriers + * (see "Notes on Program-Order guarantees on SMP systems." in + * kernel/sched/core.c). + * + * Uncacheable stores in the case of involuntary preemption must + * be taken care of. The smp_mb__after_spinlock() in __schedule() + * is implemented as hwsync on powerpc, which orders MMIO too. So + * long as there is an hwsync in the context switch path, it will + * be executed on the source CPU after the task has performed + * all MMIO ops on that CPU, and on the destination CPU before the + * task performs any MMIO ops there. + */ + + /* + * The kernel context switch path must contain a spin_lock, + * which contains larx/stcx, which will clear any reservation + * of the task being switched. + */ - /* r3-r13 are destroyed -- Cort */ - REST_NVGPRS(r1) +#ifdef CONFIG_PPC32 + do_switch_32 +#else + do_switch_64 +#endif - /* convert old thread to its task_struct for return value */ - addi r3,r3,-THREAD - ld r7,_NIP(r1) /* Return to _switch caller in new task */ - mtlr r7 + lwz r0,_CCR(r1) + mtcrf 0xFF,r0 + REST_NVGPRS(r1) /* volatiles are destroyed -- Cort */ + PPC_LL r0,_NIP(r1) /* Return to _switch caller in new task */ + mtlr r0 addi r1,r1,SWITCH_FRAME_SIZE blr - -_GLOBAL(enter_prom) - mflr r0 - std r0,16(r1) - stdu r1,-SWITCH_FRAME_SIZE(r1) /* Save SP and create stack space */ - - /* Because PROM is running in 32b mode, it clobbers the high order half - * of all registers that it saves. We therefore save those registers - * PROM might touch to the stack. (r0, r3-r13 are caller saved) - */ - SAVE_GPR(2, r1) - SAVE_GPR(13, r1) - SAVE_NVGPRS(r1) - mfcr r10 - mfmsr r11 - std r10,_CCR(r1) - std r11,_MSR(r1) - - /* Put PROM address in SRR0 */ - mtsrr0 r4 - - /* Setup our trampoline return addr in LR */ - bcl 20,31,$+4 -0: mflr r4 - addi r4,r4,(1f - 0b) - mtlr r4 - - /* Prepare a 32-bit mode big endian MSR - */ -#ifdef CONFIG_PPC_BOOK3E_64 - rlwinm r11,r11,0,1,31 - mtsrr1 r11 - rfi -#else /* CONFIG_PPC_BOOK3E_64 */ - LOAD_REG_IMMEDIATE(r12, MSR_SF | MSR_LE) - andc r11,r11,r12 - mtsrr1 r11 - RFI_TO_KERNEL -#endif /* CONFIG_PPC_BOOK3E_64 */ - -1: /* Return from OF */ - FIXUP_ENDIAN - - /* Just make sure that r1 top 32 bits didn't get - * corrupt by OF - */ - rldicl r1,r1,0,32 - - /* Restore the MSR (back to 64 bits) */ - ld r0,_MSR(r1) - MTMSRD(r0) - isync - - /* Restore other registers */ - REST_GPR(2, r1) - REST_GPR(13, r1) - REST_NVGPRS(r1) - ld r4,_CCR(r1) - mtcr r4 - - addi r1,r1,SWITCH_FRAME_SIZE - ld r0,16(r1) - mtlr r0 - blr diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c index 9bdd79aa51cf..e59ec6d32d37 100644 --- a/arch/powerpc/kernel/traps.c +++ b/arch/powerpc/kernel/traps.c @@ -1516,6 +1516,22 @@ static void do_program_check(struct pt_regs *regs) return; } } + + if (cpu_has_feature(CPU_FTR_DEXCR_NPHIE) && user_mode(regs)) { + ppc_inst_t insn; + + if (get_user_instr(insn, (void __user *)regs->nip)) { + _exception(SIGSEGV, regs, SEGV_MAPERR, regs->nip); + return; + } + + if (ppc_inst_primary_opcode(insn) == 31 && + get_xop(ppc_inst_val(insn)) == OP_31_XOP_HASHCHK) { + _exception(SIGILL, regs, ILL_ILLOPN, regs->nip); + return; + } + } + _exception(SIGTRAP, regs, TRAP_BRKPT, regs->nip); return; } diff --git a/arch/powerpc/kernel/vdso/Makefile b/arch/powerpc/kernel/vdso/Makefile index 4c3f34485f08..0c7d82c270c3 100644 --- a/arch/powerpc/kernel/vdso/Makefile +++ b/arch/powerpc/kernel/vdso/Makefile @@ -54,8 +54,10 @@ KASAN_SANITIZE := n KCSAN_SANITIZE := n ccflags-y := -fno-common -fno-builtin -ldflags-y := -Wl,--hash-style=both -nostdlib -shared -z noexecstack +ldflags-y := -Wl,--hash-style=both -nostdlib -shared -z noexecstack $(CLANG_FLAGS) ldflags-$(CONFIG_LD_IS_LLD) += $(call cc-option,--ld-path=$(LD),-fuse-ld=lld) +ldflags-$(CONFIG_LD_ORPHAN_WARN) += -Wl,--orphan-handling=$(CONFIG_LD_ORPHAN_WARN_LEVEL) + # Filter flags that clang will warn are unused for linking ldflags-y += $(filter-out $(CC_AUTO_VAR_INIT_ZERO_ENABLER) $(CC_FLAGS_FTRACE) -Wa$(comma)%, $(KBUILD_CFLAGS)) diff --git a/arch/powerpc/kernel/vdso/vdso32.lds.S b/arch/powerpc/kernel/vdso/vdso32.lds.S index bc0be274a9ac..426e1ccc6971 100644 --- a/arch/powerpc/kernel/vdso/vdso32.lds.S +++ b/arch/powerpc/kernel/vdso/vdso32.lds.S @@ -83,9 +83,11 @@ SECTIONS /DISCARD/ : { *(.note.GNU-stack) + *(*.EMB.apuinfo) + *(.branch_lt) *(.data .data.* .gnu.linkonce.d.* .sdata*) *(.bss .sbss .dynbss .dynsbss) - *(.got1) + *(.got1 .glink .iplt .rela*) } } diff --git a/arch/powerpc/kernel/vdso/vdso64.lds.S b/arch/powerpc/kernel/vdso/vdso64.lds.S index 744ae5363e6c..bda6c8cdd459 100644 --- a/arch/powerpc/kernel/vdso/vdso64.lds.S +++ b/arch/powerpc/kernel/vdso/vdso64.lds.S @@ -32,7 +32,7 @@ SECTIONS . = ALIGN(16); .text : { *(.text .stub .text.* .gnu.linkonce.t.* __ftr_alt_*) - *(.sfpr .glink) + *(.sfpr) } :text PROVIDE(__etext = .); PROVIDE(_etext = .); @@ -81,10 +81,12 @@ SECTIONS /DISCARD/ : { *(.note.GNU-stack) + *(*.EMB.apuinfo) *(.branch_lt) *(.data .data.* .gnu.linkonce.d.* .sdata*) *(.bss .sbss .dynbss .dynsbss) *(.opd) + *(.glink .iplt .plt .rela*) } } diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c index da85f046377a..0f5b021fa559 100644 --- a/arch/powerpc/kvm/book3s_hv_builtin.c +++ b/arch/powerpc/kvm/book3s_hv_builtin.c @@ -406,7 +406,7 @@ static long kvmppc_read_one_intr(bool *again) return 1; /* see if a host IPI is pending */ - host_ipi = local_paca->kvm_hstate.host_ipi; + host_ipi = READ_ONCE(local_paca->kvm_hstate.host_ipi); if (host_ipi) return 1; @@ -466,7 +466,7 @@ static long kvmppc_read_one_intr(bool *again) * meantime. If it's clear, we bounce the interrupt to the * guest */ - host_ipi = local_paca->kvm_hstate.host_ipi; + host_ipi = READ_ONCE(local_paca->kvm_hstate.host_ipi); if (unlikely(host_ipi != 0)) { /* We raced with the host, * we need to resend that IPI, bummer diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile index c4db459d304a..9aa8286c9687 100644 --- a/arch/powerpc/lib/Makefile +++ b/arch/powerpc/lib/Makefile @@ -44,7 +44,7 @@ obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o # 64-bit linker creates .sfpr on demand for final link (vmlinux), # so it is only needed for modules, and only for older linkers which # do not support --save-restore-funcs -ifeq ($(call ld-ifversion, -lt, 22500, y),y) +ifndef CONFIG_LD_IS_BFD extra-$(CONFIG_PPC64) += crtsavres.o endif diff --git a/arch/powerpc/lib/qspinlock.c b/arch/powerpc/lib/qspinlock.c index e4bd145255d0..253620979d0c 100644 --- a/arch/powerpc/lib/qspinlock.c +++ b/arch/powerpc/lib/qspinlock.c @@ -161,6 +161,8 @@ static __always_inline u32 publish_tail_cpu(struct qspinlock *lock, u32 tail) { u32 prev, tmp; + kcsan_release(); + asm volatile( "\t" PPC_RELEASE_BARRIER " \n" "1: lwarx %0,0,%2 # publish_tail_cpu \n" @@ -435,7 +437,7 @@ yield_prev: smp_rmb(); /* See __yield_to_locked_owner comment */ - if (!node->locked) { + if (!READ_ONCE(node->locked)) { yield_to_preempted(prev_cpu, yield_count); spin_begin(); return preempted; @@ -570,6 +572,11 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b tail = encode_tail_cpu(node->cpu); + /* + * Assign all attributes of a node before it can be published. + * Issues an lwsync, serving as a release barrier, as well as a + * compiler barrier. + */ old = publish_tail_cpu(lock, tail); /* @@ -584,7 +591,7 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b /* Wait for mcs node lock to be released */ spin_begin(); - while (!node->locked) { + while (!READ_ONCE(node->locked)) { spec_barrier(); if (yield_to_prev(lock, node, old, paravirt)) diff --git a/arch/powerpc/mm/book3s32/hash_low.S b/arch/powerpc/mm/book3s32/hash_low.S index 6925ce998557..a5a21d444e72 100644 --- a/arch/powerpc/mm/book3s32/hash_low.S +++ b/arch/powerpc/mm/book3s32/hash_low.S @@ -199,12 +199,12 @@ _GLOBAL(add_hash_page) lis r6, (mmu_hash_lock - PAGE_OFFSET)@ha addi r6, r6, (mmu_hash_lock - PAGE_OFFSET)@l 10: lwarx r0,0,r6 /* take the mmu_hash_lock */ - cmpi 0,r0,0 + cmpwi 0,r0,0 bne- 11f stwcx. r8,0,r6 beq+ 12f 11: lwz r0,0(r6) - cmpi 0,r0,0 + cmpwi 0,r0,0 beq 10b b 11b 12: isync @@ -512,12 +512,12 @@ _GLOBAL(flush_hash_pages) lwz r8, TASK_CPU(r8) oris r8,r8,9 10: lwarx r0,0,r9 - cmpi 0,r0,0 + cmpwi 0,r0,0 bne- 11f stwcx. r8,0,r9 beq+ 12f 11: lwz r0,0(r9) - cmpi 0,r0,0 + cmpwi 0,r0,0 beq 10b b 11b 12: isync diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c index 2297aa764ecd..e7ea492ac510 100644 --- a/arch/powerpc/mm/book3s64/radix_pgtable.c +++ b/arch/powerpc/mm/book3s64/radix_pgtable.c @@ -745,9 +745,9 @@ static void free_pud_table(pud_t *pud_start, p4d_t *p4d) } static void remove_pte_table(pte_t *pte_start, unsigned long addr, - unsigned long end) + unsigned long end, bool direct) { - unsigned long next; + unsigned long next, pages = 0; pte_t *pte; pte = pte_start + pte_index(addr); @@ -769,13 +769,16 @@ static void remove_pte_table(pte_t *pte_start, unsigned long addr, } pte_clear(&init_mm, addr, pte); + pages++; } + if (direct) + update_page_count(mmu_virtual_psize, -pages); } static void __meminit remove_pmd_table(pmd_t *pmd_start, unsigned long addr, - unsigned long end) + unsigned long end, bool direct) { - unsigned long next; + unsigned long next, pages = 0; pte_t *pte_base; pmd_t *pmd; @@ -793,19 +796,22 @@ static void __meminit remove_pmd_table(pmd_t *pmd_start, unsigned long addr, continue; } pte_clear(&init_mm, addr, (pte_t *)pmd); + pages++; continue; } pte_base = (pte_t *)pmd_page_vaddr(*pmd); - remove_pte_table(pte_base, addr, next); + remove_pte_table(pte_base, addr, next, direct); free_pte_table(pte_base, pmd); } + if (direct) + update_page_count(MMU_PAGE_2M, -pages); } static void __meminit remove_pud_table(pud_t *pud_start, unsigned long addr, - unsigned long end) + unsigned long end, bool direct) { - unsigned long next; + unsigned long next, pages = 0; pmd_t *pmd_base; pud_t *pud; @@ -823,16 +829,20 @@ static void __meminit remove_pud_table(pud_t *pud_start, unsigned long addr, continue; } pte_clear(&init_mm, addr, (pte_t *)pud); + pages++; continue; } pmd_base = pud_pgtable(*pud); - remove_pmd_table(pmd_base, addr, next); + remove_pmd_table(pmd_base, addr, next, direct); free_pmd_table(pmd_base, pud); } + if (direct) + update_page_count(MMU_PAGE_1G, -pages); } -static void __meminit remove_pagetable(unsigned long start, unsigned long end) +static void __meminit remove_pagetable(unsigned long start, unsigned long end, + bool direct) { unsigned long addr, next; pud_t *pud_base; @@ -861,7 +871,7 @@ static void __meminit remove_pagetable(unsigned long start, unsigned long end) } pud_base = p4d_pgtable(*p4d); - remove_pud_table(pud_base, addr, next); + remove_pud_table(pud_base, addr, next, direct); free_pud_table(pud_base, p4d); } @@ -884,7 +894,7 @@ int __meminit radix__create_section_mapping(unsigned long start, int __meminit radix__remove_section_mapping(unsigned long start, unsigned long end) { - remove_pagetable(start, end); + remove_pagetable(start, end, true); return 0; } #endif /* CONFIG_MEMORY_HOTPLUG */ @@ -902,7 +912,6 @@ int __meminit radix__vmemmap_create_mapping(unsigned long start, unsigned long phys) { /* Create a PTE encoding */ - unsigned long flags = _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_KERNEL_RW; int nid = early_pfn_to_nid(phys >> PAGE_SHIFT); int ret; @@ -911,7 +920,7 @@ int __meminit radix__vmemmap_create_mapping(unsigned long start, return -1; } - ret = __map_kernel_page_nid(start, phys, __pgprot(flags), page_size, nid); + ret = __map_kernel_page_nid(start, phys, PAGE_KERNEL, page_size, nid); BUG_ON(ret); return 0; @@ -920,7 +929,7 @@ int __meminit radix__vmemmap_create_mapping(unsigned long start, #ifdef CONFIG_MEMORY_HOTPLUG void __meminit radix__vmemmap_remove_mapping(unsigned long start, unsigned long page_size) { - remove_pagetable(start, start + page_size); + remove_pagetable(start, start + page_size, false); } #endif #endif @@ -952,7 +961,7 @@ unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long add assert_spin_locked(pmd_lockptr(mm, pmdp)); #endif - old = radix__pte_update(mm, addr, (pte_t *)pmdp, clr, set, 1); + old = radix__pte_update(mm, addr, pmdp_ptep(pmdp), clr, set, 1); trace_hugepage_update(addr, old, clr, set); return old; diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c index 05b0d584e50b..fe1b83020e0d 100644 --- a/arch/powerpc/mm/init_64.c +++ b/arch/powerpc/mm/init_64.c @@ -189,7 +189,7 @@ static bool altmap_cross_boundary(struct vmem_altmap *altmap, unsigned long star unsigned long nr_pfn = page_size / sizeof(struct page); unsigned long start_pfn = page_to_pfn((struct page *)start); - if ((start_pfn + nr_pfn) > altmap->end_pfn) + if ((start_pfn + nr_pfn - 1) > altmap->end_pfn) return true; if (start_pfn < altmap->base_pfn) diff --git a/arch/powerpc/platforms/44x/ppc476.c b/arch/powerpc/platforms/44x/ppc476.c index fbc6edad481f..164cbcd4588e 100644 --- a/arch/powerpc/platforms/44x/ppc476.c +++ b/arch/powerpc/platforms/44x/ppc476.c @@ -103,7 +103,7 @@ static struct i2c_driver avr_driver = { .driver = { .name = "akebono-avr", }, - .probe_new = avr_probe, + .probe = avr_probe, .id_table = avr_id, }; diff --git a/arch/powerpc/platforms/512x/mpc512x_lpbfifo.c b/arch/powerpc/platforms/512x/mpc512x_lpbfifo.c index 04bf6ecf7d55..1bfb29574caa 100644 --- a/arch/powerpc/platforms/512x/mpc512x_lpbfifo.c +++ b/arch/powerpc/platforms/512x/mpc512x_lpbfifo.c @@ -373,50 +373,32 @@ static int get_cs_ranges(struct device *dev) { int ret = -ENODEV; struct device_node *lb_node; - const u32 *addr_cells_p; - const u32 *size_cells_p; - int proplen; - size_t i; + size_t i = 0; + struct of_range_parser parser; + struct of_range range; lb_node = of_find_compatible_node(NULL, NULL, "fsl,mpc5121-localbus"); if (!lb_node) return ret; - /* - * The node defined as compatible with 'fsl,mpc5121-localbus' - * should have two address cells and one size cell. - * Every item of its ranges property should consist of: - * - the first address cell which is the chipselect number; - * - the second address cell which is the offset in the chipselect, - * must be zero. - * - CPU address of the beginning of an access window; - * - the only size cell which is the size of an access window. - */ - addr_cells_p = of_get_property(lb_node, "#address-cells", NULL); - size_cells_p = of_get_property(lb_node, "#size-cells", NULL); - if (addr_cells_p == NULL || *addr_cells_p != 2 || - size_cells_p == NULL || *size_cells_p != 1) { - goto end; - } - - proplen = of_property_count_u32_elems(lb_node, "ranges"); - if (proplen <= 0 || proplen % 4 != 0) - goto end; + of_range_parser_init(&parser, lb_node); + lpbfifo.cs_n = of_range_count(&parser); - lpbfifo.cs_n = proplen / 4; lpbfifo.cs_ranges = devm_kcalloc(dev, lpbfifo.cs_n, sizeof(struct cs_range), GFP_KERNEL); if (!lpbfifo.cs_ranges) goto end; - if (of_property_read_u32_array(lb_node, "ranges", - (u32 *)lpbfifo.cs_ranges, proplen) != 0) { - goto end; - } - - for (i = 0; i < lpbfifo.cs_n; i++) { - if (lpbfifo.cs_ranges[i].base != 0) + for_each_of_range(&parser, &range) { + u32 base = lower_32_bits(range.bus_addr); + if (base) goto end; + + lpbfifo.cs_ranges[i].csnum = upper_32_bits(range.bus_addr); + lpbfifo.cs_ranges[i].base = base; + lpbfifo.cs_ranges[i].addr = range.cpu_addr; + lpbfifo.cs_ranges[i].size = range.size; + i++; } ret = 0; diff --git a/arch/powerpc/platforms/52xx/Kconfig b/arch/powerpc/platforms/52xx/Kconfig index b72ed2950ca8..384e4bef2c28 100644 --- a/arch/powerpc/platforms/52xx/Kconfig +++ b/arch/powerpc/platforms/52xx/Kconfig @@ -54,8 +54,3 @@ config PPC_MPC5200_BUGFIX for MPC5200B based boards. It is safe to say 'Y' here - -config PPC_MPC5200_LPBFIFO - tristate "MPC5200 LocalPlus bus FIFO driver" - depends on PPC_MPC52xx && PPC_BESTCOMM - select PPC_BESTCOMM_GEN_BD diff --git a/arch/powerpc/platforms/52xx/Makefile b/arch/powerpc/platforms/52xx/Makefile index f40d48eab779..1b1f72d83342 100644 --- a/arch/powerpc/platforms/52xx/Makefile +++ b/arch/powerpc/platforms/52xx/Makefile @@ -14,5 +14,3 @@ obj-$(CONFIG_PM) += mpc52xx_sleep.o mpc52xx_pm.o ifdef CONFIG_PPC_LITE5200 obj-$(CONFIG_PM) += lite5200_sleep.o lite5200_pm.o endif - -obj-$(CONFIG_PPC_MPC5200_LPBFIFO) += mpc52xx_lpbfifo.o diff --git a/arch/powerpc/platforms/52xx/lite5200_pm.c b/arch/powerpc/platforms/52xx/lite5200_pm.c index ee29b63fca16..4900f5f48cce 100644 --- a/arch/powerpc/platforms/52xx/lite5200_pm.c +++ b/arch/powerpc/platforms/52xx/lite5200_pm.c @@ -47,7 +47,7 @@ static int lite5200_pm_begin(suspend_state_t state) static int lite5200_pm_prepare(void) { struct device_node *np; - const struct of_device_id immr_ids[] = { + static const struct of_device_id immr_ids[] = { { .compatible = "fsl,mpc5200-immr", }, { .compatible = "fsl,mpc5200b-immr", }, { .type = "soc", .compatible = "mpc5200", }, /* lite5200 */ diff --git a/arch/powerpc/platforms/52xx/mpc52xx_lpbfifo.c b/arch/powerpc/platforms/52xx/mpc52xx_lpbfifo.c deleted file mode 100644 index 6d1dd6e87478..000000000000 --- a/arch/powerpc/platforms/52xx/mpc52xx_lpbfifo.c +++ /dev/null @@ -1,594 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * LocalPlus Bus FIFO driver for the Freescale MPC52xx. - * - * Copyright (C) 2009 Secret Lab Technologies Ltd. - * - * Todo: - * - Add support for multiple requests to be queued. - */ - -#include <linux/interrupt.h> -#include <linux/kernel.h> -#include <linux/of.h> -#include <linux/of_address.h> -#include <linux/of_irq.h> -#include <linux/of_platform.h> -#include <linux/spinlock.h> -#include <linux/module.h> -#include <asm/io.h> -#include <asm/mpc52xx.h> -#include <asm/time.h> - -#include <linux/fsl/bestcomm/bestcomm.h> -#include <linux/fsl/bestcomm/bestcomm_priv.h> -#include <linux/fsl/bestcomm/gen_bd.h> - -MODULE_AUTHOR("Grant Likely <grant.likely@secretlab.ca>"); -MODULE_DESCRIPTION("MPC5200 LocalPlus FIFO device driver"); -MODULE_LICENSE("GPL"); - -#define LPBFIFO_REG_PACKET_SIZE (0x00) -#define LPBFIFO_REG_START_ADDRESS (0x04) -#define LPBFIFO_REG_CONTROL (0x08) -#define LPBFIFO_REG_ENABLE (0x0C) -#define LPBFIFO_REG_BYTES_DONE_STATUS (0x14) -#define LPBFIFO_REG_FIFO_DATA (0x40) -#define LPBFIFO_REG_FIFO_STATUS (0x44) -#define LPBFIFO_REG_FIFO_CONTROL (0x48) -#define LPBFIFO_REG_FIFO_ALARM (0x4C) - -struct mpc52xx_lpbfifo { - struct device *dev; - phys_addr_t regs_phys; - void __iomem *regs; - int irq; - spinlock_t lock; - - struct bcom_task *bcom_tx_task; - struct bcom_task *bcom_rx_task; - struct bcom_task *bcom_cur_task; - - /* Current state data */ - struct mpc52xx_lpbfifo_request *req; - int dma_irqs_enabled; -}; - -/* The MPC5200 has only one fifo, so only need one instance structure */ -static struct mpc52xx_lpbfifo lpbfifo; - -/** - * mpc52xx_lpbfifo_kick - Trigger the next block of data to be transferred - * - * @req: Pointer to request structure - */ -static void mpc52xx_lpbfifo_kick(struct mpc52xx_lpbfifo_request *req) -{ - size_t transfer_size = req->size - req->pos; - struct bcom_bd *bd; - void __iomem *reg; - u32 *data; - int i; - int bit_fields; - int dma = !(req->flags & MPC52XX_LPBFIFO_FLAG_NO_DMA); - int write = req->flags & MPC52XX_LPBFIFO_FLAG_WRITE; - int poll_dma = req->flags & MPC52XX_LPBFIFO_FLAG_POLL_DMA; - - /* Set and clear the reset bits; is good practice in User Manual */ - out_be32(lpbfifo.regs + LPBFIFO_REG_ENABLE, 0x01010000); - - /* set master enable bit */ - out_be32(lpbfifo.regs + LPBFIFO_REG_ENABLE, 0x00000001); - if (!dma) { - /* While the FIFO can be setup for transfer sizes as large as - * 16M-1, the FIFO itself is only 512 bytes deep and it does - * not generate interrupts for FIFO full events (only transfer - * complete will raise an IRQ). Therefore when not using - * Bestcomm to drive the FIFO it needs to either be polled, or - * transfers need to constrained to the size of the fifo. - * - * This driver restricts the size of the transfer - */ - if (transfer_size > 512) - transfer_size = 512; - - /* Load the FIFO with data */ - if (write) { - reg = lpbfifo.regs + LPBFIFO_REG_FIFO_DATA; - data = req->data + req->pos; - for (i = 0; i < transfer_size; i += 4) - out_be32(reg, *data++); - } - - /* Unmask both error and completion irqs */ - out_be32(lpbfifo.regs + LPBFIFO_REG_ENABLE, 0x00000301); - } else { - /* Choose the correct direction - * - * Configure the watermarks so DMA will always complete correctly. - * It may be worth experimenting with the ALARM value to see if - * there is a performance impact. However, if it is wrong there - * is a risk of DMA not transferring the last chunk of data - */ - if (write) { - out_be32(lpbfifo.regs + LPBFIFO_REG_FIFO_ALARM, 0x1e4); - out_8(lpbfifo.regs + LPBFIFO_REG_FIFO_CONTROL, 7); - lpbfifo.bcom_cur_task = lpbfifo.bcom_tx_task; - } else { - out_be32(lpbfifo.regs + LPBFIFO_REG_FIFO_ALARM, 0x1ff); - out_8(lpbfifo.regs + LPBFIFO_REG_FIFO_CONTROL, 0); - lpbfifo.bcom_cur_task = lpbfifo.bcom_rx_task; - - if (poll_dma) { - if (lpbfifo.dma_irqs_enabled) { - disable_irq(bcom_get_task_irq(lpbfifo.bcom_rx_task)); - lpbfifo.dma_irqs_enabled = 0; - } - } else { - if (!lpbfifo.dma_irqs_enabled) { - enable_irq(bcom_get_task_irq(lpbfifo.bcom_rx_task)); - lpbfifo.dma_irqs_enabled = 1; - } - } - } - - bd = bcom_prepare_next_buffer(lpbfifo.bcom_cur_task); - bd->status = transfer_size; - if (!write) { - /* - * In the DMA read case, the DMA doesn't complete, - * possibly due to incorrect watermarks in the ALARM - * and CONTROL regs. For now instead of trying to - * determine the right watermarks that will make this - * work, just increase the number of bytes the FIFO is - * expecting. - * - * When submitting another operation, the FIFO will get - * reset, so the condition of the FIFO waiting for a - * non-existent 4 bytes will get cleared. - */ - transfer_size += 4; /* BLECH! */ - } - bd->data[0] = req->data_phys + req->pos; - bcom_submit_next_buffer(lpbfifo.bcom_cur_task, NULL); - - /* error irq & master enabled bit */ - bit_fields = 0x00000201; - - /* Unmask irqs */ - if (write && (!poll_dma)) - bit_fields |= 0x00000100; /* completion irq too */ - out_be32(lpbfifo.regs + LPBFIFO_REG_ENABLE, bit_fields); - } - - /* Set transfer size, width, chip select and READ mode */ - out_be32(lpbfifo.regs + LPBFIFO_REG_START_ADDRESS, - req->offset + req->pos); - out_be32(lpbfifo.regs + LPBFIFO_REG_PACKET_SIZE, transfer_size); - - bit_fields = req->cs << 24 | 0x000008; - if (!write) - bit_fields |= 0x010000; /* read mode */ - out_be32(lpbfifo.regs + LPBFIFO_REG_CONTROL, bit_fields); - - /* Kick it off */ - if (!lpbfifo.req->defer_xfer_start) - out_8(lpbfifo.regs + LPBFIFO_REG_PACKET_SIZE, 0x01); - if (dma) - bcom_enable(lpbfifo.bcom_cur_task); -} - -/** - * mpc52xx_lpbfifo_irq - IRQ handler for LPB FIFO - * @irq: IRQ number to be handled - * @dev_id: device ID cookie - * - * On transmit, the dma completion irq triggers before the fifo completion - * triggers. Handle the dma completion here instead of the LPB FIFO Bestcomm - * task completion irq because everything is not really done until the LPB FIFO - * completion irq triggers. - * - * In other words: - * For DMA, on receive, the "Fat Lady" is the bestcom completion irq. on - * transmit, the fifo completion irq is the "Fat Lady". The opera (or in this - * case the DMA/FIFO operation) is not finished until the "Fat Lady" sings. - * - * Reasons for entering this routine: - * 1) PIO mode rx and tx completion irq - * 2) DMA interrupt mode tx completion irq - * 3) DMA polled mode tx - * - * Exit conditions: - * 1) Transfer aborted - * 2) FIFO complete without DMA; more data to do - * 3) FIFO complete without DMA; all data transferred - * 4) FIFO complete using DMA - * - * Condition 1 can occur regardless of whether or not DMA is used. - * It requires executing the callback to report the error and exiting - * immediately. - * - * Condition 2 requires programming the FIFO with the next block of data - * - * Condition 3 requires executing the callback to report completion - * - * Condition 4 means the same as 3, except that we also retrieve the bcom - * buffer so DMA doesn't get clogged up. - * - * To make things trickier, the spinlock must be dropped before - * executing the callback, otherwise we could end up with a deadlock - * or nested spinlock condition. The out path is non-trivial, so - * extra fiddling is done to make sure all paths lead to the same - * outbound code. - * - * Return: irqreturn code (%IRQ_HANDLED) - */ -static irqreturn_t mpc52xx_lpbfifo_irq(int irq, void *dev_id) -{ - struct mpc52xx_lpbfifo_request *req; - u32 status = in_8(lpbfifo.regs + LPBFIFO_REG_BYTES_DONE_STATUS); - void __iomem *reg; - u32 *data; - int count, i; - int do_callback = 0; - u32 ts; - unsigned long flags; - int dma, write, poll_dma; - - spin_lock_irqsave(&lpbfifo.lock, flags); - ts = mftb(); - - req = lpbfifo.req; - if (!req) { - spin_unlock_irqrestore(&lpbfifo.lock, flags); - pr_err("bogus LPBFIFO IRQ\n"); - return IRQ_HANDLED; - } - - dma = !(req->flags & MPC52XX_LPBFIFO_FLAG_NO_DMA); - write = req->flags & MPC52XX_LPBFIFO_FLAG_WRITE; - poll_dma = req->flags & MPC52XX_LPBFIFO_FLAG_POLL_DMA; - - if (dma && !write) { - spin_unlock_irqrestore(&lpbfifo.lock, flags); - pr_err("bogus LPBFIFO IRQ (dma and not writing)\n"); - return IRQ_HANDLED; - } - - if ((status & 0x01) == 0) { - goto out; - } - - /* check abort bit */ - if (status & 0x10) { - out_be32(lpbfifo.regs + LPBFIFO_REG_ENABLE, 0x01010000); - do_callback = 1; - goto out; - } - - /* Read result from hardware */ - count = in_be32(lpbfifo.regs + LPBFIFO_REG_BYTES_DONE_STATUS); - count &= 0x00ffffff; - - if (!dma && !write) { - /* copy the data out of the FIFO */ - reg = lpbfifo.regs + LPBFIFO_REG_FIFO_DATA; - data = req->data + req->pos; - for (i = 0; i < count; i += 4) - *data++ = in_be32(reg); - } - - /* Update transfer position and count */ - req->pos += count; - - /* Decide what to do next */ - if (req->size - req->pos) - mpc52xx_lpbfifo_kick(req); /* more work to do */ - else - do_callback = 1; - - out: - /* Clear the IRQ */ - out_8(lpbfifo.regs + LPBFIFO_REG_BYTES_DONE_STATUS, 0x01); - - if (dma && (status & 0x11)) { - /* - * Count the DMA as complete only when the FIFO completion - * status or abort bits are set. - * - * (status & 0x01) should always be the case except sometimes - * when using polled DMA. - * - * (status & 0x10) {transfer aborted}: This case needs more - * testing. - */ - bcom_retrieve_buffer(lpbfifo.bcom_cur_task, &status, NULL); - } - req->last_byte = ((u8 *)req->data)[req->size - 1]; - - /* When the do_callback flag is set; it means the transfer is finished - * so set the FIFO as idle */ - if (do_callback) - lpbfifo.req = NULL; - - if (irq != 0) /* don't increment on polled case */ - req->irq_count++; - - req->irq_ticks += mftb() - ts; - spin_unlock_irqrestore(&lpbfifo.lock, flags); - - /* Spinlock is released; it is now safe to call the callback */ - if (do_callback && req->callback) - req->callback(req); - - return IRQ_HANDLED; -} - -/** - * mpc52xx_lpbfifo_bcom_irq - IRQ handler for LPB FIFO Bestcomm task - * @irq: IRQ number to be handled - * @dev_id: device ID cookie - * - * Only used when receiving data. - * - * Return: irqreturn code (%IRQ_HANDLED) - */ -static irqreturn_t mpc52xx_lpbfifo_bcom_irq(int irq, void *dev_id) -{ - struct mpc52xx_lpbfifo_request *req; - unsigned long flags; - u32 status; - u32 ts; - - spin_lock_irqsave(&lpbfifo.lock, flags); - ts = mftb(); - - req = lpbfifo.req; - if (!req || (req->flags & MPC52XX_LPBFIFO_FLAG_NO_DMA)) { - spin_unlock_irqrestore(&lpbfifo.lock, flags); - return IRQ_HANDLED; - } - - if (irq != 0) /* don't increment on polled case */ - req->irq_count++; - - if (!bcom_buffer_done(lpbfifo.bcom_cur_task)) { - spin_unlock_irqrestore(&lpbfifo.lock, flags); - - req->buffer_not_done_cnt++; - if ((req->buffer_not_done_cnt % 1000) == 0) - pr_err("transfer stalled\n"); - - return IRQ_HANDLED; - } - - bcom_retrieve_buffer(lpbfifo.bcom_cur_task, &status, NULL); - - req->last_byte = ((u8 *)req->data)[req->size - 1]; - - req->pos = status & 0x00ffffff; - - /* Mark the FIFO as idle */ - lpbfifo.req = NULL; - - /* Release the lock before calling out to the callback. */ - req->irq_ticks += mftb() - ts; - spin_unlock_irqrestore(&lpbfifo.lock, flags); - - if (req->callback) - req->callback(req); - - return IRQ_HANDLED; -} - -/** - * mpc52xx_lpbfifo_poll - Poll for DMA completion - */ -void mpc52xx_lpbfifo_poll(void) -{ - struct mpc52xx_lpbfifo_request *req = lpbfifo.req; - int dma = !(req->flags & MPC52XX_LPBFIFO_FLAG_NO_DMA); - int write = req->flags & MPC52XX_LPBFIFO_FLAG_WRITE; - - /* - * For more information, see comments on the "Fat Lady" - */ - if (dma && write) - mpc52xx_lpbfifo_irq(0, NULL); - else - mpc52xx_lpbfifo_bcom_irq(0, NULL); -} -EXPORT_SYMBOL(mpc52xx_lpbfifo_poll); - -/** - * mpc52xx_lpbfifo_submit - Submit an LPB FIFO transfer request. - * @req: Pointer to request structure - * - * Return: %0 on success, -errno code on error - */ -int mpc52xx_lpbfifo_submit(struct mpc52xx_lpbfifo_request *req) -{ - unsigned long flags; - - if (!lpbfifo.regs) - return -ENODEV; - - spin_lock_irqsave(&lpbfifo.lock, flags); - - /* If the req pointer is already set, then a transfer is in progress */ - if (lpbfifo.req) { - spin_unlock_irqrestore(&lpbfifo.lock, flags); - return -EBUSY; - } - - /* Setup the transfer */ - lpbfifo.req = req; - req->irq_count = 0; - req->irq_ticks = 0; - req->buffer_not_done_cnt = 0; - req->pos = 0; - - mpc52xx_lpbfifo_kick(req); - spin_unlock_irqrestore(&lpbfifo.lock, flags); - return 0; -} -EXPORT_SYMBOL(mpc52xx_lpbfifo_submit); - -int mpc52xx_lpbfifo_start_xfer(struct mpc52xx_lpbfifo_request *req) -{ - unsigned long flags; - - if (!lpbfifo.regs) - return -ENODEV; - - spin_lock_irqsave(&lpbfifo.lock, flags); - - /* - * If the req pointer is already set and a transfer was - * started on submit, then this transfer is in progress - */ - if (lpbfifo.req && !lpbfifo.req->defer_xfer_start) { - spin_unlock_irqrestore(&lpbfifo.lock, flags); - return -EBUSY; - } - - /* - * If the req was previously submitted but not - * started, start it now - */ - if (lpbfifo.req && lpbfifo.req == req && - lpbfifo.req->defer_xfer_start) { - out_8(lpbfifo.regs + LPBFIFO_REG_PACKET_SIZE, 0x01); - } - - spin_unlock_irqrestore(&lpbfifo.lock, flags); - return 0; -} -EXPORT_SYMBOL(mpc52xx_lpbfifo_start_xfer); - -void mpc52xx_lpbfifo_abort(struct mpc52xx_lpbfifo_request *req) -{ - unsigned long flags; - - spin_lock_irqsave(&lpbfifo.lock, flags); - if (lpbfifo.req == req) { - /* Put it into reset and clear the state */ - bcom_gen_bd_rx_reset(lpbfifo.bcom_rx_task); - bcom_gen_bd_tx_reset(lpbfifo.bcom_tx_task); - out_be32(lpbfifo.regs + LPBFIFO_REG_ENABLE, 0x01010000); - lpbfifo.req = NULL; - } - spin_unlock_irqrestore(&lpbfifo.lock, flags); -} -EXPORT_SYMBOL(mpc52xx_lpbfifo_abort); - -static int mpc52xx_lpbfifo_probe(struct platform_device *op) -{ - struct resource res; - int rc = -ENOMEM; - - if (lpbfifo.dev != NULL) - return -ENOSPC; - - lpbfifo.irq = irq_of_parse_and_map(op->dev.of_node, 0); - if (!lpbfifo.irq) - return -ENODEV; - - if (of_address_to_resource(op->dev.of_node, 0, &res)) - return -ENODEV; - lpbfifo.regs_phys = res.start; - lpbfifo.regs = of_iomap(op->dev.of_node, 0); - if (!lpbfifo.regs) - return -ENOMEM; - - spin_lock_init(&lpbfifo.lock); - - /* Put FIFO into reset */ - out_be32(lpbfifo.regs + LPBFIFO_REG_ENABLE, 0x01010000); - - /* Register the interrupt handler */ - rc = request_irq(lpbfifo.irq, mpc52xx_lpbfifo_irq, 0, - "mpc52xx-lpbfifo", &lpbfifo); - if (rc) - goto err_irq; - - /* Request the Bestcomm receive (fifo --> memory) task and IRQ */ - lpbfifo.bcom_rx_task = - bcom_gen_bd_rx_init(2, res.start + LPBFIFO_REG_FIFO_DATA, - BCOM_INITIATOR_SCLPC, BCOM_IPR_SCLPC, - 16*1024*1024); - if (!lpbfifo.bcom_rx_task) - goto err_bcom_rx; - - rc = request_irq(bcom_get_task_irq(lpbfifo.bcom_rx_task), - mpc52xx_lpbfifo_bcom_irq, 0, - "mpc52xx-lpbfifo-rx", &lpbfifo); - if (rc) - goto err_bcom_rx_irq; - - lpbfifo.dma_irqs_enabled = 1; - - /* Request the Bestcomm transmit (memory --> fifo) task and IRQ */ - lpbfifo.bcom_tx_task = - bcom_gen_bd_tx_init(2, res.start + LPBFIFO_REG_FIFO_DATA, - BCOM_INITIATOR_SCLPC, BCOM_IPR_SCLPC); - if (!lpbfifo.bcom_tx_task) - goto err_bcom_tx; - - lpbfifo.dev = &op->dev; - return 0; - - err_bcom_tx: - free_irq(bcom_get_task_irq(lpbfifo.bcom_rx_task), &lpbfifo); - err_bcom_rx_irq: - bcom_gen_bd_rx_release(lpbfifo.bcom_rx_task); - err_bcom_rx: - free_irq(lpbfifo.irq, &lpbfifo); - err_irq: - iounmap(lpbfifo.regs); - lpbfifo.regs = NULL; - - dev_err(&op->dev, "mpc52xx_lpbfifo_probe() failed\n"); - return -ENODEV; -} - - -static int mpc52xx_lpbfifo_remove(struct platform_device *op) -{ - if (lpbfifo.dev != &op->dev) - return 0; - - /* Put FIFO in reset */ - out_be32(lpbfifo.regs + LPBFIFO_REG_ENABLE, 0x01010000); - - /* Release the bestcomm transmit task */ - free_irq(bcom_get_task_irq(lpbfifo.bcom_tx_task), &lpbfifo); - bcom_gen_bd_tx_release(lpbfifo.bcom_tx_task); - - /* Release the bestcomm receive task */ - free_irq(bcom_get_task_irq(lpbfifo.bcom_rx_task), &lpbfifo); - bcom_gen_bd_rx_release(lpbfifo.bcom_rx_task); - - free_irq(lpbfifo.irq, &lpbfifo); - iounmap(lpbfifo.regs); - lpbfifo.regs = NULL; - lpbfifo.dev = NULL; - - return 0; -} - -static const struct of_device_id mpc52xx_lpbfifo_match[] = { - { .compatible = "fsl,mpc5200-lpbfifo", }, - {}, -}; -MODULE_DEVICE_TABLE(of, mpc52xx_lpbfifo_match); - -static struct platform_driver mpc52xx_lpbfifo_driver = { - .driver = { - .name = "mpc52xx-lpbfifo", - .of_match_table = mpc52xx_lpbfifo_match, - }, - .probe = mpc52xx_lpbfifo_probe, - .remove = mpc52xx_lpbfifo_remove, -}; -module_platform_driver(mpc52xx_lpbfifo_driver); diff --git a/arch/powerpc/platforms/52xx/mpc52xx_pm.c b/arch/powerpc/platforms/52xx/mpc52xx_pm.c index 549b3629e39a..f0c31ae15da5 100644 --- a/arch/powerpc/platforms/52xx/mpc52xx_pm.c +++ b/arch/powerpc/platforms/52xx/mpc52xx_pm.c @@ -60,7 +60,7 @@ int mpc52xx_set_wakeup_gpio(u8 pin, u8 level) int mpc52xx_pm_prepare(void) { struct device_node *np; - const struct of_device_id immr_ids[] = { + static const struct of_device_id immr_ids[] = { { .compatible = "fsl,mpc5200-immr", }, { .compatible = "fsl,mpc5200b-immr", }, { .type = "soc", .compatible = "mpc5200", }, /* lite5200 */ diff --git a/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c b/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c index 77ed61306a73..4d8fa9ed1a67 100644 --- a/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c +++ b/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c @@ -211,7 +211,7 @@ static struct i2c_driver mcu_driver = { .name = "mcu-mpc8349emitx", .of_match_table = mcu_of_match_table, }, - .probe_new = mcu_probe, + .probe = mcu_probe, .remove = mcu_remove, .id_table = mcu_ids, }; diff --git a/arch/powerpc/platforms/85xx/Makefile b/arch/powerpc/platforms/85xx/Makefile index e3d977624e33..43c34f26f108 100644 --- a/arch/powerpc/platforms/85xx/Makefile +++ b/arch/powerpc/platforms/85xx/Makefile @@ -12,9 +12,6 @@ obj-y += common.o obj-$(CONFIG_BSC9131_RDB) += bsc913x_rdb.o obj-$(CONFIG_BSC9132_QDS) += bsc913x_qds.o obj-$(CONFIG_C293_PCIE) += c293pcie.o -obj-$(CONFIG_MPC8540_ADS) += mpc85xx_ads.o -obj-$(CONFIG_MPC8560_ADS) += mpc85xx_ads.o -obj-$(CONFIG_MPC85xx_CDS) += mpc85xx_cds.o obj-$(CONFIG_MPC8536_DS) += mpc8536_ds.o obj8259-$(CONFIG_PPC_I8259) += mpc85xx_8259.o obj-$(CONFIG_MPC85xx_DS) += mpc85xx_ds.o $(obj8259-y) diff --git a/arch/powerpc/platforms/85xx/mpc85xx_ads.c b/arch/powerpc/platforms/85xx/mpc85xx_ads.c deleted file mode 100644 index 7c67438e76f8..000000000000 --- a/arch/powerpc/platforms/85xx/mpc85xx_ads.c +++ /dev/null @@ -1,162 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * MPC85xx setup and early boot code plus other random bits. - * - * Maintained by Kumar Gala (see MAINTAINERS for contact information) - * - * Copyright 2005 Freescale Semiconductor Inc. - */ - -#include <linux/stddef.h> -#include <linux/kernel.h> -#include <linux/pci.h> -#include <linux/kdev_t.h> -#include <linux/delay.h> -#include <linux/seq_file.h> -#include <linux/of_platform.h> - -#include <asm/time.h> -#include <asm/machdep.h> -#include <asm/pci-bridge.h> -#include <asm/mpic.h> -#include <mm/mmu_decl.h> -#include <asm/udbg.h> - -#include <sysdev/fsl_soc.h> -#include <sysdev/fsl_pci.h> - -#ifdef CONFIG_CPM2 -#include <asm/cpm2.h> -#include <sysdev/cpm2_pic.h> -#endif - -#include "mpc85xx.h" - -static void __init mpc85xx_ads_pic_init(void) -{ - struct mpic *mpic = mpic_alloc(NULL, 0, MPIC_BIG_ENDIAN, - 0, 256, " OpenPIC "); - BUG_ON(mpic == NULL); - mpic_init(mpic); - - mpc85xx_cpm2_pic_init(); -} - -/* - * Setup the architecture - */ -#ifdef CONFIG_CPM2 -struct cpm_pin { - int port, pin, flags; -}; - -static const struct cpm_pin mpc8560_ads_pins[] = { - /* SCC1 */ - {3, 29, CPM_PIN_OUTPUT | CPM_PIN_PRIMARY}, - {3, 30, CPM_PIN_OUTPUT | CPM_PIN_SECONDARY}, - {3, 31, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - - /* SCC2 */ - {2, 12, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - {2, 13, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - {3, 26, CPM_PIN_OUTPUT | CPM_PIN_PRIMARY}, - {3, 27, CPM_PIN_OUTPUT | CPM_PIN_PRIMARY}, - {3, 28, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - - /* FCC2 */ - {1, 18, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - {1, 19, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - {1, 20, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - {1, 21, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - {1, 22, CPM_PIN_OUTPUT | CPM_PIN_PRIMARY}, - {1, 23, CPM_PIN_OUTPUT | CPM_PIN_PRIMARY}, - {1, 24, CPM_PIN_OUTPUT | CPM_PIN_PRIMARY}, - {1, 25, CPM_PIN_OUTPUT | CPM_PIN_PRIMARY}, - {1, 26, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - {1, 27, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - {1, 28, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - {1, 29, CPM_PIN_OUTPUT | CPM_PIN_SECONDARY}, - {1, 30, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - {1, 31, CPM_PIN_OUTPUT | CPM_PIN_PRIMARY}, - {2, 18, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, /* CLK14 */ - {2, 19, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, /* CLK13 */ - - /* FCC3 */ - {1, 4, CPM_PIN_OUTPUT | CPM_PIN_PRIMARY}, - {1, 5, CPM_PIN_OUTPUT | CPM_PIN_PRIMARY}, - {1, 6, CPM_PIN_OUTPUT | CPM_PIN_PRIMARY}, - {1, 8, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - {1, 9, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - {1, 10, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - {1, 11, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - {1, 12, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - {1, 13, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - {1, 14, CPM_PIN_OUTPUT | CPM_PIN_PRIMARY}, - {1, 15, CPM_PIN_OUTPUT | CPM_PIN_PRIMARY}, - {1, 16, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - {1, 17, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, - {2, 16, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, /* CLK16 */ - {2, 17, CPM_PIN_INPUT | CPM_PIN_PRIMARY}, /* CLK15 */ - {2, 27, CPM_PIN_OUTPUT | CPM_PIN_PRIMARY}, -}; - -static void __init init_ioports(void) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(mpc8560_ads_pins); i++) { - const struct cpm_pin *pin = &mpc8560_ads_pins[i]; - cpm2_set_pin(pin->port, pin->pin, pin->flags); - } - - cpm2_clk_setup(CPM_CLK_SCC1, CPM_BRG1, CPM_CLK_RX); - cpm2_clk_setup(CPM_CLK_SCC1, CPM_BRG1, CPM_CLK_TX); - cpm2_clk_setup(CPM_CLK_SCC2, CPM_BRG2, CPM_CLK_RX); - cpm2_clk_setup(CPM_CLK_SCC2, CPM_BRG2, CPM_CLK_TX); - cpm2_clk_setup(CPM_CLK_FCC2, CPM_CLK13, CPM_CLK_RX); - cpm2_clk_setup(CPM_CLK_FCC2, CPM_CLK14, CPM_CLK_TX); - cpm2_clk_setup(CPM_CLK_FCC3, CPM_CLK15, CPM_CLK_RX); - cpm2_clk_setup(CPM_CLK_FCC3, CPM_CLK16, CPM_CLK_TX); -} -#endif - -static void __init mpc85xx_ads_setup_arch(void) -{ - if (ppc_md.progress) - ppc_md.progress("mpc85xx_ads_setup_arch()", 0); - -#ifdef CONFIG_CPM2 - cpm2_reset(); - init_ioports(); -#endif - - fsl_pci_assign_primary(); -} - -static void mpc85xx_ads_show_cpuinfo(struct seq_file *m) -{ - uint pvid, svid, phid1; - - pvid = mfspr(SPRN_PVR); - svid = mfspr(SPRN_SVR); - - seq_printf(m, "Vendor\t\t: Freescale Semiconductor\n"); - seq_printf(m, "PVR\t\t: 0x%x\n", pvid); - seq_printf(m, "SVR\t\t: 0x%x\n", svid); - - /* Display cpu Pll setting */ - phid1 = mfspr(SPRN_HID1); - seq_printf(m, "PLL setting\t: 0x%x\n", ((phid1 >> 24) & 0x3f)); -} - -machine_arch_initcall(mpc85xx_ads, mpc85xx_common_publish_devices); - -define_machine(mpc85xx_ads) { - .name = "MPC85xx ADS", - .compatible = "MPC85xxADS", - .setup_arch = mpc85xx_ads_setup_arch, - .init_IRQ = mpc85xx_ads_pic_init, - .show_cpuinfo = mpc85xx_ads_show_cpuinfo, - .get_irq = mpic_get_irq, - .progress = udbg_progress, -}; diff --git a/arch/powerpc/platforms/85xx/mpc85xx_cds.c b/arch/powerpc/platforms/85xx/mpc85xx_cds.c deleted file mode 100644 index 0e6964c7fdd6..000000000000 --- a/arch/powerpc/platforms/85xx/mpc85xx_cds.c +++ /dev/null @@ -1,387 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * MPC85xx setup and early boot code plus other random bits. - * - * Maintained by Kumar Gala (see MAINTAINERS for contact information) - * - * Copyright 2005, 2011-2012 Freescale Semiconductor Inc. - */ - -#include <linux/stddef.h> -#include <linux/kernel.h> -#include <linux/init.h> -#include <linux/errno.h> -#include <linux/reboot.h> -#include <linux/pci.h> -#include <linux/kdev_t.h> -#include <linux/major.h> -#include <linux/console.h> -#include <linux/delay.h> -#include <linux/seq_file.h> -#include <linux/initrd.h> -#include <linux/interrupt.h> -#include <linux/fsl_devices.h> -#include <linux/of_address.h> -#include <linux/of_irq.h> -#include <linux/of_platform.h> -#include <linux/pgtable.h> - -#include <asm/page.h> -#include <linux/atomic.h> -#include <asm/time.h> -#include <asm/io.h> -#include <asm/machdep.h> -#include <asm/ipic.h> -#include <asm/pci-bridge.h> -#include <asm/irq.h> -#include <mm/mmu_decl.h> -#include <asm/udbg.h> -#include <asm/mpic.h> -#include <asm/i8259.h> - -#include <sysdev/fsl_soc.h> -#include <sysdev/fsl_pci.h> - -#include "mpc85xx.h" - -/* - * The CDS board contains an FPGA/CPLD called "Cadmus", which collects - * various logic and performs system control functions. - * Here is the FPGA/CPLD register map. - */ -struct cadmus_reg { - u8 cm_ver; /* Board version */ - u8 cm_csr; /* General control/status */ - u8 cm_rst; /* Reset control */ - u8 cm_hsclk; /* High speed clock */ - u8 cm_hsxclk; /* High speed clock extended */ - u8 cm_led; /* LED data */ - u8 cm_pci; /* PCI control/status */ - u8 cm_dma; /* DMA control */ - u8 res[248]; /* Total 256 bytes */ -}; - -static struct cadmus_reg *cadmus; - -#ifdef CONFIG_PCI - -#define ARCADIA_HOST_BRIDGE_IDSEL 17 -#define ARCADIA_2ND_BRIDGE_IDSEL 3 - -static int mpc85xx_exclude_device(struct pci_controller *hose, - u_char bus, u_char devfn) -{ - /* We explicitly do not go past the Tundra 320 Bridge */ - if ((bus == 1) && (PCI_SLOT(devfn) == ARCADIA_2ND_BRIDGE_IDSEL)) - return PCIBIOS_DEVICE_NOT_FOUND; - if ((bus == 0) && (PCI_SLOT(devfn) == ARCADIA_2ND_BRIDGE_IDSEL)) - return PCIBIOS_DEVICE_NOT_FOUND; - else - return PCIBIOS_SUCCESSFUL; -} - -static int mpc85xx_cds_restart(struct notifier_block *this, - unsigned long mode, void *cmd) -{ - struct pci_dev *dev; - u_char tmp; - - if ((dev = pci_get_device(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_82C686, - NULL))) { - - /* Use the VIA Super Southbridge to force a PCI reset */ - pci_read_config_byte(dev, 0x47, &tmp); - pci_write_config_byte(dev, 0x47, tmp | 1); - - /* Flush the outbound PCI write queues */ - pci_read_config_byte(dev, 0x47, &tmp); - - /* - * At this point, the hardware reset should have triggered. - * However, if it doesn't work for some mysterious reason, - * just fall through to the default reset below. - */ - - pci_dev_put(dev); - } - - /* - * If we can't find the VIA chip (maybe the P2P bridge is - * disabled) or the VIA chip reset didn't work, just return - * and let default reset sequence happen. - */ - return NOTIFY_DONE; -} - -static int mpc85xx_cds_restart_register(void) -{ - static struct notifier_block restart_handler; - - restart_handler.notifier_call = mpc85xx_cds_restart; - restart_handler.priority = 192; - - return register_restart_handler(&restart_handler); -} -machine_arch_initcall(mpc85xx_cds, mpc85xx_cds_restart_register); - - -static void __init mpc85xx_cds_pci_irq_fixup(struct pci_dev *dev) -{ - u_char c; - if (dev->vendor == PCI_VENDOR_ID_VIA) { - switch (dev->device) { - case PCI_DEVICE_ID_VIA_82C586_1: - /* - * U-Boot does not set the enable bits - * for the IDE device. Force them on here. - */ - pci_read_config_byte(dev, 0x40, &c); - c |= 0x03; /* IDE: Chip Enable Bits */ - pci_write_config_byte(dev, 0x40, c); - - /* - * Since only primary interface works, force the - * IDE function to standard primary IDE interrupt - * w/ 8259 offset - */ - dev->irq = 14; - pci_write_config_byte(dev, PCI_INTERRUPT_LINE, dev->irq); - break; - /* - * Force legacy USB interrupt routing - */ - case PCI_DEVICE_ID_VIA_82C586_2: - /* There are two USB controllers. - * Identify them by function number - */ - if (PCI_FUNC(dev->devfn) == 3) - dev->irq = 11; - else - dev->irq = 10; - pci_write_config_byte(dev, PCI_INTERRUPT_LINE, dev->irq); - break; - default: - break; - } - } -} - -static void skip_fake_bridge(struct pci_dev *dev) -{ - /* Make it an error to skip the fake bridge - * in pci_setup_device() in probe.c */ - dev->hdr_type = 0x7f; -} -DECLARE_PCI_FIXUP_EARLY(0x1957, 0x3fff, skip_fake_bridge); -DECLARE_PCI_FIXUP_EARLY(0x3fff, 0x1957, skip_fake_bridge); -DECLARE_PCI_FIXUP_EARLY(0xff3f, 0x5719, skip_fake_bridge); - -#define PCI_DEVICE_ID_IDT_TSI310 0x01a7 - -/* - * Fix Tsi310 PCI-X bridge resource. - * Force the bridge to open a window from 0x0000-0x1fff in PCI I/O space. - * This allows legacy I/O(i8259, etc) on the VIA southbridge to be accessed. - */ -void mpc85xx_cds_fixup_bus(struct pci_bus *bus) -{ - struct pci_dev *dev = bus->self; - struct resource *res = bus->resource[0]; - - if (dev != NULL && - dev->vendor == PCI_VENDOR_ID_IBM && - dev->device == PCI_DEVICE_ID_IDT_TSI310) { - if (res) { - res->start = 0; - res->end = 0x1fff; - res->flags = IORESOURCE_IO; - pr_info("mpc85xx_cds: PCI bridge resource fixup applied\n"); - pr_info("mpc85xx_cds: %pR\n", res); - } - } - - fsl_pcibios_fixup_bus(bus); -} - -#ifdef CONFIG_PPC_I8259 -static void mpc85xx_8259_cascade_handler(struct irq_desc *desc) -{ - unsigned int cascade_irq = i8259_irq(); - - if (cascade_irq) - /* handle an interrupt from the 8259 */ - generic_handle_irq(cascade_irq); - - /* check for any interrupts from the shared IRQ line */ - handle_fasteoi_irq(desc); -} - -static irqreturn_t mpc85xx_8259_cascade_action(int irq, void *dev_id) -{ - return IRQ_HANDLED; -} -#endif /* PPC_I8259 */ -#endif /* CONFIG_PCI */ - -static void __init mpc85xx_cds_pic_init(void) -{ - struct mpic *mpic; - mpic = mpic_alloc(NULL, 0, MPIC_BIG_ENDIAN, - 0, 256, " OpenPIC "); - BUG_ON(mpic == NULL); - mpic_init(mpic); -} - -#if defined(CONFIG_PPC_I8259) && defined(CONFIG_PCI) -static int mpc85xx_cds_8259_attach(void) -{ - int ret; - struct device_node *np = NULL; - struct device_node *cascade_node = NULL; - int cascade_irq; - - /* Initialize the i8259 controller */ - for_each_node_by_type(np, "interrupt-controller") - if (of_device_is_compatible(np, "chrp,iic")) { - cascade_node = np; - break; - } - - if (cascade_node == NULL) { - printk(KERN_DEBUG "Could not find i8259 PIC\n"); - return -ENODEV; - } - - cascade_irq = irq_of_parse_and_map(cascade_node, 0); - if (!cascade_irq) { - printk(KERN_ERR "Failed to map cascade interrupt\n"); - return -ENXIO; - } - - i8259_init(cascade_node, 0); - of_node_put(cascade_node); - - /* - * Hook the interrupt to make sure desc->action is never NULL. - * This is required to ensure that the interrupt does not get - * disabled when the last user of the shared IRQ line frees their - * interrupt. - */ - ret = request_irq(cascade_irq, mpc85xx_8259_cascade_action, - IRQF_SHARED | IRQF_NO_THREAD, "8259 cascade", - cascade_node); - if (ret) { - printk(KERN_ERR "Failed to setup cascade interrupt\n"); - return ret; - } - - /* Success. Connect our low-level cascade handler. */ - irq_set_handler(cascade_irq, mpc85xx_8259_cascade_handler); - - return 0; -} -machine_device_initcall(mpc85xx_cds, mpc85xx_cds_8259_attach); - -#endif /* CONFIG_PPC_I8259 */ - -static void __init mpc85xx_cds_pci_assign_primary(void) -{ -#ifdef CONFIG_PCI - struct device_node *np; - - if (fsl_pci_primary) - return; - - /* - * MPC85xx_CDS has ISA bridge but unfortunately there is no - * isa node in device tree. We now looking for i8259 node as - * a workaround for such a broken device tree. This routine - * is for complying to all device trees. - */ - np = of_find_node_by_name(NULL, "i8259"); - while ((fsl_pci_primary = of_get_parent(np))) { - of_node_put(np); - np = fsl_pci_primary; - - if ((of_device_is_compatible(np, "fsl,mpc8540-pci") || - of_device_is_compatible(np, "fsl,mpc8548-pcie")) && - of_device_is_available(np)) - return; - } -#endif -} - -/* - * Setup the architecture - */ -static void __init mpc85xx_cds_setup_arch(void) -{ - struct device_node *np; - int cds_pci_slot; - - if (ppc_md.progress) - ppc_md.progress("mpc85xx_cds_setup_arch()", 0); - - np = of_find_compatible_node(NULL, NULL, "fsl,mpc8548cds-fpga"); - if (!np) { - pr_err("Could not find FPGA node.\n"); - return; - } - - cadmus = of_iomap(np, 0); - of_node_put(np); - if (!cadmus) { - pr_err("Fail to map FPGA area.\n"); - return; - } - - if (ppc_md.progress) { - char buf[40]; - cds_pci_slot = ((in_8(&cadmus->cm_csr) >> 6) & 0x3) + 1; - snprintf(buf, 40, "CDS Version = 0x%x in slot %d\n", - in_8(&cadmus->cm_ver), cds_pci_slot); - ppc_md.progress(buf, 0); - } - -#ifdef CONFIG_PCI - ppc_md.pci_irq_fixup = mpc85xx_cds_pci_irq_fixup; - ppc_md.pci_exclude_device = mpc85xx_exclude_device; -#endif - - mpc85xx_cds_pci_assign_primary(); - fsl_pci_assign_primary(); -} - -static void mpc85xx_cds_show_cpuinfo(struct seq_file *m) -{ - uint pvid, svid, phid1; - - pvid = mfspr(SPRN_PVR); - svid = mfspr(SPRN_SVR); - - seq_printf(m, "Vendor\t\t: Freescale Semiconductor\n"); - seq_printf(m, "Machine\t\t: MPC85xx CDS (0x%x)\n", - in_8(&cadmus->cm_ver)); - seq_printf(m, "PVR\t\t: 0x%x\n", pvid); - seq_printf(m, "SVR\t\t: 0x%x\n", svid); - - /* Display cpu Pll setting */ - phid1 = mfspr(SPRN_HID1); - seq_printf(m, "PLL setting\t: 0x%x\n", ((phid1 >> 24) & 0x3f)); -} - -machine_arch_initcall(mpc85xx_cds, mpc85xx_common_publish_devices); - -define_machine(mpc85xx_cds) { - .name = "MPC85xx CDS", - .compatible = "MPC85xxCDS", - .setup_arch = mpc85xx_cds_setup_arch, - .init_IRQ = mpc85xx_cds_pic_init, - .show_cpuinfo = mpc85xx_cds_show_cpuinfo, - .get_irq = mpic_get_irq, -#ifdef CONFIG_PCI - .pcibios_fixup_bus = mpc85xx_cds_fixup_bus, - .pcibios_fixup_phb = fsl_pcibios_fixup_phb, -#endif - .progress = udbg_progress, -}; diff --git a/arch/powerpc/platforms/86xx/Kconfig b/arch/powerpc/platforms/86xx/Kconfig index 8bfafc9d2bf7..67467cd6f34c 100644 --- a/arch/powerpc/platforms/86xx/Kconfig +++ b/arch/powerpc/platforms/86xx/Kconfig @@ -1,5 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -config PPC_86xx menuconfig PPC_86xx bool "86xx-based boards" depends on PPC_BOOK3S_32 diff --git a/arch/powerpc/platforms/cell/spu_base.c b/arch/powerpc/platforms/cell/spu_base.c index 7bd0b563e163..dea6f0f25897 100644 --- a/arch/powerpc/platforms/cell/spu_base.c +++ b/arch/powerpc/platforms/cell/spu_base.c @@ -326,12 +326,6 @@ spu_irq_class_1(int irq, void *data) if (stat & CLASS1_STORAGE_FAULT_INTR) __spu_trap_data_map(spu, dar, dsisr); - if (stat & CLASS1_LS_COMPARE_SUSPEND_ON_GET_INTR) - ; - - if (stat & CLASS1_LS_COMPARE_SUSPEND_ON_PUT_INTR) - ; - spu->class_1_dsisr = 0; spu->class_1_dar = 0; diff --git a/arch/powerpc/platforms/embedded6xx/Kconfig b/arch/powerpc/platforms/embedded6xx/Kconfig index a57424d6ef20..c6adff216fe6 100644 --- a/arch/powerpc/platforms/embedded6xx/Kconfig +++ b/arch/powerpc/platforms/embedded6xx/Kconfig @@ -10,7 +10,7 @@ config LINKSTATION select FSL_SOC select PPC_UDBG_16550 if SERIAL_8250 select DEFAULT_UIMAGE - select MPC10X_BRIDGE + imply MPC10X_BRIDGE if PCI help Select LINKSTATION if configuring for one of PPC- (MPC8241) based NAS systems from Buffalo Technology. So far only @@ -24,7 +24,7 @@ config STORCENTER select MPIC select FSL_SOC select PPC_UDBG_16550 if SERIAL_8250 - select MPC10X_BRIDGE + imply MPC10X_BRIDGE if PCI help Select STORCENTER if configuring for the iomega StorCenter with an 8241 CPU in it. diff --git a/arch/powerpc/platforms/powermac/feature.c b/arch/powerpc/platforms/powermac/feature.c index a195d5faa4e5..ed58928469b5 100644 --- a/arch/powerpc/platforms/powermac/feature.c +++ b/arch/powerpc/platforms/powermac/feature.c @@ -1053,11 +1053,11 @@ core99_reset_cpu(struct device_node *node, long param, long value) return -ENODEV; for_each_of_cpu_node(np) { - const u32 *num = of_get_property(np, "reg", NULL); const u32 *rst = of_get_property(np, "soft-reset", NULL); - if (num == NULL || rst == NULL) + if (!rst) continue; - if (param == *num) { + if (param == of_get_cpu_hwid(np, 0)) { + of_node_put(np); reset_io = *rst; break; } @@ -1499,11 +1499,11 @@ static long g5_reset_cpu(struct device_node *node, long param, long value) return -ENODEV; for_each_of_cpu_node(np) { - const u32 *num = of_get_property(np, "reg", NULL); const u32 *rst = of_get_property(np, "soft-reset", NULL); - if (num == NULL || rst == NULL) + if (!rst) continue; - if (param == *num) { + if (param == of_get_cpu_hwid(np, 0)) { + of_node_put(np); reset_io = *rst; break; } diff --git a/arch/powerpc/platforms/powernv/idle.c b/arch/powerpc/platforms/powernv/idle.c index 6dfe8d611164..ad41dffe4d92 100644 --- a/arch/powerpc/platforms/powernv/idle.c +++ b/arch/powerpc/platforms/powernv/idle.c @@ -246,9 +246,9 @@ static inline void atomic_lock_thread_idle(void) { int cpu = raw_smp_processor_id(); int first = cpu_first_thread_sibling(cpu); - unsigned long *state = &paca_ptrs[first]->idle_state; + unsigned long *lock = &paca_ptrs[first]->idle_lock; - while (unlikely(test_and_set_bit_lock(NR_PNV_CORE_IDLE_LOCK_BIT, state))) + while (unlikely(test_and_set_bit_lock(NR_PNV_CORE_IDLE_LOCK_BIT, lock))) barrier(); } @@ -258,29 +258,31 @@ static inline void atomic_unlock_and_stop_thread_idle(void) int first = cpu_first_thread_sibling(cpu); unsigned long thread = 1UL << cpu_thread_in_core(cpu); unsigned long *state = &paca_ptrs[first]->idle_state; + unsigned long *lock = &paca_ptrs[first]->idle_lock; u64 s = READ_ONCE(*state); u64 new, tmp; - BUG_ON(!(s & PNV_CORE_IDLE_LOCK_BIT)); + BUG_ON(!(READ_ONCE(*lock) & PNV_CORE_IDLE_LOCK_BIT)); BUG_ON(s & thread); again: - new = (s | thread) & ~PNV_CORE_IDLE_LOCK_BIT; + new = s | thread; tmp = cmpxchg(state, s, new); if (unlikely(tmp != s)) { s = tmp; goto again; } + clear_bit_unlock(NR_PNV_CORE_IDLE_LOCK_BIT, lock); } static inline void atomic_unlock_thread_idle(void) { int cpu = raw_smp_processor_id(); int first = cpu_first_thread_sibling(cpu); - unsigned long *state = &paca_ptrs[first]->idle_state; + unsigned long *lock = &paca_ptrs[first]->idle_lock; - BUG_ON(!test_bit(NR_PNV_CORE_IDLE_LOCK_BIT, state)); - clear_bit_unlock(NR_PNV_CORE_IDLE_LOCK_BIT, state); + BUG_ON(!test_bit(NR_PNV_CORE_IDLE_LOCK_BIT, lock)); + clear_bit_unlock(NR_PNV_CORE_IDLE_LOCK_BIT, lock); } /* P7 and P8 */ diff --git a/arch/powerpc/platforms/powernv/opal-call.c b/arch/powerpc/platforms/powernv/opal-call.c index f812c74c61e5..021b0ec29e24 100644 --- a/arch/powerpc/platforms/powernv/opal-call.c +++ b/arch/powerpc/platforms/powernv/opal-call.c @@ -167,8 +167,6 @@ OPAL_CALL(opal_pci_map_pe_mmio_window, OPAL_PCI_MAP_PE_MMIO_WINDOW); OPAL_CALL(opal_pci_set_phb_table_memory, OPAL_PCI_SET_PHB_TABLE_MEMORY); OPAL_CALL(opal_pci_set_pe, OPAL_PCI_SET_PE); OPAL_CALL(opal_pci_set_peltv, OPAL_PCI_SET_PELTV); -OPAL_CALL(opal_pci_set_mve, OPAL_PCI_SET_MVE); -OPAL_CALL(opal_pci_set_mve_enable, OPAL_PCI_SET_MVE_ENABLE); OPAL_CALL(opal_pci_get_xive_reissue, OPAL_PCI_GET_XIVE_REISSUE); OPAL_CALL(opal_pci_set_xive_reissue, OPAL_PCI_SET_XIVE_REISSUE); OPAL_CALL(opal_pci_set_xive_pe, OPAL_PCI_SET_XIVE_PE); diff --git a/arch/powerpc/platforms/powernv/opal-irqchip.c b/arch/powerpc/platforms/powernv/opal-irqchip.c index d55652b5f6fa..f9a7001dacb7 100644 --- a/arch/powerpc/platforms/powernv/opal-irqchip.c +++ b/arch/powerpc/platforms/powernv/opal-irqchip.c @@ -59,7 +59,7 @@ again: cond_resched(); } - last_outstanding_events = 0; + WRITE_ONCE(last_outstanding_events, 0); if (opal_poll_events(&events) != OPAL_SUCCESS) return; e = be64_to_cpu(events) & opal_event_irqchip.mask; @@ -69,7 +69,7 @@ again: bool opal_have_pending_events(void) { - if (last_outstanding_events & opal_event_irqchip.mask) + if (READ_ONCE(last_outstanding_events) & opal_event_irqchip.mask) return true; return false; } @@ -124,7 +124,7 @@ static irqreturn_t opal_interrupt(int irq, void *data) __be64 events; opal_handle_interrupt(virq_to_hw(irq), &events); - last_outstanding_events = be64_to_cpu(events); + WRITE_ONCE(last_outstanding_events, be64_to_cpu(events)); if (opal_have_pending_events()) opal_wake_poller(); diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index a02e9cdb5b5d..cb637827bc58 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -45,11 +45,8 @@ #include "pci.h" #include "../../../../drivers/pci/pci.h" -#define PNV_IODA1_M64_NUM 16 /* Number of M64 BARs */ -#define PNV_IODA1_M64_SEGS 8 /* Segments per M64 BAR */ -#define PNV_IODA1_DMA32_SEGSIZE 0x10000000 - -static const char * const pnv_phb_names[] = { "IODA1", "IODA2", "NPU_OCAPI" }; +/* This array is indexed with enum pnv_phb_type */ +static const char * const pnv_phb_names[] = { "IODA2", "NPU_OCAPI" }; static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable); static void pnv_pci_configure_bus(struct pci_bus *bus); @@ -280,86 +277,6 @@ static void pnv_ioda_reserve_dev_m64_pe(struct pci_dev *pdev, } } -static int pnv_ioda1_init_m64(struct pnv_phb *phb) -{ - struct resource *r; - int index; - - /* - * There are 16 M64 BARs, each of which has 8 segments. So - * there are as many M64 segments as the maximum number of - * PEs, which is 128. - */ - for (index = 0; index < PNV_IODA1_M64_NUM; index++) { - unsigned long base, segsz = phb->ioda.m64_segsize; - int64_t rc; - - base = phb->ioda.m64_base + - index * PNV_IODA1_M64_SEGS * segsz; - rc = opal_pci_set_phb_mem_window(phb->opal_id, - OPAL_M64_WINDOW_TYPE, index, base, 0, - PNV_IODA1_M64_SEGS * segsz); - if (rc != OPAL_SUCCESS) { - pr_warn(" Error %lld setting M64 PHB#%x-BAR#%d\n", - rc, phb->hose->global_number, index); - goto fail; - } - - rc = opal_pci_phb_mmio_enable(phb->opal_id, - OPAL_M64_WINDOW_TYPE, index, - OPAL_ENABLE_M64_SPLIT); - if (rc != OPAL_SUCCESS) { - pr_warn(" Error %lld enabling M64 PHB#%x-BAR#%d\n", - rc, phb->hose->global_number, index); - goto fail; - } - } - - for (index = 0; index < phb->ioda.total_pe_num; index++) { - int64_t rc; - - /* - * P7IOC supports M64DT, which helps mapping M64 segment - * to one particular PE#. However, PHB3 has fixed mapping - * between M64 segment and PE#. In order to have same logic - * for P7IOC and PHB3, we enforce fixed mapping between M64 - * segment and PE# on P7IOC. - */ - rc = opal_pci_map_pe_mmio_window(phb->opal_id, - index, OPAL_M64_WINDOW_TYPE, - index / PNV_IODA1_M64_SEGS, - index % PNV_IODA1_M64_SEGS); - if (rc != OPAL_SUCCESS) { - pr_warn("%s: Error %lld mapping M64 for PHB#%x-PE#%x\n", - __func__, rc, phb->hose->global_number, - index); - goto fail; - } - } - - /* - * Exclude the segments for reserved and root bus PE, which - * are first or last two PEs. - */ - r = &phb->hose->mem_resources[1]; - if (phb->ioda.reserved_pe_idx == 0) - r->start += (2 * phb->ioda.m64_segsize); - else if (phb->ioda.reserved_pe_idx == (phb->ioda.total_pe_num - 1)) - r->end -= (2 * phb->ioda.m64_segsize); - else - WARN(1, "Wrong reserved PE#%x on PHB#%x\n", - phb->ioda.reserved_pe_idx, phb->hose->global_number); - - return 0; - -fail: - for ( ; index >= 0; index--) - opal_pci_phb_mmio_enable(phb->opal_id, - OPAL_M64_WINDOW_TYPE, index, OPAL_DISABLE_M64); - - return -EIO; -} - static void pnv_ioda_reserve_m64_pe(struct pci_bus *bus, unsigned long *pe_bitmap, bool all) @@ -443,7 +360,7 @@ static void __init pnv_ioda_parse_m64_window(struct pnv_phb *phb) const __be32 *r; u64 pci_addr; - if (phb->type != PNV_PHB_IODA1 && phb->type != PNV_PHB_IODA2) { + if (phb->type != PNV_PHB_IODA2) { pr_info(" Not support M64 window\n"); return; } @@ -518,10 +435,7 @@ static void __init pnv_ioda_parse_m64_window(struct pnv_phb *phb) * Setup init functions for M64 based on IODA version, IODA3 uses * the IODA2 code. */ - if (phb->type == PNV_PHB_IODA1) - phb->init_m64 = pnv_ioda1_init_m64; - else - phb->init_m64 = pnv_ioda2_init_m64; + phb->init_m64 = pnv_ioda2_init_m64; } static void pnv_ioda_freeze_pe(struct pnv_phb *phb, int pe_no) @@ -952,29 +866,8 @@ int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe) for (rid = pe->rid; rid < rid_end; rid++) phb->ioda.pe_rmap[rid] = pe->pe_number; - /* Setup one MVTs on IODA1 */ - if (phb->type != PNV_PHB_IODA1) { - pe->mve_number = 0; - goto out; - } - - pe->mve_number = pe->pe_number; - rc = opal_pci_set_mve(phb->opal_id, pe->mve_number, pe->pe_number); - if (rc != OPAL_SUCCESS) { - pe_err(pe, "OPAL error %ld setting up MVE %x\n", - rc, pe->mve_number); - pe->mve_number = -1; - } else { - rc = opal_pci_set_mve_enable(phb->opal_id, - pe->mve_number, OPAL_ENABLE_MVE); - if (rc) { - pe_err(pe, "OPAL error %ld enabling MVE %x\n", - rc, pe->mve_number); - pe->mve_number = -1; - } - } + pe->mve_number = 0; -out: return 0; } @@ -1097,9 +990,6 @@ static struct pnv_ioda_pe *pnv_ioda_setup_bus_PE(struct pci_bus *bus, bool all) return pe; } -static void pnv_pci_ioda1_setup_dma_pe(struct pnv_phb *phb, - struct pnv_ioda_pe *pe); - static void pnv_pci_ioda_dma_dev_setup(struct pci_dev *pdev) { struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus); @@ -1134,9 +1024,6 @@ static void pnv_pci_ioda_dma_dev_setup(struct pci_dev *pdev) */ if (!pe->dma_setup_done && !pci_is_bridge(pdev)) { switch (phb->type) { - case PNV_PHB_IODA1: - pnv_pci_ioda1_setup_dma_pe(phb, pe); - break; case PNV_PHB_IODA2: pnv_pci_ioda2_setup_dma_pe(phb, pe); break; @@ -1273,53 +1160,6 @@ static inline __be64 __iomem *pnv_ioda_get_inval_reg(struct pnv_phb *phb) return phb->regs + 0x210; } -static void pnv_pci_p7ioc_tce_invalidate(struct iommu_table *tbl, - unsigned long index, unsigned long npages) -{ - struct iommu_table_group_link *tgl = list_first_entry_or_null( - &tbl->it_group_list, struct iommu_table_group_link, - next); - struct pnv_ioda_pe *pe = container_of(tgl->table_group, - struct pnv_ioda_pe, table_group); - __be64 __iomem *invalidate = pnv_ioda_get_inval_reg(pe->phb); - unsigned long start, end, inc; - - start = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset); - end = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset + - npages - 1); - - /* p7ioc-style invalidation, 2 TCEs per write */ - start |= (1ull << 63); - end |= (1ull << 63); - inc = 16; - end |= inc - 1; /* round up end to be different than start */ - - mb(); /* Ensure above stores are visible */ - while (start <= end) { - __raw_writeq_be(start, invalidate); - start += inc; - } - - /* - * The iommu layer will do another mb() for us on build() - * and we don't care on free() - */ -} - -static int pnv_ioda1_tce_build(struct iommu_table *tbl, long index, - long npages, unsigned long uaddr, - enum dma_data_direction direction, - unsigned long attrs) -{ - int ret = pnv_tce_build(tbl, index, npages, uaddr, direction, - attrs); - - if (!ret) - pnv_pci_p7ioc_tce_invalidate(tbl, index, npages); - - return ret; -} - #ifdef CONFIG_IOMMU_API /* Common for IODA1 and IODA2 */ static int pnv_ioda_tce_xchg_no_kill(struct iommu_table *tbl, long index, @@ -1329,25 +1169,6 @@ static int pnv_ioda_tce_xchg_no_kill(struct iommu_table *tbl, long index, } #endif -static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index, - long npages) -{ - pnv_tce_free(tbl, index, npages); - - pnv_pci_p7ioc_tce_invalidate(tbl, index, npages); -} - -static struct iommu_table_ops pnv_ioda1_iommu_ops = { - .set = pnv_ioda1_tce_build, -#ifdef CONFIG_IOMMU_API - .xchg_no_kill = pnv_ioda_tce_xchg_no_kill, - .tce_kill = pnv_pci_p7ioc_tce_invalidate, - .useraddrptr = pnv_tce_useraddrptr, -#endif - .clear = pnv_ioda1_tce_free, - .get = pnv_tce_get, -}; - #define PHB3_TCE_KILL_INVAL_ALL PPC_BIT(0) #define PHB3_TCE_KILL_INVAL_PE PPC_BIT(1) #define PHB3_TCE_KILL_INVAL_ONE PPC_BIT(2) @@ -1453,182 +1274,6 @@ static struct iommu_table_ops pnv_ioda2_iommu_ops = { .free = pnv_pci_ioda2_table_free_pages, }; -static int pnv_pci_ioda_dev_dma_weight(struct pci_dev *dev, void *data) -{ - unsigned int *weight = (unsigned int *)data; - - /* This is quite simplistic. The "base" weight of a device - * is 10. 0 means no DMA is to be accounted for it. - */ - if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL) - return 0; - - if (dev->class == PCI_CLASS_SERIAL_USB_UHCI || - dev->class == PCI_CLASS_SERIAL_USB_OHCI || - dev->class == PCI_CLASS_SERIAL_USB_EHCI) - *weight += 3; - else if ((dev->class >> 8) == PCI_CLASS_STORAGE_RAID) - *weight += 15; - else - *weight += 10; - - return 0; -} - -static unsigned int pnv_pci_ioda_pe_dma_weight(struct pnv_ioda_pe *pe) -{ - unsigned int weight = 0; - - /* SRIOV VF has same DMA32 weight as its PF */ -#ifdef CONFIG_PCI_IOV - if ((pe->flags & PNV_IODA_PE_VF) && pe->parent_dev) { - pnv_pci_ioda_dev_dma_weight(pe->parent_dev, &weight); - return weight; - } -#endif - - if ((pe->flags & PNV_IODA_PE_DEV) && pe->pdev) { - pnv_pci_ioda_dev_dma_weight(pe->pdev, &weight); - } else if ((pe->flags & PNV_IODA_PE_BUS) && pe->pbus) { - struct pci_dev *pdev; - - list_for_each_entry(pdev, &pe->pbus->devices, bus_list) - pnv_pci_ioda_dev_dma_weight(pdev, &weight); - } else if ((pe->flags & PNV_IODA_PE_BUS_ALL) && pe->pbus) { - pci_walk_bus(pe->pbus, pnv_pci_ioda_dev_dma_weight, &weight); - } - - return weight; -} - -static void pnv_pci_ioda1_setup_dma_pe(struct pnv_phb *phb, - struct pnv_ioda_pe *pe) -{ - - struct page *tce_mem = NULL; - struct iommu_table *tbl; - unsigned int weight, total_weight = 0; - unsigned int tce32_segsz, base, segs, avail, i; - int64_t rc; - void *addr; - - /* XXX FIXME: Handle 64-bit only DMA devices */ - /* XXX FIXME: Provide 64-bit DMA facilities & non-4K TCE tables etc.. */ - /* XXX FIXME: Allocate multi-level tables on PHB3 */ - weight = pnv_pci_ioda_pe_dma_weight(pe); - if (!weight) - return; - - pci_walk_bus(phb->hose->bus, pnv_pci_ioda_dev_dma_weight, - &total_weight); - segs = (weight * phb->ioda.dma32_count) / total_weight; - if (!segs) - segs = 1; - - /* - * Allocate contiguous DMA32 segments. We begin with the expected - * number of segments. With one more attempt, the number of DMA32 - * segments to be allocated is decreased by one until one segment - * is allocated successfully. - */ - do { - for (base = 0; base <= phb->ioda.dma32_count - segs; base++) { - for (avail = 0, i = base; i < base + segs; i++) { - if (phb->ioda.dma32_segmap[i] == - IODA_INVALID_PE) - avail++; - } - - if (avail == segs) - goto found; - } - } while (--segs); - - if (!segs) { - pe_warn(pe, "No available DMA32 segments\n"); - return; - } - -found: - tbl = pnv_pci_table_alloc(phb->hose->node); - if (WARN_ON(!tbl)) - return; - -#ifdef CONFIG_IOMMU_API - pe->table_group.ops = &spapr_tce_table_group_ops; - pe->table_group.pgsizes = SZ_4K; -#endif - iommu_register_group(&pe->table_group, phb->hose->global_number, - pe->pe_number); - pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group); - - /* Grab a 32-bit TCE table */ - pe_info(pe, "DMA weight %d (%d), assigned (%d) %d DMA32 segments\n", - weight, total_weight, base, segs); - pe_info(pe, " Setting up 32-bit TCE table at %08x..%08x\n", - base * PNV_IODA1_DMA32_SEGSIZE, - (base + segs) * PNV_IODA1_DMA32_SEGSIZE - 1); - - /* XXX Currently, we allocate one big contiguous table for the - * TCEs. We only really need one chunk per 256M of TCE space - * (ie per segment) but that's an optimization for later, it - * requires some added smarts with our get/put_tce implementation - * - * Each TCE page is 4KB in size and each TCE entry occupies 8 - * bytes - */ - tce32_segsz = PNV_IODA1_DMA32_SEGSIZE >> (IOMMU_PAGE_SHIFT_4K - 3); - tce_mem = alloc_pages_node(phb->hose->node, GFP_KERNEL, - get_order(tce32_segsz * segs)); - if (!tce_mem) { - pe_err(pe, " Failed to allocate a 32-bit TCE memory\n"); - goto fail; - } - addr = page_address(tce_mem); - memset(addr, 0, tce32_segsz * segs); - - /* Configure HW */ - for (i = 0; i < segs; i++) { - rc = opal_pci_map_pe_dma_window(phb->opal_id, - pe->pe_number, - base + i, 1, - __pa(addr) + tce32_segsz * i, - tce32_segsz, IOMMU_PAGE_SIZE_4K); - if (rc) { - pe_err(pe, " Failed to configure 32-bit TCE table, err %lld\n", - rc); - goto fail; - } - } - - /* Setup DMA32 segment mapping */ - for (i = base; i < base + segs; i++) - phb->ioda.dma32_segmap[i] = pe->pe_number; - - /* Setup linux iommu table */ - pnv_pci_setup_iommu_table(tbl, addr, tce32_segsz * segs, - base * PNV_IODA1_DMA32_SEGSIZE, - IOMMU_PAGE_SHIFT_4K); - - tbl->it_ops = &pnv_ioda1_iommu_ops; - pe->table_group.tce32_start = tbl->it_offset << tbl->it_page_shift; - pe->table_group.tce32_size = tbl->it_size << tbl->it_page_shift; - tbl->it_index = (phb->hose->global_number << 16) | pe->pe_number; - if (!iommu_init_table(tbl, phb->hose->node, 0, 0)) - panic("Failed to initialize iommu table"); - - pe->dma_setup_done = true; - return; - fail: - /* XXX Failure: Try to fallback to 64-bit only ? */ - if (tce_mem) - __free_pages(tce_mem, get_order(tce32_segsz * segs)); - if (tbl) { - pnv_pci_unlink_table_and_group(tbl, &pe->table_group); - iommu_tce_table_put(tbl); - } -} - static long pnv_pci_ioda2_set_window(struct iommu_table_group *table_group, int num, struct iommu_table *tbl) { @@ -2707,57 +2352,6 @@ static bool pnv_ocapi_enable_device_hook(struct pci_dev *dev) return true; } -static long pnv_pci_ioda1_unset_window(struct iommu_table_group *table_group, - int num) -{ - struct pnv_ioda_pe *pe = container_of(table_group, - struct pnv_ioda_pe, table_group); - struct pnv_phb *phb = pe->phb; - unsigned int idx; - long rc; - - pe_info(pe, "Removing DMA window #%d\n", num); - for (idx = 0; idx < phb->ioda.dma32_count; idx++) { - if (phb->ioda.dma32_segmap[idx] != pe->pe_number) - continue; - - rc = opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number, - idx, 0, 0ul, 0ul, 0ul); - if (rc != OPAL_SUCCESS) { - pe_warn(pe, "Failure %ld unmapping DMA32 segment#%d\n", - rc, idx); - return rc; - } - - phb->ioda.dma32_segmap[idx] = IODA_INVALID_PE; - } - - pnv_pci_unlink_table_and_group(table_group->tables[num], table_group); - return OPAL_SUCCESS; -} - -static void pnv_pci_ioda1_release_pe_dma(struct pnv_ioda_pe *pe) -{ - struct iommu_table *tbl = pe->table_group.tables[0]; - int64_t rc; - - if (!pe->dma_setup_done) - return; - - rc = pnv_pci_ioda1_unset_window(&pe->table_group, 0); - if (rc != OPAL_SUCCESS) - return; - - pnv_pci_p7ioc_tce_invalidate(tbl, tbl->it_offset, tbl->it_size); - if (pe->table_group.group) { - iommu_group_put(pe->table_group.group); - WARN_ON(pe->table_group.group); - } - - free_pages(tbl->it_base, get_order(tbl->it_size << 3)); - iommu_tce_table_put(tbl); -} - void pnv_pci_ioda2_release_pe_dma(struct pnv_ioda_pe *pe) { struct iommu_table *tbl = pe->table_group.tables[0]; @@ -2806,13 +2400,7 @@ static void pnv_ioda_release_pe_seg(struct pnv_ioda_pe *pe) { struct pnv_phb *phb = pe->phb; - if (phb->type == PNV_PHB_IODA1) { - pnv_ioda_free_pe_seg(pe, OPAL_IO_WINDOW_TYPE, - phb->ioda.io_segmap); - pnv_ioda_free_pe_seg(pe, OPAL_M32_WINDOW_TYPE, - phb->ioda.m32_segmap); - /* M64 is pre-configured by pnv_ioda1_init_m64() */ - } else if (phb->type == PNV_PHB_IODA2) { + if (phb->type == PNV_PHB_IODA2) { pnv_ioda_free_pe_seg(pe, OPAL_M32_WINDOW_TYPE, phb->ioda.m32_segmap); } @@ -2830,9 +2418,6 @@ static void pnv_ioda_release_pe(struct pnv_ioda_pe *pe) mutex_unlock(&phb->ioda.pe_list_mutex); switch (phb->type) { - case PNV_PHB_IODA1: - pnv_pci_ioda1_release_pe_dma(pe); - break; case PNV_PHB_IODA2: pnv_pci_ioda2_release_pe_dma(pe); break; @@ -2981,7 +2566,6 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, struct pci_controller *hose; struct pnv_phb *phb; unsigned long size, m64map_off, m32map_off, pemap_off; - unsigned long iomap_off = 0, dma32map_off = 0; struct pnv_ioda_pe *root_pe; struct resource r; const __be64 *prop64; @@ -3092,10 +2676,6 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, phb->ioda.io_segsize = phb->ioda.io_size / phb->ioda.total_pe_num; phb->ioda.io_pci_base = 0; /* XXX calculate this ? */ - /* Calculate how many 32-bit TCE segments we have */ - phb->ioda.dma32_count = phb->ioda.m32_pci_base / - PNV_IODA1_DMA32_SEGSIZE; - /* Allocate aux data & arrays. We don't have IO ports on PHB3 */ size = ALIGN(max_t(unsigned, phb->ioda.total_pe_num, 8) / 8, sizeof(unsigned long)); @@ -3103,13 +2683,6 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, size += phb->ioda.total_pe_num * sizeof(phb->ioda.m64_segmap[0]); m32map_off = size; size += phb->ioda.total_pe_num * sizeof(phb->ioda.m32_segmap[0]); - if (phb->type == PNV_PHB_IODA1) { - iomap_off = size; - size += phb->ioda.total_pe_num * sizeof(phb->ioda.io_segmap[0]); - dma32map_off = size; - size += phb->ioda.dma32_count * - sizeof(phb->ioda.dma32_segmap[0]); - } pemap_off = size; size += phb->ioda.total_pe_num * sizeof(struct pnv_ioda_pe); aux = kzalloc(size, GFP_KERNEL); @@ -3123,15 +2696,6 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, phb->ioda.m64_segmap[segno] = IODA_INVALID_PE; phb->ioda.m32_segmap[segno] = IODA_INVALID_PE; } - if (phb->type == PNV_PHB_IODA1) { - phb->ioda.io_segmap = aux + iomap_off; - for (segno = 0; segno < phb->ioda.total_pe_num; segno++) - phb->ioda.io_segmap[segno] = IODA_INVALID_PE; - - phb->ioda.dma32_segmap = aux + dma32map_off; - for (segno = 0; segno < phb->ioda.dma32_count; segno++) - phb->ioda.dma32_segmap[segno] = IODA_INVALID_PE; - } phb->ioda.pe_array = aux + pemap_off; /* @@ -3155,10 +2719,6 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, INIT_LIST_HEAD(&phb->ioda.pe_list); mutex_init(&phb->ioda.pe_list_mutex); - /* Calculate how many 32-bit TCE segments we have */ - phb->ioda.dma32_count = phb->ioda.m32_pci_base / - PNV_IODA1_DMA32_SEGSIZE; - #if 0 /* We should really do that ... */ rc = opal_pci_set_phb_mem_window(opal->phb_id, window_type, @@ -3265,27 +2825,3 @@ static void pnv_npu2_opencapi_cfg_size_fixup(struct pci_dev *dev) dev->cfg_size = PCI_CFG_SPACE_EXP_SIZE; } DECLARE_PCI_FIXUP_EARLY(PCI_ANY_ID, PCI_ANY_ID, pnv_npu2_opencapi_cfg_size_fixup); - -void __init pnv_pci_init_ioda_hub(struct device_node *np) -{ - struct device_node *phbn; - const __be64 *prop64; - u64 hub_id; - - pr_info("Probing IODA IO-Hub %pOF\n", np); - - prop64 = of_get_property(np, "ibm,opal-hubid", NULL); - if (!prop64) { - pr_err(" Missing \"ibm,opal-hubid\" property !\n"); - return; - } - hub_id = be64_to_cpup(prop64); - pr_devel(" HUB-ID : 0x%016llx\n", hub_id); - - /* Count child PHBs */ - for_each_child_of_node(np, phbn) { - /* Look for IODA1 PHBs */ - if (of_device_is_compatible(phbn, "ibm,ioda-phb")) - pnv_pci_init_ioda_phb(phbn, hub_id, PNV_PHB_IODA1); - } -} diff --git a/arch/powerpc/platforms/powernv/pci-sriov.c b/arch/powerpc/platforms/powernv/pci-sriov.c index 7195133b26bb..59882da3e742 100644 --- a/arch/powerpc/platforms/powernv/pci-sriov.c +++ b/arch/powerpc/platforms/powernv/pci-sriov.c @@ -594,12 +594,12 @@ static void pnv_pci_sriov_disable(struct pci_dev *pdev) struct pnv_iov_data *iov; iov = pnv_iov_get(pdev); - num_vfs = iov->num_vfs; - base_pe = iov->vf_pe_arr[0].pe_number; - if (WARN_ON(!iov)) return; + num_vfs = iov->num_vfs; + base_pe = iov->vf_pe_arr[0].pe_number; + /* Release VF PEs */ pnv_ioda_release_vf_PE(pdev); diff --git a/arch/powerpc/platforms/powernv/pci.c b/arch/powerpc/platforms/powernv/pci.c index 7725492097b6..35f566aa0424 100644 --- a/arch/powerpc/platforms/powernv/pci.c +++ b/arch/powerpc/platforms/powernv/pci.c @@ -845,11 +845,6 @@ void __init pnv_pci_init(void) pcie_ports_disabled = true; #endif - /* Look for IODA IO-Hubs. */ - for_each_compatible_node(np, NULL, "ibm,ioda-hub") { - pnv_pci_init_ioda_hub(np); - } - /* Look for ioda2 built-in PHB3's */ for_each_compatible_node(np, NULL, "ibm,ioda2-phb") pnv_pci_init_ioda2_phb(np); diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h index f12643958b8d..957f2b47a3c0 100644 --- a/arch/powerpc/platforms/powernv/pci.h +++ b/arch/powerpc/platforms/powernv/pci.h @@ -10,7 +10,6 @@ struct pci_dn; enum pnv_phb_type { - PNV_PHB_IODA1, PNV_PHB_IODA2, PNV_PHB_NPU_OCAPI, }; @@ -163,10 +162,6 @@ struct pnv_phb { unsigned int *m32_segmap; unsigned int *io_segmap; - /* DMA32 segment maps - IODA1 only */ - unsigned int dma32_count; - unsigned int *dma32_segmap; - /* IRQ chip */ int irq_chip_init; struct irq_chip irq_chip; diff --git a/arch/powerpc/platforms/powernv/vas-window.c b/arch/powerpc/platforms/powernv/vas-window.c index 0072682531d8..b664838008c1 100644 --- a/arch/powerpc/platforms/powernv/vas-window.c +++ b/arch/powerpc/platforms/powernv/vas-window.c @@ -1310,8 +1310,8 @@ int vas_win_close(struct vas_window *vwin) /* if send window, drop reference to matching receive window */ if (window->tx_win) { if (window->user_win) { - put_vas_user_win_ref(&vwin->task_ref); mm_context_remove_vas_window(vwin->task_ref.mm); + put_vas_user_win_ref(&vwin->task_ref); } put_rx_win(window->rxwin); } diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c index d59e8a98a200..d593a7227dc9 100644 --- a/arch/powerpc/platforms/pseries/iommu.c +++ b/arch/powerpc/platforms/pseries/iommu.c @@ -372,6 +372,7 @@ struct dynamic_dma_window_prop { struct dma_win { struct device_node *device; const struct dynamic_dma_window_prop *prop; + bool direct; struct list_head list; }; @@ -948,6 +949,7 @@ static struct dma_win *ddw_list_new_entry(struct device_node *pdn, window->device = pdn; window->prop = dma64; + window->direct = false; return window; } @@ -1418,6 +1420,8 @@ static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn) goto out_del_prop; if (direct_mapping) { + window->direct = true; + /* DDW maps the whole partition, so enable direct DMA mapping */ ret = walk_system_ram_range(0, memblock_end_of_DRAM() >> PAGE_SHIFT, win64->value, tce_setrange_multi_pSeriesLP_walk); @@ -1434,6 +1438,8 @@ static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn) int i; unsigned long start = 0, end = 0; + window->direct = false; + for (i = 0; i < ARRAY_SIZE(pci->phb->mem_resources); i++) { const unsigned long mask = IORESOURCE_MEM_64 | IORESOURCE_MEM; @@ -1596,8 +1602,10 @@ static int iommu_mem_notifier(struct notifier_block *nb, unsigned long action, case MEM_GOING_ONLINE: spin_lock(&dma_win_list_lock); list_for_each_entry(window, &dma_win_list, list) { - ret |= tce_setrange_multi_pSeriesLP(arg->start_pfn, - arg->nr_pages, window->prop); + if (window->direct) { + ret |= tce_setrange_multi_pSeriesLP(arg->start_pfn, + arg->nr_pages, window->prop); + } /* XXX log error */ } spin_unlock(&dma_win_list_lock); @@ -1606,8 +1614,10 @@ static int iommu_mem_notifier(struct notifier_block *nb, unsigned long action, case MEM_OFFLINE: spin_lock(&dma_win_list_lock); list_for_each_entry(window, &dma_win_list, list) { - ret |= tce_clearrange_multi_pSeriesLP(arg->start_pfn, - arg->nr_pages, window->prop); + if (window->direct) { + ret |= tce_clearrange_multi_pSeriesLP(arg->start_pfn, + arg->nr_pages, window->prop); + } /* XXX log error */ } spin_unlock(&dma_win_list_lock); diff --git a/arch/powerpc/platforms/pseries/vas.c b/arch/powerpc/platforms/pseries/vas.c index 513180467562..9a44a98ba342 100644 --- a/arch/powerpc/platforms/pseries/vas.c +++ b/arch/powerpc/platforms/pseries/vas.c @@ -507,8 +507,8 @@ static int vas_deallocate_window(struct vas_window *vwin) vascaps[win->win_type].nr_open_windows--; mutex_unlock(&vas_pseries_mutex); - put_vas_user_win_ref(&vwin->task_ref); mm_context_remove_vas_window(vwin->task_ref.mm); + put_vas_user_win_ref(&vwin->task_ref); kfree(win); return 0; diff --git a/arch/powerpc/sysdev/fsl_pci.c b/arch/powerpc/sysdev/fsl_pci.c index b7232c46b244..6daf620b63a4 100644 --- a/arch/powerpc/sysdev/fsl_pci.c +++ b/arch/powerpc/sysdev/fsl_pci.c @@ -1353,6 +1353,7 @@ static struct platform_driver fsl_pci_driver = { .of_match_table = pci_ids, }, .probe = fsl_pci_probe, + .driver_managed_dma = true, }; static int __init fsl_pci_init(void) diff --git a/arch/powerpc/sysdev/fsl_rio.c b/arch/powerpc/sysdev/fsl_rio.c index f8e492ee54cc..0331962bc6d2 100644 --- a/arch/powerpc/sysdev/fsl_rio.c +++ b/arch/powerpc/sysdev/fsl_rio.c @@ -448,13 +448,11 @@ int fsl_rio_setup(struct platform_device *dev) struct rio_mport *port; struct rio_priv *priv; int rc = 0; - const u32 *dt_range, *cell, *port_index; + const u32 *port_index; u32 active_ports = 0; struct device_node *np, *rmu_node; - int rlen; u32 ccsr; - u64 range_start, range_size; - int paw, aw, sw; + u64 range_start; u32 i; static int tmp; struct device_node *rmu_np[MAX_MSG_UNIT_NUM] = {NULL}; @@ -528,15 +526,12 @@ int fsl_rio_setup(struct platform_device *dev) dbell->bellirq = irq_of_parse_and_map(np, 1); dev_info(&dev->dev, "bellirq: %d\n", dbell->bellirq); - aw = of_n_addr_cells(np); - dt_range = of_get_property(np, "reg", &rlen); - if (!dt_range) { + if (of_property_read_reg(np, 0, &range_start, NULL)) { pr_err("%pOF: unable to find 'reg' property\n", np); rc = -ENOMEM; goto err_pw; } - range_start = of_read_number(dt_range, aw); dbell->dbell_regs = (struct rio_dbell_regs *)(rmu_regs_win + (u32)range_start); @@ -556,19 +551,18 @@ int fsl_rio_setup(struct platform_device *dev) pw->dev = &dev->dev; pw->pwirq = irq_of_parse_and_map(np, 0); dev_info(&dev->dev, "pwirq: %d\n", pw->pwirq); - aw = of_n_addr_cells(np); - dt_range = of_get_property(np, "reg", &rlen); - if (!dt_range) { + if (of_property_read_reg(np, 0, &range_start, NULL)) { pr_err("%pOF: unable to find 'reg' property\n", np); rc = -ENOMEM; goto err; } - range_start = of_read_number(dt_range, aw); pw->pw_regs = (struct rio_pw_regs *)(rmu_regs_win + (u32)range_start); /*set up ports node*/ for_each_child_of_node(dev->dev.of_node, np) { + struct resource res; + port_index = of_get_property(np, "cell-index", NULL); if (!port_index) { dev_err(&dev->dev, "Can't get %pOF property 'cell-index'\n", @@ -576,32 +570,14 @@ int fsl_rio_setup(struct platform_device *dev) continue; } - dt_range = of_get_property(np, "ranges", &rlen); - if (!dt_range) { + if (of_range_to_resource(np, 0, &res)) { dev_err(&dev->dev, "Can't get %pOF property 'ranges'\n", np); continue; } - /* Get node address wide */ - cell = of_get_property(np, "#address-cells", NULL); - if (cell) - aw = *cell; - else - aw = of_n_addr_cells(np); - /* Get node size wide */ - cell = of_get_property(np, "#size-cells", NULL); - if (cell) - sw = *cell; - else - sw = of_n_size_cells(np); - /* Get parent address wide wide */ - paw = of_n_addr_cells(np); - range_start = of_read_number(dt_range + aw, paw); - range_size = of_read_number(dt_range + aw + paw, sw); - - dev_info(&dev->dev, "%pOF: LAW start 0x%016llx, size 0x%016llx.\n", - np, range_start, range_size); + dev_info(&dev->dev, "%pOF: LAW %pR\n", + np, &res); port = kzalloc(sizeof(struct rio_mport), GFP_KERNEL); if (!port) @@ -624,9 +600,7 @@ int fsl_rio_setup(struct platform_device *dev) } INIT_LIST_HEAD(&port->dbells); - port->iores.start = range_start; - port->iores.end = port->iores.start + range_size - 1; - port->iores.flags = IORESOURCE_MEM; + port->iores = res; /* struct copy */ port->iores.name = "rio_io_win"; if (request_resource(&iomem_resource, &port->iores) < 0) { diff --git a/arch/powerpc/sysdev/fsl_rmu.c b/arch/powerpc/sysdev/fsl_rmu.c index 7a5e2e2b9d06..c1f724973589 100644 --- a/arch/powerpc/sysdev/fsl_rmu.c +++ b/arch/powerpc/sysdev/fsl_rmu.c @@ -23,6 +23,7 @@ #include <linux/types.h> #include <linux/dma-mapping.h> #include <linux/interrupt.h> +#include <linux/of_address.h> #include <linux/of_irq.h> #include <linux/of_platform.h> #include <linux/slab.h> @@ -1067,9 +1068,6 @@ int fsl_rio_setup_rmu(struct rio_mport *mport, struct device_node *node) struct rio_priv *priv; struct fsl_rmu *rmu; u64 msg_start; - const u32 *msg_addr; - int mlen; - int aw; if (!mport || !mport->priv) return -EINVAL; @@ -1086,16 +1084,12 @@ int fsl_rio_setup_rmu(struct rio_mport *mport, struct device_node *node) if (!rmu) return -ENOMEM; - aw = of_n_addr_cells(node); - msg_addr = of_get_property(node, "reg", &mlen); - if (!msg_addr) { + if (of_property_read_reg(node, 0, &msg_start, NULL)) { pr_err("%pOF: unable to find 'reg' property of message-unit\n", node); kfree(rmu); return -ENOMEM; } - msg_start = of_read_number(msg_addr, aw); - rmu->msg_regs = (struct rio_msg_regs *) (rmu_regs_win + (u32)msg_start); diff --git a/arch/powerpc/sysdev/fsl_soc.c b/arch/powerpc/sysdev/fsl_soc.c index 6ebbbca41065..68709743450e 100644 --- a/arch/powerpc/sysdev/fsl_soc.c +++ b/arch/powerpc/sysdev/fsl_soc.c @@ -51,18 +51,10 @@ phys_addr_t get_immrbase(void) soc = of_find_node_by_type(NULL, "soc"); if (soc) { - int size; - u32 naddr; - const __be32 *prop = of_get_property(soc, "#address-cells", &size); - - if (prop && size == 4) - naddr = be32_to_cpup(prop); - else - naddr = 2; - - prop = of_get_property(soc, "ranges", &size); - if (prop) - immrbase = of_translate_address(soc, prop + naddr); + struct resource res; + + if (!of_range_to_resource(soc, 0, &res)) + immrbase = res.start; of_node_put(soc); } diff --git a/arch/powerpc/tools/gcc-check-mprofile-kernel.sh b/arch/powerpc/tools/gcc-check-mprofile-kernel.sh index 137f3376ac2b..a31a56016c09 100755 --- a/arch/powerpc/tools/gcc-check-mprofile-kernel.sh +++ b/arch/powerpc/tools/gcc-check-mprofile-kernel.sh @@ -7,20 +7,21 @@ set -o pipefail # To debug, uncomment the following line # set -x -# -mprofile-kernel is only supported on 64le, so this should not be invoked -# for other targets. Therefore we can pass in -m64 and -mlittle-endian -# explicitly, to take care of toolchains defaulting to other targets. +# -mprofile-kernel is only supported on 64-bit, so this should not be invoked +# for 32-bit. We pass in -m64 explicitly, and -mbig-endian and -mlittle-endian +# are passed in from Kconfig, which takes care of toolchains defaulting to +# other targets. # Test whether the compile option -mprofile-kernel exists and generates # profiling code (ie. a call to _mcount()). echo "int func() { return 0; }" | \ - $* -m64 -mlittle-endian -S -x c -O2 -p -mprofile-kernel - -o - \ + $* -m64 -S -x c -O2 -p -mprofile-kernel - -o - \ 2> /dev/null | grep -q "_mcount" # Test whether the notrace attribute correctly suppresses calls to _mcount(). echo -e "#include <linux/compiler.h>\nnotrace int func() { return 0; }" | \ - $* -m64 -mlittle-endian -S -x c -O2 -p -mprofile-kernel - -o - \ + $* -m64 -S -x c -O2 -p -mprofile-kernel - -o - \ 2> /dev/null | grep -q "_mcount" && \ exit 1 diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index a08917f681af..b49793cf34eb 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -12,6 +12,8 @@ config 32BIT config RISCV def_bool y + select ACPI_GENERIC_GSI if ACPI + select ACPI_REDUCED_HARDWARE_ONLY if ACPI select ARCH_DMA_DEFAULT_COHERENT select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 @@ -43,6 +45,7 @@ config RISCV select ARCH_SUPPORTS_DEBUG_PAGEALLOC if MMU select ARCH_SUPPORTS_HUGETLBFS if MMU select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU + select ARCH_SUPPORTS_PER_VMA_LOCK if MMU select ARCH_USE_MEMTEST select ARCH_USE_QUEUED_RWLOCKS select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU @@ -149,6 +152,7 @@ config RISCV select HAVE_DYNAMIC_FTRACE_WITH_REGS if HAVE_DYNAMIC_FTRACE select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL select HAVE_FUNCTION_GRAPH_TRACER + select HAVE_FUNCTION_GRAPH_RETVAL if HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_TRACER if !XIP_KERNEL && !PREEMPTION config CLANG_SUPPORTS_DYNAMIC_FTRACE @@ -265,6 +269,12 @@ config RISCV_DMA_NONCOHERENT config AS_HAS_INSN def_bool $(as-instr,.insn r 51$(comma) 0$(comma) 0$(comma) t0$(comma) t0$(comma) zero) +config AS_HAS_OPTION_ARCH + # https://reviews.llvm.org/D123515 + def_bool y + depends on $(as-instr, .option arch$(comma) +m) + depends on !$(as-instr, .option arch$(comma) -i) + source "arch/riscv/Kconfig.socs" source "arch/riscv/Kconfig.errata" @@ -463,13 +473,44 @@ config RISCV_ISA_SVPBMT If you don't know what to do here, say Y. +config TOOLCHAIN_HAS_V + bool + default y + depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64iv) + depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32iv) + depends on LLD_VERSION >= 140000 || LD_VERSION >= 23800 + depends on AS_HAS_OPTION_ARCH + +config RISCV_ISA_V + bool "VECTOR extension support" + depends on TOOLCHAIN_HAS_V + depends on FPU + select DYNAMIC_SIGFRAME + default y + help + Say N here if you want to disable all vector related procedure + in the kernel. + + If you don't know what to do here, say Y. + +config RISCV_ISA_V_DEFAULT_ENABLE + bool "Enable userspace Vector by default" + depends on RISCV_ISA_V + default y + help + Say Y here if you want to enable Vector in userspace by default. + Otherwise, userspace has to make explicit prctl() call to enable + Vector, or enable it via the sysctl interface. + + If you don't know what to do here, say Y. + config TOOLCHAIN_HAS_ZBB bool default y depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zbb) depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zbb) depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900 - depends on AS_IS_GNU + depends on AS_HAS_OPTION_ARCH config RISCV_ISA_ZBB bool "Zbb extension support for bit manipulation instructions" @@ -554,6 +595,25 @@ config FPU If you don't know what to do here, say Y. +config IRQ_STACKS + bool "Independent irq & softirq stacks" if EXPERT + default y + select HAVE_IRQ_EXIT_ON_IRQ_STACK + select HAVE_SOFTIRQ_ON_OWN_STACK + help + Add independent irq & softirq stacks for percpu to prevent kernel stack + overflows. We may save some memory footprint by disabling IRQ_STACKS. + +config THREAD_SIZE_ORDER + int "Kernel stack size (in power-of-two numbers of page size)" if VMAP_STACK && EXPERT + range 0 4 + default 1 if 32BIT && !KASAN + default 3 if 64BIT && KASAN + default 2 + help + Specify the Pages of thread stack size (from 4KB to 64KB), which also + affects irq stack size, which is equal to thread stack size. + endmenu # "Platform type" menu "Kernel features" @@ -710,6 +770,7 @@ config EFI depends on OF && !XIP_KERNEL depends on MMU default y + select ARCH_SUPPORTS_ACPI if 64BIT select EFI_GENERIC_STUB select EFI_PARAMS_FROM_FDT select EFI_RUNTIME_WRAPPERS @@ -822,3 +883,5 @@ source "drivers/cpufreq/Kconfig" endmenu # "CPU Power Management" source "arch/riscv/kvm/Kconfig" + +source "drivers/acpi/Kconfig" diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index 0fb256bf8270..6ec6d52a4180 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -60,6 +60,7 @@ riscv-march-$(CONFIG_ARCH_RV32I) := rv32ima riscv-march-$(CONFIG_ARCH_RV64I) := rv64ima riscv-march-$(CONFIG_FPU) := $(riscv-march-y)fd riscv-march-$(CONFIG_RISCV_ISA_C) := $(riscv-march-y)c +riscv-march-$(CONFIG_RISCV_ISA_V) := $(riscv-march-y)v ifdef CONFIG_TOOLCHAIN_NEEDS_OLD_ISA_SPEC KBUILD_CFLAGS += -Wa,-misa-spec=2.2 @@ -71,7 +72,10 @@ endif # Check if the toolchain supports Zihintpause extension riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZIHINTPAUSE) := $(riscv-march-y)_zihintpause -KBUILD_CFLAGS += -march=$(subst fd,,$(riscv-march-y)) +# Remove F,D,V from isa string for all. Keep extensions between "fd" and "v" by +# matching non-v and non-multi-letter extensions out with the filter ([^v_]*) +KBUILD_CFLAGS += -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64ima)fd([^v_]*)v?/\1\2/') + KBUILD_AFLAGS += -march=$(riscv-march-y) KBUILD_CFLAGS += -mno-save-restore diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index 109e4b5b003c..0a0107460a5c 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -38,6 +38,7 @@ CONFIG_PM=y CONFIG_CPU_IDLE=y CONFIG_VIRTUALIZATION=y CONFIG_KVM=m +CONFIG_ACPI=y CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y diff --git a/arch/riscv/include/asm/acenv.h b/arch/riscv/include/asm/acenv.h new file mode 100644 index 000000000000..43ae2e32c779 --- /dev/null +++ b/arch/riscv/include/asm/acenv.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * RISC-V specific ACPICA environments and implementation + */ + +#ifndef _ASM_ACENV_H +#define _ASM_ACENV_H + +/* This header is required unconditionally by the ACPI core */ + +#endif /* _ASM_ACENV_H */ diff --git a/arch/riscv/include/asm/acpi.h b/arch/riscv/include/asm/acpi.h new file mode 100644 index 000000000000..f71ce21ff684 --- /dev/null +++ b/arch/riscv/include/asm/acpi.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2013-2014, Linaro Ltd. + * Author: Al Stone <al.stone@linaro.org> + * Author: Graeme Gregory <graeme.gregory@linaro.org> + * Author: Hanjun Guo <hanjun.guo@linaro.org> + * + * Copyright (C) 2021-2023, Ventana Micro Systems Inc. + * Author: Sunil V L <sunilvl@ventanamicro.com> + */ + +#ifndef _ASM_ACPI_H +#define _ASM_ACPI_H + +/* Basic configuration for ACPI */ +#ifdef CONFIG_ACPI + +typedef u64 phys_cpuid_t; +#define PHYS_CPUID_INVALID INVALID_HARTID + +/* ACPI table mapping after acpi_permanent_mmap is set */ +void *acpi_os_ioremap(acpi_physical_address phys, acpi_size size); +#define acpi_os_ioremap acpi_os_ioremap + +#define acpi_strict 1 /* No out-of-spec workarounds on RISC-V */ +extern int acpi_disabled; +extern int acpi_noirq; +extern int acpi_pci_disabled; + +static inline void disable_acpi(void) +{ + acpi_disabled = 1; + acpi_pci_disabled = 1; + acpi_noirq = 1; +} + +static inline void enable_acpi(void) +{ + acpi_disabled = 0; + acpi_pci_disabled = 0; + acpi_noirq = 0; +} + +/* + * The ACPI processor driver for ACPI core code needs this macro + * to find out whether this cpu was already mapped (mapping from CPU hardware + * ID to CPU logical ID) or not. + */ +#define cpu_physical_id(cpu) cpuid_to_hartid_map(cpu) + +/* + * Since MADT must provide at least one RINTC structure, the + * CPU will be always available in MADT on RISC-V. + */ +static inline bool acpi_has_cpu_in_madt(void) +{ + return true; +} + +static inline void arch_fix_phys_package_id(int num, u32 slot) { } + +void acpi_init_rintc_map(void); +struct acpi_madt_rintc *acpi_cpu_get_madt_rintc(int cpu); +u32 get_acpi_id_for_cpu(int cpu); +int acpi_get_riscv_isa(struct acpi_table_header *table, + unsigned int cpu, const char **isa); + +static inline int acpi_numa_get_nid(unsigned int cpu) { return NUMA_NO_NODE; } +#else +static inline void acpi_init_rintc_map(void) { } +static inline struct acpi_madt_rintc *acpi_cpu_get_madt_rintc(int cpu) +{ + return NULL; +} + +static inline int acpi_get_riscv_isa(struct acpi_table_header *table, + unsigned int cpu, const char **isa) +{ + return -EINVAL; +} + +#endif /* CONFIG_ACPI */ + +#endif /*_ASM_ACPI_H*/ diff --git a/arch/riscv/include/asm/asm-extable.h b/arch/riscv/include/asm/asm-extable.h index 14be0673f5b5..00a96e7a9664 100644 --- a/arch/riscv/include/asm/asm-extable.h +++ b/arch/riscv/include/asm/asm-extable.h @@ -7,6 +7,8 @@ #define EX_TYPE_BPF 2 #define EX_TYPE_UACCESS_ERR_ZERO 3 +#ifdef CONFIG_MMU + #ifdef __ASSEMBLY__ #define __ASM_EXTABLE_RAW(insn, fixup, type, data) \ @@ -62,4 +64,8 @@ #endif /* __ASSEMBLY__ */ +#else /* CONFIG_MMU */ + #define _ASM_EXTABLE_UACCESS_ERR(insn, fixup, err) +#endif /* CONFIG_MMU */ + #endif /* __ASM_ASM_EXTABLE_H */ diff --git a/arch/riscv/include/asm/cpu.h b/arch/riscv/include/asm/cpu.h new file mode 100644 index 000000000000..28d45a6678ce --- /dev/null +++ b/arch/riscv/include/asm/cpu.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _ASM_CPU_H +#define _ASM_CPU_H + +/* This header is required unconditionally by the ACPI core */ + +#endif /* _ASM_CPU_H */ diff --git a/arch/riscv/include/asm/cpufeature.h b/arch/riscv/include/asm/cpufeature.h index 808d5403f2ac..23fed53b8815 100644 --- a/arch/riscv/include/asm/cpufeature.h +++ b/arch/riscv/include/asm/cpufeature.h @@ -6,6 +6,9 @@ #ifndef _ASM_CPUFEATURE_H #define _ASM_CPUFEATURE_H +#include <linux/bitmap.h> +#include <asm/hwcap.h> + /* * These are probed via a device_initcall(), via either the SBI or directly * from the corresponding CSRs. @@ -16,8 +19,15 @@ struct riscv_cpuinfo { unsigned long mimpid; }; +struct riscv_isainfo { + DECLARE_BITMAP(isa, RISCV_ISA_EXT_MAX); +}; + DECLARE_PER_CPU(struct riscv_cpuinfo, riscv_cpuinfo); DECLARE_PER_CPU(long, misaligned_access_speed); +/* Per-cpu ISA extensions. */ +extern struct riscv_isainfo hart_isa[NR_CPUS]; + #endif diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h index b6acb7ed115f..7bac43a3176e 100644 --- a/arch/riscv/include/asm/csr.h +++ b/arch/riscv/include/asm/csr.h @@ -24,16 +24,24 @@ #define SR_FS_CLEAN _AC(0x00004000, UL) #define SR_FS_DIRTY _AC(0x00006000, UL) +#define SR_VS _AC(0x00000600, UL) /* Vector Status */ +#define SR_VS_OFF _AC(0x00000000, UL) +#define SR_VS_INITIAL _AC(0x00000200, UL) +#define SR_VS_CLEAN _AC(0x00000400, UL) +#define SR_VS_DIRTY _AC(0x00000600, UL) + #define SR_XS _AC(0x00018000, UL) /* Extension Status */ #define SR_XS_OFF _AC(0x00000000, UL) #define SR_XS_INITIAL _AC(0x00008000, UL) #define SR_XS_CLEAN _AC(0x00010000, UL) #define SR_XS_DIRTY _AC(0x00018000, UL) +#define SR_FS_VS (SR_FS | SR_VS) /* Vector and Floating-Point Unit */ + #ifndef CONFIG_64BIT -#define SR_SD _AC(0x80000000, UL) /* FS/XS dirty */ +#define SR_SD _AC(0x80000000, UL) /* FS/VS/XS dirty */ #else -#define SR_SD _AC(0x8000000000000000, UL) /* FS/XS dirty */ +#define SR_SD _AC(0x8000000000000000, UL) /* FS/VS/XS dirty */ #endif #ifdef CONFIG_64BIT @@ -82,7 +90,9 @@ #define EXC_INST_ACCESS 1 #define EXC_INST_ILLEGAL 2 #define EXC_BREAKPOINT 3 +#define EXC_LOAD_MISALIGNED 4 #define EXC_LOAD_ACCESS 5 +#define EXC_STORE_MISALIGNED 6 #define EXC_STORE_ACCESS 7 #define EXC_SYSCALL 8 #define EXC_HYPERVISOR_SYSCALL 9 @@ -375,6 +385,12 @@ #define CSR_MVIPH 0x319 #define CSR_MIPH 0x354 +#define CSR_VSTART 0x8 +#define CSR_VCSR 0xf +#define CSR_VL 0xc20 +#define CSR_VTYPE 0xc21 +#define CSR_VLENB 0xc22 + #ifdef CONFIG_RISCV_M_MODE # define CSR_STATUS CSR_MSTATUS # define CSR_IE CSR_MIE diff --git a/arch/riscv/include/asm/elf.h b/arch/riscv/include/asm/elf.h index 30e7d2455960..c24280774caf 100644 --- a/arch/riscv/include/asm/elf.h +++ b/arch/riscv/include/asm/elf.h @@ -66,7 +66,7 @@ extern bool compat_elf_check_arch(Elf32_Ehdr *hdr); * via a bitmap that coorespends to each single-letter ISA extension. This is * essentially defunct, but will remain for compatibility with userspace. */ -#define ELF_HWCAP (elf_hwcap & ((1UL << RISCV_ISA_EXT_BASE) - 1)) +#define ELF_HWCAP riscv_get_elf_hwcap() extern unsigned long elf_hwcap; /* @@ -105,6 +105,15 @@ do { \ get_cache_size(3, CACHE_TYPE_UNIFIED)); \ NEW_AUX_ENT(AT_L3_CACHEGEOMETRY, \ get_cache_geometry(3, CACHE_TYPE_UNIFIED)); \ + /* \ + * Should always be nonzero unless there's a kernel bug. \ + * If we haven't determined a sensible value to give to \ + * userspace, omit the entry: \ + */ \ + if (likely(signal_minsigstksz)) \ + NEW_AUX_ENT(AT_MINSIGSTKSZ, signal_minsigstksz); \ + else \ + NEW_AUX_ENT(AT_IGNORE, 0); \ } while (0) #define ARCH_HAS_SETUP_ADDITIONAL_PAGES struct linux_binprm; diff --git a/arch/riscv/include/asm/extable.h b/arch/riscv/include/asm/extable.h index 512012d193dc..3eb5c1f7bf34 100644 --- a/arch/riscv/include/asm/extable.h +++ b/arch/riscv/include/asm/extable.h @@ -32,7 +32,11 @@ do { \ (b)->data = (tmp).data; \ } while (0) +#ifdef CONFIG_MMU bool fixup_exception(struct pt_regs *regs); +#else +static inline bool fixup_exception(struct pt_regs *regs) { return false; } +#endif #if defined(CONFIG_BPF_JIT) && defined(CONFIG_ARCH_RV64I) bool ex_handler_bpf(const struct exception_table_entry *ex, struct pt_regs *regs); diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h index d47d87c2d7e3..740a979171e5 100644 --- a/arch/riscv/include/asm/ftrace.h +++ b/arch/riscv/include/asm/ftrace.h @@ -111,4 +111,25 @@ int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec); #endif /* CONFIG_DYNAMIC_FTRACE */ +#ifndef __ASSEMBLY__ +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +struct fgraph_ret_regs { + unsigned long a1; + unsigned long a0; + unsigned long s0; + unsigned long ra; +}; + +static inline unsigned long fgraph_ret_regs_return_value(struct fgraph_ret_regs *ret_regs) +{ + return ret_regs->a0; +} + +static inline unsigned long fgraph_ret_regs_frame_pointer(struct fgraph_ret_regs *ret_regs) +{ + return ret_regs->s0; +} +#endif /* ifdef CONFIG_FUNCTION_GRAPH_TRACER */ +#endif + #endif /* _ASM_RISCV_FTRACE_H */ diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index e0c40a4c63d5..f041bfa7f6a0 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -22,6 +22,7 @@ #define RISCV_ISA_EXT_m ('m' - 'a') #define RISCV_ISA_EXT_s ('s' - 'a') #define RISCV_ISA_EXT_u ('u' - 'a') +#define RISCV_ISA_EXT_v ('v' - 'a') /* * These macros represent the logical IDs of each multi-letter RISC-V ISA @@ -46,6 +47,12 @@ #define RISCV_ISA_EXT_ZICBOZ 34 #define RISCV_ISA_EXT_SMAIA 35 #define RISCV_ISA_EXT_SSAIA 36 +#define RISCV_ISA_EXT_ZBA 37 +#define RISCV_ISA_EXT_ZBS 38 +#define RISCV_ISA_EXT_ZICNTR 39 +#define RISCV_ISA_EXT_ZICSR 40 +#define RISCV_ISA_EXT_ZIFENCEI 41 +#define RISCV_ISA_EXT_ZIHPM 42 #define RISCV_ISA_EXT_MAX 64 #define RISCV_ISA_EXT_NAME_LEN_MAX 32 @@ -60,6 +67,8 @@ #include <linux/jump_label.h> +unsigned long riscv_get_elf_hwcap(void); + struct riscv_isa_ext_data { /* Name of the extension displayed to userspace via /proc/cpuinfo */ char uprop[RISCV_ISA_EXT_NAME_LEN_MAX]; diff --git a/arch/riscv/include/asm/insn.h b/arch/riscv/include/asm/insn.h index 8d5c84f2d5ef..4e1505cef8aa 100644 --- a/arch/riscv/include/asm/insn.h +++ b/arch/riscv/include/asm/insn.h @@ -137,6 +137,26 @@ #define RVG_OPCODE_JALR 0x67 #define RVG_OPCODE_JAL 0x6f #define RVG_OPCODE_SYSTEM 0x73 +#define RVG_SYSTEM_CSR_OFF 20 +#define RVG_SYSTEM_CSR_MASK GENMASK(12, 0) + +/* parts of opcode for RVF, RVD and RVQ */ +#define RVFDQ_FL_FS_WIDTH_OFF 12 +#define RVFDQ_FL_FS_WIDTH_MASK GENMASK(3, 0) +#define RVFDQ_FL_FS_WIDTH_W 2 +#define RVFDQ_FL_FS_WIDTH_D 3 +#define RVFDQ_LS_FS_WIDTH_Q 4 +#define RVFDQ_OPCODE_FL 0x07 +#define RVFDQ_OPCODE_FS 0x27 + +/* parts of opcode for RVV */ +#define RVV_OPCODE_VECTOR 0x57 +#define RVV_VL_VS_WIDTH_8 0 +#define RVV_VL_VS_WIDTH_16 5 +#define RVV_VL_VS_WIDTH_32 6 +#define RVV_VL_VS_WIDTH_64 7 +#define RVV_OPCODE_VL RVFDQ_OPCODE_FL +#define RVV_OPCODE_VS RVFDQ_OPCODE_FS /* parts of opcode for RVC*/ #define RVC_OPCODE_C0 0x0 @@ -304,6 +324,15 @@ static __always_inline bool riscv_insn_is_branch(u32 code) (RVC_X(x_, RVC_B_IMM_7_6_OPOFF, RVC_B_IMM_7_6_MASK) << RVC_B_IMM_7_6_OFF) | \ (RVC_IMM_SIGN(x_) << RVC_B_IMM_SIGN_OFF); }) +#define RVG_EXTRACT_SYSTEM_CSR(x) \ + ({typeof(x) x_ = (x); RV_X(x_, RVG_SYSTEM_CSR_OFF, RVG_SYSTEM_CSR_MASK); }) + +#define RVFDQ_EXTRACT_FL_FS_WIDTH(x) \ + ({typeof(x) x_ = (x); RV_X(x_, RVFDQ_FL_FS_WIDTH_OFF, \ + RVFDQ_FL_FS_WIDTH_MASK); }) + +#define RVV_EXRACT_VL_VS_WIDTH(x) RVFDQ_EXTRACT_FL_FS_WIDTH(x) + /* * Get the immediate from a J-type instruction. * diff --git a/arch/riscv/include/asm/irq_stack.h b/arch/riscv/include/asm/irq_stack.h new file mode 100644 index 000000000000..e4042d297580 --- /dev/null +++ b/arch/riscv/include/asm/irq_stack.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_RISCV_IRQ_STACK_H +#define _ASM_RISCV_IRQ_STACK_H + +#include <linux/bug.h> +#include <linux/gfp.h> +#include <linux/kconfig.h> +#include <linux/vmalloc.h> +#include <linux/pgtable.h> +#include <asm/thread_info.h> + +DECLARE_PER_CPU(ulong *, irq_stack_ptr); + +#ifdef CONFIG_VMAP_STACK +/* + * To ensure that VMAP'd stack overflow detection works correctly, all VMAP'd + * stacks need to have the same alignment. + */ +static inline unsigned long *arch_alloc_vmap_stack(size_t stack_size, int node) +{ + void *p; + + p = __vmalloc_node(stack_size, THREAD_ALIGN, THREADINFO_GFP, node, + __builtin_return_address(0)); + return kasan_reset_tag(p); +} +#endif /* CONFIG_VMAP_STACK */ + +#endif /* _ASM_RISCV_IRQ_STACK_H */ diff --git a/arch/riscv/include/asm/kvm_aia.h b/arch/riscv/include/asm/kvm_aia.h index 1de0717112e5..1f37b600ca47 100644 --- a/arch/riscv/include/asm/kvm_aia.h +++ b/arch/riscv/include/asm/kvm_aia.h @@ -20,6 +20,33 @@ struct kvm_aia { /* In-kernel irqchip initialized */ bool initialized; + + /* Virtualization mode (Emulation, HW Accelerated, or Auto) */ + u32 mode; + + /* Number of MSIs */ + u32 nr_ids; + + /* Number of wired IRQs */ + u32 nr_sources; + + /* Number of group bits in IMSIC address */ + u32 nr_group_bits; + + /* Position of group bits in IMSIC address */ + u32 nr_group_shift; + + /* Number of hart bits in IMSIC address */ + u32 nr_hart_bits; + + /* Number of guest bits in IMSIC address */ + u32 nr_guest_bits; + + /* Guest physical address of APLIC */ + gpa_t aplic_addr; + + /* Internal state of APLIC */ + void *aplic_state; }; struct kvm_vcpu_aia_csr { @@ -38,25 +65,53 @@ struct kvm_vcpu_aia { /* CPU AIA CSR context upon Guest VCPU reset */ struct kvm_vcpu_aia_csr guest_reset_csr; + + /* Guest physical address of IMSIC for this VCPU */ + gpa_t imsic_addr; + + /* HART index of IMSIC extacted from guest physical address */ + u32 hart_index; + + /* Internal state of IMSIC for this VCPU */ + void *imsic_state; }; +#define KVM_RISCV_AIA_UNDEF_ADDR (-1) + #define kvm_riscv_aia_initialized(k) ((k)->arch.aia.initialized) #define irqchip_in_kernel(k) ((k)->arch.aia.in_kernel) +extern unsigned int kvm_riscv_aia_nr_hgei; +extern unsigned int kvm_riscv_aia_max_ids; DECLARE_STATIC_KEY_FALSE(kvm_riscv_aia_available); #define kvm_riscv_aia_available() \ static_branch_unlikely(&kvm_riscv_aia_available) +extern struct kvm_device_ops kvm_riscv_aia_device_ops; + +void kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu *vcpu); +int kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu *vcpu); + #define KVM_RISCV_AIA_IMSIC_TOPEI (ISELECT_MASK + 1) -static inline int kvm_riscv_vcpu_aia_imsic_rmw(struct kvm_vcpu *vcpu, - unsigned long isel, - unsigned long *val, - unsigned long new_val, - unsigned long wr_mask) -{ - return 0; -} +int kvm_riscv_vcpu_aia_imsic_rmw(struct kvm_vcpu *vcpu, unsigned long isel, + unsigned long *val, unsigned long new_val, + unsigned long wr_mask); +int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type, + bool write, unsigned long *val); +int kvm_riscv_aia_imsic_has_attr(struct kvm *kvm, unsigned long type); +void kvm_riscv_vcpu_aia_imsic_reset(struct kvm_vcpu *vcpu); +int kvm_riscv_vcpu_aia_imsic_inject(struct kvm_vcpu *vcpu, + u32 guest_index, u32 offset, u32 iid); +int kvm_riscv_vcpu_aia_imsic_init(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_aia_imsic_cleanup(struct kvm_vcpu *vcpu); + +int kvm_riscv_aia_aplic_set_attr(struct kvm *kvm, unsigned long type, u32 v); +int kvm_riscv_aia_aplic_get_attr(struct kvm *kvm, unsigned long type, u32 *v); +int kvm_riscv_aia_aplic_has_attr(struct kvm *kvm, unsigned long type); +int kvm_riscv_aia_aplic_inject(struct kvm *kvm, u32 source, bool level); +int kvm_riscv_aia_aplic_init(struct kvm *kvm); +void kvm_riscv_aia_aplic_cleanup(struct kvm *kvm); #ifdef CONFIG_32BIT void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu); @@ -93,31 +148,23 @@ int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num, { .base = CSR_SIREG, .count = 1, .func = kvm_riscv_vcpu_aia_rmw_ireg }, \ { .base = CSR_STOPEI, .count = 1, .func = kvm_riscv_vcpu_aia_rmw_topei }, -static inline int kvm_riscv_vcpu_aia_update(struct kvm_vcpu *vcpu) -{ - return 1; -} - -static inline void kvm_riscv_vcpu_aia_reset(struct kvm_vcpu *vcpu) -{ -} - -static inline int kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu) -{ - return 0; -} +int kvm_riscv_vcpu_aia_update(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_aia_reset(struct kvm_vcpu *vcpu); +int kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_aia_deinit(struct kvm_vcpu *vcpu); -static inline void kvm_riscv_vcpu_aia_deinit(struct kvm_vcpu *vcpu) -{ -} +int kvm_riscv_aia_inject_msi_by_id(struct kvm *kvm, u32 hart_index, + u32 guest_index, u32 iid); +int kvm_riscv_aia_inject_msi(struct kvm *kvm, struct kvm_msi *msi); +int kvm_riscv_aia_inject_irq(struct kvm *kvm, unsigned int irq, bool level); -static inline void kvm_riscv_aia_init_vm(struct kvm *kvm) -{ -} +void kvm_riscv_aia_init_vm(struct kvm *kvm); +void kvm_riscv_aia_destroy_vm(struct kvm *kvm); -static inline void kvm_riscv_aia_destroy_vm(struct kvm *kvm) -{ -} +int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner, + void __iomem **hgei_va, phys_addr_t *hgei_pa); +void kvm_riscv_aia_free_hgei(int cpu, int hgei); +void kvm_riscv_aia_wakeon_hgei(struct kvm_vcpu *owner, bool enable); void kvm_riscv_aia_enable(void); void kvm_riscv_aia_disable(void); diff --git a/arch/riscv/include/asm/kvm_aia_aplic.h b/arch/riscv/include/asm/kvm_aia_aplic.h new file mode 100644 index 000000000000..6dd1a4809ec1 --- /dev/null +++ b/arch/riscv/include/asm/kvm_aia_aplic.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + */ +#ifndef __KVM_RISCV_AIA_IMSIC_H +#define __KVM_RISCV_AIA_IMSIC_H + +#include <linux/bitops.h> + +#define APLIC_MAX_IDC BIT(14) +#define APLIC_MAX_SOURCE 1024 + +#define APLIC_DOMAINCFG 0x0000 +#define APLIC_DOMAINCFG_RDONLY 0x80000000 +#define APLIC_DOMAINCFG_IE BIT(8) +#define APLIC_DOMAINCFG_DM BIT(2) +#define APLIC_DOMAINCFG_BE BIT(0) + +#define APLIC_SOURCECFG_BASE 0x0004 +#define APLIC_SOURCECFG_D BIT(10) +#define APLIC_SOURCECFG_CHILDIDX_MASK 0x000003ff +#define APLIC_SOURCECFG_SM_MASK 0x00000007 +#define APLIC_SOURCECFG_SM_INACTIVE 0x0 +#define APLIC_SOURCECFG_SM_DETACH 0x1 +#define APLIC_SOURCECFG_SM_EDGE_RISE 0x4 +#define APLIC_SOURCECFG_SM_EDGE_FALL 0x5 +#define APLIC_SOURCECFG_SM_LEVEL_HIGH 0x6 +#define APLIC_SOURCECFG_SM_LEVEL_LOW 0x7 + +#define APLIC_IRQBITS_PER_REG 32 + +#define APLIC_SETIP_BASE 0x1c00 +#define APLIC_SETIPNUM 0x1cdc + +#define APLIC_CLRIP_BASE 0x1d00 +#define APLIC_CLRIPNUM 0x1ddc + +#define APLIC_SETIE_BASE 0x1e00 +#define APLIC_SETIENUM 0x1edc + +#define APLIC_CLRIE_BASE 0x1f00 +#define APLIC_CLRIENUM 0x1fdc + +#define APLIC_SETIPNUM_LE 0x2000 +#define APLIC_SETIPNUM_BE 0x2004 + +#define APLIC_GENMSI 0x3000 + +#define APLIC_TARGET_BASE 0x3004 +#define APLIC_TARGET_HART_IDX_SHIFT 18 +#define APLIC_TARGET_HART_IDX_MASK 0x3fff +#define APLIC_TARGET_GUEST_IDX_SHIFT 12 +#define APLIC_TARGET_GUEST_IDX_MASK 0x3f +#define APLIC_TARGET_IPRIO_MASK 0xff +#define APLIC_TARGET_EIID_MASK 0x7ff + +#endif diff --git a/arch/riscv/include/asm/kvm_aia_imsic.h b/arch/riscv/include/asm/kvm_aia_imsic.h new file mode 100644 index 000000000000..da5881d2bde0 --- /dev/null +++ b/arch/riscv/include/asm/kvm_aia_imsic.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + */ +#ifndef __KVM_RISCV_AIA_IMSIC_H +#define __KVM_RISCV_AIA_IMSIC_H + +#include <linux/types.h> +#include <asm/csr.h> + +#define IMSIC_MMIO_PAGE_SHIFT 12 +#define IMSIC_MMIO_PAGE_SZ (1UL << IMSIC_MMIO_PAGE_SHIFT) +#define IMSIC_MMIO_PAGE_LE 0x00 +#define IMSIC_MMIO_PAGE_BE 0x04 + +#define IMSIC_MIN_ID 63 +#define IMSIC_MAX_ID 2048 + +#define IMSIC_EIDELIVERY 0x70 + +#define IMSIC_EITHRESHOLD 0x72 + +#define IMSIC_EIP0 0x80 +#define IMSIC_EIP63 0xbf +#define IMSIC_EIPx_BITS 32 + +#define IMSIC_EIE0 0xc0 +#define IMSIC_EIE63 0xff +#define IMSIC_EIEx_BITS 32 + +#define IMSIC_FIRST IMSIC_EIDELIVERY +#define IMSIC_LAST IMSIC_EIE63 + +#define IMSIC_MMIO_SETIPNUM_LE 0x00 +#define IMSIC_MMIO_SETIPNUM_BE 0x04 + +#endif diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index ee0acccb1d3b..2d8ee53b66c7 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -15,6 +15,7 @@ #include <linux/spinlock.h> #include <asm/hwcap.h> #include <asm/kvm_aia.h> +#include <asm/ptrace.h> #include <asm/kvm_vcpu_fp.h> #include <asm/kvm_vcpu_insn.h> #include <asm/kvm_vcpu_sbi.h> @@ -27,6 +28,8 @@ #define KVM_VCPU_MAX_FEATURES 0 +#define KVM_IRQCHIP_NUM_PINS 1024 + #define KVM_REQ_SLEEP \ KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_VCPU_RESET KVM_ARCH_REQ(1) @@ -145,6 +148,7 @@ struct kvm_cpu_context { unsigned long sstatus; unsigned long hstatus; union __riscv_fp_state fp; + struct __riscv_v_ext_state vector; }; struct kvm_vcpu_csr { @@ -318,6 +322,8 @@ int kvm_riscv_gstage_vmid_init(struct kvm *kvm); bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid); void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu); +int kvm_riscv_setup_default_irq_routing(struct kvm *kvm, u32 lines); + void __kvm_riscv_unpriv_trap(void); unsigned long kvm_riscv_vcpu_unpriv_read(struct kvm_vcpu *vcpu, diff --git a/arch/riscv/include/asm/kvm_vcpu_sbi.h b/arch/riscv/include/asm/kvm_vcpu_sbi.h index 4278125a38a5..cdcf0ff07be7 100644 --- a/arch/riscv/include/asm/kvm_vcpu_sbi.h +++ b/arch/riscv/include/asm/kvm_vcpu_sbi.h @@ -14,9 +14,15 @@ #define KVM_SBI_VERSION_MAJOR 1 #define KVM_SBI_VERSION_MINOR 0 +enum kvm_riscv_sbi_ext_status { + KVM_RISCV_SBI_EXT_UNINITIALIZED, + KVM_RISCV_SBI_EXT_AVAILABLE, + KVM_RISCV_SBI_EXT_UNAVAILABLE, +}; + struct kvm_vcpu_sbi_context { int return_handled; - bool extension_disabled[KVM_RISCV_SBI_EXT_MAX]; + enum kvm_riscv_sbi_ext_status ext_status[KVM_RISCV_SBI_EXT_MAX]; }; struct kvm_vcpu_sbi_return { @@ -66,4 +72,7 @@ extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_hsm; extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_experimental; extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_vendor; +#ifdef CONFIG_RISCV_PMU_SBI +extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_pmu; +#endif #endif /* __RISCV_KVM_VCPU_SBI_H__ */ diff --git a/arch/riscv/include/asm/kvm_vcpu_vector.h b/arch/riscv/include/asm/kvm_vcpu_vector.h new file mode 100644 index 000000000000..ff994fdd6d0d --- /dev/null +++ b/arch/riscv/include/asm/kvm_vcpu_vector.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2022 SiFive + * + * Authors: + * Vincent Chen <vincent.chen@sifive.com> + * Greentime Hu <greentime.hu@sifive.com> + */ + +#ifndef __KVM_VCPU_RISCV_VECTOR_H +#define __KVM_VCPU_RISCV_VECTOR_H + +#include <linux/types.h> + +#ifdef CONFIG_RISCV_ISA_V +#include <asm/vector.h> +#include <asm/kvm_host.h> + +static __always_inline void __kvm_riscv_vector_save(struct kvm_cpu_context *context) +{ + __riscv_v_vstate_save(&context->vector, context->vector.datap); +} + +static __always_inline void __kvm_riscv_vector_restore(struct kvm_cpu_context *context) +{ + __riscv_v_vstate_restore(&context->vector, context->vector.datap); +} + +void kvm_riscv_vcpu_vector_reset(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_guest_vector_save(struct kvm_cpu_context *cntx, + unsigned long *isa); +void kvm_riscv_vcpu_guest_vector_restore(struct kvm_cpu_context *cntx, + unsigned long *isa); +void kvm_riscv_vcpu_host_vector_save(struct kvm_cpu_context *cntx); +void kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cntx); +int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu, + struct kvm_cpu_context *cntx); +void kvm_riscv_vcpu_free_vector_context(struct kvm_vcpu *vcpu); +#else + +struct kvm_cpu_context; + +static inline void kvm_riscv_vcpu_vector_reset(struct kvm_vcpu *vcpu) +{ +} + +static inline void kvm_riscv_vcpu_guest_vector_save(struct kvm_cpu_context *cntx, + unsigned long *isa) +{ +} + +static inline void kvm_riscv_vcpu_guest_vector_restore(struct kvm_cpu_context *cntx, + unsigned long *isa) +{ +} + +static inline void kvm_riscv_vcpu_host_vector_save(struct kvm_cpu_context *cntx) +{ +} + +static inline void kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cntx) +{ +} + +static inline int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu, + struct kvm_cpu_context *cntx) +{ + return 0; +} + +static inline void kvm_riscv_vcpu_free_vector_context(struct kvm_vcpu *vcpu) +{ +} +#endif + +int kvm_riscv_vcpu_get_reg_vector(struct kvm_vcpu *vcpu, + const struct kvm_one_reg *reg, + unsigned long rtype); +int kvm_riscv_vcpu_set_reg_vector(struct kvm_vcpu *vcpu, + const struct kvm_one_reg *reg, + unsigned long rtype); +#endif diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index 94a0590c6971..c950a8d9edef 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -7,6 +7,7 @@ #define _ASM_RISCV_PROCESSOR_H #include <linux/const.h> +#include <linux/cache.h> #include <vdso/processor.h> @@ -39,6 +40,8 @@ struct thread_struct { unsigned long s[12]; /* s[0]: frame pointer */ struct __riscv_d_ext_state fstate; unsigned long bad_cause; + unsigned long vstate_ctrl; + struct __riscv_v_ext_state vstate; }; /* Whitelist the fstate from the task_struct for hardened usercopy */ @@ -75,11 +78,22 @@ static inline void wait_for_interrupt(void) struct device_node; int riscv_of_processor_hartid(struct device_node *node, unsigned long *hartid); +int riscv_early_of_processor_hartid(struct device_node *node, unsigned long *hartid); int riscv_of_parent_hartid(struct device_node *node, unsigned long *hartid); extern void riscv_fill_hwcap(void); extern int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src); +extern unsigned long signal_minsigstksz __ro_after_init; + +#ifdef CONFIG_RISCV_ISA_V +/* Userspace interface for PR_RISCV_V_{SET,GET}_VS prctl()s: */ +#define RISCV_V_SET_CONTROL(arg) riscv_v_vstate_ctrl_set_current(arg) +#define RISCV_V_GET_CONTROL() riscv_v_vstate_ctrl_get_current() +extern long riscv_v_vstate_ctrl_set_current(unsigned long arg); +extern long riscv_v_vstate_ctrl_get_current(void); +#endif /* CONFIG_RISCV_ISA_V */ + #endif /* __ASSEMBLY__ */ #endif /* _ASM_RISCV_PROCESSOR_H */ diff --git a/arch/riscv/include/asm/switch_to.h b/arch/riscv/include/asm/switch_to.h index 60f8ca01d36e..a727be723c56 100644 --- a/arch/riscv/include/asm/switch_to.h +++ b/arch/riscv/include/asm/switch_to.h @@ -8,6 +8,7 @@ #include <linux/jump_label.h> #include <linux/sched/task_stack.h> +#include <asm/vector.h> #include <asm/hwcap.h> #include <asm/processor.h> #include <asm/ptrace.h> @@ -46,7 +47,7 @@ static inline void fstate_restore(struct task_struct *task, } } -static inline void __switch_to_aux(struct task_struct *prev, +static inline void __switch_to_fpu(struct task_struct *prev, struct task_struct *next) { struct pt_regs *regs; @@ -66,7 +67,7 @@ static __always_inline bool has_fpu(void) static __always_inline bool has_fpu(void) { return false; } #define fstate_save(task, regs) do { } while (0) #define fstate_restore(task, regs) do { } while (0) -#define __switch_to_aux(__prev, __next) do { } while (0) +#define __switch_to_fpu(__prev, __next) do { } while (0) #endif extern struct task_struct *__switch_to(struct task_struct *, @@ -77,7 +78,9 @@ do { \ struct task_struct *__prev = (prev); \ struct task_struct *__next = (next); \ if (has_fpu()) \ - __switch_to_aux(__prev, __next); \ + __switch_to_fpu(__prev, __next); \ + if (has_vector()) \ + __switch_to_vector(__prev, __next); \ ((last) = __switch_to(__prev, __next)); \ } while (0) diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index e0d202134b44..1833beb00489 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -11,18 +11,8 @@ #include <asm/page.h> #include <linux/const.h> -#ifdef CONFIG_KASAN -#define KASAN_STACK_ORDER 1 -#else -#define KASAN_STACK_ORDER 0 -#endif - /* thread information allocation */ -#ifdef CONFIG_64BIT -#define THREAD_SIZE_ORDER (2 + KASAN_STACK_ORDER) -#else -#define THREAD_SIZE_ORDER (1 + KASAN_STACK_ORDER) -#endif +#define THREAD_SIZE_ORDER CONFIG_THREAD_SIZE_ORDER #define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER) /* @@ -40,6 +30,8 @@ #define OVERFLOW_STACK_SIZE SZ_4K #define SHADOW_OVERFLOW_STACK_SIZE (1024) +#define IRQ_STACK_SIZE THREAD_SIZE + #ifndef __ASSEMBLY__ extern long shadow_stack[SHADOW_OVERFLOW_STACK_SIZE / sizeof(long)]; @@ -81,6 +73,9 @@ struct thread_info { .preempt_count = INIT_PREEMPT_COUNT, \ } +void arch_release_task_struct(struct task_struct *tsk); +int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src); + #endif /* !__ASSEMBLY__ */ /* diff --git a/arch/riscv/include/asm/vector.h b/arch/riscv/include/asm/vector.h new file mode 100644 index 000000000000..04c0b07bf6cd --- /dev/null +++ b/arch/riscv/include/asm/vector.h @@ -0,0 +1,184 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020 SiFive + */ + +#ifndef __ASM_RISCV_VECTOR_H +#define __ASM_RISCV_VECTOR_H + +#include <linux/types.h> +#include <uapi/asm-generic/errno.h> + +#ifdef CONFIG_RISCV_ISA_V + +#include <linux/stringify.h> +#include <linux/sched.h> +#include <linux/sched/task_stack.h> +#include <asm/ptrace.h> +#include <asm/hwcap.h> +#include <asm/csr.h> +#include <asm/asm.h> + +extern unsigned long riscv_v_vsize; +int riscv_v_setup_vsize(void); +bool riscv_v_first_use_handler(struct pt_regs *regs); + +static __always_inline bool has_vector(void) +{ + return riscv_has_extension_unlikely(RISCV_ISA_EXT_v); +} + +static inline void __riscv_v_vstate_clean(struct pt_regs *regs) +{ + regs->status = (regs->status & ~SR_VS) | SR_VS_CLEAN; +} + +static inline void riscv_v_vstate_off(struct pt_regs *regs) +{ + regs->status = (regs->status & ~SR_VS) | SR_VS_OFF; +} + +static inline void riscv_v_vstate_on(struct pt_regs *regs) +{ + regs->status = (regs->status & ~SR_VS) | SR_VS_INITIAL; +} + +static inline bool riscv_v_vstate_query(struct pt_regs *regs) +{ + return (regs->status & SR_VS) != 0; +} + +static __always_inline void riscv_v_enable(void) +{ + csr_set(CSR_SSTATUS, SR_VS); +} + +static __always_inline void riscv_v_disable(void) +{ + csr_clear(CSR_SSTATUS, SR_VS); +} + +static __always_inline void __vstate_csr_save(struct __riscv_v_ext_state *dest) +{ + asm volatile ( + "csrr %0, " __stringify(CSR_VSTART) "\n\t" + "csrr %1, " __stringify(CSR_VTYPE) "\n\t" + "csrr %2, " __stringify(CSR_VL) "\n\t" + "csrr %3, " __stringify(CSR_VCSR) "\n\t" + : "=r" (dest->vstart), "=r" (dest->vtype), "=r" (dest->vl), + "=r" (dest->vcsr) : :); +} + +static __always_inline void __vstate_csr_restore(struct __riscv_v_ext_state *src) +{ + asm volatile ( + ".option push\n\t" + ".option arch, +v\n\t" + "vsetvl x0, %2, %1\n\t" + ".option pop\n\t" + "csrw " __stringify(CSR_VSTART) ", %0\n\t" + "csrw " __stringify(CSR_VCSR) ", %3\n\t" + : : "r" (src->vstart), "r" (src->vtype), "r" (src->vl), + "r" (src->vcsr) :); +} + +static inline void __riscv_v_vstate_save(struct __riscv_v_ext_state *save_to, + void *datap) +{ + unsigned long vl; + + riscv_v_enable(); + __vstate_csr_save(save_to); + asm volatile ( + ".option push\n\t" + ".option arch, +v\n\t" + "vsetvli %0, x0, e8, m8, ta, ma\n\t" + "vse8.v v0, (%1)\n\t" + "add %1, %1, %0\n\t" + "vse8.v v8, (%1)\n\t" + "add %1, %1, %0\n\t" + "vse8.v v16, (%1)\n\t" + "add %1, %1, %0\n\t" + "vse8.v v24, (%1)\n\t" + ".option pop\n\t" + : "=&r" (vl) : "r" (datap) : "memory"); + riscv_v_disable(); +} + +static inline void __riscv_v_vstate_restore(struct __riscv_v_ext_state *restore_from, + void *datap) +{ + unsigned long vl; + + riscv_v_enable(); + asm volatile ( + ".option push\n\t" + ".option arch, +v\n\t" + "vsetvli %0, x0, e8, m8, ta, ma\n\t" + "vle8.v v0, (%1)\n\t" + "add %1, %1, %0\n\t" + "vle8.v v8, (%1)\n\t" + "add %1, %1, %0\n\t" + "vle8.v v16, (%1)\n\t" + "add %1, %1, %0\n\t" + "vle8.v v24, (%1)\n\t" + ".option pop\n\t" + : "=&r" (vl) : "r" (datap) : "memory"); + __vstate_csr_restore(restore_from); + riscv_v_disable(); +} + +static inline void riscv_v_vstate_save(struct task_struct *task, + struct pt_regs *regs) +{ + if ((regs->status & SR_VS) == SR_VS_DIRTY) { + struct __riscv_v_ext_state *vstate = &task->thread.vstate; + + __riscv_v_vstate_save(vstate, vstate->datap); + __riscv_v_vstate_clean(regs); + } +} + +static inline void riscv_v_vstate_restore(struct task_struct *task, + struct pt_regs *regs) +{ + if ((regs->status & SR_VS) != SR_VS_OFF) { + struct __riscv_v_ext_state *vstate = &task->thread.vstate; + + __riscv_v_vstate_restore(vstate, vstate->datap); + __riscv_v_vstate_clean(regs); + } +} + +static inline void __switch_to_vector(struct task_struct *prev, + struct task_struct *next) +{ + struct pt_regs *regs; + + regs = task_pt_regs(prev); + riscv_v_vstate_save(prev, regs); + riscv_v_vstate_restore(next, task_pt_regs(next)); +} + +void riscv_v_vstate_ctrl_init(struct task_struct *tsk); +bool riscv_v_vstate_ctrl_user_allowed(void); + +#else /* ! CONFIG_RISCV_ISA_V */ + +struct pt_regs; + +static inline int riscv_v_setup_vsize(void) { return -EOPNOTSUPP; } +static __always_inline bool has_vector(void) { return false; } +static inline bool riscv_v_first_use_handler(struct pt_regs *regs) { return false; } +static inline bool riscv_v_vstate_query(struct pt_regs *regs) { return false; } +static inline bool riscv_v_vstate_ctrl_user_allowed(void) { return false; } +#define riscv_v_vsize (0) +#define riscv_v_vstate_save(task, regs) do {} while (0) +#define riscv_v_vstate_restore(task, regs) do {} while (0) +#define __switch_to_vector(__prev, __next) do {} while (0) +#define riscv_v_vstate_off(regs) do {} while (0) +#define riscv_v_vstate_on(regs) do {} while (0) + +#endif /* CONFIG_RISCV_ISA_V */ + +#endif /* ! __ASM_RISCV_VECTOR_H */ diff --git a/arch/riscv/include/uapi/asm/auxvec.h b/arch/riscv/include/uapi/asm/auxvec.h index fb187a33ce58..10aaa83db89e 100644 --- a/arch/riscv/include/uapi/asm/auxvec.h +++ b/arch/riscv/include/uapi/asm/auxvec.h @@ -35,5 +35,6 @@ /* entries in ARCH_DLINFO */ #define AT_VECTOR_SIZE_ARCH 9 +#define AT_MINSIGSTKSZ 51 #endif /* _UAPI_ASM_RISCV_AUXVEC_H */ diff --git a/arch/riscv/include/uapi/asm/hwcap.h b/arch/riscv/include/uapi/asm/hwcap.h index 46dc3f5ee99f..c52bb7bbbabe 100644 --- a/arch/riscv/include/uapi/asm/hwcap.h +++ b/arch/riscv/include/uapi/asm/hwcap.h @@ -21,5 +21,6 @@ #define COMPAT_HWCAP_ISA_F (1 << ('F' - 'A')) #define COMPAT_HWCAP_ISA_D (1 << ('D' - 'A')) #define COMPAT_HWCAP_ISA_C (1 << ('C' - 'A')) +#define COMPAT_HWCAP_ISA_V (1 << ('V' - 'A')) #endif /* _UAPI_ASM_RISCV_HWCAP_H */ diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index 8d745a4ad8a2..006bfb48343d 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -25,6 +25,10 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_KEY_IMA_EXT_0 4 #define RISCV_HWPROBE_IMA_FD (1 << 0) #define RISCV_HWPROBE_IMA_C (1 << 1) +#define RISCV_HWPROBE_IMA_V (1 << 2) +#define RISCV_HWPROBE_EXT_ZBA (1 << 3) +#define RISCV_HWPROBE_EXT_ZBB (1 << 4) +#define RISCV_HWPROBE_EXT_ZBS (1 << 5) #define RISCV_HWPROBE_KEY_CPUPERF_0 5 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) #define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0) diff --git a/arch/riscv/include/uapi/asm/kvm.h b/arch/riscv/include/uapi/asm/kvm.h index f92790c9481a..930fdc4101cd 100644 --- a/arch/riscv/include/uapi/asm/kvm.h +++ b/arch/riscv/include/uapi/asm/kvm.h @@ -15,6 +15,7 @@ #include <asm/bitsperlong.h> #include <asm/ptrace.h> +#define __KVM_HAVE_IRQ_LINE #define __KVM_HAVE_READONLY_MEM #define KVM_COALESCED_MMIO_PAGE_OFFSET 1 @@ -121,6 +122,8 @@ enum KVM_RISCV_ISA_EXT_ID { KVM_RISCV_ISA_EXT_ZICBOZ, KVM_RISCV_ISA_EXT_ZBB, KVM_RISCV_ISA_EXT_SSAIA, + KVM_RISCV_ISA_EXT_V, + KVM_RISCV_ISA_EXT_SVNAPOT, KVM_RISCV_ISA_EXT_MAX, }; @@ -203,6 +206,84 @@ enum KVM_RISCV_SBI_EXT_ID { #define KVM_REG_RISCV_SBI_MULTI_REG_LAST \ KVM_REG_RISCV_SBI_MULTI_REG(KVM_RISCV_SBI_EXT_MAX - 1) +/* V extension registers are mapped as type 9 */ +#define KVM_REG_RISCV_VECTOR (0x09 << KVM_REG_RISCV_TYPE_SHIFT) +#define KVM_REG_RISCV_VECTOR_CSR_REG(name) \ + (offsetof(struct __riscv_v_ext_state, name) / sizeof(unsigned long)) +#define KVM_REG_RISCV_VECTOR_REG(n) \ + ((n) + sizeof(struct __riscv_v_ext_state) / sizeof(unsigned long)) + +/* Device Control API: RISC-V AIA */ +#define KVM_DEV_RISCV_APLIC_ALIGN 0x1000 +#define KVM_DEV_RISCV_APLIC_SIZE 0x4000 +#define KVM_DEV_RISCV_APLIC_MAX_HARTS 0x4000 +#define KVM_DEV_RISCV_IMSIC_ALIGN 0x1000 +#define KVM_DEV_RISCV_IMSIC_SIZE 0x1000 + +#define KVM_DEV_RISCV_AIA_GRP_CONFIG 0 +#define KVM_DEV_RISCV_AIA_CONFIG_MODE 0 +#define KVM_DEV_RISCV_AIA_CONFIG_IDS 1 +#define KVM_DEV_RISCV_AIA_CONFIG_SRCS 2 +#define KVM_DEV_RISCV_AIA_CONFIG_GROUP_BITS 3 +#define KVM_DEV_RISCV_AIA_CONFIG_GROUP_SHIFT 4 +#define KVM_DEV_RISCV_AIA_CONFIG_HART_BITS 5 +#define KVM_DEV_RISCV_AIA_CONFIG_GUEST_BITS 6 + +/* + * Modes of RISC-V AIA device: + * 1) EMUL (aka Emulation): Trap-n-emulate IMSIC + * 2) HWACCEL (aka HW Acceleration): Virtualize IMSIC using IMSIC guest files + * 3) AUTO (aka Automatic): Virtualize IMSIC using IMSIC guest files whenever + * available otherwise fallback to trap-n-emulation + */ +#define KVM_DEV_RISCV_AIA_MODE_EMUL 0 +#define KVM_DEV_RISCV_AIA_MODE_HWACCEL 1 +#define KVM_DEV_RISCV_AIA_MODE_AUTO 2 + +#define KVM_DEV_RISCV_AIA_IDS_MIN 63 +#define KVM_DEV_RISCV_AIA_IDS_MAX 2048 +#define KVM_DEV_RISCV_AIA_SRCS_MAX 1024 +#define KVM_DEV_RISCV_AIA_GROUP_BITS_MAX 8 +#define KVM_DEV_RISCV_AIA_GROUP_SHIFT_MIN 24 +#define KVM_DEV_RISCV_AIA_GROUP_SHIFT_MAX 56 +#define KVM_DEV_RISCV_AIA_HART_BITS_MAX 16 +#define KVM_DEV_RISCV_AIA_GUEST_BITS_MAX 8 + +#define KVM_DEV_RISCV_AIA_GRP_ADDR 1 +#define KVM_DEV_RISCV_AIA_ADDR_APLIC 0 +#define KVM_DEV_RISCV_AIA_ADDR_IMSIC(__vcpu) (1 + (__vcpu)) +#define KVM_DEV_RISCV_AIA_ADDR_MAX \ + (1 + KVM_DEV_RISCV_APLIC_MAX_HARTS) + +#define KVM_DEV_RISCV_AIA_GRP_CTRL 2 +#define KVM_DEV_RISCV_AIA_CTRL_INIT 0 + +/* + * The device attribute type contains the memory mapped offset of the + * APLIC register (range 0x0000-0x3FFF) and it must be 4-byte aligned. + */ +#define KVM_DEV_RISCV_AIA_GRP_APLIC 3 + +/* + * The lower 12-bits of the device attribute type contains the iselect + * value of the IMSIC register (range 0x70-0xFF) whereas the higher order + * bits contains the VCPU id. + */ +#define KVM_DEV_RISCV_AIA_GRP_IMSIC 4 +#define KVM_DEV_RISCV_AIA_IMSIC_ISEL_BITS 12 +#define KVM_DEV_RISCV_AIA_IMSIC_ISEL_MASK \ + ((1U << KVM_DEV_RISCV_AIA_IMSIC_ISEL_BITS) - 1) +#define KVM_DEV_RISCV_AIA_IMSIC_MKATTR(__vcpu, __isel) \ + (((__vcpu) << KVM_DEV_RISCV_AIA_IMSIC_ISEL_BITS) | \ + ((__isel) & KVM_DEV_RISCV_AIA_IMSIC_ISEL_MASK)) +#define KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(__attr) \ + ((__attr) & KVM_DEV_RISCV_AIA_IMSIC_ISEL_MASK) +#define KVM_DEV_RISCV_AIA_IMSIC_GET_VCPU(__attr) \ + ((__attr) >> KVM_DEV_RISCV_AIA_IMSIC_ISEL_BITS) + +/* One single KVM irqchip, ie. the AIA */ +#define KVM_NR_IRQCHIPS 1 + #endif #endif /* __LINUX_KVM_RISCV_H */ diff --git a/arch/riscv/include/uapi/asm/ptrace.h b/arch/riscv/include/uapi/asm/ptrace.h index 882547f6bd5c..e17c550986a6 100644 --- a/arch/riscv/include/uapi/asm/ptrace.h +++ b/arch/riscv/include/uapi/asm/ptrace.h @@ -71,12 +71,51 @@ struct __riscv_q_ext_state { __u32 reserved[3]; }; +struct __riscv_ctx_hdr { + __u32 magic; + __u32 size; +}; + +struct __riscv_extra_ext_header { + __u32 __padding[129] __attribute__((aligned(16))); + /* + * Reserved for expansion of sigcontext structure. Currently zeroed + * upon signal, and must be zero upon sigreturn. + */ + __u32 reserved; + struct __riscv_ctx_hdr hdr; +}; + union __riscv_fp_state { struct __riscv_f_ext_state f; struct __riscv_d_ext_state d; struct __riscv_q_ext_state q; }; +struct __riscv_v_ext_state { + unsigned long vstart; + unsigned long vl; + unsigned long vtype; + unsigned long vcsr; + void *datap; + /* + * In signal handler, datap will be set a correct user stack offset + * and vector registers will be copied to the address of datap + * pointer. + * + * In ptrace syscall, datap will be set to zero and the vector + * registers will be copied to the address right after this + * structure. + */ +}; + +/* + * According to spec: The number of bits in a single vector register, + * VLEN >= ELEN, which must be a power of 2, and must be no greater than + * 2^16 = 65536bits = 8192bytes + */ +#define RISCV_MAX_VLENB (8192) + #endif /* __ASSEMBLY__ */ #endif /* _UAPI_ASM_RISCV_PTRACE_H */ diff --git a/arch/riscv/include/uapi/asm/sigcontext.h b/arch/riscv/include/uapi/asm/sigcontext.h index 84f2dfcfdbce..8b8a8541673a 100644 --- a/arch/riscv/include/uapi/asm/sigcontext.h +++ b/arch/riscv/include/uapi/asm/sigcontext.h @@ -8,6 +8,17 @@ #include <asm/ptrace.h> +/* The Magic number for signal context frame header. */ +#define RISCV_V_MAGIC 0x53465457 +#define END_MAGIC 0x0 + +/* The size of END signal context header. */ +#define END_HDR_SIZE 0x0 + +struct __sc_riscv_v_state { + struct __riscv_v_ext_state v_state; +} __attribute__((aligned(16))); + /* * Signal context structure * @@ -16,7 +27,10 @@ */ struct sigcontext { struct user_regs_struct sc_regs; - union __riscv_fp_state sc_fpregs; + union { + union __riscv_fp_state sc_fpregs; + struct __riscv_extra_ext_header sc_extdesc; + }; }; #endif /* _UAPI_ASM_RISCV_SIGCONTEXT_H */ diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index 153864e4f399..506cc4a9a45a 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -60,6 +60,7 @@ obj-$(CONFIG_MMU) += vdso.o vdso/ obj-$(CONFIG_RISCV_M_MODE) += traps_misaligned.o obj-$(CONFIG_FPU) += fpu.o +obj-$(CONFIG_RISCV_ISA_V) += vector.o obj-$(CONFIG_SMP) += smpboot.o obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_SMP) += cpu_ops.o @@ -96,3 +97,4 @@ obj-$(CONFIG_COMPAT) += compat_signal.o obj-$(CONFIG_COMPAT) += compat_vdso/ obj-$(CONFIG_64BIT) += pi/ +obj-$(CONFIG_ACPI) += acpi.o diff --git a/arch/riscv/kernel/acpi.c b/arch/riscv/kernel/acpi.c new file mode 100644 index 000000000000..5ee03ebab80e --- /dev/null +++ b/arch/riscv/kernel/acpi.c @@ -0,0 +1,251 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * RISC-V Specific Low-Level ACPI Boot Support + * + * Copyright (C) 2013-2014, Linaro Ltd. + * Author: Al Stone <al.stone@linaro.org> + * Author: Graeme Gregory <graeme.gregory@linaro.org> + * Author: Hanjun Guo <hanjun.guo@linaro.org> + * Author: Tomasz Nowicki <tomasz.nowicki@linaro.org> + * Author: Naresh Bhat <naresh.bhat@linaro.org> + * + * Copyright (C) 2021-2023, Ventana Micro Systems Inc. + * Author: Sunil V L <sunilvl@ventanamicro.com> + */ + +#include <linux/acpi.h> +#include <linux/io.h> +#include <linux/pci.h> +#include <linux/efi.h> + +int acpi_noirq = 1; /* skip ACPI IRQ initialization */ +int acpi_disabled = 1; +EXPORT_SYMBOL(acpi_disabled); + +int acpi_pci_disabled = 1; /* skip ACPI PCI scan and IRQ initialization */ +EXPORT_SYMBOL(acpi_pci_disabled); + +static bool param_acpi_off __initdata; +static bool param_acpi_on __initdata; +static bool param_acpi_force __initdata; + +static struct acpi_madt_rintc cpu_madt_rintc[NR_CPUS]; + +static int __init parse_acpi(char *arg) +{ + if (!arg) + return -EINVAL; + + /* "acpi=off" disables both ACPI table parsing and interpreter */ + if (strcmp(arg, "off") == 0) + param_acpi_off = true; + else if (strcmp(arg, "on") == 0) /* prefer ACPI over DT */ + param_acpi_on = true; + else if (strcmp(arg, "force") == 0) /* force ACPI to be enabled */ + param_acpi_force = true; + else + return -EINVAL; /* Core will print when we return error */ + + return 0; +} +early_param("acpi", parse_acpi); + +/* + * acpi_fadt_sanity_check() - Check FADT presence and carry out sanity + * checks on it + * + * Return 0 on success, <0 on failure + */ +static int __init acpi_fadt_sanity_check(void) +{ + struct acpi_table_header *table; + struct acpi_table_fadt *fadt; + acpi_status status; + int ret = 0; + + /* + * FADT is required on riscv; retrieve it to check its presence + * and carry out revision and ACPI HW reduced compliancy tests + */ + status = acpi_get_table(ACPI_SIG_FADT, 0, &table); + if (ACPI_FAILURE(status)) { + const char *msg = acpi_format_exception(status); + + pr_err("Failed to get FADT table, %s\n", msg); + return -ENODEV; + } + + fadt = (struct acpi_table_fadt *)table; + + /* + * The revision in the table header is the FADT's Major revision. The + * FADT also has a minor revision, which is stored in the FADT itself. + * + * TODO: Currently, we check for 6.5 as the minimum version to check + * for HW_REDUCED flag. However, once RISC-V updates are released in + * the ACPI spec, we need to update this check for exact minor revision + */ + if (table->revision < 6 || (table->revision == 6 && fadt->minor_revision < 5)) + pr_err(FW_BUG "Unsupported FADT revision %d.%d, should be 6.5+\n", + table->revision, fadt->minor_revision); + + if (!(fadt->flags & ACPI_FADT_HW_REDUCED)) { + pr_err("FADT not ACPI hardware reduced compliant\n"); + ret = -EINVAL; + } + + /* + * acpi_get_table() creates FADT table mapping that + * should be released after parsing and before resuming boot + */ + acpi_put_table(table); + return ret; +} + +/* + * acpi_boot_table_init() called from setup_arch(), always. + * 1. find RSDP and get its address, and then find XSDT + * 2. extract all tables and checksums them all + * 3. check ACPI FADT HW reduced flag + * + * We can parse ACPI boot-time tables such as MADT after + * this function is called. + * + * On return ACPI is enabled if either: + * + * - ACPI tables are initialized and sanity checks passed + * - acpi=force was passed in the command line and ACPI was not disabled + * explicitly through acpi=off command line parameter + * + * ACPI is disabled on function return otherwise + */ +void __init acpi_boot_table_init(void) +{ + /* + * Enable ACPI instead of device tree unless + * - ACPI has been disabled explicitly (acpi=off), or + * - firmware has not populated ACPI ptr in EFI system table + * and ACPI has not been [force] enabled (acpi=on|force) + */ + if (param_acpi_off || + (!param_acpi_on && !param_acpi_force && + efi.acpi20 == EFI_INVALID_TABLE_ADDR)) + return; + + /* + * ACPI is disabled at this point. Enable it in order to parse + * the ACPI tables and carry out sanity checks + */ + enable_acpi(); + + /* + * If ACPI tables are initialized and FADT sanity checks passed, + * leave ACPI enabled and carry on booting; otherwise disable ACPI + * on initialization error. + * If acpi=force was passed on the command line it forces ACPI + * to be enabled even if its initialization failed. + */ + if (acpi_table_init() || acpi_fadt_sanity_check()) { + pr_err("Failed to init ACPI tables\n"); + if (!param_acpi_force) + disable_acpi(); + } +} + +static int acpi_parse_madt_rintc(union acpi_subtable_headers *header, const unsigned long end) +{ + struct acpi_madt_rintc *rintc = (struct acpi_madt_rintc *)header; + int cpuid; + + if (!(rintc->flags & ACPI_MADT_ENABLED)) + return 0; + + cpuid = riscv_hartid_to_cpuid(rintc->hart_id); + /* + * When CONFIG_SMP is disabled, mapping won't be created for + * all cpus. + * CPUs more than num_possible_cpus, will be ignored. + */ + if (cpuid >= 0 && cpuid < num_possible_cpus()) + cpu_madt_rintc[cpuid] = *rintc; + + return 0; +} + +/* + * Instead of parsing (and freeing) the ACPI table, cache + * the RINTC structures since they are frequently used + * like in cpuinfo. + */ +void __init acpi_init_rintc_map(void) +{ + if (acpi_table_parse_madt(ACPI_MADT_TYPE_RINTC, acpi_parse_madt_rintc, 0) <= 0) { + pr_err("No valid RINTC entries exist\n"); + BUG(); + } +} + +struct acpi_madt_rintc *acpi_cpu_get_madt_rintc(int cpu) +{ + return &cpu_madt_rintc[cpu]; +} + +u32 get_acpi_id_for_cpu(int cpu) +{ + return acpi_cpu_get_madt_rintc(cpu)->uid; +} + +/* + * __acpi_map_table() will be called before paging_init(), so early_ioremap() + * or early_memremap() should be called here to for ACPI table mapping. + */ +void __init __iomem *__acpi_map_table(unsigned long phys, unsigned long size) +{ + if (!size) + return NULL; + + return early_ioremap(phys, size); +} + +void __init __acpi_unmap_table(void __iomem *map, unsigned long size) +{ + if (!map || !size) + return; + + early_iounmap(map, size); +} + +void *acpi_os_ioremap(acpi_physical_address phys, acpi_size size) +{ + return memremap(phys, size, MEMREMAP_WB); +} + +#ifdef CONFIG_PCI + +/* + * These interfaces are defined just to enable building ACPI core. + * TODO: Update it with actual implementation when external interrupt + * controller support is added in RISC-V ACPI. + */ +int raw_pci_read(unsigned int domain, unsigned int bus, unsigned int devfn, + int reg, int len, u32 *val) +{ + return PCIBIOS_DEVICE_NOT_FOUND; +} + +int raw_pci_write(unsigned int domain, unsigned int bus, unsigned int devfn, + int reg, int len, u32 val) +{ + return PCIBIOS_DEVICE_NOT_FOUND; +} + +int acpi_pci_bus_find_domain_nr(struct pci_bus *bus) +{ + return -1; +} + +struct pci_bus *pci_acpi_scan_root(struct acpi_pci_root *root) +{ + return NULL; +} +#endif /* CONFIG_PCI */ diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index c96aa56cf1c7..a2fc952318e9 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -3,10 +3,13 @@ * Copyright (C) 2012 Regents of the University of California */ +#include <linux/acpi.h> #include <linux/cpu.h> +#include <linux/ctype.h> #include <linux/init.h> #include <linux/seq_file.h> #include <linux/of.h> +#include <asm/acpi.h> #include <asm/cpufeature.h> #include <asm/csr.h> #include <asm/hwcap.h> @@ -20,6 +23,26 @@ */ int riscv_of_processor_hartid(struct device_node *node, unsigned long *hart) { + int cpu; + + *hart = (unsigned long)of_get_cpu_hwid(node, 0); + if (*hart == ~0UL) { + pr_warn("Found CPU without hart ID\n"); + return -ENODEV; + } + + cpu = riscv_hartid_to_cpuid(*hart); + if (cpu < 0) + return cpu; + + if (!cpu_possible(cpu)) + return -ENODEV; + + return 0; +} + +int riscv_early_of_processor_hartid(struct device_node *node, unsigned long *hart) +{ const char *isa; if (!of_device_is_compatible(node, "riscv")) { @@ -27,7 +50,7 @@ int riscv_of_processor_hartid(struct device_node *node, unsigned long *hart) return -ENODEV; } - *hart = (unsigned long) of_get_cpu_hwid(node, 0); + *hart = (unsigned long)of_get_cpu_hwid(node, 0); if (*hart == ~0UL) { pr_warn("Found CPU without hart ID\n"); return -ENODEV; @@ -42,10 +65,12 @@ int riscv_of_processor_hartid(struct device_node *node, unsigned long *hart) pr_warn("CPU with hartid=%lu has no \"riscv,isa\" property\n", *hart); return -ENODEV; } - if (isa[0] != 'r' || isa[1] != 'v') { - pr_warn("CPU with hartid=%lu has an invalid ISA of \"%s\"\n", *hart, isa); + + if (IS_ENABLED(CONFIG_32BIT) && strncasecmp(isa, "rv32ima", 7)) + return -ENODEV; + + if (IS_ENABLED(CONFIG_64BIT) && strncasecmp(isa, "rv64ima", 7)) return -ENODEV; - } return 0; } @@ -183,8 +208,14 @@ arch_initcall(riscv_cpuinfo_init); static struct riscv_isa_ext_data isa_ext_arr[] = { __RISCV_ISA_EXT_DATA(zicbom, RISCV_ISA_EXT_ZICBOM), __RISCV_ISA_EXT_DATA(zicboz, RISCV_ISA_EXT_ZICBOZ), + __RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR), + __RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR), + __RISCV_ISA_EXT_DATA(zifencei, RISCV_ISA_EXT_ZIFENCEI), __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), + __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), + __RISCV_ISA_EXT_DATA(zba, RISCV_ISA_EXT_ZBA), __RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB), + __RISCV_ISA_EXT_DATA(zbs, RISCV_ISA_EXT_ZBS), __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA), __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA), __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF), @@ -283,23 +314,35 @@ static void c_stop(struct seq_file *m, void *v) static int c_show(struct seq_file *m, void *v) { unsigned long cpu_id = (unsigned long)v - 1; - struct device_node *node = of_get_cpu_node(cpu_id, NULL); struct riscv_cpuinfo *ci = per_cpu_ptr(&riscv_cpuinfo, cpu_id); + struct device_node *node; const char *compat, *isa; seq_printf(m, "processor\t: %lu\n", cpu_id); seq_printf(m, "hart\t\t: %lu\n", cpuid_to_hartid_map(cpu_id)); - if (!of_property_read_string(node, "riscv,isa", &isa)) - print_isa(m, isa); - print_mmu(m); - if (!of_property_read_string(node, "compatible", &compat) - && strcmp(compat, "riscv")) - seq_printf(m, "uarch\t\t: %s\n", compat); + + if (acpi_disabled) { + node = of_get_cpu_node(cpu_id, NULL); + if (!of_property_read_string(node, "riscv,isa", &isa)) + print_isa(m, isa); + + print_mmu(m); + if (!of_property_read_string(node, "compatible", &compat) && + strcmp(compat, "riscv")) + seq_printf(m, "uarch\t\t: %s\n", compat); + + of_node_put(node); + } else { + if (!acpi_get_riscv_isa(NULL, cpu_id, &isa)) + print_isa(m, isa); + + print_mmu(m); + } + seq_printf(m, "mvendorid\t: 0x%lx\n", ci->mvendorid); seq_printf(m, "marchid\t\t: 0x%lx\n", ci->marchid); seq_printf(m, "mimpid\t\t: 0x%lx\n", ci->mimpid); seq_puts(m, "\n"); - of_node_put(node); return 0; } diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index b1d6b7e4b829..bdcf460ea53d 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -6,18 +6,22 @@ * Copyright (C) 2017 SiFive */ +#include <linux/acpi.h> #include <linux/bitmap.h> #include <linux/ctype.h> #include <linux/log2.h> #include <linux/memory.h> #include <linux/module.h> #include <linux/of.h> +#include <linux/of_device.h> +#include <asm/acpi.h> #include <asm/alternative.h> #include <asm/cacheflush.h> #include <asm/cpufeature.h> #include <asm/hwcap.h> #include <asm/patch.h> #include <asm/processor.h> +#include <asm/vector.h> #define NUM_ALPHA_EXTS ('z' - 'a' + 1) @@ -26,6 +30,9 @@ unsigned long elf_hwcap __read_mostly; /* Host ISA bitmap */ static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; +/* Per-cpu ISA extensions. */ +struct riscv_isainfo hart_isa[NR_CPUS]; + /* Performance information */ DEFINE_PER_CPU(long, misaligned_access_speed); @@ -71,10 +78,10 @@ static bool riscv_isa_extension_check(int id) switch (id) { case RISCV_ISA_EXT_ZICBOM: if (!riscv_cbom_block_size) { - pr_err("Zicbom detected in ISA string, but no cbom-block-size found\n"); + pr_err("Zicbom detected in ISA string, disabling as no cbom-block-size found\n"); return false; } else if (!is_power_of_2(riscv_cbom_block_size)) { - pr_err("cbom-block-size present, but is not a power-of-2\n"); + pr_err("Zicbom disabled as cbom-block-size present, but is not a power-of-2\n"); return false; } return true; @@ -99,7 +106,9 @@ void __init riscv_fill_hwcap(void) char print_str[NUM_ALPHA_EXTS + 1]; int i, j, rc; unsigned long isa2hwcap[26] = {0}; - unsigned long hartid; + struct acpi_table_header *rhct; + acpi_status status; + unsigned int cpu; isa2hwcap['i' - 'a'] = COMPAT_HWCAP_ISA_I; isa2hwcap['m' - 'a'] = COMPAT_HWCAP_ISA_M; @@ -107,45 +116,59 @@ void __init riscv_fill_hwcap(void) isa2hwcap['f' - 'a'] = COMPAT_HWCAP_ISA_F; isa2hwcap['d' - 'a'] = COMPAT_HWCAP_ISA_D; isa2hwcap['c' - 'a'] = COMPAT_HWCAP_ISA_C; + isa2hwcap['v' - 'a'] = COMPAT_HWCAP_ISA_V; elf_hwcap = 0; bitmap_zero(riscv_isa, RISCV_ISA_EXT_MAX); - for_each_of_cpu_node(node) { + if (!acpi_disabled) { + status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct); + if (ACPI_FAILURE(status)) + return; + } + + for_each_possible_cpu(cpu) { + struct riscv_isainfo *isainfo = &hart_isa[cpu]; unsigned long this_hwcap = 0; - DECLARE_BITMAP(this_isa, RISCV_ISA_EXT_MAX); - const char *temp; - rc = riscv_of_processor_hartid(node, &hartid); - if (rc < 0) - continue; + if (acpi_disabled) { + node = of_cpu_device_node_get(cpu); + if (!node) { + pr_warn("Unable to find cpu node\n"); + continue; + } - if (of_property_read_string(node, "riscv,isa", &isa)) { - pr_warn("Unable to find \"riscv,isa\" devicetree entry\n"); - continue; + rc = of_property_read_string(node, "riscv,isa", &isa); + of_node_put(node); + if (rc) { + pr_warn("Unable to find \"riscv,isa\" devicetree entry\n"); + continue; + } + } else { + rc = acpi_get_riscv_isa(rhct, cpu, &isa); + if (rc < 0) { + pr_warn("Unable to get ISA for the hart - %d\n", cpu); + continue; + } } - temp = isa; -#if IS_ENABLED(CONFIG_32BIT) - if (!strncmp(isa, "rv32", 4)) - isa += 4; -#elif IS_ENABLED(CONFIG_64BIT) - if (!strncmp(isa, "rv64", 4)) - isa += 4; -#endif - /* The riscv,isa DT property must start with rv64 or rv32 */ - if (temp == isa) - continue; - bitmap_zero(this_isa, RISCV_ISA_EXT_MAX); - for (; *isa; ++isa) { + /* + * For all possible cpus, we have already validated in + * the boot process that they at least contain "rv" and + * whichever of "32"/"64" this kernel supports, and so this + * section can be skipped. + */ + isa += 4; + + while (*isa) { const char *ext = isa++; const char *ext_end = isa; bool ext_long = false, ext_err = false; switch (*ext) { case 's': - /** + /* * Workaround for invalid single-letter 's' & 'u'(QEMU). * No need to set the bit in riscv_isa as 's' & 'u' are * not valid ISA extensions. It works until multi-letter @@ -157,75 +180,123 @@ void __init riscv_fill_hwcap(void) break; } fallthrough; + case 'S': case 'x': + case 'X': case 'z': + case 'Z': + /* + * Before attempting to parse the extension itself, we find its end. + * As multi-letter extensions must be split from other multi-letter + * extensions with an "_", the end of a multi-letter extension will + * either be the null character or the "_" at the start of the next + * multi-letter extension. + * + * Next, as the extensions version is currently ignored, we + * eliminate that portion. This is done by parsing backwards from + * the end of the extension, removing any numbers. This may be a + * major or minor number however, so the process is repeated if a + * minor number was found. + * + * ext_end is intended to represent the first character *after* the + * name portion of an extension, but will be decremented to the last + * character itself while eliminating the extensions version number. + * A simple re-increment solves this problem. + */ ext_long = true; - /* Multi-letter extension must be delimited */ for (; *isa && *isa != '_'; ++isa) - if (unlikely(!islower(*isa) - && !isdigit(*isa))) + if (unlikely(!isalnum(*isa))) ext_err = true; - /* Parse backwards */ + ext_end = isa; if (unlikely(ext_err)) break; + if (!isdigit(ext_end[-1])) break; - /* Skip the minor version */ + while (isdigit(*--ext_end)) ; - if (ext_end[0] != 'p' - || !isdigit(ext_end[-1])) { - /* Advance it to offset the pre-decrement */ + + if (tolower(ext_end[0]) != 'p' || !isdigit(ext_end[-1])) { ++ext_end; break; } - /* Skip the major version */ + while (isdigit(*--ext_end)) ; + ++ext_end; break; default: - if (unlikely(!islower(*ext))) { + /* + * Things are a little easier for single-letter extensions, as they + * are parsed forwards. + * + * After checking that our starting position is valid, we need to + * ensure that, when isa was incremented at the start of the loop, + * that it arrived at the start of the next extension. + * + * If we are already on a non-digit, there is nothing to do. Either + * we have a multi-letter extension's _, or the start of an + * extension. + * + * Otherwise we have found the current extension's major version + * number. Parse past it, and a subsequent p/minor version number + * if present. The `p` extension must not appear immediately after + * a number, so there is no fear of missing it. + * + */ + if (unlikely(!isalpha(*ext))) { ext_err = true; break; } - /* Find next extension */ + if (!isdigit(*isa)) break; - /* Skip the minor version */ + while (isdigit(*++isa)) ; - if (*isa != 'p') + + if (tolower(*isa) != 'p') break; + if (!isdigit(*++isa)) { --isa; break; } - /* Skip the major version */ + while (isdigit(*++isa)) ; + break; } - if (*isa != '_') - --isa; -#define SET_ISA_EXT_MAP(name, bit) \ - do { \ - if ((ext_end - ext == sizeof(name) - 1) && \ - !memcmp(ext, name, sizeof(name) - 1) && \ - riscv_isa_extension_check(bit)) \ - set_bit(bit, this_isa); \ - } while (false) \ + /* + * The parser expects that at the start of an iteration isa points to the + * first character of the next extension. As we stop parsing an extension + * on meeting a non-alphanumeric character, an extra increment is needed + * where the succeeding extension is a multi-letter prefixed with an "_". + */ + if (*isa == '_') + ++isa; + +#define SET_ISA_EXT_MAP(name, bit) \ + do { \ + if ((ext_end - ext == sizeof(name) - 1) && \ + !strncasecmp(ext, name, sizeof(name) - 1) && \ + riscv_isa_extension_check(bit)) \ + set_bit(bit, isainfo->isa); \ + } while (false) \ if (unlikely(ext_err)) continue; if (!ext_long) { - int nr = *ext - 'a'; + int nr = tolower(*ext) - 'a'; if (riscv_isa_extension_check(nr)) { this_hwcap |= isa2hwcap[nr]; - set_bit(nr, this_isa); + set_bit(nr, isainfo->isa); } } else { /* sorted alphabetically */ @@ -236,7 +307,9 @@ void __init riscv_fill_hwcap(void) SET_ISA_EXT_MAP("svinval", RISCV_ISA_EXT_SVINVAL); SET_ISA_EXT_MAP("svnapot", RISCV_ISA_EXT_SVNAPOT); SET_ISA_EXT_MAP("svpbmt", RISCV_ISA_EXT_SVPBMT); + SET_ISA_EXT_MAP("zba", RISCV_ISA_EXT_ZBA); SET_ISA_EXT_MAP("zbb", RISCV_ISA_EXT_ZBB); + SET_ISA_EXT_MAP("zbs", RISCV_ISA_EXT_ZBS); SET_ISA_EXT_MAP("zicbom", RISCV_ISA_EXT_ZICBOM); SET_ISA_EXT_MAP("zicboz", RISCV_ISA_EXT_ZICBOZ); SET_ISA_EXT_MAP("zihintpause", RISCV_ISA_EXT_ZIHINTPAUSE); @@ -245,6 +318,23 @@ void __init riscv_fill_hwcap(void) } /* + * Linux requires the following extensions, so we may as well + * always set them. + */ + set_bit(RISCV_ISA_EXT_ZICSR, isainfo->isa); + set_bit(RISCV_ISA_EXT_ZIFENCEI, isainfo->isa); + + /* + * These ones were as they were part of the base ISA when the + * port & dt-bindings were upstreamed, and so can be set + * unconditionally where `i` is in riscv,isa on DT systems. + */ + if (acpi_disabled) { + set_bit(RISCV_ISA_EXT_ZICNTR, isainfo->isa); + set_bit(RISCV_ISA_EXT_ZIHPM, isainfo->isa); + } + + /* * All "okay" hart should have same isa. Set HWCAP based on * common capabilities of every "okay" hart, in case they don't * have. @@ -255,11 +345,14 @@ void __init riscv_fill_hwcap(void) elf_hwcap = this_hwcap; if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) - bitmap_copy(riscv_isa, this_isa, RISCV_ISA_EXT_MAX); + bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); else - bitmap_and(riscv_isa, riscv_isa, this_isa, RISCV_ISA_EXT_MAX); + bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); } + if (!acpi_disabled && rhct) + acpi_put_table((struct acpi_table_header *)rhct); + /* We don't support systems with F but without D, so mask those out * here. */ if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) { @@ -267,6 +360,17 @@ void __init riscv_fill_hwcap(void) elf_hwcap &= ~COMPAT_HWCAP_ISA_F; } + if (elf_hwcap & COMPAT_HWCAP_ISA_V) { + riscv_v_setup_vsize(); + /* + * ISA string in device tree might have 'v' flag, but + * CONFIG_RISCV_ISA_V is disabled in kernel. + * Clear V flag in elf_hwcap if CONFIG_RISCV_ISA_V is disabled. + */ + if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) + elf_hwcap &= ~COMPAT_HWCAP_ISA_V; + } + memset(print_str, 0, sizeof(print_str)); for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) if (riscv_isa[0] & BIT_MASK(i)) @@ -280,6 +384,18 @@ void __init riscv_fill_hwcap(void) pr_info("riscv: ELF capabilities %s\n", print_str); } +unsigned long riscv_get_elf_hwcap(void) +{ + unsigned long hwcap; + + hwcap = (elf_hwcap & ((1UL << RISCV_ISA_EXT_BASE) - 1)); + + if (!riscv_v_vstate_ctrl_user_allowed()) + hwcap &= ~COMPAT_HWCAP_ISA_V; + + return hwcap; +} + #ifdef CONFIG_RISCV_ALTERNATIVE /* * Alternative patch sites consider 48 bits when determining when to patch diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 3fbb100bc9e4..143a2bb3e697 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -48,10 +48,10 @@ _save_context: * Disable user-mode memory access as it should only be set in the * actual user copy routines. * - * Disable the FPU to detect illegal usage of floating point in kernel - * space. + * Disable the FPU/Vector to detect illegal usage of floating point + * or vector in kernel space. */ - li t0, SR_SUM | SR_FS + li t0, SR_SUM | SR_FS_VS REG_L s0, TASK_TI_USER_SP(tp) csrrc s1, CSR_STATUS, t0 @@ -348,6 +348,6 @@ SYM_CODE_END(excp_vect_table) #ifndef CONFIG_MMU SYM_CODE_START(__user_rt_sigreturn) li a7, __NR_rt_sigreturn - scall + ecall SYM_CODE_END(__user_rt_sigreturn) #endif diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index 4bf6c449d78b..11c3b94c4534 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -140,10 +140,10 @@ secondary_start_sbi: .option pop /* - * Disable FPU to detect illegal usage of - * floating point in kernel space + * Disable FPU & VECTOR to detect illegal usage of + * floating point or vector in kernel space */ - li t0, SR_FS + li t0, SR_FS_VS csrc CSR_STATUS, t0 /* Set trap vector to spin forever to help debug */ @@ -234,10 +234,10 @@ pmp_done: .option pop /* - * Disable FPU to detect illegal usage of - * floating point in kernel space + * Disable FPU & VECTOR to detect illegal usage of + * floating point or vector in kernel space */ - li t0, SR_FS + li t0, SR_FS_VS csrc CSR_STATUS, t0 #ifdef CONFIG_RISCV_BOOT_SPINWAIT @@ -301,6 +301,7 @@ clear_bss_done: la tp, init_task la sp, init_thread_union + THREAD_SIZE XIP_FIXUP_OFFSET sp + addi sp, sp, -PT_SIZE_ON_STACK #ifdef CONFIG_BUILTIN_DTB la a0, __dtb_start XIP_FIXUP_OFFSET a0 @@ -318,6 +319,7 @@ clear_bss_done: /* Restore C environment */ la tp, init_task la sp, init_thread_union + THREAD_SIZE + addi sp, sp, -PT_SIZE_ON_STACK #ifdef CONFIG_KASAN call kasan_early_init @@ -392,7 +394,7 @@ ENTRY(reset_regs) #ifdef CONFIG_FPU csrr t0, CSR_MISA andi t0, t0, (COMPAT_HWCAP_ISA_F | COMPAT_HWCAP_ISA_D) - beqz t0, .Lreset_regs_done + beqz t0, .Lreset_regs_done_fpu li t1, SR_FS csrs CSR_STATUS, t1 @@ -430,8 +432,31 @@ ENTRY(reset_regs) fmv.s.x f31, zero csrw fcsr, 0 /* note that the caller must clear SR_FS */ +.Lreset_regs_done_fpu: #endif /* CONFIG_FPU */ -.Lreset_regs_done: + +#ifdef CONFIG_RISCV_ISA_V + csrr t0, CSR_MISA + li t1, COMPAT_HWCAP_ISA_V + and t0, t0, t1 + beqz t0, .Lreset_regs_done_vector + + /* + * Clear vector registers and reset vcsr + * VLMAX has a defined value, VLEN is a constant, + * and this form of vsetvli is defined to set vl to VLMAX. + */ + li t1, SR_VS + csrs CSR_STATUS, t1 + csrs CSR_VCSR, x0 + vsetvli t1, x0, e8, m8, ta, ma + vmv.v.i v0, 0 + vmv.v.i v8, 0 + vmv.v.i v16, 0 + vmv.v.i v24, 0 + /* note that the caller must clear SR_VS */ +.Lreset_regs_done_vector: +#endif /* CONFIG_RISCV_ISA_V */ ret END(reset_regs) #endif /* CONFIG_RISCV_M_MODE */ diff --git a/arch/riscv/kernel/hibernate-asm.S b/arch/riscv/kernel/hibernate-asm.S index effaf5ca5da0..d698dd7df637 100644 --- a/arch/riscv/kernel/hibernate-asm.S +++ b/arch/riscv/kernel/hibernate-asm.S @@ -28,7 +28,6 @@ ENTRY(__hibernate_cpu_resume) REG_L a0, hibernate_cpu_context - suspend_restore_csrs suspend_restore_regs /* Return zero value. */ @@ -50,7 +49,7 @@ ENTRY(hibernate_restore_image) REG_L s4, restore_pblist REG_L a1, relocated_restore_code - jalr a1 + jr a1 END(hibernate_restore_image) /* @@ -73,5 +72,5 @@ ENTRY(hibernate_core_restore_code) REG_L s4, HIBERN_PBE_NEXT(s4) bnez s4, .Lcopy - jalr s2 + jr s2 END(hibernate_core_restore_code) diff --git a/arch/riscv/kernel/hibernate.c b/arch/riscv/kernel/hibernate.c index 264b2dcdd67e..671b686c0158 100644 --- a/arch/riscv/kernel/hibernate.c +++ b/arch/riscv/kernel/hibernate.c @@ -80,7 +80,6 @@ int pfn_is_nosave(unsigned long pfn) void notrace save_processor_state(void) { - WARN_ON(num_online_cpus() != 1); } void notrace restore_processor_state(void) diff --git a/arch/riscv/kernel/irq.c b/arch/riscv/kernel/irq.c index eb9a68a539e6..d0577cc6a081 100644 --- a/arch/riscv/kernel/irq.c +++ b/arch/riscv/kernel/irq.c @@ -11,6 +11,9 @@ #include <linux/module.h> #include <linux/seq_file.h> #include <asm/sbi.h> +#include <asm/smp.h> +#include <asm/softirq_stack.h> +#include <asm/stacktrace.h> static struct fwnode_handle *(*__get_intc_node)(void); @@ -28,6 +31,70 @@ struct fwnode_handle *riscv_get_intc_hwnode(void) } EXPORT_SYMBOL_GPL(riscv_get_intc_hwnode); +#ifdef CONFIG_IRQ_STACKS +#include <asm/irq_stack.h> + +DEFINE_PER_CPU(ulong *, irq_stack_ptr); + +#ifdef CONFIG_VMAP_STACK +static void init_irq_stacks(void) +{ + int cpu; + ulong *p; + + for_each_possible_cpu(cpu) { + p = arch_alloc_vmap_stack(IRQ_STACK_SIZE, cpu_to_node(cpu)); + per_cpu(irq_stack_ptr, cpu) = p; + } +} +#else +/* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */ +DEFINE_PER_CPU_ALIGNED(ulong [IRQ_STACK_SIZE/sizeof(ulong)], irq_stack); + +static void init_irq_stacks(void) +{ + int cpu; + + for_each_possible_cpu(cpu) + per_cpu(irq_stack_ptr, cpu) = per_cpu(irq_stack, cpu); +} +#endif /* CONFIG_VMAP_STACK */ + +#ifdef CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK +void do_softirq_own_stack(void) +{ +#ifdef CONFIG_IRQ_STACKS + if (on_thread_stack()) { + ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id()) + + IRQ_STACK_SIZE/sizeof(ulong); + __asm__ __volatile( + "addi sp, sp, -"RISCV_SZPTR "\n" + REG_S" ra, (sp) \n" + "addi sp, sp, -"RISCV_SZPTR "\n" + REG_S" s0, (sp) \n" + "addi s0, sp, 2*"RISCV_SZPTR "\n" + "move sp, %[sp] \n" + "call __do_softirq \n" + "addi sp, s0, -2*"RISCV_SZPTR"\n" + REG_L" s0, (sp) \n" + "addi sp, sp, "RISCV_SZPTR "\n" + REG_L" ra, (sp) \n" + "addi sp, sp, "RISCV_SZPTR "\n" + : + : [sp] "r" (sp) + : "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", + "t0", "t1", "t2", "t3", "t4", "t5", "t6", + "memory"); + } else +#endif + __do_softirq(); +} +#endif /* CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK */ + +#else +static void init_irq_stacks(void) {} +#endif /* CONFIG_IRQ_STACKS */ + int arch_show_interrupts(struct seq_file *p, int prec) { show_ipi_stats(p, prec); @@ -36,6 +103,7 @@ int arch_show_interrupts(struct seq_file *p, int prec) void __init init_IRQ(void) { + init_irq_stacks(); irqchip_init(); if (!handle_arch_irq) panic("No interrupt controller found."); diff --git a/arch/riscv/kernel/mcount.S b/arch/riscv/kernel/mcount.S index 30102aadc4d7..8a6e5a9e842a 100644 --- a/arch/riscv/kernel/mcount.S +++ b/arch/riscv/kernel/mcount.S @@ -65,13 +65,8 @@ ENTRY(return_to_handler) * So alternatively we check the *old* frame pointer position, that is, the * value stored in -16(s0) on entry, and the s0 on return. */ -#ifdef HAVE_FUNCTION_GRAPH_FP_TEST - mv t6, s0 -#endif SAVE_RET_ABI_STATE -#ifdef HAVE_FUNCTION_GRAPH_FP_TEST - mv a0, t6 -#endif + mv a0, sp call ftrace_return_to_handler mv a2, a0 RESTORE_RET_ABI_STATE diff --git a/arch/riscv/kernel/probes/uprobes.c b/arch/riscv/kernel/probes/uprobes.c index c976a21cd4bd..194f166b2cc4 100644 --- a/arch/riscv/kernel/probes/uprobes.c +++ b/arch/riscv/kernel/probes/uprobes.c @@ -67,6 +67,7 @@ int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) struct uprobe_task *utask = current->utask; WARN_ON_ONCE(current->thread.bad_cause != UPROBE_TRAP_NR); + current->thread.bad_cause = utask->autask.saved_cause; instruction_pointer_set(regs, utask->vaddr + auprobe->insn_size); @@ -102,6 +103,7 @@ void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) { struct uprobe_task *utask = current->utask; + current->thread.bad_cause = utask->autask.saved_cause; /* * Task has received a fatal signal, so reset back to probbed * address. diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index e2a060066730..e32d737e039f 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -24,6 +24,7 @@ #include <asm/switch_to.h> #include <asm/thread_info.h> #include <asm/cpuidle.h> +#include <asm/vector.h> register unsigned long gp_in_global __asm__("gp"); @@ -146,12 +147,29 @@ void flush_thread(void) fstate_off(current, task_pt_regs(current)); memset(¤t->thread.fstate, 0, sizeof(current->thread.fstate)); #endif +#ifdef CONFIG_RISCV_ISA_V + /* Reset vector state */ + riscv_v_vstate_ctrl_init(current); + riscv_v_vstate_off(task_pt_regs(current)); + kfree(current->thread.vstate.datap); + memset(¤t->thread.vstate, 0, sizeof(struct __riscv_v_ext_state)); +#endif +} + +void arch_release_task_struct(struct task_struct *tsk) +{ + /* Free the vector context of datap. */ + if (has_vector()) + kfree(tsk->thread.vstate.datap); } int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { fstate_save(src, task_pt_regs(src)); *dst = *src; + /* clear entire V context, including datap for a new task */ + memset(&dst->thread.vstate, 0, sizeof(struct __riscv_v_ext_state)); + return 0; } @@ -176,6 +194,8 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) p->thread.s[1] = (unsigned long)args->fn_arg; } else { *childregs = *(current_pt_regs()); + /* Turn off status.VS */ + riscv_v_vstate_off(childregs); if (usp) /* User fork */ childregs->sp = usp; if (clone_flags & CLONE_SETTLS) diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c index 23c48b14a0e7..1d572cf3140f 100644 --- a/arch/riscv/kernel/ptrace.c +++ b/arch/riscv/kernel/ptrace.c @@ -7,6 +7,7 @@ * Copied from arch/tile/kernel/ptrace.c */ +#include <asm/vector.h> #include <asm/ptrace.h> #include <asm/syscall.h> #include <asm/thread_info.h> @@ -24,6 +25,9 @@ enum riscv_regset { #ifdef CONFIG_FPU REGSET_F, #endif +#ifdef CONFIG_RISCV_ISA_V + REGSET_V, +#endif }; static int riscv_gpr_get(struct task_struct *target, @@ -80,6 +84,61 @@ static int riscv_fpr_set(struct task_struct *target, } #endif +#ifdef CONFIG_RISCV_ISA_V +static int riscv_vr_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + struct __riscv_v_ext_state *vstate = &target->thread.vstate; + + if (!riscv_v_vstate_query(task_pt_regs(target))) + return -EINVAL; + + /* + * Ensure the vector registers have been saved to the memory before + * copying them to membuf. + */ + if (target == current) + riscv_v_vstate_save(current, task_pt_regs(current)); + + /* Copy vector header from vstate. */ + membuf_write(&to, vstate, offsetof(struct __riscv_v_ext_state, datap)); + membuf_zero(&to, sizeof(vstate->datap)); + + /* Copy all the vector registers from vstate. */ + return membuf_write(&to, vstate->datap, riscv_v_vsize); +} + +static int riscv_vr_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + int ret, size; + struct __riscv_v_ext_state *vstate = &target->thread.vstate; + + if (!riscv_v_vstate_query(task_pt_regs(target))) + return -EINVAL; + + /* Copy rest of the vstate except datap */ + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, vstate, 0, + offsetof(struct __riscv_v_ext_state, datap)); + if (unlikely(ret)) + return ret; + + /* Skip copy datap. */ + size = sizeof(vstate->datap); + count -= size; + ubuf += size; + + /* Copy all the vector registers. */ + pos = 0; + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, vstate->datap, + 0, riscv_v_vsize); + return ret; +} +#endif + static const struct user_regset riscv_user_regset[] = { [REGSET_X] = { .core_note_type = NT_PRSTATUS, @@ -99,6 +158,17 @@ static const struct user_regset riscv_user_regset[] = { .set = riscv_fpr_set, }, #endif +#ifdef CONFIG_RISCV_ISA_V + [REGSET_V] = { + .core_note_type = NT_RISCV_VECTOR, + .align = 16, + .n = ((32 * RISCV_MAX_VLENB) + + sizeof(struct __riscv_v_ext_state)) / sizeof(__u32), + .size = sizeof(__u32), + .regset_get = riscv_vr_get, + .set = riscv_vr_set, + }, +#endif }; static const struct user_regset_view riscv_user_native_view = { diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index 36b026057503..971fe776e2f8 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -8,6 +8,7 @@ * Nick Kossifidis <mick@ics.forth.gr> */ +#include <linux/acpi.h> #include <linux/cpu.h> #include <linux/init.h> #include <linux/mm.h> @@ -21,6 +22,7 @@ #include <linux/efi.h> #include <linux/crash_dump.h> +#include <asm/acpi.h> #include <asm/alternative.h> #include <asm/cacheflush.h> #include <asm/cpu_ops.h> @@ -262,6 +264,8 @@ static void __init parse_dtb(void) #endif } +extern void __init init_rt_signal_env(void); + void __init setup_arch(char **cmdline_p) { parse_dtb(); @@ -270,11 +274,16 @@ void __init setup_arch(char **cmdline_p) *cmdline_p = boot_command_line; early_ioremap_setup(); + sbi_init(); jump_label_init(); parse_early_param(); efi_init(); paging_init(); + + /* Parse the ACPI tables for possible boot-time configuration */ + acpi_boot_table_init(); + #if IS_ENABLED(CONFIG_BUILTIN_DTB) unflatten_and_copy_device_tree(); #else @@ -283,7 +292,6 @@ void __init setup_arch(char **cmdline_p) misc_mem_init(); init_resources(); - sbi_init(); #ifdef CONFIG_KASAN kasan_init(); @@ -293,8 +301,12 @@ void __init setup_arch(char **cmdline_p) setup_smp(); #endif + if (!acpi_disabled) + acpi_init_rintc_map(); + riscv_init_cbo_blocksizes(); riscv_fill_hwcap(); + init_rt_signal_env(); apply_boot_alternatives(); if (IS_ENABLED(CONFIG_RISCV_ISA_ZICBOM) && riscv_isa_extension_available(NULL, ZICBOM)) diff --git a/arch/riscv/kernel/signal.c b/arch/riscv/kernel/signal.c index 9aff9d720590..180d951d3624 100644 --- a/arch/riscv/kernel/signal.c +++ b/arch/riscv/kernel/signal.c @@ -19,10 +19,14 @@ #include <asm/signal.h> #include <asm/signal32.h> #include <asm/switch_to.h> +#include <asm/vector.h> #include <asm/csr.h> #include <asm/cacheflush.h> +unsigned long signal_minsigstksz __ro_after_init; + extern u32 __user_rt_sigreturn[2]; +static size_t riscv_v_sc_size __ro_after_init; #define DEBUG_SIG 0 @@ -40,26 +44,13 @@ static long restore_fp_state(struct pt_regs *regs, { long err; struct __riscv_d_ext_state __user *state = &sc_fpregs->d; - size_t i; err = __copy_from_user(¤t->thread.fstate, state, sizeof(*state)); if (unlikely(err)) return err; fstate_restore(current, regs); - - /* We support no other extension state at this time. */ - for (i = 0; i < ARRAY_SIZE(sc_fpregs->q.reserved); i++) { - u32 value; - - err = __get_user(value, &sc_fpregs->q.reserved[i]); - if (unlikely(err)) - break; - if (value != 0) - return -EINVAL; - } - - return err; + return 0; } static long save_fp_state(struct pt_regs *regs, @@ -67,52 +58,186 @@ static long save_fp_state(struct pt_regs *regs, { long err; struct __riscv_d_ext_state __user *state = &sc_fpregs->d; - size_t i; fstate_save(current, regs); err = __copy_to_user(state, ¤t->thread.fstate, sizeof(*state)); + return err; +} +#else +#define save_fp_state(task, regs) (0) +#define restore_fp_state(task, regs) (0) +#endif + +#ifdef CONFIG_RISCV_ISA_V + +static long save_v_state(struct pt_regs *regs, void __user **sc_vec) +{ + struct __riscv_ctx_hdr __user *hdr; + struct __sc_riscv_v_state __user *state; + void __user *datap; + long err; + + hdr = *sc_vec; + /* Place state to the user's signal context space after the hdr */ + state = (struct __sc_riscv_v_state __user *)(hdr + 1); + /* Point datap right after the end of __sc_riscv_v_state */ + datap = state + 1; + + /* datap is designed to be 16 byte aligned for better performance */ + WARN_ON(unlikely(!IS_ALIGNED((unsigned long)datap, 16))); + + riscv_v_vstate_save(current, regs); + /* Copy everything of vstate but datap. */ + err = __copy_to_user(&state->v_state, ¤t->thread.vstate, + offsetof(struct __riscv_v_ext_state, datap)); + /* Copy the pointer datap itself. */ + err |= __put_user(datap, &state->v_state.datap); + /* Copy the whole vector content to user space datap. */ + err |= __copy_to_user(datap, current->thread.vstate.datap, riscv_v_vsize); + /* Copy magic to the user space after saving all vector conetext */ + err |= __put_user(RISCV_V_MAGIC, &hdr->magic); + err |= __put_user(riscv_v_sc_size, &hdr->size); if (unlikely(err)) return err; - /* We support no other extension state at this time. */ - for (i = 0; i < ARRAY_SIZE(sc_fpregs->q.reserved); i++) { - err = __put_user(0, &sc_fpregs->q.reserved[i]); - if (unlikely(err)) - break; - } + /* Only progress the sv_vec if everything has done successfully */ + *sc_vec += riscv_v_sc_size; + return 0; +} + +/* + * Restore Vector extension context from the user's signal frame. This function + * assumes a valid extension header. So magic and size checking must be done by + * the caller. + */ +static long __restore_v_state(struct pt_regs *regs, void __user *sc_vec) +{ + long err; + struct __sc_riscv_v_state __user *state = sc_vec; + void __user *datap; + + /* Copy everything of __sc_riscv_v_state except datap. */ + err = __copy_from_user(¤t->thread.vstate, &state->v_state, + offsetof(struct __riscv_v_ext_state, datap)); + if (unlikely(err)) + return err; + + /* Copy the pointer datap itself. */ + err = __get_user(datap, &state->v_state.datap); + if (unlikely(err)) + return err; + /* + * Copy the whole vector content from user space datap. Use + * copy_from_user to prevent information leak. + */ + err = copy_from_user(current->thread.vstate.datap, datap, riscv_v_vsize); + if (unlikely(err)) + return err; + + riscv_v_vstate_restore(current, regs); return err; } #else -#define save_fp_state(task, regs) (0) -#define restore_fp_state(task, regs) (0) +#define save_v_state(task, regs) (0) +#define __restore_v_state(task, regs) (0) #endif static long restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc) { + void __user *sc_ext_ptr = &sc->sc_extdesc.hdr; + __u32 rsvd; long err; /* sc_regs is structured the same as the start of pt_regs */ err = __copy_from_user(regs, &sc->sc_regs, sizeof(sc->sc_regs)); + if (unlikely(err)) + return err; + /* Restore the floating-point state. */ - if (has_fpu()) - err |= restore_fp_state(regs, &sc->sc_fpregs); + if (has_fpu()) { + err = restore_fp_state(regs, &sc->sc_fpregs); + if (unlikely(err)) + return err; + } + + /* Check the reserved word before extensions parsing */ + err = __get_user(rsvd, &sc->sc_extdesc.reserved); + if (unlikely(err)) + return err; + if (unlikely(rsvd)) + return -EINVAL; + + while (!err) { + __u32 magic, size; + struct __riscv_ctx_hdr __user *head = sc_ext_ptr; + + err |= __get_user(magic, &head->magic); + err |= __get_user(size, &head->size); + if (unlikely(err)) + return err; + + sc_ext_ptr += sizeof(*head); + switch (magic) { + case END_MAGIC: + if (size != END_HDR_SIZE) + return -EINVAL; + + return 0; + case RISCV_V_MAGIC: + if (!has_vector() || !riscv_v_vstate_query(regs) || + size != riscv_v_sc_size) + return -EINVAL; + + err = __restore_v_state(regs, sc_ext_ptr); + break; + default: + return -EINVAL; + } + sc_ext_ptr = (void __user *)head + size; + } return err; } +static size_t get_rt_frame_size(bool cal_all) +{ + struct rt_sigframe __user *frame; + size_t frame_size; + size_t total_context_size = 0; + + frame_size = sizeof(*frame); + + if (has_vector()) { + if (cal_all || riscv_v_vstate_query(task_pt_regs(current))) + total_context_size += riscv_v_sc_size; + } + /* + * Preserved a __riscv_ctx_hdr for END signal context header if an + * extension uses __riscv_extra_ext_header + */ + if (total_context_size) + total_context_size += sizeof(struct __riscv_ctx_hdr); + + frame_size += total_context_size; + + frame_size = round_up(frame_size, 16); + return frame_size; +} + SYSCALL_DEFINE0(rt_sigreturn) { struct pt_regs *regs = current_pt_regs(); struct rt_sigframe __user *frame; struct task_struct *task; sigset_t set; + size_t frame_size = get_rt_frame_size(false); /* Always make any pending restarted system calls return -EINTR */ current->restart_block.fn = do_no_restart_syscall; frame = (struct rt_sigframe __user *)regs->sp; - if (!access_ok(frame, sizeof(*frame))) + if (!access_ok(frame, frame_size)) goto badframe; if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) @@ -146,12 +271,23 @@ static long setup_sigcontext(struct rt_sigframe __user *frame, struct pt_regs *regs) { struct sigcontext __user *sc = &frame->uc.uc_mcontext; + struct __riscv_ctx_hdr __user *sc_ext_ptr = &sc->sc_extdesc.hdr; long err; + /* sc_regs is structured the same as the start of pt_regs */ err = __copy_to_user(&sc->sc_regs, regs, sizeof(sc->sc_regs)); /* Save the floating-point state. */ if (has_fpu()) err |= save_fp_state(regs, &sc->sc_fpregs); + /* Save the vector state. */ + if (has_vector() && riscv_v_vstate_query(regs)) + err |= save_v_state(regs, (void __user **)&sc_ext_ptr); + /* Write zero to fp-reserved space and check it on restore_sigcontext */ + err |= __put_user(0, &sc->sc_extdesc.reserved); + /* And put END __riscv_ctx_hdr at the end. */ + err |= __put_user(END_MAGIC, &sc_ext_ptr->magic); + err |= __put_user(END_HDR_SIZE, &sc_ext_ptr->size); + return err; } @@ -175,6 +311,13 @@ static inline void __user *get_sigframe(struct ksignal *ksig, /* Align the stack frame. */ sp &= ~0xfUL; + /* + * Fail if the size of the altstack is not large enough for the + * sigframe construction. + */ + if (current->sas_ss_size && sp < current->sas_ss_sp) + return (void __user __force *)-1UL; + return (void __user *)sp; } @@ -184,9 +327,10 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct rt_sigframe __user *frame; long err = 0; unsigned long __maybe_unused addr; + size_t frame_size = get_rt_frame_size(false); - frame = get_sigframe(ksig, regs, sizeof(*frame)); - if (!access_ok(frame, sizeof(*frame))) + frame = get_sigframe(ksig, regs, frame_size); + if (!access_ok(frame, frame_size)) return -EFAULT; err |= copy_siginfo_to_user(&frame->info, &ksig->info); @@ -319,3 +463,23 @@ void arch_do_signal_or_restart(struct pt_regs *regs) */ restore_saved_sigmask(); } + +void init_rt_signal_env(void); +void __init init_rt_signal_env(void) +{ + riscv_v_sc_size = sizeof(struct __riscv_ctx_hdr) + + sizeof(struct __sc_riscv_v_state) + riscv_v_vsize; + /* + * Determine the stack space required for guaranteed signal delivery. + * The signal_minsigstksz will be populated into the AT_MINSIGSTKSZ entry + * in the auxiliary array at process startup. + */ + signal_minsigstksz = get_rt_frame_size(true); +} + +#ifdef CONFIG_DYNAMIC_SIGFRAME +bool sigaltstack_size_valid(size_t ss_size) +{ + return ss_size > get_rt_frame_size(false); +} +#endif /* CONFIG_DYNAMIC_SIGFRAME */ diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index 445a4efee267..bb0b76e1a6d4 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -8,6 +8,7 @@ * Copyright (C) 2017 SiFive */ +#include <linux/acpi.h> #include <linux/arch_topology.h> #include <linux/module.h> #include <linux/init.h> @@ -31,6 +32,8 @@ #include <asm/tlbflush.h> #include <asm/sections.h> #include <asm/smp.h> +#include <uapi/asm/hwcap.h> +#include <asm/vector.h> #include "head.h" @@ -70,7 +73,73 @@ void __init smp_prepare_cpus(unsigned int max_cpus) } } -void __init setup_smp(void) +#ifdef CONFIG_ACPI +static unsigned int cpu_count = 1; + +static int __init acpi_parse_rintc(union acpi_subtable_headers *header, const unsigned long end) +{ + unsigned long hart; + static bool found_boot_cpu; + struct acpi_madt_rintc *processor = (struct acpi_madt_rintc *)header; + + /* + * Each RINTC structure in MADT will have a flag. If ACPI_MADT_ENABLED + * bit in the flag is not enabled, it means OS should not try to enable + * the cpu to which RINTC belongs. + */ + if (!(processor->flags & ACPI_MADT_ENABLED)) + return 0; + + if (BAD_MADT_ENTRY(processor, end)) + return -EINVAL; + + acpi_table_print_madt_entry(&header->common); + + hart = processor->hart_id; + if (hart == INVALID_HARTID) { + pr_warn("Invalid hartid\n"); + return 0; + } + + if (hart == cpuid_to_hartid_map(0)) { + BUG_ON(found_boot_cpu); + found_boot_cpu = true; + early_map_cpu_to_node(0, acpi_numa_get_nid(cpu_count)); + return 0; + } + + if (cpu_count >= NR_CPUS) { + pr_warn("NR_CPUS is too small for the number of ACPI tables.\n"); + return 0; + } + + cpuid_to_hartid_map(cpu_count) = hart; + early_map_cpu_to_node(cpu_count, acpi_numa_get_nid(cpu_count)); + cpu_count++; + + return 0; +} + +static void __init acpi_parse_and_init_cpus(void) +{ + int cpuid; + + cpu_set_ops(0); + + acpi_table_parse_madt(ACPI_MADT_TYPE_RINTC, acpi_parse_rintc, 0); + + for (cpuid = 1; cpuid < nr_cpu_ids; cpuid++) { + if (cpuid_to_hartid_map(cpuid) != INVALID_HARTID) { + cpu_set_ops(cpuid); + set_cpu_possible(cpuid, true); + } + } +} +#else +#define acpi_parse_and_init_cpus(...) do { } while (0) +#endif + +static void __init of_parse_and_init_cpus(void) { struct device_node *dn; unsigned long hart; @@ -81,7 +150,7 @@ void __init setup_smp(void) cpu_set_ops(0); for_each_of_cpu_node(dn) { - rc = riscv_of_processor_hartid(dn, &hart); + rc = riscv_early_of_processor_hartid(dn, &hart); if (rc < 0) continue; @@ -116,6 +185,14 @@ void __init setup_smp(void) } } +void __init setup_smp(void) +{ + if (acpi_disabled) + of_parse_and_init_cpus(); + else + acpi_parse_and_init_cpus(); +} + static int start_secondary_cpu(int cpu, struct task_struct *tidle) { if (cpu_ops[cpu]->cpu_start) @@ -169,6 +246,11 @@ asmlinkage __visible void smp_callin(void) set_cpu_online(curr_cpuid, 1); probe_vendor_features(curr_cpuid); + if (has_vector()) { + if (riscv_v_setup_vsize()) + elf_hwcap &= ~COMPAT_HWCAP_ISA_V; + } + /* * Remote TLB flushes are ignored while the CPU is offline, so emit * a local TLB flush right now just in case. diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c index 5db29683ebee..26ef5526bfb4 100644 --- a/arch/riscv/kernel/sys_riscv.c +++ b/arch/riscv/kernel/sys_riscv.c @@ -10,6 +10,7 @@ #include <asm/cpufeature.h> #include <asm/hwprobe.h> #include <asm/sbi.h> +#include <asm/vector.h> #include <asm/switch_to.h> #include <asm/uaccess.h> #include <asm/unistd.h> @@ -121,6 +122,49 @@ static void hwprobe_arch_id(struct riscv_hwprobe *pair, pair->value = id; } +static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, + const struct cpumask *cpus) +{ + int cpu; + u64 missing = 0; + + pair->value = 0; + if (has_fpu()) + pair->value |= RISCV_HWPROBE_IMA_FD; + + if (riscv_isa_extension_available(NULL, c)) + pair->value |= RISCV_HWPROBE_IMA_C; + + if (has_vector()) + pair->value |= RISCV_HWPROBE_IMA_V; + + /* + * Loop through and record extensions that 1) anyone has, and 2) anyone + * doesn't have. + */ + for_each_cpu(cpu, cpus) { + struct riscv_isainfo *isainfo = &hart_isa[cpu]; + + if (riscv_isa_extension_available(isainfo->isa, ZBA)) + pair->value |= RISCV_HWPROBE_EXT_ZBA; + else + missing |= RISCV_HWPROBE_EXT_ZBA; + + if (riscv_isa_extension_available(isainfo->isa, ZBB)) + pair->value |= RISCV_HWPROBE_EXT_ZBB; + else + missing |= RISCV_HWPROBE_EXT_ZBB; + + if (riscv_isa_extension_available(isainfo->isa, ZBS)) + pair->value |= RISCV_HWPROBE_EXT_ZBS; + else + missing |= RISCV_HWPROBE_EXT_ZBS; + } + + /* Now turn off reporting features if any CPU is missing it. */ + pair->value &= ~missing; +} + static u64 hwprobe_misaligned(const struct cpumask *cpus) { int cpu; @@ -164,13 +208,7 @@ static void hwprobe_one_pair(struct riscv_hwprobe *pair, break; case RISCV_HWPROBE_KEY_IMA_EXT_0: - pair->value = 0; - if (has_fpu()) - pair->value |= RISCV_HWPROBE_IMA_FD; - - if (riscv_isa_extension_available(NULL, c)) - pair->value |= RISCV_HWPROBE_IMA_C; - + hwprobe_isa_ext0(pair, cpus); break; case RISCV_HWPROBE_KEY_CPUPERF_0: diff --git a/arch/riscv/kernel/time.c b/arch/riscv/kernel/time.c index babaf3b48ba8..23641e82a9df 100644 --- a/arch/riscv/kernel/time.c +++ b/arch/riscv/kernel/time.c @@ -4,6 +4,7 @@ * Copyright (C) 2017 SiFive */ +#include <linux/acpi.h> #include <linux/of_clk.h> #include <linux/clockchips.h> #include <linux/clocksource.h> @@ -18,17 +19,29 @@ EXPORT_SYMBOL_GPL(riscv_timebase); void __init time_init(void) { struct device_node *cpu; + struct acpi_table_rhct *rhct; + acpi_status status; u32 prop; - cpu = of_find_node_by_path("/cpus"); - if (!cpu || of_property_read_u32(cpu, "timebase-frequency", &prop)) - panic(KERN_WARNING "RISC-V system with no 'timebase-frequency' in DTS\n"); - of_node_put(cpu); - riscv_timebase = prop; + if (acpi_disabled) { + cpu = of_find_node_by_path("/cpus"); + if (!cpu || of_property_read_u32(cpu, "timebase-frequency", &prop)) + panic("RISC-V system with no 'timebase-frequency' in DTS\n"); + + of_node_put(cpu); + riscv_timebase = prop; + of_clk_init(NULL); + } else { + status = acpi_get_table(ACPI_SIG_RHCT, 0, (struct acpi_table_header **)&rhct); + if (ACPI_FAILURE(status)) + panic("RISC-V ACPI system with no RHCT table\n"); + + riscv_timebase = rhct->time_base_freq; + acpi_put_table((struct acpi_table_header *)rhct); + } lpj_fine = riscv_timebase / HZ; - of_clk_init(NULL); timer_probe(); tick_setup_hrtimer_broadcast(); diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index 8c258b78c925..5158961ea977 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -26,6 +26,8 @@ #include <asm/ptrace.h> #include <asm/syscall.h> #include <asm/thread_info.h> +#include <asm/vector.h> +#include <asm/irq_stack.h> int show_unhandled_signals = 1; @@ -145,8 +147,29 @@ DO_ERROR_INFO(do_trap_insn_misaligned, SIGBUS, BUS_ADRALN, "instruction address misaligned"); DO_ERROR_INFO(do_trap_insn_fault, SIGSEGV, SEGV_ACCERR, "instruction access fault"); -DO_ERROR_INFO(do_trap_insn_illegal, - SIGILL, ILL_ILLOPC, "illegal instruction"); + +asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *regs) +{ + if (user_mode(regs)) { + irqentry_enter_from_user_mode(regs); + + local_irq_enable(); + + if (!riscv_v_first_use_handler(regs)) + do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc, + "Oops - illegal instruction"); + + irqentry_exit_to_user_mode(regs); + } else { + irqentry_state_t state = irqentry_nmi_enter(regs); + + do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc, + "Oops - illegal instruction"); + + irqentry_nmi_exit(regs, state); + } +} + DO_ERROR_INFO(do_trap_load_fault, SIGSEGV, SEGV_ACCERR, "load access fault"); #ifndef CONFIG_RISCV_M_MODE @@ -305,16 +328,46 @@ asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs) } #endif -asmlinkage __visible noinstr void do_irq(struct pt_regs *regs) +static void noinstr handle_riscv_irq(struct pt_regs *regs) { struct pt_regs *old_regs; - irqentry_state_t state = irqentry_enter(regs); irq_enter_rcu(); old_regs = set_irq_regs(regs); handle_arch_irq(regs); set_irq_regs(old_regs); irq_exit_rcu(); +} + +asmlinkage void noinstr do_irq(struct pt_regs *regs) +{ + irqentry_state_t state = irqentry_enter(regs); +#ifdef CONFIG_IRQ_STACKS + if (on_thread_stack()) { + ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id()) + + IRQ_STACK_SIZE/sizeof(ulong); + __asm__ __volatile( + "addi sp, sp, -"RISCV_SZPTR "\n" + REG_S" ra, (sp) \n" + "addi sp, sp, -"RISCV_SZPTR "\n" + REG_S" s0, (sp) \n" + "addi s0, sp, 2*"RISCV_SZPTR "\n" + "move sp, %[sp] \n" + "move a0, %[regs] \n" + "call handle_riscv_irq \n" + "addi sp, s0, -2*"RISCV_SZPTR"\n" + REG_L" s0, (sp) \n" + "addi sp, sp, "RISCV_SZPTR "\n" + REG_L" ra, (sp) \n" + "addi sp, sp, "RISCV_SZPTR "\n" + : + : [sp] "r" (sp), [regs] "r" (regs) + : "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", + "t0", "t1", "t2", "t3", "t4", "t5", "t6", + "memory"); + } else +#endif + handle_riscv_irq(regs); irqentry_exit(regs, state); } diff --git a/arch/riscv/kernel/vdso/rt_sigreturn.S b/arch/riscv/kernel/vdso/rt_sigreturn.S index 0573705eac76..10438c7c626a 100644 --- a/arch/riscv/kernel/vdso/rt_sigreturn.S +++ b/arch/riscv/kernel/vdso/rt_sigreturn.S @@ -11,6 +11,6 @@ ENTRY(__vdso_rt_sigreturn) .cfi_startproc .cfi_signal_frame li a7, __NR_rt_sigreturn - scall + ecall .cfi_endproc ENDPROC(__vdso_rt_sigreturn) diff --git a/arch/riscv/kernel/vector.c b/arch/riscv/kernel/vector.c new file mode 100644 index 000000000000..f9c8e19ab301 --- /dev/null +++ b/arch/riscv/kernel/vector.c @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2023 SiFive + * Author: Andy Chiu <andy.chiu@sifive.com> + */ +#include <linux/export.h> +#include <linux/sched/signal.h> +#include <linux/types.h> +#include <linux/slab.h> +#include <linux/sched.h> +#include <linux/uaccess.h> +#include <linux/prctl.h> + +#include <asm/thread_info.h> +#include <asm/processor.h> +#include <asm/insn.h> +#include <asm/vector.h> +#include <asm/csr.h> +#include <asm/elf.h> +#include <asm/ptrace.h> +#include <asm/bug.h> + +static bool riscv_v_implicit_uacc = IS_ENABLED(CONFIG_RISCV_ISA_V_DEFAULT_ENABLE); + +unsigned long riscv_v_vsize __read_mostly; +EXPORT_SYMBOL_GPL(riscv_v_vsize); + +int riscv_v_setup_vsize(void) +{ + unsigned long this_vsize; + + /* There are 32 vector registers with vlenb length. */ + riscv_v_enable(); + this_vsize = csr_read(CSR_VLENB) * 32; + riscv_v_disable(); + + if (!riscv_v_vsize) { + riscv_v_vsize = this_vsize; + return 0; + } + + if (riscv_v_vsize != this_vsize) { + WARN(1, "RISCV_ISA_V only supports one vlenb on SMP systems"); + return -EOPNOTSUPP; + } + + return 0; +} + +static bool insn_is_vector(u32 insn_buf) +{ + u32 opcode = insn_buf & __INSN_OPCODE_MASK; + u32 width, csr; + + /* + * All V-related instructions, including CSR operations are 4-Byte. So, + * do not handle if the instruction length is not 4-Byte. + */ + if (unlikely(GET_INSN_LENGTH(insn_buf) != 4)) + return false; + + switch (opcode) { + case RVV_OPCODE_VECTOR: + return true; + case RVV_OPCODE_VL: + case RVV_OPCODE_VS: + width = RVV_EXRACT_VL_VS_WIDTH(insn_buf); + if (width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 || + width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64) + return true; + + break; + case RVG_OPCODE_SYSTEM: + csr = RVG_EXTRACT_SYSTEM_CSR(insn_buf); + if ((csr >= CSR_VSTART && csr <= CSR_VCSR) || + (csr >= CSR_VL && csr <= CSR_VLENB)) + return true; + } + + return false; +} + +static int riscv_v_thread_zalloc(void) +{ + void *datap; + + datap = kzalloc(riscv_v_vsize, GFP_KERNEL); + if (!datap) + return -ENOMEM; + + current->thread.vstate.datap = datap; + memset(¤t->thread.vstate, 0, offsetof(struct __riscv_v_ext_state, + datap)); + return 0; +} + +#define VSTATE_CTRL_GET_CUR(x) ((x) & PR_RISCV_V_VSTATE_CTRL_CUR_MASK) +#define VSTATE_CTRL_GET_NEXT(x) (((x) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) >> 2) +#define VSTATE_CTRL_MAKE_NEXT(x) (((x) << 2) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) +#define VSTATE_CTRL_GET_INHERIT(x) (!!((x) & PR_RISCV_V_VSTATE_CTRL_INHERIT)) +static inline int riscv_v_ctrl_get_cur(struct task_struct *tsk) +{ + return VSTATE_CTRL_GET_CUR(tsk->thread.vstate_ctrl); +} + +static inline int riscv_v_ctrl_get_next(struct task_struct *tsk) +{ + return VSTATE_CTRL_GET_NEXT(tsk->thread.vstate_ctrl); +} + +static inline bool riscv_v_ctrl_test_inherit(struct task_struct *tsk) +{ + return VSTATE_CTRL_GET_INHERIT(tsk->thread.vstate_ctrl); +} + +static inline void riscv_v_ctrl_set(struct task_struct *tsk, int cur, int nxt, + bool inherit) +{ + unsigned long ctrl; + + ctrl = cur & PR_RISCV_V_VSTATE_CTRL_CUR_MASK; + ctrl |= VSTATE_CTRL_MAKE_NEXT(nxt); + if (inherit) + ctrl |= PR_RISCV_V_VSTATE_CTRL_INHERIT; + tsk->thread.vstate_ctrl = ctrl; +} + +bool riscv_v_vstate_ctrl_user_allowed(void) +{ + return riscv_v_ctrl_get_cur(current) == PR_RISCV_V_VSTATE_CTRL_ON; +} +EXPORT_SYMBOL_GPL(riscv_v_vstate_ctrl_user_allowed); + +bool riscv_v_first_use_handler(struct pt_regs *regs) +{ + u32 __user *epc = (u32 __user *)regs->epc; + u32 insn = (u32)regs->badaddr; + + /* Do not handle if V is not supported, or disabled */ + if (!(ELF_HWCAP & COMPAT_HWCAP_ISA_V)) + return false; + + /* If V has been enabled then it is not the first-use trap */ + if (riscv_v_vstate_query(regs)) + return false; + + /* Get the instruction */ + if (!insn) { + if (__get_user(insn, epc)) + return false; + } + + /* Filter out non-V instructions */ + if (!insn_is_vector(insn)) + return false; + + /* Sanity check. datap should be null by the time of the first-use trap */ + WARN_ON(current->thread.vstate.datap); + + /* + * Now we sure that this is a V instruction. And it executes in the + * context where VS has been off. So, try to allocate the user's V + * context and resume execution. + */ + if (riscv_v_thread_zalloc()) { + force_sig(SIGBUS); + return true; + } + riscv_v_vstate_on(regs); + return true; +} + +void riscv_v_vstate_ctrl_init(struct task_struct *tsk) +{ + bool inherit; + int cur, next; + + if (!has_vector()) + return; + + next = riscv_v_ctrl_get_next(tsk); + if (!next) { + if (READ_ONCE(riscv_v_implicit_uacc)) + cur = PR_RISCV_V_VSTATE_CTRL_ON; + else + cur = PR_RISCV_V_VSTATE_CTRL_OFF; + } else { + cur = next; + } + /* Clear next mask if inherit-bit is not set */ + inherit = riscv_v_ctrl_test_inherit(tsk); + if (!inherit) + next = PR_RISCV_V_VSTATE_CTRL_DEFAULT; + + riscv_v_ctrl_set(tsk, cur, next, inherit); +} + +long riscv_v_vstate_ctrl_get_current(void) +{ + if (!has_vector()) + return -EINVAL; + + return current->thread.vstate_ctrl & PR_RISCV_V_VSTATE_CTRL_MASK; +} + +long riscv_v_vstate_ctrl_set_current(unsigned long arg) +{ + bool inherit; + int cur, next; + + if (!has_vector()) + return -EINVAL; + + if (arg & ~PR_RISCV_V_VSTATE_CTRL_MASK) + return -EINVAL; + + cur = VSTATE_CTRL_GET_CUR(arg); + switch (cur) { + case PR_RISCV_V_VSTATE_CTRL_OFF: + /* Do not allow user to turn off V if current is not off */ + if (riscv_v_ctrl_get_cur(current) != PR_RISCV_V_VSTATE_CTRL_OFF) + return -EPERM; + + break; + case PR_RISCV_V_VSTATE_CTRL_ON: + break; + case PR_RISCV_V_VSTATE_CTRL_DEFAULT: + cur = riscv_v_ctrl_get_cur(current); + break; + default: + return -EINVAL; + } + + next = VSTATE_CTRL_GET_NEXT(arg); + inherit = VSTATE_CTRL_GET_INHERIT(arg); + switch (next) { + case PR_RISCV_V_VSTATE_CTRL_DEFAULT: + case PR_RISCV_V_VSTATE_CTRL_OFF: + case PR_RISCV_V_VSTATE_CTRL_ON: + riscv_v_ctrl_set(current, cur, next, inherit); + return 0; + } + + return -EINVAL; +} + +#ifdef CONFIG_SYSCTL + +static struct ctl_table riscv_v_default_vstate_table[] = { + { + .procname = "riscv_v_default_allow", + .data = &riscv_v_implicit_uacc, + .maxlen = sizeof(riscv_v_implicit_uacc), + .mode = 0644, + .proc_handler = proc_dobool, + }, + { } +}; + +static int __init riscv_v_sysctl_init(void) +{ + if (has_vector()) + if (!register_sysctl("abi", riscv_v_default_vstate_table)) + return -EINVAL; + return 0; +} + +#else /* ! CONFIG_SYSCTL */ +static int __init riscv_v_sysctl_init(void) { return 0; } +#endif /* ! CONFIG_SYSCTL */ + +static int riscv_v_init(void) +{ + return riscv_v_sysctl_init(); +} +core_initcall(riscv_v_init); diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig index 28891e583259..dfc237d7875b 100644 --- a/arch/riscv/kvm/Kconfig +++ b/arch/riscv/kvm/Kconfig @@ -21,6 +21,10 @@ config KVM tristate "Kernel-based Virtual Machine (KVM) support (EXPERIMENTAL)" depends on RISCV_SBI && MMU select HAVE_KVM_EVENTFD + select HAVE_KVM_IRQCHIP + select HAVE_KVM_IRQFD + select HAVE_KVM_IRQ_ROUTING + select HAVE_KVM_MSI select HAVE_KVM_VCPU_ASYNC_IOCTL select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_GENERIC_HARDWARE_ENABLING diff --git a/arch/riscv/kvm/Makefile b/arch/riscv/kvm/Makefile index 8031b8912a0d..fee0671e2dc1 100644 --- a/arch/riscv/kvm/Makefile +++ b/arch/riscv/kvm/Makefile @@ -17,6 +17,7 @@ kvm-y += mmu.o kvm-y += vcpu.o kvm-y += vcpu_exit.o kvm-y += vcpu_fp.o +kvm-y += vcpu_vector.o kvm-y += vcpu_insn.o kvm-y += vcpu_switch.o kvm-y += vcpu_sbi.o @@ -27,3 +28,6 @@ kvm-y += vcpu_sbi_hsm.o kvm-y += vcpu_timer.o kvm-$(CONFIG_RISCV_PMU_SBI) += vcpu_pmu.o vcpu_sbi_pmu.o kvm-y += aia.o +kvm-y += aia_device.o +kvm-y += aia_aplic.o +kvm-y += aia_imsic.o diff --git a/arch/riscv/kvm/aia.c b/arch/riscv/kvm/aia.c index 4f1286fc7f17..585a3b42c52c 100644 --- a/arch/riscv/kvm/aia.c +++ b/arch/riscv/kvm/aia.c @@ -8,11 +8,49 @@ */ #include <linux/kernel.h> +#include <linux/bitops.h> +#include <linux/irq.h> +#include <linux/irqdomain.h> #include <linux/kvm_host.h> +#include <linux/percpu.h> +#include <linux/spinlock.h> #include <asm/hwcap.h> +#include <asm/kvm_aia_imsic.h> +struct aia_hgei_control { + raw_spinlock_t lock; + unsigned long free_bitmap; + struct kvm_vcpu *owners[BITS_PER_LONG]; +}; +static DEFINE_PER_CPU(struct aia_hgei_control, aia_hgei); +static int hgei_parent_irq; + +unsigned int kvm_riscv_aia_nr_hgei; +unsigned int kvm_riscv_aia_max_ids; DEFINE_STATIC_KEY_FALSE(kvm_riscv_aia_available); +static int aia_find_hgei(struct kvm_vcpu *owner) +{ + int i, hgei; + unsigned long flags; + struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei); + + raw_spin_lock_irqsave(&hgctrl->lock, flags); + + hgei = -1; + for (i = 1; i <= kvm_riscv_aia_nr_hgei; i++) { + if (hgctrl->owners[i] == owner) { + hgei = i; + break; + } + } + + raw_spin_unlock_irqrestore(&hgctrl->lock, flags); + + put_cpu_ptr(&aia_hgei); + return hgei; +} + static void aia_set_hvictl(bool ext_irq_pending) { unsigned long hvictl; @@ -56,6 +94,7 @@ void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu) bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) { + int hgei; unsigned long seip; if (!kvm_riscv_aia_available()) @@ -74,6 +113,10 @@ bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) if (!kvm_riscv_aia_initialized(vcpu->kvm) || !seip) return false; + hgei = aia_find_hgei(vcpu); + if (hgei > 0) + return !!(csr_read(CSR_HGEIP) & BIT(hgei)); + return false; } @@ -323,8 +366,6 @@ static int aia_rmw_iprio(struct kvm_vcpu *vcpu, unsigned int isel, return KVM_INSN_CONTINUE_NEXT_SEPC; } -#define IMSIC_FIRST 0x70 -#define IMSIC_LAST 0xff int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num, unsigned long *val, unsigned long new_val, unsigned long wr_mask) @@ -348,6 +389,143 @@ int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num, return KVM_INSN_EXIT_TO_USER_SPACE; } +int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner, + void __iomem **hgei_va, phys_addr_t *hgei_pa) +{ + int ret = -ENOENT; + unsigned long flags; + struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu); + + if (!kvm_riscv_aia_available() || !hgctrl) + return -ENODEV; + + raw_spin_lock_irqsave(&hgctrl->lock, flags); + + if (hgctrl->free_bitmap) { + ret = __ffs(hgctrl->free_bitmap); + hgctrl->free_bitmap &= ~BIT(ret); + hgctrl->owners[ret] = owner; + } + + raw_spin_unlock_irqrestore(&hgctrl->lock, flags); + + /* TODO: To be updated later by AIA IMSIC HW guest file support */ + if (hgei_va) + *hgei_va = NULL; + if (hgei_pa) + *hgei_pa = 0; + + return ret; +} + +void kvm_riscv_aia_free_hgei(int cpu, int hgei) +{ + unsigned long flags; + struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu); + + if (!kvm_riscv_aia_available() || !hgctrl) + return; + + raw_spin_lock_irqsave(&hgctrl->lock, flags); + + if (hgei > 0 && hgei <= kvm_riscv_aia_nr_hgei) { + if (!(hgctrl->free_bitmap & BIT(hgei))) { + hgctrl->free_bitmap |= BIT(hgei); + hgctrl->owners[hgei] = NULL; + } + } + + raw_spin_unlock_irqrestore(&hgctrl->lock, flags); +} + +void kvm_riscv_aia_wakeon_hgei(struct kvm_vcpu *owner, bool enable) +{ + int hgei; + + if (!kvm_riscv_aia_available()) + return; + + hgei = aia_find_hgei(owner); + if (hgei > 0) { + if (enable) + csr_set(CSR_HGEIE, BIT(hgei)); + else + csr_clear(CSR_HGEIE, BIT(hgei)); + } +} + +static irqreturn_t hgei_interrupt(int irq, void *dev_id) +{ + int i; + unsigned long hgei_mask, flags; + struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei); + + hgei_mask = csr_read(CSR_HGEIP) & csr_read(CSR_HGEIE); + csr_clear(CSR_HGEIE, hgei_mask); + + raw_spin_lock_irqsave(&hgctrl->lock, flags); + + for_each_set_bit(i, &hgei_mask, BITS_PER_LONG) { + if (hgctrl->owners[i]) + kvm_vcpu_kick(hgctrl->owners[i]); + } + + raw_spin_unlock_irqrestore(&hgctrl->lock, flags); + + put_cpu_ptr(&aia_hgei); + return IRQ_HANDLED; +} + +static int aia_hgei_init(void) +{ + int cpu, rc; + struct irq_domain *domain; + struct aia_hgei_control *hgctrl; + + /* Initialize per-CPU guest external interrupt line management */ + for_each_possible_cpu(cpu) { + hgctrl = per_cpu_ptr(&aia_hgei, cpu); + raw_spin_lock_init(&hgctrl->lock); + if (kvm_riscv_aia_nr_hgei) { + hgctrl->free_bitmap = + BIT(kvm_riscv_aia_nr_hgei + 1) - 1; + hgctrl->free_bitmap &= ~BIT(0); + } else + hgctrl->free_bitmap = 0; + } + + /* Find INTC irq domain */ + domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), + DOMAIN_BUS_ANY); + if (!domain) { + kvm_err("unable to find INTC domain\n"); + return -ENOENT; + } + + /* Map per-CPU SGEI interrupt from INTC domain */ + hgei_parent_irq = irq_create_mapping(domain, IRQ_S_GEXT); + if (!hgei_parent_irq) { + kvm_err("unable to map SGEI IRQ\n"); + return -ENOMEM; + } + + /* Request per-CPU SGEI interrupt */ + rc = request_percpu_irq(hgei_parent_irq, hgei_interrupt, + "riscv-kvm", &aia_hgei); + if (rc) { + kvm_err("failed to request SGEI IRQ\n"); + return rc; + } + + return 0; +} + +static void aia_hgei_exit(void) +{ + /* Free per-CPU SGEI interrupt */ + free_percpu_irq(hgei_parent_irq, &aia_hgei); +} + void kvm_riscv_aia_enable(void) { if (!kvm_riscv_aia_available()) @@ -362,21 +540,105 @@ void kvm_riscv_aia_enable(void) csr_write(CSR_HVIPRIO1H, 0x0); csr_write(CSR_HVIPRIO2H, 0x0); #endif + + /* Enable per-CPU SGEI interrupt */ + enable_percpu_irq(hgei_parent_irq, + irq_get_trigger_type(hgei_parent_irq)); + csr_set(CSR_HIE, BIT(IRQ_S_GEXT)); } void kvm_riscv_aia_disable(void) { + int i; + unsigned long flags; + struct kvm_vcpu *vcpu; + struct aia_hgei_control *hgctrl; + if (!kvm_riscv_aia_available()) return; + hgctrl = get_cpu_ptr(&aia_hgei); + + /* Disable per-CPU SGEI interrupt */ + csr_clear(CSR_HIE, BIT(IRQ_S_GEXT)); + disable_percpu_irq(hgei_parent_irq); aia_set_hvictl(false); + + raw_spin_lock_irqsave(&hgctrl->lock, flags); + + for (i = 0; i <= kvm_riscv_aia_nr_hgei; i++) { + vcpu = hgctrl->owners[i]; + if (!vcpu) + continue; + + /* + * We release hgctrl->lock before notifying IMSIC + * so that we don't have lock ordering issues. + */ + raw_spin_unlock_irqrestore(&hgctrl->lock, flags); + + /* Notify IMSIC */ + kvm_riscv_vcpu_aia_imsic_release(vcpu); + + /* + * Wakeup VCPU if it was blocked so that it can + * run on other HARTs + */ + if (csr_read(CSR_HGEIE) & BIT(i)) { + csr_clear(CSR_HGEIE, BIT(i)); + kvm_vcpu_kick(vcpu); + } + + raw_spin_lock_irqsave(&hgctrl->lock, flags); + } + + raw_spin_unlock_irqrestore(&hgctrl->lock, flags); + + put_cpu_ptr(&aia_hgei); } int kvm_riscv_aia_init(void) { + int rc; + if (!riscv_isa_extension_available(NULL, SxAIA)) return -ENODEV; + /* Figure-out number of bits in HGEIE */ + csr_write(CSR_HGEIE, -1UL); + kvm_riscv_aia_nr_hgei = fls_long(csr_read(CSR_HGEIE)); + csr_write(CSR_HGEIE, 0); + if (kvm_riscv_aia_nr_hgei) + kvm_riscv_aia_nr_hgei--; + + /* + * Number of usable HGEI lines should be minimum of per-HART + * IMSIC guest files and number of bits in HGEIE + * + * TODO: To be updated later by AIA IMSIC HW guest file support + */ + kvm_riscv_aia_nr_hgei = 0; + + /* + * Find number of guest MSI IDs + * + * TODO: To be updated later by AIA IMSIC HW guest file support + */ + kvm_riscv_aia_max_ids = IMSIC_MAX_ID; + + /* Initialize guest external interrupt line management */ + rc = aia_hgei_init(); + if (rc) + return rc; + + /* Register device operations */ + rc = kvm_register_device_ops(&kvm_riscv_aia_device_ops, + KVM_DEV_TYPE_RISCV_AIA); + if (rc) { + aia_hgei_exit(); + return rc; + } + /* Enable KVM AIA support */ static_branch_enable(&kvm_riscv_aia_available); @@ -385,4 +647,12 @@ int kvm_riscv_aia_init(void) void kvm_riscv_aia_exit(void) { + if (!kvm_riscv_aia_available()) + return; + + /* Unregister device operations */ + kvm_unregister_device_ops(KVM_DEV_TYPE_RISCV_AIA); + + /* Cleanup the HGEI state */ + aia_hgei_exit(); } diff --git a/arch/riscv/kvm/aia_aplic.c b/arch/riscv/kvm/aia_aplic.c new file mode 100644 index 000000000000..39e72aa016a4 --- /dev/null +++ b/arch/riscv/kvm/aia_aplic.c @@ -0,0 +1,619 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + * + * Authors: + * Anup Patel <apatel@ventanamicro.com> + */ + +#include <linux/kvm_host.h> +#include <linux/math.h> +#include <linux/spinlock.h> +#include <linux/swab.h> +#include <kvm/iodev.h> +#include <asm/kvm_aia_aplic.h> + +struct aplic_irq { + raw_spinlock_t lock; + u32 sourcecfg; + u32 state; +#define APLIC_IRQ_STATE_PENDING BIT(0) +#define APLIC_IRQ_STATE_ENABLED BIT(1) +#define APLIC_IRQ_STATE_ENPEND (APLIC_IRQ_STATE_PENDING | \ + APLIC_IRQ_STATE_ENABLED) +#define APLIC_IRQ_STATE_INPUT BIT(8) + u32 target; +}; + +struct aplic { + struct kvm_io_device iodev; + + u32 domaincfg; + u32 genmsi; + + u32 nr_irqs; + u32 nr_words; + struct aplic_irq *irqs; +}; + +static u32 aplic_read_sourcecfg(struct aplic *aplic, u32 irq) +{ + u32 ret; + unsigned long flags; + struct aplic_irq *irqd; + + if (!irq || aplic->nr_irqs <= irq) + return 0; + irqd = &aplic->irqs[irq]; + + raw_spin_lock_irqsave(&irqd->lock, flags); + ret = irqd->sourcecfg; + raw_spin_unlock_irqrestore(&irqd->lock, flags); + + return ret; +} + +static void aplic_write_sourcecfg(struct aplic *aplic, u32 irq, u32 val) +{ + unsigned long flags; + struct aplic_irq *irqd; + + if (!irq || aplic->nr_irqs <= irq) + return; + irqd = &aplic->irqs[irq]; + + if (val & APLIC_SOURCECFG_D) + val = 0; + else + val &= APLIC_SOURCECFG_SM_MASK; + + raw_spin_lock_irqsave(&irqd->lock, flags); + irqd->sourcecfg = val; + raw_spin_unlock_irqrestore(&irqd->lock, flags); +} + +static u32 aplic_read_target(struct aplic *aplic, u32 irq) +{ + u32 ret; + unsigned long flags; + struct aplic_irq *irqd; + + if (!irq || aplic->nr_irqs <= irq) + return 0; + irqd = &aplic->irqs[irq]; + + raw_spin_lock_irqsave(&irqd->lock, flags); + ret = irqd->target; + raw_spin_unlock_irqrestore(&irqd->lock, flags); + + return ret; +} + +static void aplic_write_target(struct aplic *aplic, u32 irq, u32 val) +{ + unsigned long flags; + struct aplic_irq *irqd; + + if (!irq || aplic->nr_irqs <= irq) + return; + irqd = &aplic->irqs[irq]; + + val &= APLIC_TARGET_EIID_MASK | + (APLIC_TARGET_HART_IDX_MASK << APLIC_TARGET_HART_IDX_SHIFT) | + (APLIC_TARGET_GUEST_IDX_MASK << APLIC_TARGET_GUEST_IDX_SHIFT); + + raw_spin_lock_irqsave(&irqd->lock, flags); + irqd->target = val; + raw_spin_unlock_irqrestore(&irqd->lock, flags); +} + +static bool aplic_read_pending(struct aplic *aplic, u32 irq) +{ + bool ret; + unsigned long flags; + struct aplic_irq *irqd; + + if (!irq || aplic->nr_irqs <= irq) + return false; + irqd = &aplic->irqs[irq]; + + raw_spin_lock_irqsave(&irqd->lock, flags); + ret = (irqd->state & APLIC_IRQ_STATE_PENDING) ? true : false; + raw_spin_unlock_irqrestore(&irqd->lock, flags); + + return ret; +} + +static void aplic_write_pending(struct aplic *aplic, u32 irq, bool pending) +{ + unsigned long flags, sm; + struct aplic_irq *irqd; + + if (!irq || aplic->nr_irqs <= irq) + return; + irqd = &aplic->irqs[irq]; + + raw_spin_lock_irqsave(&irqd->lock, flags); + + sm = irqd->sourcecfg & APLIC_SOURCECFG_SM_MASK; + if (!pending && + ((sm == APLIC_SOURCECFG_SM_LEVEL_HIGH) || + (sm == APLIC_SOURCECFG_SM_LEVEL_LOW))) + goto skip_write_pending; + + if (pending) + irqd->state |= APLIC_IRQ_STATE_PENDING; + else + irqd->state &= ~APLIC_IRQ_STATE_PENDING; + +skip_write_pending: + raw_spin_unlock_irqrestore(&irqd->lock, flags); +} + +static bool aplic_read_enabled(struct aplic *aplic, u32 irq) +{ + bool ret; + unsigned long flags; + struct aplic_irq *irqd; + + if (!irq || aplic->nr_irqs <= irq) + return false; + irqd = &aplic->irqs[irq]; + + raw_spin_lock_irqsave(&irqd->lock, flags); + ret = (irqd->state & APLIC_IRQ_STATE_ENABLED) ? true : false; + raw_spin_unlock_irqrestore(&irqd->lock, flags); + + return ret; +} + +static void aplic_write_enabled(struct aplic *aplic, u32 irq, bool enabled) +{ + unsigned long flags; + struct aplic_irq *irqd; + + if (!irq || aplic->nr_irqs <= irq) + return; + irqd = &aplic->irqs[irq]; + + raw_spin_lock_irqsave(&irqd->lock, flags); + if (enabled) + irqd->state |= APLIC_IRQ_STATE_ENABLED; + else + irqd->state &= ~APLIC_IRQ_STATE_ENABLED; + raw_spin_unlock_irqrestore(&irqd->lock, flags); +} + +static bool aplic_read_input(struct aplic *aplic, u32 irq) +{ + bool ret; + unsigned long flags; + struct aplic_irq *irqd; + + if (!irq || aplic->nr_irqs <= irq) + return false; + irqd = &aplic->irqs[irq]; + + raw_spin_lock_irqsave(&irqd->lock, flags); + ret = (irqd->state & APLIC_IRQ_STATE_INPUT) ? true : false; + raw_spin_unlock_irqrestore(&irqd->lock, flags); + + return ret; +} + +static void aplic_inject_msi(struct kvm *kvm, u32 irq, u32 target) +{ + u32 hart_idx, guest_idx, eiid; + + hart_idx = target >> APLIC_TARGET_HART_IDX_SHIFT; + hart_idx &= APLIC_TARGET_HART_IDX_MASK; + guest_idx = target >> APLIC_TARGET_GUEST_IDX_SHIFT; + guest_idx &= APLIC_TARGET_GUEST_IDX_MASK; + eiid = target & APLIC_TARGET_EIID_MASK; + kvm_riscv_aia_inject_msi_by_id(kvm, hart_idx, guest_idx, eiid); +} + +static void aplic_update_irq_range(struct kvm *kvm, u32 first, u32 last) +{ + bool inject; + u32 irq, target; + unsigned long flags; + struct aplic_irq *irqd; + struct aplic *aplic = kvm->arch.aia.aplic_state; + + if (!(aplic->domaincfg & APLIC_DOMAINCFG_IE)) + return; + + for (irq = first; irq <= last; irq++) { + if (!irq || aplic->nr_irqs <= irq) + continue; + irqd = &aplic->irqs[irq]; + + raw_spin_lock_irqsave(&irqd->lock, flags); + + inject = false; + target = irqd->target; + if ((irqd->state & APLIC_IRQ_STATE_ENPEND) == + APLIC_IRQ_STATE_ENPEND) { + irqd->state &= ~APLIC_IRQ_STATE_PENDING; + inject = true; + } + + raw_spin_unlock_irqrestore(&irqd->lock, flags); + + if (inject) + aplic_inject_msi(kvm, irq, target); + } +} + +int kvm_riscv_aia_aplic_inject(struct kvm *kvm, u32 source, bool level) +{ + u32 target; + bool inject = false, ie; + unsigned long flags; + struct aplic_irq *irqd; + struct aplic *aplic = kvm->arch.aia.aplic_state; + + if (!aplic || !source || (aplic->nr_irqs <= source)) + return -ENODEV; + irqd = &aplic->irqs[source]; + ie = (aplic->domaincfg & APLIC_DOMAINCFG_IE) ? true : false; + + raw_spin_lock_irqsave(&irqd->lock, flags); + + if (irqd->sourcecfg & APLIC_SOURCECFG_D) + goto skip_unlock; + + switch (irqd->sourcecfg & APLIC_SOURCECFG_SM_MASK) { + case APLIC_SOURCECFG_SM_EDGE_RISE: + if (level && !(irqd->state & APLIC_IRQ_STATE_INPUT) && + !(irqd->state & APLIC_IRQ_STATE_PENDING)) + irqd->state |= APLIC_IRQ_STATE_PENDING; + break; + case APLIC_SOURCECFG_SM_EDGE_FALL: + if (!level && (irqd->state & APLIC_IRQ_STATE_INPUT) && + !(irqd->state & APLIC_IRQ_STATE_PENDING)) + irqd->state |= APLIC_IRQ_STATE_PENDING; + break; + case APLIC_SOURCECFG_SM_LEVEL_HIGH: + if (level && !(irqd->state & APLIC_IRQ_STATE_PENDING)) + irqd->state |= APLIC_IRQ_STATE_PENDING; + break; + case APLIC_SOURCECFG_SM_LEVEL_LOW: + if (!level && !(irqd->state & APLIC_IRQ_STATE_PENDING)) + irqd->state |= APLIC_IRQ_STATE_PENDING; + break; + } + + if (level) + irqd->state |= APLIC_IRQ_STATE_INPUT; + else + irqd->state &= ~APLIC_IRQ_STATE_INPUT; + + target = irqd->target; + if (ie && ((irqd->state & APLIC_IRQ_STATE_ENPEND) == + APLIC_IRQ_STATE_ENPEND)) { + irqd->state &= ~APLIC_IRQ_STATE_PENDING; + inject = true; + } + +skip_unlock: + raw_spin_unlock_irqrestore(&irqd->lock, flags); + + if (inject) + aplic_inject_msi(kvm, source, target); + + return 0; +} + +static u32 aplic_read_input_word(struct aplic *aplic, u32 word) +{ + u32 i, ret = 0; + + for (i = 0; i < 32; i++) + ret |= aplic_read_input(aplic, word * 32 + i) ? BIT(i) : 0; + + return ret; +} + +static u32 aplic_read_pending_word(struct aplic *aplic, u32 word) +{ + u32 i, ret = 0; + + for (i = 0; i < 32; i++) + ret |= aplic_read_pending(aplic, word * 32 + i) ? BIT(i) : 0; + + return ret; +} + +static void aplic_write_pending_word(struct aplic *aplic, u32 word, + u32 val, bool pending) +{ + u32 i; + + for (i = 0; i < 32; i++) { + if (val & BIT(i)) + aplic_write_pending(aplic, word * 32 + i, pending); + } +} + +static u32 aplic_read_enabled_word(struct aplic *aplic, u32 word) +{ + u32 i, ret = 0; + + for (i = 0; i < 32; i++) + ret |= aplic_read_enabled(aplic, word * 32 + i) ? BIT(i) : 0; + + return ret; +} + +static void aplic_write_enabled_word(struct aplic *aplic, u32 word, + u32 val, bool enabled) +{ + u32 i; + + for (i = 0; i < 32; i++) { + if (val & BIT(i)) + aplic_write_enabled(aplic, word * 32 + i, enabled); + } +} + +static int aplic_mmio_read_offset(struct kvm *kvm, gpa_t off, u32 *val32) +{ + u32 i; + struct aplic *aplic = kvm->arch.aia.aplic_state; + + if ((off & 0x3) != 0) + return -EOPNOTSUPP; + + if (off == APLIC_DOMAINCFG) { + *val32 = APLIC_DOMAINCFG_RDONLY | + aplic->domaincfg | APLIC_DOMAINCFG_DM; + } else if ((off >= APLIC_SOURCECFG_BASE) && + (off < (APLIC_SOURCECFG_BASE + (aplic->nr_irqs - 1) * 4))) { + i = ((off - APLIC_SOURCECFG_BASE) >> 2) + 1; + *val32 = aplic_read_sourcecfg(aplic, i); + } else if ((off >= APLIC_SETIP_BASE) && + (off < (APLIC_SETIP_BASE + aplic->nr_words * 4))) { + i = (off - APLIC_SETIP_BASE) >> 2; + *val32 = aplic_read_pending_word(aplic, i); + } else if (off == APLIC_SETIPNUM) { + *val32 = 0; + } else if ((off >= APLIC_CLRIP_BASE) && + (off < (APLIC_CLRIP_BASE + aplic->nr_words * 4))) { + i = (off - APLIC_CLRIP_BASE) >> 2; + *val32 = aplic_read_input_word(aplic, i); + } else if (off == APLIC_CLRIPNUM) { + *val32 = 0; + } else if ((off >= APLIC_SETIE_BASE) && + (off < (APLIC_SETIE_BASE + aplic->nr_words * 4))) { + i = (off - APLIC_SETIE_BASE) >> 2; + *val32 = aplic_read_enabled_word(aplic, i); + } else if (off == APLIC_SETIENUM) { + *val32 = 0; + } else if ((off >= APLIC_CLRIE_BASE) && + (off < (APLIC_CLRIE_BASE + aplic->nr_words * 4))) { + *val32 = 0; + } else if (off == APLIC_CLRIENUM) { + *val32 = 0; + } else if (off == APLIC_SETIPNUM_LE) { + *val32 = 0; + } else if (off == APLIC_SETIPNUM_BE) { + *val32 = 0; + } else if (off == APLIC_GENMSI) { + *val32 = aplic->genmsi; + } else if ((off >= APLIC_TARGET_BASE) && + (off < (APLIC_TARGET_BASE + (aplic->nr_irqs - 1) * 4))) { + i = ((off - APLIC_TARGET_BASE) >> 2) + 1; + *val32 = aplic_read_target(aplic, i); + } else + return -ENODEV; + + return 0; +} + +static int aplic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, + gpa_t addr, int len, void *val) +{ + if (len != 4) + return -EOPNOTSUPP; + + return aplic_mmio_read_offset(vcpu->kvm, + addr - vcpu->kvm->arch.aia.aplic_addr, + val); +} + +static int aplic_mmio_write_offset(struct kvm *kvm, gpa_t off, u32 val32) +{ + u32 i; + struct aplic *aplic = kvm->arch.aia.aplic_state; + + if ((off & 0x3) != 0) + return -EOPNOTSUPP; + + if (off == APLIC_DOMAINCFG) { + /* Only IE bit writeable */ + aplic->domaincfg = val32 & APLIC_DOMAINCFG_IE; + } else if ((off >= APLIC_SOURCECFG_BASE) && + (off < (APLIC_SOURCECFG_BASE + (aplic->nr_irqs - 1) * 4))) { + i = ((off - APLIC_SOURCECFG_BASE) >> 2) + 1; + aplic_write_sourcecfg(aplic, i, val32); + } else if ((off >= APLIC_SETIP_BASE) && + (off < (APLIC_SETIP_BASE + aplic->nr_words * 4))) { + i = (off - APLIC_SETIP_BASE) >> 2; + aplic_write_pending_word(aplic, i, val32, true); + } else if (off == APLIC_SETIPNUM) { + aplic_write_pending(aplic, val32, true); + } else if ((off >= APLIC_CLRIP_BASE) && + (off < (APLIC_CLRIP_BASE + aplic->nr_words * 4))) { + i = (off - APLIC_CLRIP_BASE) >> 2; + aplic_write_pending_word(aplic, i, val32, false); + } else if (off == APLIC_CLRIPNUM) { + aplic_write_pending(aplic, val32, false); + } else if ((off >= APLIC_SETIE_BASE) && + (off < (APLIC_SETIE_BASE + aplic->nr_words * 4))) { + i = (off - APLIC_SETIE_BASE) >> 2; + aplic_write_enabled_word(aplic, i, val32, true); + } else if (off == APLIC_SETIENUM) { + aplic_write_enabled(aplic, val32, true); + } else if ((off >= APLIC_CLRIE_BASE) && + (off < (APLIC_CLRIE_BASE + aplic->nr_words * 4))) { + i = (off - APLIC_CLRIE_BASE) >> 2; + aplic_write_enabled_word(aplic, i, val32, false); + } else if (off == APLIC_CLRIENUM) { + aplic_write_enabled(aplic, val32, false); + } else if (off == APLIC_SETIPNUM_LE) { + aplic_write_pending(aplic, val32, true); + } else if (off == APLIC_SETIPNUM_BE) { + aplic_write_pending(aplic, __swab32(val32), true); + } else if (off == APLIC_GENMSI) { + aplic->genmsi = val32 & ~(APLIC_TARGET_GUEST_IDX_MASK << + APLIC_TARGET_GUEST_IDX_SHIFT); + kvm_riscv_aia_inject_msi_by_id(kvm, + val32 >> APLIC_TARGET_HART_IDX_SHIFT, 0, + val32 & APLIC_TARGET_EIID_MASK); + } else if ((off >= APLIC_TARGET_BASE) && + (off < (APLIC_TARGET_BASE + (aplic->nr_irqs - 1) * 4))) { + i = ((off - APLIC_TARGET_BASE) >> 2) + 1; + aplic_write_target(aplic, i, val32); + } else + return -ENODEV; + + aplic_update_irq_range(kvm, 1, aplic->nr_irqs - 1); + + return 0; +} + +static int aplic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, + gpa_t addr, int len, const void *val) +{ + if (len != 4) + return -EOPNOTSUPP; + + return aplic_mmio_write_offset(vcpu->kvm, + addr - vcpu->kvm->arch.aia.aplic_addr, + *((const u32 *)val)); +} + +static struct kvm_io_device_ops aplic_iodoev_ops = { + .read = aplic_mmio_read, + .write = aplic_mmio_write, +}; + +int kvm_riscv_aia_aplic_set_attr(struct kvm *kvm, unsigned long type, u32 v) +{ + int rc; + + if (!kvm->arch.aia.aplic_state) + return -ENODEV; + + rc = aplic_mmio_write_offset(kvm, type, v); + if (rc) + return rc; + + return 0; +} + +int kvm_riscv_aia_aplic_get_attr(struct kvm *kvm, unsigned long type, u32 *v) +{ + int rc; + + if (!kvm->arch.aia.aplic_state) + return -ENODEV; + + rc = aplic_mmio_read_offset(kvm, type, v); + if (rc) + return rc; + + return 0; +} + +int kvm_riscv_aia_aplic_has_attr(struct kvm *kvm, unsigned long type) +{ + int rc; + u32 val; + + if (!kvm->arch.aia.aplic_state) + return -ENODEV; + + rc = aplic_mmio_read_offset(kvm, type, &val); + if (rc) + return rc; + + return 0; +} + +int kvm_riscv_aia_aplic_init(struct kvm *kvm) +{ + int i, ret = 0; + struct aplic *aplic; + + /* Do nothing if we have zero sources */ + if (!kvm->arch.aia.nr_sources) + return 0; + + /* Allocate APLIC global state */ + aplic = kzalloc(sizeof(*aplic), GFP_KERNEL); + if (!aplic) + return -ENOMEM; + kvm->arch.aia.aplic_state = aplic; + + /* Setup APLIC IRQs */ + aplic->nr_irqs = kvm->arch.aia.nr_sources + 1; + aplic->nr_words = DIV_ROUND_UP(aplic->nr_irqs, 32); + aplic->irqs = kcalloc(aplic->nr_irqs, + sizeof(*aplic->irqs), GFP_KERNEL); + if (!aplic->irqs) { + ret = -ENOMEM; + goto fail_free_aplic; + } + for (i = 0; i < aplic->nr_irqs; i++) + raw_spin_lock_init(&aplic->irqs[i].lock); + + /* Setup IO device */ + kvm_iodevice_init(&aplic->iodev, &aplic_iodoev_ops); + mutex_lock(&kvm->slots_lock); + ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, + kvm->arch.aia.aplic_addr, + KVM_DEV_RISCV_APLIC_SIZE, + &aplic->iodev); + mutex_unlock(&kvm->slots_lock); + if (ret) + goto fail_free_aplic_irqs; + + /* Setup default IRQ routing */ + ret = kvm_riscv_setup_default_irq_routing(kvm, aplic->nr_irqs); + if (ret) + goto fail_unreg_iodev; + + return 0; + +fail_unreg_iodev: + mutex_lock(&kvm->slots_lock); + kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &aplic->iodev); + mutex_unlock(&kvm->slots_lock); +fail_free_aplic_irqs: + kfree(aplic->irqs); +fail_free_aplic: + kvm->arch.aia.aplic_state = NULL; + kfree(aplic); + return ret; +} + +void kvm_riscv_aia_aplic_cleanup(struct kvm *kvm) +{ + struct aplic *aplic = kvm->arch.aia.aplic_state; + + if (!aplic) + return; + + mutex_lock(&kvm->slots_lock); + kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &aplic->iodev); + mutex_unlock(&kvm->slots_lock); + + kfree(aplic->irqs); + + kvm->arch.aia.aplic_state = NULL; + kfree(aplic); +} diff --git a/arch/riscv/kvm/aia_device.c b/arch/riscv/kvm/aia_device.c new file mode 100644 index 000000000000..0eb689351b7d --- /dev/null +++ b/arch/riscv/kvm/aia_device.c @@ -0,0 +1,673 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + * + * Authors: + * Anup Patel <apatel@ventanamicro.com> + */ + +#include <linux/bits.h> +#include <linux/kvm_host.h> +#include <linux/uaccess.h> +#include <asm/kvm_aia_imsic.h> + +static void unlock_vcpus(struct kvm *kvm, int vcpu_lock_idx) +{ + struct kvm_vcpu *tmp_vcpu; + + for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) { + tmp_vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx); + mutex_unlock(&tmp_vcpu->mutex); + } +} + +static void unlock_all_vcpus(struct kvm *kvm) +{ + unlock_vcpus(kvm, atomic_read(&kvm->online_vcpus) - 1); +} + +static bool lock_all_vcpus(struct kvm *kvm) +{ + struct kvm_vcpu *tmp_vcpu; + unsigned long c; + + kvm_for_each_vcpu(c, tmp_vcpu, kvm) { + if (!mutex_trylock(&tmp_vcpu->mutex)) { + unlock_vcpus(kvm, c - 1); + return false; + } + } + + return true; +} + +static int aia_create(struct kvm_device *dev, u32 type) +{ + int ret; + unsigned long i; + struct kvm *kvm = dev->kvm; + struct kvm_vcpu *vcpu; + + if (irqchip_in_kernel(kvm)) + return -EEXIST; + + ret = -EBUSY; + if (!lock_all_vcpus(kvm)) + return ret; + + kvm_for_each_vcpu(i, vcpu, kvm) { + if (vcpu->arch.ran_atleast_once) + goto out_unlock; + } + ret = 0; + + kvm->arch.aia.in_kernel = true; + +out_unlock: + unlock_all_vcpus(kvm); + return ret; +} + +static void aia_destroy(struct kvm_device *dev) +{ + kfree(dev); +} + +static int aia_config(struct kvm *kvm, unsigned long type, + u32 *nr, bool write) +{ + struct kvm_aia *aia = &kvm->arch.aia; + + /* Writes can only be done before irqchip is initialized */ + if (write && kvm_riscv_aia_initialized(kvm)) + return -EBUSY; + + switch (type) { + case KVM_DEV_RISCV_AIA_CONFIG_MODE: + if (write) { + switch (*nr) { + case KVM_DEV_RISCV_AIA_MODE_EMUL: + break; + case KVM_DEV_RISCV_AIA_MODE_HWACCEL: + case KVM_DEV_RISCV_AIA_MODE_AUTO: + /* + * HW Acceleration and Auto modes only + * supported on host with non-zero guest + * external interrupts (i.e. non-zero + * VS-level IMSIC pages). + */ + if (!kvm_riscv_aia_nr_hgei) + return -EINVAL; + break; + default: + return -EINVAL; + } + aia->mode = *nr; + } else + *nr = aia->mode; + break; + case KVM_DEV_RISCV_AIA_CONFIG_IDS: + if (write) { + if ((*nr < KVM_DEV_RISCV_AIA_IDS_MIN) || + (*nr >= KVM_DEV_RISCV_AIA_IDS_MAX) || + ((*nr & KVM_DEV_RISCV_AIA_IDS_MIN) != + KVM_DEV_RISCV_AIA_IDS_MIN) || + (kvm_riscv_aia_max_ids <= *nr)) + return -EINVAL; + aia->nr_ids = *nr; + } else + *nr = aia->nr_ids; + break; + case KVM_DEV_RISCV_AIA_CONFIG_SRCS: + if (write) { + if ((*nr >= KVM_DEV_RISCV_AIA_SRCS_MAX) || + (*nr >= kvm_riscv_aia_max_ids)) + return -EINVAL; + aia->nr_sources = *nr; + } else + *nr = aia->nr_sources; + break; + case KVM_DEV_RISCV_AIA_CONFIG_GROUP_BITS: + if (write) { + if (*nr >= KVM_DEV_RISCV_AIA_GROUP_BITS_MAX) + return -EINVAL; + aia->nr_group_bits = *nr; + } else + *nr = aia->nr_group_bits; + break; + case KVM_DEV_RISCV_AIA_CONFIG_GROUP_SHIFT: + if (write) { + if ((*nr < KVM_DEV_RISCV_AIA_GROUP_SHIFT_MIN) || + (*nr >= KVM_DEV_RISCV_AIA_GROUP_SHIFT_MAX)) + return -EINVAL; + aia->nr_group_shift = *nr; + } else + *nr = aia->nr_group_shift; + break; + case KVM_DEV_RISCV_AIA_CONFIG_HART_BITS: + if (write) { + if (*nr >= KVM_DEV_RISCV_AIA_HART_BITS_MAX) + return -EINVAL; + aia->nr_hart_bits = *nr; + } else + *nr = aia->nr_hart_bits; + break; + case KVM_DEV_RISCV_AIA_CONFIG_GUEST_BITS: + if (write) { + if (*nr >= KVM_DEV_RISCV_AIA_GUEST_BITS_MAX) + return -EINVAL; + aia->nr_guest_bits = *nr; + } else + *nr = aia->nr_guest_bits; + break; + default: + return -ENXIO; + } + + return 0; +} + +static int aia_aplic_addr(struct kvm *kvm, u64 *addr, bool write) +{ + struct kvm_aia *aia = &kvm->arch.aia; + + if (write) { + /* Writes can only be done before irqchip is initialized */ + if (kvm_riscv_aia_initialized(kvm)) + return -EBUSY; + + if (*addr & (KVM_DEV_RISCV_APLIC_ALIGN - 1)) + return -EINVAL; + + aia->aplic_addr = *addr; + } else + *addr = aia->aplic_addr; + + return 0; +} + +static int aia_imsic_addr(struct kvm *kvm, u64 *addr, + unsigned long vcpu_idx, bool write) +{ + struct kvm_vcpu *vcpu; + struct kvm_vcpu_aia *vcpu_aia; + + vcpu = kvm_get_vcpu(kvm, vcpu_idx); + if (!vcpu) + return -EINVAL; + vcpu_aia = &vcpu->arch.aia_context; + + if (write) { + /* Writes can only be done before irqchip is initialized */ + if (kvm_riscv_aia_initialized(kvm)) + return -EBUSY; + + if (*addr & (KVM_DEV_RISCV_IMSIC_ALIGN - 1)) + return -EINVAL; + } + + mutex_lock(&vcpu->mutex); + if (write) + vcpu_aia->imsic_addr = *addr; + else + *addr = vcpu_aia->imsic_addr; + mutex_unlock(&vcpu->mutex); + + return 0; +} + +static gpa_t aia_imsic_ppn(struct kvm_aia *aia, gpa_t addr) +{ + u32 h, l; + gpa_t mask = 0; + + h = aia->nr_hart_bits + aia->nr_guest_bits + + IMSIC_MMIO_PAGE_SHIFT - 1; + mask = GENMASK_ULL(h, 0); + + if (aia->nr_group_bits) { + h = aia->nr_group_bits + aia->nr_group_shift - 1; + l = aia->nr_group_shift; + mask |= GENMASK_ULL(h, l); + } + + return (addr & ~mask) >> IMSIC_MMIO_PAGE_SHIFT; +} + +static u32 aia_imsic_hart_index(struct kvm_aia *aia, gpa_t addr) +{ + u32 hart, group = 0; + + hart = (addr >> (aia->nr_guest_bits + IMSIC_MMIO_PAGE_SHIFT)) & + GENMASK_ULL(aia->nr_hart_bits - 1, 0); + if (aia->nr_group_bits) + group = (addr >> aia->nr_group_shift) & + GENMASK_ULL(aia->nr_group_bits - 1, 0); + + return (group << aia->nr_hart_bits) | hart; +} + +static int aia_init(struct kvm *kvm) +{ + int ret, i; + unsigned long idx; + struct kvm_vcpu *vcpu; + struct kvm_vcpu_aia *vaia; + struct kvm_aia *aia = &kvm->arch.aia; + gpa_t base_ppn = KVM_RISCV_AIA_UNDEF_ADDR; + + /* Irqchip can be initialized only once */ + if (kvm_riscv_aia_initialized(kvm)) + return -EBUSY; + + /* We might be in the middle of creating a VCPU? */ + if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus)) + return -EBUSY; + + /* Number of sources should be less than or equals number of IDs */ + if (aia->nr_ids < aia->nr_sources) + return -EINVAL; + + /* APLIC base is required for non-zero number of sources */ + if (aia->nr_sources && aia->aplic_addr == KVM_RISCV_AIA_UNDEF_ADDR) + return -EINVAL; + + /* Initialize APLIC */ + ret = kvm_riscv_aia_aplic_init(kvm); + if (ret) + return ret; + + /* Iterate over each VCPU */ + kvm_for_each_vcpu(idx, vcpu, kvm) { + vaia = &vcpu->arch.aia_context; + + /* IMSIC base is required */ + if (vaia->imsic_addr == KVM_RISCV_AIA_UNDEF_ADDR) { + ret = -EINVAL; + goto fail_cleanup_imsics; + } + + /* All IMSICs should have matching base PPN */ + if (base_ppn == KVM_RISCV_AIA_UNDEF_ADDR) + base_ppn = aia_imsic_ppn(aia, vaia->imsic_addr); + if (base_ppn != aia_imsic_ppn(aia, vaia->imsic_addr)) { + ret = -EINVAL; + goto fail_cleanup_imsics; + } + + /* Update HART index of the IMSIC based on IMSIC base */ + vaia->hart_index = aia_imsic_hart_index(aia, + vaia->imsic_addr); + + /* Initialize IMSIC for this VCPU */ + ret = kvm_riscv_vcpu_aia_imsic_init(vcpu); + if (ret) + goto fail_cleanup_imsics; + } + + /* Set the initialized flag */ + kvm->arch.aia.initialized = true; + + return 0; + +fail_cleanup_imsics: + for (i = idx - 1; i >= 0; i--) { + vcpu = kvm_get_vcpu(kvm, i); + if (!vcpu) + continue; + kvm_riscv_vcpu_aia_imsic_cleanup(vcpu); + } + kvm_riscv_aia_aplic_cleanup(kvm); + return ret; +} + +static int aia_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) +{ + u32 nr; + u64 addr; + int nr_vcpus, r = -ENXIO; + unsigned long v, type = (unsigned long)attr->attr; + void __user *uaddr = (void __user *)(long)attr->addr; + + switch (attr->group) { + case KVM_DEV_RISCV_AIA_GRP_CONFIG: + if (copy_from_user(&nr, uaddr, sizeof(nr))) + return -EFAULT; + + mutex_lock(&dev->kvm->lock); + r = aia_config(dev->kvm, type, &nr, true); + mutex_unlock(&dev->kvm->lock); + + break; + + case KVM_DEV_RISCV_AIA_GRP_ADDR: + if (copy_from_user(&addr, uaddr, sizeof(addr))) + return -EFAULT; + + nr_vcpus = atomic_read(&dev->kvm->online_vcpus); + mutex_lock(&dev->kvm->lock); + if (type == KVM_DEV_RISCV_AIA_ADDR_APLIC) + r = aia_aplic_addr(dev->kvm, &addr, true); + else if (type < KVM_DEV_RISCV_AIA_ADDR_IMSIC(nr_vcpus)) + r = aia_imsic_addr(dev->kvm, &addr, + type - KVM_DEV_RISCV_AIA_ADDR_IMSIC(0), true); + mutex_unlock(&dev->kvm->lock); + + break; + + case KVM_DEV_RISCV_AIA_GRP_CTRL: + switch (type) { + case KVM_DEV_RISCV_AIA_CTRL_INIT: + mutex_lock(&dev->kvm->lock); + r = aia_init(dev->kvm); + mutex_unlock(&dev->kvm->lock); + break; + } + + break; + case KVM_DEV_RISCV_AIA_GRP_APLIC: + if (copy_from_user(&nr, uaddr, sizeof(nr))) + return -EFAULT; + + mutex_lock(&dev->kvm->lock); + r = kvm_riscv_aia_aplic_set_attr(dev->kvm, type, nr); + mutex_unlock(&dev->kvm->lock); + + break; + case KVM_DEV_RISCV_AIA_GRP_IMSIC: + if (copy_from_user(&v, uaddr, sizeof(v))) + return -EFAULT; + + mutex_lock(&dev->kvm->lock); + r = kvm_riscv_aia_imsic_rw_attr(dev->kvm, type, true, &v); + mutex_unlock(&dev->kvm->lock); + + break; + } + + return r; +} + +static int aia_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr) +{ + u32 nr; + u64 addr; + int nr_vcpus, r = -ENXIO; + void __user *uaddr = (void __user *)(long)attr->addr; + unsigned long v, type = (unsigned long)attr->attr; + + switch (attr->group) { + case KVM_DEV_RISCV_AIA_GRP_CONFIG: + if (copy_from_user(&nr, uaddr, sizeof(nr))) + return -EFAULT; + + mutex_lock(&dev->kvm->lock); + r = aia_config(dev->kvm, type, &nr, false); + mutex_unlock(&dev->kvm->lock); + if (r) + return r; + + if (copy_to_user(uaddr, &nr, sizeof(nr))) + return -EFAULT; + + break; + case KVM_DEV_RISCV_AIA_GRP_ADDR: + if (copy_from_user(&addr, uaddr, sizeof(addr))) + return -EFAULT; + + nr_vcpus = atomic_read(&dev->kvm->online_vcpus); + mutex_lock(&dev->kvm->lock); + if (type == KVM_DEV_RISCV_AIA_ADDR_APLIC) + r = aia_aplic_addr(dev->kvm, &addr, false); + else if (type < KVM_DEV_RISCV_AIA_ADDR_IMSIC(nr_vcpus)) + r = aia_imsic_addr(dev->kvm, &addr, + type - KVM_DEV_RISCV_AIA_ADDR_IMSIC(0), false); + mutex_unlock(&dev->kvm->lock); + if (r) + return r; + + if (copy_to_user(uaddr, &addr, sizeof(addr))) + return -EFAULT; + + break; + case KVM_DEV_RISCV_AIA_GRP_APLIC: + if (copy_from_user(&nr, uaddr, sizeof(nr))) + return -EFAULT; + + mutex_lock(&dev->kvm->lock); + r = kvm_riscv_aia_aplic_get_attr(dev->kvm, type, &nr); + mutex_unlock(&dev->kvm->lock); + if (r) + return r; + + if (copy_to_user(uaddr, &nr, sizeof(nr))) + return -EFAULT; + + break; + case KVM_DEV_RISCV_AIA_GRP_IMSIC: + if (copy_from_user(&v, uaddr, sizeof(v))) + return -EFAULT; + + mutex_lock(&dev->kvm->lock); + r = kvm_riscv_aia_imsic_rw_attr(dev->kvm, type, false, &v); + mutex_unlock(&dev->kvm->lock); + if (r) + return r; + + if (copy_to_user(uaddr, &v, sizeof(v))) + return -EFAULT; + + break; + } + + return r; +} + +static int aia_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr) +{ + int nr_vcpus; + + switch (attr->group) { + case KVM_DEV_RISCV_AIA_GRP_CONFIG: + switch (attr->attr) { + case KVM_DEV_RISCV_AIA_CONFIG_MODE: + case KVM_DEV_RISCV_AIA_CONFIG_IDS: + case KVM_DEV_RISCV_AIA_CONFIG_SRCS: + case KVM_DEV_RISCV_AIA_CONFIG_GROUP_BITS: + case KVM_DEV_RISCV_AIA_CONFIG_GROUP_SHIFT: + case KVM_DEV_RISCV_AIA_CONFIG_HART_BITS: + case KVM_DEV_RISCV_AIA_CONFIG_GUEST_BITS: + return 0; + } + break; + case KVM_DEV_RISCV_AIA_GRP_ADDR: + nr_vcpus = atomic_read(&dev->kvm->online_vcpus); + if (attr->attr == KVM_DEV_RISCV_AIA_ADDR_APLIC) + return 0; + else if (attr->attr < KVM_DEV_RISCV_AIA_ADDR_IMSIC(nr_vcpus)) + return 0; + break; + case KVM_DEV_RISCV_AIA_GRP_CTRL: + switch (attr->attr) { + case KVM_DEV_RISCV_AIA_CTRL_INIT: + return 0; + } + break; + case KVM_DEV_RISCV_AIA_GRP_APLIC: + return kvm_riscv_aia_aplic_has_attr(dev->kvm, attr->attr); + case KVM_DEV_RISCV_AIA_GRP_IMSIC: + return kvm_riscv_aia_imsic_has_attr(dev->kvm, attr->attr); + } + + return -ENXIO; +} + +struct kvm_device_ops kvm_riscv_aia_device_ops = { + .name = "kvm-riscv-aia", + .create = aia_create, + .destroy = aia_destroy, + .set_attr = aia_set_attr, + .get_attr = aia_get_attr, + .has_attr = aia_has_attr, +}; + +int kvm_riscv_vcpu_aia_update(struct kvm_vcpu *vcpu) +{ + /* Proceed only if AIA was initialized successfully */ + if (!kvm_riscv_aia_initialized(vcpu->kvm)) + return 1; + + /* Update the IMSIC HW state before entering guest mode */ + return kvm_riscv_vcpu_aia_imsic_update(vcpu); +} + +void kvm_riscv_vcpu_aia_reset(struct kvm_vcpu *vcpu) +{ + struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; + struct kvm_vcpu_aia_csr *reset_csr = + &vcpu->arch.aia_context.guest_reset_csr; + + if (!kvm_riscv_aia_available()) + return; + memcpy(csr, reset_csr, sizeof(*csr)); + + /* Proceed only if AIA was initialized successfully */ + if (!kvm_riscv_aia_initialized(vcpu->kvm)) + return; + + /* Reset the IMSIC context */ + kvm_riscv_vcpu_aia_imsic_reset(vcpu); +} + +int kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu) +{ + struct kvm_vcpu_aia *vaia = &vcpu->arch.aia_context; + + if (!kvm_riscv_aia_available()) + return 0; + + /* + * We don't do any memory allocations over here because these + * will be done after AIA device is initialized by the user-space. + * + * Refer, aia_init() implementation for more details. + */ + + /* Initialize default values in AIA vcpu context */ + vaia->imsic_addr = KVM_RISCV_AIA_UNDEF_ADDR; + vaia->hart_index = vcpu->vcpu_idx; + + return 0; +} + +void kvm_riscv_vcpu_aia_deinit(struct kvm_vcpu *vcpu) +{ + /* Proceed only if AIA was initialized successfully */ + if (!kvm_riscv_aia_initialized(vcpu->kvm)) + return; + + /* Cleanup IMSIC context */ + kvm_riscv_vcpu_aia_imsic_cleanup(vcpu); +} + +int kvm_riscv_aia_inject_msi_by_id(struct kvm *kvm, u32 hart_index, + u32 guest_index, u32 iid) +{ + unsigned long idx; + struct kvm_vcpu *vcpu; + + /* Proceed only if AIA was initialized successfully */ + if (!kvm_riscv_aia_initialized(kvm)) + return -EBUSY; + + /* Inject MSI to matching VCPU */ + kvm_for_each_vcpu(idx, vcpu, kvm) { + if (vcpu->arch.aia_context.hart_index == hart_index) + return kvm_riscv_vcpu_aia_imsic_inject(vcpu, + guest_index, + 0, iid); + } + + return 0; +} + +int kvm_riscv_aia_inject_msi(struct kvm *kvm, struct kvm_msi *msi) +{ + gpa_t tppn, ippn; + unsigned long idx; + struct kvm_vcpu *vcpu; + u32 g, toff, iid = msi->data; + struct kvm_aia *aia = &kvm->arch.aia; + gpa_t target = (((gpa_t)msi->address_hi) << 32) | msi->address_lo; + + /* Proceed only if AIA was initialized successfully */ + if (!kvm_riscv_aia_initialized(kvm)) + return -EBUSY; + + /* Convert target address to target PPN */ + tppn = target >> IMSIC_MMIO_PAGE_SHIFT; + + /* Extract and clear Guest ID from target PPN */ + g = tppn & (BIT(aia->nr_guest_bits) - 1); + tppn &= ~((gpa_t)(BIT(aia->nr_guest_bits) - 1)); + + /* Inject MSI to matching VCPU */ + kvm_for_each_vcpu(idx, vcpu, kvm) { + ippn = vcpu->arch.aia_context.imsic_addr >> + IMSIC_MMIO_PAGE_SHIFT; + if (ippn == tppn) { + toff = target & (IMSIC_MMIO_PAGE_SZ - 1); + return kvm_riscv_vcpu_aia_imsic_inject(vcpu, g, + toff, iid); + } + } + + return 0; +} + +int kvm_riscv_aia_inject_irq(struct kvm *kvm, unsigned int irq, bool level) +{ + /* Proceed only if AIA was initialized successfully */ + if (!kvm_riscv_aia_initialized(kvm)) + return -EBUSY; + + /* Inject interrupt level change in APLIC */ + return kvm_riscv_aia_aplic_inject(kvm, irq, level); +} + +void kvm_riscv_aia_init_vm(struct kvm *kvm) +{ + struct kvm_aia *aia = &kvm->arch.aia; + + if (!kvm_riscv_aia_available()) + return; + + /* + * We don't do any memory allocations over here because these + * will be done after AIA device is initialized by the user-space. + * + * Refer, aia_init() implementation for more details. + */ + + /* Initialize default values in AIA global context */ + aia->mode = (kvm_riscv_aia_nr_hgei) ? + KVM_DEV_RISCV_AIA_MODE_AUTO : KVM_DEV_RISCV_AIA_MODE_EMUL; + aia->nr_ids = kvm_riscv_aia_max_ids - 1; + aia->nr_sources = 0; + aia->nr_group_bits = 0; + aia->nr_group_shift = KVM_DEV_RISCV_AIA_GROUP_SHIFT_MIN; + aia->nr_hart_bits = 0; + aia->nr_guest_bits = 0; + aia->aplic_addr = KVM_RISCV_AIA_UNDEF_ADDR; +} + +void kvm_riscv_aia_destroy_vm(struct kvm *kvm) +{ + /* Proceed only if AIA was initialized successfully */ + if (!kvm_riscv_aia_initialized(kvm)) + return; + + /* Cleanup APLIC context */ + kvm_riscv_aia_aplic_cleanup(kvm); +} diff --git a/arch/riscv/kvm/aia_imsic.c b/arch/riscv/kvm/aia_imsic.c new file mode 100644 index 000000000000..6cf23b8adb71 --- /dev/null +++ b/arch/riscv/kvm/aia_imsic.c @@ -0,0 +1,1084 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + * + * Authors: + * Anup Patel <apatel@ventanamicro.com> + */ + +#include <linux/atomic.h> +#include <linux/bitmap.h> +#include <linux/kvm_host.h> +#include <linux/math.h> +#include <linux/spinlock.h> +#include <linux/swab.h> +#include <kvm/iodev.h> +#include <asm/csr.h> +#include <asm/kvm_aia_imsic.h> + +#define IMSIC_MAX_EIX (IMSIC_MAX_ID / BITS_PER_TYPE(u64)) + +struct imsic_mrif_eix { + unsigned long eip[BITS_PER_TYPE(u64) / BITS_PER_LONG]; + unsigned long eie[BITS_PER_TYPE(u64) / BITS_PER_LONG]; +}; + +struct imsic_mrif { + struct imsic_mrif_eix eix[IMSIC_MAX_EIX]; + unsigned long eithreshold; + unsigned long eidelivery; +}; + +struct imsic { + struct kvm_io_device iodev; + + u32 nr_msis; + u32 nr_eix; + u32 nr_hw_eix; + + /* + * At any point in time, the register state is in + * one of the following places: + * + * 1) Hardware: IMSIC VS-file (vsfile_cpu >= 0) + * 2) Software: IMSIC SW-file (vsfile_cpu < 0) + */ + + /* IMSIC VS-file */ + rwlock_t vsfile_lock; + int vsfile_cpu; + int vsfile_hgei; + void __iomem *vsfile_va; + phys_addr_t vsfile_pa; + + /* IMSIC SW-file */ + struct imsic_mrif *swfile; + phys_addr_t swfile_pa; +}; + +#define imsic_vs_csr_read(__c) \ +({ \ + unsigned long __r; \ + csr_write(CSR_VSISELECT, __c); \ + __r = csr_read(CSR_VSIREG); \ + __r; \ +}) + +#define imsic_read_switchcase(__ireg) \ + case __ireg: \ + return imsic_vs_csr_read(__ireg); +#define imsic_read_switchcase_2(__ireg) \ + imsic_read_switchcase(__ireg + 0) \ + imsic_read_switchcase(__ireg + 1) +#define imsic_read_switchcase_4(__ireg) \ + imsic_read_switchcase_2(__ireg + 0) \ + imsic_read_switchcase_2(__ireg + 2) +#define imsic_read_switchcase_8(__ireg) \ + imsic_read_switchcase_4(__ireg + 0) \ + imsic_read_switchcase_4(__ireg + 4) +#define imsic_read_switchcase_16(__ireg) \ + imsic_read_switchcase_8(__ireg + 0) \ + imsic_read_switchcase_8(__ireg + 8) +#define imsic_read_switchcase_32(__ireg) \ + imsic_read_switchcase_16(__ireg + 0) \ + imsic_read_switchcase_16(__ireg + 16) +#define imsic_read_switchcase_64(__ireg) \ + imsic_read_switchcase_32(__ireg + 0) \ + imsic_read_switchcase_32(__ireg + 32) + +static unsigned long imsic_eix_read(int ireg) +{ + switch (ireg) { + imsic_read_switchcase_64(IMSIC_EIP0) + imsic_read_switchcase_64(IMSIC_EIE0) + } + + return 0; +} + +#define imsic_vs_csr_swap(__c, __v) \ +({ \ + unsigned long __r; \ + csr_write(CSR_VSISELECT, __c); \ + __r = csr_swap(CSR_VSIREG, __v); \ + __r; \ +}) + +#define imsic_swap_switchcase(__ireg, __v) \ + case __ireg: \ + return imsic_vs_csr_swap(__ireg, __v); +#define imsic_swap_switchcase_2(__ireg, __v) \ + imsic_swap_switchcase(__ireg + 0, __v) \ + imsic_swap_switchcase(__ireg + 1, __v) +#define imsic_swap_switchcase_4(__ireg, __v) \ + imsic_swap_switchcase_2(__ireg + 0, __v) \ + imsic_swap_switchcase_2(__ireg + 2, __v) +#define imsic_swap_switchcase_8(__ireg, __v) \ + imsic_swap_switchcase_4(__ireg + 0, __v) \ + imsic_swap_switchcase_4(__ireg + 4, __v) +#define imsic_swap_switchcase_16(__ireg, __v) \ + imsic_swap_switchcase_8(__ireg + 0, __v) \ + imsic_swap_switchcase_8(__ireg + 8, __v) +#define imsic_swap_switchcase_32(__ireg, __v) \ + imsic_swap_switchcase_16(__ireg + 0, __v) \ + imsic_swap_switchcase_16(__ireg + 16, __v) +#define imsic_swap_switchcase_64(__ireg, __v) \ + imsic_swap_switchcase_32(__ireg + 0, __v) \ + imsic_swap_switchcase_32(__ireg + 32, __v) + +static unsigned long imsic_eix_swap(int ireg, unsigned long val) +{ + switch (ireg) { + imsic_swap_switchcase_64(IMSIC_EIP0, val) + imsic_swap_switchcase_64(IMSIC_EIE0, val) + } + + return 0; +} + +#define imsic_vs_csr_write(__c, __v) \ +do { \ + csr_write(CSR_VSISELECT, __c); \ + csr_write(CSR_VSIREG, __v); \ +} while (0) + +#define imsic_write_switchcase(__ireg, __v) \ + case __ireg: \ + imsic_vs_csr_write(__ireg, __v); \ + break; +#define imsic_write_switchcase_2(__ireg, __v) \ + imsic_write_switchcase(__ireg + 0, __v) \ + imsic_write_switchcase(__ireg + 1, __v) +#define imsic_write_switchcase_4(__ireg, __v) \ + imsic_write_switchcase_2(__ireg + 0, __v) \ + imsic_write_switchcase_2(__ireg + 2, __v) +#define imsic_write_switchcase_8(__ireg, __v) \ + imsic_write_switchcase_4(__ireg + 0, __v) \ + imsic_write_switchcase_4(__ireg + 4, __v) +#define imsic_write_switchcase_16(__ireg, __v) \ + imsic_write_switchcase_8(__ireg + 0, __v) \ + imsic_write_switchcase_8(__ireg + 8, __v) +#define imsic_write_switchcase_32(__ireg, __v) \ + imsic_write_switchcase_16(__ireg + 0, __v) \ + imsic_write_switchcase_16(__ireg + 16, __v) +#define imsic_write_switchcase_64(__ireg, __v) \ + imsic_write_switchcase_32(__ireg + 0, __v) \ + imsic_write_switchcase_32(__ireg + 32, __v) + +static void imsic_eix_write(int ireg, unsigned long val) +{ + switch (ireg) { + imsic_write_switchcase_64(IMSIC_EIP0, val) + imsic_write_switchcase_64(IMSIC_EIE0, val) + } +} + +#define imsic_vs_csr_set(__c, __v) \ +do { \ + csr_write(CSR_VSISELECT, __c); \ + csr_set(CSR_VSIREG, __v); \ +} while (0) + +#define imsic_set_switchcase(__ireg, __v) \ + case __ireg: \ + imsic_vs_csr_set(__ireg, __v); \ + break; +#define imsic_set_switchcase_2(__ireg, __v) \ + imsic_set_switchcase(__ireg + 0, __v) \ + imsic_set_switchcase(__ireg + 1, __v) +#define imsic_set_switchcase_4(__ireg, __v) \ + imsic_set_switchcase_2(__ireg + 0, __v) \ + imsic_set_switchcase_2(__ireg + 2, __v) +#define imsic_set_switchcase_8(__ireg, __v) \ + imsic_set_switchcase_4(__ireg + 0, __v) \ + imsic_set_switchcase_4(__ireg + 4, __v) +#define imsic_set_switchcase_16(__ireg, __v) \ + imsic_set_switchcase_8(__ireg + 0, __v) \ + imsic_set_switchcase_8(__ireg + 8, __v) +#define imsic_set_switchcase_32(__ireg, __v) \ + imsic_set_switchcase_16(__ireg + 0, __v) \ + imsic_set_switchcase_16(__ireg + 16, __v) +#define imsic_set_switchcase_64(__ireg, __v) \ + imsic_set_switchcase_32(__ireg + 0, __v) \ + imsic_set_switchcase_32(__ireg + 32, __v) + +static void imsic_eix_set(int ireg, unsigned long val) +{ + switch (ireg) { + imsic_set_switchcase_64(IMSIC_EIP0, val) + imsic_set_switchcase_64(IMSIC_EIE0, val) + } +} + +static unsigned long imsic_mrif_atomic_rmw(struct imsic_mrif *mrif, + unsigned long *ptr, + unsigned long new_val, + unsigned long wr_mask) +{ + unsigned long old_val = 0, tmp = 0; + + __asm__ __volatile__ ( + "0: lr.w.aq %1, %0\n" + " and %2, %1, %3\n" + " or %2, %2, %4\n" + " sc.w.rl %2, %2, %0\n" + " bnez %2, 0b" + : "+A" (*ptr), "+r" (old_val), "+r" (tmp) + : "r" (~wr_mask), "r" (new_val & wr_mask) + : "memory"); + + return old_val; +} + +static unsigned long imsic_mrif_atomic_or(struct imsic_mrif *mrif, + unsigned long *ptr, + unsigned long val) +{ + return atomic_long_fetch_or(val, (atomic_long_t *)ptr); +} + +#define imsic_mrif_atomic_write(__mrif, __ptr, __new_val) \ + imsic_mrif_atomic_rmw(__mrif, __ptr, __new_val, -1UL) +#define imsic_mrif_atomic_read(__mrif, __ptr) \ + imsic_mrif_atomic_or(__mrif, __ptr, 0) + +static u32 imsic_mrif_topei(struct imsic_mrif *mrif, u32 nr_eix, u32 nr_msis) +{ + struct imsic_mrif_eix *eix; + u32 i, imin, imax, ei, max_msi; + unsigned long eipend[BITS_PER_TYPE(u64) / BITS_PER_LONG]; + unsigned long eithreshold = imsic_mrif_atomic_read(mrif, + &mrif->eithreshold); + + max_msi = (eithreshold && (eithreshold <= nr_msis)) ? + eithreshold : nr_msis; + for (ei = 0; ei < nr_eix; ei++) { + eix = &mrif->eix[ei]; + eipend[0] = imsic_mrif_atomic_read(mrif, &eix->eie[0]) & + imsic_mrif_atomic_read(mrif, &eix->eip[0]); +#ifdef CONFIG_32BIT + eipend[1] = imsic_mrif_atomic_read(mrif, &eix->eie[1]) & + imsic_mrif_atomic_read(mrif, &eix->eip[1]); + if (!eipend[0] && !eipend[1]) +#else + if (!eipend[0]) +#endif + continue; + + imin = ei * BITS_PER_TYPE(u64); + imax = ((imin + BITS_PER_TYPE(u64)) < max_msi) ? + imin + BITS_PER_TYPE(u64) : max_msi; + for (i = (!imin) ? 1 : imin; i < imax; i++) { + if (test_bit(i - imin, eipend)) + return (i << TOPEI_ID_SHIFT) | i; + } + } + + return 0; +} + +static int imsic_mrif_isel_check(u32 nr_eix, unsigned long isel) +{ + u32 num = 0; + + switch (isel) { + case IMSIC_EIDELIVERY: + case IMSIC_EITHRESHOLD: + break; + case IMSIC_EIP0 ... IMSIC_EIP63: + num = isel - IMSIC_EIP0; + break; + case IMSIC_EIE0 ... IMSIC_EIE63: + num = isel - IMSIC_EIE0; + break; + default: + return -ENOENT; + } +#ifndef CONFIG_32BIT + if (num & 0x1) + return -EINVAL; +#endif + if ((num / 2) >= nr_eix) + return -EINVAL; + + return 0; +} + +static int imsic_mrif_rmw(struct imsic_mrif *mrif, u32 nr_eix, + unsigned long isel, unsigned long *val, + unsigned long new_val, unsigned long wr_mask) +{ + bool pend; + struct imsic_mrif_eix *eix; + unsigned long *ei, num, old_val = 0; + + switch (isel) { + case IMSIC_EIDELIVERY: + old_val = imsic_mrif_atomic_rmw(mrif, &mrif->eidelivery, + new_val, wr_mask & 0x1); + break; + case IMSIC_EITHRESHOLD: + old_val = imsic_mrif_atomic_rmw(mrif, &mrif->eithreshold, + new_val, wr_mask & (IMSIC_MAX_ID - 1)); + break; + case IMSIC_EIP0 ... IMSIC_EIP63: + case IMSIC_EIE0 ... IMSIC_EIE63: + if (isel >= IMSIC_EIP0 && isel <= IMSIC_EIP63) { + pend = true; + num = isel - IMSIC_EIP0; + } else { + pend = false; + num = isel - IMSIC_EIE0; + } + + if ((num / 2) >= nr_eix) + return -EINVAL; + eix = &mrif->eix[num / 2]; + +#ifndef CONFIG_32BIT + if (num & 0x1) + return -EINVAL; + ei = (pend) ? &eix->eip[0] : &eix->eie[0]; +#else + ei = (pend) ? &eix->eip[num & 0x1] : &eix->eie[num & 0x1]; +#endif + + /* Bit0 of EIP0 or EIE0 is read-only */ + if (!num) + wr_mask &= ~BIT(0); + + old_val = imsic_mrif_atomic_rmw(mrif, ei, new_val, wr_mask); + break; + default: + return -ENOENT; + } + + if (val) + *val = old_val; + + return 0; +} + +struct imsic_vsfile_read_data { + int hgei; + u32 nr_eix; + bool clear; + struct imsic_mrif *mrif; +}; + +static void imsic_vsfile_local_read(void *data) +{ + u32 i; + struct imsic_mrif_eix *eix; + struct imsic_vsfile_read_data *idata = data; + struct imsic_mrif *mrif = idata->mrif; + unsigned long new_hstatus, old_hstatus, old_vsiselect; + + old_vsiselect = csr_read(CSR_VSISELECT); + old_hstatus = csr_read(CSR_HSTATUS); + new_hstatus = old_hstatus & ~HSTATUS_VGEIN; + new_hstatus |= ((unsigned long)idata->hgei) << HSTATUS_VGEIN_SHIFT; + csr_write(CSR_HSTATUS, new_hstatus); + + /* + * We don't use imsic_mrif_atomic_xyz() functions to store + * values in MRIF because imsic_vsfile_read() is always called + * with pointer to temporary MRIF on stack. + */ + + if (idata->clear) { + mrif->eidelivery = imsic_vs_csr_swap(IMSIC_EIDELIVERY, 0); + mrif->eithreshold = imsic_vs_csr_swap(IMSIC_EITHRESHOLD, 0); + for (i = 0; i < idata->nr_eix; i++) { + eix = &mrif->eix[i]; + eix->eip[0] = imsic_eix_swap(IMSIC_EIP0 + i * 2, 0); + eix->eie[0] = imsic_eix_swap(IMSIC_EIE0 + i * 2, 0); +#ifdef CONFIG_32BIT + eix->eip[1] = imsic_eix_swap(IMSIC_EIP0 + i * 2 + 1, 0); + eix->eie[1] = imsic_eix_swap(IMSIC_EIE0 + i * 2 + 1, 0); +#endif + } + } else { + mrif->eidelivery = imsic_vs_csr_read(IMSIC_EIDELIVERY); + mrif->eithreshold = imsic_vs_csr_read(IMSIC_EITHRESHOLD); + for (i = 0; i < idata->nr_eix; i++) { + eix = &mrif->eix[i]; + eix->eip[0] = imsic_eix_read(IMSIC_EIP0 + i * 2); + eix->eie[0] = imsic_eix_read(IMSIC_EIE0 + i * 2); +#ifdef CONFIG_32BIT + eix->eip[1] = imsic_eix_read(IMSIC_EIP0 + i * 2 + 1); + eix->eie[1] = imsic_eix_read(IMSIC_EIE0 + i * 2 + 1); +#endif + } + } + + csr_write(CSR_HSTATUS, old_hstatus); + csr_write(CSR_VSISELECT, old_vsiselect); +} + +static void imsic_vsfile_read(int vsfile_hgei, int vsfile_cpu, u32 nr_eix, + bool clear, struct imsic_mrif *mrif) +{ + struct imsic_vsfile_read_data idata; + + /* We can only read clear if we have a IMSIC VS-file */ + if (vsfile_cpu < 0 || vsfile_hgei <= 0) + return; + + /* We can only read clear on local CPU */ + idata.hgei = vsfile_hgei; + idata.nr_eix = nr_eix; + idata.clear = clear; + idata.mrif = mrif; + on_each_cpu_mask(cpumask_of(vsfile_cpu), + imsic_vsfile_local_read, &idata, 1); +} + +struct imsic_vsfile_rw_data { + int hgei; + int isel; + bool write; + unsigned long val; +}; + +static void imsic_vsfile_local_rw(void *data) +{ + struct imsic_vsfile_rw_data *idata = data; + unsigned long new_hstatus, old_hstatus, old_vsiselect; + + old_vsiselect = csr_read(CSR_VSISELECT); + old_hstatus = csr_read(CSR_HSTATUS); + new_hstatus = old_hstatus & ~HSTATUS_VGEIN; + new_hstatus |= ((unsigned long)idata->hgei) << HSTATUS_VGEIN_SHIFT; + csr_write(CSR_HSTATUS, new_hstatus); + + switch (idata->isel) { + case IMSIC_EIDELIVERY: + if (idata->write) + imsic_vs_csr_write(IMSIC_EIDELIVERY, idata->val); + else + idata->val = imsic_vs_csr_read(IMSIC_EIDELIVERY); + break; + case IMSIC_EITHRESHOLD: + if (idata->write) + imsic_vs_csr_write(IMSIC_EITHRESHOLD, idata->val); + else + idata->val = imsic_vs_csr_read(IMSIC_EITHRESHOLD); + break; + case IMSIC_EIP0 ... IMSIC_EIP63: + case IMSIC_EIE0 ... IMSIC_EIE63: +#ifndef CONFIG_32BIT + if (idata->isel & 0x1) + break; +#endif + if (idata->write) + imsic_eix_write(idata->isel, idata->val); + else + idata->val = imsic_eix_read(idata->isel); + break; + default: + break; + } + + csr_write(CSR_HSTATUS, old_hstatus); + csr_write(CSR_VSISELECT, old_vsiselect); +} + +static int imsic_vsfile_rw(int vsfile_hgei, int vsfile_cpu, u32 nr_eix, + unsigned long isel, bool write, + unsigned long *val) +{ + int rc; + struct imsic_vsfile_rw_data rdata; + + /* We can only access register if we have a IMSIC VS-file */ + if (vsfile_cpu < 0 || vsfile_hgei <= 0) + return -EINVAL; + + /* Check IMSIC register iselect */ + rc = imsic_mrif_isel_check(nr_eix, isel); + if (rc) + return rc; + + /* We can only access register on local CPU */ + rdata.hgei = vsfile_hgei; + rdata.isel = isel; + rdata.write = write; + rdata.val = (write) ? *val : 0; + on_each_cpu_mask(cpumask_of(vsfile_cpu), + imsic_vsfile_local_rw, &rdata, 1); + + if (!write) + *val = rdata.val; + + return 0; +} + +static void imsic_vsfile_local_clear(int vsfile_hgei, u32 nr_eix) +{ + u32 i; + unsigned long new_hstatus, old_hstatus, old_vsiselect; + + /* We can only zero-out if we have a IMSIC VS-file */ + if (vsfile_hgei <= 0) + return; + + old_vsiselect = csr_read(CSR_VSISELECT); + old_hstatus = csr_read(CSR_HSTATUS); + new_hstatus = old_hstatus & ~HSTATUS_VGEIN; + new_hstatus |= ((unsigned long)vsfile_hgei) << HSTATUS_VGEIN_SHIFT; + csr_write(CSR_HSTATUS, new_hstatus); + + imsic_vs_csr_write(IMSIC_EIDELIVERY, 0); + imsic_vs_csr_write(IMSIC_EITHRESHOLD, 0); + for (i = 0; i < nr_eix; i++) { + imsic_eix_write(IMSIC_EIP0 + i * 2, 0); + imsic_eix_write(IMSIC_EIE0 + i * 2, 0); +#ifdef CONFIG_32BIT + imsic_eix_write(IMSIC_EIP0 + i * 2 + 1, 0); + imsic_eix_write(IMSIC_EIE0 + i * 2 + 1, 0); +#endif + } + + csr_write(CSR_HSTATUS, old_hstatus); + csr_write(CSR_VSISELECT, old_vsiselect); +} + +static void imsic_vsfile_local_update(int vsfile_hgei, u32 nr_eix, + struct imsic_mrif *mrif) +{ + u32 i; + struct imsic_mrif_eix *eix; + unsigned long new_hstatus, old_hstatus, old_vsiselect; + + /* We can only update if we have a HW IMSIC context */ + if (vsfile_hgei <= 0) + return; + + /* + * We don't use imsic_mrif_atomic_xyz() functions to read values + * from MRIF in this function because it is always called with + * pointer to temporary MRIF on stack. + */ + + old_vsiselect = csr_read(CSR_VSISELECT); + old_hstatus = csr_read(CSR_HSTATUS); + new_hstatus = old_hstatus & ~HSTATUS_VGEIN; + new_hstatus |= ((unsigned long)vsfile_hgei) << HSTATUS_VGEIN_SHIFT; + csr_write(CSR_HSTATUS, new_hstatus); + + for (i = 0; i < nr_eix; i++) { + eix = &mrif->eix[i]; + imsic_eix_set(IMSIC_EIP0 + i * 2, eix->eip[0]); + imsic_eix_set(IMSIC_EIE0 + i * 2, eix->eie[0]); +#ifdef CONFIG_32BIT + imsic_eix_set(IMSIC_EIP0 + i * 2 + 1, eix->eip[1]); + imsic_eix_set(IMSIC_EIE0 + i * 2 + 1, eix->eie[1]); +#endif + } + imsic_vs_csr_write(IMSIC_EITHRESHOLD, mrif->eithreshold); + imsic_vs_csr_write(IMSIC_EIDELIVERY, mrif->eidelivery); + + csr_write(CSR_HSTATUS, old_hstatus); + csr_write(CSR_VSISELECT, old_vsiselect); +} + +static void imsic_vsfile_cleanup(struct imsic *imsic) +{ + int old_vsfile_hgei, old_vsfile_cpu; + unsigned long flags; + + /* + * We don't use imsic_mrif_atomic_xyz() functions to clear the + * SW-file in this function because it is always called when the + * VCPU is being destroyed. + */ + + write_lock_irqsave(&imsic->vsfile_lock, flags); + old_vsfile_hgei = imsic->vsfile_hgei; + old_vsfile_cpu = imsic->vsfile_cpu; + imsic->vsfile_cpu = imsic->vsfile_hgei = -1; + imsic->vsfile_va = NULL; + imsic->vsfile_pa = 0; + write_unlock_irqrestore(&imsic->vsfile_lock, flags); + + memset(imsic->swfile, 0, sizeof(*imsic->swfile)); + + if (old_vsfile_cpu >= 0) + kvm_riscv_aia_free_hgei(old_vsfile_cpu, old_vsfile_hgei); +} + +static void imsic_swfile_extirq_update(struct kvm_vcpu *vcpu) +{ + struct imsic *imsic = vcpu->arch.aia_context.imsic_state; + struct imsic_mrif *mrif = imsic->swfile; + + if (imsic_mrif_atomic_read(mrif, &mrif->eidelivery) && + imsic_mrif_topei(mrif, imsic->nr_eix, imsic->nr_msis)) + kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_EXT); + else + kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT); +} + +static void imsic_swfile_read(struct kvm_vcpu *vcpu, bool clear, + struct imsic_mrif *mrif) +{ + struct imsic *imsic = vcpu->arch.aia_context.imsic_state; + + /* + * We don't use imsic_mrif_atomic_xyz() functions to read and + * write SW-file and MRIF in this function because it is always + * called when VCPU is not using SW-file and the MRIF points to + * a temporary MRIF on stack. + */ + + memcpy(mrif, imsic->swfile, sizeof(*mrif)); + if (clear) { + memset(imsic->swfile, 0, sizeof(*imsic->swfile)); + kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT); + } +} + +static void imsic_swfile_update(struct kvm_vcpu *vcpu, + struct imsic_mrif *mrif) +{ + u32 i; + struct imsic_mrif_eix *seix, *eix; + struct imsic *imsic = vcpu->arch.aia_context.imsic_state; + struct imsic_mrif *smrif = imsic->swfile; + + imsic_mrif_atomic_write(smrif, &smrif->eidelivery, mrif->eidelivery); + imsic_mrif_atomic_write(smrif, &smrif->eithreshold, mrif->eithreshold); + for (i = 0; i < imsic->nr_eix; i++) { + seix = &smrif->eix[i]; + eix = &mrif->eix[i]; + imsic_mrif_atomic_or(smrif, &seix->eip[0], eix->eip[0]); + imsic_mrif_atomic_or(smrif, &seix->eie[0], eix->eie[0]); +#ifdef CONFIG_32BIT + imsic_mrif_atomic_or(smrif, &seix->eip[1], eix->eip[1]); + imsic_mrif_atomic_or(smrif, &seix->eie[1], eix->eie[1]); +#endif + } + + imsic_swfile_extirq_update(vcpu); +} + +void kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu *vcpu) +{ + unsigned long flags; + struct imsic_mrif tmrif; + int old_vsfile_hgei, old_vsfile_cpu; + struct imsic *imsic = vcpu->arch.aia_context.imsic_state; + + /* Read and clear IMSIC VS-file details */ + write_lock_irqsave(&imsic->vsfile_lock, flags); + old_vsfile_hgei = imsic->vsfile_hgei; + old_vsfile_cpu = imsic->vsfile_cpu; + imsic->vsfile_cpu = imsic->vsfile_hgei = -1; + imsic->vsfile_va = NULL; + imsic->vsfile_pa = 0; + write_unlock_irqrestore(&imsic->vsfile_lock, flags); + + /* Do nothing, if no IMSIC VS-file to release */ + if (old_vsfile_cpu < 0) + return; + + /* + * At this point, all interrupt producers are still using + * the old IMSIC VS-file so we first re-direct all interrupt + * producers. + */ + + /* Purge the G-stage mapping */ + kvm_riscv_gstage_iounmap(vcpu->kvm, + vcpu->arch.aia_context.imsic_addr, + IMSIC_MMIO_PAGE_SZ); + + /* TODO: Purge the IOMMU mapping ??? */ + + /* + * At this point, all interrupt producers have been re-directed + * to somewhere else so we move register state from the old IMSIC + * VS-file to the IMSIC SW-file. + */ + + /* Read and clear register state from old IMSIC VS-file */ + memset(&tmrif, 0, sizeof(tmrif)); + imsic_vsfile_read(old_vsfile_hgei, old_vsfile_cpu, imsic->nr_hw_eix, + true, &tmrif); + + /* Update register state in IMSIC SW-file */ + imsic_swfile_update(vcpu, &tmrif); + + /* Free-up old IMSIC VS-file */ + kvm_riscv_aia_free_hgei(old_vsfile_cpu, old_vsfile_hgei); +} + +int kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu *vcpu) +{ + unsigned long flags; + phys_addr_t new_vsfile_pa; + struct imsic_mrif tmrif; + void __iomem *new_vsfile_va; + struct kvm *kvm = vcpu->kvm; + struct kvm_run *run = vcpu->run; + struct kvm_vcpu_aia *vaia = &vcpu->arch.aia_context; + struct imsic *imsic = vaia->imsic_state; + int ret = 0, new_vsfile_hgei = -1, old_vsfile_hgei, old_vsfile_cpu; + + /* Do nothing for emulation mode */ + if (kvm->arch.aia.mode == KVM_DEV_RISCV_AIA_MODE_EMUL) + return 1; + + /* Read old IMSIC VS-file details */ + read_lock_irqsave(&imsic->vsfile_lock, flags); + old_vsfile_hgei = imsic->vsfile_hgei; + old_vsfile_cpu = imsic->vsfile_cpu; + read_unlock_irqrestore(&imsic->vsfile_lock, flags); + + /* Do nothing if we are continuing on same CPU */ + if (old_vsfile_cpu == vcpu->cpu) + return 1; + + /* Allocate new IMSIC VS-file */ + ret = kvm_riscv_aia_alloc_hgei(vcpu->cpu, vcpu, + &new_vsfile_va, &new_vsfile_pa); + if (ret <= 0) { + /* For HW acceleration mode, we can't continue */ + if (kvm->arch.aia.mode == KVM_DEV_RISCV_AIA_MODE_HWACCEL) { + run->fail_entry.hardware_entry_failure_reason = + CSR_HSTATUS; + run->fail_entry.cpu = vcpu->cpu; + run->exit_reason = KVM_EXIT_FAIL_ENTRY; + return 0; + } + + /* Release old IMSIC VS-file */ + if (old_vsfile_cpu >= 0) + kvm_riscv_vcpu_aia_imsic_release(vcpu); + + /* For automatic mode, we continue */ + goto done; + } + new_vsfile_hgei = ret; + + /* + * At this point, all interrupt producers are still using + * to the old IMSIC VS-file so we first move all interrupt + * producers to the new IMSIC VS-file. + */ + + /* Zero-out new IMSIC VS-file */ + imsic_vsfile_local_clear(new_vsfile_hgei, imsic->nr_hw_eix); + + /* Update G-stage mapping for the new IMSIC VS-file */ + ret = kvm_riscv_gstage_ioremap(kvm, vcpu->arch.aia_context.imsic_addr, + new_vsfile_pa, IMSIC_MMIO_PAGE_SZ, + true, true); + if (ret) + goto fail_free_vsfile_hgei; + + /* TODO: Update the IOMMU mapping ??? */ + + /* Update new IMSIC VS-file details in IMSIC context */ + write_lock_irqsave(&imsic->vsfile_lock, flags); + imsic->vsfile_hgei = new_vsfile_hgei; + imsic->vsfile_cpu = vcpu->cpu; + imsic->vsfile_va = new_vsfile_va; + imsic->vsfile_pa = new_vsfile_pa; + write_unlock_irqrestore(&imsic->vsfile_lock, flags); + + /* + * At this point, all interrupt producers have been moved + * to the new IMSIC VS-file so we move register state from + * the old IMSIC VS/SW-file to the new IMSIC VS-file. + */ + + memset(&tmrif, 0, sizeof(tmrif)); + if (old_vsfile_cpu >= 0) { + /* Read and clear register state from old IMSIC VS-file */ + imsic_vsfile_read(old_vsfile_hgei, old_vsfile_cpu, + imsic->nr_hw_eix, true, &tmrif); + + /* Free-up old IMSIC VS-file */ + kvm_riscv_aia_free_hgei(old_vsfile_cpu, old_vsfile_hgei); + } else { + /* Read and clear register state from IMSIC SW-file */ + imsic_swfile_read(vcpu, true, &tmrif); + } + + /* Restore register state in the new IMSIC VS-file */ + imsic_vsfile_local_update(new_vsfile_hgei, imsic->nr_hw_eix, &tmrif); + +done: + /* Set VCPU HSTATUS.VGEIN to new IMSIC VS-file */ + vcpu->arch.guest_context.hstatus &= ~HSTATUS_VGEIN; + if (new_vsfile_hgei > 0) + vcpu->arch.guest_context.hstatus |= + ((unsigned long)new_vsfile_hgei) << HSTATUS_VGEIN_SHIFT; + + /* Continue run-loop */ + return 1; + +fail_free_vsfile_hgei: + kvm_riscv_aia_free_hgei(vcpu->cpu, new_vsfile_hgei); + return ret; +} + +int kvm_riscv_vcpu_aia_imsic_rmw(struct kvm_vcpu *vcpu, unsigned long isel, + unsigned long *val, unsigned long new_val, + unsigned long wr_mask) +{ + u32 topei; + struct imsic_mrif_eix *eix; + int r, rc = KVM_INSN_CONTINUE_NEXT_SEPC; + struct imsic *imsic = vcpu->arch.aia_context.imsic_state; + + if (isel == KVM_RISCV_AIA_IMSIC_TOPEI) { + /* Read pending and enabled interrupt with highest priority */ + topei = imsic_mrif_topei(imsic->swfile, imsic->nr_eix, + imsic->nr_msis); + if (val) + *val = topei; + + /* Writes ignore value and clear top pending interrupt */ + if (topei && wr_mask) { + topei >>= TOPEI_ID_SHIFT; + if (topei) { + eix = &imsic->swfile->eix[topei / + BITS_PER_TYPE(u64)]; + clear_bit(topei & (BITS_PER_TYPE(u64) - 1), + eix->eip); + } + } + } else { + r = imsic_mrif_rmw(imsic->swfile, imsic->nr_eix, isel, + val, new_val, wr_mask); + /* Forward unknown IMSIC register to user-space */ + if (r) + rc = (r == -ENOENT) ? 0 : KVM_INSN_ILLEGAL_TRAP; + } + + if (wr_mask) + imsic_swfile_extirq_update(vcpu); + + return rc; +} + +int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type, + bool write, unsigned long *val) +{ + u32 isel, vcpu_id; + unsigned long flags; + struct imsic *imsic; + struct kvm_vcpu *vcpu; + int rc, vsfile_hgei, vsfile_cpu; + + if (!kvm_riscv_aia_initialized(kvm)) + return -ENODEV; + + vcpu_id = KVM_DEV_RISCV_AIA_IMSIC_GET_VCPU(type); + vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); + if (!vcpu) + return -ENODEV; + + isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type); + imsic = vcpu->arch.aia_context.imsic_state; + + read_lock_irqsave(&imsic->vsfile_lock, flags); + + rc = 0; + vsfile_hgei = imsic->vsfile_hgei; + vsfile_cpu = imsic->vsfile_cpu; + if (vsfile_cpu < 0) { + if (write) { + rc = imsic_mrif_rmw(imsic->swfile, imsic->nr_eix, + isel, NULL, *val, -1UL); + imsic_swfile_extirq_update(vcpu); + } else + rc = imsic_mrif_rmw(imsic->swfile, imsic->nr_eix, + isel, val, 0, 0); + } + + read_unlock_irqrestore(&imsic->vsfile_lock, flags); + + if (!rc && vsfile_cpu >= 0) + rc = imsic_vsfile_rw(vsfile_hgei, vsfile_cpu, imsic->nr_eix, + isel, write, val); + + return rc; +} + +int kvm_riscv_aia_imsic_has_attr(struct kvm *kvm, unsigned long type) +{ + u32 isel, vcpu_id; + struct imsic *imsic; + struct kvm_vcpu *vcpu; + + if (!kvm_riscv_aia_initialized(kvm)) + return -ENODEV; + + vcpu_id = KVM_DEV_RISCV_AIA_IMSIC_GET_VCPU(type); + vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); + if (!vcpu) + return -ENODEV; + + isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type); + imsic = vcpu->arch.aia_context.imsic_state; + return imsic_mrif_isel_check(imsic->nr_eix, isel); +} + +void kvm_riscv_vcpu_aia_imsic_reset(struct kvm_vcpu *vcpu) +{ + struct imsic *imsic = vcpu->arch.aia_context.imsic_state; + + if (!imsic) + return; + + kvm_riscv_vcpu_aia_imsic_release(vcpu); + + memset(imsic->swfile, 0, sizeof(*imsic->swfile)); +} + +int kvm_riscv_vcpu_aia_imsic_inject(struct kvm_vcpu *vcpu, + u32 guest_index, u32 offset, u32 iid) +{ + unsigned long flags; + struct imsic_mrif_eix *eix; + struct imsic *imsic = vcpu->arch.aia_context.imsic_state; + + /* We only emulate one IMSIC MMIO page for each Guest VCPU */ + if (!imsic || !iid || guest_index || + (offset != IMSIC_MMIO_SETIPNUM_LE && + offset != IMSIC_MMIO_SETIPNUM_BE)) + return -ENODEV; + + iid = (offset == IMSIC_MMIO_SETIPNUM_BE) ? __swab32(iid) : iid; + if (imsic->nr_msis <= iid) + return -EINVAL; + + read_lock_irqsave(&imsic->vsfile_lock, flags); + + if (imsic->vsfile_cpu >= 0) { + writel(iid, imsic->vsfile_va + IMSIC_MMIO_SETIPNUM_LE); + kvm_vcpu_kick(vcpu); + } else { + eix = &imsic->swfile->eix[iid / BITS_PER_TYPE(u64)]; + set_bit(iid & (BITS_PER_TYPE(u64) - 1), eix->eip); + imsic_swfile_extirq_update(vcpu); + } + + read_unlock_irqrestore(&imsic->vsfile_lock, flags); + + return 0; +} + +static int imsic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, + gpa_t addr, int len, void *val) +{ + if (len != 4 || (addr & 0x3) != 0) + return -EOPNOTSUPP; + + *((u32 *)val) = 0; + + return 0; +} + +static int imsic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, + gpa_t addr, int len, const void *val) +{ + struct kvm_msi msi = { 0 }; + + if (len != 4 || (addr & 0x3) != 0) + return -EOPNOTSUPP; + + msi.address_hi = addr >> 32; + msi.address_lo = (u32)addr; + msi.data = *((const u32 *)val); + kvm_riscv_aia_inject_msi(vcpu->kvm, &msi); + + return 0; +}; + +static struct kvm_io_device_ops imsic_iodoev_ops = { + .read = imsic_mmio_read, + .write = imsic_mmio_write, +}; + +int kvm_riscv_vcpu_aia_imsic_init(struct kvm_vcpu *vcpu) +{ + int ret = 0; + struct imsic *imsic; + struct page *swfile_page; + struct kvm *kvm = vcpu->kvm; + + /* Fail if we have zero IDs */ + if (!kvm->arch.aia.nr_ids) + return -EINVAL; + + /* Allocate IMSIC context */ + imsic = kzalloc(sizeof(*imsic), GFP_KERNEL); + if (!imsic) + return -ENOMEM; + vcpu->arch.aia_context.imsic_state = imsic; + + /* Setup IMSIC context */ + imsic->nr_msis = kvm->arch.aia.nr_ids + 1; + rwlock_init(&imsic->vsfile_lock); + imsic->nr_eix = BITS_TO_U64(imsic->nr_msis); + imsic->nr_hw_eix = BITS_TO_U64(kvm_riscv_aia_max_ids); + imsic->vsfile_hgei = imsic->vsfile_cpu = -1; + + /* Setup IMSIC SW-file */ + swfile_page = alloc_pages(GFP_KERNEL | __GFP_ZERO, + get_order(sizeof(*imsic->swfile))); + if (!swfile_page) { + ret = -ENOMEM; + goto fail_free_imsic; + } + imsic->swfile = page_to_virt(swfile_page); + imsic->swfile_pa = page_to_phys(swfile_page); + + /* Setup IO device */ + kvm_iodevice_init(&imsic->iodev, &imsic_iodoev_ops); + mutex_lock(&kvm->slots_lock); + ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, + vcpu->arch.aia_context.imsic_addr, + KVM_DEV_RISCV_IMSIC_SIZE, + &imsic->iodev); + mutex_unlock(&kvm->slots_lock); + if (ret) + goto fail_free_swfile; + + return 0; + +fail_free_swfile: + free_pages((unsigned long)imsic->swfile, + get_order(sizeof(*imsic->swfile))); +fail_free_imsic: + vcpu->arch.aia_context.imsic_state = NULL; + kfree(imsic); + return ret; +} + +void kvm_riscv_vcpu_aia_imsic_cleanup(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = vcpu->kvm; + struct imsic *imsic = vcpu->arch.aia_context.imsic_state; + + if (!imsic) + return; + + imsic_vsfile_cleanup(imsic); + + mutex_lock(&kvm->slots_lock); + kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &imsic->iodev); + mutex_unlock(&kvm->slots_lock); + + free_pages((unsigned long)imsic->swfile, + get_order(sizeof(*imsic->swfile))); + + vcpu->arch.aia_context.imsic_state = NULL; + kfree(imsic); +} diff --git a/arch/riscv/kvm/main.c b/arch/riscv/kvm/main.c index a7112d583637..48ae0d4b3932 100644 --- a/arch/riscv/kvm/main.c +++ b/arch/riscv/kvm/main.c @@ -116,7 +116,8 @@ static int __init riscv_kvm_init(void) kvm_info("VMID %ld bits available\n", kvm_riscv_gstage_vmid_bits()); if (kvm_riscv_aia_available()) - kvm_info("AIA available\n"); + kvm_info("AIA available with %d guest external interrupts\n", + kvm_riscv_aia_nr_hgei); rc = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); if (rc) { diff --git a/arch/riscv/kvm/tlb.c b/arch/riscv/kvm/tlb.c index 0e5479600695..44bc324aeeb0 100644 --- a/arch/riscv/kvm/tlb.c +++ b/arch/riscv/kvm/tlb.c @@ -296,7 +296,7 @@ static void make_xfence_request(struct kvm *kvm, unsigned int actual_req = req; DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS); - bitmap_clear(vcpu_mask, 0, KVM_MAX_VCPUS); + bitmap_zero(vcpu_mask, KVM_MAX_VCPUS); kvm_for_each_vcpu(i, vcpu, kvm) { if (hbase != -1UL) { if (vcpu->vcpu_id < hbase) diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index 8bd9f2a8a0b9..d12ef99901fc 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -22,6 +22,8 @@ #include <asm/cacheflush.h> #include <asm/hwcap.h> #include <asm/sbi.h> +#include <asm/vector.h> +#include <asm/kvm_vcpu_vector.h> const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), @@ -57,10 +59,12 @@ static const unsigned long kvm_isa_ext_arr[] = { [KVM_RISCV_ISA_EXT_H] = RISCV_ISA_EXT_h, [KVM_RISCV_ISA_EXT_I] = RISCV_ISA_EXT_i, [KVM_RISCV_ISA_EXT_M] = RISCV_ISA_EXT_m, + [KVM_RISCV_ISA_EXT_V] = RISCV_ISA_EXT_v, KVM_ISA_EXT_ARR(SSAIA), KVM_ISA_EXT_ARR(SSTC), KVM_ISA_EXT_ARR(SVINVAL), + KVM_ISA_EXT_ARR(SVNAPOT), KVM_ISA_EXT_ARR(SVPBMT), KVM_ISA_EXT_ARR(ZBB), KVM_ISA_EXT_ARR(ZIHINTPAUSE), @@ -85,6 +89,8 @@ static bool kvm_riscv_vcpu_isa_enable_allowed(unsigned long ext) switch (ext) { case KVM_RISCV_ISA_EXT_H: return false; + case KVM_RISCV_ISA_EXT_V: + return riscv_v_vstate_ctrl_user_allowed(); default: break; } @@ -102,6 +108,7 @@ static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext) case KVM_RISCV_ISA_EXT_SSAIA: case KVM_RISCV_ISA_EXT_SSTC: case KVM_RISCV_ISA_EXT_SVINVAL: + case KVM_RISCV_ISA_EXT_SVNAPOT: case KVM_RISCV_ISA_EXT_ZIHINTPAUSE: case KVM_RISCV_ISA_EXT_ZBB: return false; @@ -138,6 +145,8 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu) kvm_riscv_vcpu_fp_reset(vcpu); + kvm_riscv_vcpu_vector_reset(vcpu); + kvm_riscv_vcpu_timer_reset(vcpu); kvm_riscv_vcpu_aia_reset(vcpu); @@ -198,6 +207,9 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) cntx->hstatus |= HSTATUS_SPVP; cntx->hstatus |= HSTATUS_SPV; + if (kvm_riscv_vcpu_alloc_vector_context(vcpu, cntx)) + return -ENOMEM; + /* By default, make CY, TM, and IR counters accessible in VU mode */ reset_csr->scounteren = 0x7; @@ -241,6 +253,9 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) /* Free unused pages pre-allocated for G-stage page table mappings */ kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); + + /* Free vector context space for host and guest kernel */ + kvm_riscv_vcpu_free_vector_context(vcpu); } int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) @@ -250,10 +265,12 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) { + kvm_riscv_aia_wakeon_hgei(vcpu, true); } void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) { + kvm_riscv_aia_wakeon_hgei(vcpu, false); } int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) @@ -679,6 +696,9 @@ static int kvm_riscv_vcpu_set_reg(struct kvm_vcpu *vcpu, return kvm_riscv_vcpu_set_reg_isa_ext(vcpu, reg); case KVM_REG_RISCV_SBI_EXT: return kvm_riscv_vcpu_set_reg_sbi_ext(vcpu, reg); + case KVM_REG_RISCV_VECTOR: + return kvm_riscv_vcpu_set_reg_vector(vcpu, reg, + KVM_REG_RISCV_VECTOR); default: break; } @@ -708,6 +728,9 @@ static int kvm_riscv_vcpu_get_reg(struct kvm_vcpu *vcpu, return kvm_riscv_vcpu_get_reg_isa_ext(vcpu, reg); case KVM_REG_RISCV_SBI_EXT: return kvm_riscv_vcpu_get_reg_sbi_ext(vcpu, reg); + case KVM_REG_RISCV_VECTOR: + return kvm_riscv_vcpu_get_reg_vector(vcpu, reg, + KVM_REG_RISCV_VECTOR); default: break; } @@ -1002,6 +1025,9 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) kvm_riscv_vcpu_host_fp_save(&vcpu->arch.host_context); kvm_riscv_vcpu_guest_fp_restore(&vcpu->arch.guest_context, vcpu->arch.isa); + kvm_riscv_vcpu_host_vector_save(&vcpu->arch.host_context); + kvm_riscv_vcpu_guest_vector_restore(&vcpu->arch.guest_context, + vcpu->arch.isa); kvm_riscv_vcpu_aia_load(vcpu, cpu); @@ -1021,6 +1047,9 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) kvm_riscv_vcpu_host_fp_restore(&vcpu->arch.host_context); kvm_riscv_vcpu_timer_save(vcpu); + kvm_riscv_vcpu_guest_vector_save(&vcpu->arch.guest_context, + vcpu->arch.isa); + kvm_riscv_vcpu_host_vector_restore(&vcpu->arch.host_context); csr->vsstatus = csr_read(CSR_VSSTATUS); csr->vsie = csr_read(CSR_VSIE); diff --git a/arch/riscv/kvm/vcpu_exit.c b/arch/riscv/kvm/vcpu_exit.c index 4ea101a73d8b..2415722c01b8 100644 --- a/arch/riscv/kvm/vcpu_exit.c +++ b/arch/riscv/kvm/vcpu_exit.c @@ -183,6 +183,8 @@ int kvm_riscv_vcpu_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, run->exit_reason = KVM_EXIT_UNKNOWN; switch (trap->scause) { case EXC_INST_ILLEGAL: + case EXC_LOAD_MISALIGNED: + case EXC_STORE_MISALIGNED: if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV) { kvm_riscv_vcpu_trap_redirect(vcpu, trap); ret = 1; diff --git a/arch/riscv/kvm/vcpu_sbi.c b/arch/riscv/kvm/vcpu_sbi.c index e52fde504433..7b46e04fb667 100644 --- a/arch/riscv/kvm/vcpu_sbi.c +++ b/arch/riscv/kvm/vcpu_sbi.c @@ -20,9 +20,7 @@ static const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_v01 = { }; #endif -#ifdef CONFIG_RISCV_PMU_SBI -extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_pmu; -#else +#ifndef CONFIG_RISCV_PMU_SBI static const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_pmu = { .extid_start = -1UL, .extid_end = -1UL, @@ -31,49 +29,49 @@ static const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_pmu = { #endif struct kvm_riscv_sbi_extension_entry { - enum KVM_RISCV_SBI_EXT_ID dis_idx; + enum KVM_RISCV_SBI_EXT_ID ext_idx; const struct kvm_vcpu_sbi_extension *ext_ptr; }; static const struct kvm_riscv_sbi_extension_entry sbi_ext[] = { { - .dis_idx = KVM_RISCV_SBI_EXT_V01, + .ext_idx = KVM_RISCV_SBI_EXT_V01, .ext_ptr = &vcpu_sbi_ext_v01, }, { - .dis_idx = KVM_RISCV_SBI_EXT_MAX, /* Can't be disabled */ + .ext_idx = KVM_RISCV_SBI_EXT_MAX, /* Can't be disabled */ .ext_ptr = &vcpu_sbi_ext_base, }, { - .dis_idx = KVM_RISCV_SBI_EXT_TIME, + .ext_idx = KVM_RISCV_SBI_EXT_TIME, .ext_ptr = &vcpu_sbi_ext_time, }, { - .dis_idx = KVM_RISCV_SBI_EXT_IPI, + .ext_idx = KVM_RISCV_SBI_EXT_IPI, .ext_ptr = &vcpu_sbi_ext_ipi, }, { - .dis_idx = KVM_RISCV_SBI_EXT_RFENCE, + .ext_idx = KVM_RISCV_SBI_EXT_RFENCE, .ext_ptr = &vcpu_sbi_ext_rfence, }, { - .dis_idx = KVM_RISCV_SBI_EXT_SRST, + .ext_idx = KVM_RISCV_SBI_EXT_SRST, .ext_ptr = &vcpu_sbi_ext_srst, }, { - .dis_idx = KVM_RISCV_SBI_EXT_HSM, + .ext_idx = KVM_RISCV_SBI_EXT_HSM, .ext_ptr = &vcpu_sbi_ext_hsm, }, { - .dis_idx = KVM_RISCV_SBI_EXT_PMU, + .ext_idx = KVM_RISCV_SBI_EXT_PMU, .ext_ptr = &vcpu_sbi_ext_pmu, }, { - .dis_idx = KVM_RISCV_SBI_EXT_EXPERIMENTAL, + .ext_idx = KVM_RISCV_SBI_EXT_EXPERIMENTAL, .ext_ptr = &vcpu_sbi_ext_experimental, }, { - .dis_idx = KVM_RISCV_SBI_EXT_VENDOR, + .ext_idx = KVM_RISCV_SBI_EXT_VENDOR, .ext_ptr = &vcpu_sbi_ext_vendor, }, }; @@ -147,7 +145,7 @@ static int riscv_vcpu_set_sbi_ext_single(struct kvm_vcpu *vcpu, return -EINVAL; for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) { - if (sbi_ext[i].dis_idx == reg_num) { + if (sbi_ext[i].ext_idx == reg_num) { sext = &sbi_ext[i]; break; } @@ -155,7 +153,15 @@ static int riscv_vcpu_set_sbi_ext_single(struct kvm_vcpu *vcpu, if (!sext) return -ENOENT; - scontext->extension_disabled[sext->dis_idx] = !reg_val; + /* + * We can't set the extension status to available here, since it may + * have a probe() function which needs to confirm availability first, + * but it may be too early to call that here. We can set the status to + * unavailable, though. + */ + if (!reg_val) + scontext->ext_status[sext->ext_idx] = + KVM_RISCV_SBI_EXT_UNAVAILABLE; return 0; } @@ -172,7 +178,7 @@ static int riscv_vcpu_get_sbi_ext_single(struct kvm_vcpu *vcpu, return -EINVAL; for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) { - if (sbi_ext[i].dis_idx == reg_num) { + if (sbi_ext[i].ext_idx == reg_num) { sext = &sbi_ext[i]; break; } @@ -180,7 +186,15 @@ static int riscv_vcpu_get_sbi_ext_single(struct kvm_vcpu *vcpu, if (!sext) return -ENOENT; - *reg_val = !scontext->extension_disabled[sext->dis_idx]; + /* + * If the extension status is still uninitialized, then we should probe + * to determine if it's available, but it may be too early to do that + * here. The best we can do is report that the extension has not been + * disabled, i.e. we return 1 when the extension is available and also + * when it only may be available. + */ + *reg_val = scontext->ext_status[sext->ext_idx] != + KVM_RISCV_SBI_EXT_UNAVAILABLE; return 0; } @@ -307,18 +321,32 @@ int kvm_riscv_vcpu_get_reg_sbi_ext(struct kvm_vcpu *vcpu, const struct kvm_vcpu_sbi_extension *kvm_vcpu_sbi_find_ext( struct kvm_vcpu *vcpu, unsigned long extid) { - int i; - const struct kvm_riscv_sbi_extension_entry *sext; struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context; + const struct kvm_riscv_sbi_extension_entry *entry; + const struct kvm_vcpu_sbi_extension *ext; + int i; for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) { - sext = &sbi_ext[i]; - if (sext->ext_ptr->extid_start <= extid && - sext->ext_ptr->extid_end >= extid) { - if (sext->dis_idx < KVM_RISCV_SBI_EXT_MAX && - scontext->extension_disabled[sext->dis_idx]) + entry = &sbi_ext[i]; + ext = entry->ext_ptr; + + if (ext->extid_start <= extid && ext->extid_end >= extid) { + if (entry->ext_idx >= KVM_RISCV_SBI_EXT_MAX || + scontext->ext_status[entry->ext_idx] == + KVM_RISCV_SBI_EXT_AVAILABLE) + return ext; + if (scontext->ext_status[entry->ext_idx] == + KVM_RISCV_SBI_EXT_UNAVAILABLE) return NULL; - return sbi_ext[i].ext_ptr; + if (ext->probe && !ext->probe(vcpu)) { + scontext->ext_status[entry->ext_idx] = + KVM_RISCV_SBI_EXT_UNAVAILABLE; + return NULL; + } + + scontext->ext_status[entry->ext_idx] = + KVM_RISCV_SBI_EXT_AVAILABLE; + return ext; } } diff --git a/arch/riscv/kvm/vcpu_vector.c b/arch/riscv/kvm/vcpu_vector.c new file mode 100644 index 000000000000..edd2eecbddc2 --- /dev/null +++ b/arch/riscv/kvm/vcpu_vector.c @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 SiFive + * + * Authors: + * Vincent Chen <vincent.chen@sifive.com> + * Greentime Hu <greentime.hu@sifive.com> + */ + +#include <linux/errno.h> +#include <linux/err.h> +#include <linux/kvm_host.h> +#include <linux/uaccess.h> +#include <asm/hwcap.h> +#include <asm/kvm_vcpu_vector.h> +#include <asm/vector.h> + +#ifdef CONFIG_RISCV_ISA_V +void kvm_riscv_vcpu_vector_reset(struct kvm_vcpu *vcpu) +{ + unsigned long *isa = vcpu->arch.isa; + struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; + + cntx->sstatus &= ~SR_VS; + if (riscv_isa_extension_available(isa, v)) { + cntx->sstatus |= SR_VS_INITIAL; + WARN_ON(!cntx->vector.datap); + memset(cntx->vector.datap, 0, riscv_v_vsize); + } else { + cntx->sstatus |= SR_VS_OFF; + } +} + +static void kvm_riscv_vcpu_vector_clean(struct kvm_cpu_context *cntx) +{ + cntx->sstatus &= ~SR_VS; + cntx->sstatus |= SR_VS_CLEAN; +} + +void kvm_riscv_vcpu_guest_vector_save(struct kvm_cpu_context *cntx, + unsigned long *isa) +{ + if ((cntx->sstatus & SR_VS) == SR_VS_DIRTY) { + if (riscv_isa_extension_available(isa, v)) + __kvm_riscv_vector_save(cntx); + kvm_riscv_vcpu_vector_clean(cntx); + } +} + +void kvm_riscv_vcpu_guest_vector_restore(struct kvm_cpu_context *cntx, + unsigned long *isa) +{ + if ((cntx->sstatus & SR_VS) != SR_VS_OFF) { + if (riscv_isa_extension_available(isa, v)) + __kvm_riscv_vector_restore(cntx); + kvm_riscv_vcpu_vector_clean(cntx); + } +} + +void kvm_riscv_vcpu_host_vector_save(struct kvm_cpu_context *cntx) +{ + /* No need to check host sstatus as it can be modified outside */ + if (riscv_isa_extension_available(NULL, v)) + __kvm_riscv_vector_save(cntx); +} + +void kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cntx) +{ + if (riscv_isa_extension_available(NULL, v)) + __kvm_riscv_vector_restore(cntx); +} + +int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu, + struct kvm_cpu_context *cntx) +{ + cntx->vector.datap = kmalloc(riscv_v_vsize, GFP_KERNEL); + if (!cntx->vector.datap) + return -ENOMEM; + + vcpu->arch.host_context.vector.datap = kzalloc(riscv_v_vsize, GFP_KERNEL); + if (!vcpu->arch.host_context.vector.datap) + return -ENOMEM; + + return 0; +} + +void kvm_riscv_vcpu_free_vector_context(struct kvm_vcpu *vcpu) +{ + kfree(vcpu->arch.guest_reset_context.vector.datap); + kfree(vcpu->arch.host_context.vector.datap); +} +#endif + +static void *kvm_riscv_vcpu_vreg_addr(struct kvm_vcpu *vcpu, + unsigned long reg_num, + size_t reg_size) +{ + struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; + void *reg_val; + size_t vlenb = riscv_v_vsize / 32; + + if (reg_num < KVM_REG_RISCV_VECTOR_REG(0)) { + if (reg_size != sizeof(unsigned long)) + return NULL; + switch (reg_num) { + case KVM_REG_RISCV_VECTOR_CSR_REG(vstart): + reg_val = &cntx->vector.vstart; + break; + case KVM_REG_RISCV_VECTOR_CSR_REG(vl): + reg_val = &cntx->vector.vl; + break; + case KVM_REG_RISCV_VECTOR_CSR_REG(vtype): + reg_val = &cntx->vector.vtype; + break; + case KVM_REG_RISCV_VECTOR_CSR_REG(vcsr): + reg_val = &cntx->vector.vcsr; + break; + case KVM_REG_RISCV_VECTOR_CSR_REG(datap): + default: + return NULL; + } + } else if (reg_num <= KVM_REG_RISCV_VECTOR_REG(31)) { + if (reg_size != vlenb) + return NULL; + reg_val = cntx->vector.datap + + (reg_num - KVM_REG_RISCV_VECTOR_REG(0)) * vlenb; + } else { + return NULL; + } + + return reg_val; +} + +int kvm_riscv_vcpu_get_reg_vector(struct kvm_vcpu *vcpu, + const struct kvm_one_reg *reg, + unsigned long rtype) +{ + unsigned long *isa = vcpu->arch.isa; + unsigned long __user *uaddr = + (unsigned long __user *)(unsigned long)reg->addr; + unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | + KVM_REG_SIZE_MASK | + rtype); + void *reg_val = NULL; + size_t reg_size = KVM_REG_SIZE(reg->id); + + if (rtype == KVM_REG_RISCV_VECTOR && + riscv_isa_extension_available(isa, v)) { + reg_val = kvm_riscv_vcpu_vreg_addr(vcpu, reg_num, reg_size); + } + + if (!reg_val) + return -EINVAL; + + if (copy_to_user(uaddr, reg_val, reg_size)) + return -EFAULT; + + return 0; +} + +int kvm_riscv_vcpu_set_reg_vector(struct kvm_vcpu *vcpu, + const struct kvm_one_reg *reg, + unsigned long rtype) +{ + unsigned long *isa = vcpu->arch.isa; + unsigned long __user *uaddr = + (unsigned long __user *)(unsigned long)reg->addr; + unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | + KVM_REG_SIZE_MASK | + rtype); + void *reg_val = NULL; + size_t reg_size = KVM_REG_SIZE(reg->id); + + if (rtype == KVM_REG_RISCV_VECTOR && + riscv_isa_extension_available(isa, v)) { + reg_val = kvm_riscv_vcpu_vreg_addr(vcpu, reg_num, reg_size); + } + + if (!reg_val) + return -EINVAL; + + if (copy_from_user(reg_val, uaddr, reg_size)) + return -EFAULT; + + return 0; +} diff --git a/arch/riscv/kvm/vm.c b/arch/riscv/kvm/vm.c index 6ef15f78e80f..7e2b50c692c1 100644 --- a/arch/riscv/kvm/vm.c +++ b/arch/riscv/kvm/vm.c @@ -55,11 +55,129 @@ void kvm_arch_destroy_vm(struct kvm *kvm) kvm_riscv_aia_destroy_vm(kvm); } +int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irql, + bool line_status) +{ + if (!irqchip_in_kernel(kvm)) + return -ENXIO; + + return kvm_riscv_aia_inject_irq(kvm, irql->irq, irql->level); +} + +int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm, int irq_source_id, + int level, bool line_status) +{ + struct kvm_msi msi; + + if (!level) + return -1; + + msi.address_lo = e->msi.address_lo; + msi.address_hi = e->msi.address_hi; + msi.data = e->msi.data; + msi.flags = e->msi.flags; + msi.devid = e->msi.devid; + + return kvm_riscv_aia_inject_msi(kvm, &msi); +} + +static int kvm_riscv_set_irq(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm, int irq_source_id, + int level, bool line_status) +{ + return kvm_riscv_aia_inject_irq(kvm, e->irqchip.pin, level); +} + +int kvm_riscv_setup_default_irq_routing(struct kvm *kvm, u32 lines) +{ + struct kvm_irq_routing_entry *ents; + int i, rc; + + ents = kcalloc(lines, sizeof(*ents), GFP_KERNEL); + if (!ents) + return -ENOMEM; + + for (i = 0; i < lines; i++) { + ents[i].gsi = i; + ents[i].type = KVM_IRQ_ROUTING_IRQCHIP; + ents[i].u.irqchip.irqchip = 0; + ents[i].u.irqchip.pin = i; + } + rc = kvm_set_irq_routing(kvm, ents, lines, 0); + kfree(ents); + + return rc; +} + +bool kvm_arch_can_set_irq_routing(struct kvm *kvm) +{ + return irqchip_in_kernel(kvm); +} + +int kvm_set_routing_entry(struct kvm *kvm, + struct kvm_kernel_irq_routing_entry *e, + const struct kvm_irq_routing_entry *ue) +{ + int r = -EINVAL; + + switch (ue->type) { + case KVM_IRQ_ROUTING_IRQCHIP: + e->set = kvm_riscv_set_irq; + e->irqchip.irqchip = ue->u.irqchip.irqchip; + e->irqchip.pin = ue->u.irqchip.pin; + if ((e->irqchip.pin >= KVM_IRQCHIP_NUM_PINS) || + (e->irqchip.irqchip >= KVM_NR_IRQCHIPS)) + goto out; + break; + case KVM_IRQ_ROUTING_MSI: + e->set = kvm_set_msi; + e->msi.address_lo = ue->u.msi.address_lo; + e->msi.address_hi = ue->u.msi.address_hi; + e->msi.data = ue->u.msi.data; + e->msi.flags = ue->flags; + e->msi.devid = ue->u.msi.devid; + break; + default: + goto out; + } + r = 0; +out: + return r; +} + +int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm, int irq_source_id, int level, + bool line_status) +{ + if (!level) + return -EWOULDBLOCK; + + switch (e->type) { + case KVM_IRQ_ROUTING_MSI: + return kvm_set_msi(e, kvm, irq_source_id, level, line_status); + + case KVM_IRQ_ROUTING_IRQCHIP: + return kvm_riscv_set_irq(e, kvm, irq_source_id, + level, line_status); + } + + return -EWOULDBLOCK; +} + +bool kvm_arch_irqchip_in_kernel(struct kvm *kvm) +{ + return irqchip_in_kernel(kvm); +} + int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) { int r; switch (ext) { + case KVM_CAP_IRQCHIP: + r = kvm_riscv_aia_available(); + break; case KVM_CAP_IOEVENTFD: case KVM_CAP_DEVICE_CTRL: case KVM_CAP_USER_MEMORY: diff --git a/arch/riscv/mm/Makefile b/arch/riscv/mm/Makefile index b85e9e82f082..9c454f90fd3d 100644 --- a/arch/riscv/mm/Makefile +++ b/arch/riscv/mm/Makefile @@ -13,8 +13,7 @@ endif KCOV_INSTRUMENT_init.o := n obj-y += init.o -obj-y += extable.o -obj-$(CONFIG_MMU) += fault.o pageattr.o +obj-$(CONFIG_MMU) += extable.o fault.o pageattr.o obj-y += cacheflush.o obj-y += context.o obj-y += pgtable.o diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index 35a84ec69a9f..6ea2cce4cc17 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -247,24 +247,12 @@ void handle_page_fault(struct pt_regs *regs) * only copy the information from the master page table, * nothing more. */ - if (unlikely((addr >= VMALLOC_START) && (addr < VMALLOC_END))) { + if ((!IS_ENABLED(CONFIG_MMU) || !IS_ENABLED(CONFIG_64BIT)) && + unlikely(addr >= VMALLOC_START && addr < VMALLOC_END)) { vmalloc_fault(regs, code, addr); return; } -#ifdef CONFIG_64BIT - /* - * Modules in 64bit kernels lie in their own virtual region which is not - * in the vmalloc region, but dealing with page faults in this region - * or the vmalloc region amounts to doing the same thing: checking that - * the mapping exists in init_mm.pgd and updating user page table, so - * just use vmalloc_fault. - */ - if (unlikely(addr >= MODULES_VADDR && addr < MODULES_END)) { - vmalloc_fault(regs, code, addr); - return; - } -#endif /* Enable interrupts if they were enabled in the parent context. */ if (!regs_irqs_disabled(regs)) local_irq_enable(); @@ -295,6 +283,36 @@ void handle_page_fault(struct pt_regs *regs) flags |= FAULT_FLAG_WRITE; else if (cause == EXC_INST_PAGE_FAULT) flags |= FAULT_FLAG_INSTRUCTION; +#ifdef CONFIG_PER_VMA_LOCK + if (!(flags & FAULT_FLAG_USER)) + goto lock_mmap; + + vma = lock_vma_under_rcu(mm, addr); + if (!vma) + goto lock_mmap; + + if (unlikely(access_error(cause, vma))) { + vma_end_read(vma); + goto lock_mmap; + } + + fault = handle_mm_fault(vma, addr, flags | FAULT_FLAG_VMA_LOCK, regs); + vma_end_read(vma); + + if (!(fault & VM_FAULT_RETRY)) { + count_vm_vma_lock_event(VMA_LOCK_SUCCESS); + goto done; + } + count_vm_vma_lock_event(VMA_LOCK_RETRY); + + if (fault_signal_pending(fault, regs)) { + if (!user_mode(regs)) + no_context(regs, addr); + return; + } +lock_mmap: +#endif /* CONFIG_PER_VMA_LOCK */ + retry: vma = lock_mm_and_find_vma(mm, addr, regs); if (unlikely(!vma)) { @@ -350,6 +368,9 @@ retry: mmap_read_unlock(mm); +#ifdef CONFIG_PER_VMA_LOCK +done: +#endif if (unlikely(fault & VM_FAULT_ERROR)) { tsk->thread.bad_cause = cause; mm_fault_error(regs, addr, fault); diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index 4fa420faa780..3b1e927a06b7 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -1389,3 +1389,61 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, return vmemmap_populate_basepages(start, end, node, NULL); } #endif + +#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) +/* + * Pre-allocates page-table pages for a specific area in the kernel + * page-table. Only the level which needs to be synchronized between + * all page-tables is allocated because the synchronization can be + * expensive. + */ +static void __init preallocate_pgd_pages_range(unsigned long start, unsigned long end, + const char *area) +{ + unsigned long addr; + const char *lvl; + + for (addr = start; addr < end && addr >= start; addr = ALIGN(addr + 1, PGDIR_SIZE)) { + pgd_t *pgd = pgd_offset_k(addr); + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + + lvl = "p4d"; + p4d = p4d_alloc(&init_mm, pgd, addr); + if (!p4d) + goto failed; + + if (pgtable_l5_enabled) + continue; + + lvl = "pud"; + pud = pud_alloc(&init_mm, p4d, addr); + if (!pud) + goto failed; + + if (pgtable_l4_enabled) + continue; + + lvl = "pmd"; + pmd = pmd_alloc(&init_mm, pud, addr); + if (!pmd) + goto failed; + } + return; + +failed: + /* + * The pages have to be there now or they will be missing in + * process page-tables later. + */ + panic("Failed to pre-allocate %s pages for %s area\n", lvl, area); +} + +void __init pgtable_cache_init(void) +{ + preallocate_pgd_pages_range(VMALLOC_START, VMALLOC_END, "vmalloc"); + if (IS_ENABLED(CONFIG_MODULES)) + preallocate_pgd_pages_range(MODULES_VADDR, MODULES_END, "bpf/modules"); +} +#endif diff --git a/arch/s390/boot/uv.c b/arch/s390/boot/uv.c index 0a077c0a2056..1e66d2cbb096 100644 --- a/arch/s390/boot/uv.c +++ b/arch/s390/boot/uv.c @@ -47,6 +47,10 @@ void uv_query_info(void) uv_info.conf_dump_finalize_len = uvcb.conf_dump_finalize_len; uv_info.supp_att_req_hdr_ver = uvcb.supp_att_req_hdr_ver; uv_info.supp_att_pflags = uvcb.supp_att_pflags; + uv_info.supp_add_secret_req_ver = uvcb.supp_add_secret_req_ver; + uv_info.supp_add_secret_pcf = uvcb.supp_add_secret_pcf; + uv_info.supp_secret_types = uvcb.supp_secret_types; + uv_info.max_secrets = uvcb.max_secrets; } #ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST diff --git a/arch/s390/include/asm/uv.h b/arch/s390/include/asm/uv.h index 28a9ad57b6f1..d6bb2f4f78d1 100644 --- a/arch/s390/include/asm/uv.h +++ b/arch/s390/include/asm/uv.h @@ -58,6 +58,9 @@ #define UVC_CMD_SET_SHARED_ACCESS 0x1000 #define UVC_CMD_REMOVE_SHARED_ACCESS 0x1001 #define UVC_CMD_RETR_ATTEST 0x1020 +#define UVC_CMD_ADD_SECRET 0x1031 +#define UVC_CMD_LIST_SECRETS 0x1033 +#define UVC_CMD_LOCK_SECRETS 0x1034 /* Bits in installed uv calls */ enum uv_cmds_inst { @@ -88,6 +91,9 @@ enum uv_cmds_inst { BIT_UVC_CMD_DUMP_CPU = 26, BIT_UVC_CMD_DUMP_COMPLETE = 27, BIT_UVC_CMD_RETR_ATTEST = 28, + BIT_UVC_CMD_ADD_SECRET = 29, + BIT_UVC_CMD_LIST_SECRETS = 30, + BIT_UVC_CMD_LOCK_SECRETS = 31, }; enum uv_feat_ind { @@ -117,7 +123,7 @@ struct uv_cb_qui { u32 reserved70[3]; /* 0x0070 */ u32 max_num_sec_conf; /* 0x007c */ u64 max_guest_stor_addr; /* 0x0080 */ - u8 reserved88[158 - 136]; /* 0x0088 */ + u8 reserved88[0x9e - 0x88]; /* 0x0088 */ u16 max_guest_cpu_id; /* 0x009e */ u64 uv_feature_indications; /* 0x00a0 */ u64 reserveda8; /* 0x00a8 */ @@ -129,7 +135,12 @@ struct uv_cb_qui { u64 reservedd8; /* 0x00d8 */ u64 supp_att_req_hdr_ver; /* 0x00e0 */ u64 supp_att_pflags; /* 0x00e8 */ - u8 reservedf0[256 - 240]; /* 0x00f0 */ + u64 reservedf0; /* 0x00f0 */ + u64 supp_add_secret_req_ver; /* 0x00f8 */ + u64 supp_add_secret_pcf; /* 0x0100 */ + u64 supp_secret_types; /* 0x0180 */ + u16 max_secrets; /* 0x0110 */ + u8 reserved112[0x120 - 0x112]; /* 0x0112 */ } __packed __aligned(8); /* Initialize Ultravisor */ @@ -292,6 +303,19 @@ struct uv_cb_dump_complete { u64 reserved30[5]; } __packed __aligned(8); +/* + * A common UV call struct for pv guests that contains a single address + * Examples: + * Add Secret + * List Secrets + */ +struct uv_cb_guest_addr { + struct uv_cb_header header; + u64 reserved08[3]; + u64 addr; + u64 reserved28[4]; +} __packed __aligned(8); + static inline int __uv_call(unsigned long r1, unsigned long r2) { int cc; @@ -365,6 +389,10 @@ struct uv_info { unsigned long conf_dump_finalize_len; unsigned long supp_att_req_hdr_ver; unsigned long supp_att_pflags; + unsigned long supp_add_secret_req_ver; + unsigned long supp_add_secret_pcf; + unsigned long supp_secret_types; + unsigned short max_secrets; }; extern struct uv_info uv_info; diff --git a/arch/s390/include/uapi/asm/uvdevice.h b/arch/s390/include/uapi/asm/uvdevice.h index 10a5ac918e02..b9c2f14a6af3 100644 --- a/arch/s390/include/uapi/asm/uvdevice.h +++ b/arch/s390/include/uapi/asm/uvdevice.h @@ -32,6 +32,33 @@ struct uvio_attest { __u16 reserved136; /* 0x0136 */ }; +/** + * uvio_uvdev_info - Information of supported functions + * @supp_uvio_cmds - supported IOCTLs by this device + * @supp_uv_cmds - supported UVCs corresponding to the IOCTL + * + * UVIO request to get information about supported request types by this + * uvdevice and the Ultravisor. Everything is output. Bits are in LSB0 + * ordering. If the bit is set in both, @supp_uvio_cmds and @supp_uv_cmds, the + * uvdevice and the Ultravisor support that call. + * + * Note that bit 0 (UVIO_IOCTL_UVDEV_INFO_NR) is always zero for `supp_uv_cmds` + * as there is no corresponding UV-call. + */ +struct uvio_uvdev_info { + /* + * If bit `n` is set, this device supports the IOCTL with nr `n`. + */ + __u64 supp_uvio_cmds; + /* + * If bit `n` is set, the Ultravisor(UV) supports the UV-call + * corresponding to the IOCTL with nr `n` in the calling contextx (host + * or guest). The value is only valid if the corresponding bit in + * @supp_uvio_cmds is set as well. + */ + __u64 supp_uv_cmds; +}; + /* * The following max values define an upper length for the IOCTL in/out buffers. * However, they do not represent the maximum the Ultravisor allows which is @@ -42,10 +69,34 @@ struct uvio_attest { #define UVIO_ATT_ARCB_MAX_LEN 0x100000 #define UVIO_ATT_MEASUREMENT_MAX_LEN 0x8000 #define UVIO_ATT_ADDITIONAL_MAX_LEN 0x8000 +#define UVIO_ADD_SECRET_MAX_LEN 0x100000 +#define UVIO_LIST_SECRETS_LEN 0x1000 #define UVIO_DEVICE_NAME "uv" #define UVIO_TYPE_UVC 'u' -#define UVIO_IOCTL_ATT _IOWR(UVIO_TYPE_UVC, 0x01, struct uvio_ioctl_cb) +enum UVIO_IOCTL_NR { + UVIO_IOCTL_UVDEV_INFO_NR = 0x00, + UVIO_IOCTL_ATT_NR, + UVIO_IOCTL_ADD_SECRET_NR, + UVIO_IOCTL_LIST_SECRETS_NR, + UVIO_IOCTL_LOCK_SECRETS_NR, + /* must be the last entry */ + UVIO_IOCTL_NUM_IOCTLS +}; + +#define UVIO_IOCTL(nr) _IOWR(UVIO_TYPE_UVC, nr, struct uvio_ioctl_cb) +#define UVIO_IOCTL_UVDEV_INFO UVIO_IOCTL(UVIO_IOCTL_UVDEV_INFO_NR) +#define UVIO_IOCTL_ATT UVIO_IOCTL(UVIO_IOCTL_ATT_NR) +#define UVIO_IOCTL_ADD_SECRET UVIO_IOCTL(UVIO_IOCTL_ADD_SECRET_NR) +#define UVIO_IOCTL_LIST_SECRETS UVIO_IOCTL(UVIO_IOCTL_LIST_SECRETS_NR) +#define UVIO_IOCTL_LOCK_SECRETS UVIO_IOCTL(UVIO_IOCTL_LOCK_SECRETS_NR) + +#define UVIO_SUPP_CALL(nr) (1ULL << (nr)) +#define UVIO_SUPP_UDEV_INFO UVIO_SUPP_CALL(UVIO_IOCTL_UDEV_INFO_NR) +#define UVIO_SUPP_ATT UVIO_SUPP_CALL(UVIO_IOCTL_ATT_NR) +#define UVIO_SUPP_ADD_SECRET UVIO_SUPP_CALL(UVIO_IOCTL_ADD_SECRET_NR) +#define UVIO_SUPP_LIST_SECRETS UVIO_SUPP_CALL(UVIO_IOCTL_LIST_SECRETS_NR) +#define UVIO_SUPP_LOCK_SECRETS UVIO_SUPP_CALL(UVIO_IOCTL_LOCK_SECRETS_NR) #endif /* __S390_ASM_UVDEVICE_H */ diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c index 3c62d1b218b1..66f0eb1c872b 100644 --- a/arch/s390/kernel/uv.c +++ b/arch/s390/kernel/uv.c @@ -23,12 +23,20 @@ int __bootdata_preserved(prot_virt_guest); #endif +/* + * uv_info contains both host and guest information but it's currently only + * expected to be used within modules if it's the KVM module or for + * any PV guest module. + * + * The kernel itself will write these values once in uv_query_info() + * and then make some of them readable via a sysfs interface. + */ struct uv_info __bootdata_preserved(uv_info); +EXPORT_SYMBOL(uv_info); #if IS_ENABLED(CONFIG_KVM) int __bootdata_preserved(prot_virt_host); EXPORT_SYMBOL(prot_virt_host); -EXPORT_SYMBOL(uv_info); static int __init uv_init(phys_addr_t stor_base, unsigned long stor_len) { @@ -462,13 +470,13 @@ EXPORT_SYMBOL_GPL(arch_make_page_accessible); #if defined(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) || IS_ENABLED(CONFIG_KVM) static ssize_t uv_query_facilities(struct kobject *kobj, - struct kobj_attribute *attr, char *page) + struct kobj_attribute *attr, char *buf) { - return scnprintf(page, PAGE_SIZE, "%lx\n%lx\n%lx\n%lx\n", - uv_info.inst_calls_list[0], - uv_info.inst_calls_list[1], - uv_info.inst_calls_list[2], - uv_info.inst_calls_list[3]); + return sysfs_emit(buf, "%lx\n%lx\n%lx\n%lx\n", + uv_info.inst_calls_list[0], + uv_info.inst_calls_list[1], + uv_info.inst_calls_list[2], + uv_info.inst_calls_list[3]); } static struct kobj_attribute uv_query_facilities_attr = @@ -493,30 +501,27 @@ static struct kobj_attribute uv_query_supp_se_hdr_pcf_attr = __ATTR(supp_se_hdr_pcf, 0444, uv_query_supp_se_hdr_pcf, NULL); static ssize_t uv_query_dump_cpu_len(struct kobject *kobj, - struct kobj_attribute *attr, char *page) + struct kobj_attribute *attr, char *buf) { - return scnprintf(page, PAGE_SIZE, "%lx\n", - uv_info.guest_cpu_stor_len); + return sysfs_emit(buf, "%lx\n", uv_info.guest_cpu_stor_len); } static struct kobj_attribute uv_query_dump_cpu_len_attr = __ATTR(uv_query_dump_cpu_len, 0444, uv_query_dump_cpu_len, NULL); static ssize_t uv_query_dump_storage_state_len(struct kobject *kobj, - struct kobj_attribute *attr, char *page) + struct kobj_attribute *attr, char *buf) { - return scnprintf(page, PAGE_SIZE, "%lx\n", - uv_info.conf_dump_storage_state_len); + return sysfs_emit(buf, "%lx\n", uv_info.conf_dump_storage_state_len); } static struct kobj_attribute uv_query_dump_storage_state_len_attr = __ATTR(dump_storage_state_len, 0444, uv_query_dump_storage_state_len, NULL); static ssize_t uv_query_dump_finalize_len(struct kobject *kobj, - struct kobj_attribute *attr, char *page) + struct kobj_attribute *attr, char *buf) { - return scnprintf(page, PAGE_SIZE, "%lx\n", - uv_info.conf_dump_finalize_len); + return sysfs_emit(buf, "%lx\n", uv_info.conf_dump_finalize_len); } static struct kobj_attribute uv_query_dump_finalize_len_attr = @@ -532,53 +537,86 @@ static struct kobj_attribute uv_query_feature_indications_attr = __ATTR(feature_indications, 0444, uv_query_feature_indications, NULL); static ssize_t uv_query_max_guest_cpus(struct kobject *kobj, - struct kobj_attribute *attr, char *page) + struct kobj_attribute *attr, char *buf) { - return scnprintf(page, PAGE_SIZE, "%d\n", - uv_info.max_guest_cpu_id + 1); + return sysfs_emit(buf, "%d\n", uv_info.max_guest_cpu_id + 1); } static struct kobj_attribute uv_query_max_guest_cpus_attr = __ATTR(max_cpus, 0444, uv_query_max_guest_cpus, NULL); static ssize_t uv_query_max_guest_vms(struct kobject *kobj, - struct kobj_attribute *attr, char *page) + struct kobj_attribute *attr, char *buf) { - return scnprintf(page, PAGE_SIZE, "%d\n", - uv_info.max_num_sec_conf); + return sysfs_emit(buf, "%d\n", uv_info.max_num_sec_conf); } static struct kobj_attribute uv_query_max_guest_vms_attr = __ATTR(max_guests, 0444, uv_query_max_guest_vms, NULL); static ssize_t uv_query_max_guest_addr(struct kobject *kobj, - struct kobj_attribute *attr, char *page) + struct kobj_attribute *attr, char *buf) { - return scnprintf(page, PAGE_SIZE, "%lx\n", - uv_info.max_sec_stor_addr); + return sysfs_emit(buf, "%lx\n", uv_info.max_sec_stor_addr); } static struct kobj_attribute uv_query_max_guest_addr_attr = __ATTR(max_address, 0444, uv_query_max_guest_addr, NULL); static ssize_t uv_query_supp_att_req_hdr_ver(struct kobject *kobj, - struct kobj_attribute *attr, char *page) + struct kobj_attribute *attr, char *buf) { - return scnprintf(page, PAGE_SIZE, "%lx\n", uv_info.supp_att_req_hdr_ver); + return sysfs_emit(buf, "%lx\n", uv_info.supp_att_req_hdr_ver); } static struct kobj_attribute uv_query_supp_att_req_hdr_ver_attr = __ATTR(supp_att_req_hdr_ver, 0444, uv_query_supp_att_req_hdr_ver, NULL); static ssize_t uv_query_supp_att_pflags(struct kobject *kobj, - struct kobj_attribute *attr, char *page) + struct kobj_attribute *attr, char *buf) { - return scnprintf(page, PAGE_SIZE, "%lx\n", uv_info.supp_att_pflags); + return sysfs_emit(buf, "%lx\n", uv_info.supp_att_pflags); } static struct kobj_attribute uv_query_supp_att_pflags_attr = __ATTR(supp_att_pflags, 0444, uv_query_supp_att_pflags, NULL); +static ssize_t uv_query_supp_add_secret_req_ver(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%lx\n", uv_info.supp_add_secret_req_ver); +} + +static struct kobj_attribute uv_query_supp_add_secret_req_ver_attr = + __ATTR(supp_add_secret_req_ver, 0444, uv_query_supp_add_secret_req_ver, NULL); + +static ssize_t uv_query_supp_add_secret_pcf(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%lx\n", uv_info.supp_add_secret_pcf); +} + +static struct kobj_attribute uv_query_supp_add_secret_pcf_attr = + __ATTR(supp_add_secret_pcf, 0444, uv_query_supp_add_secret_pcf, NULL); + +static ssize_t uv_query_supp_secret_types(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%lx\n", uv_info.supp_secret_types); +} + +static struct kobj_attribute uv_query_supp_secret_types_attr = + __ATTR(supp_secret_types, 0444, uv_query_supp_secret_types, NULL); + +static ssize_t uv_query_max_secrets(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%d\n", uv_info.max_secrets); +} + +static struct kobj_attribute uv_query_max_secrets_attr = + __ATTR(max_secrets, 0444, uv_query_max_secrets, NULL); + static struct attribute *uv_query_attrs[] = { &uv_query_facilities_attr.attr, &uv_query_feature_indications_attr.attr, @@ -592,6 +630,10 @@ static struct attribute *uv_query_attrs[] = { &uv_query_dump_cpu_len_attr.attr, &uv_query_supp_att_req_hdr_ver_attr.attr, &uv_query_supp_att_pflags_attr.attr, + &uv_query_supp_add_secret_req_ver_attr.attr, + &uv_query_supp_add_secret_pcf_attr.attr, + &uv_query_supp_secret_types_attr.attr, + &uv_query_max_secrets_attr.attr, NULL, }; @@ -600,18 +642,18 @@ static struct attribute_group uv_query_attr_group = { }; static ssize_t uv_is_prot_virt_guest(struct kobject *kobj, - struct kobj_attribute *attr, char *page) + struct kobj_attribute *attr, char *buf) { int val = 0; #ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST val = prot_virt_guest; #endif - return scnprintf(page, PAGE_SIZE, "%d\n", val); + return sysfs_emit(buf, "%d\n", val); } static ssize_t uv_is_prot_virt_host(struct kobject *kobj, - struct kobj_attribute *attr, char *page) + struct kobj_attribute *attr, char *buf) { int val = 0; @@ -619,7 +661,7 @@ static ssize_t uv_is_prot_virt_host(struct kobject *kobj, val = prot_virt_host; #endif - return scnprintf(page, PAGE_SIZE, "%d\n", val); + return sysfs_emit(buf, "%d\n", val); } static struct kobj_attribute uv_prot_virt_guest = diff --git a/arch/s390/kvm/diag.c b/arch/s390/kvm/diag.c index 807fa9da1e72..3c65b8258ae6 100644 --- a/arch/s390/kvm/diag.c +++ b/arch/s390/kvm/diag.c @@ -166,6 +166,7 @@ static int diag9c_forwarding_overrun(void) static int __diag_time_slice_end_directed(struct kvm_vcpu *vcpu) { struct kvm_vcpu *tcpu; + int tcpu_cpu; int tid; tid = vcpu->run->s.regs.gprs[(vcpu->arch.sie_block->ipa & 0xf0) >> 4]; @@ -181,14 +182,15 @@ static int __diag_time_slice_end_directed(struct kvm_vcpu *vcpu) goto no_yield; /* target guest VCPU already running */ - if (READ_ONCE(tcpu->cpu) >= 0) { + tcpu_cpu = READ_ONCE(tcpu->cpu); + if (tcpu_cpu >= 0) { if (!diag9c_forwarding_hz || diag9c_forwarding_overrun()) goto no_yield; /* target host CPU already running */ - if (!vcpu_is_preempted(tcpu->cpu)) + if (!vcpu_is_preempted(tcpu_cpu)) goto no_yield; - smp_yield_cpu(tcpu->cpu); + smp_yield_cpu(tcpu_cpu); VCPU_EVENT(vcpu, 5, "diag time slice end directed to %d: yield forwarded", tid); diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 17b81659cdb2..670019696464 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -2156,6 +2156,10 @@ static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots, ms = container_of(mnode, struct kvm_memory_slot, gfn_node[slots->node_idx]); ofs = 0; } + + if (cur_gfn < ms->base_gfn) + ofs = 0; + ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, ofs); while (ofs >= ms->npages && (mnode = rb_next(mnode))) { ms = container_of(mnode, struct kvm_memory_slot, gfn_node[slots->node_idx]); diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index 8d6b765abf29..0333ee482eb8 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -177,7 +177,8 @@ static int setup_apcb00(struct kvm_vcpu *vcpu, unsigned long *apcb_s, sizeof(struct kvm_s390_apcb0))) return -EFAULT; - bitmap_and(apcb_s, apcb_s, apcb_h, sizeof(struct kvm_s390_apcb0)); + bitmap_and(apcb_s, apcb_s, apcb_h, + BITS_PER_BYTE * sizeof(struct kvm_s390_apcb0)); return 0; } @@ -203,7 +204,8 @@ static int setup_apcb11(struct kvm_vcpu *vcpu, unsigned long *apcb_s, sizeof(struct kvm_s390_apcb1))) return -EFAULT; - bitmap_and(apcb_s, apcb_s, apcb_h, sizeof(struct kvm_s390_apcb1)); + bitmap_and(apcb_s, apcb_s, apcb_h, + BITS_PER_BYTE * sizeof(struct kvm_s390_apcb1)); return 0; } diff --git a/arch/sh/Kbuild b/arch/sh/Kbuild index be171880977e..056efec72c2a 100644 --- a/arch/sh/Kbuild +++ b/arch/sh/Kbuild @@ -3,5 +3,7 @@ obj-y += kernel/ mm/ boards/ obj-$(CONFIG_SH_FPU_EMU) += math-emu/ obj-$(CONFIG_USE_BUILTIN_DTB) += boot/dts/ +obj-$(CONFIG_HD6446X_SERIES) += cchips/hd6446x/ + # for cleaning subdir- += boot diff --git a/arch/sh/Makefile b/arch/sh/Makefile index 5c8776482530..cab2f9c011a8 100644 --- a/arch/sh/Makefile +++ b/arch/sh/Makefile @@ -116,34 +116,15 @@ export ld-bfd # Mach groups machdir-$(CONFIG_SOLUTION_ENGINE) += mach-se -machdir-$(CONFIG_SH_HP6XX) += mach-hp6xx machdir-$(CONFIG_SH_DREAMCAST) += mach-dreamcast machdir-$(CONFIG_SH_SH03) += mach-sh03 -machdir-$(CONFIG_SH_RTS7751R2D) += mach-r2d -machdir-$(CONFIG_SH_HIGHLANDER) += mach-highlander machdir-$(CONFIG_SH_MIGOR) += mach-migor -machdir-$(CONFIG_SH_AP325RXA) += mach-ap325rxa machdir-$(CONFIG_SH_KFR2R09) += mach-kfr2r09 machdir-$(CONFIG_SH_ECOVEC) += mach-ecovec24 -machdir-$(CONFIG_SH_SDK7780) += mach-sdk7780 machdir-$(CONFIG_SH_SDK7786) += mach-sdk7786 machdir-$(CONFIG_SH_X3PROTO) += mach-x3proto -machdir-$(CONFIG_SH_SH7763RDP) += mach-sh7763rdp -machdir-$(CONFIG_SH_SH4202_MICRODEV) += mach-microdev machdir-$(CONFIG_SH_LANDISK) += mach-landisk -machdir-$(CONFIG_SH_LBOX_RE2) += mach-lboxre2 -machdir-$(CONFIG_SH_RSK) += mach-rsk - -ifneq ($(machdir-y),) -core-y += $(addprefix arch/sh/boards/, \ - $(filter-out ., $(patsubst %,%/,$(machdir-y)))) -endif - -# Common machine type headers. Not part of the arch/sh/boards/ hierarchy. -machdir-y += mach-common - -# Companion chips -core-$(CONFIG_HD6446X_SERIES) += arch/sh/cchips/hd6446x/ +machdir-y += mach-common # # CPU header paths @@ -164,11 +145,8 @@ cpuincdir-y += cpu-common # Must be last drivers-y += arch/sh/drivers/ -cflags-y += $(foreach d, $(cpuincdir-y), -I $(srctree)/arch/sh/include/$(d)) \ - $(foreach d, $(machdir-y), -I $(srctree)/arch/sh/include/$(d)) - +KBUILD_CPPFLAGS += $(addprefix -I $(srctree)/arch/sh/include/, $(cpuincdir-y) $(machdir-y)) KBUILD_CFLAGS += -pipe $(cflags-y) -KBUILD_CPPFLAGS += $(cflags-y) KBUILD_AFLAGS += $(cflags-y) ifeq ($(CONFIG_MCOUNT),y) diff --git a/arch/sh/boards/Makefile b/arch/sh/boards/Makefile index 4002a22a7c40..b57219436ace 100644 --- a/arch/sh/boards/Makefile +++ b/arch/sh/boards/Makefile @@ -18,3 +18,22 @@ obj-$(CONFIG_SH_APSH4A3A) += board-apsh4a3a.o obj-$(CONFIG_SH_APSH4AD0A) += board-apsh4ad0a.o obj-$(CONFIG_SH_DEVICE_TREE) += of-generic.o + +obj-$(CONFIG_SOLUTION_ENGINE) += mach-se/ +obj-$(CONFIG_SH_HP6XX) += mach-hp6xx/ +obj-$(CONFIG_SH_DREAMCAST) += mach-dreamcast/ +obj-$(CONFIG_SH_SH03) += mach-sh03/ +obj-$(CONFIG_SH_RTS7751R2D) += mach-r2d/ +obj-$(CONFIG_SH_HIGHLANDER) += mach-highlander/ +obj-$(CONFIG_SH_MIGOR) += mach-migor/ +obj-$(CONFIG_SH_AP325RXA) += mach-ap325rxa/ +obj-$(CONFIG_SH_KFR2R09) += mach-kfr2r09/ +obj-$(CONFIG_SH_ECOVEC) += mach-ecovec24/ +obj-$(CONFIG_SH_SDK7780) += mach-sdk7780/ +obj-$(CONFIG_SH_SDK7786) += mach-sdk7786/ +obj-$(CONFIG_SH_X3PROTO) += mach-x3proto/ +obj-$(CONFIG_SH_SH7763RDP) += mach-sh7763rdp/ +obj-$(CONFIG_SH_SH4202_MICRODEV)+= mach-microdev/ +obj-$(CONFIG_SH_LANDISK) += mach-landisk/ +obj-$(CONFIG_SH_LBOX_RE2) += mach-lboxre2/ +obj-$(CONFIG_SH_RSK) += mach-rsk/ diff --git a/arch/sh/configs/ap325rxa_defconfig b/arch/sh/configs/ap325rxa_defconfig index 4d83576b89c6..4464a2ad42ed 100644 --- a/arch/sh/configs/ap325rxa_defconfig +++ b/arch/sh/configs/ap325rxa_defconfig @@ -3,7 +3,6 @@ CONFIG_SYSVIPC=y CONFIG_BSD_PROCESS_ACCT=y CONFIG_LOG_BUF_SHIFT=14 # CONFIG_KALLSYMS is not set -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/sh/configs/apsh4a3a_defconfig b/arch/sh/configs/apsh4a3a_defconfig index 99931a13a74d..cc909f347877 100644 --- a/arch/sh/configs/apsh4a3a_defconfig +++ b/arch/sh/configs/apsh4a3a_defconfig @@ -6,7 +6,6 @@ CONFIG_LOG_BUF_SHIFT=14 CONFIG_SYSFS_DEPRECATED=y CONFIG_SYSFS_DEPRECATED_V2=y CONFIG_BLK_DEV_INITRD=y -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y diff --git a/arch/sh/configs/apsh4ad0a_defconfig b/arch/sh/configs/apsh4ad0a_defconfig index d9fb124bf015..64558bf60e10 100644 --- a/arch/sh/configs/apsh4ad0a_defconfig +++ b/arch/sh/configs/apsh4ad0a_defconfig @@ -15,7 +15,6 @@ CONFIG_NAMESPACES=y CONFIG_BLK_DEV_INITRD=y CONFIG_KALLSYMS_ALL=y # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y diff --git a/arch/sh/configs/dreamcast_defconfig b/arch/sh/configs/dreamcast_defconfig index 6a82c7b8ff32..0c9f2030bb7c 100644 --- a/arch/sh/configs/dreamcast_defconfig +++ b/arch/sh/configs/dreamcast_defconfig @@ -1,7 +1,6 @@ CONFIG_SYSVIPC=y CONFIG_BSD_PROCESS_ACCT=y CONFIG_LOG_BUF_SHIFT=14 -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y diff --git a/arch/sh/configs/ecovec24-romimage_defconfig b/arch/sh/configs/ecovec24-romimage_defconfig index 5c60e71d839e..13fec4338416 100644 --- a/arch/sh/configs/ecovec24-romimage_defconfig +++ b/arch/sh/configs/ecovec24-romimage_defconfig @@ -6,7 +6,6 @@ CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y # CONFIG_KALLSYMS is not set -CONFIG_SLAB=y # CONFIG_BLK_DEV_BSG is not set CONFIG_CPU_SUBTYPE_SH7724=y CONFIG_MEMORY_SIZE=0x10000000 diff --git a/arch/sh/configs/ecovec24_defconfig b/arch/sh/configs/ecovec24_defconfig index 4d655e8d4d74..ee1b36682155 100644 --- a/arch/sh/configs/ecovec24_defconfig +++ b/arch/sh/configs/ecovec24_defconfig @@ -3,7 +3,6 @@ CONFIG_SYSVIPC=y CONFIG_BSD_PROCESS_ACCT=y CONFIG_LOG_BUF_SHIFT=14 # CONFIG_KALLSYMS is not set -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/sh/configs/espt_defconfig b/arch/sh/configs/espt_defconfig index 2804cb760a76..623012d9af9d 100644 --- a/arch/sh/configs/espt_defconfig +++ b/arch/sh/configs/espt_defconfig @@ -5,7 +5,6 @@ CONFIG_LOG_BUF_SHIFT=14 CONFIG_NAMESPACES=y CONFIG_UTS_NS=y CONFIG_IPC_NS=y -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_MODULES=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/sh/configs/hp6xx_defconfig b/arch/sh/configs/hp6xx_defconfig index 70e6605d7f7e..0c45f2a0f9bd 100644 --- a/arch/sh/configs/hp6xx_defconfig +++ b/arch/sh/configs/hp6xx_defconfig @@ -3,7 +3,6 @@ CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=14 # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set -CONFIG_SLAB=y # CONFIG_BLK_DEV_BSG is not set CONFIG_CPU_SUBTYPE_SH7709=y CONFIG_MEMORY_START=0x0d000000 diff --git a/arch/sh/configs/kfr2r09-romimage_defconfig b/arch/sh/configs/kfr2r09-romimage_defconfig index 04436b4fbd76..42bf34181a3e 100644 --- a/arch/sh/configs/kfr2r09-romimage_defconfig +++ b/arch/sh/configs/kfr2r09-romimage_defconfig @@ -6,7 +6,6 @@ CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y # CONFIG_KALLSYMS is not set -CONFIG_SLAB=y # CONFIG_BLOCK is not set CONFIG_CPU_SUBTYPE_SH7724=y CONFIG_MEMORY_SIZE=0x08000000 diff --git a/arch/sh/configs/kfr2r09_defconfig b/arch/sh/configs/kfr2r09_defconfig index 833404490cfe..d80e83e7ec38 100644 --- a/arch/sh/configs/kfr2r09_defconfig +++ b/arch/sh/configs/kfr2r09_defconfig @@ -6,7 +6,6 @@ CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y # CONFIG_KALLSYMS is not set -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/sh/configs/landisk_defconfig b/arch/sh/configs/landisk_defconfig index 7037320b654a..541082090918 100644 --- a/arch/sh/configs/landisk_defconfig +++ b/arch/sh/configs/landisk_defconfig @@ -1,7 +1,6 @@ CONFIG_SYSVIPC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_KALLSYMS_EXTRA_PASS=y -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y diff --git a/arch/sh/configs/lboxre2_defconfig b/arch/sh/configs/lboxre2_defconfig index 05e4ac6fed5f..6a234761bfd7 100644 --- a/arch/sh/configs/lboxre2_defconfig +++ b/arch/sh/configs/lboxre2_defconfig @@ -1,7 +1,6 @@ CONFIG_SYSVIPC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_KALLSYMS_EXTRA_PASS=y -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y diff --git a/arch/sh/configs/magicpanelr2_defconfig b/arch/sh/configs/magicpanelr2_defconfig index ef1d98e35c91..52937f9cc2ab 100644 --- a/arch/sh/configs/magicpanelr2_defconfig +++ b/arch/sh/configs/magicpanelr2_defconfig @@ -8,7 +8,6 @@ CONFIG_RELAY=y CONFIG_BLK_DEV_INITRD=y # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set CONFIG_KALLSYMS_ALL=y -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODVERSIONS=y diff --git a/arch/sh/configs/microdev_defconfig b/arch/sh/configs/microdev_defconfig index e9825196dd66..39a83d832cac 100644 --- a/arch/sh/configs/microdev_defconfig +++ b/arch/sh/configs/microdev_defconfig @@ -2,7 +2,6 @@ CONFIG_BSD_PROCESS_ACCT=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set -CONFIG_SLAB=y # CONFIG_BLK_DEV_BSG is not set CONFIG_CPU_SUBTYPE_SH4_202=y CONFIG_FLATMEM_MANUAL=y diff --git a/arch/sh/configs/migor_defconfig b/arch/sh/configs/migor_defconfig index 4859cd30cfc4..2d1e65cad239 100644 --- a/arch/sh/configs/migor_defconfig +++ b/arch/sh/configs/migor_defconfig @@ -4,7 +4,6 @@ CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_MODULES=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/sh/configs/polaris_defconfig b/arch/sh/configs/polaris_defconfig index f42e4867ddc1..4f0396c2ac79 100644 --- a/arch/sh/configs/polaris_defconfig +++ b/arch/sh/configs/polaris_defconfig @@ -7,7 +7,6 @@ CONFIG_BSD_PROCESS_ACCT_V3=y CONFIG_AUDIT=y CONFIG_LOG_BUF_SHIFT=14 # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODVERSIONS=y diff --git a/arch/sh/configs/r7780mp_defconfig b/arch/sh/configs/r7780mp_defconfig index e527cd60a191..6bd6c0ae85d7 100644 --- a/arch/sh/configs/r7780mp_defconfig +++ b/arch/sh/configs/r7780mp_defconfig @@ -5,7 +5,6 @@ CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=14 # CONFIG_FUTEX is not set # CONFIG_EPOLL is not set -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y diff --git a/arch/sh/configs/r7785rp_defconfig b/arch/sh/configs/r7785rp_defconfig index a3f952a83d97..cde668569cc1 100644 --- a/arch/sh/configs/r7785rp_defconfig +++ b/arch/sh/configs/r7785rp_defconfig @@ -7,7 +7,6 @@ CONFIG_RCU_TRACE=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=14 -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_KPROBES=y CONFIG_MODULES=y diff --git a/arch/sh/configs/rsk7264_defconfig b/arch/sh/configs/rsk7264_defconfig index f7b9c528c6df..de9e350cdf28 100644 --- a/arch/sh/configs/rsk7264_defconfig +++ b/arch/sh/configs/rsk7264_defconfig @@ -12,7 +12,6 @@ CONFIG_KALLSYMS_ALL=y CONFIG_EMBEDDED=y CONFIG_PERF_COUNTERS=y # CONFIG_VM_EVENT_COUNTERS is not set -CONFIG_SLAB=y CONFIG_MMAP_ALLOW_UNINITIALIZED=y CONFIG_PROFILING=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/sh/configs/rsk7269_defconfig b/arch/sh/configs/rsk7269_defconfig index 4bff14fb185d..7ca6a5c25a9f 100644 --- a/arch/sh/configs/rsk7269_defconfig +++ b/arch/sh/configs/rsk7269_defconfig @@ -2,7 +2,6 @@ CONFIG_LOG_BUF_SHIFT=14 CONFIG_CC_OPTIMIZE_FOR_SIZE=y CONFIG_EMBEDDED=y # CONFIG_VM_EVENT_COUNTERS is not set -CONFIG_SLAB=y # CONFIG_BLK_DEV_BSG is not set CONFIG_SWAP_IO_SPACE=y CONFIG_CPU_SUBTYPE_SH7269=y diff --git a/arch/sh/configs/rts7751r2d1_defconfig b/arch/sh/configs/rts7751r2d1_defconfig index 96263a4912b7..c863a11c7592 100644 --- a/arch/sh/configs/rts7751r2d1_defconfig +++ b/arch/sh/configs/rts7751r2d1_defconfig @@ -1,7 +1,6 @@ CONFIG_SYSVIPC=y CONFIG_LOG_BUF_SHIFT=14 # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_MODULES=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/sh/configs/rts7751r2dplus_defconfig b/arch/sh/configs/rts7751r2dplus_defconfig index 92e586e6c974..7e4f710d46c7 100644 --- a/arch/sh/configs/rts7751r2dplus_defconfig +++ b/arch/sh/configs/rts7751r2dplus_defconfig @@ -1,7 +1,6 @@ CONFIG_SYSVIPC=y CONFIG_LOG_BUF_SHIFT=14 # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_MODULES=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/sh/configs/sdk7786_defconfig b/arch/sh/configs/sdk7786_defconfig index 97b7356639ed..a080c5d05a7b 100644 --- a/arch/sh/configs/sdk7786_defconfig +++ b/arch/sh/configs/sdk7786_defconfig @@ -31,7 +31,6 @@ CONFIG_RD_BZIP2=y CONFIG_RD_LZMA=y CONFIG_RD_LZO=y # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_KPROBES=y CONFIG_MODULES=y diff --git a/arch/sh/configs/se7343_defconfig b/arch/sh/configs/se7343_defconfig index 5d6c19338ebf..75db12fb9ad1 100644 --- a/arch/sh/configs/se7343_defconfig +++ b/arch/sh/configs/se7343_defconfig @@ -5,7 +5,6 @@ CONFIG_LOG_BUF_SHIFT=14 # CONFIG_FUTEX is not set # CONFIG_EPOLL is not set # CONFIG_SHMEM is not set -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y diff --git a/arch/sh/configs/se7619_defconfig b/arch/sh/configs/se7619_defconfig index 71a672c30716..14d0f5ead502 100644 --- a/arch/sh/configs/se7619_defconfig +++ b/arch/sh/configs/se7619_defconfig @@ -8,7 +8,6 @@ CONFIG_LOG_BUF_SHIFT=14 # CONFIG_FUTEX is not set # CONFIG_EPOLL is not set # CONFIG_VM_EVENT_COUNTERS is not set -CONFIG_SLAB=y # CONFIG_BLK_DEV_BSG is not set CONFIG_MEMORY_START=0x0c000000 CONFIG_FLATMEM_MANUAL=y diff --git a/arch/sh/configs/se7705_defconfig b/arch/sh/configs/se7705_defconfig index ed00a6eeadf5..16a0f72f0822 100644 --- a/arch/sh/configs/se7705_defconfig +++ b/arch/sh/configs/se7705_defconfig @@ -4,7 +4,6 @@ CONFIG_BLK_DEV_INITRD=y # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set # CONFIG_KALLSYMS is not set # CONFIG_HOTPLUG is not set -CONFIG_SLAB=y CONFIG_MODULES=y # CONFIG_BLK_DEV_BSG is not set CONFIG_CPU_SUBTYPE_SH7705=y diff --git a/arch/sh/configs/se7712_defconfig b/arch/sh/configs/se7712_defconfig index 36356223d51c..dc854293da43 100644 --- a/arch/sh/configs/se7712_defconfig +++ b/arch/sh/configs/se7712_defconfig @@ -9,7 +9,6 @@ CONFIG_KALLSYMS_ALL=y # CONFIG_BUG is not set # CONFIG_BASE_FULL is not set # CONFIG_SHMEM is not set -CONFIG_SLAB=y CONFIG_MODULES=y # CONFIG_BLK_DEV_BSG is not set CONFIG_CPU_SUBTYPE_SH7712=y diff --git a/arch/sh/configs/se7721_defconfig b/arch/sh/configs/se7721_defconfig index 46c5a263a239..c891945b8a90 100644 --- a/arch/sh/configs/se7721_defconfig +++ b/arch/sh/configs/se7721_defconfig @@ -9,7 +9,6 @@ CONFIG_KALLSYMS_ALL=y # CONFIG_BUG is not set # CONFIG_BASE_FULL is not set # CONFIG_SHMEM is not set -CONFIG_SLAB=y CONFIG_MODULES=y # CONFIG_BLK_DEV_BSG is not set CONFIG_CPU_SUBTYPE_SH7721=y diff --git a/arch/sh/configs/se7724_defconfig b/arch/sh/configs/se7724_defconfig index d817df7cc4a7..96521271758c 100644 --- a/arch/sh/configs/se7724_defconfig +++ b/arch/sh/configs/se7724_defconfig @@ -3,7 +3,6 @@ CONFIG_SYSVIPC=y CONFIG_BSD_PROCESS_ACCT=y CONFIG_LOG_BUF_SHIFT=14 # CONFIG_KALLSYMS is not set -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/sh/configs/se7750_defconfig b/arch/sh/configs/se7750_defconfig index 4defc7628a49..5fa6239ae4ea 100644 --- a/arch/sh/configs/se7750_defconfig +++ b/arch/sh/configs/se7750_defconfig @@ -6,7 +6,6 @@ CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=14 # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set # CONFIG_HOTPLUG is not set -CONFIG_SLAB=y CONFIG_MODULES=y # CONFIG_BLK_DEV_BSG is not set CONFIG_CPU_SUBTYPE_SH7750=y diff --git a/arch/sh/configs/se7751_defconfig b/arch/sh/configs/se7751_defconfig index 4a024065bb75..8b5fe4ec16bc 100644 --- a/arch/sh/configs/se7751_defconfig +++ b/arch/sh/configs/se7751_defconfig @@ -4,7 +4,6 @@ CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set # CONFIG_HOTPLUG is not set -CONFIG_SLAB=y CONFIG_MODULES=y # CONFIG_BLK_DEV_BSG is not set CONFIG_CPU_SUBTYPE_SH7751=y diff --git a/arch/sh/configs/se7780_defconfig b/arch/sh/configs/se7780_defconfig index dcd85b858ac8..21303304eda2 100644 --- a/arch/sh/configs/se7780_defconfig +++ b/arch/sh/configs/se7780_defconfig @@ -6,7 +6,6 @@ CONFIG_LOG_BUF_SHIFT=14 # CONFIG_KALLSYMS is not set # CONFIG_HOTPLUG is not set # CONFIG_EPOLL is not set -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_CPU_SUBTYPE_SH7780=y diff --git a/arch/sh/configs/secureedge5410_defconfig b/arch/sh/configs/secureedge5410_defconfig index 8422599cfb04..120176afe3f6 100644 --- a/arch/sh/configs/secureedge5410_defconfig +++ b/arch/sh/configs/secureedge5410_defconfig @@ -2,7 +2,6 @@ CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y # CONFIG_HOTPLUG is not set -CONFIG_SLAB=y # CONFIG_BLK_DEV_BSG is not set CONFIG_CPU_SUBTYPE_SH7751R=y CONFIG_MEMORY_SIZE=0x01000000 diff --git a/arch/sh/configs/sh03_defconfig b/arch/sh/configs/sh03_defconfig index 9fcf68b22dba..f661ef816f09 100644 --- a/arch/sh/configs/sh03_defconfig +++ b/arch/sh/configs/sh03_defconfig @@ -3,7 +3,6 @@ CONFIG_POSIX_MQUEUE=y CONFIG_BSD_PROCESS_ACCT=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y diff --git a/arch/sh/configs/sh2007_defconfig b/arch/sh/configs/sh2007_defconfig index 259c69e3fa22..977ae405a75e 100644 --- a/arch/sh/configs/sh2007_defconfig +++ b/arch/sh/configs/sh2007_defconfig @@ -8,7 +8,6 @@ CONFIG_IKCONFIG=y CONFIG_LOG_BUF_SHIFT=14 # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set CONFIG_KALLSYMS_ALL=y -CONFIG_SLAB=y # CONFIG_BLK_DEV_BSG is not set CONFIG_CPU_SUBTYPE_SH7780=y CONFIG_MEMORY_SIZE=0x08000000 diff --git a/arch/sh/configs/sh7710voipgw_defconfig b/arch/sh/configs/sh7710voipgw_defconfig index 0d814770b07f..7f742729df69 100644 --- a/arch/sh/configs/sh7710voipgw_defconfig +++ b/arch/sh/configs/sh7710voipgw_defconfig @@ -5,7 +5,6 @@ CONFIG_LOG_BUF_SHIFT=14 # CONFIG_FUTEX is not set # CONFIG_EPOLL is not set # CONFIG_SHMEM is not set -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y diff --git a/arch/sh/configs/sh7757lcr_defconfig b/arch/sh/configs/sh7757lcr_defconfig index 2579dc4bc0c8..48a0f9beb116 100644 --- a/arch/sh/configs/sh7757lcr_defconfig +++ b/arch/sh/configs/sh7757lcr_defconfig @@ -9,7 +9,6 @@ CONFIG_TASK_IO_ACCOUNTING=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y CONFIG_KALLSYMS_ALL=y -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/sh/configs/sh7763rdp_defconfig b/arch/sh/configs/sh7763rdp_defconfig index 8a6a446f9eb8..ae726745d9a0 100644 --- a/arch/sh/configs/sh7763rdp_defconfig +++ b/arch/sh/configs/sh7763rdp_defconfig @@ -5,7 +5,6 @@ CONFIG_LOG_BUF_SHIFT=14 CONFIG_NAMESPACES=y CONFIG_UTS_NS=y CONFIG_IPC_NS=y -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_MODULES=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/sh/configs/sh7785lcr_32bit_defconfig b/arch/sh/configs/sh7785lcr_32bit_defconfig index 781ff13227fc..59262f42abe6 100644 --- a/arch/sh/configs/sh7785lcr_32bit_defconfig +++ b/arch/sh/configs/sh7785lcr_32bit_defconfig @@ -6,7 +6,6 @@ CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=16 CONFIG_PERF_EVENTS=y # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_GCOV_KERNEL=y CONFIG_MODULES=y diff --git a/arch/sh/configs/sh7785lcr_defconfig b/arch/sh/configs/sh7785lcr_defconfig index 1b88929083f7..94381f8268ff 100644 --- a/arch/sh/configs/sh7785lcr_defconfig +++ b/arch/sh/configs/sh7785lcr_defconfig @@ -3,7 +3,6 @@ CONFIG_BSD_PROCESS_ACCT=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=14 -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y diff --git a/arch/sh/configs/titan_defconfig b/arch/sh/configs/titan_defconfig index 73a0d68b0de6..871092753591 100644 --- a/arch/sh/configs/titan_defconfig +++ b/arch/sh/configs/titan_defconfig @@ -6,7 +6,6 @@ CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=16 CONFIG_BLK_DEV_INITRD=y # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y diff --git a/arch/sh/configs/urquell_defconfig b/arch/sh/configs/urquell_defconfig index 8fc687c98fd1..445bb451a5ec 100644 --- a/arch/sh/configs/urquell_defconfig +++ b/arch/sh/configs/urquell_defconfig @@ -17,7 +17,6 @@ CONFIG_CGROUP_MEMCG=y CONFIG_CGROUP_SCHED=y CONFIG_RT_GROUP_SCHED=y CONFIG_BLK_DEV_INITRD=y -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y diff --git a/arch/sh/drivers/dma/Kconfig b/arch/sh/drivers/dma/Kconfig index 7d54f284ce10..08d937a6d249 100644 --- a/arch/sh/drivers/dma/Kconfig +++ b/arch/sh/drivers/dma/Kconfig @@ -28,17 +28,19 @@ config SH_DMA_API config NR_ONCHIP_DMA_CHANNELS int depends on SH_DMA - default "4" if CPU_SUBTYPE_SH7750 || CPU_SUBTYPE_SH7751 || \ - CPU_SUBTYPE_SH7750S || CPU_SUBTYPE_SH7091 + default "4" if CPU_SUBTYPE_SH7709 || CPU_SUBTYPE_SH7750 || \ + CPU_SUBTYPE_SH7750S || CPU_SUBTYPE_SH7751 || \ + CPU_SUBTYPE_SH7091 default "8" if CPU_SUBTYPE_SH7750R || CPU_SUBTYPE_SH7751R || \ CPU_SUBTYPE_SH7760 - default "12" if CPU_SUBTYPE_SH7723 || CPU_SUBTYPE_SH7780 || \ - CPU_SUBTYPE_SH7785 || CPU_SUBTYPE_SH7724 + default "12" if CPU_SUBTYPE_SH7723 || CPU_SUBTYPE_SH7724 || \ + CPU_SUBTYPE_SH7780 || CPU_SUBTYPE_SH7785 default "6" help This allows you to specify the number of channels that the on-chip - DMAC supports. This will be 4 for SH7750/SH7751/Sh7750S/SH7091 and 8 for the - SH7750R/SH7751R/SH7760, 12 for the SH7723/SH7780/SH7785/SH7724, default is 6. + DMAC supports. This will be 4 for SH7709/SH7750/SH7750S/SH7751/SH7091, + 8 for SH7750R/SH7751R/SH7760, and 12 for SH7723/SH7724/SH7780/SH7785. + Default is 6. config SH_DMABRG bool "SH7760 DMABRG support" diff --git a/arch/sh/drivers/dma/dma-sh.c b/arch/sh/drivers/dma/dma-sh.c index 96c626c2cd0a..306fba1564e5 100644 --- a/arch/sh/drivers/dma/dma-sh.c +++ b/arch/sh/drivers/dma/dma-sh.c @@ -19,6 +19,18 @@ #include <cpu/dma.h> /* + * Some of the SoCs feature two DMAC modules. In such a case, the channels are + * distributed equally among them. + */ +#ifdef SH_DMAC_BASE1 +#define SH_DMAC_NR_MD_CH (CONFIG_NR_ONCHIP_DMA_CHANNELS / 2) +#else +#define SH_DMAC_NR_MD_CH CONFIG_NR_ONCHIP_DMA_CHANNELS +#endif + +#define SH_DMAC_CH_SZ 0x10 + +/* * Define the default configuration for dual address memory-memory transfer. * The 0x400 value represents auto-request, external->external. */ @@ -29,7 +41,7 @@ static unsigned long dma_find_base(unsigned int chan) unsigned long base = SH_DMAC_BASE0; #ifdef SH_DMAC_BASE1 - if (chan >= 6) + if (chan >= SH_DMAC_NR_MD_CH) base = SH_DMAC_BASE1; #endif @@ -40,13 +52,13 @@ static unsigned long dma_base_addr(unsigned int chan) { unsigned long base = dma_find_base(chan); - /* Normalize offset calculation */ - if (chan >= 9) - chan -= 6; - if (chan >= 4) - base += 0x10; + chan = (chan % SH_DMAC_NR_MD_CH) * SH_DMAC_CH_SZ; + + /* DMAOR is placed inside the channel register space. Step over it. */ + if (chan >= DMAOR) + base += SH_DMAC_CH_SZ; - return base + (chan * 0x10); + return base + chan; } #ifdef CONFIG_SH_DMA_IRQ_MULTI @@ -250,12 +262,11 @@ static int sh_dmac_get_dma_residue(struct dma_channel *chan) #define NR_DMAOR 1 #endif -/* - * DMAOR bases are broken out amongst channel groups. DMAOR0 manages - * channels 0 - 5, DMAOR1 6 - 11 (optional). - */ -#define dmaor_read_reg(n) __raw_readw(dma_find_base((n)*6)) -#define dmaor_write_reg(n, data) __raw_writew(data, dma_find_base(n)*6) +#define dmaor_read_reg(n) __raw_readw(dma_find_base((n) * \ + SH_DMAC_NR_MD_CH) + DMAOR) +#define dmaor_write_reg(n, data) __raw_writew(data, \ + dma_find_base((n) * \ + SH_DMAC_NR_MD_CH) + DMAOR) static inline int dmaor_reset(int no) { diff --git a/arch/sh/include/asm/io.h b/arch/sh/include/asm/io.h index fba90e670ed4..d8f3537ef57f 100644 --- a/arch/sh/include/asm/io.h +++ b/arch/sh/include/asm/io.h @@ -286,6 +286,7 @@ static inline void iounmap(volatile void __iomem *addr) { } * access */ #define xlate_dev_mem_ptr(p) __va(p) +#define unxlate_dev_mem_ptr(p, v) do { } while (0) #define ARCH_HAS_VALID_PHYS_ADDR_RANGE int valid_phys_addr_range(phys_addr_t addr, size_t size); diff --git a/arch/sh/include/cpu-sh4/cpu/dma.h b/arch/sh/include/cpu-sh4/cpu/dma.h index 38187d06b234..e97fb2c79177 100644 --- a/arch/sh/include/cpu-sh4/cpu/dma.h +++ b/arch/sh/include/cpu-sh4/cpu/dma.h @@ -13,6 +13,5 @@ #define DMAE0_IRQ evt2irq(0x6c0) #define SH_DMAC_BASE0 0xffa00000 -#define SH_DMAC_BASE1 0xffa00070 #endif /* __ASM_CPU_SH4_DMA_H */ diff --git a/arch/sh/include/mach-common/mach/highlander.h b/arch/sh/include/mach-common/mach/highlander.h index fb44c299d033..b12c79558422 100644 --- a/arch/sh/include/mach-common/mach/highlander.h +++ b/arch/sh/include/mach-common/mach/highlander.h @@ -176,7 +176,7 @@ #define IVDR_CK_ON 4 /* iVDR Clock ON */ #endif -#define HL_FPGA_IRQ_BASE 200 +#define HL_FPGA_IRQ_BASE (200 + 16) #define HL_NR_IRL 15 #define IRQ_AX88796 (HL_FPGA_IRQ_BASE + 0) diff --git a/arch/sh/include/mach-common/mach/r2d.h b/arch/sh/include/mach-common/mach/r2d.h index 0d7e483c7d3f..69bc1907c563 100644 --- a/arch/sh/include/mach-common/mach/r2d.h +++ b/arch/sh/include/mach-common/mach/r2d.h @@ -47,7 +47,7 @@ #define IRLCNTR1 (PA_BCR + 0) /* Interrupt Control Register1 */ -#define R2D_FPGA_IRQ_BASE 100 +#define R2D_FPGA_IRQ_BASE (100 + 16) #define IRQ_VOYAGER (R2D_FPGA_IRQ_BASE + 0) #define IRQ_EXT (R2D_FPGA_IRQ_BASE + 1) diff --git a/arch/sh/include/mach-dreamcast/mach/sysasic.h b/arch/sh/include/mach-dreamcast/mach/sysasic.h index ed69ce7f2030..3b27be9a527e 100644 --- a/arch/sh/include/mach-dreamcast/mach/sysasic.h +++ b/arch/sh/include/mach-dreamcast/mach/sysasic.h @@ -22,7 +22,7 @@ takes. */ -#define HW_EVENT_IRQ_BASE 48 +#define HW_EVENT_IRQ_BASE (48 + 16) /* IRQ 13 */ #define HW_EVENT_VSYNC (HW_EVENT_IRQ_BASE + 5) /* VSync */ diff --git a/arch/sh/include/mach-se/mach/se7724.h b/arch/sh/include/mach-se/mach/se7724.h index 1fe28820dfa9..ea6c46633b33 100644 --- a/arch/sh/include/mach-se/mach/se7724.h +++ b/arch/sh/include/mach-se/mach/se7724.h @@ -37,7 +37,7 @@ #define IRQ2_IRQ evt2irq(0x640) /* Bits in IRQ012 registers */ -#define SE7724_FPGA_IRQ_BASE 220 +#define SE7724_FPGA_IRQ_BASE (220 + 16) /* IRQ0 */ #define IRQ0_BASE SE7724_FPGA_IRQ_BASE diff --git a/arch/sh/kernel/cpu/sh2/probe.c b/arch/sh/kernel/cpu/sh2/probe.c index d342ea08843f..70a07f4f2142 100644 --- a/arch/sh/kernel/cpu/sh2/probe.c +++ b/arch/sh/kernel/cpu/sh2/probe.c @@ -21,7 +21,7 @@ static int __init scan_cache(unsigned long node, const char *uname, if (!of_flat_dt_is_compatible(node, "jcore,cache")) return 0; - j2_ccr_base = (u32 __iomem *)of_flat_dt_translate_address(node); + j2_ccr_base = ioremap(of_flat_dt_translate_address(node), 4); return 1; } diff --git a/arch/sh/kernel/cpu/sh3/entry.S b/arch/sh/kernel/cpu/sh3/entry.S index e48b3dd996f5..b1f5b3c58a01 100644 --- a/arch/sh/kernel/cpu/sh3/entry.S +++ b/arch/sh/kernel/cpu/sh3/entry.S @@ -470,9 +470,9 @@ ENTRY(handle_interrupt) mov r4, r0 ! save vector->jmp table offset for later shlr2 r4 ! vector to IRQ# conversion - add #-0x10, r4 - cmp/pz r4 ! is it a valid IRQ? + mov #0x10, r5 + cmp/hs r5, r4 ! is it a valid IRQ? bt 10f /* diff --git a/arch/sparc/configs/sparc32_defconfig b/arch/sparc/configs/sparc32_defconfig index 7b3efe5edc1a..7c489e7f19b7 100644 --- a/arch/sparc/configs/sparc32_defconfig +++ b/arch/sparc/configs/sparc32_defconfig @@ -4,7 +4,6 @@ CONFIG_LOG_BUF_SHIFT=14 CONFIG_SYSFS_DEPRECATED_V2=y CONFIG_BLK_DEV_INITRD=y # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/sparc/configs/sparc64_defconfig b/arch/sparc/configs/sparc64_defconfig index 18099099583e..0bb0d4da5227 100644 --- a/arch/sparc/configs/sparc64_defconfig +++ b/arch/sparc/configs/sparc64_defconfig @@ -6,7 +6,6 @@ CONFIG_LOG_BUF_SHIFT=18 CONFIG_BLK_DEV_INITRD=y CONFIG_PERF_EVENTS=y # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_PROFILING=y CONFIG_KPROBES=y CONFIG_MODULES=y diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c index a3ccc0267bc2..86a831ebd8c8 100644 --- a/arch/sparc/mm/fault_32.c +++ b/arch/sparc/mm/fault_32.c @@ -312,7 +312,7 @@ static void force_user_fault(unsigned long address, int write) code = SEGV_MAPERR; - vma = lock_mm_and_find_vma(mm, address, regs); + vma = lock_mm_and_find_vma(mm, address, NULL); if (!vma) goto bad_area_nosemaphore; code = SEGV_ACCERR; diff --git a/arch/um/configs/i386_defconfig b/arch/um/configs/i386_defconfig index c0162286d68b..05ae535ba0dc 100644 --- a/arch/um/configs/i386_defconfig +++ b/arch/um/configs/i386_defconfig @@ -22,7 +22,6 @@ CONFIG_BLK_CGROUP=y # CONFIG_PID_NS is not set CONFIG_SYSFS_DEPRECATED=y CONFIG_CC_OPTIMIZE_FOR_SIZE=y -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/um/configs/x86_64_defconfig b/arch/um/configs/x86_64_defconfig index bec6e5d95687..1c757238be7f 100644 --- a/arch/um/configs/x86_64_defconfig +++ b/arch/um/configs/x86_64_defconfig @@ -20,7 +20,6 @@ CONFIG_BLK_CGROUP=y # CONFIG_PID_NS is not set CONFIG_SYSFS_DEPRECATED=y CONFIG_CC_OPTIMIZE_FOR_SIZE=y -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/um/os-Linux/user_syms.c b/arch/um/os-Linux/user_syms.c index 9b62a9d352b3..a310ae27b479 100644 --- a/arch/um/os-Linux/user_syms.c +++ b/arch/um/os-Linux/user_syms.c @@ -37,13 +37,6 @@ EXPORT_SYMBOL(vsyscall_ehdr); EXPORT_SYMBOL(vsyscall_end); #endif -/* Export symbols used by GCC for the stack protector. */ -extern void __stack_smash_handler(void *) __attribute__((weak)); -EXPORT_SYMBOL(__stack_smash_handler); - -extern long __guard __attribute__((weak)); -EXPORT_SYMBOL(__guard); - #ifdef _FORTIFY_SOURCE extern int __sprintf_chk(char *str, int flag, size_t len, const char *format); EXPORT_SYMBOL(__sprintf_chk); diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index b3ebf58cf77e..7422db409770 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -217,6 +217,7 @@ config X86 select HAVE_FAST_GUP select HAVE_FENTRY if X86_64 || DYNAMIC_FTRACE select HAVE_FTRACE_MCOUNT_RECORD + select HAVE_FUNCTION_GRAPH_RETVAL if HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_GRAPH_TRACER if X86_32 || (X86_64 && DYNAMIC_FTRACE) select HAVE_FUNCTION_TRACER select HAVE_GCC_PLUGINS diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h index b8d4a07f9595..897cf02c20b1 100644 --- a/arch/x86/include/asm/ftrace.h +++ b/arch/x86/include/asm/ftrace.h @@ -150,4 +150,24 @@ static inline bool arch_trace_is_compat_syscall(struct pt_regs *regs) #endif /* !COMPILE_OFFSETS */ #endif /* !__ASSEMBLY__ */ +#ifndef __ASSEMBLY__ +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +struct fgraph_ret_regs { + unsigned long ax; + unsigned long dx; + unsigned long bp; +}; + +static inline unsigned long fgraph_ret_regs_return_value(struct fgraph_ret_regs *ret_regs) +{ + return ret_regs->ax; +} + +static inline unsigned long fgraph_ret_regs_frame_pointer(struct fgraph_ret_regs *ret_regs) +{ + return ret_regs->bp; +} +#endif /* ifdef CONFIG_FUNCTION_GRAPH_TRACER */ +#endif + #endif /* _ASM_X86_FTRACE_H */ diff --git a/arch/x86/include/asm/kvm-x86-pmu-ops.h b/arch/x86/include/asm/kvm-x86-pmu-ops.h index c17e3e96fc1d..6c98f4bb4228 100644 --- a/arch/x86/include/asm/kvm-x86-pmu-ops.h +++ b/arch/x86/include/asm/kvm-x86-pmu-ops.h @@ -13,7 +13,6 @@ BUILD_BUG_ON(1) * at the call sites. */ KVM_X86_PMU_OP(hw_event_available) -KVM_X86_PMU_OP(pmc_is_enabled) KVM_X86_PMU_OP(pmc_idx_to_pmc) KVM_X86_PMU_OP(rdpmc_ecx_to_pmc) KVM_X86_PMU_OP(msr_idx_to_pmc) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index fb9d1f2d6136..28bd38303d70 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -523,7 +523,7 @@ struct kvm_pmu { u64 global_status; u64 counter_bitmask[2]; u64 global_ctrl_mask; - u64 global_ovf_ctrl_mask; + u64 global_status_mask; u64 reserved_bits; u64 raw_event_mask; struct kvm_pmc gp_counters[KVM_INTEL_PMC_MAX_GENERIC]; diff --git a/arch/x86/kernel/ftrace_32.S b/arch/x86/kernel/ftrace_32.S index 0d9a14528176..24c1175a47e2 100644 --- a/arch/x86/kernel/ftrace_32.S +++ b/arch/x86/kernel/ftrace_32.S @@ -187,12 +187,14 @@ SYM_CODE_END(ftrace_graph_caller) .globl return_to_handler return_to_handler: - pushl %eax + pushl $0 pushl %edx - movl $0, %eax + pushl %eax + movl %esp, %eax call ftrace_return_to_handler movl %eax, %ecx - popl %edx popl %eax + popl %edx + addl $4, %esp # skip ebp JMP_NOSPEC ecx #endif diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S index b8c720b5dab2..945cfa5f7239 100644 --- a/arch/x86/kernel/ftrace_64.S +++ b/arch/x86/kernel/ftrace_64.S @@ -348,12 +348,13 @@ STACK_FRAME_NON_STANDARD_FP(__fentry__) SYM_CODE_START(return_to_handler) UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR - subq $16, %rsp + subq $24, %rsp /* Save the return values */ movq %rax, (%rsp) movq %rdx, 8(%rsp) - movq %rbp, %rdi + movq %rbp, 16(%rsp) + movq %rsp, %rdi call ftrace_return_to_handler @@ -361,7 +362,7 @@ SYM_CODE_START(return_to_handler) movq 8(%rsp), %rdx movq (%rsp), %rax - addq $16, %rsp + addq $24, %rsp /* * Jump back to the old return address. This cannot be JMP_NOSPEC rdi * since IBT would demand that contain ENDBR, which simply isn't so for diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 0c9660a07b23..7f4d13383cf2 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -501,20 +501,15 @@ int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, struct kvm_cpuid_entry2 __user *entries) { - int r; - - r = -E2BIG; if (cpuid->nent < vcpu->arch.cpuid_nent) - goto out; - r = -EFAULT; + return -E2BIG; + if (copy_to_user(entries, vcpu->arch.cpuid_entries, vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2))) - goto out; - return 0; + return -EFAULT; -out: cpuid->nent = vcpu->arch.cpuid_nent; - return r; + return 0; } /* Mask kvm_cpu_caps for @leaf with the raw CPUID capabilities of this CPU. */ @@ -734,6 +729,10 @@ void kvm_set_cpu_caps(void) F(NULL_SEL_CLR_BASE) | F(AUTOIBRS) | 0 /* PrefetchCtlMsr */ ); + kvm_cpu_cap_init_kvm_defined(CPUID_8000_0022_EAX, + F(PERFMON_V2) + ); + /* * Synthesize "LFENCE is serializing" into the AMD-defined entry in * KVM's supported CPUID if the feature is reported as supported by the @@ -948,7 +947,7 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) union cpuid10_eax eax; union cpuid10_edx edx; - if (!static_cpu_has(X86_FEATURE_ARCH_PERFMON)) { + if (!enable_pmu || !static_cpu_has(X86_FEATURE_ARCH_PERFMON)) { entry->eax = entry->ebx = entry->ecx = entry->edx = 0; break; } @@ -1128,7 +1127,7 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) entry->edx = 0; break; case 0x80000000: - entry->eax = min(entry->eax, 0x80000021); + entry->eax = min(entry->eax, 0x80000022); /* * Serializing LFENCE is reported in a multitude of ways, and * NullSegClearsBase is not reported in CPUID on Zen2; help @@ -1233,6 +1232,28 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) entry->ebx = entry->ecx = entry->edx = 0; cpuid_entry_override(entry, CPUID_8000_0021_EAX); break; + /* AMD Extended Performance Monitoring and Debug */ + case 0x80000022: { + union cpuid_0x80000022_ebx ebx; + + entry->ecx = entry->edx = 0; + if (!enable_pmu || !kvm_cpu_cap_has(X86_FEATURE_PERFMON_V2)) { + entry->eax = entry->ebx; + break; + } + + cpuid_entry_override(entry, CPUID_8000_0022_EAX); + + if (kvm_cpu_cap_has(X86_FEATURE_PERFMON_V2)) + ebx.split.num_core_pmc = kvm_pmu_cap.num_counters_gp; + else if (kvm_cpu_cap_has(X86_FEATURE_PERFCTR_CORE)) + ebx.split.num_core_pmc = AMD64_NUM_COUNTERS_CORE; + else + ebx.split.num_core_pmc = AMD64_NUM_COUNTERS; + + entry->ebx = ebx.full; + break; + } /*Add support for Centaur's CPUID instruction*/ case 0xC0000000: /*Just support up to 0xC0000004 now*/ diff --git a/arch/x86/kvm/i8259.c b/arch/x86/kvm/i8259.c index 4756bcb5724f..8dec646e764b 100644 --- a/arch/x86/kvm/i8259.c +++ b/arch/x86/kvm/i8259.c @@ -411,7 +411,10 @@ static u32 pic_poll_read(struct kvm_kpic_state *s, u32 addr1) pic_clear_isr(s, ret); if (addr1 >> 7 || ret != 2) pic_update_irq(s->pics_state); + /* Bit 7 is 1, means there's an interrupt */ + ret |= 0x80; } else { + /* Bit 7 is 0, means there's no interrupt */ ret = 0x07; pic_update_irq(s->pics_state); } diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 3c300a196bdf..113ca9661ab2 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -51,11 +51,6 @@ #define mod_64(x, y) ((x) % (y)) #endif -#define PRId64 "d" -#define PRIx64 "llx" -#define PRIu64 "u" -#define PRIo64 "o" - /* 14 is the version for Xeon and Pentium 8.4.8*/ #define APIC_VERSION 0x14UL #define LAPIC_MMIO_LENGTH (1 << 12) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 6eaa3d6994ae..ec169f5c7dce 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -58,6 +58,8 @@ extern bool itlb_multihit_kvm_mitigation; +static bool nx_hugepage_mitigation_hard_disabled; + int __read_mostly nx_huge_pages = -1; static uint __read_mostly nx_huge_pages_recovery_period_ms; #ifdef CONFIG_PREEMPT_RT @@ -67,12 +69,13 @@ static uint __read_mostly nx_huge_pages_recovery_ratio = 0; static uint __read_mostly nx_huge_pages_recovery_ratio = 60; #endif +static int get_nx_huge_pages(char *buffer, const struct kernel_param *kp); static int set_nx_huge_pages(const char *val, const struct kernel_param *kp); static int set_nx_huge_pages_recovery_param(const char *val, const struct kernel_param *kp); static const struct kernel_param_ops nx_huge_pages_ops = { .set = set_nx_huge_pages, - .get = param_get_bool, + .get = get_nx_huge_pages, }; static const struct kernel_param_ops nx_huge_pages_recovery_param_ops = { @@ -1600,6 +1603,10 @@ bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) if (tdp_mmu_enabled) flush = kvm_tdp_mmu_unmap_gfn_range(kvm, range, flush); + if (kvm_x86_ops.set_apic_access_page_addr && + range->slot->id == APIC_ACCESS_PAGE_PRIVATE_MEMSLOT) + kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD); + return flush; } @@ -5797,6 +5804,14 @@ static void __kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu vcpu_clear_mmio_info(vcpu, addr); + /* + * Walking and synchronizing SPTEs both assume they are operating in + * the context of the current MMU, and would need to be reworked if + * this is ever used to sync the guest_mmu, e.g. to emulate INVEPT. + */ + if (WARN_ON_ONCE(mmu != vcpu->arch.mmu)) + return; + if (!VALID_PAGE(root_hpa)) return; @@ -6844,6 +6859,14 @@ static void mmu_destroy_caches(void) kmem_cache_destroy(mmu_page_header_cache); } +static int get_nx_huge_pages(char *buffer, const struct kernel_param *kp) +{ + if (nx_hugepage_mitigation_hard_disabled) + return sprintf(buffer, "never\n"); + + return param_get_bool(buffer, kp); +} + static bool get_nx_auto_mode(void) { /* Return true when CPU has the bug, and mitigations are ON */ @@ -6860,15 +6883,29 @@ static int set_nx_huge_pages(const char *val, const struct kernel_param *kp) bool old_val = nx_huge_pages; bool new_val; + if (nx_hugepage_mitigation_hard_disabled) + return -EPERM; + /* In "auto" mode deploy workaround only if CPU has the bug. */ - if (sysfs_streq(val, "off")) + if (sysfs_streq(val, "off")) { new_val = 0; - else if (sysfs_streq(val, "force")) + } else if (sysfs_streq(val, "force")) { new_val = 1; - else if (sysfs_streq(val, "auto")) + } else if (sysfs_streq(val, "auto")) { new_val = get_nx_auto_mode(); - else if (kstrtobool(val, &new_val) < 0) + } else if (sysfs_streq(val, "never")) { + new_val = 0; + + mutex_lock(&kvm_lock); + if (!list_empty(&vm_list)) { + mutex_unlock(&kvm_lock); + return -EBUSY; + } + nx_hugepage_mitigation_hard_disabled = true; + mutex_unlock(&kvm_lock); + } else if (kstrtobool(val, &new_val) < 0) { return -EINVAL; + } __set_nx_huge_pages(new_val); @@ -7006,6 +7043,9 @@ static int set_nx_huge_pages_recovery_param(const char *val, const struct kernel uint old_period, new_period; int err; + if (nx_hugepage_mitigation_hard_disabled) + return -EPERM; + was_recovery_enabled = calc_nx_huge_pages_recovery_period(&old_period); err = param_set_uint(val, kp); @@ -7164,6 +7204,9 @@ int kvm_mmu_post_init_vm(struct kvm *kvm) { int err; + if (nx_hugepage_mitigation_hard_disabled) + return 0; + err = kvm_vm_create_worker_thread(kvm, kvm_nx_huge_page_recovery_worker, 0, "kvm-nx-lpage-recovery", &kvm->arch.nx_huge_page_recovery_thread); diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 08340219c35a..512163d52194 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -592,7 +592,10 @@ static inline int tdp_mmu_set_spte_atomic(struct kvm *kvm, /* * Note, fast_pf_fix_direct_spte() can also modify TDP MMU SPTEs and - * does not hold the mmu_lock. + * does not hold the mmu_lock. On failure, i.e. if a different logical + * CPU modified the SPTE, try_cmpxchg64() updates iter->old_spte with + * the current value, so the caller operates on fresh data, e.g. if it + * retries tdp_mmu_set_spte_atomic() */ if (!try_cmpxchg64(sptep, &iter->old_spte, new_spte)) return -EBUSY; diff --git a/arch/x86/kvm/mtrr.c b/arch/x86/kvm/mtrr.c index 9fac1ec03463..3eb6e7f47e96 100644 --- a/arch/x86/kvm/mtrr.c +++ b/arch/x86/kvm/mtrr.c @@ -25,10 +25,24 @@ #define IA32_MTRR_DEF_TYPE_FE (1ULL << 10) #define IA32_MTRR_DEF_TYPE_TYPE_MASK (0xff) +static bool is_mtrr_base_msr(unsigned int msr) +{ + /* MTRR base MSRs use even numbers, masks use odd numbers. */ + return !(msr & 0x1); +} + +static struct kvm_mtrr_range *var_mtrr_msr_to_range(struct kvm_vcpu *vcpu, + unsigned int msr) +{ + int index = (msr - MTRRphysBase_MSR(0)) / 2; + + return &vcpu->arch.mtrr_state.var_ranges[index]; +} + static bool msr_mtrr_valid(unsigned msr) { switch (msr) { - case 0x200 ... 0x200 + 2 * KVM_NR_VAR_MTRR - 1: + case MTRRphysBase_MSR(0) ... MTRRphysMask_MSR(KVM_NR_VAR_MTRR - 1): case MSR_MTRRfix64K_00000: case MSR_MTRRfix16K_80000: case MSR_MTRRfix16K_A0000: @@ -41,7 +55,6 @@ static bool msr_mtrr_valid(unsigned msr) case MSR_MTRRfix4K_F0000: case MSR_MTRRfix4K_F8000: case MSR_MTRRdefType: - case MSR_IA32_CR_PAT: return true; } return false; @@ -52,7 +65,7 @@ static bool valid_mtrr_type(unsigned t) return t < 8 && (1 << t) & 0x73; /* 0, 1, 4, 5, 6 */ } -bool kvm_mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data) +static bool kvm_mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data) { int i; u64 mask; @@ -60,9 +73,7 @@ bool kvm_mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data) if (!msr_mtrr_valid(msr)) return false; - if (msr == MSR_IA32_CR_PAT) { - return kvm_pat_valid(data); - } else if (msr == MSR_MTRRdefType) { + if (msr == MSR_MTRRdefType) { if (data & ~0xcff) return false; return valid_mtrr_type(data & 0xff); @@ -74,7 +85,8 @@ bool kvm_mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data) } /* variable MTRRs */ - WARN_ON(!(msr >= 0x200 && msr < 0x200 + 2 * KVM_NR_VAR_MTRR)); + WARN_ON(!(msr >= MTRRphysBase_MSR(0) && + msr <= MTRRphysMask_MSR(KVM_NR_VAR_MTRR - 1))); mask = kvm_vcpu_reserved_gpa_bits_raw(vcpu); if ((msr & 1) == 0) { @@ -88,7 +100,6 @@ bool kvm_mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data) return (data & mask) == 0; } -EXPORT_SYMBOL_GPL(kvm_mtrr_valid); static bool mtrr_is_enabled(struct kvm_mtrr *mtrr_state) { @@ -308,10 +319,8 @@ static void update_mtrr(struct kvm_vcpu *vcpu, u32 msr) { struct kvm_mtrr *mtrr_state = &vcpu->arch.mtrr_state; gfn_t start, end; - int index; - if (msr == MSR_IA32_CR_PAT || !tdp_enabled || - !kvm_arch_has_noncoherent_dma(vcpu->kvm)) + if (!tdp_enabled || !kvm_arch_has_noncoherent_dma(vcpu->kvm)) return; if (!mtrr_is_enabled(mtrr_state) && msr != MSR_MTRRdefType) @@ -326,8 +335,7 @@ static void update_mtrr(struct kvm_vcpu *vcpu, u32 msr) end = ~0ULL; } else { /* variable range MTRRs. */ - index = (msr - 0x200) / 2; - var_mtrr_range(&mtrr_state->var_ranges[index], &start, &end); + var_mtrr_range(var_mtrr_msr_to_range(vcpu, msr), &start, &end); } kvm_zap_gfn_range(vcpu->kvm, gpa_to_gfn(start), gpa_to_gfn(end)); @@ -342,21 +350,18 @@ static void set_var_mtrr_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data) { struct kvm_mtrr *mtrr_state = &vcpu->arch.mtrr_state; struct kvm_mtrr_range *tmp, *cur; - int index, is_mtrr_mask; - index = (msr - 0x200) / 2; - is_mtrr_mask = msr - 0x200 - 2 * index; - cur = &mtrr_state->var_ranges[index]; + cur = var_mtrr_msr_to_range(vcpu, msr); /* remove the entry if it's in the list. */ if (var_mtrr_range_is_valid(cur)) - list_del(&mtrr_state->var_ranges[index].node); + list_del(&cur->node); /* * Set all illegal GPA bits in the mask, since those bits must * implicitly be 0. The bits are then cleared when reading them. */ - if (!is_mtrr_mask) + if (is_mtrr_base_msr(msr)) cur->base = data; else cur->mask = data | kvm_vcpu_reserved_gpa_bits_raw(vcpu); @@ -382,8 +387,6 @@ int kvm_mtrr_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data) *(u64 *)&vcpu->arch.mtrr_state.fixed_ranges[index] = data; else if (msr == MSR_MTRRdefType) vcpu->arch.mtrr_state.deftype = data; - else if (msr == MSR_IA32_CR_PAT) - vcpu->arch.pat = data; else set_var_mtrr_msr(vcpu, msr, data); @@ -411,21 +414,16 @@ int kvm_mtrr_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) return 1; index = fixed_msr_to_range_index(msr); - if (index >= 0) + if (index >= 0) { *pdata = *(u64 *)&vcpu->arch.mtrr_state.fixed_ranges[index]; - else if (msr == MSR_MTRRdefType) + } else if (msr == MSR_MTRRdefType) { *pdata = vcpu->arch.mtrr_state.deftype; - else if (msr == MSR_IA32_CR_PAT) - *pdata = vcpu->arch.pat; - else { /* Variable MTRRs */ - int is_mtrr_mask; - - index = (msr - 0x200) / 2; - is_mtrr_mask = msr - 0x200 - 2 * index; - if (!is_mtrr_mask) - *pdata = vcpu->arch.mtrr_state.var_ranges[index].base; + } else { + /* Variable MTRRs */ + if (is_mtrr_base_msr(msr)) + *pdata = var_mtrr_msr_to_range(vcpu, msr)->base; else - *pdata = vcpu->arch.mtrr_state.var_ranges[index].mask; + *pdata = var_mtrr_msr_to_range(vcpu, msr)->mask; *pdata &= ~kvm_vcpu_reserved_gpa_bits_raw(vcpu); } diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index 1690d41c1830..bf653df86112 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -93,11 +93,6 @@ void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops) #undef __KVM_X86_PMU_OP } -static inline bool pmc_is_globally_enabled(struct kvm_pmc *pmc) -{ - return static_call(kvm_x86_pmu_pmc_is_enabled)(pmc); -} - static void kvm_pmi_trigger_fn(struct irq_work *irq_work) { struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work); @@ -562,6 +557,14 @@ void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu) bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) { + switch (msr) { + case MSR_CORE_PERF_GLOBAL_STATUS: + case MSR_CORE_PERF_GLOBAL_CTRL: + case MSR_CORE_PERF_GLOBAL_OVF_CTRL: + return kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)); + default: + break; + } return static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr) || static_call(kvm_x86_pmu_is_valid_msr)(vcpu, msr); } @@ -577,13 +580,86 @@ static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr) int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) { - return static_call(kvm_x86_pmu_get_msr)(vcpu, msr_info); + struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); + u32 msr = msr_info->index; + + switch (msr) { + case MSR_CORE_PERF_GLOBAL_STATUS: + case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS: + msr_info->data = pmu->global_status; + break; + case MSR_AMD64_PERF_CNTR_GLOBAL_CTL: + case MSR_CORE_PERF_GLOBAL_CTRL: + msr_info->data = pmu->global_ctrl; + break; + case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR: + case MSR_CORE_PERF_GLOBAL_OVF_CTRL: + msr_info->data = 0; + break; + default: + return static_call(kvm_x86_pmu_get_msr)(vcpu, msr_info); + } + + return 0; } int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) { - kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index); - return static_call(kvm_x86_pmu_set_msr)(vcpu, msr_info); + struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); + u32 msr = msr_info->index; + u64 data = msr_info->data; + u64 diff; + + /* + * Note, AMD ignores writes to reserved bits and read-only PMU MSRs, + * whereas Intel generates #GP on attempts to write reserved/RO MSRs. + */ + switch (msr) { + case MSR_CORE_PERF_GLOBAL_STATUS: + if (!msr_info->host_initiated) + return 1; /* RO MSR */ + fallthrough; + case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS: + /* Per PPR, Read-only MSR. Writes are ignored. */ + if (!msr_info->host_initiated) + break; + + if (data & pmu->global_status_mask) + return 1; + + pmu->global_status = data; + break; + case MSR_AMD64_PERF_CNTR_GLOBAL_CTL: + data &= ~pmu->global_ctrl_mask; + fallthrough; + case MSR_CORE_PERF_GLOBAL_CTRL: + if (!kvm_valid_perf_global_ctrl(pmu, data)) + return 1; + + if (pmu->global_ctrl != data) { + diff = pmu->global_ctrl ^ data; + pmu->global_ctrl = data; + reprogram_counters(pmu, diff); + } + break; + case MSR_CORE_PERF_GLOBAL_OVF_CTRL: + /* + * GLOBAL_OVF_CTRL, a.k.a. GLOBAL STATUS_RESET, clears bits in + * GLOBAL_STATUS, and so the set of reserved bits is the same. + */ + if (data & pmu->global_status_mask) + return 1; + fallthrough; + case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR: + if (!msr_info->host_initiated) + pmu->global_status &= ~data; + break; + default: + kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index); + return static_call(kvm_x86_pmu_set_msr)(vcpu, msr_info); + } + + return 0; } /* refresh PMU settings. This function generally is called when underlying diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h index 5c7bbf03b599..7d9ba301c090 100644 --- a/arch/x86/kvm/pmu.h +++ b/arch/x86/kvm/pmu.h @@ -20,7 +20,6 @@ struct kvm_pmu_ops { bool (*hw_event_available)(struct kvm_pmc *pmc); - bool (*pmc_is_enabled)(struct kvm_pmc *pmc); struct kvm_pmc *(*pmc_idx_to_pmc)(struct kvm_pmu *pmu, int pmc_idx); struct kvm_pmc *(*rdpmc_ecx_to_pmc)(struct kvm_vcpu *vcpu, unsigned int idx, u64 *mask); @@ -37,10 +36,25 @@ struct kvm_pmu_ops { const u64 EVENTSEL_EVENT; const int MAX_NR_GP_COUNTERS; + const int MIN_NR_GP_COUNTERS; }; void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops); +static inline bool kvm_pmu_has_perf_global_ctrl(struct kvm_pmu *pmu) +{ + /* + * Architecturally, Intel's SDM states that IA32_PERF_GLOBAL_CTRL is + * supported if "CPUID.0AH: EAX[7:0] > 0", i.e. if the PMU version is + * greater than zero. However, KVM only exposes and emulates the MSR + * to/for the guest if the guest PMU supports at least "Architectural + * Performance Monitoring Version 2". + * + * AMD's version of PERF_GLOBAL_CTRL conveniently shows up with v2. + */ + return pmu->version > 1; +} + static inline u64 pmc_bitmask(struct kvm_pmc *pmc) { struct kvm_pmu *pmu = pmc_to_pmu(pmc); @@ -161,6 +175,7 @@ extern struct x86_pmu_capability kvm_pmu_cap; static inline void kvm_init_pmu_capability(const struct kvm_pmu_ops *pmu_ops) { bool is_intel = boot_cpu_data.x86_vendor == X86_VENDOR_INTEL; + int min_nr_gp_ctrs = pmu_ops->MIN_NR_GP_COUNTERS; /* * Hybrid PMUs don't play nice with virtualization without careful @@ -175,11 +190,15 @@ static inline void kvm_init_pmu_capability(const struct kvm_pmu_ops *pmu_ops) perf_get_x86_pmu_capability(&kvm_pmu_cap); /* - * For Intel, only support guest architectural pmu - * on a host with architectural pmu. + * WARN if perf did NOT disable hardware PMU if the number of + * architecturally required GP counters aren't present, i.e. if + * there are a non-zero number of counters, but fewer than what + * is architecturally required. */ - if ((is_intel && !kvm_pmu_cap.version) || - !kvm_pmu_cap.num_counters_gp) + if (!kvm_pmu_cap.num_counters_gp || + WARN_ON_ONCE(kvm_pmu_cap.num_counters_gp < min_nr_gp_ctrs)) + enable_pmu = false; + else if (is_intel && !kvm_pmu_cap.version) enable_pmu = false; } @@ -201,6 +220,33 @@ static inline void kvm_pmu_request_counter_reprogram(struct kvm_pmc *pmc) kvm_make_request(KVM_REQ_PMU, pmc->vcpu); } +static inline void reprogram_counters(struct kvm_pmu *pmu, u64 diff) +{ + int bit; + + if (!diff) + return; + + for_each_set_bit(bit, (unsigned long *)&diff, X86_PMC_IDX_MAX) + set_bit(bit, pmu->reprogram_pmi); + kvm_make_request(KVM_REQ_PMU, pmu_to_vcpu(pmu)); +} + +/* + * Check if a PMC is enabled by comparing it against global_ctrl bits. + * + * If the vPMU doesn't have global_ctrl MSR, all vPMCs are enabled. + */ +static inline bool pmc_is_globally_enabled(struct kvm_pmc *pmc) +{ + struct kvm_pmu *pmu = pmc_to_pmu(pmc); + + if (!kvm_pmu_has_perf_global_ctrl(pmu)) + return true; + + return test_bit(pmc->idx, (unsigned long *)&pmu->global_ctrl); +} + void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu); void kvm_pmu_handle_event(struct kvm_vcpu *vcpu); int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned pmc, u64 *data); diff --git a/arch/x86/kvm/reverse_cpuid.h b/arch/x86/kvm/reverse_cpuid.h index a5717282bb9c..56cbdb24400a 100644 --- a/arch/x86/kvm/reverse_cpuid.h +++ b/arch/x86/kvm/reverse_cpuid.h @@ -15,6 +15,7 @@ enum kvm_only_cpuid_leafs { CPUID_12_EAX = NCAPINTS, CPUID_7_1_EDX, CPUID_8000_0007_EDX, + CPUID_8000_0022_EAX, NR_KVM_CPU_CAPS, NKVMCAPINTS = NR_KVM_CPU_CAPS - NCAPINTS, @@ -47,6 +48,9 @@ enum kvm_only_cpuid_leafs { /* CPUID level 0x80000007 (EDX). */ #define KVM_X86_FEATURE_CONSTANT_TSC KVM_X86_FEATURE(CPUID_8000_0007_EDX, 8) +/* CPUID level 0x80000022 (EAX) */ +#define KVM_X86_FEATURE_PERFMON_V2 KVM_X86_FEATURE(CPUID_8000_0022_EAX, 0) + struct cpuid_reg { u32 function; u32 index; @@ -74,6 +78,7 @@ static const struct cpuid_reg reverse_cpuid[] = { [CPUID_7_1_EDX] = { 7, 1, CPUID_EDX}, [CPUID_8000_0007_EDX] = {0x80000007, 0, CPUID_EDX}, [CPUID_8000_0021_EAX] = {0x80000021, 0, CPUID_EAX}, + [CPUID_8000_0022_EAX] = {0x80000022, 0, CPUID_EAX}, }; /* @@ -108,6 +113,8 @@ static __always_inline u32 __feature_translate(int x86_feature) return KVM_X86_FEATURE_SGX_EDECCSSA; else if (x86_feature == X86_FEATURE_CONSTANT_TSC) return KVM_X86_FEATURE_CONSTANT_TSC; + else if (x86_feature == X86_FEATURE_PERFMON_V2) + return KVM_X86_FEATURE_PERFMON_V2; return x86_feature; } diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c index 5fa939e411d8..cef5a3d0abd0 100644 --- a/arch/x86/kvm/svm/pmu.c +++ b/arch/x86/kvm/svm/pmu.c @@ -78,14 +78,6 @@ static bool amd_hw_event_available(struct kvm_pmc *pmc) return true; } -/* check if a PMC is enabled by comparing it against global_ctrl bits. Because - * AMD CPU doesn't have global_ctrl MSR, all PMCs are enabled (return TRUE). - */ -static bool amd_pmc_is_enabled(struct kvm_pmc *pmc) -{ - return true; -} - static bool amd_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); @@ -102,12 +94,6 @@ static struct kvm_pmc *amd_rdpmc_ecx_to_pmc(struct kvm_vcpu *vcpu, return amd_pmc_idx_to_pmc(vcpu_to_pmu(vcpu), idx & ~(3u << 30)); } -static bool amd_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) -{ - /* All MSRs refer to exactly one PMC, so msr_idx_to_pmc is enough. */ - return false; -} - static struct kvm_pmc *amd_msr_idx_to_pmc(struct kvm_vcpu *vcpu, u32 msr) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); @@ -119,6 +105,29 @@ static struct kvm_pmc *amd_msr_idx_to_pmc(struct kvm_vcpu *vcpu, u32 msr) return pmc; } +static bool amd_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) +{ + struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); + + switch (msr) { + case MSR_K7_EVNTSEL0 ... MSR_K7_PERFCTR3: + return pmu->version > 0; + case MSR_F15H_PERF_CTL0 ... MSR_F15H_PERF_CTR5: + return guest_cpuid_has(vcpu, X86_FEATURE_PERFCTR_CORE); + case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS: + case MSR_AMD64_PERF_CNTR_GLOBAL_CTL: + case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR: + return pmu->version > 1; + default: + if (msr > MSR_F15H_PERF_CTR5 && + msr < MSR_F15H_PERF_CTL0 + 2 * pmu->nr_arch_gp_counters) + return pmu->version > 1; + break; + } + + return amd_msr_idx_to_pmc(vcpu, msr); +} + static int amd_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); @@ -172,20 +181,39 @@ static int amd_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) static void amd_pmu_refresh(struct kvm_vcpu *vcpu) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); + union cpuid_0x80000022_ebx ebx; - if (guest_cpuid_has(vcpu, X86_FEATURE_PERFCTR_CORE)) + pmu->version = 1; + if (guest_cpuid_has(vcpu, X86_FEATURE_PERFMON_V2)) { + pmu->version = 2; + /* + * Note, PERFMON_V2 is also in 0x80000022.0x0, i.e. the guest + * CPUID entry is guaranteed to be non-NULL. + */ + BUILD_BUG_ON(x86_feature_cpuid(X86_FEATURE_PERFMON_V2).function != 0x80000022 || + x86_feature_cpuid(X86_FEATURE_PERFMON_V2).index); + ebx.full = kvm_find_cpuid_entry_index(vcpu, 0x80000022, 0)->ebx; + pmu->nr_arch_gp_counters = ebx.split.num_core_pmc; + } else if (guest_cpuid_has(vcpu, X86_FEATURE_PERFCTR_CORE)) { pmu->nr_arch_gp_counters = AMD64_NUM_COUNTERS_CORE; - else + } else { pmu->nr_arch_gp_counters = AMD64_NUM_COUNTERS; + } + + pmu->nr_arch_gp_counters = min_t(unsigned int, pmu->nr_arch_gp_counters, + kvm_pmu_cap.num_counters_gp); + + if (pmu->version > 1) { + pmu->global_ctrl_mask = ~((1ull << pmu->nr_arch_gp_counters) - 1); + pmu->global_status_mask = pmu->global_ctrl_mask; + } pmu->counter_bitmask[KVM_PMC_GP] = ((u64)1 << 48) - 1; pmu->reserved_bits = 0xfffffff000280000ull; pmu->raw_event_mask = AMD64_RAW_EVENT_MASK; - pmu->version = 1; /* not applicable to AMD; but clean them to prevent any fall out */ pmu->counter_bitmask[KVM_PMC_FIXED] = 0; pmu->nr_arch_fixed_counters = 0; - pmu->global_status = 0; bitmap_set(pmu->all_valid_pmc_idx, 0, pmu->nr_arch_gp_counters); } @@ -216,11 +244,12 @@ static void amd_pmu_reset(struct kvm_vcpu *vcpu) pmc_stop_counter(pmc); pmc->counter = pmc->prev_counter = pmc->eventsel = 0; } + + pmu->global_ctrl = pmu->global_status = 0; } struct kvm_pmu_ops amd_pmu_ops __initdata = { .hw_event_available = amd_hw_event_available, - .pmc_is_enabled = amd_pmc_is_enabled, .pmc_idx_to_pmc = amd_pmc_idx_to_pmc, .rdpmc_ecx_to_pmc = amd_rdpmc_ecx_to_pmc, .msr_idx_to_pmc = amd_msr_idx_to_pmc, @@ -233,4 +262,5 @@ struct kvm_pmu_ops amd_pmu_ops __initdata = { .reset = amd_pmu_reset, .EVENTSEL_EVENT = AMD64_EVENTSEL_EVENT, .MAX_NR_GP_COUNTERS = KVM_AMD_PMC_MAX_GENERIC, + .MIN_NR_GP_COUNTERS = AMD64_NUM_COUNTERS, }; diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 69ae5e1b3120..07756b7348ae 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2216,10 +2216,7 @@ void __init sev_hardware_setup(void) } sev_asid_count = max_sev_asid - min_sev_asid + 1; - if (misc_cg_set_capacity(MISC_CG_RES_SEV, sev_asid_count)) - goto out; - - pr_info("SEV supported: %u ASIDs\n", sev_asid_count); + WARN_ON_ONCE(misc_cg_set_capacity(MISC_CG_RES_SEV, sev_asid_count)); sev_supported = true; /* SEV-ES support requested? */ @@ -2244,13 +2241,19 @@ void __init sev_hardware_setup(void) goto out; sev_es_asid_count = min_sev_asid - 1; - if (misc_cg_set_capacity(MISC_CG_RES_SEV_ES, sev_es_asid_count)) - goto out; - - pr_info("SEV-ES supported: %u ASIDs\n", sev_es_asid_count); + WARN_ON_ONCE(misc_cg_set_capacity(MISC_CG_RES_SEV_ES, sev_es_asid_count)); sev_es_supported = true; out: + if (boot_cpu_has(X86_FEATURE_SEV)) + pr_info("SEV %s (ASIDs %u - %u)\n", + sev_supported ? "enabled" : "disabled", + min_sev_asid, max_sev_asid); + if (boot_cpu_has(X86_FEATURE_SEV_ES)) + pr_info("SEV-ES %s (ASIDs %u - %u)\n", + sev_es_supported ? "enabled" : "disabled", + min_sev_asid > 1 ? 1 : 0, min_sev_asid - 1); + sev_enabled = sev_supported; sev_es_enabled = sev_es_supported; #endif diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 54089f990c8f..d381ad424554 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -244,15 +244,6 @@ static u8 rsm_ins_bytes[] = "\x0f\xaa"; static unsigned long iopm_base; -struct kvm_ldttss_desc { - u16 limit0; - u16 base0; - unsigned base1:8, type:5, dpl:2, p:1; - unsigned limit1:4, zero0:3, g:1, base2:8; - u32 base3; - u32 zero1; -} __attribute__((packed)); - DEFINE_PER_CPU(struct svm_cpu_data, svm_data); /* @@ -588,7 +579,6 @@ static int svm_hardware_enable(void) struct svm_cpu_data *sd; uint64_t efer; - struct desc_struct *gdt; int me = raw_smp_processor_id(); rdmsrl(MSR_EFER, efer); @@ -601,9 +591,6 @@ static int svm_hardware_enable(void) sd->next_asid = sd->max_asid + 1; sd->min_asid = max_sev_asid + 1; - gdt = get_current_gdt_rw(); - sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS); - wrmsrl(MSR_EFER, efer | EFER_SVME); wrmsrl(MSR_VM_HSAVE_PA, sd->save_area_pa); @@ -752,7 +739,7 @@ static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr) BUG_ON(offset == MSR_INVALID); - return !!test_bit(bit_write, &tmp); + return test_bit(bit_write, &tmp); } static void set_msr_interception_bitmap(struct kvm_vcpu *vcpu, u32 *msrpm, @@ -2939,9 +2926,10 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) break; case MSR_IA32_CR_PAT: - if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data)) - return 1; - vcpu->arch.pat = data; + ret = kvm_set_msr_common(vcpu, msr); + if (ret) + break; + svm->vmcb01.ptr->save.g_pat = data; if (is_guest_mode(vcpu)) nested_vmcb02_compute_g_pat(svm); @@ -3418,8 +3406,6 @@ static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) struct kvm_run *kvm_run = vcpu->run; u32 exit_code = svm->vmcb->control.exit_code; - trace_kvm_exit(vcpu, KVM_ISA_SVM); - /* SEV-ES guests must use the CR write traps to track CR registers. */ if (!sev_es_guest(vcpu->kvm)) { if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE)) @@ -3457,14 +3443,6 @@ static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) return svm_invoke_exit_handler(vcpu, exit_code); } -static void reload_tss(struct kvm_vcpu *vcpu) -{ - struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); - - sd->tss_desc->type = 9; /* available 32/64-bit TSS */ - load_TR_desc(); -} - static void pre_svm_run(struct kvm_vcpu *vcpu) { struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); @@ -4099,9 +4077,6 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) svm_vcpu_enter_exit(vcpu, spec_ctrl_intercepted); - if (!sev_es_guest(vcpu->kvm)) - reload_tss(vcpu); - if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL)) x86_spec_ctrl_restore_host(svm->virt_spec_ctrl); @@ -4156,6 +4131,8 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) SVM_EXIT_EXCP_BASE + MC_VECTOR)) svm_handle_mce(vcpu); + trace_kvm_exit(vcpu, KVM_ISA_SVM); + svm_complete_interrupts(vcpu); if (is_guest_mode(vcpu)) @@ -5025,9 +5002,22 @@ static __init void svm_set_cpu_caps(void) boot_cpu_has(X86_FEATURE_AMD_SSBD)) kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD); - /* AMD PMU PERFCTR_CORE CPUID */ - if (enable_pmu && boot_cpu_has(X86_FEATURE_PERFCTR_CORE)) - kvm_cpu_cap_set(X86_FEATURE_PERFCTR_CORE); + if (enable_pmu) { + /* + * Enumerate support for PERFCTR_CORE if and only if KVM has + * access to enough counters to virtualize "core" support, + * otherwise limit vPMU support to the legacy number of counters. + */ + if (kvm_pmu_cap.num_counters_gp < AMD64_NUM_COUNTERS_CORE) + kvm_pmu_cap.num_counters_gp = min(AMD64_NUM_COUNTERS, + kvm_pmu_cap.num_counters_gp); + else + kvm_cpu_cap_check_and_set(X86_FEATURE_PERFCTR_CORE); + + if (kvm_pmu_cap.version != 2 || + !kvm_cpu_cap_has(X86_FEATURE_PERFCTR_CORE)) + kvm_cpu_cap_clear(X86_FEATURE_PERFMON_V2); + } /* CPUID 0x8000001F (SME/SEV features) */ sev_set_cpu_caps(); diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index f44751dd8d5d..18af7e712a5a 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -303,7 +303,6 @@ struct svm_cpu_data { u32 max_asid; u32 next_asid; u32 min_asid; - struct kvm_ldttss_desc *tss_desc; struct page *save_area; unsigned long save_area_pa; diff --git a/arch/x86/kvm/vmx/capabilities.h b/arch/x86/kvm/vmx/capabilities.h index 45162c1bcd8f..d0abee35d7ba 100644 --- a/arch/x86/kvm/vmx/capabilities.h +++ b/arch/x86/kvm/vmx/capabilities.h @@ -152,8 +152,8 @@ static inline bool cpu_has_vmx_ept(void) static inline bool vmx_umip_emulated(void) { - return vmcs_config.cpu_based_2nd_exec_ctrl & - SECONDARY_EXEC_DESC; + return !boot_cpu_has(X86_FEATURE_UMIP) && + (vmcs_config.cpu_based_2nd_exec_ctrl & SECONDARY_EXEC_DESC); } static inline bool cpu_has_vmx_rdtscp(void) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index e35cf0bd0df9..516391cc0d64 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -2328,8 +2328,7 @@ static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct loaded_vmcs *vmcs0 * Preset *DT exiting when emulating UMIP, so that vmx_set_cr4() * will not have to rewrite the controls just for this bit. */ - if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated() && - (vmcs12->guest_cr4 & X86_CR4_UMIP)) + if (vmx_umip_emulated() && (vmcs12->guest_cr4 & X86_CR4_UMIP)) exec_control |= SECONDARY_EXEC_DESC; if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) @@ -2649,7 +2648,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, } if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) && - intel_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)) && + kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)) && WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL, vmcs12->guest_ia32_perf_global_ctrl))) { *entry_failure_code = ENTRY_FAIL_DEFAULT; @@ -4524,7 +4523,7 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, vcpu->arch.pat = vmcs12->host_ia32_pat; } if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) && - intel_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu))) + kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu))) WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL, vmcs12->host_ia32_perf_global_ctrl)); diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 741efe2c497b..80c769c58a87 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -73,18 +73,6 @@ static struct kvm_pmc *intel_pmc_idx_to_pmc(struct kvm_pmu *pmu, int pmc_idx) } } -static void reprogram_counters(struct kvm_pmu *pmu, u64 diff) -{ - int bit; - - if (!diff) - return; - - for_each_set_bit(bit, (unsigned long *)&diff, X86_PMC_IDX_MAX) - set_bit(bit, pmu->reprogram_pmi); - kvm_make_request(KVM_REQ_PMU, pmu_to_vcpu(pmu)); -} - static bool intel_hw_event_available(struct kvm_pmc *pmc) { struct kvm_pmu *pmu = pmc_to_pmu(pmc); @@ -107,17 +95,6 @@ static bool intel_hw_event_available(struct kvm_pmc *pmc) return true; } -/* check if a PMC is enabled by comparing it with globl_ctrl bits. */ -static bool intel_pmc_is_enabled(struct kvm_pmc *pmc) -{ - struct kvm_pmu *pmu = pmc_to_pmu(pmc); - - if (!intel_pmu_has_perf_global_ctrl(pmu)) - return true; - - return test_bit(pmc->idx, (unsigned long *)&pmu->global_ctrl); -} - static bool intel_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); @@ -198,11 +175,7 @@ static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) switch (msr) { case MSR_CORE_PERF_FIXED_CTR_CTRL: - case MSR_CORE_PERF_GLOBAL_STATUS: - case MSR_CORE_PERF_GLOBAL_CTRL: - case MSR_CORE_PERF_GLOBAL_OVF_CTRL: - return intel_pmu_has_perf_global_ctrl(pmu); - break; + return kvm_pmu_has_perf_global_ctrl(pmu); case MSR_IA32_PEBS_ENABLE: ret = vcpu_get_perf_capabilities(vcpu) & PERF_CAP_PEBS_FORMAT; break; @@ -352,15 +325,6 @@ static int intel_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) case MSR_CORE_PERF_FIXED_CTR_CTRL: msr_info->data = pmu->fixed_ctr_ctrl; break; - case MSR_CORE_PERF_GLOBAL_STATUS: - msr_info->data = pmu->global_status; - break; - case MSR_CORE_PERF_GLOBAL_CTRL: - msr_info->data = pmu->global_ctrl; - break; - case MSR_CORE_PERF_GLOBAL_OVF_CTRL: - msr_info->data = 0; - break; case MSR_IA32_PEBS_ENABLE: msr_info->data = pmu->pebs_enable; break; @@ -410,29 +374,6 @@ static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (pmu->fixed_ctr_ctrl != data) reprogram_fixed_counters(pmu, data); break; - case MSR_CORE_PERF_GLOBAL_STATUS: - if (!msr_info->host_initiated) - return 1; /* RO MSR */ - - pmu->global_status = data; - break; - case MSR_CORE_PERF_GLOBAL_CTRL: - if (!kvm_valid_perf_global_ctrl(pmu, data)) - return 1; - - if (pmu->global_ctrl != data) { - diff = pmu->global_ctrl ^ data; - pmu->global_ctrl = data; - reprogram_counters(pmu, diff); - } - break; - case MSR_CORE_PERF_GLOBAL_OVF_CTRL: - if (data & pmu->global_ovf_ctrl_mask) - return 1; - - if (!msr_info->host_initiated) - pmu->global_status &= ~data; - break; case MSR_IA32_PEBS_ENABLE: if (data & pmu->pebs_enable_mask) return 1; @@ -444,8 +385,6 @@ static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) } break; case MSR_IA32_DS_AREA: - if (msr_info->host_initiated && data && !guest_cpuid_has(vcpu, X86_FEATURE_DS)) - return 1; if (is_noncanonical_address(data, vcpu)) return 1; @@ -531,7 +470,7 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) pmu->reserved_bits = 0xffffffff00200000ull; pmu->raw_event_mask = X86_RAW_EVENT_MASK; pmu->global_ctrl_mask = ~0ull; - pmu->global_ovf_ctrl_mask = ~0ull; + pmu->global_status_mask = ~0ull; pmu->fixed_ctr_ctrl_mask = ~0ull; pmu->pebs_enable_mask = ~0ull; pmu->pebs_data_cfg_mask = ~0ull; @@ -585,11 +524,17 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) counter_mask = ~(((1ull << pmu->nr_arch_gp_counters) - 1) | (((1ull << pmu->nr_arch_fixed_counters) - 1) << INTEL_PMC_IDX_FIXED)); pmu->global_ctrl_mask = counter_mask; - pmu->global_ovf_ctrl_mask = pmu->global_ctrl_mask + + /* + * GLOBAL_STATUS and GLOBAL_OVF_CONTROL (a.k.a. GLOBAL_STATUS_RESET) + * share reserved bit definitions. The kernel just happens to use + * OVF_CTRL for the names. + */ + pmu->global_status_mask = pmu->global_ctrl_mask & ~(MSR_CORE_PERF_GLOBAL_OVF_CTRL_OVF_BUF | MSR_CORE_PERF_GLOBAL_OVF_CTRL_COND_CHGD); if (vmx_pt_mode_is_host_guest()) - pmu->global_ovf_ctrl_mask &= + pmu->global_status_mask &= ~MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI; entry = kvm_find_cpuid_entry_index(vcpu, 7, 0); @@ -801,7 +746,7 @@ void intel_pmu_cross_mapped_check(struct kvm_pmu *pmu) pmc = intel_pmc_idx_to_pmc(pmu, bit); if (!pmc || !pmc_speculative_in_use(pmc) || - !intel_pmc_is_enabled(pmc) || !pmc->perf_event) + !pmc_is_globally_enabled(pmc) || !pmc->perf_event) continue; /* @@ -816,7 +761,6 @@ void intel_pmu_cross_mapped_check(struct kvm_pmu *pmu) struct kvm_pmu_ops intel_pmu_ops __initdata = { .hw_event_available = intel_hw_event_available, - .pmc_is_enabled = intel_pmc_is_enabled, .pmc_idx_to_pmc = intel_pmc_idx_to_pmc, .rdpmc_ecx_to_pmc = intel_rdpmc_ecx_to_pmc, .msr_idx_to_pmc = intel_msr_idx_to_pmc, @@ -831,4 +775,5 @@ struct kvm_pmu_ops intel_pmu_ops __initdata = { .cleanup = intel_pmu_cleanup, .EVENTSEL_EVENT = ARCH_PERFMON_EVENTSEL_EVENT, .MAX_NR_GP_COUNTERS = KVM_INTEL_PMC_MAX_GENERIC, + .MIN_NR_GP_COUNTERS = 1, }; diff --git a/arch/x86/kvm/vmx/sgx.c b/arch/x86/kvm/vmx/sgx.c index 2261b684a7d4..3e822e582497 100644 --- a/arch/x86/kvm/vmx/sgx.c +++ b/arch/x86/kvm/vmx/sgx.c @@ -357,11 +357,12 @@ static int handle_encls_einit(struct kvm_vcpu *vcpu) static inline bool encls_leaf_enabled_in_guest(struct kvm_vcpu *vcpu, u32 leaf) { - if (!enable_sgx || !guest_cpuid_has(vcpu, X86_FEATURE_SGX)) - return false; - + /* + * ENCLS generates a #UD if SGX1 isn't supported, i.e. this point will + * be reached if and only if the SGX1 leafs are enabled. + */ if (leaf >= ECREATE && leaf <= ETRACK) - return guest_cpuid_has(vcpu, X86_FEATURE_SGX1); + return true; if (leaf >= EAUG && leaf <= EMODT) return guest_cpuid_has(vcpu, X86_FEATURE_SGX2); @@ -380,9 +381,11 @@ int handle_encls(struct kvm_vcpu *vcpu) { u32 leaf = (u32)kvm_rax_read(vcpu); - if (!encls_leaf_enabled_in_guest(vcpu, leaf)) { + if (!enable_sgx || !guest_cpuid_has(vcpu, X86_FEATURE_SGX) || + !guest_cpuid_has(vcpu, X86_FEATURE_SGX1)) { kvm_queue_exception(vcpu, UD_VECTOR); - } else if (!sgx_enabled_in_guest_bios(vcpu)) { + } else if (!encls_leaf_enabled_in_guest(vcpu, leaf) || + !sgx_enabled_in_guest_bios(vcpu) || !is_paging(vcpu)) { kvm_inject_gp(vcpu, 0); } else { if (leaf == ECREATE) diff --git a/arch/x86/kvm/vmx/vmenter.S b/arch/x86/kvm/vmx/vmenter.S index 631fd7da2bc3..07e927d4d099 100644 --- a/arch/x86/kvm/vmx/vmenter.S +++ b/arch/x86/kvm/vmx/vmenter.S @@ -187,7 +187,7 @@ SYM_FUNC_START(__vmx_vcpu_run) _ASM_EXTABLE(.Lvmresume, .Lfixup) _ASM_EXTABLE(.Lvmlaunch, .Lfixup) -SYM_INNER_LABEL(vmx_vmexit, SYM_L_GLOBAL) +SYM_INNER_LABEL_ALIGN(vmx_vmexit, SYM_L_GLOBAL) /* Restore unwind state from before the VMRESUME/VMLAUNCH. */ UNWIND_HINT_RESTORE diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 44fb619803b8..0ecf4be2c6af 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2287,19 +2287,16 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) return 1; goto find_uret_msr; case MSR_IA32_CR_PAT: - if (!kvm_pat_valid(data)) - return 1; + ret = kvm_set_msr_common(vcpu, msr_info); + if (ret) + break; if (is_guest_mode(vcpu) && get_vmcs12(vcpu)->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT) get_vmcs12(vcpu)->guest_ia32_pat = data; - if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) { + if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) vmcs_write64(GUEST_IA32_PAT, data); - vcpu->arch.pat = data; - break; - } - ret = kvm_set_msr_common(vcpu, msr_info); break; case MSR_IA32_MCG_EXT_CTL: if ((!msr_info->host_initiated && @@ -3387,15 +3384,15 @@ static bool vmx_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) { - unsigned long old_cr4 = vcpu->arch.cr4; + unsigned long old_cr4 = kvm_read_cr4(vcpu); struct vcpu_vmx *vmx = to_vmx(vcpu); + unsigned long hw_cr4; + /* * Pass through host's Machine Check Enable value to hw_cr4, which * is in force while we are in guest mode. Do not let guests control * this bit, even if host CR4.MCE == 0. */ - unsigned long hw_cr4; - hw_cr4 = (cr4_read_shadow() & X86_CR4_MCE) | (cr4 & ~X86_CR4_MCE); if (is_unrestricted_guest(vcpu)) hw_cr4 |= KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST; @@ -3404,7 +3401,7 @@ void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) else hw_cr4 |= KVM_PMODE_VM_CR4_ALWAYS_ON; - if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated()) { + if (vmx_umip_emulated()) { if (cr4 & X86_CR4_UMIP) { secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_DESC); hw_cr4 &= ~X86_CR4_UMIP; @@ -5402,7 +5399,13 @@ static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val) static int handle_desc(struct kvm_vcpu *vcpu) { - WARN_ON(!(vcpu->arch.cr4 & X86_CR4_UMIP)); + /* + * UMIP emulation relies on intercepting writes to CR4.UMIP, i.e. this + * and other code needs to be updated if UMIP can be guest owned. + */ + BUILD_BUG_ON(KVM_POSSIBLE_CR4_GUEST_BITS & X86_CR4_UMIP); + + WARN_ON_ONCE(!kvm_is_cr4_bit_set(vcpu, X86_CR4_UMIP)); return kvm_emulate_instruction(vcpu, 0); } @@ -6708,7 +6711,12 @@ void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu) static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu) { - struct page *page; + const gfn_t gfn = APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT; + struct kvm *kvm = vcpu->kvm; + struct kvm_memslots *slots = kvm_memslots(kvm); + struct kvm_memory_slot *slot; + unsigned long mmu_seq; + kvm_pfn_t pfn; /* Defer reload until vmcs01 is the current VMCS. */ if (is_guest_mode(vcpu)) { @@ -6720,18 +6728,53 @@ static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu) SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) return; - page = gfn_to_page(vcpu->kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT); - if (is_error_page(page)) + /* + * Grab the memslot so that the hva lookup for the mmu_notifier retry + * is guaranteed to use the same memslot as the pfn lookup, i.e. rely + * on the pfn lookup's validation of the memslot to ensure a valid hva + * is used for the retry check. + */ + slot = id_to_memslot(slots, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT); + if (!slot || slot->flags & KVM_MEMSLOT_INVALID) return; - vmcs_write64(APIC_ACCESS_ADDR, page_to_phys(page)); + /* + * Ensure that the mmu_notifier sequence count is read before KVM + * retrieves the pfn from the primary MMU. Note, the memslot is + * protected by SRCU, not the mmu_notifier. Pairs with the smp_wmb() + * in kvm_mmu_invalidate_end(). + */ + mmu_seq = kvm->mmu_invalidate_seq; + smp_rmb(); + + /* + * No need to retry if the memslot does not exist or is invalid. KVM + * controls the APIC-access page memslot, and only deletes the memslot + * if APICv is permanently inhibited, i.e. the memslot won't reappear. + */ + pfn = gfn_to_pfn_memslot(slot, gfn); + if (is_error_noslot_pfn(pfn)) + return; + + read_lock(&vcpu->kvm->mmu_lock); + if (mmu_invalidate_retry_hva(kvm, mmu_seq, + gfn_to_hva_memslot(slot, gfn))) { + kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); + read_unlock(&vcpu->kvm->mmu_lock); + goto out; + } + + vmcs_write64(APIC_ACCESS_ADDR, pfn_to_hpa(pfn)); + read_unlock(&vcpu->kvm->mmu_lock); + vmx_flush_tlb_current(vcpu); +out: /* * Do not pin apic access page in memory, the MMU notifier * will call us again if it is migrated or swapped out. */ - put_page(page); + kvm_release_pfn_clean(pfn); } static void vmx_hwapic_isr_update(int max_isr) diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 9e66531861cf..32384ba38499 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -93,18 +93,6 @@ union vmx_exit_reason { u32 full; }; -static inline bool intel_pmu_has_perf_global_ctrl(struct kvm_pmu *pmu) -{ - /* - * Architecturally, Intel's SDM states that IA32_PERF_GLOBAL_CTRL is - * supported if "CPUID.0AH: EAX[7:0] > 0", i.e. if the PMU version is - * greater than zero. However, KVM only exposes and emulates the MSR - * to/for the guest if the guest PMU supports at least "Architectural - * Performance Monitoring Version 2". - */ - return pmu->version > 1; -} - struct lbr_desc { /* Basic info about guest LBR records. */ struct x86_pmu_lbr records; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 7f70207e8689..a6b9bea62fb8 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1017,13 +1017,11 @@ void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu) wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss); } -#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS - if (static_cpu_has(X86_FEATURE_PKU) && + if (cpu_feature_enabled(X86_FEATURE_PKU) && vcpu->arch.pkru != vcpu->arch.host_pkru && ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) || kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE))) write_pkru(vcpu->arch.pkru); -#endif /* CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS */ } EXPORT_SYMBOL_GPL(kvm_load_guest_xsave_state); @@ -1032,15 +1030,13 @@ void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu) if (vcpu->arch.guest_state_protected) return; -#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS - if (static_cpu_has(X86_FEATURE_PKU) && + if (cpu_feature_enabled(X86_FEATURE_PKU) && ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) || kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE))) { vcpu->arch.pkru = rdpkru(); if (vcpu->arch.pkru != vcpu->arch.host_pkru) write_pkru(vcpu->arch.host_pkru); } -#endif /* CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS */ if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) { @@ -1427,15 +1423,14 @@ int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu) EXPORT_SYMBOL_GPL(kvm_emulate_rdpmc); /* - * List of msr numbers which we expose to userspace through KVM_GET_MSRS - * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. - * - * The three MSR lists(msrs_to_save, emulated_msrs, msr_based_features) - * extract the supported MSRs from the related const lists. - * msrs_to_save is selected from the msrs_to_save_all to reflect the - * capabilities of the host cpu. This capabilities test skips MSRs that are - * kvm-specific. Those are put in emulated_msrs_all; filtering of emulated_msrs - * may depend on host virtualization features rather than host cpu features. + * The three MSR lists(msrs_to_save, emulated_msrs, msr_based_features) track + * the set of MSRs that KVM exposes to userspace through KVM_GET_MSRS, + * KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. msrs_to_save holds MSRs that + * require host support, i.e. should be probed via RDMSR. emulated_msrs holds + * MSRs that KVM emulates without strictly requiring host support. + * msr_based_features holds MSRs that enumerate features, i.e. are effectively + * CPUID leafs. Note, msr_based_features isn't mutually exclusive with + * msrs_to_save and emulated_msrs. */ static const u32 msrs_to_save_base[] = { @@ -1483,6 +1478,10 @@ static const u32 msrs_to_save_pmu[] = { MSR_F15H_PERF_CTL3, MSR_F15H_PERF_CTL4, MSR_F15H_PERF_CTL5, MSR_F15H_PERF_CTR0, MSR_F15H_PERF_CTR1, MSR_F15H_PERF_CTR2, MSR_F15H_PERF_CTR3, MSR_F15H_PERF_CTR4, MSR_F15H_PERF_CTR5, + + MSR_AMD64_PERF_CNTR_GLOBAL_CTL, + MSR_AMD64_PERF_CNTR_GLOBAL_STATUS, + MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, }; static u32 msrs_to_save[ARRAY_SIZE(msrs_to_save_base) + @@ -1531,11 +1530,11 @@ static const u32 emulated_msrs_all[] = { MSR_IA32_UCODE_REV, /* - * The following list leaves out MSRs whose values are determined - * by arch/x86/kvm/vmx/nested.c based on CPUID or other MSRs. - * We always support the "true" VMX control MSRs, even if the host - * processor does not, so I am putting these registers here rather - * than in msrs_to_save_all. + * KVM always supports the "true" VMX control MSRs, even if the host + * does not. The VMX MSRs as a whole are considered "emulated" as KVM + * doesn't strictly require them to exist in the host (ignoring that + * KVM would refuse to load in the first place if the core set of MSRs + * aren't supported). */ MSR_IA32_VMX_BASIC, MSR_IA32_VMX_TRUE_PINBASED_CTLS, @@ -1631,7 +1630,7 @@ static u64 kvm_get_arch_capabilities(void) * If we're doing cache flushes (either "always" or "cond") * we will do one whenever the guest does a vmlaunch/vmresume. * If an outer hypervisor is doing the cache flush for us - * (VMENTER_L1D_FLUSH_NESTED_VM), we can safely pass that + * (ARCH_CAP_SKIP_VMENTRY_L1DFLUSH), we can safely pass that * capability to the guest too, and if EPT is disabled we're not * vulnerable. Overall, only VMENTER_L1D_FLUSH_NEVER will * require a nested hypervisor to do a flush of its own. @@ -1809,7 +1808,7 @@ bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type) unsigned long *bitmap = ranges[i].bitmap; if ((index >= start) && (index < end) && (flags & type)) { - allowed = !!test_bit(index - start, bitmap); + allowed = test_bit(index - start, bitmap); break; } } @@ -3701,8 +3700,14 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) return 1; } break; - case 0x200 ... MSR_IA32_MC0_CTL2 - 1: - case MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) ... 0x2ff: + case MSR_IA32_CR_PAT: + if (!kvm_pat_valid(data)) + return 1; + + vcpu->arch.pat = data; + break; + case MTRRphysBase_MSR(0) ... MSR_MTRRfix4K_F8000: + case MSR_MTRRdefType: return kvm_mtrr_set_msr(vcpu, msr, data); case MSR_IA32_APICBASE: return kvm_set_apic_base(vcpu, msr_info); @@ -4109,9 +4114,12 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) msr_info->data = kvm_scale_tsc(rdtsc(), ratio) + offset; break; } + case MSR_IA32_CR_PAT: + msr_info->data = vcpu->arch.pat; + break; case MSR_MTRRcap: - case 0x200 ... MSR_IA32_MC0_CTL2 - 1: - case MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) ... 0x2ff: + case MTRRphysBase_MSR(0) ... MSR_MTRRfix4K_F8000: + case MSR_MTRRdefType: return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data); case 0xcd: /* fsb frequency */ msr_info->data = 3; @@ -7149,6 +7157,12 @@ static void kvm_probe_msr_to_save(u32 msr_index) kvm_pmu_cap.num_counters_fixed) return; break; + case MSR_AMD64_PERF_CNTR_GLOBAL_CTL: + case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS: + case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR: + if (!kvm_cpu_cap_has(X86_FEATURE_PERFMON_V2)) + return; + break; case MSR_IA32_XFD: case MSR_IA32_XFD_ERR: if (!kvm_cpu_cap_has(X86_FEATURE_XFD)) @@ -10434,20 +10448,6 @@ static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu) vcpu, (u64 *)vcpu->arch.ioapic_handled_vectors); } -void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, - unsigned long start, unsigned long end) -{ - unsigned long apic_address; - - /* - * The physical address of apic access page is stored in the VMCS. - * Update it when it becomes invalid. - */ - apic_address = gfn_to_hva(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT); - if (start <= apic_address && apic_address < end) - kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD); -} - void kvm_arch_guest_memory_reclaimed(struct kvm *kvm) { static_call_cond(kvm_x86_guest_memory_reclaimed)(kvm); diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index c544602d07a3..82e3dafc5453 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -309,7 +309,6 @@ void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, void kvm_vcpu_mtrr_init(struct kvm_vcpu *vcpu); u8 kvm_mtrr_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn); -bool kvm_mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data); int kvm_mtrr_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data); int kvm_mtrr_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata); bool kvm_mtrr_check_gfn_range_consistency(struct kvm_vcpu *vcpu, gfn_t gfn, diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c index 232acf418cfb..77f7ac3668cb 100644 --- a/arch/x86/platform/efi/efi_64.c +++ b/arch/x86/platform/efi/efi_64.c @@ -853,9 +853,9 @@ efi_set_virtual_address_map(unsigned long memory_map_size, /* Disable interrupts around EFI calls: */ local_irq_save(flags); - status = efi_call(efi.runtime->set_virtual_address_map, - memory_map_size, descriptor_size, - descriptor_version, virtual_map); + status = arch_efi_call_virt(efi.runtime, set_virtual_address_map, + memory_map_size, descriptor_size, + descriptor_version, virtual_map); local_irq_restore(flags); efi_fpu_end(); diff --git a/arch/x86/platform/efi/quirks.c b/arch/x86/platform/efi/quirks.c index b0b848d6933a..f0cc00032751 100644 --- a/arch/x86/platform/efi/quirks.c +++ b/arch/x86/platform/efi/quirks.c @@ -114,6 +114,14 @@ void efi_delete_dummy_variable(void) EFI_VARIABLE_RUNTIME_ACCESS, 0, NULL); } +u64 efivar_reserved_space(void) +{ + if (efi_no_storage_paranoia) + return 0; + return EFI_MIN_RESERVE; +} +EXPORT_SYMBOL_GPL(efivar_reserved_space); + /* * In the nonblocking case we do not attempt to perform garbage * collection if we do not have enough free space. Rather, we do the diff --git a/arch/x86/platform/efi/runtime-map.c b/arch/x86/platform/efi/runtime-map.c index bbee682ef8cd..a6f02cef3ca2 100644 --- a/arch/x86/platform/efi/runtime-map.c +++ b/arch/x86/platform/efi/runtime-map.c @@ -93,7 +93,7 @@ static void map_release(struct kobject *kobj) kfree(entry); } -static struct kobj_type __refdata map_ktype = { +static const struct kobj_type __refconst map_ktype = { .sysfs_ops = &map_attr_ops, .default_groups = def_groups, .release = map_release, diff --git a/arch/xtensa/lib/Makefile b/arch/xtensa/lib/Makefile index 6e5b2232668c..ec0d8d233c74 100644 --- a/arch/xtensa/lib/Makefile +++ b/arch/xtensa/lib/Makefile @@ -9,5 +9,3 @@ lib-y += memcopy.o memset.o checksum.o \ usercopy.o strnlen_user.o lib-$(CONFIG_ARCH_HAS_STRNCPY_FROM_USER) += strncpy_user.o lib-$(CONFIG_PCI) += pci-auto.o -lib-$(CONFIG_KCSAN) += kcsan-stubs.o -KCSAN_SANITIZE_kcsan-stubs.o := n diff --git a/arch/xtensa/lib/kcsan-stubs.c b/arch/xtensa/lib/kcsan-stubs.c deleted file mode 100644 index 2b08faa62b86..000000000000 --- a/arch/xtensa/lib/kcsan-stubs.c +++ /dev/null @@ -1,54 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include <linux/bug.h> -#include <linux/types.h> - -void __atomic_store_8(volatile void *p, u64 v, int i) -{ - BUG(); -} - -u64 __atomic_load_8(const volatile void *p, int i) -{ - BUG(); -} - -u64 __atomic_exchange_8(volatile void *p, u64 v, int i) -{ - BUG(); -} - -bool __atomic_compare_exchange_8(volatile void *p1, void *p2, u64 v, bool b, int i1, int i2) -{ - BUG(); -} - -u64 __atomic_fetch_add_8(volatile void *p, u64 v, int i) -{ - BUG(); -} - -u64 __atomic_fetch_sub_8(volatile void *p, u64 v, int i) -{ - BUG(); -} - -u64 __atomic_fetch_and_8(volatile void *p, u64 v, int i) -{ - BUG(); -} - -u64 __atomic_fetch_or_8(volatile void *p, u64 v, int i) -{ - BUG(); -} - -u64 __atomic_fetch_xor_8(volatile void *p, u64 v, int i) -{ - BUG(); -} - -u64 __atomic_fetch_nand_8(volatile void *p, u64 v, int i) -{ - BUG(); -} |