Deal with the non-standard XTheadMae (Memory Attribute Extensions)

thorpej · thorpej · commit de08c0b952db · 2025-10-12T04:08:26.000Z
present on some T-Head CPU cores, notably the one found in the
Allwinner D1 SoC.

This extension allows memory attributes (cacheable, bufferable,
strongly-ordered, etc.) to be specified on a per-mapping basis.
Alas, it has two unfortunate properties:
- It uses the same bits as the standard Svpbmt (Page Based Memory Types)
  extension, and the bits are completely incompatible.
- Unlike Svpbmt, which if none of its extension bits are set in the
  PTE, results in rational behavior, if you don't set the XTheadMae
  attibutes in the PTE correctly, the system will blow up in your face
  in interesting and unpredictable ways.

So, we have to probe for XTheadMae early, when we're setting up the
initial MMU tables so that the kernel is mapped correctly when the
MMU is enabled, and then we probe for it again to set some globals
that are used when creating PTEs for new kernel and user mappings
going forward.  Luckily, there are combinations of XTheadMae attributes
that map reasonably well to the Svpbmt types, so Svpbmt is used as
the abstraction.

With this, my D1 Nezha board boots to the root device prompt. \o/
diff --git a/sys/arch/riscv/include/pmap.h b/sys/arch/riscv/include/pmap.h
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.24 2024/08/04 08:16:25 skrll Exp $ */
+/* $NetBSD: pmap.h,v 1.25 2025/10/12 04:08:26 thorpej Exp $ */
 
 /*
  * Copyright (c) 2014, 2019, 2021 The NetBSD Foundation, Inc.
@@ -153,6 +153,7 @@ void	pmap_md_tlb_info_attach(struct pmap_tlb_info *, struct cpu_info *);
 void	pmap_md_xtab_activate(struct pmap *, struct lwp *);
 void	pmap_md_xtab_deactivate(struct pmap *);
 
+void	pmap_probe_pbmt(void);
 void	pmap_bootstrap(vaddr_t, vaddr_t);
 
 vsize_t	pmap_kenter_range(vaddr_t, paddr_t, vsize_t, vm_prot_t, u_int);
diff --git a/sys/arch/riscv/include/pte.h b/sys/arch/riscv/include/pte.h
@@ -1,4 +1,4 @@
-/* $NetBSD: pte.h,v 1.14 2024/10/12 12:27:33 skrll Exp $ */
+/* $NetBSD: pte.h,v 1.15 2025/10/12 04:08:26 thorpej Exp $ */
 
 /*
  * Copyright (c) 2014, 2019, 2021 The NetBSD Foundation, Inc.
@@ -62,6 +62,36 @@ typedef uint32_t pd_entry_t;
 #define PTE_PBMT	__BITS(62, 61)	// Svpbmt
 #define PTE_reserved0	__BITS(60, 54)	//
 
+/*
+ * Svpbmt (Page Based Memory Types) extension:
+ *
+ * PMA --> adhere to physical memory attributes
+ * NC  --> non-cacheable, idempotent, weakly-ordered
+ * IO  --> non-cacheable, non-idempotent, strongly-ordered
+ */
+#define PTE_PBMT_PMA	__SHIFTIN(0, PTE_PBMT)
+#define PTE_PBMT_NC	__SHIFTIN(1, PTE_PBMT)
+#define PTE_PBMT_IO	__SHIFTIN(2, PTE_PBMT)
+
+/* XTheadMae (Memory Attribute Extensions) */
+#define PTE_XMAE	__BITS(63,59)
+#define PTE_XMAE_SO	__BIT(63)	// Strong Order
+#define PTE_XMAE_C	__BIT(62)	// Cacheable
+#define PTE_XMAE_B	__BIT(61)	// Bufferable
+#define PTE_XMAE_SH	__BIT(60)	// Shareable
+#define PTE_XMAE_T	__BIT(59)	// Trustable
+
+/*
+ * Map to the rough PBMT equivalent:
+ *
+ * PMA (i.e. no specific attribute) -->     C  B  SH
+ * NC                               -->        B  SH
+ * IO                               --> SO        SH
+ */
+#define PTE_XMAE_PMA	(              PTE_XMAE_C | PTE_XMAE_B | PTE_XMAE_SH)
+#define PTE_XMAE_NC	(                           PTE_XMAE_B | PTE_XMAE_SH)
+#define PTE_XMAE_IO	(PTE_XMAE_SO                           | PTE_XMAE_SH)
+
 /* Software PTE bits. */
 #define	PTE_RSW		__BITS(9, 8)
 #define	PTE_WIRED	__BIT(9)
@@ -87,7 +117,7 @@ typedef uint32_t pd_entry_t;
 #define	PTE_ISLEAF_P(pte) (((pte) & PTE_RWX) != 0)
 
 #define	PA_TO_PTE(pa)	(((pa) >> PGSHIFT) << PTE_PPN_SHIFT)
-#define	PTE_TO_PA(pte)	(((pte) >> PTE_PPN_SHIFT) << PGSHIFT)
+#define	PTE_TO_PA(pte)	(__SHIFTOUT((pte), PTE_PPN) << PGSHIFT)
 
 #if defined(_KERNEL)
 
@@ -186,6 +216,8 @@ pte_flag_bits(struct vm_page_md *mdpg, int flags, bool kernel_p)
 	return 0;
 }
 
+pt_entry_t	pte_enter_flags_to_pbmt(int);
+
 static inline pt_entry_t
 pte_make_enter(paddr_t pa, struct vm_page_md *mdpg, vm_prot_t prot,
     int flags, bool kernel_p)
@@ -195,6 +227,7 @@ pte_make_enter(paddr_t pa, struct vm_page_md *mdpg, vm_prot_t prot,
 	pte |= kernel_p ? PTE_KERN : PTE_USER;
 	pte |= pte_flag_bits(mdpg, flags, kernel_p);
 	pte |= pte_prot_bits(mdpg, prot, kernel_p);
+	pte |= pte_enter_flags_to_pbmt(flags);
 
 	if (mdpg != NULL) {
 
@@ -232,6 +265,7 @@ pte_make_kenter_pa(paddr_t pa, struct vm_page_md *mdpg, vm_prot_t prot,
 	pte |= PTE_KERN | PTE_HARDWIRED | PTE_WIRED;
 	pte |= pte_flag_bits(NULL, flags, true);
 	pte |= pte_prot_bits(NULL, prot, true);
+	pte |= pte_enter_flags_to_pbmt(flags);
 
 	return pte;
 }
diff --git a/sys/arch/riscv/include/sysreg.h b/sys/arch/riscv/include/sysreg.h
@@ -1,4 +1,4 @@
-/* $NetBSD: sysreg.h,v 1.33 2024/05/14 15:17:57 riastradh Exp $ */
+/* $NetBSD: sysreg.h,v 1.34 2025/10/12 04:08:26 thorpej Exp $ */
 
 /*
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -38,6 +38,12 @@
 
 #include <riscv/reg.h>
 
+/* CPU vendors (get CSR value from SBI). */
+#define CPU_VENDOR_SIFIVE		0x489
+#define  CPU_SIFIVE_ARCH_7SERIES	0x8000000000000007
+
+#define CPU_VENDOR_THEAD		0x5b7
+
 #define	FCSR_FMASK	0	// no exception bits
 #define	FCSR_FRM	__BITS(7, 5)
 #define	 FCSR_FRM_RNE	0b000	// Round Nearest, ties to Even
@@ -333,4 +339,16 @@ csr_asid_write(uint32_t asid)
 	csr_satp_write(satp);
 }
 
+/* Non-standard CSRs. */
+static inline uintptr_t
+csr_thead_sxstatus_read(void)
+{
+	uintptr_t __rv;
+	asm volatile("csrr %0, 0x5c0" : "=r"(__rv) :: "memory");
+	return __rv;
+}               
+
+#define	TX_SXSTATUS_MAEE	__BIT(21)
+#define	TH_SXSTATUS_THEADISAEE	__BIT(22)
+
 #endif /* _RISCV_SYSREG_H_ */
diff --git a/sys/arch/riscv/riscv/bus_space.c b/sys/arch/riscv/riscv/bus_space.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: bus_space.c,v 1.2 2023/05/07 12:41:48 skrll Exp $	*/
+/*	$NetBSD: bus_space.c,v 1.3 2025/10/12 04:08:26 thorpej Exp $	*/
 
 /*-
  * Copyright (c) 2022 The NetBSD Foundation, Inc.
@@ -32,7 +32,7 @@
 #include "opt_console.h"
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: bus_space.c,v 1.2 2023/05/07 12:41:48 skrll Exp $");
+__KERNEL_RCSID(1, "$NetBSD: bus_space.c,v 1.3 2025/10/12 04:08:26 thorpej Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -326,10 +326,8 @@ generic_bs_map(void *t, bus_addr_t bpa, bus_size_t size, int flag,
 	else if ((flag & _RISCV_BUS_SPACE_MAP_STRONGLY_ORDERED) != 0)
 		pmapflags = PMAP_DEV_NP;
 	else
-		pmapflags = PMAP_DEV;
 #endif
-
-	pmapflags = 0;
+		pmapflags = PMAP_DEV;
 
 	for (pa = startpa; pa < endpa; pa += PAGE_SIZE, va += PAGE_SIZE) {
 		pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE, pmapflags);
diff --git a/sys/arch/riscv/riscv/cpu.c b/sys/arch/riscv/riscv/cpu.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: cpu.c,v 1.9 2025/10/07 06:40:33 skrll Exp $	*/
+/*	$NetBSD: cpu.c,v 1.10 2025/10/12 04:08:26 thorpej Exp $	*/
 
 /*-
  * Copyright (c) 2023 The NetBSD Foundation, Inc.
@@ -32,7 +32,7 @@
 #include "opt_multiprocessor.h"
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.9 2025/10/07 06:40:33 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.10 2025/10/12 04:08:26 thorpej Exp $");
 
 #include <sys/param.h>
 
@@ -46,6 +46,7 @@ __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.9 2025/10/07 06:40:33 skrll Exp $");
 #include <riscv/cpuvar.h>
 #include <riscv/machdep.h>
 #include <riscv/sbi.h>
+#include <riscv/sysreg.h>
 
 #ifdef MULTIPROCESSOR
 #define NCPUINFO	MAXCPUS
@@ -65,20 +66,14 @@ void (*cpu_sdcache_wb_range)(vaddr_t, paddr_t, psize_t) = cache_nullop;
 u_int   riscv_dcache_align = CACHE_LINE_SIZE;
 u_int   riscv_dcache_align_mask = CACHE_LINE_SIZE - 1;
 
-#define CPU_VENDOR_SIFIVE	0x489
-
-#define CPU_ARCH_7SERIES	0x8000000000000007
-
-#define CPU_VENDOR_THEAD	0x5b7
-
 struct cpu_arch {
 	uint64_t	 ca_id;
 	const char	*ca_name;
 };
 
 struct cpu_arch cpu_arch_sifive[] = {
     {
-	.ca_id = CPU_ARCH_7SERIES,
+	.ca_id = CPU_SIFIVE_ARCH_7SERIES,
 	.ca_name = "7-Series Processor (E7, S7, U7 series)",
     },
     { },	// terminator
diff --git a/sys/arch/riscv/riscv/genassym.cf b/sys/arch/riscv/riscv/genassym.cf
@@ -1,4 +1,4 @@
-#	$NetBSD: genassym.cf,v 1.16 2024/04/06 10:08:54 skrll Exp $
+#	$NetBSD: genassym.cf,v 1.17 2025/10/12 04:08:26 thorpej Exp $
 
 #-
 # Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -227,6 +227,9 @@ define	PTE_V		PTE_V
 define	PTE_KERN	PTE_KERN
 define	PTE_HARDWIRED	PTE_HARDWIRED
 
+define	PTE_XMAE_PMA	PTE_XMAE_PMA
+define	PTE_XMAE_IO	PTE_XMAE_IO
+
 #define	PTE_PPN0_S	PTE_PPN0_S
 #define	PTE_PPN1_S	PTE_PPN1_S
 #define	PTE_PPN2_S	PTE_PPN2_S
@@ -237,6 +240,11 @@ define	PTE_PPN_SHIFT	PTE_PPN_SHIFT
 define	PM_PDETAB	offsetof(struct pmap, pm_pdetab)
 define	PM_MD_PPN	offsetof(struct pmap, pm_md.md_ppn)
 
+define	CPU_VENDOR_THEAD		CPU_VENDOR_THEAD
+define	TX_SXSTATUS_MAEE		TX_SXSTATUS_MAEE
+
+define	SBI_EID_BASE			SBI_EID_BASE
+define	SBI_FID_BASE_GETMVENDORID	SBI_FID_BASE_GETMVENDORID
 define	SBI_LEGACY_CONSOLE_PUTCHAR	SBI_LEGACY_CONSOLE_PUTCHAR
 
 # for bus_space_asm
diff --git a/sys/arch/riscv/riscv/locore.S b/sys/arch/riscv/riscv/locore.S
@@ -1,4 +1,4 @@
-/* $NetBSD: locore.S,v 1.46 2025/03/02 08:14:26 skrll Exp $ */
+/* $NetBSD: locore.S,v 1.47 2025/10/12 04:08:26 thorpej Exp $ */
 
 /*-
  * Copyright (c) 2014, 2022 The NetBSD Foundation, Inc.
@@ -118,6 +118,45 @@ ENTRY_NP(start)
 	PTR_LA	s5, _C_LABEL(lwp0uspace)
 	PTR_LA	s6, _C_LABEL(bootstk)
 
+	/*
+	 * Before we construct the initial MMU tables, we need to know
+	 * if there are any extra PTE bits that we need to include.
+	 * (Grumble, mumble, T-Head XMAE).
+	 *
+	 * We'll push space for 2 PTE prototypes, and fill them with
+	 * zeros for now.  sp[0] is for "regular memory", sp[1] is
+	 * for "devices".
+	 */
+	addi	sp, sp, -(2 * SZREG)
+	REG_S	zero, 0(sp)
+	REG_S	zero, SZREG(sp)
+
+	/* Get the CPU vendor ID, which requires an SBI call. */
+	VPRINTS("vendor:  ")
+	li	a7, SBI_EID_BASE
+	li	a6, SBI_FID_BASE_GETMVENDORID
+	ecall
+	VPRINTXNL(a1)
+
+#ifdef _LP64
+	li	a0, CPU_VENDOR_THEAD
+	bne	a0, a1, 9f		/* not T-Head, skip */
+	csrr	a1, 0x5c0		/* read T-Head TX.SXSTATUS reg */
+	li	a0, TX_SXSTATUS_MAEE	/* is MAEE bit set? */
+	and	a0, a0, a1
+	beqz	a0, .Lpdetab		/* nope, carry on */
+	VPRINTS("extsn:   T-Head XMAE\r\n")
+	li	a0, PTE_XMAE_PMA
+	REG_S	a0, 0(sp)
+	li	a0, PTE_XMAE_IO
+	REG_S	a0, SZREG(sp)
+	j	.Lpdetab		/* done. */
+9:
+#endif /* _LP64 */
+
+	/* Next vendor quirk check goes here. */
+
+.Lpdetab:
 	/*
 	 * Our load address is not fixed, but our VA is.  We need to construct
 	 * an initial PDETAB.
@@ -218,6 +257,8 @@ ENTRY_NP(start)
 	li	s5, (VM_KERNEL_SIZE >> SEGSHIFT)		// # of megapages
 	li	s6, (NBSEG >> (PGSHIFT - PTE_PPN_SHIFT))	// load for ease
 	li	s7, PTE_KERN | PTE_HARDWIRED | PTE_R | PTE_W | PTE_X
+	REG_L	a0, 0(sp)
+	or	s7, s7, a0
 
 	//
 	// Fill in the PDEs for kernel.
@@ -243,6 +284,8 @@ ENTRY_NP(start)
 	add	s9, s2, t1
 
 	li	s7, PTE_KERN | PTE_HARDWIRED | PTE_R | PTE_W
+	REG_L	a0, 0(sp)
+	or	s7, s7, a0
 
 	//
 	// Fill in the PDE for the DTB. Only do one - if any more are required
@@ -261,6 +304,9 @@ ENTRY_NP(start)
 	REG_S	s0, 0(s9)
 
 #ifdef CONSADDR
+	li	s7, PTE_KERN | PTE_HARDWIRED | PTE_R | PTE_W
+	REG_L	a0, SZREG(sp)
+	or	s7, s7, a0
 	li	t1,  ((VM_KERNEL_IO_BASE >> SEGSHIFT) & (NPDEPG - 1)) * SZREG
 	add	s9, s2, t1
 
@@ -279,6 +325,9 @@ ENTRY_NP(start)
 	REG_S	s0, 0(s9)
 #endif
 
+	/* Pop the PTE prototypes pushed way up above. */
+	addi	sp, sp, (2 * SZREG)
+
 	li	a0, 'P'
 	call	_C_LABEL(uartputc)
 
diff --git a/sys/arch/riscv/riscv/pmap_machdep.c b/sys/arch/riscv/riscv/pmap_machdep.c
diff --git a/sys/arch/riscv/riscv/riscv_machdep.c b/sys/arch/riscv/riscv/riscv_machdep.c
