Pull up following revision(s) (requested by skrll in ticket #1166):

	sys/arch/arm/arm32/bus_dma.c: revision 1.142
	sys/arch/arm/include/cpufunc.h: revision 1.92
	sys/arch/arm/arm32/bus_dma.c: revision 1.150

More KNF


arm: Adjust barriers issued in bus_dmamap_sync for the coherent case.
PR/59654

This change adjusts the memory barriers issued by bus_dmamap_sync for
the coherent case. In the non-coherent case the CPU cache operations
are expected to provide any, and all required barriers.

These barriers are emitted after bouncing for PREWRITE and before
bouncing for POSTREAD.

A new indrection is introduced to deal with the differencs in barrier
(dsb) options between Arm V8 and V7, the lack of options in Arm V6,
and the need to use cpu_drain_writebuf for all other earlier versions.

The Arm V8 Net change is:
op                      old                     new
---------------------   ----------------------  ------------
PREREAD                 none                    dsb(osh)
PREWRITE                cpu_drain_writebuf      dsb(oshst)
PREREAD|PREWRITE        cpu_drain_writebuf      dsb(osh)
POSTREAD                cpu_drain_writebuf      dsb(oshld)
POSTWRITE               none                    none
POSTREAD|POSTWRITE      cpu_drain_writebuf      dsb(oshld)
where cpu_drain_writebuf is a dsb(sy) or CPU equivalent.

Author: MartinHusemann

sys/arch/arm/arm32/bus_dma.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: bus_dma.c,v 1.138 2022/10/11 22:03:37 andvar Exp $	*/
+/*	$NetBSD: bus_dma.c,v 1.138.2.1 2025/10/01 16:54:19 martin Exp $	*/
 
 /*-
  * Copyright (c) 1996, 1997, 1998, 2020 The NetBSD Foundation, Inc.
@@ -36,7 +36,7 @@
 #include "opt_cputypes.h"
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: bus_dma.c,v 1.138 2022/10/11 22:03:37 andvar Exp $");
+__KERNEL_RCSID(0, "$NetBSD: bus_dma.c,v 1.138.2.1 2025/10/01 16:54:19 martin Exp $");
 
 #include <sys/param.h>
 
@@ -875,7 +875,7 @@ _bus_dmamap_sync_segment(vaddr_t va, paddr_t pa, vsize_t len, int ops,
 #endif
 
 	switch (ops) {
-	case BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE:
+	case BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE:
 		if (!readonly_p) {
 			STAT_INCR(sync_prereadwrite);
 			cpu_dcache_wbinv_range(va, len);
@@ -932,7 +932,7 @@ _bus_dmamap_sync_segment(vaddr_t va, paddr_t pa, vsize_t len, int ops,
 	 * Since these can't be dirty, we can just invalidate them and don't
 	 * have to worry about having to write back their contents.
 	 */
-	case BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE:
+	case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE:
 		STAT_INCR(sync_postreadwrite);
 		cpu_dcache_inv_range(va, len);
 		cpu_sdcache_inv_range(va, pa, len);
@@ -1102,8 +1102,8 @@ _bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
 	/*
 	 * Mixing of PRE and POST operations is not allowed.
 	 */
-	if ((ops & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)) != 0 &&
-	    (ops & (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)) != 0)
+	if ((ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) != 0 &&
+	    (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE)) != 0)
 		panic("%s: mix PRE and POST", __func__);
 
 	KASSERTMSG(offset < map->dm_mapsize,
@@ -1121,7 +1121,7 @@ _bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
 	 *	here in case a write-back is required by the back-end.
 	 *
 	 *	PREWRITE -- Write-back the D-cache.  Note that if
-	 *	we are doing a PREREAD|PREWRITE, we can collapse
+	 *	we are doing a PREREAD | PREWRITE, we can collapse
 	 *	the whole thing into a single Wb-Inv.
 	 *
 	 *	POSTREAD -- Re-invalidate the D-cache in case speculative
@@ -1136,9 +1136,9 @@ _bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
 	const bool bouncing = false;
 #endif
 
-	const int pre_ops = ops & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
+	const int pre_ops = ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 #if defined(CPU_CORTEX) || defined(CPU_ARMV8)
-	const int post_ops = ops & (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
+	const int post_ops = ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 #else
 	const int post_ops = 0;
 #endif
@@ -1196,38 +1196,77 @@ _bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
 		}
 	}
 
-	/* Skip cache frobbing if mapping was COHERENT */
+	/*
+	 * Provide appropriate memory barriers, and skip cache frobbing
+	 * if mapping is COHERENT.
+	 *
+	 * The case of PREREAD is as follows:
+	 *
+	 * 1. loads and stores before DMA buffer may be allocated for the purpose
+	 * 2. bus_dmamap_sync(BUS_DMASYNC_PREREAD)
+	 * 3. store to register or DMA descriptor to trigger DMA
+	 *
+	 *    The load/store-before-store ordering is ensured by dma_rw_w().
+	 *
+	 * The case of PREWRITE is as follows:
+	 *
+	 * 1. stores to DMA buffer. loads can happen later as the buffer is not changed
+	 *    by the device.
+	 * 2. bus_dmamap_sync(BUS_DMASYNC_PREWRITE)
+	 * 3. store to register or DMA descriptor to trigger DMA
+	 *
+	 *    The store-before-store ordering is ensured by dma_w_w().
+	 *
+	 * The case of POSTREAD is as follows:
+	 *
+	 * 1. load from register or DMA descriptor notifying DMA completion
+	 * 2. bus_dmamap_sync(BUS_DMASYNC_POSTREAD)
+	 * 3. loads from DMA buffer to use data, and stores to reuse buffer
+	 *
+	 *    The stores in (3) will not be speculated and, therefore, don't need
+	 *    specific handling. The load-before-load ordering is ensured by
+	 *    dma_r_r().
+	 *
+	 * The case of POSTWRITE is as follows:
+	 *
+	 * 1. load from register or DMA descriptor notifying DMA completion
+	 * 2. bus_dmamap_sync(BUS_DMASYNC_POSTWRITE)
+	 * 3. loads and stores to reuse buffer
+	 *
+	 *    The stores in (3) will not be speculated, and the load can happen at any
+	 *    time as the DMA buffer is not changed by the device so no barrier is
+	 *    required.
+	 */
+
 	if ((map->_dm_flags & _BUS_DMAMAP_COHERENT)) {
 		switch (ops) {
-		case BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE:
+		case BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE:
+			dma_rw_w();
 			STAT_INCR(sync_coherent_prereadwrite);
 			break;
 
 		case BUS_DMASYNC_PREREAD:
+			dma_rw_w();
 			STAT_INCR(sync_coherent_preread);
 			break;
 
 		case BUS_DMASYNC_PREWRITE:
+			dma_w_w();
 			STAT_INCR(sync_coherent_prewrite);
 			break;
 
-		case BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE:
+		case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE:
+			dma_r_r();
 			STAT_INCR(sync_coherent_postreadwrite);
 			break;
 
 		case BUS_DMASYNC_POSTREAD:
+			dma_r_r();
 			STAT_INCR(sync_coherent_postread);
 			break;
 
 		/* BUS_DMASYNC_POSTWRITE was aleady handled as a fastpath */
 		}
-		/*
-		 * Drain the write buffer of DMA operators.
-		 * 1) when cpu->device (prewrite)
-		 * 2) when device->cpu (postread)
-		 */
-		if ((pre_ops & BUS_DMASYNC_PREWRITE) || (post_ops & BUS_DMASYNC_POSTREAD))
-			cpu_drain_writebuf();
 
 		/*
 		 * Only thing left to do for COHERENT mapping is copy from bounce

sys/arch/arm/include/cpufunc.h

@@ -75,6 +75,20 @@
 
 #endif
 
+#if defined(_ARM_ARCH_8)
+#define	dma_r_r()	dsb(oshld)	// actually r_rw
+#define	dma_w_w()	dsb(oshst)
+#define	dma_rw_w()	dsb(osh)	// actually rw_rw
+#elif defined(_ARM_ARCH_6)
+#define	dma_r_r()	dsb(osh)	// actually rw_rw
+#define	dma_w_w()	dsb(oshst)
+#define	dma_rw_w()	dsb(osh)	// actually rw_rw
+#else
+#define	dma_r_r()	__nothing
+#define	dma_w_w()	cpu_drain_writebuf()
+#define	dma_rw_w()	cpu_drain_writebuf()
+#endif
+
 #ifdef __arm__
 
 #ifdef _KERNEL