From 75d8b8dbc2d312bf77c1b9b994a0ff8f93cf122d Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 18 Oct 2026 16:28:13 +0200 Subject: [PATCH] perf: float32 pairwise sum blocks on VSX (ppc64le) blocks8VSX keeps the eight accumulators in two VSX registers (two dependency chains) and adds each block with xvaddsp, one IEEE float32 addition per lane, so the result is the scalar loop's bit for bit. LXVW4X/STXVW4X keep every word in its lane. VSX is the ppc64le baseline. Go's assembler has no xvaddsp; go-asmgen v0.18.0 emits it as a WORD pinned against GNU as and llvm-mc (go-asmgen/asmgen#27; the cmd/asm side is CL 846285). POWER9 (cfarm29), alternating base/head, GOMAXPROCS=1, pinned core, benchstat n=10: float32 Sum -36% (2^14) / -48% (2^20); float64 and int32 unchanged. The kernel tests and the PROT_NONE fence pass there; a sabotaged kernel (xvsubsp) fails them. docs/perf.md: the ppc64le section; s390x is what is left (vfasb, CL 846286). SECURITY.md: audit entry for the float32 block kernels. Co-Authored-By: Claude Opus 5.5 --- SECURITY.md | 11 +++++++++++ docs/perf.md | 20 ++++++++++++++++---- internal/kernels/asmgen/go.mod | 2 +- internal/kernels/asmgen/go.sum | 4 ++-- internal/kernels/asmgen/ppc64le/gen.go | 25 +++++++++++++++++++++++++ internal/kernels/blocks8_generic.go | 2 +- internal/kernels/blocks8_ppc64le.go | 12 ++++++++++++ internal/kernels/sum_ppc64le.s | 20 ++++++++++++++++++++ 8 files changed, 88 insertions(+), 8 deletions(-) create mode 100644 internal/kernels/blocks8_ppc64le.go diff --git a/SECURITY.md b/SECURITY.md index f60f0d9..299bf08 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -66,6 +66,17 @@ standard library's known vulnerabilities; `govulncheck ./...` reports none. ## Audit log +**2026-10-18 (v0.16.0).** The float32 pairwise-sum block kernels (SSE2, NEON, +RVV, LSX since v0.14.0/v0.15.0, VSX now) take raw pointers and a block count. +`AddBlocks8F32` refuses a length that is not a positive multiple of 8, and +each wrapper touches the last element before the call, so a kernel never sees +a count its operand cannot hold. `TestBlocks8StaysInsideItsOperands` puts the +accumulators and the input against PROT_NONE pages, and it passes on real +hardware for each kernel: Zen 3, Neoverse N1, SpacemiT X60, Loongson +3C5000L and POWER9. The ppc64le kernel's `xvaddsp` is a `WORD` whose encoding +go-asmgen pins against GNU as and llvm-mc. A sabotaged one (`xvsubsp`) failed +the bit-identity test on the POWER9. + **2026-10-17 (after v0.12.0, tests only).** v0.10.0 added thirteen dtypes, and `FuzzOps` builds only float64 arrays; `values.json` checks every operation on operands of one dtype, so the values of a mixed-dtype operation diff --git a/docs/perf.md b/docs/perf.md index efc1321..478a45a 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -897,10 +897,22 @@ accumulators, and was **26–47% slower** than the scalar loop. Every block's addition waited on the previous one, a single dependency chain, where the scalar loop has eight. Two registers give two chains, as SSE2 does on amd64. -Left: ppc64le and s390x run the scalar block loop. Go's assembler lacks -`xvaddsp` (ppc64) and `vfasb` (s390x). go-asmgen/asmgen#27 adds the VSX -single-precision family to go-asmgen's registry, with references from GNU as -and llvm-mc, and needs a cmd/asm patch of its own. +**ppc64le (2026-10-18).** Go's assembler has no `xvaddsp`, so go-asmgen +emits it as a `WORD`. The encoding is pinned against GNU as 2.44 and llvm-mc +on a POWER9 (go-asmgen/asmgen#27). The cmd/asm spelling is +[CL 846285](https://go.dev/cl/846285), and go-asmgen's registry check +assembles every entry with that CL applied. The eight accumulators sit in two +VSX registers, loaded and stored with `LXVW4X`/`STXVW4X`, so every word keeps +its lane. VSX is the ppc64le baseline, so there is no feature check. The kernel +tests passed on the POWER9, and a sabotaged kernel (one `xvsubsp`) failed them. + +| float32 `Sum`, 2¹⁴ / 2²⁰ | change | controls | +|---|---:|---| +| POWER9 (cfarm29) | −36% / −48% | float64, int32 ~ | + +Left: s390x runs the scalar block loop. Go's assembler lacks `vfasb` +(vector-enhancements facility 1, z14), which [CL 846286](https://go.dev/cl/846286) +adds, and go-asmgen's registry has no s390x entries yet to pin its encoding. ## SIMD coverage diff --git a/internal/kernels/asmgen/go.mod b/internal/kernels/asmgen/go.mod index 7a09d65..2cd36d5 100644 --- a/internal/kernels/asmgen/go.mod +++ b/internal/kernels/asmgen/go.mod @@ -6,4 +6,4 @@ module github.com/go-ndarray/ndarray/internal/kernels/asmgen go 1.27.2 -require github.com/go-asmgen/asmgen v0.16.0 +require github.com/go-asmgen/asmgen v0.18.0 diff --git a/internal/kernels/asmgen/go.sum b/internal/kernels/asmgen/go.sum index e9719a8..3244dab 100644 --- a/internal/kernels/asmgen/go.sum +++ b/internal/kernels/asmgen/go.sum @@ -1,2 +1,2 @@ -github.com/go-asmgen/asmgen v0.16.0 h1:fjrFWDDNxdSJvaTcN7uTIlawvrXe2PECdtScIZqX5bg= -github.com/go-asmgen/asmgen v0.16.0/go.mod h1:FHWXZt1MrejOwSShl4tPYjI2GPEfNAJuXJUX6Z+ruyQ= +github.com/go-asmgen/asmgen v0.18.0 h1:fo2agcSeLGgS69guRwlL7V0kGN7FtkQAg/NhLI0fE+4= +github.com/go-asmgen/asmgen v0.18.0/go.mod h1:FHWXZt1MrejOwSShl4tPYjI2GPEfNAJuXJUX6Z+ruyQ= diff --git a/internal/kernels/asmgen/ppc64le/gen.go b/internal/kernels/asmgen/ppc64le/gen.go index 67e2b9a..7c10e9b 100644 --- a/internal/kernels/asmgen/ppc64le/gen.go +++ b/internal/kernels/asmgen/ppc64le/gen.go @@ -40,6 +40,7 @@ func main() { f.Add(binKernel(k.name, k.vec, k.sca)) } f.Add(gemmKernel()) + f.Add(blocks8Kernel()) if err := os.WriteFile("sum_ppc64le.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -282,3 +283,27 @@ func gemmKernel() *emit.Function { b.Ret() return b.Func() } + +// blocks8Kernel builds blocks8VSX(r, a *float32, blocks int): for each of the +// blocks of eight float32 at a, in order, r[j] += a[8i+j] (NumPy's pairwise-sum +// block). The eight accumulators are split over two VSX registers, VS0 and VS1 +// (two dependency chains, as SSE2 on amd64 and LSX on loong64), and XVADDSP is +// one IEEE float32 addition per lane, so the result is the scalar loop's bit +// for bit. LXVW4X/STXVW4X put each word at the same lane in the accumulators +// and the input, and back at its own index on the store. blocks >= 1. +func blocks8Kernel() *emit.Function { + sig := ppc64.Layout([]string{"r", "a", "blocks"}, + []ppc64.Type{ppc64.Ptr, ppc64.Ptr, ppc64.Int64}, nil, nil) + b := ppc64.NewFunc("blocks8VSX", sig, 0) + b.LoadArg("r", "R3").LoadArg("a", "R4").LoadArg("blocks", "R5") + b.Raw("MOVD $16, R14") + b.Raw("LXVW4X (R3)(R0), VS0").Raw("LXVW4X (R3)(R14), VS1") + b.Label("b8loop"). + Raw("LXVW4X (R4)(R0), VS32"). + Raw("LXVW4X (R4)(R14), VS33") + b.XVADDSP(0, 0, 32).XVADDSP(1, 1, 33) + b.Raw("ADD $32, R4").Raw("ADD $-1, R5").Raw("CMP R5, $0").Raw("BNE b8loop") + b.Raw("STXVW4X VS0, (R3)(R0)").Raw("STXVW4X VS1, (R3)(R14)") + b.Ret() + return b.Func() +} diff --git a/internal/kernels/blocks8_generic.go b/internal/kernels/blocks8_generic.go index 489ac6c..102e1b5 100644 --- a/internal/kernels/blocks8_generic.go +++ b/internal/kernels/blocks8_generic.go @@ -1,4 +1,4 @@ -//go:build !amd64 && !arm64 && !loong64 && !riscv64 +//go:build !amd64 && !arm64 && !loong64 && !ppc64le && !riscv64 package kernels diff --git a/internal/kernels/blocks8_ppc64le.go b/internal/kernels/blocks8_ppc64le.go new file mode 100644 index 0000000..a904c49 --- /dev/null +++ b/internal/kernels/blocks8_ppc64le.go @@ -0,0 +1,12 @@ +package kernels + +// addBlocks8F32 runs the VSX kernel (sum_ppc64le.s): the eight accumulators in +// two VSX registers, one XVADDSP per register per block. VSX is the ppc64le +// baseline. len(a) is a positive multiple of 8 (AddBlocks8F32 checks). +func addBlocks8F32(r *[8]float32, a []float32) { + _ = a[len(a)-1] // the kernel reads len(a) of a + blocks8VSX(&r[0], &a[0], len(a)/8) +} + +//go:noescape +func blocks8VSX(r, a *float32, blocks int) diff --git a/internal/kernels/sum_ppc64le.s b/internal/kernels/sum_ppc64le.s index 4e9cbae..70ef94e 100644 --- a/internal/kernels/sum_ppc64le.s +++ b/internal/kernels/sum_ppc64le.s @@ -650,3 +650,23 @@ gemmMicro8x8VSXstore: STXVD2X VS57, (R6)(R16) RET +TEXT ·blocks8VSX(SB), NOSPLIT, $0-24 + MOVD r+0(FP), R3 + MOVD a+8(FP), R4 + MOVD blocks+16(FP), R5 + MOVD $16, R14 + LXVW4X (R3)(R0), VS0 + LXVW4X (R3)(R14), VS1 +b8loop: + LXVW4X (R4)(R0), VS32 + LXVW4X (R4)(R14), VS33 + WORD $0xf0000202 // XVADDSP VS0, VS32, VS0 + WORD $0xf0210a02 // XVADDSP VS1, VS33, VS1 + ADD $32, R4 + ADD $-1, R5 + CMP R5, $0 + BNE b8loop + STXVW4X VS0, (R3)(R0) + STXVW4X VS1, (R3)(R14) + RET +