diff --git a/SECURITY.md b/SECURITY.md index f60f0d9..299bf08 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -66,6 +66,17 @@ standard library's known vulnerabilities; `govulncheck ./...` reports none. ## Audit log +**2026-10-18 (v0.16.0).** The float32 pairwise-sum block kernels (SSE2, NEON, +RVV, LSX since v0.14.0/v0.15.0, VSX now) take raw pointers and a block count. +`AddBlocks8F32` refuses a length that is not a positive multiple of 8, and +each wrapper touches the last element before the call, so a kernel never sees +a count its operand cannot hold. `TestBlocks8StaysInsideItsOperands` puts the +accumulators and the input against PROT_NONE pages, and it passes on real +hardware for each kernel: Zen 3, Neoverse N1, SpacemiT X60, Loongson +3C5000L and POWER9. The ppc64le kernel's `xvaddsp` is a `WORD` whose encoding +go-asmgen pins against GNU as and llvm-mc. A sabotaged one (`xvsubsp`) failed +the bit-identity test on the POWER9. + **2026-10-17 (after v0.12.0, tests only).** v0.10.0 added thirteen dtypes, and `FuzzOps` builds only float64 arrays; `values.json` checks every operation on operands of one dtype, so the values of a mixed-dtype operation diff --git a/docs/perf.md b/docs/perf.md index efc1321..478a45a 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -897,10 +897,22 @@ accumulators, and was **26–47% slower** than the scalar loop. Every block's addition waited on the previous one, a single dependency chain, where the scalar loop has eight. Two registers give two chains, as SSE2 does on amd64. -Left: ppc64le and s390x run the scalar block loop. Go's assembler lacks -`xvaddsp` (ppc64) and `vfasb` (s390x). go-asmgen/asmgen#27 adds the VSX -single-precision family to go-asmgen's registry, with references from GNU as -and llvm-mc, and needs a cmd/asm patch of its own. +**ppc64le (2026-10-18).** Go's assembler has no `xvaddsp`, so go-asmgen +emits it as a `WORD`. The encoding is pinned against GNU as 2.44 and llvm-mc +on a POWER9 (go-asmgen/asmgen#27). The cmd/asm spelling is +[CL 846285](https://go.dev/cl/846285), and go-asmgen's registry check +assembles every entry with that CL applied. The eight accumulators sit in two +VSX registers, loaded and stored with `LXVW4X`/`STXVW4X`, so every word keeps +its lane. VSX is the ppc64le baseline, so there is no feature check. The kernel +tests passed on the POWER9, and a sabotaged kernel (one `xvsubsp`) failed them. + +| float32 `Sum`, 2¹⁴ / 2²⁰ | change | controls | +|---|---:|---| +| POWER9 (cfarm29) | −36% / −48% | float64, int32 ~ | + +Left: s390x runs the scalar block loop. Go's assembler lacks `vfasb` +(vector-enhancements facility 1, z14), which [CL 846286](https://go.dev/cl/846286) +adds, and go-asmgen's registry has no s390x entries yet to pin its encoding. ## SIMD coverage diff --git a/internal/kernels/asmgen/go.mod b/internal/kernels/asmgen/go.mod index 7a09d65..2cd36d5 100644 --- a/internal/kernels/asmgen/go.mod +++ b/internal/kernels/asmgen/go.mod @@ -6,4 +6,4 @@ module github.com/go-ndarray/ndarray/internal/kernels/asmgen go 1.27.2 -require github.com/go-asmgen/asmgen v0.16.0 +require github.com/go-asmgen/asmgen v0.18.0 diff --git a/internal/kernels/asmgen/go.sum b/internal/kernels/asmgen/go.sum index e9719a8..3244dab 100644 --- a/internal/kernels/asmgen/go.sum +++ b/internal/kernels/asmgen/go.sum @@ -1,2 +1,2 @@ -github.com/go-asmgen/asmgen v0.16.0 h1:fjrFWDDNxdSJvaTcN7uTIlawvrXe2PECdtScIZqX5bg= -github.com/go-asmgen/asmgen v0.16.0/go.mod h1:FHWXZt1MrejOwSShl4tPYjI2GPEfNAJuXJUX6Z+ruyQ= +github.com/go-asmgen/asmgen v0.18.0 h1:fo2agcSeLGgS69guRwlL7V0kGN7FtkQAg/NhLI0fE+4= +github.com/go-asmgen/asmgen v0.18.0/go.mod h1:FHWXZt1MrejOwSShl4tPYjI2GPEfNAJuXJUX6Z+ruyQ= diff --git a/internal/kernels/asmgen/ppc64le/gen.go b/internal/kernels/asmgen/ppc64le/gen.go index 67e2b9a..7c10e9b 100644 --- a/internal/kernels/asmgen/ppc64le/gen.go +++ b/internal/kernels/asmgen/ppc64le/gen.go @@ -40,6 +40,7 @@ func main() { f.Add(binKernel(k.name, k.vec, k.sca)) } f.Add(gemmKernel()) + f.Add(blocks8Kernel()) if err := os.WriteFile("sum_ppc64le.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -282,3 +283,27 @@ func gemmKernel() *emit.Function { b.Ret() return b.Func() } + +// blocks8Kernel builds blocks8VSX(r, a *float32, blocks int): for each of the +// blocks of eight float32 at a, in order, r[j] += a[8i+j] (NumPy's pairwise-sum +// block). The eight accumulators are split over two VSX registers, VS0 and VS1 +// (two dependency chains, as SSE2 on amd64 and LSX on loong64), and XVADDSP is +// one IEEE float32 addition per lane, so the result is the scalar loop's bit +// for bit. LXVW4X/STXVW4X put each word at the same lane in the accumulators +// and the input, and back at its own index on the store. blocks >= 1. +func blocks8Kernel() *emit.Function { + sig := ppc64.Layout([]string{"r", "a", "blocks"}, + []ppc64.Type{ppc64.Ptr, ppc64.Ptr, ppc64.Int64}, nil, nil) + b := ppc64.NewFunc("blocks8VSX", sig, 0) + b.LoadArg("r", "R3").LoadArg("a", "R4").LoadArg("blocks", "R5") + b.Raw("MOVD $16, R14") + b.Raw("LXVW4X (R3)(R0), VS0").Raw("LXVW4X (R3)(R14), VS1") + b.Label("b8loop"). + Raw("LXVW4X (R4)(R0), VS32"). + Raw("LXVW4X (R4)(R14), VS33") + b.XVADDSP(0, 0, 32).XVADDSP(1, 1, 33) + b.Raw("ADD $32, R4").Raw("ADD $-1, R5").Raw("CMP R5, $0").Raw("BNE b8loop") + b.Raw("STXVW4X VS0, (R3)(R0)").Raw("STXVW4X VS1, (R3)(R14)") + b.Ret() + return b.Func() +} diff --git a/internal/kernels/blocks8_generic.go b/internal/kernels/blocks8_generic.go index 489ac6c..102e1b5 100644 --- a/internal/kernels/blocks8_generic.go +++ b/internal/kernels/blocks8_generic.go @@ -1,4 +1,4 @@ -//go:build !amd64 && !arm64 && !loong64 && !riscv64 +//go:build !amd64 && !arm64 && !loong64 && !ppc64le && !riscv64 package kernels diff --git a/internal/kernels/blocks8_ppc64le.go b/internal/kernels/blocks8_ppc64le.go new file mode 100644 index 0000000..a904c49 --- /dev/null +++ b/internal/kernels/blocks8_ppc64le.go @@ -0,0 +1,12 @@ +package kernels + +// addBlocks8F32 runs the VSX kernel (sum_ppc64le.s): the eight accumulators in +// two VSX registers, one XVADDSP per register per block. VSX is the ppc64le +// baseline. len(a) is a positive multiple of 8 (AddBlocks8F32 checks). +func addBlocks8F32(r *[8]float32, a []float32) { + _ = a[len(a)-1] // the kernel reads len(a) of a + blocks8VSX(&r[0], &a[0], len(a)/8) +} + +//go:noescape +func blocks8VSX(r, a *float32, blocks int) diff --git a/internal/kernels/sum_ppc64le.s b/internal/kernels/sum_ppc64le.s index 4e9cbae..70ef94e 100644 --- a/internal/kernels/sum_ppc64le.s +++ b/internal/kernels/sum_ppc64le.s @@ -650,3 +650,23 @@ gemmMicro8x8VSXstore: STXVD2X VS57, (R6)(R16) RET +TEXT ·blocks8VSX(SB), NOSPLIT, $0-24 + MOVD r+0(FP), R3 + MOVD a+8(FP), R4 + MOVD blocks+16(FP), R5 + MOVD $16, R14 + LXVW4X (R3)(R0), VS0 + LXVW4X (R3)(R14), VS1 +b8loop: + LXVW4X (R4)(R0), VS32 + LXVW4X (R4)(R14), VS33 + WORD $0xf0000202 // XVADDSP VS0, VS32, VS0 + WORD $0xf0210a02 // XVADDSP VS1, VS33, VS1 + ADD $32, R4 + ADD $-1, R5 + CMP R5, $0 + BNE b8loop + STXVW4X VS0, (R3)(R0) + STXVW4X VS1, (R3)(R14) + RET +