diff --git a/SECURITY.md b/SECURITY.md index 299bf08..e3fcd76 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -77,6 +77,14 @@ hardware for each kernel: Zen 3, Neoverse N1, SpacemiT X60, Loongson go-asmgen pins against GNU as and llvm-mc. A sabotaged one (`xvsubsp`) failed the bit-identity test on the POWER9. +**2026-10-18 (v0.17.0).** The s390x block kernel (`blocks8VXE`) follows the +same guards: `AddBlocks8F32`'s length check, the wrapper's touch of the last +element, and the `useVXE` gate, so a CPU without vector-enhancements facility +1 never reaches `vfasb`. On an IBM Z (LinuxONE) the PROT_NONE fence test and +the whole suite pass, and a sabotaged kernel (`vfssb`) failed the bit-identity +test. Its `vfasb` is `WORD`+`BYTE`+`BYTE`, pinned in go-asmgen against GNU as +and llvm-mc and executed bit-exact on that machine. + **2026-10-17 (after v0.12.0, tests only).** v0.10.0 added thirteen dtypes, and `FuzzOps` builds only float64 arrays; `values.json` checks every operation on operands of one dtype, so the values of a mixed-dtype operation diff --git a/docs/perf.md b/docs/perf.md index 478a45a..37daa03 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -910,9 +910,24 @@ tests passed on the POWER9, and a sabotaged kernel (one `xvsubsp`) failed them. |---|---:|---| | POWER9 (cfarm29) | −36% / −48% | float64, int32 ~ | -Left: s390x runs the scalar block loop. Go's assembler lacks `vfasb` -(vector-enhancements facility 1, z14), which [CL 846286](https://go.dev/cl/846286) -adds, and go-asmgen's registry has no s390x entries yet to pin its encoding. +**s390x (2026-10-18).** Go's assembler lacks `vfasb` (vector-enhancements +facility 1, z14), which [CL 846286](https://go.dev/cl/846286) adds. go-asmgen +v0.19.0 emits it as `WORD`+`BYTE`+`BYTE`, the 6 bytes pinned against GNU as +(go-asmgen/asmgen#29). The eight accumulators sit in two vector registers; +`VL`/`VST` keep memory order on this big-endian machine. The kernel runs +behind the `useVXE` gate of max/min, so a z13 takes the scalar loop. The +kernel tests passed on the z machine, and a sabotaged kernel (one `vfssb`) +failed them. Same A/B protocol: + +| float32 `Sum`, 2¹⁴ / 2²⁰ | change | controls | +|---|---:|---| +| IBM Z, LinuxONE (vxe, vxe2) | −73% / −78% | int32 ~; float64 ~ / +2.0% | + +The float64 control at 2²⁰ moved +2.0% with its kernel byte-identical: the +change moved `sumVX` by 32 bytes (0x18f920 → 0x18f940). It is attributed to +layout, not proven. + +Every 64-bit target now adds the float32 block with SIMD. ## SIMD coverage diff --git a/internal/kernels/asmgen/go.mod b/internal/kernels/asmgen/go.mod index 2cd36d5..5bd679b 100644 --- a/internal/kernels/asmgen/go.mod +++ b/internal/kernels/asmgen/go.mod @@ -6,4 +6,4 @@ module github.com/go-ndarray/ndarray/internal/kernels/asmgen go 1.27.2 -require github.com/go-asmgen/asmgen v0.18.0 +require github.com/go-asmgen/asmgen v0.19.0 diff --git a/internal/kernels/asmgen/go.sum b/internal/kernels/asmgen/go.sum index 3244dab..208aa81 100644 --- a/internal/kernels/asmgen/go.sum +++ b/internal/kernels/asmgen/go.sum @@ -1,2 +1,2 @@ -github.com/go-asmgen/asmgen v0.18.0 h1:fo2agcSeLGgS69guRwlL7V0kGN7FtkQAg/NhLI0fE+4= -github.com/go-asmgen/asmgen v0.18.0/go.mod h1:FHWXZt1MrejOwSShl4tPYjI2GPEfNAJuXJUX6Z+ruyQ= +github.com/go-asmgen/asmgen v0.19.0 h1:ESiyTkTS1KOi0uMsjmOo7pnBw+IOxvnGkTuVmipZmDo= +github.com/go-asmgen/asmgen v0.19.0/go.mod h1:FHWXZt1MrejOwSShl4tPYjI2GPEfNAJuXJUX6Z+ruyQ= diff --git a/internal/kernels/asmgen/s390x/gen.go b/internal/kernels/asmgen/s390x/gen.go index d7e007b..aa46b0c 100644 --- a/internal/kernels/asmgen/s390x/gen.go +++ b/internal/kernels/asmgen/s390x/gen.go @@ -39,6 +39,7 @@ func main() { f.Add(gemmKernel()) f.Add(extremumKernel("maxVXE", "VFMAXDB", "WFMAXDB")) f.Add(extremumKernel("minVXE", "VFMINDB", "WFMINDB")) + f.Add(blocks8Kernel()) if err := os.WriteFile("sum_s390x.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -216,3 +217,27 @@ func extremumKernel(name, vec, sca string) *emit.Function { Ret() return b.Func() } + +// blocks8Kernel builds blocks8VXE(r, a *float32, blocks int): for each of the +// blocks of eight float32 at a, in order, r[j] += a[8i+j] (NumPy's pairwise-sum +// block). The eight accumulators are split over two vector registers, V0 and +// V1 (two dependency chains, as on the other targets), and VFASB is one IEEE +// float32 addition per lane, so the result is the scalar loop's bit for bit. +// VL/VST keep memory order on this big-endian machine, so every float keeps +// its lane. VFASB is the vector-enhancements facility 1 (z14); Go's assembler +// lacks it, and go-asmgen emits it as WORD+BYTE+BYTE pinned against GNU as +// (the cmd/asm spelling is CL 846286). blocks >= 1. +func blocks8Kernel() *emit.Function { + sig := s390x.Layout([]string{"r", "a", "blocks"}, + []s390x.Type{s390x.Ptr, s390x.Ptr, s390x.Int64}, nil, nil) + b := s390x.NewFunc("blocks8VXE", sig, 0) + b.LoadArg("r", "R1").LoadArg("a", "R2").LoadArg("blocks", "R3"). + Raw("VL 0(R1), V0").Raw("VL 16(R1), V1"). + Raw("b8loop:"). + Raw("VL 0(R2), V2").Raw("VL 16(R2), V3") + b.VFASB(0, 0, 2).VFASB(1, 1, 3) + b.Raw("ADD $32, R2").Raw("SUB $1, R3").Raw("CMPBNE R3, $0, b8loop"). + Raw("VST V0, 0(R1)").Raw("VST V1, 16(R1)"). + Ret() + return b.Func() +} diff --git a/internal/kernels/blocks8_generic.go b/internal/kernels/blocks8_generic.go index 102e1b5..a5bc9da 100644 --- a/internal/kernels/blocks8_generic.go +++ b/internal/kernels/blocks8_generic.go @@ -1,4 +1,4 @@ -//go:build !amd64 && !arm64 && !loong64 && !ppc64le && !riscv64 +//go:build !amd64 && !arm64 && !loong64 && !ppc64le && !riscv64 && !s390x package kernels diff --git a/internal/kernels/blocks8_s390x.go b/internal/kernels/blocks8_s390x.go new file mode 100644 index 0000000..852336e --- /dev/null +++ b/internal/kernels/blocks8_s390x.go @@ -0,0 +1,18 @@ +package kernels + +// addBlocks8F32 runs the VXE kernel (sum_s390x.s), the eight accumulators in +// two vector registers, one VFASB per register per block, on a z14 or later +// (VFASB is vector-enhancements facility 1, the useVXE gate of max/min); the +// scalar loop otherwise. len(a) is a positive multiple of 8 (AddBlocks8F32 +// checks). +func addBlocks8F32(r *[8]float32, a []float32) { + if !useVXE { + addBlocks8F32Go(r, a) + return + } + _ = a[len(a)-1] // the kernel reads len(a) of a + blocks8VXE(&r[0], &a[0], len(a)/8) +} + +//go:noescape +func blocks8VXE(r, a *float32, blocks int) diff --git a/internal/kernels/sum_s390x.s b/internal/kernels/sum_s390x.s index 5ff3933..04b17d5 100644 --- a/internal/kernels/sum_s390x.s +++ b/internal/kernels/sum_s390x.s @@ -401,3 +401,21 @@ TEXT ·minVXE(SB), NOSPLIT, $0-24 FMOVD F0, ret+16(FP) RET +TEXT ·blocks8VXE(SB), NOSPLIT, $0-24 + MOVD r+0(FP), R1 + MOVD a+8(FP), R2 + MOVD blocks+16(FP), R3 + VL 0(R1), V0 + VL 16(R1), V1 + b8loop: + VL 0(R2), V2 + VL 16(R2), V3 + WORD $0xe7002000; BYTE $0x20; BYTE $0xe3 // VFASB V0, V2, V0 + WORD $0xe7113000; BYTE $0x20; BYTE $0xe3 // VFASB V1, V3, V1 + ADD $32, R2 + SUB $1, R3 + CMPBNE R3, $0, b8loop + VST V0, 0(R1) + VST V1, 16(R1) + RET +