[tailscale] runtime, runtime/metrics: add goroutine stack size histogram

src/runtime/metrics.go

@@ -26,6 +26,10 @@ var (
 
 	sizeClassBuckets []float64
 	timeHistBuckets  []float64
+
+	// tailscaleStackSizeBuckets is a Tailscale addition; see
+	// tailscaleStackHistBuckets.
+	tailscaleStackSizeBuckets []float64
 )
 
 type metricData struct {
@@ -86,6 +90,7 @@ func initMetrics() {
 	sizeClassBuckets = append(sizeClassBuckets, float64Inf())
 
 	timeHistBuckets = timeHistogramMetricsBuckets()
+	tailscaleStackSizeBuckets = tailscaleStackHistBuckets()
 	metrics = map[string]metricData{
 		"/cgo/go-to-c-calls:calls": {
 			compute: func(_ *statAggregate, out *metricValue) {
@@ -545,6 +550,30 @@ func initMetrics() {
 				out.scalar = float64bits(nsToSec(totalMutexWaitTimeNanos()))
 			},
 		},
+		"/tailscale/sched/goroutines-by-stack-size:bytes": {
+			compute: func(_ *statAggregate, out *metricValue) {
+				hist := out.float64HistOrInit(tailscaleStackSizeBuckets)
+				tailscaleStackHistRead(hist.counts)
+			},
+		},
+		"/tailscale/sched/stacks/copied:bytes": {
+			compute: func(_ *statAggregate, out *metricValue) {
+				out.kind = metricKindUint64
+				out.scalar = tailscaleStackBytesCopied.Load()
+			},
+		},
+		"/tailscale/sched/stacks/growths:events": {
+			compute: func(_ *statAggregate, out *metricValue) {
+				out.kind = metricKindUint64
+				out.scalar = tailscaleStackGrowths.Load()
+			},
+		},
+		"/tailscale/sched/stacks/shrinks:events": {
+			compute: func(_ *statAggregate, out *metricValue) {
+				out.kind = metricKindUint64
+				out.scalar = tailscaleStackShrinks.Load()
+			},
+		},
 	}
 
 	for _, info := range godebugs.All {

src/runtime/metrics/description.go

@@ -509,6 +509,29 @@ var allDesc = []Description{
 		Kind:        KindFloat64,
 		Cumulative:  true,
 	},
+	{
+		Name:        "/tailscale/sched/goroutines-by-stack-size:bytes",
+		Description: "Distribution of live goroutines by the current size of their stacks, including runtime system goroutines. Stack sizes are always powers of two, so each bucket but the last counts exactly the goroutines whose stack size is the bucket's lower bound, and the last bucket counts every goroutine with a larger stack. The runtime updates these counts as goroutines are created, exit, and have their stacks grown or shrunk, so reading this metric does not visit every goroutine. The bucket counts sum to approximately /sched/goroutines:goroutines. This metric is specific to the Tailscale fork of Go.",
+		Kind:        KindFloat64Histogram,
+	},
+	{
+		Name:        "/tailscale/sched/stacks/copied:bytes",
+		Description: "Cumulative bytes of goroutine stack copied while growing or shrinking stacks since program start. Every stack growth or shrink copies the in-use part of the goroutine's stack to a new allocation, so this measures the runtime's stack copying work. This metric is specific to the Tailscale fork of Go.",
+		Kind:        KindUint64,
+		Cumulative:  true,
+	},
+	{
+		Name:        "/tailscale/sched/stacks/growths:events",
+		Description: "Count of goroutine stack growths since program start. A stack doubles in size when a function call would overflow it. Compared with /tailscale/sched/stacks/shrinks:events this shows whether stacks are repeatedly growing and then being shrunk back by the garbage collector. This metric is specific to the Tailscale fork of Go.",
+		Kind:        KindUint64,
+		Cumulative:  true,
+	},
+	{
+		Name:        "/tailscale/sched/stacks/shrinks:events",
+		Description: "Count of goroutine stack shrinks since program start. The garbage collector halves a goroutine's stack when it finds the goroutine using less than a quarter of it. This metric is specific to the Tailscale fork of Go.",
+		Kind:        KindUint64,
+		Cumulative:  true,
+	},
 }
 
 func init() {

src/runtime/metrics/doc.go

@@ -587,5 +587,38 @@ Below is the full list of supported metrics, ordered lexicographically.
 		is useful for identifying global changes in lock contention.
 		Collect a mutex or block profile using the runtime/pprof package
 		for more detailed contention data.
+
+	/tailscale/sched/goroutines-by-stack-size:bytes
+		Distribution of live goroutines by the current size of their
+		stacks, including runtime system goroutines. Stack sizes are
+		always powers of two, so each bucket but the last counts exactly
+		the goroutines whose stack size is the bucket's lower bound,
+		and the last bucket counts every goroutine with a larger stack.
+		The runtime updates these counts as goroutines are created,
+		exit, and have their stacks grown or shrunk, so reading this
+		metric does not visit every goroutine. The bucket counts sum
+		to approximately /sched/goroutines:goroutines. This metric is
+		specific to the Tailscale fork of Go.
+
+	/tailscale/sched/stacks/copied:bytes
+		Cumulative bytes of goroutine stack copied while growing or
+		shrinking stacks since program start. Every stack growth or
+		shrink copies the in-use part of the goroutine's stack to a new
+		allocation, so this measures the runtime's stack copying work.
+		This metric is specific to the Tailscale fork of Go.
+
+	/tailscale/sched/stacks/growths:events
+		Count of goroutine stack growths since program start. A stack
+		doubles in size when a function call would overflow it. Compared
+		with /tailscale/sched/stacks/shrinks:events this shows whether
+		stacks are repeatedly growing and then being shrunk back by the
+		garbage collector. This metric is specific to the Tailscale fork
+		of Go.
+
+	/tailscale/sched/stacks/shrinks:events
+		Count of goroutine stack shrinks since program start.
+		The garbage collector halves a goroutine's stack when it finds
+		the goroutine using less than a quarter of it. This metric is
+		specific to the Tailscale fork of Go.
 */
 package metrics

src/runtime/proc.go

@@ -2561,6 +2561,13 @@ func oneNewExtraM() {
 	// has the same effect.
 	sched.ngsys.Add(1)
 
+	// The goroutine's stack can grow while it runs cgo callbacks, and
+	// copystack records that in the stack size histogram, so record
+	// its initial size too or the histogram would go negative. Extra
+	// Ms are never destroyed, so there is no matching decrement. They
+	// are also rare, so the global counter is fine here.
+	tailscaleStackHistAdd(nil, gp.stack.hi-gp.stack.lo, 1)
+
 	// Add m to the extra list.
 	addExtraM(mp)
 }
@@ -4521,6 +4528,7 @@ func gdestroy(gp *g) {
 
 	casgstatus(gp, _Grunning, _Gdead)
 	gcController.addScannableStack(pp, -int64(gp.stack.hi-gp.stack.lo))
+	tailscaleStackHistAdd(pp, gp.stack.hi-gp.stack.lo, -1)
 	if isSystemGoroutine(gp, false) {
 		sched.ngsys.Add(-1)
 	}
@@ -5420,6 +5428,7 @@ func newproc1(fn *funcval, callergp *g, callerpc uintptr, parked bool, waitreaso
 		newg.tracking = true
 	}
 	gcController.addScannableStack(pp, int64(newg.stack.hi-newg.stack.lo))
+	tailscaleStackHistAdd(pp, newg.stack.hi-newg.stack.lo, 1)
 
 	// Get a goid and switch to runnable. This needs to happen under traceAcquire
 	// since it's a goroutine transition. See tracer invariants in trace.go.
@@ -6069,6 +6078,7 @@ func (pp *p) destroy() {
 	pp.cleanupsQueued = 0
 	sched.goroutinesCreated.Add(int64(pp.goroutinesCreated))
 	pp.goroutinesCreated = 0
+	tailscaleStackHistFlush(pp)
 	pp.xRegs.free()
 	pp.status = _Pdead
 }

src/runtime/runtime2.go

@@ -920,6 +920,20 @@ type p struct {
 	// scheduler ASAP (regardless of what G is running on it).
 	preempt bool
 
+	// tsStackHist counts live goroutines by stack size class. It is a
+	// Tailscale addition backing the
+	// /tailscale/sched/goroutines-by-stack-size:bytes metric; see the
+	// comment above tailscaleStackHistBaseOrder for the indexing and
+	// how the counts are used. It sits here, in the padding after
+	// preempt, so that it shares a cache line with goroutinesCreated,
+	// which newproc1 also writes, rather than dirtying one of its own.
+	// Only the P's owner writes it, and tailscaleStackHistRead reads
+	// it from other Ps without synchronization, like goroutinesCreated.
+	// The entries are int16 so that the whole array fits in that cache
+	// line; they are flushed to tailscaleStackHist once they drift
+	// tailscaleStackHistSlack from zero, so they cannot overflow.
+	tsStackHist [tailscaleStackHistLen]int16
+
 	// gcStopTime is the nanotime timestamp that this P last entered _Pgcstop.
 	gcStopTime int64
 

src/runtime/stack.go

@@ -940,6 +940,7 @@ func copystack(gp *g, newsize uintptr) {
 	// It's also fine if we have no P, addScannableStack can deal with
 	// that case.
 	gcController.addScannableStack(getg().m.p.ptr(), int64(newsize)-int64(old.hi-old.lo))
+	tailscaleNoteStackCopy(getg().m.p.ptr(), old.hi-old.lo, newsize, used)
 
 	// allocate new stack
 	new := stackalloc(uint32(newsize))

src/runtime/tailscale_export_test.go

@@ -0,0 +1,40 @@
+// Copyright 2026 Tailscale. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime
+
+import "unsafe"
+
+// TailscaleStackHistSlow stops the world and returns the goroutine
+// stack size histogram computed two ways: want, by walking every
+// goroutine, and got, from the incrementally maintained counts via the
+// same code path the /tailscale/sched/goroutines-by-stack-size:bytes
+// metric uses. Both are indexed like the metric's buckets. Nothing can
+// change while the world is stopped, so the two must match exactly.
+func TailscaleStackHistSlow() (want, got []uint64) {
+	stw := stopTheWorld(stwForTestReadMetricsSlow)
+	first := tailscaleStackHistFirst()
+	want = make([]uint64, tailscaleStackHistLen-first)
+	got = make([]uint64, tailscaleStackHistLen-first)
+	forEachG(func(gp *g) {
+		// Dead goroutines waiting on a free list still own a stack,
+		// but they are not live and are not counted. Goroutines that
+		// belong to extra Ms are _Gdeadextra rather than _Gdead
+		// between cgo callbacks, and those are counted.
+		if readgstatus(gp)&^_Gscan == _Gdead {
+			return
+		}
+		want[tailscaleStackHistIndex(gp.stack.hi-gp.stack.lo)-first]++
+	})
+	tailscaleStackHistRead(got)
+	startTheWorld(stw)
+	return want, got
+}
+
+// TailscaleStackHistLayout reports where p.tsStackHist and
+// p.goroutinesCreated live within p, so that a test can check that
+// they share a cache line.
+func TailscaleStackHistLayout() (histOff, histSize, goroutinesCreatedOff uintptr) {
+	return unsafe.Offsetof(p{}.tsStackHist), unsafe.Sizeof(p{}.tsStackHist), unsafe.Offsetof(p{}.goroutinesCreated)
+}