diff --git a/crates/gpui/src/elements/list.rs b/crates/gpui/src/elements/list.rs index 85b38d5234efec..76be4cd09653fc 100644 --- a/crates/gpui/src/elements/list.rs +++ b/crates/gpui/src/elements/list.rs @@ -296,6 +296,7 @@ struct ListItemSummary { height: Pixels, has_focus_handles: bool, has_unknown_height: bool, + unknown_height_count: usize, } #[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] @@ -685,6 +686,26 @@ impl ListState { None } + /// Top of item `ix` within the list content, and the content height. + /// Items with no known height count as the mean known height, so the + /// numbers stay proportional before every item has been laid out. + pub fn item_top_and_content_height(&self, ix: usize) -> (Pixels, Pixels) { + let state = &*self.0.borrow(); + let total = state.items.summary(); + let known = total.count - total.unknown_height_count; + let mean = if known > 0 { + total.height.0 / known as f32 + } else { + 0. + }; + let mut cursor = state.items.cursor::(()); + let before: ListItemSummary = cursor.summary(&Count(ix), Bias::Right); + ( + px(before.height.0 + mean * before.unknown_height_count as f32), + px(total.height.0 + mean * total.unknown_height_count as f32), + ) + } + /// Call this method when the user starts dragging the scrollbar. /// /// This will prevent the height reported to the scrollbar from changing during the drag @@ -1597,6 +1618,7 @@ impl sum_tree::Item for ListItem { }, has_focus_handles: focus_handle.is_some(), has_unknown_height: size_hint.is_none(), + unknown_height_count: usize::from(size_hint.is_none()), }, ListItem::Measured { size, focus_handle, .. @@ -1607,6 +1629,7 @@ impl sum_tree::Item for ListItem { height: size.height, has_focus_handles: focus_handle.is_some(), has_unknown_height: false, + unknown_height_count: 0, }, } } @@ -1624,6 +1647,7 @@ impl sum_tree::ContextLessSummary for ListItemSummary { self.height += summary.height; self.has_focus_handles |= summary.has_focus_handles; self.has_unknown_height |= summary.has_unknown_height; + self.unknown_height_count += summary.unknown_height_count; } } @@ -1951,6 +1975,45 @@ mod test { assert_eq!(state.item_is_below_viewport(0), Some(false)); } + #[gpui::test] + fn test_item_top_tracks_measured_and_unmeasured_heights(cx: &mut TestAppContext) { + let cx = cx.add_empty_window(); + + struct TestView(ListState); + impl Render for TestView { + fn render(&mut self, _: &mut Window, _: &mut Context) -> impl IntoElement { + list(self.0.clone(), |ix, _, _| { + div().h(px(10. * (ix + 1) as f32)).w_full().into_any() + }) + .w_full() + .h_full() + } + } + + let state = ListState::new(4, crate::ListAlignment::Top, px(0.)).measure_all(); + let view = cx.update(|_, cx| cx.new(|_| TestView(state.clone()))); + cx.draw(point(px(0.), px(0.)), size(px(100.), px(200.)), |_, _| { + view.into_any_element() + }); + assert_eq!(state.item_top_and_content_height(2), (px(30.), px(100.))); + + struct Uniform(ListState); + impl Render for Uniform { + fn render(&mut self, _: &mut Window, _: &mut Context) -> impl IntoElement { + list(self.0.clone(), |_, _, _| div().h(px(50.)).w_full().into_any()) + .w_full() + .h_full() + } + } + let state = ListState::new(10, crate::ListAlignment::Top, px(0.)); + let view = cx.update(|_, cx| cx.new(|_| Uniform(state.clone()))); + cx.draw(point(px(0.), px(0.)), size(px(100.), px(100.)), |_, _| { + view.into_any_element() + }); + // Only the visible items are measured; the rest count as the mean. + assert_eq!(state.item_top_and_content_height(5), (px(250.), px(500.))); + } + #[gpui::test] fn test_measure_all_after_width_change(cx: &mut TestAppContext) { let cx = cx.add_empty_window(); diff --git a/crates/gpui/src/external_texture.rs b/crates/gpui/src/external_texture.rs new file mode 100644 index 00000000000000..c216a4b4b6e3fc --- /dev/null +++ b/crates/gpui/src/external_texture.rs @@ -0,0 +1,1048 @@ +//! Caller-owned GPU textures composited **inside** the GPUI scene. +//! +//! This exists for one reason: Cherry Pick's embedded browser paints Chromium +//! off-screen and needs those pixels inside the swap chain, at 60fps, without +//! going anywhere near the sprite atlas. +//! +//! # Why not the sprite atlas +//! +//! [`crate::Window::paint_image`] packs its pixels into a shared atlas texture +//! alongside every icon and glyph on screen. That is right for content that is +//! uploaded once and drawn many times. A live web page is the opposite: a +//! 1920x1080 BGRA frame is 8 MB, it changes on its own schedule, and pushing it +//! through the atlas would evict every icon in the app on the first frame and +//! then thrash the allocator forever after. It would also serialise browser +//! uploads behind unrelated text rendering. +//! +//! So an external texture gets its own GPU texture, keyed by [`ExternalTextureId`], +//! that the renderer creates once and re-uses. Only the pixels that changed are +//! re-uploaded, and a page that has settled uploads nothing at all. +//! +//! # Threading +//! +//! The producer (a Chromium OSR paint callback, on a Chromium thread) writes +//! into its own buffer and bumps [`ExternalFrameView::sequence`]. The consumer +//! (a GPUI renderer, on the UI thread during paint) calls +//! [`ExternalTextureSource::with_frame`], which is expected to take whatever +//! lock the producer uses. Implementations must keep that critical section to a +//! memcpy's worth of work — it runs inside the frame budget. +//! +//! # Resize +//! +//! Reallocating the buffer bumps [`ExternalFrameView::generation`]. A renderer +//! that sees a new generation throws its cached texture away and creates a new +//! one, which is what keeps a stale-sized texture from being sampled with new +//! dimensions after a pane resize. + +use std::fmt; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::{Bounds, DevicePixels, Size}; + +/// Stable identity for one external texture across frames. +/// +/// The renderer caches a GPU texture per id, so this must be stable for the +/// life of the producing surface and must never be reused by a different +/// producer while the old one could still be in a scene. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct ExternalTextureId(pub u64); + +impl ExternalTextureId { + /// Hand out an id no other caller in this process will get. + pub fn next() -> Self { + static NEXT: AtomicU64 = AtomicU64::new(1); + Self(NEXT.fetch_add(1, Ordering::Relaxed)) + } +} + +/// Byte order of an external frame's pixels. +/// +/// Both are 8 bits per channel and 4 bytes per pixel; only the channel order +/// differs. Chromium OSR hands out `Bgra8`. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum ExternalTextureFormat { + /// Blue, green, red, alpha. What Chromium's `OnPaint` produces. + Bgra8, + /// Red, green, blue, alpha. + Rgba8, +} + +impl ExternalTextureFormat { + /// Bytes per pixel. Both variants are 4. + pub const fn bytes_per_pixel(self) -> usize { + 4 + } +} + +/// A borrowed look at the producer's current pixels. +/// +/// Only valid for the duration of the [`ExternalTextureSource::with_frame`] +/// callback that produced it. +pub struct ExternalFrameView<'a> { + /// Pixel dimensions of the whole buffer. + pub size: Size, + /// Bytes between the start of one row and the start of the next. May be + /// larger than `size.width * 4` when the producer pads rows. + pub stride: usize, + /// Channel order of `bytes`. + pub format: ExternalTextureFormat, + /// The pixels. At least `stride * size.height` long. + pub bytes: &'a [u8], + /// Bumped whenever the buffer is reallocated, which for a browser means a + /// resize. A renderer must recreate its cached texture when this changes. + pub generation: u64, + /// Bumped on every painted frame. A renderer that has already uploaded this + /// sequence must not upload again — this is what makes a settled page cost + /// zero bandwidth. + pub sequence: u64, + /// The regions that changed since `sequence - 1`, in texture pixels. Empty + /// means "assume everything changed", which is correct but expensive; a + /// producer should always fill this in when it knows. + pub dirty: &'a [Bounds], +} + +impl ExternalFrameView<'_> { + /// Whether the buffer is large enough for its own declared geometry. + /// + /// A renderer must check this before indexing. A producer that gets this + /// wrong is a bug, but it must not be a GPU crash or an out-of-bounds read. + pub fn is_well_formed(&self) -> bool { + let width = self.size.width.0.max(0) as usize; + let height = self.size.height.0.max(0) as usize; + if width == 0 || height == 0 { + return false; + } + let Some(row_bytes) = width.checked_mul(self.format.bytes_per_pixel()) else { + return false; + }; + // Both native upload APIs represent the row pitch as u32. + self.stride >= row_bytes + && u32::try_from(self.stride).is_ok() + && self + .stride + .checked_mul(height) + .is_some_and(|needed| needed <= self.bytes.len()) + } + + /// The dirty regions, clamped to the buffer, with an empty list meaning the + /// whole frame. Renderers should upload exactly these. + pub fn dirty_regions(&self) -> Vec> { + let full = Bounds { + origin: crate::point(DevicePixels(0), DevicePixels(0)), + size: self.size, + }; + if self.dirty.is_empty() { + return vec![full]; + } + let regions: Vec<_> = self + .dirty + .iter() + .filter_map(|rect| { + // External coordinates are untrusted. Widen before adding so + // malformed i32 endpoints cannot wrap into a native GPU box. + if rect.size.width.0 <= 0 || rect.size.height.0 <= 0 { + return None; + } + let left = i64::from(rect.origin.x.0).clamp(0, i64::from(self.size.width.0.max(0))); + let top = i64::from(rect.origin.y.0).clamp(0, i64::from(self.size.height.0.max(0))); + let right = (i64::from(rect.origin.x.0) + i64::from(rect.size.width.0)) + .clamp(left, i64::from(self.size.width.0.max(0))); + let bottom = (i64::from(rect.origin.y.0) + i64::from(rect.size.height.0)) + .clamp(top, i64::from(self.size.height.0.max(0))); + (right > left && bottom > top).then_some(Bounds { + origin: crate::point(DevicePixels(left as i32), DevicePixels(top as i32)), + size: crate::size( + DevicePixels((right - left) as i32), + DevicePixels((bottom - top) as i32), + ), + }) + }) + .collect(); + // Dirty rectangles are hints about a complete frame. If every hint + // falls outside the current texture, acknowledging the sequence with + // no upload would leave stale pixels cached indefinitely. + if regions.is_empty() { + vec![full] + } else { + regions + } + } +} + +/// Something that produces frames for an external texture. +/// +/// Implemented outside GPUI (Cherry Pick's browser engine implements it over a +/// Chromium OSR buffer). GPUI only reads. +pub trait ExternalTextureSource: fmt::Debug + Send + Sync + 'static { + /// Stable identity, so the renderer can cache one GPU texture per source. + fn id(&self) -> ExternalTextureId; + + /// Show the renderer the current frame, if there is one. + /// + /// Called on the UI thread inside paint. `visit` is not called when no + /// frame has been produced yet, which is the normal state for a pane whose + /// page has not painted. + fn with_frame(&self, visit: &mut dyn FnMut(ExternalFrameView<'_>)); + + /// Told to the source after the renderer uploads `sequence`. + /// + /// Sources use it for the upload counter the performance gates read + /// (PERF-B04: a static page must settle at zero uploads per second). + /// + /// **Called from inside [`Self::with_frame`]**, because that is the only + /// place a renderer can see the sequence it just uploaded. An + /// implementation must therefore not take any lock that `with_frame` + /// holds, or the render thread deadlocks on the first painted frame. + fn mark_uploaded(&self, sequence: u64); + + /// How many uploads this source has served. Instrumentation only. + fn upload_count(&self) -> u64; +} + +/// A frame producer plus the bookkeeping every implementation would otherwise +/// duplicate: double buffering, dirty-rect accumulation, sequence and +/// generation counters, and the upload counter the perf gates read. +/// +/// Producers call [`Self::submit`]; GPUI calls the [`ExternalTextureSource`] +/// methods. Neither side needs to know about the other's thread. +pub struct ExternalTextureBuffer { + id: ExternalTextureId, + state: parking_lot::Mutex, + /// Read by perf assertions from another thread, so it lives outside the + /// lock. + uploads: AtomicU64, + /// Mirrors `state.sequence` for lock-free reads. + sequence: AtomicU64, + /// The last sequence a renderer said it uploaded. + /// + /// Outside the mutex on purpose. Renderers acknowledge from inside the + /// [`ExternalTextureSource::with_frame`] visitor, because that is the only + /// place they can see the sequence they just uploaded, and the lock is held + /// for the whole visit. An acknowledgement that took the lock would + /// deadlock the render thread on the first painted frame. + acknowledged: AtomicU64, +} + +#[derive(Default)] +struct BufferState { + bytes: Vec, + size: Size, + stride: usize, + format: Option, + generation: u64, + sequence: u64, + dirty: Vec>, +} + +impl fmt::Debug for ExternalTextureBuffer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let state = self.state.lock(); + f.debug_struct("ExternalTextureBuffer") + .field("id", &self.id) + .field("size", &state.size) + .field("generation", &state.generation) + .field("sequence", &state.sequence) + .finish() + } +} + +impl Default for ExternalTextureBuffer { + fn default() -> Self { + Self::new() + } +} + +impl ExternalTextureBuffer { + /// A buffer with no frame yet. Allocates nothing until the first + /// [`Self::submit`], which is what lets a browser pane exist without + /// costing a framebuffer (PRD ENG-02). + pub fn new() -> Self { + Self { + id: ExternalTextureId::next(), + state: parking_lot::Mutex::new(BufferState::default()), + uploads: AtomicU64::new(0), + sequence: AtomicU64::new(0), + acknowledged: AtomicU64::new(0), + } + } + + /// Wrap in an `Arc` for handing to [`crate::Window::paint_external_texture`]. + pub fn shared() -> Arc { + Arc::new(Self::new()) + } + + /// Whether any frame has been produced. A pane whose page has not painted + /// yet must not draw a black rectangle over its own placeholder. + pub fn has_frame(&self) -> bool { + self.sequence.load(Ordering::Acquire) > 0 + } + + /// The frame counter, for tests and instrumentation. + pub fn sequence(&self) -> u64 { + self.sequence.load(Ordering::Acquire) + } + + /// Last frame committed by a renderer. CPU submissions do not advance it. + pub fn uploaded_sequence(&self) -> u64 { + self.acknowledged.load(Ordering::Acquire) + } + + /// Current buffer dimensions, or zero before the first frame. + pub fn size(&self) -> Size { + self.state.lock().size + } + + /// Publish a frame. + /// + /// `dirty` is in texture pixels; an empty slice means the whole frame. A + /// size or format change reallocates and bumps the generation, and the + /// dirty list is then irrelevant because the renderer will recreate the + /// texture anyway. + /// + /// Returns `false` and drops the frame when `bytes` is too small for the + /// declared geometry, so a misbehaving producer cannot make the renderer + /// read out of bounds. + pub fn submit( + &self, + size: Size, + stride: usize, + format: ExternalTextureFormat, + bytes: &[u8], + dirty: &[Bounds], + ) -> bool { + let width = size.width.0.max(0) as usize; + let height = size.height.0.max(0) as usize; + if width == 0 + || height == 0 + || u32::try_from(stride).is_err() + || width + .checked_mul(format.bytes_per_pixel()) + .is_none_or(|minimum| stride < minimum) + { + return false; + } + let needed = stride + .checked_mul(height) + .filter(|needed| *needed <= bytes.len()); + let Some(needed) = needed else { + return false; + }; + + let mut state = self.state.lock(); + let reallocated = + state.size != size || state.stride != stride || state.format != Some(format); + if reallocated { + state.bytes.clear(); + state.bytes.resize(needed, 0); + state.size = size; + state.stride = stride; + state.format = Some(format); + state.generation += 1; + state.dirty.clear(); + } else if state.bytes.len() < needed { + state.bytes.resize(needed, 0); + } + state.bytes[..needed].copy_from_slice(&bytes[..needed]); + + let acknowledged = self.acknowledged.load(Ordering::Acquire); + if reallocated { + // A recreated texture is uploaded whole; per-rect bookkeeping would + // just be thrown away. + state.dirty.clear(); + } else if state.sequence > acknowledged { + // The renderer has not consumed the previous frame, so this frame's + // dirt is added to it rather than replacing it. Dropping it would + // leave half the page showing stale pixels. + // Empty means a full repaint. It must dominate in either order; + // appending a small rectangle to it would lose that full repaint. + // Bound the list when a hidden/slow consumer misses many frames. + if dirty.is_empty() || state.dirty.len().saturating_add(dirty.len()) > 64 { + state.dirty.clear(); + } else if !state.dirty.is_empty() { + state.dirty.extend_from_slice(dirty); + } + } else { + // Everything up to here has been uploaded, so the previous rects + // are spent. This is also where an acknowledgement's clean-up + // happens, which is why `mark_uploaded` does not need the lock. + state.dirty.clear(); + if dirty.len() <= 64 { + state.dirty.extend_from_slice(dirty); + } + } + + state.sequence += 1; + self.sequence.store(state.sequence, Ordering::Release); + true + } + + /// Forget the frame without dropping the identity, so a hidden pane stops + /// holding a framebuffer while keeping its renderer (PRD HID-01). + pub fn release_frame(&self) { + let mut state = self.state.lock(); + state.bytes = Vec::new(); + state.size = Size::default(); + state.stride = 0; + state.format = None; + state.dirty.clear(); + state.generation += 1; + state.sequence = 0; + self.sequence.store(0, Ordering::Release); + self.acknowledged.store(0, Ordering::Release); + } +} + +impl ExternalTextureSource for ExternalTextureBuffer { + fn id(&self) -> ExternalTextureId { + self.id + } + + fn with_frame(&self, visit: &mut dyn FnMut(ExternalFrameView<'_>)) { + let state = self.state.lock(); + let Some(format) = state.format else { + return; + }; + if state.sequence == 0 { + return; + } + visit(ExternalFrameView { + size: state.size, + stride: state.stride, + format, + bytes: &state.bytes, + generation: state.generation, + sequence: state.sequence, + dirty: &state.dirty, + }); + } + + fn mark_uploaded(&self, sequence: u64) { + // Deliberately lock-free: this is called from inside `with_frame`, + // which holds the state lock for the whole visit. + self.acknowledged.fetch_max(sequence, Ordering::AcqRel); + self.uploads.fetch_add(1, Ordering::Relaxed); + } + + fn upload_count(&self) -> u64 { + self.uploads.load(Ordering::Relaxed) + } +} + +/// What a renderer needs to decide whether to upload, kept out of the renderers +/// so Windows and Linux cannot drift apart on the rule. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct ExternalTextureCacheKey { + /// Which source the cached texture belongs to. + pub id: ExternalTextureId, + /// Which allocation. A change means recreate. + pub generation: u64, + /// Which frame. A change means upload the dirty rects. + pub sequence: u64, + /// Cached texture dimensions. + pub size: Size, +} + +/// The decision a renderer makes for one external texture this frame. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum ExternalTextureUpdate { + /// No cached texture, or the producer reallocated. Create and upload whole. + Recreate, + /// Same texture, new pixels. Upload the dirty rects. + UploadDirty, + /// Nothing changed. Draw the cached texture and upload nothing. This is the + /// branch a settled page must take every frame (PERF-B04). + Reuse, +} + +/// Decide what to do with a cached texture given the frame on offer. +pub fn plan_external_texture_update( + cached: Option, + frame: &ExternalFrameView<'_>, +) -> ExternalTextureUpdate { + match cached { + Some(cached) + if cached.generation == frame.generation + && cached.size == frame.size + && cached.sequence == frame.sequence => + { + ExternalTextureUpdate::Reuse + } + Some(cached) if cached.generation == frame.generation && cached.size == frame.size => { + ExternalTextureUpdate::UploadDirty + } + _ => ExternalTextureUpdate::Recreate, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{point, size}; + + fn bgra(width: i32, height: i32, fill: u8) -> Vec { + vec![fill; (width * height * 4) as usize] + } + + fn dims(width: i32, height: i32) -> Size { + size(DevicePixels(width), DevicePixels(height)) + } + + #[test] + fn a_buffer_with_no_frame_never_calls_the_visitor() { + let buffer = ExternalTextureBuffer::new(); + assert!(!buffer.has_frame()); + let mut seen = 0; + buffer.with_frame(&mut |_| seen += 1); + assert_eq!(seen, 0, "a pane that never painted must draw nothing"); + } + + /// The counter PERF-B04 reads must count uploads, not draws. + /// + /// This is the shape of a bug that shipped in the Windows renderer and not + /// the wgpu one: it acknowledged whenever the cached sequence matched the + /// frame's, which is exactly the `Reuse` condition, so a still page + /// appeared to upload once per composited frame. + #[test] + fn reusing_a_cached_texture_is_not_an_upload() { + let buffer = ExternalTextureBuffer::new(); + let pixels = bgra(4, 2, 0x11); + assert!(buffer.submit(dims(4, 2), 16, ExternalTextureFormat::Bgra8, &pixels, &[])); + + // First look at the frame: nothing cached, so the renderer uploads. + let mut cached = None; + buffer.with_frame(&mut |frame| { + assert_eq!( + plan_external_texture_update(cached, &frame), + ExternalTextureUpdate::Recreate + ); + cached = Some(ExternalTextureCacheKey { + id: buffer.id(), + generation: frame.generation, + sequence: frame.sequence, + size: frame.size, + }); + buffer.mark_uploaded(frame.sequence); + }); + assert_eq!(buffer.upload_count(), 1); + + // Every later frame draws the same pixels. A renderer that + // acknowledges here is counting compositing, not uploading. + for _ in 0..30 { + buffer.with_frame(&mut |frame| { + let plan = plan_external_texture_update(cached, &frame); + assert_eq!(plan, ExternalTextureUpdate::Reuse); + if !matches!(plan, ExternalTextureUpdate::Reuse) { + buffer.mark_uploaded(frame.sequence); + } + }); + } + assert_eq!( + buffer.upload_count(), + 1, + "a settled page must stop adding to the upload counter" + ); + } + + #[test] + fn ids_are_unique_per_source() { + let a = ExternalTextureBuffer::new(); + let b = ExternalTextureBuffer::new(); + assert_ne!(a.id(), b.id()); + } + + #[test] + fn a_submitted_frame_is_visible_to_the_renderer() { + let buffer = ExternalTextureBuffer::new(); + let pixels = bgra(4, 2, 0x7f); + assert!(buffer.submit(dims(4, 2), 16, ExternalTextureFormat::Bgra8, &pixels, &[])); + assert!(buffer.has_frame()); + + let mut observed = None; + buffer.with_frame(&mut |frame| { + assert!(frame.is_well_formed()); + observed = Some((frame.size, frame.sequence, frame.generation, frame.format)); + assert!(frame.bytes.iter().all(|byte| *byte == 0x7f)); + }); + assert_eq!( + observed, + Some((dims(4, 2), 1, 1, ExternalTextureFormat::Bgra8)) + ); + } + + #[test] + fn a_short_buffer_is_refused_rather_than_read_out_of_bounds() { + let buffer = ExternalTextureBuffer::new(); + let too_small = bgra(4, 1, 0); + assert!(!buffer.submit( + dims(4, 2), + 16, + ExternalTextureFormat::Bgra8, + &too_small, + &[] + )); + assert!(!buffer.has_frame()); + } + + #[test] + fn a_stride_narrower_than_the_row_is_refused() { + let buffer = ExternalTextureBuffer::new(); + let pixels = bgra(4, 2, 0); + assert!(!buffer.submit(dims(4, 2), 8, ExternalTextureFormat::Bgra8, &pixels, &[])); + } + + #[test] + fn a_settled_page_uploads_nothing() { + // PERF-B04 in one test: submit once, paint three times, and only the + // first paint may upload. + let buffer = ExternalTextureBuffer::new(); + let pixels = bgra(8, 8, 1); + buffer.submit(dims(8, 8), 32, ExternalTextureFormat::Bgra8, &pixels, &[]); + + let mut cached: Option = None; + let mut plans = Vec::new(); + for _ in 0..3 { + buffer.with_frame(&mut |frame| { + let plan = plan_external_texture_update(cached, &frame); + plans.push(plan); + if !matches!(plan, ExternalTextureUpdate::Reuse) { + buffer.mark_uploaded(frame.sequence); + cached = Some(ExternalTextureCacheKey { + id: buffer.id(), + generation: frame.generation, + sequence: frame.sequence, + size: frame.size, + }); + } + }); + } + assert_eq!( + plans, + vec![ + ExternalTextureUpdate::Recreate, + ExternalTextureUpdate::Reuse, + ExternalTextureUpdate::Reuse + ] + ); + assert_eq!(buffer.upload_count(), 1); + } + + #[test] + fn a_new_frame_at_the_same_size_uploads_only_the_dirty_rects() { + let buffer = ExternalTextureBuffer::new(); + let pixels = bgra(8, 8, 1); + buffer.submit(dims(8, 8), 32, ExternalTextureFormat::Bgra8, &pixels, &[]); + let mut cached = None; + buffer.with_frame(&mut |frame| { + buffer.mark_uploaded(frame.sequence); + cached = Some(ExternalTextureCacheKey { + id: buffer.id(), + generation: frame.generation, + sequence: frame.sequence, + size: frame.size, + }); + }); + + let dirty = [Bounds { + origin: point(DevicePixels(2), DevicePixels(3)), + size: dims(4, 2), + }]; + buffer.submit( + dims(8, 8), + 32, + ExternalTextureFormat::Bgra8, + &bgra(8, 8, 2), + &dirty, + ); + + let mut plan = None; + let mut regions = Vec::new(); + buffer.with_frame(&mut |frame| { + plan = Some(plan_external_texture_update(cached, &frame)); + regions = frame.dirty_regions(); + }); + assert_eq!(plan, Some(ExternalTextureUpdate::UploadDirty)); + assert_eq!(regions, dirty.to_vec()); + } + + #[test] + fn a_resize_forces_the_renderer_to_recreate_its_texture() { + let buffer = ExternalTextureBuffer::new(); + buffer.submit( + dims(8, 8), + 32, + ExternalTextureFormat::Bgra8, + &bgra(8, 8, 1), + &[], + ); + let mut cached = None; + buffer.with_frame(&mut |frame| { + cached = Some(ExternalTextureCacheKey { + id: buffer.id(), + generation: frame.generation, + sequence: frame.sequence, + size: frame.size, + }); + }); + + buffer.submit( + dims(16, 8), + 64, + ExternalTextureFormat::Bgra8, + &bgra(16, 8, 1), + &[], + ); + let mut plan = None; + buffer.with_frame(&mut |frame| { + plan = Some(plan_external_texture_update(cached, &frame)); + assert_eq!(frame.generation, 2, "a reallocation bumps the generation"); + }); + assert_eq!(plan, Some(ExternalTextureUpdate::Recreate)); + } + + #[test] + fn dirt_accumulates_while_the_renderer_is_behind() { + // Two frames land between paints. The renderer must be told about both + // regions, or half the page keeps stale pixels. + let buffer = ExternalTextureBuffer::new(); + buffer.submit( + dims(8, 8), + 32, + ExternalTextureFormat::Bgra8, + &bgra(8, 8, 0), + &[], + ); + buffer.with_frame(&mut |frame| buffer.mark_uploaded(frame.sequence)); + + let first = Bounds { + origin: point(DevicePixels(0), DevicePixels(0)), + size: dims(2, 2), + }; + let second = Bounds { + origin: point(DevicePixels(4), DevicePixels(4)), + size: dims(2, 2), + }; + buffer.submit( + dims(8, 8), + 32, + ExternalTextureFormat::Bgra8, + &bgra(8, 8, 1), + &[first], + ); + buffer.submit( + dims(8, 8), + 32, + ExternalTextureFormat::Bgra8, + &bgra(8, 8, 2), + &[second], + ); + + let mut regions = Vec::new(); + buffer.with_frame(&mut |frame| regions = frame.dirty_regions()); + assert_eq!(regions, vec![first, second]); + } + + #[test] + fn an_empty_dirty_list_means_the_whole_frame() { + let buffer = ExternalTextureBuffer::new(); + buffer.submit( + dims(6, 4), + 24, + ExternalTextureFormat::Bgra8, + &bgra(6, 4, 0), + &[], + ); + let mut regions = Vec::new(); + buffer.with_frame(&mut |frame| regions = frame.dirty_regions()); + assert_eq!( + regions, + vec![Bounds { + origin: point(DevicePixels(0), DevicePixels(0)), + size: dims(6, 4), + }] + ); + } + + #[test] + fn dirty_rects_are_clamped_to_the_buffer() { + // The rect has to arrive on a frame that is *not* a reallocation, + // because a reallocation uploads the whole texture and discards + // per-rect bookkeeping. The interesting case is a stale rect from a + // larger previous frame arriving after a shrink. + let buffer = ExternalTextureBuffer::new(); + buffer.submit( + dims(8, 8), + 32, + ExternalTextureFormat::Bgra8, + &bgra(8, 8, 0), + &[], + ); + buffer.with_frame(&mut |frame| buffer.mark_uploaded(frame.sequence)); + + let outside = Bounds { + origin: point(DevicePixels(4), DevicePixels(4)), + size: dims(64, 64), + }; + buffer.submit( + dims(8, 8), + 32, + ExternalTextureFormat::Bgra8, + &bgra(8, 8, 1), + &[outside], + ); + let mut regions = Vec::new(); + buffer.with_frame(&mut |frame| regions = frame.dirty_regions()); + assert_eq!( + regions, + vec![Bounds { + origin: point(DevicePixels(4), DevicePixels(4)), + size: dims(4, 4), + }], + "a producer's rect must never index past the texture" + ); + } + + #[test] + fn the_first_frame_after_a_reallocation_uploads_everything() { + // The behaviour the test above had to work around, asserted directly: + // a new texture has no pixels, so a partial upload would leave the rest + // of the pane as whatever the driver had in that memory. + let buffer = ExternalTextureBuffer::new(); + let corner = Bounds { + origin: point(DevicePixels(0), DevicePixels(0)), + size: dims(1, 1), + }; + buffer.submit( + dims(8, 8), + 32, + ExternalTextureFormat::Bgra8, + &bgra(8, 8, 0), + &[corner], + ); + let mut regions = Vec::new(); + buffer.with_frame(&mut |frame| regions = frame.dirty_regions()); + assert_eq!( + regions, + vec![Bounds { + origin: point(DevicePixels(0), DevicePixels(0)), + size: dims(8, 8), + }] + ); + } + + #[test] + fn acknowledging_from_inside_the_visitor_does_not_deadlock() { + // The renderers do exactly this: they learn the sequence they uploaded + // from the frame view, and acknowledge it before the view goes out of + // scope. `with_frame` holds the state lock for the whole visit, so an + // acknowledgement that took the same lock would hang the render thread + // on the very first painted frame. + let buffer = ExternalTextureBuffer::new(); + buffer.submit( + dims(4, 4), + 16, + ExternalTextureFormat::Bgra8, + &bgra(4, 4, 3), + &[], + ); + buffer.with_frame(&mut |frame| { + buffer.mark_uploaded(frame.sequence); + }); + assert_eq!(buffer.upload_count(), 1); + } + + #[test] + fn an_acknowledged_frame_stops_re_reporting_its_dirty_rects() { + // The clean-up that used to live in `mark_uploaded` now happens on the + // next submit, so this is where it has to be proven. + let buffer = ExternalTextureBuffer::new(); + buffer.submit( + dims(8, 8), + 32, + ExternalTextureFormat::Bgra8, + &bgra(8, 8, 0), + &[], + ); + buffer.with_frame(&mut |frame| buffer.mark_uploaded(frame.sequence)); + + let first = Bounds { + origin: point(DevicePixels(0), DevicePixels(0)), + size: dims(2, 2), + }; + buffer.submit( + dims(8, 8), + 32, + ExternalTextureFormat::Bgra8, + &bgra(8, 8, 1), + &[first], + ); + buffer.with_frame(&mut |frame| { + assert_eq!(frame.dirty_regions(), vec![first]); + buffer.mark_uploaded(frame.sequence); + }); + + let second = Bounds { + origin: point(DevicePixels(4), DevicePixels(4)), + size: dims(2, 2), + }; + buffer.submit( + dims(8, 8), + 32, + ExternalTextureFormat::Bgra8, + &bgra(8, 8, 2), + &[second], + ); + let mut regions = Vec::new(); + buffer.with_frame(&mut |frame| regions = frame.dirty_regions()); + assert_eq!( + regions, + vec![second], + "the acknowledged rect must not be uploaded a second time" + ); + } + + #[test] + fn releasing_a_frame_keeps_the_identity_but_drops_the_pixels() { + let buffer = ExternalTextureBuffer::new(); + let id = buffer.id(); + buffer.submit( + dims(8, 8), + 32, + ExternalTextureFormat::Bgra8, + &bgra(8, 8, 1), + &[], + ); + buffer.release_frame(); + assert!(!buffer.has_frame()); + assert_eq!(buffer.id(), id); + let mut seen = 0; + buffer.with_frame(&mut |_| seen += 1); + assert_eq!(seen, 0); + } + #[test] + fn full_repaint_dominates_partial_frames_in_both_orders() { + for full_first in [true, false] { + let buffer = ExternalTextureBuffer::new(); + let pixels = bgra(8, 8, 1); + buffer.submit(dims(8, 8), 32, ExternalTextureFormat::Bgra8, &pixels, &[]); + buffer.with_frame(&mut |frame| buffer.mark_uploaded(frame.sequence)); + let partial = [Bounds { + origin: point(DevicePixels(1), DevicePixels(1)), + size: dims(2, 2), + }]; + let (first, second): (&[_], &[_]) = if full_first { + (&[], &partial) + } else { + (&partial, &[]) + }; + buffer.submit(dims(8, 8), 32, ExternalTextureFormat::Bgra8, &pixels, first); + buffer.submit( + dims(8, 8), + 32, + ExternalTextureFormat::Bgra8, + &pixels, + second, + ); + buffer.with_frame(&mut |frame| { + assert_eq!( + frame.dirty_regions(), + vec![Bounds { + origin: point(DevicePixels(0), DevicePixels(0)), + size: dims(8, 8) + }] + ); + }); + } + } + + #[test] + fn missed_partial_frames_have_bounded_dirty_bookkeeping() { + let buffer = ExternalTextureBuffer::new(); + let pixels = bgra(8, 8, 1); + buffer.submit(dims(8, 8), 32, ExternalTextureFormat::Bgra8, &pixels, &[]); + buffer.with_frame(&mut |frame| buffer.mark_uploaded(frame.sequence)); + let partial = [Bounds { + origin: point(DevicePixels(1), DevicePixels(1)), + size: dims(2, 2), + }]; + for _ in 0..1000 { + buffer.submit( + dims(8, 8), + 32, + ExternalTextureFormat::Bgra8, + &pixels, + &partial, + ); + } + buffer.with_frame(&mut |frame| assert!(frame.dirty.is_empty())); + } + + #[test] + fn overflowing_frame_geometry_is_rejected() { + let frame = ExternalFrameView { + size: dims(1, 2), + stride: usize::MAX / 2 + 1, + format: ExternalTextureFormat::Bgra8, + bytes: &[], + generation: 1, + sequence: 1, + dirty: &[], + }; + assert!(!frame.is_well_formed()); + } + + #[test] + fn dirty_rectangles_cannot_overflow_when_clipped() { + let dirty = [Bounds { + origin: point(DevicePixels(i32::MAX - 1), DevicePixels(0)), + size: dims(8, 8), + }]; + let pixels = bgra(8, 8, 0); + let frame = ExternalFrameView { + size: dims(8, 8), + stride: 32, + format: ExternalTextureFormat::Bgra8, + bytes: &pixels, + generation: 1, + sequence: 1, + dirty: &dirty, + }; + assert_eq!( + frame.dirty_regions(), + vec![Bounds { + origin: point(DevicePixels(0), DevicePixels(0)), + size: dims(8, 8), + }] + ); + } + + #[test] + fn unusable_dirty_hints_cannot_acknowledge_stale_pixels() { + let buffer = ExternalTextureBuffer::new(); + let mut texture = bgra(4, 4, 0); + buffer.submit(dims(4, 4), 16, ExternalTextureFormat::Bgra8, &texture, &[]); + buffer.with_frame(&mut |frame| buffer.mark_uploaded(frame.sequence)); + let next = bgra(4, 4, 17); + let dirty = [Bounds { + origin: point(DevicePixels(100), DevicePixels(100)), + size: dims(1, 1), + }]; + buffer.submit(dims(4, 4), 16, ExternalTextureFormat::Bgra8, &next, &dirty); + buffer.with_frame(&mut |frame| { + for region in frame.dirty_regions() { + let left = region.origin.x.0 as usize * 4; + let right = left + region.size.width.0 as usize * 4; + for y in region.origin.y.0..region.origin.y.0 + region.size.height.0 { + let row = y as usize * frame.stride; + texture[row + left..row + right] + .copy_from_slice(&frame.bytes[row + left..row + right]); + } + } + buffer.mark_uploaded(frame.sequence); + }); + assert_eq!( + texture, next, + "an acknowledged frame must update the texture" + ); + } +} diff --git a/crates/gpui/src/gpui.rs b/crates/gpui/src/gpui.rs index a81ff265c3edd8..d68d4c58dbb08f 100644 --- a/crates/gpui/src/gpui.rs +++ b/crates/gpui/src/gpui.rs @@ -21,6 +21,7 @@ pub mod colors; mod element; mod elements; mod executor; +mod external_texture; mod platform_scheduler; pub(crate) use platform_scheduler::PlatformScheduler; mod geometry; @@ -97,6 +98,7 @@ pub use ctor::ctor; pub use element::*; pub use elements::*; pub use executor::*; +pub use external_texture::*; pub use geometry::*; pub use global::*; pub use gpui_macros::{ diff --git a/crates/gpui/src/scene.rs b/crates/gpui/src/scene.rs index bc7f5d79eace55..c53cf58bbfc77c 100644 --- a/crates/gpui/src/scene.rs +++ b/crates/gpui/src/scene.rs @@ -721,8 +721,52 @@ pub struct PaintSurface { pub order: DrawOrder, pub bounds: Bounds, pub content_mask: ContentMask, + pub content: SurfaceContent, +} + +/// What a [`PaintSurface`] samples from. +/// +/// Both variants bypass the sprite atlas: a surface owns its texture. That is +/// the point of the primitive. See [`crate::external_texture`] for why a live +/// video frame or web page must never be atlased. +#[derive(Clone)] +pub enum SurfaceContent { + /// A CoreVideo buffer, as produced by macOS video decode. YUV, sampled by + /// the Metal renderer's two-plane surface shader. #[cfg(target_os = "macos")] - pub image_buffer: core_video::pixel_buffer::CVPixelBuffer, + PixelBuffer(core_video::pixel_buffer::CVPixelBuffer), + /// A caller-owned RGBA/BGRA frame, uploaded to a dedicated texture the + /// renderer caches per [`crate::ExternalTextureId`]. This is the + /// cross-platform path, and the one Chromium off-screen rendering uses. + ExternalTexture(std::sync::Arc), +} + +impl SurfaceContent { + /// The caller-owned texture this surface samples, if it is one. + /// + /// Renderers use this rather than destructuring, because off macOS the enum + /// has a single variant and a `let ... else` on it is irrefutable. It also + /// keeps the mapping in one place instead of once per backend. + pub fn external_texture(&self) -> Option<&std::sync::Arc> { + match self { + #[cfg(target_os = "macos")] + Self::PixelBuffer(_) => None, + Self::ExternalTexture(source) => Some(source), + } + } +} + +impl Debug for SurfaceContent { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + #[cfg(target_os = "macos")] + Self::PixelBuffer(_) => f.write_str("SurfaceContent::PixelBuffer"), + Self::ExternalTexture(source) => f + .debug_tuple("SurfaceContent::ExternalTexture") + .field(&source.id()) + .finish(), + } + } } impl From for Primitive { diff --git a/crates/gpui/src/window.rs b/crates/gpui/src/window.rs index 37a54bcebab17e..32522845715265 100644 --- a/crates/gpui/src/window.rs +++ b/crates/gpui/src/window.rs @@ -4120,7 +4120,7 @@ impl Window { /// This method should only be called as part of the paint phase of element drawing. #[cfg(target_os = "macos")] pub fn paint_surface(&mut self, bounds: Bounds, image_buffer: CVPixelBuffer) { - use crate::PaintSurface; + use crate::{PaintSurface, SurfaceContent}; self.invalidator.debug_assert_paint(); @@ -4130,7 +4130,37 @@ impl Window { order: 0, bounds, content_mask, - image_buffer, + content: SurfaceContent::PixelBuffer(image_buffer), + }); + } + + /// Paint a caller-owned texture into the scene at the current z-index. + /// + /// The renderer keeps one GPU texture per [`crate::ExternalTextureId`] and + /// uploads only what the source reports as dirty, so a source that has + /// stopped producing frames costs a draw call and no bandwidth. + /// + /// This deliberately does **not** go through the sprite atlas. Atlas tiles + /// are for icons and glyphs; a 1080p frame arriving at 60 Hz would evict + /// the entire atlas every frame. See [`crate::external_texture`]. + /// + /// Call only during the paint phase of element drawing. + pub fn paint_external_texture( + &mut self, + bounds: Bounds, + source: Arc, + ) { + use crate::{PaintSurface, SurfaceContent}; + + self.invalidator.debug_assert_paint(); + + let bounds = self.snap_bounds(bounds); + let content_mask = self.snapped_content_mask(); + self.next_frame.scene.insert_primitive(PaintSurface { + order: 0, + bounds, + content_mask, + content: SurfaceContent::ExternalTexture(source), }); } diff --git a/crates/gpui_macos/src/metal_renderer.rs b/crates/gpui_macos/src/metal_renderer.rs index d1f1233efe2f10..0088075811e56b 100644 --- a/crates/gpui_macos/src/metal_renderer.rs +++ b/crates/gpui_macos/src/metal_renderer.rs @@ -1521,35 +1521,43 @@ impl MetalRenderer { ); for surface in surfaces { + // Metal keeps the CoreVideo two-plane path. Caller-owned RGBA + // textures (Chromium off-screen rendering) are a Windows/Linux + // path; nothing on macOS produces one, and silently skipping is + // better than sampling a YUV shader over BGRA bytes. + let image_buffer = match &surface.content { + gpui::SurfaceContent::PixelBuffer(image_buffer) => image_buffer, + gpui::SurfaceContent::ExternalTexture(_) => continue, + }; let texture_size = size( - DevicePixels::from(surface.image_buffer.get_width() as i32), - DevicePixels::from(surface.image_buffer.get_height() as i32), + DevicePixels::from(image_buffer.get_width() as i32), + DevicePixels::from(image_buffer.get_height() as i32), ); assert_eq!( - surface.image_buffer.get_pixel_format(), + image_buffer.get_pixel_format(), kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ); let y_texture = self .core_video_texture_cache .create_texture_from_image( - surface.image_buffer.as_concrete_TypeRef(), + image_buffer.as_concrete_TypeRef(), None, MTLPixelFormat::R8Unorm, - surface.image_buffer.get_width_of_plane(0), - surface.image_buffer.get_height_of_plane(0), + image_buffer.get_width_of_plane(0), + image_buffer.get_height_of_plane(0), 0, ) .unwrap(); let cb_cr_texture = self .core_video_texture_cache .create_texture_from_image( - surface.image_buffer.as_concrete_TypeRef(), + image_buffer.as_concrete_TypeRef(), None, MTLPixelFormat::RG8Unorm, - surface.image_buffer.get_width_of_plane(1), - surface.image_buffer.get_height_of_plane(1), + image_buffer.get_width_of_plane(1), + image_buffer.get_height_of_plane(1), 1, ) .unwrap(); diff --git a/crates/gpui_wgpu/src/shaders.wgsl b/crates/gpui_wgpu/src/shaders.wgsl index 933b88e84d79a9..ed300d8d29d8bf 100644 --- a/crates/gpui_wgpu/src/shaders.wgsl +++ b/crates/gpui_wgpu/src/shaders.wgsl @@ -1362,3 +1362,58 @@ fn fs_surface(input: SurfaceVarying) -> @location(0) vec4 { return ycbcr_to_RGB * y_cb_cr; } + +// --- external textures --- // +// +// A caller-owned RGBA/BGRA texture composited inside the scene, used by the +// embedded browser's off-screen Chromium frames. Unlike polychrome sprites +// this never samples the shared sprite atlas: the texture bound at +// group(1) binding(1) belongs to one producer and holds exactly one frame, +// so a 1080p page cannot evict every icon in the app. + +struct ExternalTexture { + bounds: Bounds, + content_mask: Bounds, + opacity: f32, + swap_red_blue: u32, +} +@group(1) @binding(0) var b_external_textures: array; + +struct ExternalTextureVarying { + @builtin(position) position: vec4, + @location(0) texture_position: vec2, + @location(1) @interpolate(flat) external_id: u32, + @location(2) clip_distances: vec4, +} + +@vertex +fn vs_external_texture(@builtin(vertex_index) vertex_id: u32, @builtin(instance_index) instance_id: u32) -> ExternalTextureVarying { + let unit_vertex = vec2(f32(vertex_id & 1u), 0.5 * f32(vertex_id & 2u)); + // Not named `external`: that is a reserved keyword in WGSL, and naga + // rejects the whole shader module for it. The failure is not local to this + // quad either, because every GPUI shader is compiled as one module, so the + // app panics before it can draw anything at all. + let quad = b_external_textures[instance_id]; + + var out = ExternalTextureVarying(); + out.position = to_device_position(unit_vertex, quad.bounds); + // The texture holds exactly this rect, so unit coordinates are the + // texture coordinates. No atlas tile arithmetic. + out.texture_position = unit_vertex; + out.external_id = instance_id; + out.clip_distances = distance_from_clip_rect(unit_vertex, quad.bounds, quad.content_mask); + return out; +} + +@fragment +fn fs_external_texture(input: ExternalTextureVarying) -> @location(0) vec4 { + let sample = textureSample(t_sprite, s_sprite, input.texture_position); + // Alpha clip after using the derivatives. + if (any(input.clip_distances < vec4(0.0))) { + return vec4(0.0); + } + + let quad = b_external_textures[input.external_id]; + let color = select(sample, sample.bgra, quad.swap_red_blue != 0u); + return blend_color(color, quad.opacity); +} diff --git a/crates/gpui_wgpu/src/wgpu_renderer.rs b/crates/gpui_wgpu/src/wgpu_renderer.rs index 08f30dc0090d3a..76b33da99cdc01 100644 --- a/crates/gpui_wgpu/src/wgpu_renderer.rs +++ b/crates/gpui_wgpu/src/wgpu_renderer.rs @@ -1,14 +1,16 @@ use crate::{CompositorGpuHint, WgpuAtlas, WgpuContext}; use bytemuck::{Pod, Zeroable}; use gpui::{ - AtlasTextureId, Background, Bounds, DevicePixels, GpuSpecs, MonochromeSprite, Path, Point, - PolychromeSprite, PrimitiveBatch, Quad, ScaledPixels, Scene, Shadow, Size, SubpixelSprite, - Underline, get_gamma_correction_ratios, + AtlasTextureId, Background, Bounds, DevicePixels, ExternalTextureCacheKey, + ExternalTextureFormat, ExternalTextureId, ExternalTextureUpdate, GpuSpecs, MonochromeSprite, + PaintSurface, Path, Point, PolychromeSprite, PrimitiveBatch, Quad, ScaledPixels, Scene, Shadow, + Size, SubpixelSprite, Underline, get_gamma_correction_ratios, plan_external_texture_update, }; use log::warn; #[cfg(not(target_family = "wasm"))] use raw_window_handle::{HasDisplayHandle, HasWindowHandle}; use std::cell::RefCell; +use std::collections::HashMap; use std::num::NonZeroU64; use std::rc::Rc; use std::sync::{Arc, Mutex}; @@ -37,6 +39,25 @@ impl From> for PodBounds { } } +/// One external-texture draw. Mirrors `ExternalTexture` in `shaders.wgsl`; +/// changing either side without the other silently corrupts the geometry. +#[repr(C)] +#[derive(Clone, Copy, Pod, Zeroable)] +struct ExternalTextureInstance { + bounds: PodBounds, + content_mask: PodBounds, + opacity: f32, + swap_red_blue: u32, +} + +/// A GPU texture owned by one external source, kept across frames so a page +/// that is not repainting costs no upload bandwidth (PERF-B04). +struct CachedExternalTexture { + texture: wgpu::Texture, + view: wgpu::TextureView, + key: ExternalTextureCacheKey, +} + #[repr(C)] #[derive(Clone, Copy, Pod, Zeroable)] struct SurfaceParams { @@ -92,6 +113,7 @@ struct WgpuPipelines { poly_sprites: wgpu::RenderPipeline, #[allow(dead_code)] surfaces: wgpu::RenderPipeline, + external_textures: wgpu::RenderPipeline, } struct WgpuBindGroupLayouts { @@ -120,6 +142,11 @@ struct WgpuResources { path_intermediate_view: Option, path_msaa_texture: Option, path_msaa_view: Option, + /// One cached texture per external source. Interior mutability because the + /// draw path holds `&self` while recording a render pass, and a device loss + /// drops the whole `WgpuResources`, which is exactly when these must go. + external_textures: RefCell>, + external_texture_format: wgpu::TextureFormat, } impl WgpuResources { @@ -463,6 +490,8 @@ impl WgpuRenderer { path_intermediate_view: None, path_msaa_texture: None, path_msaa_view: None, + external_textures: RefCell::new(HashMap::new()), + external_texture_format: context.color_texture_format(), }; Ok(Self { @@ -865,6 +894,7 @@ impl WgpuRenderer { &shader_module, ); + let external_color_target = color_target.clone(); let surfaces = create_pipeline( "surfaces", "vs_surface", @@ -877,6 +907,22 @@ impl WgpuRenderer { &shader_module, ); + // Deliberately built on `instances_with_texture`, the same layout the + // atlas sprites use, but bound to a caller-owned texture rather than an + // atlas page. See `external_texture.rs` in gpui for why atlasing a live + // browser frame is not an option. + let external_textures = create_pipeline( + "external_textures", + "vs_external_texture", + "fs_external_texture", + &layouts.globals, + &layouts.instances_with_texture, + wgpu::PrimitiveTopology::TriangleStrip, + &[Some(external_color_target)], + 1, + &shader_module, + ); + WgpuPipelines { quads, shadows, @@ -887,6 +933,7 @@ impl WgpuRenderer { subpixel_sprites, poly_sprites, surfaces, + external_textures, } } @@ -1080,6 +1127,18 @@ impl WgpuRenderer { } pub fn draw(&mut self, scene: &Scene) -> bool { + // Drop GPU resources for sources absent from the complete next scene. + if let Some(resources) = self.resources.as_mut() { + resources.external_textures.get_mut().retain(|id, _| { + scene.surfaces.iter().any(|surface| { + surface + .content + .external_texture() + .is_some_and(|source| source.id() == *id) + }) + }); + } + // Bail out early if the surface has been unconfigured (e.g. during // Android background/rotation transitions). Attempting to acquire // a texture from an unconfigured surface can block indefinitely on @@ -1300,11 +1359,11 @@ impl WgpuRenderer { &mut instance_offset, &mut pass, ), - PrimitiveBatch::Surfaces(_surfaces) => { - // Surfaces are macOS-only for video playback - // Not implemented for Linux/wgpu - true - } + PrimitiveBatch::Surfaces(range) => self.draw_surfaces( + &scene.surfaces[range], + &mut instance_offset, + &mut pass, + ), }; if !ok { overflow = true; @@ -1446,6 +1505,201 @@ impl WgpuRenderer { ) } + /// Composite caller-owned textures (embedded browser frames) into the + /// scene. + /// + /// CoreVideo surfaces are a macOS path and never reach here; on Linux a + /// `PaintSurface` is always an external texture. + fn draw_surfaces( + &self, + surfaces: &[PaintSurface], + instance_offset: &mut u64, + pass: &mut wgpu::RenderPass<'_>, + ) -> bool { + for surface in surfaces { + let Some(source) = surface.content.external_texture() else { + continue; + }; + let mut ok = true; + source.with_frame(&mut |frame| { + if !frame.is_well_formed() { + // A producer that mis-declares its geometry is a bug, but + // it must not become an out-of-bounds GPU read. + log::error!( + "external texture {:?} offered a malformed frame ({:?}, stride {})", + source.id(), + frame.size, + frame.stride + ); + return; + } + + let cached_key = self + .resources() + .external_textures + .borrow() + .get(&source.id()) + .map(|cached| cached.key); + let plan = plan_external_texture_update(cached_key, &frame); + + if !matches!(plan, ExternalTextureUpdate::Reuse) { + if !self.upload_external_texture(source.id(), &frame, plan) { + ok = false; + return; + } + source.mark_uploaded(frame.sequence); + } + + let instances = [ExternalTextureInstance { + bounds: surface.bounds.into(), + content_mask: surface.content_mask.bounds.into(), + opacity: 1.0, + swap_red_blue: u32::from( + matches!(frame.format, ExternalTextureFormat::Bgra8) + != (self.resources().external_texture_format + == wgpu::TextureFormat::Bgra8Unorm), + ), + }]; + // SAFETY: ExternalTextureInstance is Pod and its layout matches + // the ExternalTexture storage-buffer element in shaders.wgsl. + let data = unsafe { Self::instance_bytes(&instances) }; + + let textures = self.resources().external_textures.borrow(); + let Some(cached) = textures.get(&source.id()) else { + return; + }; + ok = self.draw_instances_with_texture( + data, + 1, + &cached.view, + &self.resources().pipelines.external_textures, + instance_offset, + pass, + ); + }); + if !ok { + return false; + } + } + true + } + + /// Create or refresh the cached texture for one external source. Returns + /// false only when the frame cannot be represented at all. + fn upload_external_texture( + &self, + id: ExternalTextureId, + frame: &gpui::ExternalFrameView<'_>, + plan: ExternalTextureUpdate, + ) -> bool { + let resources = self.resources(); + // The context validated sampled/copy support for this format. Upload + // producer bytes directly; the draw instance swaps channels if needed. + let format = resources.external_texture_format; + let width = frame.size.width.0.max(0) as u32; + let height = frame.size.height.0.max(0) as u32; + if width == 0 || height == 0 { + return true; + } + + let mut textures = resources.external_textures.borrow_mut(); + if matches!(plan, ExternalTextureUpdate::Recreate) { + let texture = resources.device.create_texture(&wgpu::TextureDescriptor { + label: Some("external_texture"), + size: wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + let view = texture.create_view(&wgpu::TextureViewDescriptor::default()); + textures.insert( + id, + CachedExternalTexture { + texture, + view, + key: ExternalTextureCacheKey { + id, + generation: frame.generation, + sequence: 0, + size: frame.size, + }, + }, + ); + } + + let Some(cached) = textures.get_mut(&id) else { + return false; + }; + + // A recreated texture has no pixels yet, so it takes the whole frame + // regardless of what the producer reported as dirty. + let regions = if matches!(plan, ExternalTextureUpdate::Recreate) { + vec![Bounds { + origin: gpui::point(DevicePixels(0), DevicePixels(0)), + size: frame.size, + }] + } else { + frame.dirty_regions() + }; + + for region in regions { + let x = region.origin.x.0.max(0) as u32; + let y = region.origin.y.0.max(0) as u32; + let region_width = region.size.width.0.max(0) as u32; + let region_height = region.size.height.0.max(0) as u32; + if region_width == 0 || region_height == 0 { + continue; + } + let offset = y as usize * frame.stride + x as usize * 4; + // `write_texture` reads `region_height` rows of `region_width * 4` + // bytes at `frame.stride` pitch starting here. A slice shorter than + // that is a wgpu validation panic, so the whole span is checked + // rather than just its first byte. + let span_end = + (y + region_height - 1) as usize * frame.stride + (x + region_width) as usize * 4; + if span_end > frame.bytes.len() { + log::error!( + "external texture {id:?} reported a dirty region outside its own buffer" + ); + continue; + } + resources.queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &cached.texture, + mip_level: 0, + origin: wgpu::Origin3d { x, y, z: 0 }, + aspect: wgpu::TextureAspect::All, + }, + &frame.bytes[offset..], + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(frame.stride as u32), + rows_per_image: Some(region_height), + }, + wgpu::Extent3d { + width: region_width, + height: region_height, + depth_or_array_layers: 1, + }, + ); + } + + cached.key = ExternalTextureCacheKey { + id, + generation: frame.generation, + sequence: frame.sequence, + size: frame.size, + }; + true + } + fn draw_instances( &self, data: &[u8], diff --git a/crates/gpui_windows/build.rs b/crates/gpui_windows/build.rs index 1db95715e26daf..95da58cfdeeb02 100644 --- a/crates/gpui_windows/build.rs +++ b/crates/gpui_windows/build.rs @@ -38,6 +38,7 @@ mod shader_compilation { "monochrome_sprite", "subpixel_sprite", "polychrome_sprite", + "external_texture", ]; let rust_binding_path = format!("{}/shaders_bytes.rs", out_dir); diff --git a/crates/gpui_windows/src/directx_renderer.rs b/crates/gpui_windows/src/directx_renderer.rs index 03666241e674d5..259993c50f7330 100644 --- a/crates/gpui_windows/src/directx_renderer.rs +++ b/crates/gpui_windows/src/directx_renderer.rs @@ -1,4 +1,5 @@ use std::{ + collections::HashMap, slice, sync::{Arc, OnceLock}, }; @@ -44,6 +45,10 @@ pub(crate) struct DirectXRenderer { pipelines: DirectXRenderPipelines, direct_composition: Option, font_info: &'static FontInfo, + /// One cached D3D texture per external source (embedded browser panes), so + /// a page that is not repainting costs a draw call and no upload + /// (PERF-B04). Dropped with the device on a device-lost recovery. + external_textures: HashMap, width: u32, height: u32, @@ -90,6 +95,25 @@ struct DirectXRenderPipelines { mono_sprites: PipelineState, subpixel_sprites: PipelineState, poly_sprites: PipelineState, + external_textures: PipelineState, +} + +/// One external-texture draw. Mirrors `ExternalTexture` in `shaders.hlsl`; +/// changing either side without the other silently corrupts the geometry. +#[derive(Clone, Copy)] +#[repr(C)] +struct ExternalTextureInstance { + bounds: Bounds, + content_mask: Bounds, + opacity: f32, + pad: u32, +} + +/// A GPU texture owned by one external source, kept across frames. +struct CachedExternalTexture { + texture: ID3D11Texture2D, + view: Option, + key: ExternalTextureCacheKey, } struct DirectXGlobalElements { @@ -171,6 +195,7 @@ impl DirectXRenderer { pipelines, direct_composition, font_info: Self::get_font_info(), + external_textures: HashMap::new(), width: 1, height: 1, skip_draws: false, @@ -259,6 +284,8 @@ impl DirectXRenderer { self.direct_composition.take(); self.devices.take(); } + // These hold textures created on the dead device. + self.external_textures.clear(); let devices = DirectXRendererDevices::new(directx_devices, disable_direct_composition) .context("Recreating DirectX devices")?; @@ -306,6 +333,15 @@ impl DirectXRenderer { scene: &Scene, background_appearance: WindowBackgroundAppearance, ) -> Result<()> { + // Prune once against the complete scene, including frames with no surfaces. + self.external_textures.retain(|id, _| { + scene.surfaces.iter().any(|surface| { + surface + .content + .external_texture() + .is_some_and(|source| source.id() == *id) + }) + }); if self.skip_draws { // skip drawing this frame, we just recovered from a device lost event // and so likely do not have the textures anymore that are required for drawing @@ -697,13 +733,221 @@ impl DirectXRenderer { ) } + /// Composite caller-owned textures (embedded browser frames) into the + /// scene. + /// + /// CoreVideo surfaces are a macOS path and never reach here; on Windows a + /// `PaintSurface` is always an external texture. fn draw_surfaces(&mut self, surfaces: &[PaintSurface]) -> Result<()> { if surfaces.is_empty() { return Ok(()); } + for surface in surfaces { + let Some(source) = surface.content.external_texture() else { + continue; + }; + let mut result = Ok(()); + source.with_frame(&mut |frame| { + if !frame.is_well_formed() { + // A producer that mis-declares its geometry is a bug, but it + // must not become an out-of-bounds read. + log::error!( + "external texture {:?} offered a malformed frame ({:?}, stride {})", + source.id(), + frame.size, + frame.stride + ); + return; + } + match self.draw_external_frame(surface, source.id(), &frame) { + // Only an actual upload is acknowledged. Acknowledging a + // reuse would make `upload_count` report the compositing + // rate, and PERF-B04 reads that counter to prove a settled + // page stops uploading. + Ok(uploaded) => { + if uploaded { + source.mark_uploaded(frame.sequence); + } + } + Err(error) => result = Err(error), + } + }); + result?; + } Ok(()) } + /// Upload what changed and draw one external texture. + /// + /// Returns whether pixels were actually sent to the GPU, so the caller can + /// acknowledge an upload and not a reuse. + fn draw_external_frame( + &mut self, + surface: &PaintSurface, + id: ExternalTextureId, + frame: &ExternalFrameView<'_>, + ) -> Result { + let width = frame.size.width.0.max(0) as u32; + let height = frame.size.height.0.max(0) as u32; + if width == 0 || height == 0 { + return Ok(false); + } + + let plan = plan_external_texture_update( + self.external_textures.get(&id).map(|cached| cached.key), + frame, + ); + + { + let devices = self.devices.as_ref().context("devices missing")?; + if matches!(plan, ExternalTextureUpdate::Recreate) { + let format = match frame.format { + ExternalTextureFormat::Bgra8 => DXGI_FORMAT_B8G8R8A8_UNORM, + ExternalTextureFormat::Rgba8 => DXGI_FORMAT_R8G8B8A8_UNORM, + }; + let desc = D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + Format: format, + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + let mut texture = None; + // SAFETY: desc is initialized, dimensions were validated, and the + // output slot lives through the COM call; no initial data is read. + unsafe { + devices + .device + .CreateTexture2D(&desc, None, Some(&mut texture)) + } + .context("Creating an external browser texture")?; + let texture = texture.context("CreateTexture2D returned nothing")?; + let mut view = None; + // SAFETY: texture is a live COM resource with shader-resource + // binding enabled; the output slot is valid for this call. + unsafe { + devices + .device + .CreateShaderResourceView(&texture, None, Some(&mut view)) + } + .context("Creating an external browser texture view")?; + self.external_textures.insert( + id, + CachedExternalTexture { + texture, + view, + key: ExternalTextureCacheKey { + id, + generation: frame.generation, + sequence: 0, + size: frame.size, + }, + }, + ); + } + + if !matches!(plan, ExternalTextureUpdate::Reuse) { + let cached = self + .external_textures + .get_mut(&id) + .context("external texture went missing between create and upload")?; + // A recreated texture has no pixels yet, so it takes the whole + // frame regardless of what the producer reported as dirty. + let regions = if matches!(plan, ExternalTextureUpdate::Recreate) { + vec![Bounds { + origin: point(DevicePixels(0), DevicePixels(0)), + size: frame.size, + }] + } else { + frame.dirty_regions() + }; + for region in regions { + let x = region.origin.x.0.max(0) as u32; + let y = region.origin.y.0.max(0) as u32; + let region_width = region.size.width.0.max(0) as u32; + let region_height = region.size.height.0.max(0) as u32; + if region_width == 0 || region_height == 0 { + continue; + } + let offset = y as usize * frame.stride + x as usize * 4; + let last_row = (y + region_height - 1) as usize * frame.stride + + (x + region_width) as usize * 4; + if last_row > frame.bytes.len() { + continue; + } + let destination = D3D11_BOX { + left: x, + top: y, + front: 0, + right: x + region_width, + bottom: y + region_height, + back: 1, + }; + // SAFETY: `destination` is clipped to the texture by + // `dirty_regions`, and the bounds check above proves the + // source rows for that box are inside `frame.bytes`, which + // stays alive for this call because the producer holds its + // lock for the duration of `with_frame`. + unsafe { + devices.device_context.UpdateSubresource( + &cached.texture, + 0, + Some(&destination), + frame.bytes[offset..].as_ptr() as *const _, + frame.stride as u32, + 0, + ); + } + } + cached.key = ExternalTextureCacheKey { + id, + generation: frame.generation, + sequence: frame.sequence, + size: frame.size, + }; + } + } + + let instance = ExternalTextureInstance { + bounds: surface.bounds, + content_mask: surface.content_mask.bounds, + opacity: 1.0, + pad: 0, + }; + let devices = self.devices.as_ref().context("devices missing")?; + let resources = self.resources.as_ref().context("resources missing")?; + let view = self + .external_textures + .get(&id) + .context("external texture missing at draw time")? + .view + .clone(); + self.pipelines.external_textures.update_buffer( + &devices.device, + &devices.device_context, + slice::from_ref(&instance), + )?; + self.pipelines.external_textures.draw_range_with_texture( + &devices.device, + &devices.device_context, + slice::from_ref(&view), + slice::from_ref(&resources.viewport), + slice::from_ref(&self.globals.global_params_buffer), + slice::from_ref(&self.globals.sampler), + 0, + 1, + )?; + Ok(!matches!(plan, ExternalTextureUpdate::Reuse)) + } + pub(crate) fn gpu_specs(&self) -> Result { let devices = self.devices.as_ref().context("devices missing")?; let desc = unsafe { devices.adapter.GetDesc1() }?; @@ -881,6 +1125,15 @@ impl DirectXRenderPipelines { 16, create_blend_state(device)?, )?; + // One instance per visible browser pane; the 8-pane cap (PRD) is the + // real bound, so this starts small and grows if it ever needs to. + let external_textures = PipelineState::new( + device, + "external_texture_pipeline", + ShaderModule::ExternalTexture, + 8, + create_blend_state(device)?, + )?; Ok(Self { shadow_pipeline, @@ -891,6 +1144,7 @@ impl DirectXRenderPipelines { mono_sprites, subpixel_sprites, poly_sprites, + external_textures, }) } } @@ -1603,6 +1857,7 @@ pub(crate) mod shader_resources { MonochromeSprite, SubpixelSprite, PolychromeSprite, + ExternalTexture, EmojiRasterization, } @@ -1677,6 +1932,10 @@ pub(crate) mod shader_resources { ShaderTarget::Vertex => POLYCHROME_SPRITE_VERTEX_BYTES, ShaderTarget::Fragment => POLYCHROME_SPRITE_FRAGMENT_BYTES, }, + ShaderModule::ExternalTexture => match target { + ShaderTarget::Vertex => EXTERNAL_TEXTURE_VERTEX_BYTES, + ShaderTarget::Fragment => EXTERNAL_TEXTURE_FRAGMENT_BYTES, + }, ShaderModule::EmojiRasterization => match target { ShaderTarget::Vertex => EMOJI_RASTERIZATION_VERTEX_BYTES, ShaderTarget::Fragment => EMOJI_RASTERIZATION_FRAGMENT_BYTES, @@ -1767,6 +2026,7 @@ pub(crate) mod shader_resources { ShaderModule::MonochromeSprite => "monochrome_sprite", ShaderModule::SubpixelSprite => "subpixel_sprite", ShaderModule::PolychromeSprite => "polychrome_sprite", + ShaderModule::ExternalTexture => "external_texture", ShaderModule::EmojiRasterization => "emoji_rasterization", } } diff --git a/crates/gpui_windows/src/shaders.hlsl b/crates/gpui_windows/src/shaders.hlsl index 89c12489b24cf6..88329e90fca294 100644 --- a/crates/gpui_windows/src/shaders.hlsl +++ b/crates/gpui_windows/src/shaders.hlsl @@ -1256,3 +1256,58 @@ float4 polychrome_sprite_fragment(PolychromeSpriteFragmentInput input): SV_Targe color.a *= sprite.opacity * saturate(0.5 - distance); return color; } + +/* +** +** External textures +** +*/ +// A caller-owned RGBA/BGRA texture composited inside the scene, used by the +// embedded browser's off-screen Chromium frames. Unlike a polychrome sprite +// this samples a texture that belongs to one producer and holds exactly one +// frame, so a 1080p page never touches the shared sprite atlas. + +struct ExternalTexture { + Bounds bounds; + Bounds content_mask; + float opacity; + uint pad; +}; +StructuredBuffer external_textures: register(t1); + +struct ExternalTextureVertexOutput { + nointerpolation uint external_id: TEXCOORD0; + float4 position: SV_Position; + float2 texture_position: POSITION; + float4 clip_distance: SV_ClipDistance; +}; + +struct ExternalTextureFragmentInput { + nointerpolation uint external_id: TEXCOORD0; + float4 position: SV_Position; + float2 texture_position: POSITION; +}; + +ExternalTextureVertexOutput external_texture_vertex(uint vertex_id: SV_VertexID, uint external_id: SV_InstanceID) { + float2 unit_vertex = float2(float(vertex_id & 1u), 0.5 * float(vertex_id & 2u)); + ExternalTexture external = external_textures[external_id]; + float4 device_position = to_device_position(unit_vertex, external.bounds); + float4 clip_distance = distance_from_clip_rect(unit_vertex, external.bounds, + external.content_mask); + + ExternalTextureVertexOutput output; + output.position = device_position; + // The texture holds exactly this rect, so unit coordinates are the texture + // coordinates. No atlas tile arithmetic. + output.texture_position = unit_vertex; + output.external_id = external_id; + output.clip_distance = clip_distance; + return output; +} + +float4 external_texture_fragment(ExternalTextureFragmentInput input): SV_Target { + ExternalTexture external = external_textures[input.external_id]; + float4 color = t_sprite.Sample(s_sprite, input.texture_position); + color.a *= external.opacity; + return color; +}