Put every character on the cell grid, so selection agrees with what is drawn

claude.md

@@ -190,6 +190,12 @@ apart.
   enrichment on top. Copying is `IViewerWindow.SetClipboard` rather than a `ViewerActions` member,
   because a clipboard belongs to a toolkit the way a window does, and it is answered before the
   owner link: the text is already in this process.
+- Columns are cells of `CellGrid`, decided in the model rather than by any head's fonts: a wide
+  character (CJK, fullwidth, emoji) takes two, a combining mark or joiner none. Every head draws a
+  row as `CellGrid.Segments`, each at its column, rather than as one string - a row of plain text
+  is one segment at column 0 - so a character a fallback font draws at its own width moves nothing
+  after it, and the highlight, the hit test and the copy count the same cells. Selection ends snap
+  to cluster boundaries (`CellGrid.Snap`), so a wide character is taken whole or not at all.
 - An entry opens at its first change, not line 1: every path that changes what is being read goes
   through `ViewerSession.Open`, so none resets to row 0 on its own. The minimal view ("Changes
   only", `SessionState.Minimal`) is a second `DiffView` built with each entry - changes plus

native/include/deview.h

@@ -51,17 +51,38 @@ enum DeviewQueueFlags {
     DEVIEW_QUEUE_HEADER = 1 << 2
 };
 
+/*
+ * A run of a row's text and the cell column it starts at. A renderer draws a row as its segments,
+ * each at its column times the cell width, rather than as one string: a character the font draws
+ * wider or narrower than a cell - CJK in a fallback font, a combining mark - then moves nothing
+ * after it, and a column means the same thing to the highlight, the hit test and the copy. A row
+ * of plain text is one segment at column 0.
+ *
+ * textOffset and textLength are into DeviewScreen.strings, like every other text reference, and
+ * always inside the row's own text.
+ */
+typedef struct DeviewSegment {
+    int32_t textOffset;
+    int32_t textLength;
+    int32_t column;
+} DeviewSegment;
+
 typedef struct DeviewRow {
     int32_t kind;
     /* -1 when the row is filler or folded and has no line number. */
     int32_t lineNumber;
+    /* Flattened: a tab is already four spaces, so every character is drawn where it is counted. */
     int32_t textOffset;
     int32_t textLength;
 
+    /* segmentCount entries of DeviewScreen.segments from segmentOffset: how to draw the text. */
+    int32_t segmentOffset;
+    int32_t segmentCount;
+
     /*
-     * What of this row the reader has selected, in characters of the text above rather than in
-     * pixels: the managed side flattens tabs before it counts, so a column here multiplied by the
-     * cell width is where the highlight goes.
+     * What of this row the reader has selected, in cells of the grid the segments are drawn on
+     * rather than in pixels, so a column here multiplied by the cell width is where the highlight
+     * goes.
      *
      * selectLength is 0 on a row with nothing selected, which is every row of almost every frame.
      * The managed side has already resolved which side the drag is in and clipped the range to the
@@ -139,6 +160,9 @@ typedef struct DeviewScreen {
     const DeviewRow* rows;
     int32_t rowCount;
 
+    const DeviewSegment* segments;
+    int32_t segmentCount;
+
     const DeviewButton* buttons;
     int32_t buttonCount;
 
@@ -272,8 +296,12 @@ typedef struct DeviewInput {
  *    which between them are text selection. DeviewRow is a widened array element, so this is the
  *    same kind of bump 6 was. deview_set_clipboard is added beside them, because the selection is
  *    only worth having if it can be copied and each toolkit owns its own clipboard.
+ * 9: DeviewRow carries segments, drawn each at its cell column, and its text arrives flattened.
+ *    Drawing a row as one string let each renderer's fonts decide where a wide or combining
+ *    character went, while a selection counted cells, so the two disagreed past the first one.
+ *    DeviewRow is widened and DeviewScreen gains an array, the same kind of bump 6 and 8 were.
  */
-#define DEVIEW_VERSION 8
+#define DEVIEW_VERSION 9
 
 /*
  * The Swift implementation imports this header for the struct layouts, because Swift does not

native/src/deview.cpp

@@ -685,6 +685,47 @@ int GutterDigits(const DeviewScreen* screen)
     return digits;
 }
 
+/*
+ * A row's text, each segment at its cell column: see DeviewSegment. A row that is one segment is
+ * drawn as the whole row always was, through the text item that also lays the row out; that is a
+ * row of plain text, which is nearly all of them. Any other row puts its segments on the window's
+ * draw list at their columns, clipped to the table cell like the item would be, and keeps its
+ * place in the layout with an item as tall as a line.
+ */
+void RowText(const DeviewScreen* screen, const DeviewRow& row, ImVec2 textPos)
+{
+    if (row.segmentCount <= 1 ||
+        screen->segments == nullptr ||
+        row.segmentOffset < 0 ||
+        row.segmentOffset + row.segmentCount > screen->segmentCount)
+    {
+        Text(screen, row.textOffset, row.textLength);
+        return;
+    }
+
+    const float cell = ImGui::CalcTextSize("M").x;
+    ImDrawList* list = ImGui::GetWindowDrawList();
+    const ImU32 colour = ImGui::GetColorU32(ImGuiCol_Text);
+    for (int index = 0; index < row.segmentCount; index++)
+    {
+        const DeviewSegment& segment = screen->segments[row.segmentOffset + index];
+        const char* begin;
+        const char* end;
+        if (!Slice(screen, segment.textOffset, segment.textLength, &begin, &end))
+        {
+            continue;
+        }
+
+        list->AddText(
+            ImVec2(textPos.x + static_cast<float>(segment.column) * cell, textPos.y),
+            colour,
+            begin,
+            end);
+    }
+
+    ImGui::Dummy(ImVec2(0.0f, ImGui::GetTextLineHeight()));
+}
+
 void DrawRow(const DeviewScreen* screen, const DeviewPane& pane, int index, int column, int digits, PaneHit& hit)
 {
     /* Before the row count check, so a pane shorter than the body still reports where its rows
@@ -753,7 +794,7 @@ void DrawRow(const DeviewScreen* screen, const DeviewPane& pane, int index, int
     }
 
     ImGui::PushStyleColor(ImGuiCol_Text, RowColour(row.kind));
-    Text(screen, row.textOffset, row.textLength);
+    RowText(screen, row, textPos);
     ImGui::PopStyleColor();
 }
 

native/swift/Sources/Deview/Frame.swift

@@ -31,14 +31,24 @@ struct Frame: Equatable {
         var lineNumber: Int32 = -1
         var text = ""
 
-        /// What of `text` the reader has selected, in characters. Length 0 on a row with nothing
+        /// How to draw `text`: each segment at its cell column, so a character a font draws
+        /// wider or narrower than a cell moves nothing after it. One segment at column 0 for a
+        /// row of plain text.
+        var segments: [Segment] = []
+
+        /// What of `text` the reader has selected, in cells. Length 0 on a row with nothing
         /// selected, which is every row of almost every frame. The managed side has already
         /// resolved which side the drag is in and clipped the range to the visible slice, so this
         /// is a rectangle to fill rather than a range to work out.
         var selectStart: Int32 = 0
         var selectLength: Int32 = 0
     }
 
+    struct Segment: Equatable {
+        var text = ""
+        var column: Int32 = 0
+    }
+
     struct Pane: Equatable {
         var header = ""
         var rows: [Row] = []
@@ -137,11 +147,28 @@ struct Frame: Equatable {
             }
 
             let row = rows[offset]
+            var segments: [Segment] = []
+            if let source = screen.segments {
+                for index in 0 ..< Int(max(0, row.segmentCount)) {
+                    let at = Int(row.segmentOffset) + index
+                    guard at >= 0, at < Int(screen.segmentCount) else {
+                        continue
+                    }
+
+                    let segment = source[at]
+                    segments.append(
+                        Segment(
+                            text: string(screen, segment.textOffset, segment.textLength),
+                            column: segment.column))
+                }
+            }
+
             pane.rows.append(
                 Row(
                     kind: row.kind,
                     lineNumber: row.lineNumber,
                     text: string(screen, row.textOffset, row.textLength),
+                    segments: segments,
                     selectStart: row.selectStart,
                     selectLength: row.selectLength))
         }

native/swift/Sources/Deview/Renderer.swift

@@ -352,11 +352,23 @@ final class Renderer {
         }
 
         text(gutter, in: CGRect(x: bounds.minX, y: bounds.minY, width: width, height: bounds.height), Palette.dim, context)
-        text(
-            row.text,
-            in: CGRect(x: bounds.minX + width, y: bounds.minY, width: bounds.width - width, height: bounds.height),
-            Palette.foreground(row.kind),
-            context)
+
+        // Each segment at its column rather than the row as one line, so a character Core Text
+        // takes from a fallback font, at that font's width, moves nothing after it. A row of plain
+        // text is one segment at column 0, drawn exactly as the whole row was.
+        let segments = row.segments.isEmpty ? [Frame.Segment(text: row.text, column: 0)] : row.segments
+        for segment in segments {
+            let left = bounds.minX + width + CGFloat(segment.column) * cell.width
+            guard left < bounds.maxX else {
+                break
+            }
+
+            text(
+                segment.text,
+                in: CGRect(x: left, y: bounds.minY, width: bounds.maxX - left, height: bounds.height),
+                Palette.foreground(row.kind),
+                context)
+        }
     }
 
     /// The picture a pane is, one blank line under its rows — the same placement the other two

src/DiffEngineTray.Tests/DebugReportTests.Full.verified.txt

@@ -6,7 +6,7 @@ Tracking: True
 Deletes (1)
 -----------
 [1] Extra.verified.txt
-    File:               {Directory}\Extra.verified.txt (missing)
+    File:               {Directory}\Extra.verified.txt (exists)
     Group:              <null>
 
 Moves (1)

src/DiffEngineTray.Tests/DebugReportTests.cs

@@ -49,7 +49,11 @@ public async Task Full()
         // view is where the whole message is readable rather than the menu's "!"
         viewer.Queue.Add(new(@"c:\repo\failed.cs|7", "Failed.cs:7", "the file is locked"));
         await using var tracker = new RecordingTracker();
-        tracker.AddDelete(Path.Combine(directory, "Extra.verified.txt"));
+        // A file that exists, as a pending delete's is. The tracker's scan drops a delete whose file
+        // has gone, every two seconds, and on a slow runner that scan landed mid test
+        var extra = Path.Combine(directory, "Extra.verified.txt");
+        File.WriteAllText(extra, "");
+        tracker.AddDelete(extra);
         tracker.AddMove(
             received,
             verified,

src/DiffEngineViewer.Tests/CellGridTests.cs

@@ -0,0 +1,86 @@
+/// <summary>
+/// The character grid every head draws a row on and every selection counts in. What a character
+/// takes is decided here and not by any head's fonts, which is what keeps the highlight, the hit
+/// test and the copy on one run of text.
+/// </summary>
+public class CellGridTests
+{
+    [Test]
+    [Arguments("", 0)]
+    [Arguments("plain text", 10)]
+    // Latin with its extensions, Greek and Cyrillic: one cell each, like ASCII
+    [Arguments("Ωμέγα Привет żółw", 17)]
+    // Wide: two cells each
+    [Arguments("中文", 4)]
+    [Arguments("ｆｕｌｌ", 8)]
+    [Arguments("한국어", 6)]
+    // A combining mark takes no cell of its own
+    [Arguments("é", 1)]
+    [Arguments("é̂x", 2)]
+    // Outside the basic plane: a surrogate pair is one character
+    [Arguments("\U0001D400", 1)]
+    // Emoji are wide, and a joined sequence is one picture
+    [Arguments("\U0001F600", 2)]
+    [Arguments("\U0001F468‍\U0001F469‍\U0001F467", 2)]
+    // A mark with nothing before it still takes a cell, or it could not be selected
+    [Arguments("́", 1)]
+    public async Task Cells(string text, int cells) =>
+        await Assert.That(CellGrid.Cells(text)).IsEqualTo(cells);
+
+    [Test]
+    public async Task PlainTextIsOneSegmentAtColumnZero() =>
+        await Assert.That(CellGrid.Segments("the quick brown fox")).IsEquivalentTo([new CellGrid.Segment(0, 19, 0)]);
+
+    [Test]
+    public async Task EmptyTextHasNoSegments() =>
+        await Assert.That(CellGrid.Segments("")).IsEmpty();
+
+    /// <summary>
+    /// Text the monospace font has is drawn in runs; anything else on its own at its column, so the
+    /// characters after it land where the grid says whatever width it is drawn at.
+    /// </summary>
+    [Test]
+    public async Task EveryOtherClusterIsASegmentOfItsOwn() =>
+        await Assert.That(CellGrid.Segments("ab中ćd"))
+            .IsEquivalentTo(
+            [
+                new CellGrid.Segment(0, 2, 0),
+                new CellGrid.Segment(2, 1, 2),
+                new CellGrid.Segment(3, 2, 4),
+                new CellGrid.Segment(5, 1, 5)
+            ]);
+
+    [Test]
+    public async Task CyrillicIsARunLikeAscii() =>
+        await Assert.That(CellGrid.Segments("Привет, мир")).IsEquivalentTo([new CellGrid.Segment(0, 11, 0)]);
+
+    /// <summary>
+    /// A column inside a wide character moves to its end, so a selection takes it whole or not at
+    /// all, and the index it maps to is after the whole character.
+    /// </summary>
+    [Test]
+    [Arguments(0, 0, 0)]
+    [Arguments(1, 1, 1)]
+    [Arguments(2, 3, 2)]
+    [Arguments(3, 3, 2)]
+    [Arguments(4, 4, 3)]
+    [Arguments(9, 4, 3)]
+    public async Task SnapsToWholeCharacters(int cell, int snapped, int index)
+    {
+        // a is cell 0, 中 cells 1 and 2, b cell 3
+        const string text = "a中b";
+
+        await Assert.That(CellGrid.Snap(text, cell)).IsEqualTo(snapped);
+        await Assert.That(CellGrid.Index(text, cell)).IsEqualTo(index);
+    }
+
+    [Test]
+    public async Task NeverSplitsACharacterFromItsMarks()
+    {
+        // e and its two marks are cell 0, x cell 1
+        const string text = "é̂x";
+
+        await Assert.That(CellGrid.Index(text, 1)).IsEqualTo(3);
+        await Assert.That(CellGrid.Index(text, 2)).IsEqualTo(4);
+    }
+}

src/DiffEngineViewer.Tests/DeviewStructTests.cs

@@ -58,6 +58,7 @@ public async Task KeysMatchTheHeader() =>
 
     public static IEnumerable<(string, Type)> Structs()
     {
+        yield return ("DeviewSegment", typeof(DeviewSegment));
         yield return ("DeviewRow", typeof(DeviewRow));
         yield return ("DeviewPane", typeof(DeviewPane));
         yield return ("DeviewButton", typeof(DeviewButton));