From 18910b408f815c66dc21d643ab4275d3a402bcdf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 6 Sep 2026 14:58:57 +0000 Subject: [PATCH 1/2] Add GPT-6 tiktoken model mapping Co-authored-by: stephentoub <2642209+stephentoub@users.noreply.github.com> --- src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs | 2 ++ test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs b/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs index 69f2af814f..0a57805393 100644 --- a/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs +++ b/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs @@ -1044,6 +1044,7 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo ( "o4-mini-", ModelEncoding.O200kBase ), // e.g. o4-mini // chat + ( "gpt-6-", ModelEncoding.O200kBase ), ( "gpt-5.6-", ModelEncoding.O200kBase ), ( "gpt-5.5-", ModelEncoding.O200kBase ), ( "gpt-5.4-", ModelEncoding.O200kBase ), @@ -1077,6 +1078,7 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo { "o4-mini", ModelEncoding.O200kBase }, // chat + { "gpt-6", ModelEncoding.O200kBase }, { "gpt-5.6", ModelEncoding.O200kBase }, { "gpt-5.5", ModelEncoding.O200kBase }, { "gpt-5.4", ModelEncoding.O200kBase }, diff --git a/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs b/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs index 7af5732509..00fce53ed2 100644 --- a/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs +++ b/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs @@ -42,6 +42,7 @@ public class TiktokenTests public static Tokenizer GPT5_4 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.4"); public static Tokenizer GPT5_5 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.5"); public static Tokenizer GPT5_6 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.6"); + public static Tokenizer GPT6 { get; } = TiktokenTokenizer.CreateForModel("gpt-6"); public static Tokenizer Phi4 { get; } = TiktokenTokenizer.CreateForModel("phi-4"); public static TiktokenTokenizer GptOss { get; } = TiktokenTokenizer.CreateForModel("gpt-oss-20b"); @@ -292,7 +293,7 @@ public void TestEncode5() [Fact] public void TestEncodeO200kBaseEncoding() { - foreach (TiktokenTokenizer tokenizer in new[] { GPT4o, GptOss, GPT5, GPT5_1, GPT5_2 }) + foreach (TiktokenTokenizer tokenizer in new[] { GPT4o, GptOss, GPT5, GPT5_1, GPT5_2, GPT6 }) { string text = ReadAndSanitizeFile("./Data/lib.rs.txt"); IReadOnlyList encoded = tokenizer.EncodeToIds(text); @@ -433,6 +434,8 @@ public void TestEncodeR50kBase() [InlineData("gpt-5.5-mini")] [InlineData("gpt-5.6")] [InlineData("gpt-5.6-nano")] + [InlineData("gpt-6")] + [InlineData("gpt-6-mini")] [InlineData("chatgpt-4o-")] [InlineData("gpt-4")] [InlineData("gpt-4-")] @@ -557,6 +560,7 @@ public void TestEncodingNamesNegativeCases() [InlineData("gpt-5.4")] [InlineData("gpt-5.5")] [InlineData("gpt-5.6")] + [InlineData("gpt-6")] [InlineData("o1")] [InlineData("o3")] [InlineData("o4-mini")] From 3149fa7fb7f8280c901309c103880596fa914834 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 6 Sep 2026 15:11:13 +0000 Subject: [PATCH 2/2] Test GPT-6 Astra model name Co-authored-by: stephentoub <2642209+stephentoub@users.noreply.github.com> --- test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs b/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs index 00fce53ed2..535b01b3f1 100644 --- a/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs +++ b/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs @@ -435,7 +435,7 @@ public void TestEncodeR50kBase() [InlineData("gpt-5.6")] [InlineData("gpt-5.6-nano")] [InlineData("gpt-6")] - [InlineData("gpt-6-mini")] + [InlineData("gpt-6-astra")] [InlineData("chatgpt-4o-")] [InlineData("gpt-4")] [InlineData("gpt-4-")]