Read and write batch and checkpoint files as UTF-8

changelog.md

@@ -5,6 +5,7 @@ Bug Fixes
 --------
 * Open the audit log and the `\T`/`\o` output files as UTF-8, so non-ASCII output no longer fails under non-UTF-8 locales.
 * Support `DELIMITER` in batch input (`mycli < file.sql`, `--batch`), so dumps with triggers or routines restore.
+* Read `--batch` files and read and write `--checkpoint` files as UTF-8, fixing `--resume` and non-ASCII batch files on Windows.
 
 
 v2.30.4 (2026/11/12)

mycli/AUTHORS.txt

@@ -124,6 +124,7 @@ Contributors:
   * Shaurya Kesarwani
   * Alex Castillo
   * Jay Hemnani
+  * Alpenj
 
 
 Created by:

mycli/client_query.py

@@ -97,7 +97,7 @@ def run_query(
         assert self.sql_execute is not None
         self.log_query(query)
         if checkpoint and not self.checkpoint:
-            self.checkpoint = click.open_file(checkpoint, mode='a')
+            self.checkpoint = click.open_file(checkpoint, mode='a', encoding='utf-8')
         results = self.sql_execute.run(query)
         for result in results:
             self.main_formatter.query = query

mycli/main_modes/batch.py

@@ -25,6 +25,11 @@ class CheckpointReplayError(Exception):
     pass
 
 
+def batch_file_encoding(batch_path: str) -> str | None:
+    # Leave standard input to click, which picks the console encoding.
+    return None if batch_path == '-' else 'utf-8'
+
+
 def replay_checkpoint_file(
     batch_path: str,
     checkpoint_path: str | None,
@@ -48,7 +53,10 @@ def replay_checkpoint_file(
         spinner = yaspin(text='replaying checkpoint', side='right', stream=sys.stderr)
         spinner.start()
     try:
-        with click.open_file(batch_path) as batch_h, click.open_file(checkpoint_path, mode='r', encoding='utf-8') as checkpoint_h:
+        with (
+            click.open_file(batch_path, encoding=batch_file_encoding(batch_path)) as batch_h,
+            click.open_file(checkpoint_path, mode='r', encoding='utf-8') as checkpoint_h,
+        ):
             try:
                 batch_gen = statements_from_filehandle(batch_h)
             except ValueError as e:
@@ -140,13 +148,13 @@ def main_batch_with_progress_bar(mycli: 'MyCli', cli_args: 'CliArgs') -> int:
         return 1
     try:
         completed_statement_count = replay_checkpoint_file(cli_args.batch, cli_args.checkpoint, cli_args.resume, progress=True)
-        batch_count_h = click.open_file(cli_args.batch)
+        batch_count_h = click.open_file(cli_args.batch, encoding=batch_file_encoding(cli_args.batch))
         with yaspin(text='validating batch    ', side='right', stream=sys.stderr) as spinner:
             for _statement, _counter in statements_from_filehandle(batch_count_h):
                 goal_statements += 1
             spinner.ok('✔')
         batch_count_h.close()
-        batch_h = click.open_file(cli_args.batch)
+        batch_h = click.open_file(cli_args.batch, encoding=batch_file_encoding(cli_args.batch))
         batch_gen = statements_from_filehandle(batch_h)
     except (OSError, FileNotFoundError):
         click.secho(f'Failed to open --batch file: {cli_args.batch}', err=True, fg='red')
@@ -192,7 +200,7 @@ def main_batch_without_progress_bar(mycli: 'MyCli', cli_args: 'CliArgs') -> int:
         click.secho('Ignoring STDIN since --batch was also given.', err=True, fg='red')
     try:
         completed_statement_count = replay_checkpoint_file(cli_args.batch, cli_args.checkpoint, cli_args.resume)
-        batch_h = click.open_file(cli_args.batch)
+        batch_h = click.open_file(cli_args.batch, encoding=batch_file_encoding(cli_args.batch))
     except (OSError, FileNotFoundError):
         click.secho(f'Failed to open --batch file: {cli_args.batch}', err=True, fg='red')
         return 1

mycli_test/pytests/test_client_query.py

@@ -225,7 +225,7 @@ def test_on_completions_refreshed_starts_schema_prefetch() -> None:
     assert state['prefetch_started'] == [True]
 
 
-def run_query_with_state(monkeypatch, tmp_path, *, warnings_enabled: bool = True) -> dict[str, Any]:
+def run_query_with_state(monkeypatch, tmp_path, *, warnings_enabled: bool = True, query: str = 'select 1;\n') -> dict[str, Any]:
     cli = make_bare_mycli()
     normal_rows = FakeCursorBase(rows=[('one',)], warning_count=1)
     warning_rows = FakeCursorBase(rows=[('warning',)], warning_count=0)
@@ -258,7 +258,7 @@ def format_sqlresult(result: SQLResult, **kwargs: Any) -> list[str]:
     cli.log_query = lambda query: state['logged_queries'].append(query)
     cli.log_output = lambda line: state['logged_output'].append(line)
     cli.format_sqlresult = format_sqlresult
-    mycli_client.MyCli.run_query(cli, 'select 1;\n', checkpoint=str(state['checkpoint_path']), new_line=False)
+    mycli_client.MyCli.run_query(cli, query, checkpoint=str(state['checkpoint_path']), new_line=False)
     state['cli'] = cli
     return state
 
@@ -336,6 +336,12 @@ def test_run_query_writes_checkpoint(monkeypatch, tmp_path) -> None:
     assert state['checkpoint_path'].read_text(encoding='utf-8') == 'select 1;\n'
 
 
+def test_run_query_writes_checkpoint_as_utf8(monkeypatch, tmp_path) -> None:
+    state = run_query_with_state(monkeypatch, tmp_path, warnings_enabled=False, query="select 'Łukasz ✨';\n")
+
+    assert state['checkpoint_path'].read_text(encoding='utf-8') == "select 'Łukasz ✨';\n"
+
+
 def test_run_query_raises_for_error_result_when_requested(tmp_path) -> None:
     cli = make_bare_mycli()
     logged_output: list[str] = []

mycli_test/pytests/test_main_modes_batch.py

@@ -513,7 +513,7 @@ def test_main_batch_with_progress_bar_handles_open_errors(monkeypatch) -> None:
     cli_args = DummyCliArgs(batch='missing.sql')
 
     monkeypatch.setattr(batch_mode.os.path, 'exists', lambda _path: False)
-    monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path: (_ for _ in ()).throw(FileNotFoundError()))
+    monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path, **_kwargs: (_ for _ in ()).throw(FileNotFoundError()))
     monkeypatch.setattr(batch_mode.click, 'secho', lambda message, err, fg: messages.append((message, err, fg)))
     monkeypatch.setattr(batch_mode, 'sys', make_fake_sys(stdin_tty=True))
 
@@ -529,7 +529,7 @@ def test_main_batch_with_progress_bar_handles_counting_value_errors(monkeypatch)
     cli_args = DummyCliArgs(batch='statements.sql')
 
     monkeypatch.setattr(batch_mode.os.path, 'exists', lambda _path: False)
-    monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path: count_handle)
+    monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path, **_kwargs: count_handle)
     monkeypatch.setattr(batch_mode, 'statements_from_filehandle', lambda _handle: (_ for _ in ()).throw(ValueError('bad sql')))
     monkeypatch.setattr(batch_mode.click, 'secho', lambda message, err, fg: messages.append((message, err, fg)))
     monkeypatch.setattr(batch_mode, 'sys', make_fake_sys(stdin_tty=True))
@@ -548,7 +548,7 @@ def test_main_batch_with_progress_bar_processes_all_statements(monkeypatch) -> N
     dispatch_calls: list[tuple[str, int]] = []
     cli_args = DummyCliArgs(batch='statements.sql')
 
-    def fake_open_file(path: str) -> DummyFile:
+    def fake_open_file(path: str, **_kwargs: Any) -> DummyFile:
         open_calls.append(path)
         return count_handle if len(open_calls) == 1 else run_handle
 
@@ -588,7 +588,7 @@ def test_main_batch_with_progress_bar_returns_error_when_dispatch_fails(monkeypa
     open_calls = 0
     cli_args = DummyCliArgs(batch='statements.sql')
 
-    def fake_open_file(_path: str) -> DummyFile:
+    def fake_open_file(_path: str, **_kwargs: Any) -> DummyFile:
         nonlocal open_calls
         open_calls += 1
         return count_handle if open_calls == 1 else run_handle
@@ -626,7 +626,7 @@ def test_main_batch_without_progress_bar_handles_open_errors(monkeypatch) -> Non
     messages: list[tuple[str, bool, str]] = []
     cli_args = DummyCliArgs(batch='missing.sql')
 
-    monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path: (_ for _ in ()).throw(FileNotFoundError()))
+    monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path, **_kwargs: (_ for _ in ()).throw(FileNotFoundError()))
     monkeypatch.setattr(batch_mode.click, 'secho', lambda message, err, fg: messages.append((message, err, fg)))
     monkeypatch.setattr(batch_mode, 'sys', make_fake_sys(stdin_tty=True))
 
@@ -642,7 +642,7 @@ def test_main_batch_without_progress_bar_processes_statements(monkeypatch) -> No
     dispatch_calls: list[tuple[str, int]] = []
     cli_args = DummyCliArgs(batch='statements.sql')
 
-    monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path: batch_handle)
+    monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path, **_kwargs: batch_handle)
     monkeypatch.setattr(batch_mode, 'statements_from_filehandle', lambda _handle: iter([('select 1;', 0), ('select 2;', 1)]))
     monkeypatch.setattr(
         batch_mode,
@@ -680,6 +680,26 @@ def test_main_batch_without_progress_bar_skips_checkpoint_prefix(monkeypatch, tm
     assert dispatch_calls == [('select 3;', 2)]
 
 
+def test_main_batch_without_progress_bar_reads_utf8_batch_and_checkpoint(monkeypatch, tmp_path: Path) -> None:
+    batch_path = write_batch_file(tmp_path, "select 'Łukasz ✨';\nselect 2;\n")
+    dispatch_calls: list[tuple[str, int]] = []
+
+    monkeypatch.setattr(
+        batch_mode,
+        'dispatch_batch_statements',
+        lambda _mycli, _cli_args, statement, counter: dispatch_calls.append((statement, counter)),
+    )
+    monkeypatch.setattr(batch_mode, 'sys', make_fake_sys(stdin_tty=True))
+
+    checkpoint = write_checkpoint_file(tmp_path, "select 'Łukasz ✨';\n")
+
+    cli_args = DummyCliArgs(batch=batch_path, checkpoint=checkpoint, resume=True)
+    result = main_batch_without_progress_bar(DummyMyCli(), cli_args)
+
+    assert result == 0
+    assert dispatch_calls == [('select 2;', 1)]
+
+
 def test_main_batch_without_progress_bar_skips_only_matching_duplicate_prefix(monkeypatch, tmp_path: Path) -> None:
     batch_path = write_batch_file(tmp_path, 'select 1;\nselect 1;\nselect 2;\n')
     dispatch_calls: list[tuple[str, int]] = []
@@ -764,6 +784,26 @@ def test_main_batch_with_progress_bar_skips_checkpoint_prefix_and_counts_all_sta
     assert DummyProgressBar.calls == [[0, 1, 2]]
 
 
+def test_main_batch_with_progress_bar_reads_utf8_batch(monkeypatch, tmp_path: Path) -> None:
+    batch_path = write_batch_file(tmp_path, "select 'Łukasz ✨';\nselect 2;\n")
+    dispatch_calls: list[tuple[str, int]] = []
+
+    DummyProgressBar.calls.clear()
+    monkeypatch.setattr(batch_mode, 'ProgressBar', DummyProgressBar)
+    monkeypatch.setattr(batch_mode.prompt_toolkit.output, 'create_output', lambda **_kwargs: object())
+    monkeypatch.setattr(
+        batch_mode,
+        'dispatch_batch_statements',
+        lambda _mycli, _cli_args, statement, counter: dispatch_calls.append((statement, counter)),
+    )
+    monkeypatch.setattr(batch_mode, 'sys', make_fake_sys(stdin_tty=True))
+
+    result = main_batch_with_progress_bar(DummyMyCli(), DummyCliArgs(batch=batch_path))
+
+    assert result == 0
+    assert dispatch_calls == [("select 'Łukasz ✨';", 0), ('select 2;', 1)]
+
+
 def test_main_batch_with_progress_bar_returns_error_when_checkpoint_replay_fails(monkeypatch, tmp_path: Path) -> None:
     batch_path = write_batch_file(tmp_path, 'select 1;\n')
     messages: list[tuple[str, bool, str]] = []
@@ -785,7 +825,7 @@ def test_main_batch_without_progress_bar_returns_error_when_iteration_fails(monk
     batch_handle = DummyFile('run')
     cli_args = DummyCliArgs(batch='statements.sql')
 
-    monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path: batch_handle)
+    monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path, **_kwargs: batch_handle)
     monkeypatch.setattr(batch_mode, 'statements_from_filehandle', lambda _handle: (_ for _ in ()).throw(ValueError('bad sql')))
     monkeypatch.setattr(batch_mode.click, 'secho', lambda message, err, fg: messages.append((message, err, fg)))
     monkeypatch.setattr(batch_mode, 'sys', make_fake_sys(stdin_tty=True))