From 3dbecf9cf92be119ff44a0597510618a488fe0e1 Mon Sep 17 00:00:00 2001 From: Alpenj Date: Sat, 14 Nov 2026 14:44:39 +0900 Subject: [PATCH] Use UTF-8 for batch and checkpoint files --batch files were read with the locale encoding, and the --checkpoint file was written with the locale encoding but read back as UTF-8. On Windows, non-ASCII batch files failed to load and --resume could not read the checkpoint. Read batch files as UTF-8 (standard input is unchanged) and write the checkpoint as UTF-8. --- changelog.md | 1 + mycli/AUTHORS.txt | 1 + mycli/client_query.py | 2 +- mycli/main_modes/batch.py | 16 ++++-- mycli_test/pytests/test_client_query.py | 10 +++- mycli_test/pytests/test_main_modes_batch.py | 54 ++++++++++++++++++--- 6 files changed, 70 insertions(+), 14 deletions(-) diff --git a/changelog.md b/changelog.md index a34c52fa4..10af49fae 100644 --- a/changelog.md +++ b/changelog.md @@ -5,6 +5,7 @@ Bug Fixes -------- * Open the audit log and the `\T`/`\o` output files as UTF-8, so non-ASCII output no longer fails under non-UTF-8 locales. * Support `DELIMITER` in batch input (`mycli < file.sql`, `--batch`), so dumps with triggers or routines restore. +* Read `--batch` files and read and write `--checkpoint` files as UTF-8, fixing `--resume` and non-ASCII batch files on Windows. v2.30.4 (2026/11/12) diff --git a/mycli/AUTHORS.txt b/mycli/AUTHORS.txt index 1172a4ebd..731d659bd 100644 --- a/mycli/AUTHORS.txt +++ b/mycli/AUTHORS.txt @@ -124,6 +124,7 @@ Contributors: * Shaurya Kesarwani * Alex Castillo * Jay Hemnani + * Alpenj Created by: diff --git a/mycli/client_query.py b/mycli/client_query.py index 254d539a4..6838653cf 100644 --- a/mycli/client_query.py +++ b/mycli/client_query.py @@ -97,7 +97,7 @@ def run_query( assert self.sql_execute is not None self.log_query(query) if checkpoint and not self.checkpoint: - self.checkpoint = click.open_file(checkpoint, mode='a') + self.checkpoint = click.open_file(checkpoint, mode='a', encoding='utf-8') results = self.sql_execute.run(query) for result in results: self.main_formatter.query = query diff --git a/mycli/main_modes/batch.py b/mycli/main_modes/batch.py index bcb8aacca..344515cf2 100644 --- a/mycli/main_modes/batch.py +++ b/mycli/main_modes/batch.py @@ -25,6 +25,11 @@ class CheckpointReplayError(Exception): pass +def batch_file_encoding(batch_path: str) -> str | None: + # Leave standard input to click, which picks the console encoding. + return None if batch_path == '-' else 'utf-8' + + def replay_checkpoint_file( batch_path: str, checkpoint_path: str | None, @@ -48,7 +53,10 @@ def replay_checkpoint_file( spinner = yaspin(text='replaying checkpoint', side='right', stream=sys.stderr) spinner.start() try: - with click.open_file(batch_path) as batch_h, click.open_file(checkpoint_path, mode='r', encoding='utf-8') as checkpoint_h: + with ( + click.open_file(batch_path, encoding=batch_file_encoding(batch_path)) as batch_h, + click.open_file(checkpoint_path, mode='r', encoding='utf-8') as checkpoint_h, + ): try: batch_gen = statements_from_filehandle(batch_h) except ValueError as e: @@ -140,13 +148,13 @@ def main_batch_with_progress_bar(mycli: 'MyCli', cli_args: 'CliArgs') -> int: return 1 try: completed_statement_count = replay_checkpoint_file(cli_args.batch, cli_args.checkpoint, cli_args.resume, progress=True) - batch_count_h = click.open_file(cli_args.batch) + batch_count_h = click.open_file(cli_args.batch, encoding=batch_file_encoding(cli_args.batch)) with yaspin(text='validating batch ', side='right', stream=sys.stderr) as spinner: for _statement, _counter in statements_from_filehandle(batch_count_h): goal_statements += 1 spinner.ok('✔') batch_count_h.close() - batch_h = click.open_file(cli_args.batch) + batch_h = click.open_file(cli_args.batch, encoding=batch_file_encoding(cli_args.batch)) batch_gen = statements_from_filehandle(batch_h) except (OSError, FileNotFoundError): click.secho(f'Failed to open --batch file: {cli_args.batch}', err=True, fg='red') @@ -192,7 +200,7 @@ def main_batch_without_progress_bar(mycli: 'MyCli', cli_args: 'CliArgs') -> int: click.secho('Ignoring STDIN since --batch was also given.', err=True, fg='red') try: completed_statement_count = replay_checkpoint_file(cli_args.batch, cli_args.checkpoint, cli_args.resume) - batch_h = click.open_file(cli_args.batch) + batch_h = click.open_file(cli_args.batch, encoding=batch_file_encoding(cli_args.batch)) except (OSError, FileNotFoundError): click.secho(f'Failed to open --batch file: {cli_args.batch}', err=True, fg='red') return 1 diff --git a/mycli_test/pytests/test_client_query.py b/mycli_test/pytests/test_client_query.py index fdd307ac5..35fc9ef9a 100644 --- a/mycli_test/pytests/test_client_query.py +++ b/mycli_test/pytests/test_client_query.py @@ -225,7 +225,7 @@ def test_on_completions_refreshed_starts_schema_prefetch() -> None: assert state['prefetch_started'] == [True] -def run_query_with_state(monkeypatch, tmp_path, *, warnings_enabled: bool = True) -> dict[str, Any]: +def run_query_with_state(monkeypatch, tmp_path, *, warnings_enabled: bool = True, query: str = 'select 1;\n') -> dict[str, Any]: cli = make_bare_mycli() normal_rows = FakeCursorBase(rows=[('one',)], warning_count=1) warning_rows = FakeCursorBase(rows=[('warning',)], warning_count=0) @@ -258,7 +258,7 @@ def format_sqlresult(result: SQLResult, **kwargs: Any) -> list[str]: cli.log_query = lambda query: state['logged_queries'].append(query) cli.log_output = lambda line: state['logged_output'].append(line) cli.format_sqlresult = format_sqlresult - mycli_client.MyCli.run_query(cli, 'select 1;\n', checkpoint=str(state['checkpoint_path']), new_line=False) + mycli_client.MyCli.run_query(cli, query, checkpoint=str(state['checkpoint_path']), new_line=False) state['cli'] = cli return state @@ -336,6 +336,12 @@ def test_run_query_writes_checkpoint(monkeypatch, tmp_path) -> None: assert state['checkpoint_path'].read_text(encoding='utf-8') == 'select 1;\n' +def test_run_query_writes_checkpoint_as_utf8(monkeypatch, tmp_path) -> None: + state = run_query_with_state(monkeypatch, tmp_path, warnings_enabled=False, query="select 'Łukasz ✨';\n") + + assert state['checkpoint_path'].read_text(encoding='utf-8') == "select 'Łukasz ✨';\n" + + def test_run_query_raises_for_error_result_when_requested(tmp_path) -> None: cli = make_bare_mycli() logged_output: list[str] = [] diff --git a/mycli_test/pytests/test_main_modes_batch.py b/mycli_test/pytests/test_main_modes_batch.py index f0665609f..8977c606e 100644 --- a/mycli_test/pytests/test_main_modes_batch.py +++ b/mycli_test/pytests/test_main_modes_batch.py @@ -513,7 +513,7 @@ def test_main_batch_with_progress_bar_handles_open_errors(monkeypatch) -> None: cli_args = DummyCliArgs(batch='missing.sql') monkeypatch.setattr(batch_mode.os.path, 'exists', lambda _path: False) - monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path: (_ for _ in ()).throw(FileNotFoundError())) + monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path, **_kwargs: (_ for _ in ()).throw(FileNotFoundError())) monkeypatch.setattr(batch_mode.click, 'secho', lambda message, err, fg: messages.append((message, err, fg))) monkeypatch.setattr(batch_mode, 'sys', make_fake_sys(stdin_tty=True)) @@ -529,7 +529,7 @@ def test_main_batch_with_progress_bar_handles_counting_value_errors(monkeypatch) cli_args = DummyCliArgs(batch='statements.sql') monkeypatch.setattr(batch_mode.os.path, 'exists', lambda _path: False) - monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path: count_handle) + monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path, **_kwargs: count_handle) monkeypatch.setattr(batch_mode, 'statements_from_filehandle', lambda _handle: (_ for _ in ()).throw(ValueError('bad sql'))) monkeypatch.setattr(batch_mode.click, 'secho', lambda message, err, fg: messages.append((message, err, fg))) monkeypatch.setattr(batch_mode, 'sys', make_fake_sys(stdin_tty=True)) @@ -548,7 +548,7 @@ def test_main_batch_with_progress_bar_processes_all_statements(monkeypatch) -> N dispatch_calls: list[tuple[str, int]] = [] cli_args = DummyCliArgs(batch='statements.sql') - def fake_open_file(path: str) -> DummyFile: + def fake_open_file(path: str, **_kwargs: Any) -> DummyFile: open_calls.append(path) return count_handle if len(open_calls) == 1 else run_handle @@ -588,7 +588,7 @@ def test_main_batch_with_progress_bar_returns_error_when_dispatch_fails(monkeypa open_calls = 0 cli_args = DummyCliArgs(batch='statements.sql') - def fake_open_file(_path: str) -> DummyFile: + def fake_open_file(_path: str, **_kwargs: Any) -> DummyFile: nonlocal open_calls open_calls += 1 return count_handle if open_calls == 1 else run_handle @@ -626,7 +626,7 @@ def test_main_batch_without_progress_bar_handles_open_errors(monkeypatch) -> Non messages: list[tuple[str, bool, str]] = [] cli_args = DummyCliArgs(batch='missing.sql') - monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path: (_ for _ in ()).throw(FileNotFoundError())) + monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path, **_kwargs: (_ for _ in ()).throw(FileNotFoundError())) monkeypatch.setattr(batch_mode.click, 'secho', lambda message, err, fg: messages.append((message, err, fg))) monkeypatch.setattr(batch_mode, 'sys', make_fake_sys(stdin_tty=True)) @@ -642,7 +642,7 @@ def test_main_batch_without_progress_bar_processes_statements(monkeypatch) -> No dispatch_calls: list[tuple[str, int]] = [] cli_args = DummyCliArgs(batch='statements.sql') - monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path: batch_handle) + monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path, **_kwargs: batch_handle) monkeypatch.setattr(batch_mode, 'statements_from_filehandle', lambda _handle: iter([('select 1;', 0), ('select 2;', 1)])) monkeypatch.setattr( batch_mode, @@ -680,6 +680,26 @@ def test_main_batch_without_progress_bar_skips_checkpoint_prefix(monkeypatch, tm assert dispatch_calls == [('select 3;', 2)] +def test_main_batch_without_progress_bar_reads_utf8_batch_and_checkpoint(monkeypatch, tmp_path: Path) -> None: + batch_path = write_batch_file(tmp_path, "select 'Łukasz ✨';\nselect 2;\n") + dispatch_calls: list[tuple[str, int]] = [] + + monkeypatch.setattr( + batch_mode, + 'dispatch_batch_statements', + lambda _mycli, _cli_args, statement, counter: dispatch_calls.append((statement, counter)), + ) + monkeypatch.setattr(batch_mode, 'sys', make_fake_sys(stdin_tty=True)) + + checkpoint = write_checkpoint_file(tmp_path, "select 'Łukasz ✨';\n") + + cli_args = DummyCliArgs(batch=batch_path, checkpoint=checkpoint, resume=True) + result = main_batch_without_progress_bar(DummyMyCli(), cli_args) + + assert result == 0 + assert dispatch_calls == [('select 2;', 1)] + + def test_main_batch_without_progress_bar_skips_only_matching_duplicate_prefix(monkeypatch, tmp_path: Path) -> None: batch_path = write_batch_file(tmp_path, 'select 1;\nselect 1;\nselect 2;\n') dispatch_calls: list[tuple[str, int]] = [] @@ -764,6 +784,26 @@ def test_main_batch_with_progress_bar_skips_checkpoint_prefix_and_counts_all_sta assert DummyProgressBar.calls == [[0, 1, 2]] +def test_main_batch_with_progress_bar_reads_utf8_batch(monkeypatch, tmp_path: Path) -> None: + batch_path = write_batch_file(tmp_path, "select 'Łukasz ✨';\nselect 2;\n") + dispatch_calls: list[tuple[str, int]] = [] + + DummyProgressBar.calls.clear() + monkeypatch.setattr(batch_mode, 'ProgressBar', DummyProgressBar) + monkeypatch.setattr(batch_mode.prompt_toolkit.output, 'create_output', lambda **_kwargs: object()) + monkeypatch.setattr( + batch_mode, + 'dispatch_batch_statements', + lambda _mycli, _cli_args, statement, counter: dispatch_calls.append((statement, counter)), + ) + monkeypatch.setattr(batch_mode, 'sys', make_fake_sys(stdin_tty=True)) + + result = main_batch_with_progress_bar(DummyMyCli(), DummyCliArgs(batch=batch_path)) + + assert result == 0 + assert dispatch_calls == [("select 'Łukasz ✨';", 0), ('select 2;', 1)] + + def test_main_batch_with_progress_bar_returns_error_when_checkpoint_replay_fails(monkeypatch, tmp_path: Path) -> None: batch_path = write_batch_file(tmp_path, 'select 1;\n') messages: list[tuple[str, bool, str]] = [] @@ -785,7 +825,7 @@ def test_main_batch_without_progress_bar_returns_error_when_iteration_fails(monk batch_handle = DummyFile('run') cli_args = DummyCliArgs(batch='statements.sql') - monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path: batch_handle) + monkeypatch.setattr(batch_mode.click, 'open_file', lambda _path, **_kwargs: batch_handle) monkeypatch.setattr(batch_mode, 'statements_from_filehandle', lambda _handle: (_ for _ in ()).throw(ValueError('bad sql'))) monkeypatch.setattr(batch_mode.click, 'secho', lambda message, err, fg: messages.append((message, err, fg))) monkeypatch.setattr(batch_mode, 'sys', make_fake_sys(stdin_tty=True))