From 8c94643038a3e0d7dea0d7cae42929f8df515cf1 Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Fri, 13 Nov 2026 21:10:36 -0400 Subject: [PATCH] fix: read /source and editor files as UTF-8 /source and the \e editor file were opened with the locale default encoding, so UTF-8 SQL files were mis-decoded or rejected under non-UTF-8 locales (Windows cp1252/cp932, LANG=C). Pass encoding='utf-8', matching the config and history files. --- changelog.md | 1 + mycli/client_commands.py | 2 +- .../packages/special_commands/io_commands.py | 2 +- mycli_test/pytests/test_client_commands.py | 39 ++++++++++++++++--- .../pytests/test_special_io_commands.py | 23 +++++++++++ 5 files changed, 59 insertions(+), 8 deletions(-) diff --git a/changelog.md b/changelog.md index a34c52fa4..d8536af8e 100644 --- a/changelog.md +++ b/changelog.md @@ -3,6 +3,7 @@ Upcoming (TBD) Bug Fixes -------- +* Read `/source` files and the `\e` editor file as UTF-8 instead of the locale encoding, so non-ASCII SQL no longer fails or is mis-decoded under non-UTF-8 locales. * Open the audit log and the `\T`/`\o` output files as UTF-8, so non-ASCII output no longer fails under non-UTF-8 locales. * Support `DELIMITER` in batch input (`mycli < file.sql`, `--batch`), so dumps with triggers or routines restore. diff --git a/mycli/client_commands.py b/mycli/client_commands.py index 1a5089105..257e1b0f5 100644 --- a/mycli/client_commands.py +++ b/mycli/client_commands.py @@ -305,7 +305,7 @@ def execute_from_file(self, arg: str, **_) -> Generator[SQLResult, None, None]: return try: - file_h = open(os.path.expanduser(filename)) + file_h = open(os.path.expanduser(filename), encoding='utf-8') except OSError as error: yield SQLResult(status=str(error)) return diff --git a/mycli/packages/special_commands/io_commands.py b/mycli/packages/special_commands/io_commands.py index 0839797d1..1bb943c05 100644 --- a/mycli/packages/special_commands/io_commands.py +++ b/mycli/packages/special_commands/io_commands.py @@ -289,7 +289,7 @@ def open_external_editor(filename: str | None = None, sql: str | None = None) -> message = None click.edit(filename=filename) try: - with open(filename, 'r') as f: + with open(filename, 'r', encoding='utf-8') as f: query = f.read() except IOError: message = f'Error reading file: {filename}' diff --git a/mycli_test/pytests/test_client_commands.py b/mycli_test/pytests/test_client_commands.py index 99cb5a714..77e68eee5 100644 --- a/mycli_test/pytests/test_client_commands.py +++ b/mycli_test/pytests/test_client_commands.py @@ -4,6 +4,8 @@ from io import StringIO import logging from pathlib import Path +import subprocess +import sys from types import SimpleNamespace from typing import Any @@ -574,7 +576,7 @@ def test_execute_from_file_iterates_statements_without_reading_entire_file(monke file_h = IteratedFile('select 1;\nselect\n 2;\nselect 3; select 4;\nselect 5') opened_paths: list[str] = [] - def open_file(path: str) -> IteratedFile: + def open_file(path: str, **_: Any) -> IteratedFile: opened_paths.append(path) return file_h @@ -596,7 +598,7 @@ def test_execute_from_file_reports_read_errors(monkeypatch: pytest.MonkeyPatch) client = DummyClient() client.sql_execute = FakeSQLExecute() file_h = FailingFile() - monkeypatch.setattr(client_commands, 'open', lambda path: file_h, raising=False) + monkeypatch.setattr(client_commands, 'open', lambda path, **_: file_h, raising=False) assert list(client.execute_from_file('query.sql')) == [SQLResult(status='read failed')] assert file_h.closed is True @@ -655,7 +657,7 @@ def test_execute_from_file_help_is_terminal(monkeypatch: pytest.MonkeyPatch) -> client.sql_execute = FakeSQLExecute() opened_paths: list[str] = [] sleep_calls: list[float] = [] - monkeypatch.setattr(client_commands, 'open', lambda path: opened_paths.append(path), raising=False) + monkeypatch.setattr(client_commands, 'open', lambda path, **_: opened_paths.append(path), raising=False) monkeypatch.setattr(client_commands.time, 'sleep', lambda seconds: sleep_calls.append(seconds)) assert list(client.execute_from_file('--page --help ignored.sql')) == [ @@ -716,7 +718,7 @@ def test_execute_from_file_does_not_throttle_after_declined_statement(monkeypatc def test_execute_from_file_reports_invalid_throttle_without_opening_file(monkeypatch: pytest.MonkeyPatch) -> None: client = DummyClient() opened_paths: list[str] = [] - monkeypatch.setattr(client_commands, 'open', lambda path: opened_paths.append(path), raising=False) + monkeypatch.setattr(client_commands, 'open', lambda path, **_: opened_paths.append(path), raising=False) assert list(client.execute_from_file('--throttle nope query.sql')) == [ SQLResult( @@ -782,6 +784,31 @@ def test_execute_from_file_shows_each_query(capsys: pytest.CaptureFixture[str], assert capsys.readouterr().out == '> select 1;\n> select 2;\n' +def test_execute_from_file_does_not_use_locale_encoding(tmp_path: Path) -> None: + # SQL files are commonly UTF-8; the locale encoding (cp1252, cp932, ...) would + # mis-decode or reject them. EncodingWarning is raised for any open() that + # falls back to the locale encoding. + sql_file = tmp_path / 'query.sql' + sql_file.write_text('-- caf\u00e9\nselect 1;', encoding='utf-8') + script = '\n'.join(( + 'import sys', + 'from mycli.client_commands import ClientCommandsMixin', + 'class C(ClientCommandsMixin):', + ' pass', + 'try:', + ' list(C().execute_from_file(sys.argv[1]))', + 'except AttributeError:', + ' pass', + )) + result = subprocess.run( + [sys.executable, '-X', 'warn_default_encoding', '-W', 'error::EncodingWarning', '-c', script, str(sql_file)], + capture_output=True, + encoding='utf8', + errors='replace', + ) + assert 'EncodingWarning' not in result.stderr, result.stderr + + def test_execute_from_file_parses_special_option_and_preserves_filename( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -791,7 +818,7 @@ def test_execute_from_file_parses_special_option_and_preserves_filename( file_h = IteratedFile('select 1;') opened_paths: list[str] = [] - def open_file(path: str) -> IteratedFile: + def open_file(path: str, **_: Any) -> IteratedFile: opened_paths.append(path) return file_h @@ -815,7 +842,7 @@ def test_execute_from_file_reports_missing_filename_after_options(options: str) def test_execute_from_file_rejects_unquoted_filename_with_spaces(monkeypatch: pytest.MonkeyPatch) -> None: client = DummyClient() opened_paths: list[str] = [] - monkeypatch.setattr(client_commands, 'open', lambda path: opened_paths.append(path), raising=False) + monkeypatch.setattr(client_commands, 'open', lambda path, **_: opened_paths.append(path), raising=False) assert list(client.execute_from_file('query file.sql')) == [SQLResult(status=source_command.INVALID_SOURCE_FILENAME, is_error=True)] assert opened_paths == [] diff --git a/mycli_test/pytests/test_special_io_commands.py b/mycli_test/pytests/test_special_io_commands.py index b68c88c73..cda2c6d41 100644 --- a/mycli_test/pytests/test_special_io_commands.py +++ b/mycli_test/pytests/test_special_io_commands.py @@ -9,6 +9,7 @@ import signal import stat import subprocess +import sys import tempfile from time import monotonic, sleep, time from types import SimpleNamespace @@ -623,6 +624,28 @@ def raise_ioerror(*_args, **_kwargs): assert message == f'Error reading file: {filename}' +def test_open_external_editor_filename_does_not_use_locale_encoding(tmp_path: Path) -> None: + # EncodingWarning is raised for any open() that falls back to the locale encoding. + filename = tmp_path / 'query.sql' + filename.write_text('select "caf\u00e9";\n', encoding='utf-8') + script = '\n'.join(( + 'import sys', + 'import click', + 'click.edit = lambda filename: None', + 'from mycli.packages.special_commands import io_commands', + 'query, message = io_commands.open_external_editor(filename=sys.argv[1])', + 'assert message is None, message', + 'assert query == \'select "caf\\u00e9";\', query', + )) + result = subprocess.run( + [sys.executable, '-X', 'warn_default_encoding', '-W', 'error::EncodingWarning', '-c', script, str(filename)], + capture_output=True, + encoding='utf8', + errors='replace', + ) + assert result.returncode == 0, result.stderr + + def test_open_external_editor_without_filename(monkeypatch) -> None: calls: list[tuple[str, str]] = [] marker = '# Type your query above this line.\n'