diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b25a584 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,58 @@ +# Changelog + +All notable changes to this project are listed here. Versions follow [Semantic Versioning](https://semver.org/); until 1.0.0, minor versions may change CLI behaviour. + +## [0.8.0] - 2026-10-14 + +### Added + +- `-t, --to json|yaml|csv|ndjson` on every shaping command (`select`, `pick`, `omit`, `rename`, `flatten`, `unflatten`, `filter`, `sort`, `unique`, `head`, `tail`, `group-by`, `count`, `stats`, `join`), so results can be written as YAML, CSV, or NDJSON without a second `to-*` step. +- This changelog. + +### Changed + +- When `--to` is not given, shaping commands pick the output format from the `--output` file extension (for example `-o out.csv` writes CSV). Before, they always wrote JSON whatever the file name. Unknown extensions and stdout still get JSON. +- Writing a non-list result as CSV now fails with a message that points to `--to json` or `--to yaml`. + +## [0.7.0] - 2026-10-12 + +### Added + +- `join KEY --with RIGHT_FILE [file]` with `--how inner|left` and `--right-from` to join two arrays of objects on a top-level key. + +## [0.6.0] - 2026-10-10 + +### Added + +- `stats FIELD` for count / sum / min / max / avg of a numeric field, on a list or on `group-by` output. + +## [0.5.0] - 2026-09-30 + +### Added + +- `group-by KEY`, `count [KEY]`, and `unflatten` (with `--sep`). + +## [0.4.0] - 2026-09-28 + +### Added + +- `rename OLD NEW ...`, `flatten` (with `--sep`), `head N`, and `tail N`. + +## [0.3.0] - 2026-09-24 + +### Added + +- `filter KEY OP VALUE`, `sort KEY` (with `--desc`), and `unique [KEY]`. + +## [0.2.0] - 2026-09-21 + +### Added + +- NDJSON / JSON Lines input and output (`to-ndjson`, `.ndjson` / `.jsonl` detection). +- `select`, `pick`, and `omit`. + +## [0.1.0] - 2026-09-11 + +### Added + +- Python implementation of the CLI: `to-json`, `to-yaml`, `to-csv`, `pretty`, and `minify` with format detection for JSON, YAML, and CSV. diff --git a/README.md b/README.md index c1eb302..f599b7e 100644 --- a/README.md +++ b/README.md @@ -46,13 +46,45 @@ After install, the `data-transform` console script is on your `PATH`. - `[file]` — input path; omit or use `-` for stdin - `-o, --output ` — write to a file instead of stdout - `-f, --from ` — force input format: `json` | `yaml` | `csv` | `ndjson` -- `--minify` — compact JSON output (`to-json`, `select`, `pick`, `omit`, `rename`, `flatten`, `unflatten`, `filter`, `sort`, `unique`, `head`, `tail`, `group-by`, `count`, `stats`, `join`) +- `-t, --to ` — output format for the shaping commands (everything except `to-*`, `pretty`, and `minify`): `json` | `yaml` | `csv` | `ndjson`. Defaults to the `--output` file extension, else JSON. See [Output formats](#output-formats) +- `--minify` — compact JSON output (`to-json` and the shaping commands); ignored when the output format is not JSON - `--desc` — reverse sort order (`sort` only) - `--sep ` — separator between nested keys (`flatten` and `unflatten`, default `.`) - `-w, --with `, `--how inner|left`, `--right-from ` — right-hand file, join type, and right file format (`join` only) Format is taken from `--from`, else the file extension (`.ndjson` / `.jsonl` → ndjson), else simple content heuristics. +### Output formats + +Shaping commands (`select`, `pick`, `omit`, `rename`, `flatten`, `unflatten`, `filter`, `sort`, `unique`, `head`, `tail`, `group-by`, `count`, `stats`, `join`) print JSON by default. Use `-t` / `--to` to write YAML, CSV, or NDJSON directly instead of piping into a `to-*` command. + +- The format comes from `--to`, else the `--output` file extension (`.json`, `.yaml` / `.yml`, `.csv`, `.ndjson` / `.jsonl`), else JSON. `--to` always wins, and an unknown extension stays JSON. +- JSON is pretty-printed unless `--minify` is set. `--minify` does nothing for YAML, CSV, or NDJSON (NDJSON is always one compact value per line). +- CSV needs a list result. A number, object, or `group-by` result is an error (exit code 1) that suggests `--to json` or `--to yaml` instead. +- YAML output keeps key order and keeps YAML-only values such as dates, which JSON can't represent. + +```bash +# Filter a CSV and write CSV back out +data-transform filter age ge 30 people.csv -t csv +# → name,age +# Ada,36 + +# Format picked from the output file name (events.json is an array of objects) +data-transform flatten events.json -o events-flat.csv +data-transform pick id name users.json -o users.yaml + +# Unknown extension: name the format with --to +data-transform sort id users.json -t ndjson -o users.out + +# Per-team counts as YAML +data-transform count team people.json -t yaml +# → core: 1 +# web: 1 + +# Join JSON against CSV and emit CSV +data-transform join id --with teams.csv users.json -t csv -o report.csv +``` + ## Usage examples ```bash @@ -331,6 +363,10 @@ NDJSON (also known as JSON Lines / `.jsonl`) is one JSON value per non-empty lin CSV support is intentionally small and robust (RFC 4180-ish) via the Python standard library `csv` module: quoted fields, escaped quotes (`""`), and newlines inside quotes. Object arrays use a header row; all cell values are strings when parsed from CSV. +## Changelog + +See [CHANGELOG.md](./CHANGELOG.md). + ## Development ```bash diff --git a/pyproject.toml b/pyproject.toml index 91dc14a..c824479 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "data-transform-cli" -version = "0.7.0" +version = "0.8.0" description = "CLI for YAML ↔ JSON ↔ CSV ↔ NDJSON transforms for pipelines and shell use" readme = "README.md" license = { text = "MIT" } diff --git a/src/data_transform/__init__.py b/src/data_transform/__init__.py index 646a3e9..e9319db 100644 --- a/src/data_transform/__init__.py +++ b/src/data_transform/__init__.py @@ -47,4 +47,4 @@ "join_rows", ] -__version__ = "0.7.0" +__version__ = "0.8.0" diff --git a/src/data_transform/cli.py b/src/data_transform/cli.py index 1022817..6ff5c01 100644 --- a/src/data_transform/cli.py +++ b/src/data_transform/cli.py @@ -60,6 +60,41 @@ def write_output(out: str, output_path: Optional[str]) -> None: sys.stdout.write(out) +def resolve_to(to_flag: Optional[str], output_path: Optional[str]) -> str: + """Resolve output format from --to, else the --output extension, else JSON.""" + if to_flag: + return to_flag.lower() + if output_path and output_path != "-": + detected = detect_format(output_path) + if detected: + return detected + return "json" + + +def _emit( + result: object, + output: Optional[str], + to_format: Optional[str], + minify: bool, +) -> None: + """Serialize a shaping result in the chosen output format and write it. + + JSON is pretty-printed unless --minify is set. --minify has no effect on + YAML, CSV, or NDJSON (NDJSON is always compact). + """ + fmt = resolve_to(to_format, output) + if fmt == "json": + out = serialize(result, "json", pretty=not minify, minify=minify) + else: + try: + out = serialize(result, fmt) # type: ignore[arg-type] + except (TypeError, ValueError) as err: + raise ValueError( + f"cannot write {fmt}: {err}. Use --to json or --to yaml for this result" + ) from err + write_output(out, output) + + def _parse_count(token: str, command: str) -> int: """Parse a non-negative integer count from a CLI token.""" if token == "" or any(ch < "0" or ch > "9" for ch in token): @@ -131,6 +166,37 @@ def _run_json_view( sys.exit(1) +to_option = click.option( + "-t", + "--to", + "to_format", + type=click.Choice(FORMAT_CHOICES, case_sensitive=False), + default=None, + help="output format: json | yaml | csv | ndjson " + "(default: from --output extension, else json)", +) + + +def shaping_options(func): + """Shared -o/-f/-t/--minify options for commands that reshape data.""" + func = click.option( + "--minify", is_flag=True, default=False, help="emit compact JSON" + )(func) + func = to_option(func) + func = click.option( + "-f", + "--from", + "from_format", + type=click.Choice(FORMAT_CHOICES, case_sensitive=False), + default=None, + help="input format: json | yaml | csv | ndjson", + )(func) + func = click.option( + "-o", "--output", default=None, help="write to file instead of stdout" + )(func) + return func + + @click.group() @click.version_option(__version__, prog_name="data-transform") def main() -> None: @@ -253,20 +319,12 @@ def minify_cmd( @main.command("select") @click.argument("tokens", nargs=-1, required=True) -@click.option("-o", "--output", default=None, help="write to file instead of stdout") -@click.option( - "-f", - "--from", - "from_format", - type=click.Choice(FORMAT_CHOICES, case_sensitive=False), - default=None, - help="input format: json | yaml | csv | ndjson", -) -@click.option("--minify", is_flag=True, default=False, help="emit compact JSON") +@shaping_options def select_cmd( tokens: tuple[str, ...], output: Optional[str], from_format: Optional[str], + to_format: Optional[str], minify: bool, ) -> None: """Extract dotted paths from parsed data as JSON. @@ -287,9 +345,7 @@ def select_cmd( else: result = {p: get_path(data, p) for p in path_list} - write_output( - serialize(result, "json", pretty=not minify, minify=minify), output - ) + _emit(result, output, to_format, minify) except Exception as err: # noqa: BLE001 click.echo(f"error: {err}", err=True) sys.exit(1) @@ -297,20 +353,12 @@ def select_cmd( @main.command("pick") @click.argument("tokens", nargs=-1, required=True) -@click.option("-o", "--output", default=None, help="write to file instead of stdout") -@click.option( - "-f", - "--from", - "from_format", - type=click.Choice(FORMAT_CHOICES, case_sensitive=False), - default=None, - help="input format: json | yaml | csv | ndjson", -) -@click.option("--minify", is_flag=True, default=False, help="emit compact JSON") +@shaping_options def pick_cmd( tokens: tuple[str, ...], output: Optional[str], from_format: Optional[str], + to_format: Optional[str], minify: bool, ) -> None: """Keep only listed top-level keys from an object (or each object in an array). @@ -326,9 +374,7 @@ def pick_cmd( src = resolve_from(text, name, from_format) data = parse(text, src) # type: ignore[arg-type] result = pick_keys(data, key_list) - write_output( - serialize(result, "json", pretty=not minify, minify=minify), output - ) + _emit(result, output, to_format, minify) except Exception as err: # noqa: BLE001 click.echo(f"error: {err}", err=True) sys.exit(1) @@ -336,20 +382,12 @@ def pick_cmd( @main.command("omit") @click.argument("tokens", nargs=-1, required=True) -@click.option("-o", "--output", default=None, help="write to file instead of stdout") -@click.option( - "-f", - "--from", - "from_format", - type=click.Choice(FORMAT_CHOICES, case_sensitive=False), - default=None, - help="input format: json | yaml | csv | ndjson", -) -@click.option("--minify", is_flag=True, default=False, help="emit compact JSON") +@shaping_options def omit_cmd( tokens: tuple[str, ...], output: Optional[str], from_format: Optional[str], + to_format: Optional[str], minify: bool, ) -> None: """Drop listed top-level keys from an object (or each object in an array). @@ -365,9 +403,7 @@ def omit_cmd( src = resolve_from(text, name, from_format) data = parse(text, src) # type: ignore[arg-type] result = omit_keys(data, key_list) - write_output( - serialize(result, "json", pretty=not minify, minify=minify), output - ) + _emit(result, output, to_format, minify) except Exception as err: # noqa: BLE001 click.echo(f"error: {err}", err=True) sys.exit(1) @@ -376,20 +412,12 @@ def omit_cmd( @main.command("filter") @click.argument("tokens", nargs=-1, required=True) -@click.option("-o", "--output", default=None, help="write to file instead of stdout") -@click.option( - "-f", - "--from", - "from_format", - type=click.Choice(FORMAT_CHOICES, case_sensitive=False), - default=None, - help="input format: json | yaml | csv | ndjson", -) -@click.option("--minify", is_flag=True, default=False, help="emit compact JSON") +@shaping_options def filter_cmd( tokens: tuple[str, ...], output: Optional[str], from_format: Optional[str], + to_format: Optional[str], minify: bool, ) -> None: """Keep array items where KEY OP VALUE matches. @@ -419,9 +447,7 @@ def filter_cmd( src = resolve_from(text, name, from_format) data = parse(text, src) # type: ignore[arg-type] result = filter_rows(data, key, op, value) - write_output( - serialize(result, "json", pretty=not minify, minify=minify), output - ) + _emit(result, output, to_format, minify) except Exception as err: # noqa: BLE001 click.echo(f"error: {err}", err=True) sys.exit(1) @@ -429,21 +455,13 @@ def filter_cmd( @main.command("sort") @click.argument("tokens", nargs=-1, required=True) -@click.option("-o", "--output", default=None, help="write to file instead of stdout") -@click.option( - "-f", - "--from", - "from_format", - type=click.Choice(FORMAT_CHOICES, case_sensitive=False), - default=None, - help="input format: json | yaml | csv | ndjson", -) -@click.option("--minify", is_flag=True, default=False, help="emit compact JSON") +@shaping_options @click.option("--desc", is_flag=True, default=False, help="sort descending") def sort_cmd( tokens: tuple[str, ...], output: Optional[str], from_format: Optional[str], + to_format: Optional[str], minify: bool, desc: bool, ) -> None: @@ -463,9 +481,7 @@ def sort_cmd( src = resolve_from(text, name, from_format) data = parse(text, src) # type: ignore[arg-type] result = sort_rows(data, key, desc=desc) - write_output( - serialize(result, "json", pretty=not minify, minify=minify), output - ) + _emit(result, output, to_format, minify) except Exception as err: # noqa: BLE001 click.echo(f"error: {err}", err=True) sys.exit(1) @@ -473,20 +489,12 @@ def sort_cmd( @main.command("unique") @click.argument("tokens", nargs=-1, required=False) -@click.option("-o", "--output", default=None, help="write to file instead of stdout") -@click.option( - "-f", - "--from", - "from_format", - type=click.Choice(FORMAT_CHOICES, case_sensitive=False), - default=None, - help="input format: json | yaml | csv | ndjson", -) -@click.option("--minify", is_flag=True, default=False, help="emit compact JSON") +@shaping_options def unique_cmd( tokens: tuple[str, ...], output: Optional[str], from_format: Optional[str], + to_format: Optional[str], minify: bool, ) -> None: """Deduplicate an array (order-preserving, first wins). @@ -512,9 +520,7 @@ def unique_cmd( src = resolve_from(text, name, from_format) data = parse(text, src) # type: ignore[arg-type] result = unique_rows(data, key) - write_output( - serialize(result, "json", pretty=not minify, minify=minify), output - ) + _emit(result, output, to_format, minify) except Exception as err: # noqa: BLE001 click.echo(f"error: {err}", err=True) sys.exit(1) @@ -522,20 +528,12 @@ def unique_cmd( @main.command("rename") @click.argument("tokens", nargs=-1, required=True) -@click.option("-o", "--output", default=None, help="write to file instead of stdout") -@click.option( - "-f", - "--from", - "from_format", - type=click.Choice(FORMAT_CHOICES, case_sensitive=False), - default=None, - help="input format: json | yaml | csv | ndjson", -) -@click.option("--minify", is_flag=True, default=False, help="emit compact JSON") +@shaping_options def rename_cmd( tokens: tuple[str, ...], output: Optional[str], from_format: Optional[str], + to_format: Optional[str], minify: bool, ) -> None: """Rename top-level keys (OLD NEW pairs) on an object or each object in an array. @@ -553,9 +551,7 @@ def rename_cmd( src = resolve_from(text, name, from_format) data = parse(text, src) # type: ignore[arg-type] result = rename_keys(data, mapping) - write_output( - serialize(result, "json", pretty=not minify, minify=minify), output - ) + _emit(result, output, to_format, minify) except Exception as err: # noqa: BLE001 click.echo(f"error: {err}", err=True) sys.exit(1) @@ -563,16 +559,7 @@ def rename_cmd( @main.command("flatten") @click.argument("file", required=False) -@click.option("-o", "--output", default=None, help="write to file instead of stdout") -@click.option( - "-f", - "--from", - "from_format", - type=click.Choice(FORMAT_CHOICES, case_sensitive=False), - default=None, - help="input format: json | yaml | csv | ndjson", -) -@click.option("--minify", is_flag=True, default=False, help="emit compact JSON") +@shaping_options @click.option( "--sep", default=".", @@ -583,6 +570,7 @@ def flatten_cmd( file: Optional[str], output: Optional[str], from_format: Optional[str], + to_format: Optional[str], minify: bool, sep: str, ) -> None: @@ -598,9 +586,7 @@ def flatten_cmd( src = resolve_from(text, name, from_format) data = parse(text, src) # type: ignore[arg-type] result = flatten_object(data, sep) - write_output( - serialize(result, "json", pretty=not minify, minify=minify), output - ) + _emit(result, output, to_format, minify) except Exception as err: # noqa: BLE001 click.echo(f"error: {err}", err=True) sys.exit(1) @@ -608,16 +594,7 @@ def flatten_cmd( @main.command("unflatten") @click.argument("file", required=False) -@click.option("-o", "--output", default=None, help="write to file instead of stdout") -@click.option( - "-f", - "--from", - "from_format", - type=click.Choice(FORMAT_CHOICES, case_sensitive=False), - default=None, - help="input format: json | yaml | csv | ndjson", -) -@click.option("--minify", is_flag=True, default=False, help="emit compact JSON") +@shaping_options @click.option( "--sep", default=".", @@ -628,6 +605,7 @@ def unflatten_cmd( file: Optional[str], output: Optional[str], from_format: Optional[str], + to_format: Optional[str], minify: bool, sep: str, ) -> None: @@ -643,9 +621,7 @@ def unflatten_cmd( src = resolve_from(text, name, from_format) data = parse(text, src) # type: ignore[arg-type] result = unflatten_object(data, sep) - write_output( - serialize(result, "json", pretty=not minify, minify=minify), output - ) + _emit(result, output, to_format, minify) except Exception as err: # noqa: BLE001 click.echo(f"error: {err}", err=True) sys.exit(1) @@ -656,6 +632,7 @@ def _run_take( tokens: Sequence[str], output: Optional[str], from_format: Optional[str], + to_format: Optional[str], minify: bool, ) -> None: """Shared runner for head/tail.""" @@ -672,9 +649,7 @@ def _run_take( result = head_rows(data, count) else: result = tail_rows(data, count) - write_output( - serialize(result, "json", pretty=not minify, minify=minify), output - ) + _emit(result, output, to_format, minify) except Exception as err: # noqa: BLE001 click.echo(f"error: {err}", err=True) sys.exit(1) @@ -684,20 +659,12 @@ def _run_take( # can reject it, instead of Click treating it as a short flag. @main.command("head", context_settings={"ignore_unknown_options": True}) @click.argument("tokens", nargs=-1, required=True) -@click.option("-o", "--output", default=None, help="write to file instead of stdout") -@click.option( - "-f", - "--from", - "from_format", - type=click.Choice(FORMAT_CHOICES, case_sensitive=False), - default=None, - help="input format: json | yaml | csv | ndjson", -) -@click.option("--minify", is_flag=True, default=False, help="emit compact JSON") +@shaping_options def head_cmd( tokens: tuple[str, ...], output: Optional[str], from_format: Optional[str], + to_format: Optional[str], minify: bool, ) -> None: """Return the first N items of a list. @@ -707,25 +674,17 @@ def head_cmd( Usage: data-transform head N [file] """ - _run_take("head", tokens, output, from_format, minify) + _run_take("head", tokens, output, from_format, to_format, minify) @main.command("tail", context_settings={"ignore_unknown_options": True}) @click.argument("tokens", nargs=-1, required=True) -@click.option("-o", "--output", default=None, help="write to file instead of stdout") -@click.option( - "-f", - "--from", - "from_format", - type=click.Choice(FORMAT_CHOICES, case_sensitive=False), - default=None, - help="input format: json | yaml | csv | ndjson", -) -@click.option("--minify", is_flag=True, default=False, help="emit compact JSON") +@shaping_options def tail_cmd( tokens: tuple[str, ...], output: Optional[str], from_format: Optional[str], + to_format: Optional[str], minify: bool, ) -> None: """Return the last N items of a list. @@ -735,25 +694,17 @@ def tail_cmd( Usage: data-transform tail N [file] """ - _run_take("tail", tokens, output, from_format, minify) + _run_take("tail", tokens, output, from_format, to_format, minify) @main.command("group-by") @click.argument("tokens", nargs=-1, required=True) -@click.option("-o", "--output", default=None, help="write to file instead of stdout") -@click.option( - "-f", - "--from", - "from_format", - type=click.Choice(FORMAT_CHOICES, case_sensitive=False), - default=None, - help="input format: json | yaml | csv | ndjson", -) -@click.option("--minify", is_flag=True, default=False, help="emit compact JSON") +@shaping_options def group_by_cmd( tokens: tuple[str, ...], output: Optional[str], from_format: Optional[str], + to_format: Optional[str], minify: bool, ) -> None: """Group an array of objects by top-level KEY. @@ -774,9 +725,7 @@ def group_by_cmd( src = resolve_from(text, name, from_format) data = parse(text, src) # type: ignore[arg-type] result = group_rows(data, key) - write_output( - serialize(result, "json", pretty=not minify, minify=minify), output - ) + _emit(result, output, to_format, minify) except Exception as err: # noqa: BLE001 click.echo(f"error: {err}", err=True) sys.exit(1) @@ -784,20 +733,12 @@ def group_by_cmd( @main.command("count") @click.argument("tokens", nargs=-1, required=False) -@click.option("-o", "--output", default=None, help="write to file instead of stdout") -@click.option( - "-f", - "--from", - "from_format", - type=click.Choice(FORMAT_CHOICES, case_sensitive=False), - default=None, - help="input format: json | yaml | csv | ndjson", -) -@click.option("--minify", is_flag=True, default=False, help="emit compact JSON") +@shaping_options def count_cmd( tokens: tuple[str, ...], output: Optional[str], from_format: Optional[str], + to_format: Optional[str], minify: bool, ) -> None: """Count records in a list, per KEY value, or per group. @@ -820,9 +761,7 @@ def count_cmd( src = resolve_from(text, name, from_format) data = parse(text, src) # type: ignore[arg-type] result = count_rows(data, key) - write_output( - serialize(result, "json", pretty=not minify, minify=minify), output - ) + _emit(result, output, to_format, minify) except Exception as err: # noqa: BLE001 click.echo(f"error: {err}", err=True) sys.exit(1) @@ -830,20 +769,12 @@ def count_cmd( @main.command("stats") @click.argument("tokens", nargs=-1, required=True) -@click.option("-o", "--output", default=None, help="write to file instead of stdout") -@click.option( - "-f", - "--from", - "from_format", - type=click.Choice(FORMAT_CHOICES, case_sensitive=False), - default=None, - help="input format: json | yaml | csv | ndjson", -) -@click.option("--minify", is_flag=True, default=False, help="emit compact JSON") +@shaping_options def stats_cmd( tokens: tuple[str, ...], output: Optional[str], from_format: Optional[str], + to_format: Optional[str], minify: bool, ) -> None: """Summarise a numeric FIELD: count, sum, min, max, avg. @@ -861,9 +792,7 @@ def stats_cmd( src = resolve_from(text, name, from_format) data = parse(text, src) # type: ignore[arg-type] result = stats_rows(data, parts[0]) - write_output( - serialize(result, "json", pretty=not minify, minify=minify), output - ) + _emit(result, output, to_format, minify) except Exception as err: # noqa: BLE001 click.echo(f"error: {err}", err=True) sys.exit(1) @@ -908,6 +837,7 @@ def _load_right(path: str, right_from: Optional[str]) -> object: default=None, help="right file format: json | yaml | csv | ndjson", ) +@to_option @click.option( "--how", type=click.Choice(["inner", "left"], case_sensitive=False), @@ -923,6 +853,7 @@ def join_cmd( output: Optional[str], from_format: Optional[str], right_from: Optional[str], + to_format: Optional[str], how: str, minify: bool, ) -> None: @@ -942,9 +873,7 @@ def join_cmd( src = resolve_from(text, name, from_format) left = parse(text, src) # type: ignore[arg-type] result = join_rows(left, right, key, how=how) - write_output( - serialize(result, "json", pretty=not minify, minify=minify), output - ) + _emit(result, output, to_format, minify) except Exception as err: # noqa: BLE001 click.echo(f"error: {err}", err=True) sys.exit(1) diff --git a/tests/test_output_format.py b/tests/test_output_format.py new file mode 100644 index 0000000..eacd4d9 --- /dev/null +++ b/tests/test_output_format.py @@ -0,0 +1,186 @@ +import json + +import pytest +import yaml + +from data_transform.cli import resolve_to +from data_transform.csv_io import parse_csv +from test_cli import run_cli + +PEOPLE = '[{"name":"Ada","age":36,"team":"core"},{"name":"Bob","age":22,"team":"web"}]' + + +@pytest.mark.parametrize( + "to_flag, output, expected", + [ + (None, None, "json"), + (None, "-", "json"), + (None, "out.csv", "csv"), + (None, "out.YML", "yaml"), + (None, "dir/out.jsonl", "ndjson"), + (None, "out.txt", "json"), + ("yaml", "out.csv", "yaml"), + ("CSV", None, "csv"), + ], +) +def test_resolve_to_flag_then_extension_then_json(to_flag, output, expected): + assert resolve_to(to_flag, output) == expected + + +def test_filter_to_csv(): + code, stdout, stderr = run_cli( + ["filter", "age", "ge", "30", "-f", "json", "-t", "csv"], PEOPLE + ) + assert code == 0, stderr + assert stdout == "name,age,team\nAda,36,core\n" + + +def test_sort_to_yaml_keeps_key_order(): + code, stdout, stderr = run_cli(["sort", "age", "-f", "json", "--to", "yaml"], PEOPLE) + assert code == 0, stderr + assert yaml.safe_load(stdout) == [ + {"name": "Bob", "age": 22, "team": "web"}, + {"name": "Ada", "age": 36, "team": "core"}, + ] + assert stdout.startswith("- name: Bob\n age: 22\n") + + +def test_pick_to_ndjson_is_one_compact_line_per_item(): + code, stdout, stderr = run_cli(["pick", "name", "-f", "json", "-t", "ndjson"], PEOPLE) + assert code == 0, stderr + assert stdout == '{"name":"Ada"}\n{"name":"Bob"}\n' + + +def test_to_json_explicit_matches_default_and_respects_minify(): + _, default_out, _ = run_cli(["head", "1", "-f", "json"], PEOPLE) + code, explicit_out, stderr = run_cli(["head", "1", "-f", "json", "-t", "json"], PEOPLE) + assert code == 0, stderr + assert explicit_out == default_out + assert explicit_out.endswith("\n") and "\n " in explicit_out + _, compact, _ = run_cli(["head", "1", "-f", "json", "-t", "json", "--minify"], PEOPLE) + assert compact == '[{"name":"Ada","age":36,"team":"core"}]' + + +def test_minify_is_ignored_for_non_json_output(): + code, stdout, stderr = run_cli( + ["tail", "1", "-f", "json", "-t", "csv", "--minify"], PEOPLE + ) + assert code == 0, stderr + assert stdout == "name,age,team\nBob,22,web\n" + + +def test_output_extension_picks_format(tmp_path): + out_csv = tmp_path / "flat.csv" + code, stdout, stderr = run_cli( + ["flatten", "-f", "json", "-o", str(out_csv)], + '[{"id":1,"user":{"name":"Ada"}}]', + ) + assert code == 0, stderr + assert stdout == "" + assert out_csv.read_text(encoding="utf-8") == "id,user.name\n1,Ada\n" + + out_yaml = tmp_path / "people.yaml" + code, _, stderr = run_cli(["omit", "age", "-f", "json", "-o", str(out_yaml)], PEOPLE) + assert code == 0, stderr + assert yaml.safe_load(out_yaml.read_text(encoding="utf-8"))[0] == { + "name": "Ada", + "team": "core", + } + + +def test_to_flag_overrides_output_extension(tmp_path): + out = tmp_path / "data.csv" + code, _, stderr = run_cli( + ["rename", "name", "title", "-f", "json", "-t", "json", "--minify", "-o", str(out)], + PEOPLE, + ) + assert code == 0, stderr + assert json.loads(out.read_text(encoding="utf-8"))[0]["title"] == "Ada" + + +def test_unknown_output_extension_stays_json(tmp_path): + out = tmp_path / "result.out" + code, _, stderr = run_cli(["unique", "team", "-f", "json", "-o", str(out)], PEOPLE) + assert code == 0, stderr + assert len(json.loads(out.read_text(encoding="utf-8"))) == 2 + + +def test_csv_round_trip_through_filter(): + csv_in = "name,age\nAda,36\nBob,22\nCy,41\n" + code, stdout, stderr = run_cli( + ["filter", "age", "gt", "30", "-f", "csv", "-t", "csv"], csv_in + ) + assert code == 0, stderr + assert parse_csv(stdout) == [{"name": "Ada", "age": "36"}, {"name": "Cy", "age": "41"}] + + +def test_group_by_and_count_to_yaml(): + code, stdout, stderr = run_cli(["count", "team", "-f", "json", "-t", "yaml"], PEOPLE) + assert code == 0, stderr + assert stdout == "core: 1\nweb: 1\n" + code, stdout, stderr = run_cli(["group-by", "team", "-f", "json", "-t", "yaml"], PEOPLE) + assert code == 0, stderr + assert list(yaml.safe_load(stdout)) == ["core", "web"] + + +def test_stats_and_select_to_yaml(): + code, stdout, stderr = run_cli(["stats", "age", "-f", "json", "-t", "yaml"], PEOPLE) + assert code == 0, stderr + assert yaml.safe_load(stdout) == {"count": 2, "sum": 58, "min": 22, "max": 36, "avg": 29} + code, stdout, stderr = run_cli(["select", "0.name", "-f", "json", "-t", "yaml"], PEOPLE) + assert code == 0, stderr + assert yaml.safe_load(stdout) == "Ada" + + +def test_unflatten_to_ndjson(): + code, stdout, stderr = run_cli( + ["unflatten", "-f", "json", "-t", "ndjson"], '[{"a.b":1},{"a.b":2}]' + ) + assert code == 0, stderr + assert stdout == '{"a":{"b":1}}\n{"a":{"b":2}}\n' + + +def test_join_to_csv(tmp_path): + right = tmp_path / "teams.csv" + right.write_text("id,team\n1,core\n", encoding="utf-8") + code, stdout, stderr = run_cli( + ["join", "id", "-w", str(right), "-f", "json", "-t", "csv"], + '[{"id":1,"name":"Ada"},{"id":2,"name":"Bob"}]', + ) + assert code == 0, stderr + assert stdout == "id,name,team\n1,Ada,core\n" + + +def test_yaml_dates_survive_when_output_is_yaml(): + yaml_in = "- id: 1\n day: 2026-10-14\n- id: 2\n day: 2026-10-15\n" + code, stdout, stderr = run_cli(["head", "1", "-f", "yaml", "-t", "yaml"], yaml_in) + assert code == 0, stderr + assert stdout == "- id: 1\n day: 2026-10-14\n" + + +@pytest.mark.parametrize( + "args, stdin", + [ + (["count", "-f", "json", "-t", "csv"], PEOPLE), + (["group-by", "team", "-f", "json", "-t", "csv"], PEOPLE), + (["select", "0", "-f", "json", "-t", "csv"], PEOPLE), + ], +) +def test_csv_output_of_non_list_result_is_a_clear_error(args, stdin): + code, stdout, stderr = run_cli(args, stdin) + assert code == 1 + assert stdout == "" + assert "error: cannot write csv" in stderr + assert "--to json or --to yaml" in stderr + + +def test_invalid_to_choice_is_usage_error(): + code, _, stderr = run_cli(["pick", "name", "-f", "json", "-t", "xml"], PEOPLE) + assert code == 2 + assert "xml" in stderr + + +def test_conversion_commands_do_not_take_to(): + code, _, stderr = run_cli(["to-json", "-f", "json", "-t", "yaml"], "{}") + assert code == 2 + assert "No such option" in stderr diff --git a/tests/test_shape.py b/tests/test_shape.py index 7e1172e..73fd408 100644 --- a/tests/test_shape.py +++ b/tests/test_shape.py @@ -166,7 +166,7 @@ def test_unique_requires_list(): def test_public_exports_include_reshape_helpers(): - assert data_transform.__version__ == "0.7.0" + assert data_transform.__version__ == "0.8.0" for name in ( "rename_keys", "flatten_object",