diff --git a/README.md b/README.md index acfb6f3..dda8db2 100644 --- a/README.md +++ b/README.md @@ -28,13 +28,17 @@ After install, the `data-transform` console script is on your `PATH`. | `select` | Extract dotted paths from parsed data | | `pick` | Keep only listed top-level keys | | `omit` | Drop listed top-level keys | +| `filter` | Keep array items matching KEY OP VALUE | +| `sort` | Sort array of objects by top-level KEY | +| `unique` | Deduplicate array (by KEY or whole item) | ### Options (most commands) - `[file]` — input path; omit or use `-` for stdin - `-o, --output ` — write to a file instead of stdout - `-f, --from ` — force input format: `json` | `yaml` | `csv` | `ndjson` -- `--minify` — compact JSON output (`to-json`, `select`, `pick`, `omit`) +- `--minify` — compact JSON output (`to-json`, `select`, `pick`, `omit`, `filter`, `sort`, `unique`) +- `--desc` — reverse sort order (`sort` only) Format is taken from `--from`, else the file extension (`.ndjson` / `.jsonl` → ndjson), else simple content heuristics. @@ -92,16 +96,68 @@ echo '{"id":1,"secret":"x","name":"a"}' \ # → {"id":1,"name":"a"} ``` +### Filter, sort, unique + +These commands require a **list** root (JSON/YAML/CSV/NDJSON array). Non-object +items are dropped by `filter`. Missing keys sort last (first with `--desc`). + +```bash +# Keep rows where age >= 30 (numeric when both sides look like numbers) +echo '[{"name":"Ada","age":36},{"name":"Bob","age":22}]' \ + | data-transform filter age ge 30 -f json --minify +# → [{"name":"Ada","age":36}] + +# String contains / key exists (VALUE optional for exists) +echo '[{"role":"admin"},{"role":"user"}]' \ + | data-transform filter role contains adm -f json --minify +# → [{"role":"admin"}] + +echo '[{"id":1,"note":"x"},{"id":2}]' \ + | data-transform filter note exists -f json --minify +# → [{"id":1,"note":"x"}] + +# Sort by key (ascending / descending) +echo '[{"id":2},{"id":1},{"id":3}]' \ + | data-transform sort id -f json --minify +# → [{"id":1},{"id":2},{"id":3}] + +echo '[{"id":2},{"id":1}]' \ + | data-transform sort id --desc -f json --minify +# → [{"id":2},{"id":1}] + +# Deduplicate whole items, or by a key (first wins) +echo '[{"id":1},{"id":1},{"id":2}]' \ + | data-transform unique id -f json --minify +# → [{"id":1},{"id":2}] + +echo '[{"a":1},{"a":1},{"b":2}]' \ + | data-transform unique -f json --minify +# → [{"a":1},{"b":2}] +``` + ### Library use ```python -from data_transform import convert, parse, serialize, get_path, pick_keys, omit_keys +from data_transform import ( + convert, + parse, + serialize, + get_path, + pick_keys, + omit_keys, + filter_rows, + sort_rows, + unique_rows, +) yaml_text = convert('{"a":1}', "json", "yaml") data = parse("id,name\n1,x\n", "csv") json_text = serialize(data, "json", pretty=True) name = get_path({"user": {"name": "Ada"}}, "user.name") slim = pick_keys({"id": 1, "name": "a", "note": "x"}, ["id", "name"]) +adults = filter_rows([{"age": 36}, {"age": 22}], "age", "ge", "30") +ordered = sort_rows([{"id": 2}, {"id": 1}], "id") +deduped = unique_rows([{"id": 1}, {"id": 1}], "id") ``` ## NDJSON notes diff --git a/pyproject.toml b/pyproject.toml index 1ea297c..c4b6609 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "data-transform-cli" -version = "0.2.0" +version = "0.3.0" description = "CLI for YAML ↔ JSON ↔ CSV ↔ NDJSON transforms for pipelines and shell use" readme = "README.md" license = { text = "MIT" } diff --git a/src/data_transform/__init__.py b/src/data_transform/__init__.py index 5075de0..8c772b7 100644 --- a/src/data_transform/__init__.py +++ b/src/data_transform/__init__.py @@ -2,7 +2,14 @@ from .convert import convert, detect_format, infer_format, parse, serialize from .csv_io import escape_csv_field, parse_csv, parse_csv_rows, stringify_csv -from .shape import get_path, omit_keys, pick_keys +from .shape import ( + filter_rows, + get_path, + omit_keys, + pick_keys, + sort_rows, + unique_rows, +) __all__ = [ "convert", @@ -17,6 +24,9 @@ "get_path", "pick_keys", "omit_keys", + "filter_rows", + "sort_rows", + "unique_rows", ] -__version__ = "0.2.0" +__version__ = "0.3.0" diff --git a/src/data_transform/cli.py b/src/data_transform/cli.py index 4edeb3f..2115e51 100644 --- a/src/data_transform/cli.py +++ b/src/data_transform/cli.py @@ -10,7 +10,7 @@ from . import __version__ from .convert import convert, detect_format, infer_format, parse, serialize -from .shape import get_path, omit_keys, pick_keys +from .shape import filter_rows, get_path, omit_keys, pick_keys, sort_rows, unique_rows FORMAT_CHOICES = ["json", "yaml", "csv", "ndjson"] @@ -338,5 +338,153 @@ def omit_cmd( sys.exit(1) + +@main.command("filter") +@click.argument("tokens", nargs=-1, required=True) +@click.option("-o", "--output", default=None, help="write to file instead of stdout") +@click.option( + "-f", + "--from", + "from_format", + type=click.Choice(FORMAT_CHOICES, case_sensitive=False), + default=None, + help="input format: json | yaml | csv | ndjson", +) +@click.option("--minify", is_flag=True, default=False, help="emit compact JSON") +def filter_cmd( + tokens: tuple[str, ...], + output: Optional[str], + from_format: Optional[str], + minify: bool, +) -> None: + """Keep array items where KEY OP VALUE matches. + + Operators: eq, ne, gt, ge, lt, le, contains, exists. + Non-object items are dropped. Root must be a list. + + Usage: data-transform filter KEY OP VALUE [file] + """ + try: + parts, input_file = _split_tokens_and_file(tokens) + if len(parts) < 2: + raise ValueError("Usage: filter KEY OP [VALUE] [file]") + key, op = parts[0], parts[1] + op_l = op.lower() + if op_l == "exists": + if len(parts) > 3: + raise ValueError("Usage: filter KEY exists [file]") + # Optional ignored VALUE may appear as third token + value = parts[2] if len(parts) == 3 else None + else: + if len(parts) != 3: + raise ValueError("Usage: filter KEY OP VALUE [file]") + value = parts[2] + + text, name = load_input(input_file) + src = resolve_from(text, name, from_format) + data = parse(text, src) # type: ignore[arg-type] + result = filter_rows(data, key, op, value) + write_output( + serialize(result, "json", pretty=not minify, minify=minify), output + ) + except Exception as err: # noqa: BLE001 + click.echo(f"error: {err}", err=True) + sys.exit(1) + + +@main.command("sort") +@click.argument("tokens", nargs=-1, required=True) +@click.option("-o", "--output", default=None, help="write to file instead of stdout") +@click.option( + "-f", + "--from", + "from_format", + type=click.Choice(FORMAT_CHOICES, case_sensitive=False), + default=None, + help="input format: json | yaml | csv | ndjson", +) +@click.option("--minify", is_flag=True, default=False, help="emit compact JSON") +@click.option("--desc", is_flag=True, default=False, help="sort descending") +def sort_cmd( + tokens: tuple[str, ...], + output: Optional[str], + from_format: Optional[str], + minify: bool, + desc: bool, +) -> None: + """Sort an array of objects by top-level KEY. + + Missing keys sort last (first with --desc). Root must be a list. + + Usage: data-transform sort KEY [file] + """ + try: + parts, input_file = _split_tokens_and_file(tokens) + if len(parts) != 1: + raise ValueError("Usage: sort KEY [file]") + key = parts[0] + + text, name = load_input(input_file) + src = resolve_from(text, name, from_format) + data = parse(text, src) # type: ignore[arg-type] + result = sort_rows(data, key, desc=desc) + write_output( + serialize(result, "json", pretty=not minify, minify=minify), output + ) + except Exception as err: # noqa: BLE001 + click.echo(f"error: {err}", err=True) + sys.exit(1) + + +@main.command("unique") +@click.argument("tokens", nargs=-1, required=False) +@click.option("-o", "--output", default=None, help="write to file instead of stdout") +@click.option( + "-f", + "--from", + "from_format", + type=click.Choice(FORMAT_CHOICES, case_sensitive=False), + default=None, + help="input format: json | yaml | csv | ndjson", +) +@click.option("--minify", is_flag=True, default=False, help="emit compact JSON") +def unique_cmd( + tokens: tuple[str, ...], + output: Optional[str], + from_format: Optional[str], + minify: bool, +) -> None: + """Deduplicate an array (order-preserving, first wins). + + With KEY, dedupe by that top-level key. Without KEY, dedupe by whole item. + Root must be a list. + + Usage: data-transform unique [KEY] [file] + """ + try: + parts, input_file = _split_tokens_and_file(tokens) + # If a single token looks like a file and was not peeled off (e.g. path + # that does not exist yet / stdin already handled), treat it as KEY. + # When tokens was only a file path that exists, _split peels it and + # parts is empty → no key. + key: Optional[str] = None + if len(parts) == 1: + key = parts[0] + elif len(parts) > 1: + raise ValueError("Usage: unique [KEY] [file]") + + text, name = load_input(input_file) + src = resolve_from(text, name, from_format) + data = parse(text, src) # type: ignore[arg-type] + result = unique_rows(data, key) + write_output( + serialize(result, "json", pretty=not minify, minify=minify), output + ) + except Exception as err: # noqa: BLE001 + click.echo(f"error: {err}", err=True) + sys.exit(1) + + + if __name__ == "__main__": main() diff --git a/src/data_transform/shape.py b/src/data_transform/shape.py index 78030af..f60e3e6 100644 --- a/src/data_transform/shape.py +++ b/src/data_transform/shape.py @@ -2,8 +2,11 @@ from __future__ import annotations +import json from typing import Any, Mapping, Sequence +FILTER_OPS = frozenset({"eq", "ne", "gt", "ge", "lt", "le", "contains", "exists"}) + def _parse_path(path: str) -> list[str | int]: """Split a dotted path into segments; raise ValueError on invalid syntax.""" @@ -85,3 +88,159 @@ def _omit_one(item: Any, drop: set[str]) -> Any: if not isinstance(item, Mapping): return item return {k: v for k, v in item.items() if k not in drop} + + +def _require_list(data: Any, command: str) -> list[Any]: + """Raise ValueError unless data is a list; return it typed.""" + if not isinstance(data, list): + raise ValueError( + f"{command}: root must be a list (array), got {type(data).__name__}" + ) + return data + + +def _looks_number(value: Any) -> bool: + """True when value is an int/float (not bool) or a numeric string.""" + if isinstance(value, bool): + return False + if isinstance(value, (int, float)): + return True + if isinstance(value, str): + try: + float(value) + return True + except ValueError: + return False + return False + + +def _as_number(value: Any) -> float: + if isinstance(value, (int, float)) and not isinstance(value, bool): + return float(value) + return float(str(value)) + + +def _compare(left: Any, op: str, right: Any) -> bool: + """Compare left to right with op; numeric when both look like numbers.""" + if op == "exists": + return left is not None + + if op == "contains": + return str(right) in str(left) if left is not None else False + + use_numeric = _looks_number(left) and _looks_number(right) + if use_numeric: + lv: Any = _as_number(left) + rv: Any = _as_number(right) + else: + lv = "" if left is None else str(left) + rv = "" if right is None else str(right) + + if op == "eq": + return lv == rv + if op == "ne": + return lv != rv + if op == "gt": + return lv > rv + if op == "ge": + return lv >= rv + if op == "lt": + return lv < rv + if op == "le": + return lv <= rv + raise ValueError( + f"Unsupported filter operator: {op!r} " + "(use eq, ne, gt, ge, lt, le, contains, exists)" + ) + + +def filter_rows(data: Any, key: str, op: str, value: Any = None) -> list[Any]: + """ + Keep objects where top-level KEY compares via OP to VALUE. + + Root must be a list. Non-object items are dropped. For ``exists``, VALUE + is ignored and the key must be present and not null. + """ + rows = _require_list(data, "filter") + op_norm = op.lower() + if op_norm not in FILTER_OPS: + raise ValueError( + f"Unsupported filter operator: {op!r} " + "(use eq, ne, gt, ge, lt, le, contains, exists)" + ) + + out: list[Any] = [] + for item in rows: + if not isinstance(item, Mapping): + continue + if op_norm == "exists": + if key in item and item[key] is not None: + out.append(item) + continue + if key not in item: + continue + if _compare(item[key], op_norm, value): + out.append(item) + return out + + +def sort_rows(data: Any, key: str, *, desc: bool = False) -> list[Any]: + """ + Sort an array of objects by top-level KEY. + + Missing keys sort last (first when ``desc``). Non-object items are treated + as missing the key. Uses a None sentinel so ordering is stable-ish via + Python's ``sorted``. + """ + rows = _require_list(data, "sort") + + def sort_key(item: Any) -> tuple[int, int, float, str]: + # (missing, non_numeric, number, text) — always comparable across types. + if not isinstance(item, Mapping) or key not in item or item[key] is None: + # Missing / null: sort last for ascending, first for descending. + return (1, 1, 0.0, "") + val = item[key] + if _looks_number(val): + return (0, 0, _as_number(val), "") + return (0, 1, 0.0, str(val)) + + return sorted(rows, key=sort_key, reverse=desc) + + +def unique_rows(data: Any, key: str | None = None) -> list[Any]: + """ + Deduplicate an array. First occurrence wins (order-preserving). + + With KEY, dedupe by that top-level key (non-objects / missing key each keep + their own slot). Without KEY, dedupe by JSON-serialized item. + """ + rows = _require_list(data, "unique") + seen: set[Any] = set() + out: list[Any] = [] + + if key is None: + for item in rows: + try: + marker = json.dumps(item, sort_keys=True, default=str) + except (TypeError, ValueError): + marker = repr(item) + if marker in seen: + continue + seen.add(marker) + out.append(item) + return out + + for item in rows: + if isinstance(item, Mapping) and key in item: + marker: Any = ( + "k", + json.dumps(item[key], sort_keys=True, default=str), + ) + else: + # Non-object or missing key: keep each occurrence (unique by position). + marker = ("i", id(item), len(out)) + if marker in seen: + continue + seen.add(marker) + out.append(item) + return out diff --git a/tests/test_cli.py b/tests/test_cli.py index b677fdc..d0df168 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -135,3 +135,61 @@ def test_omit_keys_from_array(): ) assert code == 0, stderr assert json.loads(stdout) == [{"id": 1}, {"id": 2}] + + +def test_filter_ge_from_stdin(): + code, stdout, stderr = run_cli( + ["filter", "age", "ge", "30", "-f", "json", "--minify"], + '[{"name":"Ada","age":36},{"name":"Bob","age":22}]', + ) + assert code == 0, stderr + assert json.loads(stdout) == [{"name": "Ada", "age": 36}] + + +def test_filter_exists_and_non_list_error(): + code, stdout, stderr = run_cli( + ["filter", "note", "exists", "-f", "json", "--minify"], + '[{"id":1,"note":"x"},{"id":2}]', + ) + assert code == 0, stderr + assert json.loads(stdout) == [{"id": 1, "note": "x"}] + + code, stdout, stderr = run_cli( + ["filter", "a", "eq", "1", "-f", "json"], + '{"a":1}', + ) + assert code == 1 + assert "error:" in stderr + assert "list" in stderr.lower() + + +def test_sort_and_sort_desc(): + code, stdout, stderr = run_cli( + ["sort", "id", "-f", "json", "--minify"], + '[{"id":2},{"id":1},{"id":3}]', + ) + assert code == 0, stderr + assert json.loads(stdout) == [{"id": 1}, {"id": 2}, {"id": 3}] + + code, stdout, stderr = run_cli( + ["sort", "id", "--desc", "-f", "json", "--minify"], + '[{"id":2},{"id":1}]', + ) + assert code == 0, stderr + assert json.loads(stdout) == [{"id": 2}, {"id": 1}] + + +def test_unique_by_key_and_whole(): + code, stdout, stderr = run_cli( + ["unique", "id", "-f", "json", "--minify"], + '[{"id":1,"n":"a"},{"id":1,"n":"b"},{"id":2}]', + ) + assert code == 0, stderr + assert json.loads(stdout) == [{"id": 1, "n": "a"}, {"id": 2}] + + code, stdout, stderr = run_cli( + ["unique", "-f", "json", "--minify"], + '[{"a":1},{"a":1},{"b":2}]', + ) + assert code == 0, stderr + assert json.loads(stdout) == [{"a": 1}, {"b": 2}] diff --git a/tests/test_shape.py b/tests/test_shape.py index 1baecbb..0192463 100644 --- a/tests/test_shape.py +++ b/tests/test_shape.py @@ -1,6 +1,13 @@ import pytest -from data_transform.shape import get_path, omit_keys, pick_keys +from data_transform.shape import ( + filter_rows, + get_path, + omit_keys, + pick_keys, + sort_rows, + unique_rows, +) def test_get_path_nested_object(): @@ -61,3 +68,90 @@ def test_pick_omit_passthrough_non_objects(): assert pick_keys(42, ["a"]) == 42 assert omit_keys("hi", ["a"]) == "hi" assert pick_keys([1, {"a": 1, "b": 2}], ["a"]) == [1, {"a": 1}] + + +def test_filter_eq_and_numeric_ge(): + data = [ + {"name": "Ada", "age": 36}, + {"name": "Bob", "age": 22}, + {"name": "Cy", "age": "30"}, + ] + assert filter_rows(data, "name", "eq", "Ada") == [{"name": "Ada", "age": 36}] + assert filter_rows(data, "age", "ge", "30") == [ + {"name": "Ada", "age": 36}, + {"name": "Cy", "age": "30"}, + ] + + +def test_filter_ne_lt_contains_exists(): + data = [ + {"id": 1, "role": "admin", "note": "x"}, + {"id": 2, "role": "user"}, + {"id": 3, "role": "guest", "note": None}, + "skip-me", + ] + assert filter_rows(data, "id", "ne", "1") == [ + {"id": 2, "role": "user"}, + {"id": 3, "role": "guest", "note": None}, + ] + assert filter_rows(data, "id", "lt", "2") == [ + {"id": 1, "role": "admin", "note": "x"} + ] + assert filter_rows(data, "role", "contains", "adm") == [ + {"id": 1, "role": "admin", "note": "x"} + ] + assert filter_rows(data, "note", "exists") == [ + {"id": 1, "role": "admin", "note": "x"} + ] + + +def test_filter_drops_non_objects_and_requires_list(): + assert filter_rows([1, {"a": 1}, None], "a", "eq", "1") == [{"a": 1}] + with pytest.raises(ValueError, match="root must be a list"): + filter_rows({"a": 1}, "a", "eq", "1") + with pytest.raises(ValueError, match="Unsupported filter operator"): + filter_rows([{"a": 1}], "a", "bogus", "1") + + +def test_sort_asc_desc_and_missing_last(): + data = [{"id": 2}, {"id": 1}, {"id": 3}, {"name": "x"}] + assert sort_rows(data, "id") == [ + {"id": 1}, + {"id": 2}, + {"id": 3}, + {"name": "x"}, + ] + assert sort_rows(data, "id", desc=True) == [ + {"name": "x"}, + {"id": 3}, + {"id": 2}, + {"id": 1}, + ] + + +def test_sort_requires_list(): + with pytest.raises(ValueError, match="root must be a list"): + sort_rows({"id": 1}, "id") + + +def test_unique_whole_and_by_key(): + data = [ + {"id": 1, "n": "a"}, + {"id": 1, "n": "b"}, + {"id": 2, "n": "c"}, + {"id": 1, "n": "a"}, + ] + assert unique_rows(data) == [ + {"id": 1, "n": "a"}, + {"id": 1, "n": "b"}, + {"id": 2, "n": "c"}, + ] + assert unique_rows(data, "id") == [ + {"id": 1, "n": "a"}, + {"id": 2, "n": "c"}, + ] + + +def test_unique_requires_list(): + with pytest.raises(ValueError, match="root must be a list"): + unique_rows({"id": 1})