Add filter, sort, and unique shaping commands (v0.3.0)

README.md

@@ -28,13 +28,17 @@ After install, the `data-transform` console script is on your `PATH`.
 | `select` | Extract dotted paths from parsed data |
 | `pick` | Keep only listed top-level keys |
 | `omit` | Drop listed top-level keys |
+| `filter` | Keep array items matching KEY OP VALUE |
+| `sort` | Sort array of objects by top-level KEY |
+| `unique` | Deduplicate array (by KEY or whole item) |
 
 ### Options (most commands)
 
 - `[file]` — input path; omit or use `-` for stdin
 - `-o, --output <file>` — write to a file instead of stdout
 - `-f, --from <format>` — force input format: `json` | `yaml` | `csv` | `ndjson`
-- `--minify` — compact JSON output (`to-json`, `select`, `pick`, `omit`)
+- `--minify` — compact JSON output (`to-json`, `select`, `pick`, `omit`, `filter`, `sort`, `unique`)
+- `--desc` — reverse sort order (`sort` only)
 
 Format is taken from `--from`, else the file extension (`.ndjson` / `.jsonl` → ndjson), else simple content heuristics.
 
@@ -92,16 +96,68 @@ echo '{"id":1,"secret":"x","name":"a"}' \
 # → {"id":1,"name":"a"}
 ```
 
+### Filter, sort, unique
+
+These commands require a **list** root (JSON/YAML/CSV/NDJSON array). Non-object
+items are dropped by `filter`. Missing keys sort last (first with `--desc`).
+
+```bash
+# Keep rows where age >= 30 (numeric when both sides look like numbers)
+echo '[{"name":"Ada","age":36},{"name":"Bob","age":22}]' \
+  | data-transform filter age ge 30 -f json --minify
+# → [{"name":"Ada","age":36}]
+
+# String contains / key exists (VALUE optional for exists)
+echo '[{"role":"admin"},{"role":"user"}]' \
+  | data-transform filter role contains adm -f json --minify
+# → [{"role":"admin"}]
+
+echo '[{"id":1,"note":"x"},{"id":2}]' \
+  | data-transform filter note exists -f json --minify
+# → [{"id":1,"note":"x"}]
+
+# Sort by key (ascending / descending)
+echo '[{"id":2},{"id":1},{"id":3}]' \
+  | data-transform sort id -f json --minify
+# → [{"id":1},{"id":2},{"id":3}]
+
+echo '[{"id":2},{"id":1}]' \
+  | data-transform sort id --desc -f json --minify
+# → [{"id":2},{"id":1}]
+
+# Deduplicate whole items, or by a key (first wins)
+echo '[{"id":1},{"id":1},{"id":2}]' \
+  | data-transform unique id -f json --minify
+# → [{"id":1},{"id":2}]
+
+echo '[{"a":1},{"a":1},{"b":2}]' \
+  | data-transform unique -f json --minify
+# → [{"a":1},{"b":2}]
+```
+
 ### Library use
 
 ```python
-from data_transform import convert, parse, serialize, get_path, pick_keys, omit_keys
+from data_transform import (
+    convert,
+    parse,
+    serialize,
+    get_path,
+    pick_keys,
+    omit_keys,
+    filter_rows,
+    sort_rows,
+    unique_rows,
+)
 
 yaml_text = convert('{"a":1}', "json", "yaml")
 data = parse("id,name\n1,x\n", "csv")
 json_text = serialize(data, "json", pretty=True)
 name = get_path({"user": {"name": "Ada"}}, "user.name")
 slim = pick_keys({"id": 1, "name": "a", "note": "x"}, ["id", "name"])
+adults = filter_rows([{"age": 36}, {"age": 22}], "age", "ge", "30")
+ordered = sort_rows([{"id": 2}, {"id": 1}], "id")
+deduped = unique_rows([{"id": 1}, {"id": 1}], "id")
 ```
 
 ## NDJSON notes

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "data-transform-cli"
-version = "0.2.0"
+version = "0.3.0"
 description = "CLI for YAML ↔ JSON ↔ CSV ↔ NDJSON transforms for pipelines and shell use"
 readme = "README.md"
 license = { text = "MIT" }

src/data_transform/__init__.py

@@ -2,7 +2,14 @@
 
 from .convert import convert, detect_format, infer_format, parse, serialize
 from .csv_io import escape_csv_field, parse_csv, parse_csv_rows, stringify_csv
-from .shape import get_path, omit_keys, pick_keys
+from .shape import (
+    filter_rows,
+    get_path,
+    omit_keys,
+    pick_keys,
+    sort_rows,
+    unique_rows,
+)
 
 __all__ = [
     "convert",
@@ -17,6 +24,9 @@
     "get_path",
     "pick_keys",
     "omit_keys",
+    "filter_rows",
+    "sort_rows",
+    "unique_rows",
 ]
 
-__version__ = "0.2.0"
+__version__ = "0.3.0"

src/data_transform/cli.py

@@ -10,7 +10,7 @@
 
 from . import __version__
 from .convert import convert, detect_format, infer_format, parse, serialize
-from .shape import get_path, omit_keys, pick_keys
+from .shape import filter_rows, get_path, omit_keys, pick_keys, sort_rows, unique_rows
 
 FORMAT_CHOICES = ["json", "yaml", "csv", "ndjson"]
 
@@ -338,5 +338,153 @@ def omit_cmd(
         sys.exit(1)
 
 
+
+@main.command("filter")
+@click.argument("tokens", nargs=-1, required=True)
+@click.option("-o", "--output", default=None, help="write to file instead of stdout")
+@click.option(
+    "-f",
+    "--from",
+    "from_format",
+    type=click.Choice(FORMAT_CHOICES, case_sensitive=False),
+    default=None,
+    help="input format: json | yaml | csv | ndjson",
+)
+@click.option("--minify", is_flag=True, default=False, help="emit compact JSON")
+def filter_cmd(
+    tokens: tuple[str, ...],
+    output: Optional[str],
+    from_format: Optional[str],
+    minify: bool,
+) -> None:
+    """Keep array items where KEY OP VALUE matches.
+
+    Operators: eq, ne, gt, ge, lt, le, contains, exists.
+    Non-object items are dropped. Root must be a list.
+
+    Usage: data-transform filter KEY OP VALUE [file]
+    """
+    try:
+        parts, input_file = _split_tokens_and_file(tokens)
+        if len(parts) < 2:
+            raise ValueError("Usage: filter KEY OP [VALUE] [file]")
+        key, op = parts[0], parts[1]
+        op_l = op.lower()
+        if op_l == "exists":
+            if len(parts) > 3:
+                raise ValueError("Usage: filter KEY exists [file]")
+            # Optional ignored VALUE may appear as third token
+            value = parts[2] if len(parts) == 3 else None
+        else:
+            if len(parts) != 3:
+                raise ValueError("Usage: filter KEY OP VALUE [file]")
+            value = parts[2]
+
+        text, name = load_input(input_file)
+        src = resolve_from(text, name, from_format)
+        data = parse(text, src)  # type: ignore[arg-type]
+        result = filter_rows(data, key, op, value)
+        write_output(
+            serialize(result, "json", pretty=not minify, minify=minify), output
+        )
+    except Exception as err:  # noqa: BLE001
+        click.echo(f"error: {err}", err=True)
+        sys.exit(1)
+
+
+@main.command("sort")
+@click.argument("tokens", nargs=-1, required=True)
+@click.option("-o", "--output", default=None, help="write to file instead of stdout")
+@click.option(
+    "-f",
+    "--from",
+    "from_format",
+    type=click.Choice(FORMAT_CHOICES, case_sensitive=False),
+    default=None,
+    help="input format: json | yaml | csv | ndjson",
+)
+@click.option("--minify", is_flag=True, default=False, help="emit compact JSON")
+@click.option("--desc", is_flag=True, default=False, help="sort descending")
+def sort_cmd(
+    tokens: tuple[str, ...],
+    output: Optional[str],
+    from_format: Optional[str],
+    minify: bool,
+    desc: bool,
+) -> None:
+    """Sort an array of objects by top-level KEY.
+
+    Missing keys sort last (first with --desc). Root must be a list.
+
+    Usage: data-transform sort KEY [file]
+    """
+    try:
+        parts, input_file = _split_tokens_and_file(tokens)
+        if len(parts) != 1:
+            raise ValueError("Usage: sort KEY [file]")
+        key = parts[0]
+
+        text, name = load_input(input_file)
+        src = resolve_from(text, name, from_format)
+        data = parse(text, src)  # type: ignore[arg-type]
+        result = sort_rows(data, key, desc=desc)
+        write_output(
+            serialize(result, "json", pretty=not minify, minify=minify), output
+        )
+    except Exception as err:  # noqa: BLE001
+        click.echo(f"error: {err}", err=True)
+        sys.exit(1)
+
+
+@main.command("unique")
+@click.argument("tokens", nargs=-1, required=False)
+@click.option("-o", "--output", default=None, help="write to file instead of stdout")
+@click.option(
+    "-f",
+    "--from",
+    "from_format",
+    type=click.Choice(FORMAT_CHOICES, case_sensitive=False),
+    default=None,
+    help="input format: json | yaml | csv | ndjson",
+)
+@click.option("--minify", is_flag=True, default=False, help="emit compact JSON")
+def unique_cmd(
+    tokens: tuple[str, ...],
+    output: Optional[str],
+    from_format: Optional[str],
+    minify: bool,
+) -> None:
+    """Deduplicate an array (order-preserving, first wins).
+
+    With KEY, dedupe by that top-level key. Without KEY, dedupe by whole item.
+    Root must be a list.
+
+    Usage: data-transform unique [KEY] [file]
+    """
+    try:
+        parts, input_file = _split_tokens_and_file(tokens)
+        # If a single token looks like a file and was not peeled off (e.g. path
+        # that does not exist yet / stdin already handled), treat it as KEY.
+        # When tokens was only a file path that exists, _split peels it and
+        # parts is empty → no key.
+        key: Optional[str] = None
+        if len(parts) == 1:
+            key = parts[0]
+        elif len(parts) > 1:
+            raise ValueError("Usage: unique [KEY] [file]")
+
+        text, name = load_input(input_file)
+        src = resolve_from(text, name, from_format)
+        data = parse(text, src)  # type: ignore[arg-type]
+        result = unique_rows(data, key)
+        write_output(
+            serialize(result, "json", pretty=not minify, minify=minify), output
+        )
+    except Exception as err:  # noqa: BLE001
+        click.echo(f"error: {err}", err=True)
+        sys.exit(1)
+
+
+
 if __name__ == "__main__":
     main()