Add rename, flatten, and head/tail commands (0.4.0)

README.md

@@ -28,17 +28,22 @@ After install, the `data-transform` console script is on your `PATH`.
 | `select` | Extract dotted paths from parsed data |
 | `pick` | Keep only listed top-level keys |
 | `omit` | Drop listed top-level keys |
+| `rename` | Rename top-level keys (`OLD NEW` pairs) |
+| `flatten` | Flatten nested objects into joined keys |
 | `filter` | Keep array items matching KEY OP VALUE |
 | `sort` | Sort array of objects by top-level KEY |
 | `unique` | Deduplicate array (by KEY or whole item) |
+| `head` | First N items of a list |
+| `tail` | Last N items of a list |
 
 ### Options (most commands)
 
 - `[file]` — input path; omit or use `-` for stdin
 - `-o, --output <file>` — write to a file instead of stdout
 - `-f, --from <format>` — force input format: `json` | `yaml` | `csv` | `ndjson`
-- `--minify` — compact JSON output (`to-json`, `select`, `pick`, `omit`, `filter`, `sort`, `unique`)
+- `--minify` — compact JSON output (`to-json`, `select`, `pick`, `omit`, `rename`, `flatten`, `filter`, `sort`, `unique`, `head`, `tail`)
 - `--desc` — reverse sort order (`sort` only)
+- `--sep <text>` — separator between nested keys (`flatten` only, default `.`)
 
 Format is taken from `--from`, else the file extension (`.ndjson` / `.jsonl` → ndjson), else simple content heuristics.
 
@@ -96,6 +101,34 @@ echo '{"id":1,"secret":"x","name":"a"}' \
 # → {"id":1,"name":"a"}
 ```
 
+### Rename and flatten
+
+`rename` takes pairs of `OLD NEW` and maps over an object or each object in an array, the same way `pick` and `omit` do. Missing old keys are left alone. A later pair wins when the same old key is given twice. Renames are one pass over the original keys: `a→b` together with `b→c` leaves the value of `a` under `b`.
+
+`flatten` collapses nested objects into top-level keys joined by `--sep` (default `.`). A list root is flattened item by item. Nested lists stay as values. Empty objects stay empty. A number, string, or other non-object root is returned unchanged.
+
+```bash
+# Rename top-level keys (several pairs)
+echo '{"id":1,"name":"Ada","note":"x"}' \
+  | data-transform rename name title note comment -f json --minify
+# → {"id":1,"title":"Ada","comment":"x"}
+
+# Same mapping over an array; missing keys are a no-op
+echo '[{"name":"Ada"},{"id":2},1]' \
+  | data-transform rename name title -f json --minify
+# → [{"title":"Ada"},{"id":2},1]
+
+# Flatten nested objects; lists stay as values
+echo '{"user":{"name":"Ada","city":"London"},"tags":["a","b"]}' \
+  | data-transform flatten -f json --minify
+# → {"user.name":"Ada","user.city":"London","tags":["a","b"]}
+
+# Custom separator
+echo '{"user":{"name":"Ada"}}' \
+  | data-transform flatten --sep _ -f json --minify
+# → {"user_name":"Ada"}
+```
+
 ### Filter, sort, unique
 
 These commands require a **list** root (JSON/YAML/CSV/NDJSON array). Non-object
@@ -135,6 +168,23 @@ echo '[{"a":1},{"a":1},{"b":2}]' \
 # → [{"a":1},{"b":2}]
 ```
 
+### Head and tail
+
+`head` and `tail` require a **list** root, same as `filter`, `sort`, and `unique`. `N` is a non-negative integer. A count past the end returns the items that exist; `0` returns an empty list.
+
+```bash
+echo '[{"id":1},{"id":2},{"id":3}]' \
+  | data-transform head 2 -f json --minify
+# → [{"id":1},{"id":2}]
+
+echo '[{"id":1},{"id":2},{"id":3}]' \
+  | data-transform tail 1 -f json --minify
+# → [{"id":3}]
+
+echo '[1,2,3]' | data-transform head 0 -f json --minify
+# → []
+```
+
 ### Library use
 
 ```python
@@ -145,19 +195,27 @@ from data_transform import (
     get_path,
     pick_keys,
     omit_keys,
+    rename_keys,
+    flatten_object,
     filter_rows,
     sort_rows,
     unique_rows,
+    head_rows,
+    tail_rows,
 )
 
 yaml_text = convert('{"a":1}', "json", "yaml")
 data = parse("id,name\n1,x\n", "csv")
 json_text = serialize(data, "json", pretty=True)
 name = get_path({"user": {"name": "Ada"}}, "user.name")
 slim = pick_keys({"id": 1, "name": "a", "note": "x"}, ["id", "name"])
+renamed = rename_keys({"id": 1, "name": "a"}, {"name": "title"})
+flat = flatten_object({"user": {"name": "Ada"}, "tags": ["x"]})
 adults = filter_rows([{"age": 36}, {"age": 22}], "age", "ge", "30")
 ordered = sort_rows([{"id": 2}, {"id": 1}], "id")
 deduped = unique_rows([{"id": 1}, {"id": 1}], "id")
+first = head_rows([{"id": 1}, {"id": 2}, {"id": 3}], 2)
+last = tail_rows([{"id": 1}, {"id": 2}, {"id": 3}], 1)
 ```
 
 ## NDJSON notes

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "data-transform-cli"
-version = "0.3.0"
+version = "0.4.0"
 description = "CLI for YAML ↔ JSON ↔ CSV ↔ NDJSON transforms for pipelines and shell use"
 readme = "README.md"
 license = { text = "MIT" }

src/data_transform/__init__.py

@@ -4,10 +4,14 @@
 from .csv_io import escape_csv_field, parse_csv, parse_csv_rows, stringify_csv
 from .shape import (
     filter_rows,
+    flatten_object,
     get_path,
+    head_rows,
     omit_keys,
     pick_keys,
+    rename_keys,
     sort_rows,
+    tail_rows,
     unique_rows,
 )
 
@@ -24,9 +28,13 @@
     "get_path",
     "pick_keys",
     "omit_keys",
+    "rename_keys",
+    "flatten_object",
     "filter_rows",
     "sort_rows",
     "unique_rows",
+    "head_rows",
+    "tail_rows",
 ]
 
-__version__ = "0.3.0"
+__version__ = "0.4.0"

src/data_transform/cli.py

@@ -10,7 +10,18 @@
 
 from . import __version__
 from .convert import convert, detect_format, infer_format, parse, serialize
-from .shape import filter_rows, get_path, omit_keys, pick_keys, sort_rows, unique_rows
+from .shape import (
+    filter_rows,
+    flatten_object,
+    get_path,
+    head_rows,
+    omit_keys,
+    pick_keys,
+    rename_keys,
+    sort_rows,
+    tail_rows,
+    unique_rows,
+)
 
 FORMAT_CHOICES = ["json", "yaml", "csv", "ndjson"]
 
@@ -44,6 +55,25 @@ def write_output(out: str, output_path: Optional[str]) -> None:
         sys.stdout.write(out)
 
 
+def _parse_count(token: str, command: str) -> int:
+    """Parse a non-negative integer count from a CLI token."""
+    if token == "" or any(ch < "0" or ch > "9" for ch in token):
+        raise ValueError(
+            f"{command}: N must be a non-negative integer, got {token!r}"
+        )
+    return int(token)
+
+
+def _parse_rename_pairs(parts: Sequence[str]) -> dict[str, str]:
+    """Build an OLD→NEW map from an even list of tokens. Later pairs win."""
+    if len(parts) < 2 or len(parts) % 2 != 0:
+        raise ValueError("Usage: rename OLD NEW [OLD NEW ...] [file]")
+    mapping: dict[str, str] = {}
+    for index in range(0, len(parts), 2):
+        mapping[parts[index]] = parts[index + 1]
+    return mapping
+
+
 def _split_tokens_and_file(tokens: Sequence[str]) -> tuple[list[str], Optional[str]]:
     """
     Split trailing optional file from PATH/KEY tokens.
@@ -485,6 +515,178 @@ def unique_cmd(
         sys.exit(1)
 
 
+@main.command("rename")
+@click.argument("tokens", nargs=-1, required=True)
+@click.option("-o", "--output", default=None, help="write to file instead of stdout")
+@click.option(
+    "-f",
+    "--from",
+    "from_format",
+    type=click.Choice(FORMAT_CHOICES, case_sensitive=False),
+    default=None,
+    help="input format: json | yaml | csv | ndjson",
+)
+@click.option("--minify", is_flag=True, default=False, help="emit compact JSON")
+def rename_cmd(
+    tokens: tuple[str, ...],
+    output: Optional[str],
+    from_format: Optional[str],
+    minify: bool,
+) -> None:
+    """Rename top-level keys (OLD NEW pairs) on an object or each object in an array.
+
+    Missing OLD keys are left unchanged. Non-object items in an array are
+    unchanged. Later pairs win when OLD is repeated.
+
+    Usage: data-transform rename OLD NEW [OLD NEW ...] [file]
+    """
+    try:
+        parts, input_file = _split_tokens_and_file(tokens)
+        mapping = _parse_rename_pairs(parts)
+
+        text, name = load_input(input_file)
+        src = resolve_from(text, name, from_format)
+        data = parse(text, src)  # type: ignore[arg-type]
+        result = rename_keys(data, mapping)
+        write_output(
+            serialize(result, "json", pretty=not minify, minify=minify), output
+        )
+    except Exception as err:  # noqa: BLE001
+        click.echo(f"error: {err}", err=True)
+        sys.exit(1)
+
+
+@main.command("flatten")
+@click.argument("file", required=False)
+@click.option("-o", "--output", default=None, help="write to file instead of stdout")
+@click.option(
+    "-f",
+    "--from",
+    "from_format",
+    type=click.Choice(FORMAT_CHOICES, case_sensitive=False),
+    default=None,
+    help="input format: json | yaml | csv | ndjson",
+)
+@click.option("--minify", is_flag=True, default=False, help="emit compact JSON")
+@click.option(
+    "--sep",
+    default=".",
+    show_default=True,
+    help="separator between nested keys",
+)
+def flatten_cmd(
+    file: Optional[str],
+    output: Optional[str],
+    from_format: Optional[str],
+    minify: bool,
+    sep: str,
+) -> None:
+    """Flatten nested objects into separator-joined top-level keys.
+
+    Maps over a list root. Nested lists stay as values. Empty objects stay
+    empty. A non-object, non-list root is unchanged.
+
+    Usage: data-transform flatten [file]
+    """
+    try:
+        text, name = load_input(file)
+        src = resolve_from(text, name, from_format)
+        data = parse(text, src)  # type: ignore[arg-type]
+        result = flatten_object(data, sep)
+        write_output(
+            serialize(result, "json", pretty=not minify, minify=minify), output
+        )
+    except Exception as err:  # noqa: BLE001
+        click.echo(f"error: {err}", err=True)
+        sys.exit(1)
+
+
+def _run_take(
+    command: str,
+    tokens: Sequence[str],
+    output: Optional[str],
+    from_format: Optional[str],
+    minify: bool,
+) -> None:
+    """Shared runner for head/tail."""
+    try:
+        parts, input_file = _split_tokens_and_file(tokens)
+        if len(parts) != 1:
+            raise ValueError(f"Usage: {command} N [file]")
+        count = _parse_count(parts[0], command)
+
+        text, name = load_input(input_file)
+        src = resolve_from(text, name, from_format)
+        data = parse(text, src)  # type: ignore[arg-type]
+        if command == "head":
+            result = head_rows(data, count)
+        else:
+            result = tail_rows(data, count)
+        write_output(
+            serialize(result, "json", pretty=not minify, minify=minify), output
+        )
+    except Exception as err:  # noqa: BLE001
+        click.echo(f"error: {err}", err=True)
+        sys.exit(1)
+
+
+# ignore_unknown_options lets a leading "-1" through as N so the count check
+# can reject it, instead of Click treating it as a short flag.
+@main.command("head", context_settings={"ignore_unknown_options": True})
+@click.argument("tokens", nargs=-1, required=True)
+@click.option("-o", "--output", default=None, help="write to file instead of stdout")
+@click.option(
+    "-f",
+    "--from",
+    "from_format",
+    type=click.Choice(FORMAT_CHOICES, case_sensitive=False),
+    default=None,
+    help="input format: json | yaml | csv | ndjson",
+)
+@click.option("--minify", is_flag=True, default=False, help="emit compact JSON")
+def head_cmd(
+    tokens: tuple[str, ...],
+    output: Optional[str],
+    from_format: Optional[str],
+    minify: bool,
+) -> None:
+    """Return the first N items of a list.
+
+    N must be a non-negative integer. Counts past the end return the whole
+    list. Root must be a list.
+
+    Usage: data-transform head N [file]
+    """
+    _run_take("head", tokens, output, from_format, minify)
+
+
+@main.command("tail", context_settings={"ignore_unknown_options": True})
+@click.argument("tokens", nargs=-1, required=True)
+@click.option("-o", "--output", default=None, help="write to file instead of stdout")
+@click.option(
+    "-f",
+    "--from",
+    "from_format",
+    type=click.Choice(FORMAT_CHOICES, case_sensitive=False),
+    default=None,
+    help="input format: json | yaml | csv | ndjson",
+)
+@click.option("--minify", is_flag=True, default=False, help="emit compact JSON")
+def tail_cmd(
+    tokens: tuple[str, ...],
+    output: Optional[str],
+    from_format: Optional[str],
+    minify: bool,
+) -> None:
+    """Return the last N items of a list.
+
+    N must be a non-negative integer. Counts past the end return the whole
+    list. Root must be a list.
+
+    Usage: data-transform tail N [file]
+    """
+    _run_take("tail", tokens, output, from_format, minify)
+
 
 if __name__ == "__main__":
     main()