diff --git a/apps/docs/content/docs/en/platform/enterprise/data-retention.mdx b/apps/docs/content/docs/en/platform/enterprise/data-retention.mdx index ecd4c4321d4..c54118f790d 100644 --- a/apps/docs/content/docs/en/platform/enterprise/data-retention.mdx +++ b/apps/docs/content/docs/en/platform/enterprise/data-retention.mdx @@ -1,12 +1,18 @@ --- title: Data Retention -description: Control how long execution logs, deleted resources, and copilot data are kept before permanent deletion +description: Control how long execution logs, deleted resources, and Chat data are kept before permanent deletion — and redact PII from workflow data --- +import { Callout } from 'fumadocs-ui/components/callout' import { FAQ } from '@/components/ui/faq' import { Image } from '@/components/ui/image' -Data Retention lets organization owners and admins on Enterprise plans configure how long three categories of data are kept before they are permanently deleted. The configuration applies to every workspace in the organization. +Data Retention lets organization owners and admins on Enterprise plans control two things: + +1. **Retention periods** — how long execution logs, soft-deleted resources, and Chat data are kept before they are permanently deleted. +2. **PII redaction** — masking personally identifiable information out of your workflow data at up to three points: workflow input, block outputs, and logs. + +Both are configured once at the **organization level** and apply to every workspace, with optional **per-workspace overrides** for workspaces that need different rules. --- @@ -14,21 +20,31 @@ Data Retention lets organization owners and admins on Enterprise plans configure Go to **Settings → Enterprise → Data Retention** in your workspace. -Data Retention settings showing three dropdowns — Log retention, Soft deletion cleanup, and Task cleanup — each set to Forever +Data Retention settings showing the Retention policies list with the Organization default row and its summary of retention periods and PII stages + +The page shows your **retention policies** as a list: -You will see three independent settings, each with the same set of options: **1 day, 3 days, 7 days, 14 days, 30 days, 60 days, 90 days, 180 days, 1 year, 5 years,** or **Forever**. +- The **Organization** row (tagged *Default*) holds the settings that apply to every workspace without its own override. +- Each **workspace override** row below it applies to one or more specific workspaces. -Setting a period to **Forever** means that category of data is never automatically deleted. +Open a row to edit it, or click **Add override** to create a workspace override. Each policy has a **Retention** section, and — when PII redaction is enabled — a **PII redaction** section. --- -## Settings +## Retention periods + +Each policy has three independent retention settings, each with the same set of options: **1 day, 3 days, 7 days, 14 days, 30 days, 60 days, 90 days, 180 days, 1 year, 5 years,** or **Forever**. + +Setting a period to **Forever** means that category of data is never automatically deleted. On a workspace override, each field can also be set to **Inherit from organization** to fall back to the organization default for just that field. ### Log retention -Controls how long **workflow execution logs** are kept. +Controls how long **execution logs** are kept. -When the retention period expires, execution log records are permanently deleted, along with any files associated with those executions stored in cloud storage. +When the retention period expires, log records are permanently deleted, along with any files associated with those executions in cloud storage. This covers: + +- Workflow execution logs +- Background job logs (deployed APIs, schedules, and webhooks) ### Soft deletion cleanup @@ -40,33 +56,83 @@ Resources covered: - Workflows - Workflow folders -- Knowledge bases +- Knowledge bases (and their documents) - Tables - Files - MCP server configurations - Agent memory +- Chat conversations ### Task cleanup -Controls how long **Mothership data** is kept, including: +Controls how long **Chat data** is kept, including: -- Copilot chats and run history +- Chat conversations and run history - Run checkpoints and async tool calls -- Inbox tasks (Sim Mailer) +- Inbox tasks Each setting is independent. You can configure a short log retention period alongside a long soft deletion cleanup period, or any combination that fits your compliance requirements. --- -## Organization-wide configuration +## PII redaction + +When PII redaction is enabled for your organization, each policy gains a **PII redaction** section that masks personally identifiable information — names, emails, phone numbers, credit-card numbers, national IDs, and more — from your workflow data. Sim detects and masks PII with [Microsoft Presidio](https://microsoft.github.io/presidio/); each match is replaced with a placeholder token such as ``. + +Redaction is configured per **stage** — the point in a run where masking is applied. Select a stage, then choose which entity types and custom patterns to redact for it: + +| Stage | What it does | +|---|---| +| **Logs** | Redacts workflow logs when they are persisted. Observability-only — the workflow still runs on the original data. | +| **Workflow input** | Redacts the workflow input **before execution**. The workflow runs on the masked data, which may change its output. | +| **Block outputs** | Masks every block output **before the next block reads it**. Runs in-flight and may change output and execution performance. | + +PII redaction section with the Block outputs stage selected, showing the entity type grid grouped by Common, United States, United Kingdom, and Other regions + + +The **Workflow input** and **Block outputs** stages alter what the workflow computes on, not just what is stored. Redacted data is masked during the run and may affect workflow output. Enable them only where that trade-off is acceptable. + + +### Entity types and language + +For each stage, choose the **entity types** to redact from the searchable grid. They are grouped as: + +- **Common** — person name, email, phone, credit card, IP address, URL, IBAN, crypto wallet, medical license, VIN +- **United States** — SSN, passport, driver's license, bank account, ITIN +- **United Kingdom** — NHS number, National Insurance number +- **Other regions** — Singapore, Australian, and Indian identifiers + +The **Block outputs** stage is restricted to regex- and checksum-based recognizers, so it can run in-flight over large payloads without a performance penalty. Types that need name-model detection — person name, location, date or time — are not offered for that stage. + +Detection is language-aware: pick the **language** whose recognizers should apply. English, Spanish, Italian, Polish, and Finnish are supported, and the grid filters to the identifiers available for the selected language. + + +Some recognizers match loosely and over-redact — US Social Security Number, US bank account number, and Date or time have no checksum and match aggressively. Enable these only where false positives are acceptable. + + +### Custom patterns -Retention is configured at the **organization level**. A single configuration applies to every workspace in the organization — there are no per-workspace overrides. +Beyond the built-in entity types, each stage can redact anything a **regular expression** matches — employee IDs, internal URLs, ticket numbers. Give each pattern a name, a regex, and a replacement token; every match is replaced with the replacement text wrapped in angle brackets (e.g. `EMPLOYEE_ID` → ``). + +--- + +## Per-workspace overrides + +Retention and PII redaction are configured at the **organization level** and apply to every workspace by default. When a workspace needs different rules, add a **workspace override**. + +Add workspace override panel showing the workspace picker, retention fields set to Inherit from organization, and the PII redaction Inherit or Override switch + +- An override targets one or more workspaces (a workspace can belong to only one override). +- Each **retention** field either sets its own period or **inherits** the organization value. +- **PII redaction** on an override is either **Inherit** (use the organization's redaction) or **Override** (workspace-specific). Choosing Override replaces the organization's redaction rules entirely for that workspace — it is not merged stage-by-stage. + +Removing an override returns its workspaces to the organization defaults. --- ## Defaults -By default, all three settings are unconfigured — no data is automatically deleted in any category until you configure it. Setting a period to **Forever** has the same effect as leaving it unconfigured, but makes the intent explicit and allows you to change it later without saving from scratch. +By default, retention settings are unconfigured — no data is automatically deleted in any category, and no PII is redacted, until you configure it. Setting a retention period to **Forever** has the same effect as leaving it unconfigured, but makes the intent explicit and lets you change it later without configuring from scratch. --- @@ -84,13 +150,17 @@ By default, all three settings are unconfigured — no data is automatically del answer: "No. Once the soft deletion cleanup period expires and the cleanup job runs, resources are permanently deleted and cannot be recovered." }, { - question: "Does the retention period apply to all workspaces in my organization?", - answer: "Yes. Retention is configured once per organization and applies to every workspace in the organization." + question: "Does a policy apply to all workspaces?", + answer: "The organization policy applies to every workspace that does not have its own override. Add a per-workspace override to give specific workspaces different retention periods or PII redaction rules." }, { question: "What happens if I shorten the retention period?", answer: "The next cleanup job will delete any data that is older than the new, shorter period — including data that would have been kept under the previous setting. Shortening the period is irreversible for data that falls outside the new window." }, + { + question: "Does PII redaction change how my workflows run?", + answer: "The Logs stage does not — it only masks data as it is persisted, so the workflow runs on the original data. The Workflow input and Block outputs stages do: they mask data in-flight, so the workflow computes on the redacted values, which can change its output." + }, { question: "What is the minimum retention period?", answer: "1 day (24 hours)." @@ -105,10 +175,28 @@ By default, all three settings are unconfigured — no data is automatically del ## Self-hosted setup -### Environment variables +### Retention periods ```bash NEXT_PUBLIC_DATA_RETENTION_ENABLED=true +DATA_RETENTION_ENABLED=true +``` + +Once enabled, retention settings are configurable through **Settings → Enterprise → Data Retention** the same way as Sim Cloud. + +### PII redaction + +PII redaction runs against a standalone [Presidio](https://microsoft.github.io/presidio/) service. Deploy it (see `apps/pii`) and point Sim at it, then enable the redaction surfaces: + +```bash +# The Presidio service exposing /analyze and /anonymize +PII_URL=http://localhost:5001 + +# Expose the log-redaction stage and the Data Retention PII section +PII_REDACTION=true + +# Additionally expose the execution-altering stages (Workflow input, Block outputs) +PII_GRANULAR_REDACTION=true ``` -Once enabled, data retention settings are configurable through **Settings → Enterprise → Data Retention** the same way as Sim Cloud. +`PII_GRANULAR_REDACTION` layers on top of `PII_REDACTION` — with only `PII_REDACTION` enabled, just the **Logs** stage is configurable. diff --git a/apps/docs/public/static/enterprise/data-retention-override.png b/apps/docs/public/static/enterprise/data-retention-override.png new file mode 100644 index 00000000000..eb5771c67e5 Binary files /dev/null and b/apps/docs/public/static/enterprise/data-retention-override.png differ diff --git a/apps/docs/public/static/enterprise/data-retention-pii.png b/apps/docs/public/static/enterprise/data-retention-pii.png new file mode 100644 index 00000000000..f35688da708 Binary files /dev/null and b/apps/docs/public/static/enterprise/data-retention-pii.png differ diff --git a/apps/docs/public/static/enterprise/data-retention.png b/apps/docs/public/static/enterprise/data-retention.png index 1b8d559e764..92e3a510b44 100644 Binary files a/apps/docs/public/static/enterprise/data-retention.png and b/apps/docs/public/static/enterprise/data-retention.png differ diff --git a/apps/sim/ee/data-retention/components/data-retention-settings.tsx b/apps/sim/ee/data-retention/components/data-retention-settings.tsx index 666088d3a69..898123fc7b4 100644 --- a/apps/sim/ee/data-retention/components/data-retention-settings.tsx +++ b/apps/sim/ee/data-retention/components/data-retention-settings.tsx @@ -101,27 +101,49 @@ interface EditingPolicy { isNew: boolean } +/** Day bounds the retention contract accepts (1 day … 5 years). */ +const MIN_RETENTION_DAYS = 1 +const MAX_RETENTION_DAYS = 1825 + +/** + * Hours → display days, clamped to the contract's range. Sub-day values would + * otherwise round to `0` and be re-sent as `0`, wedging every save on the page. + */ +function clampDisplayDays(hours: number): string { + const days = Math.round(hours / 24) + return String(Math.min(MAX_RETENTION_DAYS, Math.max(MIN_RETENTION_DAYS, days))) +} + +/** Day count → hours. Throws rather than send a value the contract rejects. */ +function toRetentionHours(days: string): number { + const parsed = Number(days) + if (!Number.isFinite(parsed) || parsed < MIN_RETENTION_DAYS) { + throw new Error(`Invalid retention period: ${JSON.stringify(days)}`) + } + return Math.min(MAX_RETENTION_DAYS, Math.round(parsed)) * 24 +} + function hoursToDisplayDays(hours: number | null): string { if (hours === null) return 'never' - return String(Math.round(hours / 24)) + return clampDisplayDays(hours) } function daysToHours(days: string): number | null { if (days === 'never') return null - return Number(days) * 24 + return toRetentionHours(days) } /** Override field: `INHERIT` ⇄ undefined, `'never'` ⇄ null (forever), day count ⇄ hours. */ function hoursToOverrideValue(hours: number | null | undefined): string { if (hours === undefined) return INHERIT if (hours === null) return 'never' - return String(Math.round(hours / 24)) + return clampDisplayDays(hours) } function overrideValueToHours(value: string): number | null | undefined { if (value === INHERIT) return undefined if (value === 'never') return null - return Number(value) * 24 + return toRetentionHours(value) } function buildRetentionOverride(workspaceId: string, draft: PolicyDraft): RetentionOverride | null {