From d674924f8c0769218a81d33735de4c3db0c5a2e5 Mon Sep 17 00:00:00 2001 From: am-toledo Date: Fri, 21 Aug 2026 13:05:24 +0200 Subject: [PATCH 1/2] log attribute ingestion on datalake writes --- splight_lib/models/_v3/datalake.py | 33 ++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/splight_lib/models/_v3/datalake.py b/splight_lib/models/_v3/datalake.py index 42cf4689..7a5cf9f2 100644 --- a/splight_lib/models/_v3/datalake.py +++ b/splight_lib/models/_v3/datalake.py @@ -9,6 +9,7 @@ from splight_lib.client.datalake import DatalakeClientBuilder from splight_lib.client.datalake.common.abstract import AbstractDatalakeClient from splight_lib.client.datalake.v3.constants import StepName +from splight_lib.logging._internal import LogTags, get_splight_logger from splight_lib.models._v3.asset import Asset from splight_lib.models._v3.attribute import Attribute from splight_lib.models._v3.exceptions import TraceAlreadyExistsError @@ -21,6 +22,8 @@ MAX_NUM_TRACES = 500 T = TypeVar("T") +logger = get_splight_logger() + def hash(string: str) -> str: return sha256(string.encode("utf-8")).hexdigest() @@ -166,10 +169,40 @@ class DataRecords(BaseModel): def apply(self) -> None: dl_client = get_datalake_client() dl_client.save(self.model_dump(mode="json")) + self._log_attribute_ingestion() async def async_apply(self) -> None: dl_client = get_datalake_client() await dl_client.async_save(self.model_dump(mode="json")) + self._log_attribute_ingestion() + + def _log_attribute_ingestion(self) -> None: + try: + counters: dict[tuple[str, str], int] = {} + for record in self.records: + asset = record.get("asset") + attribute = record.get("attribute") + if asset is None or attribute is None: + continue + if isinstance(asset, dict): + asset = asset.get("id") + if isinstance(attribute, dict): + attribute = attribute.get("id") + key = (asset, attribute) + counters[key] = counters.get(key, 0) + 1 + for (asset, attribute), count in counters.items(): + logger.info( + "attribute_ingestion " + f"asset={asset} " + f"attribute={attribute} " + f"count={count} " + f"collection={self.collection}", + tags=LogTags.DATALAKE, + ) + except Exception: + logger.exception( + "Unable to log attribute ingestion", tags=LogTags.DATALAKE + ) def chunk_list( From 83295a190e089712d626dcdd005d18d682a914e0 Mon Sep 17 00:00:00 2001 From: am-toledo Date: Fri, 21 Aug 2026 13:05:29 +0200 Subject: [PATCH 2/2] bump version to 5.24.14 --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 29068789..d977d15f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "splight-lib" -version = "5.24.13" +version = "5.24.14" description = "Splight Python Library" authors = [ {name = "Splight Dev",email = "dev@splight.com"} diff --git a/uv.lock b/uv.lock index e0b4b88d..cc2f4442 100644 --- a/uv.lock +++ b/uv.lock @@ -1784,7 +1784,7 @@ wheels = [ [[package]] name = "splight-lib" -version = "5.24.13" +version = "5.24.14" source = { editable = "." } dependencies = [ { name = "apscheduler" },