Merge pull request #320 from MITLibraries/TIMX-509-run-timestamp-argument

TIMX 509 - run-timestamp argument for all transform commands

Author: ghukill

Pipfile.lock

@@ -27,6 +27,8 @@ Takes input JSON (usually from EventBridge although it can be passed to a manual
 
 - `oai-set-spec`: optional, only used when limiting the OAI-PMH record harvest to a single set from the source repository.
 - `verbose`: optional, if provided with value `"true"` (case-insensitive) will pass the `--verbose` option (debug level logging) to all pipeline task run commands.
+- `run-id`: an ETL run id that gets included for CLI commands generated; minted if not provided
+- `run-timestamp`: an ETL timestamp that gets included for CLI commands generated; minted if not provided
 
 ### Example Format Input Event
 
@@ -110,50 +112,51 @@ GitHub Actions is configured to update the Lambda function with every push to th
 
 - Run the default handler for the container
 
-   ```bash
-   docker run -e TIMDEX_ALMA_EXPORT_BUCKET_ID=alma-bucket-name \
-   -e TIMDEX_S3_EXTRACT_BUCKET_ID=timdex-bucket-name \
-   -e WORKSPACE=dev \
-   -p 9000:8080 timdex-pipeline-lambdas-dev:latest
-   ```
+ ```bash
+docker run -e TIMDEX_ALMA_EXPORT_BUCKET_ID=alma-bucket-name \
+-e TIMDEX_S3_EXTRACT_BUCKET_ID=timdex-bucket-name \
+-e WORKSPACE=dev \
+-p 9000:8080 timdex-pipeline-lambdas-dev:latest
+ ```
 
 - POST to the container
   Note: running this with next-step transform or load involves an actual S3 connection and is thus tricky to test locally. Better to push the image to Dev1 and test there.
 
-  ```bash
-  curl -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" -d '{
-    "next-step": "extract",
-    "run-date": "2022-03-10T16:30:23Z",
-    "run-type": "daily",
-    "source": "YOURSOURCE",
-    "verbose": "true",
-    "oai-pmh-host": "https://YOUR-OAI-SOURCE/oai",
-    "oai-metadata-format": "oai_dc",
-    "oai-set-spec": "YOUR-SET-SPEC"
-  }'
-  ```
+```bash
+curl -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" -d '{
+  "next-step": "extract",
+  "run-date": "2022-03-10T16:30:23Z",
+  "run-type": "daily",
+  "source": "YOURSOURCE",
+  "verbose": "true",
+  "oai-pmh-host": "https://YOUR-OAI-SOURCE/oai",
+  "oai-metadata-format": "oai_dc",
+  "oai-set-spec": "YOUR-SET-SPEC"
+}'
+```
 
 - Observe output
-  ```json
-  {
-    "run-date": "2022-03-10",
-    "run-type": "daily",
-    "source": "YOURSOURCE",
-    "verbose": true,
-    "next-step": "transform",
-    "extract": {
-      "extract-command": [
-        "--host=https://YOUR-OAI-SOURCE/oai",
-        "--output-file=s3://timdex-bucket-name/YOURSOURCE/YOURSOURCE-2022-03-09-daily-extracted-records-to-index.xml",
-        "--verbose",
-        "harvest",
-        "--metadata-format=oai_dc",
-        "--set-spec=YOUR-SET-SPEC",
-        "--from-date=2022-03-09"
-      ]
-    }
+- 
+```json
+{
+  "run-date": "2022-03-10",
+  "run-type": "daily",
+  "source": "YOURSOURCE",
+  "verbose": true,
+  "next-step": "transform",
+  "extract": {
+    "extract-command": [
+      "--host=https://YOUR-OAI-SOURCE/oai",
+      "--output-file=s3://timdex-bucket-name/YOURSOURCE/YOURSOURCE-2022-03-09-daily-extracted-records-to-index.xml",
+      "--verbose",
+      "harvest",
+      "--metadata-format=oai_dc",
+      "--set-spec=YOUR-SET-SPEC",
+      "--from-date=2022-03-09"
+    ]
   }
-  ```
+}
+```
 
 ### Running a Specific Handler Locally with Docker
 You can call any handler you copy into the container (see Dockerfile) by name as part of the `docker run` command.

README.md

@@ -72,6 +72,7 @@ def generate_transform_commands(
     input_data: dict,
     timdex_bucket: str,
     run_id: str,
+    run_timestamp: str,
 ) -> dict[str, list[dict]]:
     """Generate task run command for TIMDEX transform."""
     files_to_transform: list[dict] = []
@@ -82,6 +83,7 @@ def generate_transform_commands(
             f"--output-location=s3://{timdex_bucket}/dataset",
             f"--source={source}",
             f"--run-id={run_id}",
+            f"--run-timestamp={run_timestamp}",
         ]
         files_to_transform.append({"transform-command": transform_command})
     return {"files-to-transform": files_to_transform}

lambdas/commands.py

@@ -19,7 +19,7 @@
 VALID_STEPS = ("extract", "transform", "load")
 
 
-def check_verbosity(verbose: bool | str) -> bool:
+def check_verbosity(verbose: bool | str) -> bool:  # noqa: FBT001
     """Determine whether verbose is True or False given a boolean or string value."""
     if isinstance(verbose, bool):
         return verbose

lambdas/config.py

@@ -1,9 +1,8 @@
-# ruff: noqa: PLR0911, PLR2004
-
 import json
 import logging
 import os
 import uuid
+from datetime import UTC, datetime
 
 from lambdas import alma_prep, commands, config, errors, helpers
 
@@ -23,6 +22,7 @@ def lambda_handler(event: dict, _context: dict) -> dict:
     source = event["source"]
     next_step = event["next-step"]
     run_id = event.get("run-id", str(uuid.uuid4()))
+    run_timestamp = event.get("run-timestamp", datetime.now(UTC).isoformat())
     timdex_bucket = os.environ["TIMDEX_S3_EXTRACT_BUCKET_ID"]
 
     result = {
@@ -72,7 +72,7 @@ def lambda_handler(event: dict, _context: dict) -> dict:
         )
         result["next-step"] = "load"
         result["transform"] = commands.generate_transform_commands(
-            extract_output_files, event, timdex_bucket, run_id
+            extract_output_files, event, timdex_bucket, run_id, run_timestamp
         )
         return result
 

lambdas/format_input.py

@@ -38,7 +38,6 @@ ignore = [
     "PLR0912",
     "PLR0913",
     "PLR0915",
-    "S320",
     "S321",
 ]
 

pyproject.toml

@@ -63,3 +63,8 @@ def s3_client():
 @pytest.fixture
 def run_id():
     return "run-abc-123"
+
+
+@pytest.fixture
+def run_timestamp():
+    return "2025-06-18T12:34:56.789000"

tests/conftest.py

@@ -77,7 +77,7 @@ def test_generate_extract_command_geoharvester():
     }
 
 
-def test_generate_transform_commands_required_input_fields(run_id):
+def test_generate_transform_commands_required_input_fields(run_id, run_timestamp):
     input_data = {
         "next-step": "transform",
         "run-date": "2022-01-02T12:13:14Z",
@@ -92,6 +92,7 @@ def test_generate_transform_commands_required_input_fields(run_id):
         input_data,
         "test-timdex-bucket",
         run_id,
+        run_timestamp,
     ) == {
         "files-to-transform": [
             {
@@ -101,13 +102,14 @@ def test_generate_transform_commands_required_input_fields(run_id):
                     "--output-location=s3://test-timdex-bucket/dataset",
                     "--source=testsource",
                     f"--run-id={run_id}",
+                    f"--run-timestamp={run_timestamp}",
                 ]
             }
         ]
     }
 
 
-def test_generate_transform_commands_all_input_fields(run_id):
+def test_generate_transform_commands_all_input_fields(run_id, run_timestamp):
     input_data = {
         "next-step": "transform",
         "run-date": "2022-01-02T12:13:14Z",
@@ -120,7 +122,11 @@ def test_generate_transform_commands_all_input_fields(run_id):
         "testsource/testsource-2022-01-02-daily-extracted-records-to-delete.xml",
     ]
     assert commands.generate_transform_commands(
-        extract_output_files, input_data, "test-timdex-bucket", run_id
+        extract_output_files,
+        input_data,
+        "test-timdex-bucket",
+        run_id,
+        run_timestamp,
     ) == {
         "files-to-transform": [
             {
@@ -130,6 +136,7 @@ def test_generate_transform_commands_all_input_fields(run_id):
                     "--output-location=s3://test-timdex-bucket/dataset",
                     "--source=testsource",
                     f"--run-id={run_id}",
+                    f"--run-timestamp={run_timestamp}",
                 ]
             },
             {
@@ -139,6 +146,7 @@ def test_generate_transform_commands_all_input_fields(run_id):
                     "--output-location=s3://test-timdex-bucket/dataset",
                     "--source=testsource",
                     f"--run-id={run_id}",
+                    f"--run-timestamp={run_timestamp}",
                 ]
             },
             {
@@ -148,6 +156,7 @@ def test_generate_transform_commands_all_input_fields(run_id):
                     "--output-location=s3://test-timdex-bucket/dataset",
                     "--source=testsource",
                     f"--run-id={run_id}",
+                    f"--run-timestamp={run_timestamp}",
                 ]
             },
         ]

tests/test_commands.py

@@ -34,7 +34,7 @@ def test_lambda_handler_with_next_step_extract():
     }
 
 
-def test_lambda_handler_with_next_step_transform_files_present(s3_client):
+def test_lambda_handler_with_next_step_transform_files_present(s3_client, run_timestamp):
     s3_client.put_object(
         Bucket="test-timdex-bucket",
         Key="testsource/testsource-2022-01-02-daily-extracted-records-to-index.xml",
@@ -46,6 +46,7 @@ def test_lambda_handler_with_next_step_transform_files_present(s3_client):
         "next-step": "transform",
         "source": "testsource",
         "run-id": "run-abc-123",
+        "run-timestamp": run_timestamp,
         "verbose": "true",
     }
     assert format_input.lambda_handler(event, {}) == {
@@ -63,20 +64,22 @@ def test_lambda_handler_with_next_step_transform_files_present(s3_client):
                         "--output-location=s3://test-timdex-bucket/dataset",
                         "--source=testsource",
                         "--run-id=run-abc-123",
+                        f"--run-timestamp={run_timestamp}",
                     ]
                 }
             ]
         },
     }
 
 
-def test_lambda_handler_with_next_step_transform_alma_files_present():
+def test_lambda_handler_with_next_step_transform_alma_files_present(run_timestamp):
     event = {
         "run-date": "2022-09-12",
         "run-type": "daily",
         "next-step": "transform",
         "source": "alma",
         "run-id": "run-abc-123",
+        "run-timestamp": run_timestamp,
         "verbose": "False",
     }
     assert format_input.lambda_handler(event, {}) == {
@@ -94,6 +97,7 @@ def test_lambda_handler_with_next_step_transform_alma_files_present():
                         "--output-location=s3://test-timdex-bucket/dataset",
                         "--source=alma",
                         "--run-id=run-abc-123",
+                        f"--run-timestamp={run_timestamp}",
                     ]
                 },
                 {
@@ -103,6 +107,7 @@ def test_lambda_handler_with_next_step_transform_alma_files_present():
                         "--output-location=s3://test-timdex-bucket/dataset",
                         "--source=alma",
                         "--run-id=run-abc-123",
+                        f"--run-timestamp={run_timestamp}",
                     ]
                 },
                 {
@@ -112,13 +117,60 @@ def test_lambda_handler_with_next_step_transform_alma_files_present():
                         "--output-location=s3://test-timdex-bucket/dataset",
                         "--source=alma",
                         "--run-id=run-abc-123",
+                        f"--run-timestamp={run_timestamp}",
                     ]
                 },
             ]
         },
     }
 
 
+def test_lambda_handler_with_next_step_transform_auto_generated_timestamp(s3_client):
+    s3_client.put_object(
+        Bucket="test-timdex-bucket",
+        Key="testsource/testsource-2022-01-02-daily-extracted-records-to-index.xml",
+        Body="I am a file",
+    )
+    event = {
+        "run-date": "2022-01-02T12:13:14Z",
+        "run-type": "daily",
+        "next-step": "transform",
+        "source": "testsource",
+        "run-id": "run-abc-123",
+        "verbose": "true",
+    }
+
+    with patch("lambdas.format_input.datetime") as mock_datetime:
+        mock_datetime.now.return_value.isoformat.return_value = (
+            "2025-06-18T12:34:56.789000"
+        )
+        mock_datetime.UTC = format_input.datetime.UTC
+
+        result = format_input.lambda_handler(event, {})
+
+    assert result == {
+        "run-date": "2022-01-02",
+        "run-type": "daily",
+        "source": "testsource",
+        "verbose": True,
+        "next-step": "load",
+        "transform": {
+            "files-to-transform": [
+                {
+                    "transform-command": [
+                        "--input-file=s3://test-timdex-bucket/testsource/"
+                        "testsource-2022-01-02-daily-extracted-records-to-index.xml",
+                        "--output-location=s3://test-timdex-bucket/dataset",
+                        "--source=testsource",
+                        "--run-id=run-abc-123",
+                        "--run-timestamp=2025-06-18T12:34:56.789000",
+                    ]
+                }
+            ]
+        },
+    }
+
+
 def test_lambda_handler_with_next_step_transform_no_files_present_alma():
     event = {
         "run-date": "2022-01-02",