Remove TIMDEXDataset load and filtering, use metadata

Why these changes are being introduced:

TIMDEXDataset no longer requires .load(), and now provides information
about records in the dataset via metadata queries.  As such, we can
determine if there are records in a run just by performing a metadata
SQL query.

How this addresses that need:
* Remove `TIMDEXDataset.load()` call
* Replace row count with metadata SQL call for ETL run count

Side effects of this change:
* None

Relevant ticket(s):
* https://mitlibraries.atlassian.net/browse/TIMX-535

Author: ghukill

lambdas/format_input.py

@@ -92,10 +92,9 @@ def lambda_handler(event: dict, _context: dict) -> dict:
         return result
 
     if next_step == "load":
-        if not helpers.dataset_records_exist_for_run(run_date, run_id):
+        if not helpers.dataset_records_exist_for_run(run_id):
             result["failure"] = (
-                "No records were found in the TIMDEX dataset for run_date "
-                f"'{run_date}', run_id '{run_id}'."
+                f"No records were found in the TIMDEX dataset for run_id '{run_id}'."
             )
             return result
         result["load"] = commands.generate_load_commands(

lambdas/helpers.py

@@ -1,3 +1,5 @@
+# ruff: noqa: S608
+
 import contextlib
 import logging
 from datetime import UTC, datetime, timedelta
@@ -157,14 +159,22 @@ def list_s3_files_by_prefix(bucket: str, prefix: str) -> list[str]:
     return s3_files
 
 
-def dataset_records_exist_for_run(run_date: str, run_id: str) -> bool:
-    """Query TIMDEX dataset to confirm records to load and/or delete.
+def dataset_records_exist_for_run(run_id: str) -> bool:
+    """Query TIMDEX dataset metadata to confirm records to load and/or delete.
 
     A "run" is defined by a run-date + run-id, both provided as inputs to this lambda
     invocation provided by the StepFunction.  We are interested only in records where
     action is "index" or "delete".  If zero records exist, or have action "skip" or
     "error", we do not need to perform any load commands.
     """
     td = TIMDEXDataset(location=CONFIG.s3_timdex_dataset_location)
-    td.load(run_date=run_date, run_id=run_id, action=["index", "delete"])
-    return td.row_count > 0
+
+    etl_run_count = td.metadata.conn.query(
+        f"""
+        select count(*)
+        from metadata.records
+        where run_id = '{run_id}'
+        and action in ('index','delete')
+        """
+    ).fetchone()[0]
+    return etl_run_count > 0

tests/test_format_input.py

@@ -280,7 +280,6 @@ def test_lambda_handler_with_next_step_load_no_files_present():
         "source": "testsource",
         "verbose": False,
         "failure": (
-            "No records were found in the TIMDEX dataset for "
-            "run_date '2022-01-02', run_id 'run-abc-123'."
+            "No records were found in the TIMDEX dataset for run_id 'run-abc-123'."
         ),
     }