From c5fc2ad858797e1e51724386ed1e05acbf085b5f Mon Sep 17 00:00:00 2001 From: Matthew Carroll <28577806+MJC598@users.noreply.github.com> Date: Wed, 19 Aug 2026 13:05:41 -0400 Subject: [PATCH 1/3] adding caching of sql queried parameters --- CMakeLists.txt | 2 +- benchmarks/conftest.py | 234 ++++++++++++++++++++++------ benchmarks/test_bench_end_to_end.py | 26 ++-- pyproject.toml | 4 +- src/respondpy/data/input.py | 38 +++-- tests/test_data_input.py | 47 ++++++ 6 files changed, 273 insertions(+), 78 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f76f82d..d09f231 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,7 +62,7 @@ set(SPDLOG_INSTALL ON) FetchContent_Declare( respond GIT_REPOSITORY https://github.com/SyndemicsLab/respond.git - GIT_TAG 2bc260ef749a76b7eba73a4d64e9d38c48ed2bdc # v2.5.1 + GIT_TAG 0a5ed066ecd48f0ced927d86fb3eb42d2acb63e4 # PR 153 OVERRIDE_FIND_PACKAGE ) set(RESPOND_BUILD_DOCS OFF) diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py index ff37ed1..842dfc2 100644 --- a/benchmarks/conftest.py +++ b/benchmarks/conftest.py @@ -159,53 +159,188 @@ ); """ -_SEED_DATA = """ -INSERT INTO cohort (id, description, background_mortality_sample, behavior_transition_sample, initial_population_sample, intervention_transition_sample, overdose_sample, overdose_fatality_sample, population_change_sample, smr_sample) - VALUES (1, "Benchmark Cohort", 1, 1, 1, 1, 1, 1, 1, 1); - -INSERT INTO intervention (id, name) - VALUES (1, "no_treatment"), (2, "early_buprenorphine"), (3, "buprenorphine"), (4, "post_buprenorphine"); - -INSERT INTO behavior (id, name) - VALUES (1, "active_injection"), (2, "nonactive_injection"); - -INSERT INTO initial_population (sample, intervention, behavior, count) - VALUES (1,1,1,100),(1,1,2,150),(1,2,1,200),(1,2,2,250),(1,3,1,0),(1,3,2,0),(1,4,1,0),(1,4,2,0); - -INSERT INTO population_change (sample, intervention, behavior, time, count) - VALUES (1,1,1,1,100),(1,1,2,1,150),(1,2,1,1,200),(1,2,2,1,250),(1,3,1,1,0),(1,3,2,1,0),(1,4,1,1,0),(1,4,2,1,0); - -INSERT INTO intervention_transition (sample, behavior, time, initial_intervention, new_intervention, probability) - VALUES (1,1,1,1,1,0.8),(1,2,1,1,1,0.7),(1,1,1,1,2,0.2),(1,2,1,1,2,0.3), - (1,1,1,1,3,0.0),(1,2,1,1,3,0.0),(1,1,1,1,4,0.0),(1,2,1,1,4,0.0), - (1,1,1,2,1,0.0),(1,2,1,2,1,0.0),(1,1,1,2,2,0.7),(1,2,1,2,2,0.6), - (1,1,1,2,3,0.2),(1,2,1,2,3,0.1),(1,1,1,2,4,0.1),(1,2,1,2,4,0.3), - (1,1,1,3,1,0.0),(1,2,1,3,1,0.0),(1,1,1,3,2,0.0),(1,2,1,3,2,0.0), - (1,1,1,3,3,0.8),(1,2,1,3,3,0.8),(1,1,1,3,4,0.2),(1,2,1,3,4,0.2), - (1,1,1,4,1,0.8),(1,2,1,4,1,0.8),(1,1,1,4,2,0.0),(1,2,1,4,2,0.0), - (1,1,1,4,3,0.0),(1,2,1,4,3,0.0),(1,1,1,4,4,0.2),(1,2,1,4,4,0.2); - -INSERT INTO behavior_transition (sample, intervention, time, initial_behavior, new_behavior, probability) - VALUES (1,1,1,1,1,0.8),(1,1,1,1,2,0.2),(1,1,1,2,1,0.1),(1,1,1,2,2,0.9), - (1,2,1,1,1,0.9),(1,2,1,1,2,0.1),(1,2,1,2,1,0.7),(1,2,1,2,2,0.3), - (1,3,1,1,1,0.3),(1,3,1,1,2,0.7),(1,3,1,2,1,0.4),(1,3,1,2,2,0.6), - (1,4,1,1,1,0.3),(1,4,1,1,2,0.7),(1,4,1,2,1,0.2),(1,4,1,2,2,0.8); - -INSERT INTO smr (sample, intervention, behavior, time, ratio) - VALUES (1,1,1,1,2.0),(1,1,2,1,2.1),(1,2,1,1,2.0),(1,2,2,1,2.1), - (1,3,1,1,2.0),(1,3,2,1,2.1),(1,4,1,1,2.0),(1,4,2,1,2.1); - -INSERT INTO background_mortality (sample, time, probability) - VALUES (1,1,0.25); - -INSERT INTO overdose (sample, intervention, behavior, time, probability) - VALUES (1,1,1,1,0.8),(1,1,2,1,0.7),(1,2,1,1,0.8),(1,2,2,1,0.7), - (1,3,1,1,0.8),(1,3,2,1,0.7),(1,4,1,1,0.8),(1,4,2,1,0.7); - -INSERT INTO overdose_fatality (sample, intervention, behavior, time, probability) - VALUES (1,1,1,1,0.1),(1,1,2,1,0.2),(1,2,1,1,0.1),(1,2,2,1,0.2), - (1,3,1,1,0.1),(1,3,2,1,0.2),(1,4,1,1,0.1),(1,4,2,1,0.2); -""" + +def _seed_benchmark_database(conn: sqlite3.Connection) -> None: + """Seed a valid benchmark database across timesteps 1..52. + + The benchmark is intentionally generated programmatically so that + time-varying rows exist for multiple parameter-change schedules without + requiring a huge static SQL file. + """ + cursor = conn.cursor() + + cursor.execute( + """ + INSERT INTO cohort ( + id, description, background_mortality_sample, + behavior_transition_sample, initial_population_sample, + intervention_transition_sample, overdose_sample, + overdose_fatality_sample, population_change_sample, smr_sample + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + (1, "Benchmark Cohort", 1, 1, 1, 1, 1, 1, 1, 1), + ) + + cursor.executemany( + "INSERT INTO intervention (id, name) VALUES (?, ?)", + [ + (1, "no_treatment"), + (2, "early_buprenorphine"), + (3, "buprenorphine"), + (4, "post_buprenorphine"), + ], + ) + cursor.executemany( + "INSERT INTO behavior (id, name) VALUES (?, ?)", + [ + (1, "active_injection"), + (2, "nonactive_injection"), + ], + ) + cursor.executemany( + "INSERT INTO initial_population (sample, intervention, behavior, count) VALUES (?, ?, ?, ?)", + [ + (1, 1, 1, 100), + (1, 1, 2, 150), + (1, 2, 1, 200), + (1, 2, 2, 250), + (1, 3, 1, 0), + (1, 3, 2, 0), + (1, 4, 1, 0), + (1, 4, 2, 0), + ], + ) + + behavior_transition_rows = [] + intervention_transition_rows = [] + population_change_rows = [] + smr_rows = [] + background_mortality_rows = [] + overdose_rows = [] + overdose_fatality_rows = [] + + for t in range(1, 53): + population_change_rows.extend([ + (1, 1, 1, t, 100), + (1, 1, 2, t, 150), + (1, 2, 1, t, 200), + (1, 2, 2, t, 250), + (1, 3, 1, t, 0), + (1, 3, 2, t, 0), + (1, 4, 1, t, 0), + (1, 4, 2, t, 0), + ]) + + background_mortality_rows.append((1, t, 0.25)) + + smr_rows.extend([ + (1, 1, 1, t, 2.0), + (1, 1, 2, t, 2.1), + (1, 2, 1, t, 2.0), + (1, 2, 2, t, 2.1), + (1, 3, 1, t, 2.0), + (1, 3, 2, t, 2.1), + (1, 4, 1, t, 2.0), + (1, 4, 2, t, 2.1), + ]) + + overdose_rows.extend([ + (1, 1, 1, t, 0.8), + (1, 1, 2, t, 0.7), + (1, 2, 1, t, 0.8), + (1, 2, 2, t, 0.7), + (1, 3, 1, t, 0.8), + (1, 3, 2, t, 0.7), + (1, 4, 1, t, 0.8), + (1, 4, 2, t, 0.7), + ]) + + overdose_fatality_rows.extend([ + (1, 1, 1, t, 0.1), + (1, 1, 2, t, 0.2), + (1, 2, 1, t, 0.1), + (1, 2, 2, t, 0.2), + (1, 3, 1, t, 0.1), + (1, 3, 2, t, 0.2), + (1, 4, 1, t, 0.1), + (1, 4, 2, t, 0.2), + ]) + + for intervention in range(1, 5): + for initial_behavior in (1, 2): + for new_behavior in (1, 2): + behavior_transition_rows.append( + ( + 1, + intervention, + t, + initial_behavior, + new_behavior, + { + 1: {(1, 1): 0.8, (1, 2): 0.2, (2, 1): 0.1, (2, 2): 0.9}, + 2: {(1, 1): 0.9, (1, 2): 0.1, (2, 1): 0.7, (2, 2): 0.3}, + 3: {(1, 1): 0.3, (1, 2): 0.7, (2, 1): 0.4, (2, 2): 0.6}, + 4: {(1, 1): 0.3, (1, 2): 0.7, (2, 1): 0.2, (2, 2): 0.8}, + }[intervention][(initial_behavior, new_behavior)], + ) + ) + + for behavior in (1, 2): + for initial_intervention in range(1, 5): + for new_intervention in range(1, 5): + intervention_transition_rows.append( + ( + 1, + behavior, + t, + initial_intervention, + new_intervention, + { + 1: { + (1, 1): 0.8, (1, 2): 0.2, (1, 3): 0.0, (1, 4): 0.0, + (2, 1): 0.0, (2, 2): 0.7, (2, 3): 0.2, (2, 4): 0.1, + (3, 1): 0.0, (3, 2): 0.0, (3, 3): 0.8, (3, 4): 0.2, + (4, 1): 0.8, (4, 2): 0.0, (4, 3): 0.0, (4, 4): 0.2, + }, + 2: { + (1, 1): 0.7, (1, 2): 0.3, (1, 3): 0.0, (1, 4): 0.0, + (2, 1): 0.0, (2, 2): 0.6, (2, 3): 0.1, (2, 4): 0.3, + (3, 1): 0.0, (3, 2): 0.0, (3, 3): 0.8, (3, 4): 0.2, + (4, 1): 0.8, (4, 2): 0.0, (4, 3): 0.0, (4, 4): 0.2, + }, + }[behavior][(initial_intervention, new_intervention)], + ) + ) + + cursor.executemany( + "INSERT INTO population_change (sample, intervention, behavior, time, count) VALUES (?, ?, ?, ?, ?)", + population_change_rows, + ) + cursor.executemany( + "INSERT INTO intervention_transition (sample, behavior, time, initial_intervention, new_intervention, probability) VALUES (?, ?, ?, ?, ?, ?)", + intervention_transition_rows, + ) + cursor.executemany( + "INSERT INTO behavior_transition (sample, intervention, time, initial_behavior, new_behavior, probability) VALUES (?, ?, ?, ?, ?, ?)", + behavior_transition_rows, + ) + cursor.executemany( + "INSERT INTO smr (sample, intervention, behavior, time, ratio) VALUES (?, ?, ?, ?, ?)", + smr_rows, + ) + cursor.executemany( + "INSERT INTO background_mortality (sample, time, probability) VALUES (?, ?, ?)", + background_mortality_rows, + ) + cursor.executemany( + "INSERT INTO overdose (sample, intervention, behavior, time, probability) VALUES (?, ?, ?, ?, ?)", + overdose_rows, + ) + cursor.executemany( + "INSERT INTO overdose_fatality (sample, intervention, behavior, time, probability) VALUES (?, ?, ?, ?, ?)", + overdose_fatality_rows, + ) + conn.commit() # --------------------------------------------------------------------------- @@ -219,8 +354,7 @@ def benchmark_db(tmp_path_factory) -> Path: conn = sqlite3.connect(db_path) cursor = conn.cursor() cursor.executescript(_DB_SCHEMA) - cursor.executescript(_SEED_DATA) - conn.commit() + _seed_benchmark_database(conn) conn.close() return db_path @@ -232,7 +366,7 @@ def benchmark_config(tmp_path_factory) -> str: cfg = ConfigParser() cfg["simulation"] = { "duration": "52", - "parameter_change_times": "52", + "parameter_change_times": "1", "stratify_entering_cohort": "false", } cfg["output"] = { diff --git a/benchmarks/test_bench_end_to_end.py b/benchmarks/test_bench_end_to_end.py index b37d188..4946d55 100644 --- a/benchmarks/test_bench_end_to_end.py +++ b/benchmarks/test_bench_end_to_end.py @@ -40,19 +40,21 @@ def _write_histories_to_csv(sim: Simulation, out_dir: Path) -> None: - """Write each model's densified state histories to a separate CSV file. + """Write each model's recorded state histories to a separate CSV file. Output format per file: history_name, timestep, state_0, state_1, ..., state_n Files are named ``_histories.csv`` and written to *out_dir*. """ - all_model_histories = sim.get_model_histories() - for model_name, model_history in all_model_histories.items(): + for model_idx, model_name in enumerate(sim.get_model_names()): + model_history = sim.get_model_history(model_idx) rows: list[dict] = [] - for hist_name, state_vectors in model_history.items(): - for t, vec in enumerate(state_vectors): - row = {"history_name": hist_name, "timestep": t} + for hist_name, history in model_history.items(): + state_map = history.get_state_map() + for timestep in history.get_recorded_timesteps(): + vec = state_map[timestep] + row = {"history_name": hist_name, "timestep": int(timestep)} row.update({f"state_{i}": float(v) for i, v in enumerate(vec)}) rows.append(row) if rows: @@ -68,14 +70,16 @@ def _write_histories_to_csv(sim: Simulation, out_dir: Path) -> None: def test_bench_load_data(benchmark, benchmark_db, benchmark_config): """Benchmark SQLite data loading and Simulation construction. - Measures the time to call build_simulation() from scratch on each round, - including all database reads and model/transition setup. + Each round builds a fresh Input instance so the measurement captures the + cold load path rather than reusing cached parameter arrays or sample ids. """ - input_data = Input(db_path=benchmark_db, conf_path=benchmark_config) + + def _build_from_scratch(): + input_data = Input(db_path=benchmark_db, conf_path=benchmark_config) + return build_simulation(input_data, cohort_ids=[1]) benchmark.pedantic( - build_simulation, - kwargs={'input_data': input_data, 'cohort_ids': [1]}, + _build_from_scratch, rounds=10, iterations=1, ) diff --git a/pyproject.toml b/pyproject.toml index c68e428..7ae702a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,7 +110,7 @@ explicit = true [tool.pytest] minversion = "9.0" -testpaths = ["tests"] +testpaths = ["tests", "benchmarks"] addopts = [ "-ra", "--strict", @@ -211,7 +211,7 @@ ignore = [ ] typing-modules = ["respondpy._core"] isort.required-imports = ["from __future__ import annotations"] -exclude = ["tests/*"] +exclude = ["tests/*", "benchmarks/*", "docs/*"] [tool.ruff.lint.mccabe] max-complexity = 13 diff --git a/src/respondpy/data/input.py b/src/respondpy/data/input.py index 72beb8b..109e5da 100644 --- a/src/respondpy/data/input.py +++ b/src/respondpy/data/input.py @@ -109,6 +109,8 @@ def __init__( self.states: dict[str, list] = {} self.interventions: list[str] | None = None self.behaviors: list[str] | None = None + self._sample_id_cache: dict[tuple[ParameterType, int], int] = {} + self._parameter_cache: dict[tuple[ParameterType, int, int], np.ndarray] = {} self._log_name = log_name self._log_file = str(log_file) if log_file is not None else None @@ -208,6 +210,10 @@ def _get_sample_id_for_parameter( param: Parameter, cohort_id: int = 1 ) -> int: + cache_key = (param.get_parameter_type(), cohort_id) + if cache_key in self._sample_id_cache: + return self._sample_id_cache[cache_key] + col_name = param.get_cohort_column_name() stmt = f"SELECT {col_name} FROM cohort WHERE id = ?" _, result = self._connect_and_fetchall(stmt, (str(cohort_id),)) @@ -215,7 +221,10 @@ def _get_sample_id_for_parameter( msg = f"No sample ID found for parameter {param} and cohort ID {cohort_id}!" rpy_logging.log_error(self.log_name, msg) raise ValueError(msg) - return result[0][0] + + sample_id = result[0][0] + self._sample_id_cache[cache_key] = sample_id + return sample_id def _select_parameter_raw( self, @@ -388,18 +397,10 @@ def _get_parameter_filled( ) if complete_state_vector: - return sort_dataframes( - lf, - self._get_single_state_table("behavior"), - self._get_single_state_table("intervention") - ) + return lf if complete_transition: - return sort_dataframes( - lf, - self._get_single_state_table("behavior"), - self._get_single_state_table("intervention") - ) + return lf if param.is_state_vector_operation(): value_col = param.get_value_column_name() @@ -549,16 +550,24 @@ def select_parameter( numpy.ndarray Raw rows or model-ready shaped numpy array. """ + cache_key = (param.get_parameter_type(), cohort_id, time) + if not raw and cache_key in self._parameter_cache: + return self._parameter_cache[cache_key].copy() + sample_id = self._get_sample_id_for_parameter(param, cohort_id) + if raw: - return self._select_parameter_raw( + result = self._select_parameter_raw( param, sample_id, time).collect().to_numpy() + return result - return self._extract_values( + result = self._extract_values( param, self._get_parameter_filled(param, sample_id, time), n=len(self.get_state_names()) ) + self._parameter_cache[cache_key] = result.copy() + return result.copy() def insert_parameter( self, @@ -574,4 +583,5 @@ def insert_parameter( data : list Row tuples matching ``param`` insert statement order. """ - return self._connect_and_executemany(data, param.get_insert_statement()) + self._connect_and_executemany(data, param.get_insert_statement()) + self._parameter_cache.clear() diff --git a/tests/test_data_input.py b/tests/test_data_input.py index 009b9b9..f410a46 100644 --- a/tests/test_data_input.py +++ b/tests/test_data_input.py @@ -181,6 +181,49 @@ def test_get_cohort_ids(input_data): assert expected_ids == ids +@pytest.mark.unit +def test_select_parameter_is_cached(input_data): + param = rpydata.Parameter(rpydata.ParameterType.INITIAL_COHORT) + + original_sample_lookup = input_data._get_sample_id_for_parameter + original_parameter_fill = input_data._get_parameter_filled + calls = {"sample_lookup": 0, "parameter_fill": 0} + + def counting_sample_lookup(param_obj, cohort_id=1): + calls["sample_lookup"] += 1 + return original_sample_lookup(param_obj, cohort_id) + + def counting_parameter_fill(param_obj, sample_id=1, time=1): + calls["parameter_fill"] += 1 + return original_parameter_fill(param_obj, sample_id, time) + + input_data._get_sample_id_for_parameter = counting_sample_lookup + input_data._get_parameter_filled = counting_parameter_fill + + first = input_data.select_parameter(param, cohort_id=1, time=1) + second = input_data.select_parameter(param, cohort_id=1, time=1) + + assert isinstance(first, np.ndarray) + assert np.array_equal(first, second) + assert calls["sample_lookup"] == 1 + assert calls["parameter_fill"] == 1 + + +@pytest.mark.unit +def test_complete_parameter_avoids_sort(input_data, monkeypatch): + param = rpydata.Parameter(rpydata.ParameterType.INITIAL_COHORT) + sample_id = input_data._get_sample_id_for_parameter(param, cohort_id=1) + + def fail_sort(*args, **kwargs): + raise AssertionError("complete parameter tables should bypass sort_dataframes") + + monkeypatch.setattr("respondpy.data.input.sort_dataframes", fail_sort) + + lf = input_data._get_parameter_filled(param, sample_id=sample_id, time=1) + assert isinstance(lf, pl.LazyFrame) + assert lf.select(pl.len()).collect().item() == len(input_data.get_interventions()) * len(input_data.get_behaviors()) + + @pytest.mark.unit def test_select_parameter_intervention(input_data): param = rpydata.Parameter( @@ -366,10 +409,14 @@ def test_insert_cohorts_adds_new_row(input_data): @pytest.mark.unit def test_insert_parameter_adds_sample_row(input_data): parameter = rpydata.Parameter(rpydata.ParameterType.INITIAL_COHORT) + input_data.select_parameter(parameter, cohort_id=1) + assert input_data._parameter_cache + input_data.insert_parameter(parameter, [(2, 1, 1, 42.0)]) sample_ids = input_data._get_sample_ids_by_table("initial_population") assert 2 in sample_ids + assert not input_data._parameter_cache @pytest.mark.unit From cbef4600e46d908c62f28c9fa9b099474074c795 Mon Sep 17 00:00:00 2001 From: Matthew Carroll <28577806+MJC598@users.noreply.github.com> Date: Wed, 19 Aug 2026 13:16:20 -0400 Subject: [PATCH 2/3] updating caching sql queries --- src/respondpy/build.py | 18 +++++---- src/respondpy/data/input.py | 74 ++++++++++++++++++++++++++----------- tests/test_data_input.py | 22 +++++++++-- 3 files changed, 82 insertions(+), 32 deletions(-) diff --git a/src/respondpy/build.py b/src/respondpy/build.py index 01cf29c..e66af2d 100644 --- a/src/respondpy/build.py +++ b/src/respondpy/build.py @@ -12,6 +12,7 @@ from __future__ import annotations +import copy from collections.abc import Sequence from .data import Input, Parameter, ParameterType, validate_time_list @@ -116,16 +117,19 @@ def build_model( duration = int(input_data.config.get("simulation", "duration")) schedule_times = [1, *change_times] + timestep_templates: dict[int, Timestep] = {} for model_timestep in range(1, duration+1): parameter_time = max(t for t in schedule_times if t <= model_timestep) - model.add_timestep(build_timestep( - input_data, - cohort_id, - parameter_time, - log_name=log_name, - log_file=log_file, - )) + if parameter_time not in timestep_templates: + timestep_templates[parameter_time] = build_timestep( + input_data, + cohort_id, + parameter_time, + log_name=log_name, + log_file=log_file, + ) + model.add_timestep(copy.copy(timestep_templates[parameter_time])) return model diff --git a/src/respondpy/data/input.py b/src/respondpy/data/input.py index 109e5da..6971936 100644 --- a/src/respondpy/data/input.py +++ b/src/respondpy/data/input.py @@ -232,31 +232,35 @@ def _select_parameter_raw( sample_id: int = 1, time: int = 1 ) -> pl.LazyFrame: + cols, vals = self._select_parameter_rows(param, sample_id, time) + lzdf = pl.LazyFrame(vals, schema=cols, orient='row') + if param.is_time_varying(): + lzdf = lzdf.with_columns(sample=sample_id, time=time) + else: + lzdf = lzdf.with_columns(sample=sample_id) + return lzdf + + def _select_parameter_rows( + self, + param: Parameter, + sample_id: int = 1, + time: int = 1 + ) -> tuple[list[str], list[tuple]]: stmt = param.get_select_statement( self.get_interventions(), self.get_behaviors() ) if not param.is_time_varying(): - cols, vals = self._connect_and_fetchall( + return self._connect_and_fetchall( stmt, (str(sample_id),)) - lzdf = pl.LazyFrame( - vals, schema=cols, orient='row' - ).with_columns(sample=sample_id) - else: - cols, vals = self._connect_and_fetchall( - stmt, (str(sample_id), str(time))) - if len(vals) == 0: - msg = f"Missing time-varying parameter rows in database: parameter={param.get_parameter_name()}, sample_id={sample_id}, time={time}. Expected rows for this configured timestep but found none." - rpy_logging.log_error(self.log_name, msg) - raise ValueError(msg) - lzdf = pl.LazyFrame( - vals, schema=cols, orient='row' - ).with_columns( - sample=sample_id, - time=time - ) - return lzdf + cols, vals = self._connect_and_fetchall( + stmt, (str(sample_id), str(time))) + if len(vals) == 0: + msg = f"Missing time-varying parameter rows in database: parameter={param.get_parameter_name()}, sample_id={sample_id}, time={time}. Expected rows for this configured timestep but found none." + rpy_logging.log_error(self.log_name, msg) + raise ValueError(msg) + return cols, vals def _extract_values( self, @@ -349,7 +353,8 @@ def _get_parameter_filled( self, param: Parameter, sample_id: int = 1, - time: int = 1 + time: int = 1, + lf: pl.LazyFrame | None = None, ) -> pl.LazyFrame: """Return a complete parameter table with missing rows backfilled. @@ -376,7 +381,8 @@ def _get_parameter_filled( ValueError If required transition state columns are missing. """ - lf = self._select_parameter_raw(param, sample_id, time) + if lf is None: + lf = self._select_parameter_raw(param, sample_id, time) if param == ParameterType.INTERVENTION_TRANSITION_PROBABILITY: lf = lf.rename({"behavior": "initial_behavior"}) elif param == ParameterType.BEHAVIOR_TRANSITION_PROBABILITY: @@ -561,9 +567,35 @@ def select_parameter( param, sample_id, time).collect().to_numpy() return result + cols, vals = self._select_parameter_rows(param, sample_id, time) + n_interventions = len(self.get_interventions()) + n_behaviors = len(self.get_behaviors()) + n_states = n_interventions * n_behaviors + expected_rows = ( + n_states + if param.is_state_vector_operation() + else n_states * n_states + ) + if len(vals) == expected_rows: + value_index = cols.index(param.get_value_column_name()) + result = np.asarray( + [row[value_index] for row in vals], dtype=np.float64 + ) + shape = (n_states, 1) if param.is_state_vector_operation() else ( + n_states, n_states + ) + result = result.reshape(shape) + self._parameter_cache[cache_key] = result.copy() + return result.copy() + + lf = pl.LazyFrame(vals, schema=cols, orient='row') + if param.is_time_varying(): + lf = lf.with_columns(sample=sample_id, time=time) + else: + lf = lf.with_columns(sample=sample_id) result = self._extract_values( param, - self._get_parameter_filled(param, sample_id, time), + self._get_parameter_filled(param, sample_id, time, lf=lf), n=len(self.get_state_names()) ) self._parameter_cache[cache_key] = result.copy() diff --git a/tests/test_data_input.py b/tests/test_data_input.py index f410a46..a0a7178 100644 --- a/tests/test_data_input.py +++ b/tests/test_data_input.py @@ -186,19 +186,19 @@ def test_select_parameter_is_cached(input_data): param = rpydata.Parameter(rpydata.ParameterType.INITIAL_COHORT) original_sample_lookup = input_data._get_sample_id_for_parameter - original_parameter_fill = input_data._get_parameter_filled + original_parameter_rows = input_data._select_parameter_rows calls = {"sample_lookup": 0, "parameter_fill": 0} def counting_sample_lookup(param_obj, cohort_id=1): calls["sample_lookup"] += 1 return original_sample_lookup(param_obj, cohort_id) - def counting_parameter_fill(param_obj, sample_id=1, time=1): + def counting_parameter_rows(param_obj, sample_id=1, time=1): calls["parameter_fill"] += 1 - return original_parameter_fill(param_obj, sample_id, time) + return original_parameter_rows(param_obj, sample_id, time) input_data._get_sample_id_for_parameter = counting_sample_lookup - input_data._get_parameter_filled = counting_parameter_fill + input_data._select_parameter_rows = counting_parameter_rows first = input_data.select_parameter(param, cohort_id=1, time=1) second = input_data.select_parameter(param, cohort_id=1, time=1) @@ -224,6 +224,20 @@ def fail_sort(*args, **kwargs): assert lf.select(pl.len()).collect().item() == len(input_data.get_interventions()) * len(input_data.get_behaviors()) +@pytest.mark.unit +def test_complete_parameter_uses_direct_numpy_path(input_data, monkeypatch): + param = rpydata.Parameter(rpydata.ParameterType.INITIAL_COHORT) + + def fail_fill(*args, **kwargs): + raise AssertionError("complete parameter tables should bypass Polars filling") + + monkeypatch.setattr(input_data, "_get_parameter_filled", fail_fill) + + result = input_data.select_parameter(param, cohort_id=1, time=1) + + assert result.shape == (len(input_data.get_state_names()), 1) + + @pytest.mark.unit def test_select_parameter_intervention(input_data): param = rpydata.Parameter( From 5efe5dfd371f8a8c400984d8254df0884e4bfb8d Mon Sep 17 00:00:00 2001 From: Matthew Carroll <28577806+MJC598@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:16:33 -0400 Subject: [PATCH 3/3] fixing respond hash --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d09f231..5c10da4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,7 +62,7 @@ set(SPDLOG_INSTALL ON) FetchContent_Declare( respond GIT_REPOSITORY https://github.com/SyndemicsLab/respond.git - GIT_TAG 0a5ed066ecd48f0ced927d86fb3eb42d2acb63e4 # PR 153 + GIT_TAG cd2c0061a5e7ce27d7bdec17eacf96ef6fc5ad75 # main (Post PR 153) OVERRIDE_FIND_PACKAGE ) set(RESPOND_BUILD_DOCS OFF)