Sync junior rift_O4d into upstream rift_O4d

.gitignore

@@ -53,4 +53,4 @@ cover/
 #   in the .venv directory. It is recommended not to include this directory in version control.
 .pixi/*
 !.pixi/config.toml
-
+asimov.log

.gitlab-ci.yml

@@ -91,6 +91,7 @@ import_check:
     - python -m pytest -q MonteCarloMarginalizeCode/Code/test/test_lisa_lalsimutils_compat.py
     - python -m pytest -q MonteCarloMarginalizeCode/Code/test/test_lisa_operational_synthetic.py
     - python -m pytest -q MonteCarloMarginalizeCode/Code/test/test_lisa_helper_contract.py
+    - python -m pytest -q MonteCarloMarginalizeCode/Code/test/test_spherical_harmonics_backend.py
 
 dependency_compat_check:
   stage: system tests
@@ -129,6 +130,20 @@ test_run:
     # numpy/scipy, seconds). Runs before the heavy scripts so a wrong prior fails
     # fast rather than after the end-to-end runs.
     - python -m pytest -q MonteCarloMarginalizeCode/Code/test/test_cip_priors.py
+    # Per-extrinsic-sample time-marginalization offset in the vectorized NoLoop
+    # likelihood (issue #232).  numpy + lal, ~9 s, and it fails SILENTLY in the worst
+    # way: a batch-wide log-sum-exp offset underflows every extrinsic sample more than
+    # ~745 nats below the loudest one to lnL = -inf, which above rho ~ 40 is the bulk of
+    # the prior and collapses mcsamplerAV.  An unlisted test never runs in this CI, so
+    # the wiring ships with the fix.
+    - python -m pytest -q MonteCarloMarginalizeCode/Code/test/test_noloop_time_marg_row_offset.py
+    # Same defect on the in-loop calibration-marginalization reduction (n_cal>1,
+    # cal_method='loop', which is the DEFAULT: the fused kernel is opt-in behind
+    # --calibration-fused-kernel).  Its streaming log-sum-exp offset was a SCALAR shared
+    # by every extrinsic sample, so rows more than ~745 nats below the loudest came back
+    # lnL = -inf (issue #232).  numpy + lal, ~10 s.  An unlisted test never runs in this
+    # CI, so the wiring ships with the fix.
+    - python -m pytest -q MonteCarloMarginalizeCode/Code/test/test_calmarg_running_max_row_offset.py
     - . .travis/test-coord.sh
     - bash .travis/test-integrate.sh
     - . .travis/test-posterior.sh
@@ -160,7 +175,7 @@ gpu_integration:
       --env GW_SURROGATE="$GW_SURROGATE"
       --env RIFT_CI_REQUIRE_GPU="$RIFT_CI_REQUIRE_GPU"
       "$RIFT_CI_APPTAINER_IMAGE"
-      bash -lc 'cd "$CI_PROJECT_DIR" && export PYTHONPATH="$CI_PROJECT_DIR/MonteCarloMarginalizeCode/Code${PYTHONPATH:+:$PYTHONPATH}" && bash .travis/test-integrate.sh && bash .travis/test-calmarg-gpu.sh'
+      bash -lc 'cd "$CI_PROJECT_DIR" && export PYTHONPATH="$CI_PROJECT_DIR/MonteCarloMarginalizeCode/Code${PYTHONPATH:+:$PYTHONPATH}" && bash .travis/test-integrate.sh && bash .travis/test-calmarg-gpu.sh && bash .travis/test-lisa-gpu.sh'
   rules:
     - if: '$CI_PIPELINE_SOURCE == "web"'
       when: manual

.travis/ci_roster.txt

@@ -0,0 +1,136 @@
+# CI roster -- every test file under MonteCarloMarginalizeCode/Code that is reachable from NO
+# CI job, and the reason it is not.  Enforced by .travis/test-ci-roster.py (job ci-roster-check).
+#
+# Format:   <repo-relative path>  <STATUS>  <reason>
+#
+# STATUS is one of:
+#   HANDRUN    a hand-run study or demo, not a pytest target.  Most of these have a __main__
+#              and argparse, assert internally, and collect ZERO items under pytest -- which
+#              exits 5, "no tests ran", and reads as a pass in a log skim.  Gating them as-is
+#              would buy a green tick over an empty run.
+#   LEGACY     imports pre-package flat module names (factored_likelihood, lalsimutils, ourio,
+#              ourparams, xmlutils, common_cl, statutils, effectiveFisher, spokes) that have not
+#              existed since RIFT was packaged into RIFT.*.  These cannot be IMPORTED, let alone
+#              run, in any current environment.
+#   OPTDEP     needs a dependency CI does not install.
+#   GPU        needs a GPU; the runners have none, so it would report as skipped.
+#   EXPENSIVE  opt-in behind an env var by design.
+#   BROKEN     collects but FAILS today.  A debt, recorded as one.
+#   PENDING    waiting on a named gate that is not live yet.  The reason must say
+#              `gate:<name>`, and the entry is legal only while that gate is absent -- when
+#              it lands, the census errors on the entry by name.  An earlier version was
+#              exempt from the staleness check unconditionally, which made it the one status
+#              nothing could ever force out.  No entries currently: the last one, for
+#              test_noloop_time_interp.py on gate:q-window-stencil, expired when #242 merged.
+#
+# Everything here was collected and run individually on CIT with the IGWN conda python
+# (3.11, numpy 1.26.4, lal 7.7.0) on 2026-09-03; the counts quoted are from that run.
+#
+# Adding a line here is a DECISION, not a way to silence the check.  If a file belongs in CI,
+# register it with a job instead -- then delete its line, which the check will demand anyway.
+
+# ---------------------------------------------------------------------------------------
+# LEGACY -- the pre-package driver suite.  28 files, none importable.
+#
+# These are the original hand-run RIFT test drivers from before the code became a package.
+# `pytest --collect-only` on each reports a collection ERROR, not zero tests: e.g.
+# test_likelihood.py -> ModuleNotFoundError: No module named 'factored_likelihood'.  They are
+# not a coverage gap, because there is no coverage to lose; they are a decision nobody has
+# made.  Recommendation, deliberately NOT taken in the PR that created this roster: delete
+# them, or move them under Code/old/ where test_regions.py already sits.  That is a judgment
+# call about historical value, and it belongs to a maintainer, not to a CI audit.
+MonteCarloMarginalizeCode/Code/old/test_regions.py                            LEGACY  imports lalsimutils; also raises IndexError at collection on current numpy
+MonteCarloMarginalizeCode/Code/test/factored_likelihood_test.py               LEGACY  imports factored_likelihood, lalsimutils
+MonteCarloMarginalizeCode/Code/test/test_SpokesRefine.py                      LEGACY  imports spokes
+MonteCarloMarginalizeCode/Code/test/test_data_vs_template.py                  LEGACY  imports factored_likelihood, lalsimutils, ourio, ourparams
+MonteCarloMarginalizeCode/Code/test/test_effectiveFisher.py                   LEGACY  imports effectiveFisher, lalsimutils
+MonteCarloMarginalizeCode/Code/test/test_hlm.py                               LEGACY  imports factored_likelihood
+MonteCarloMarginalizeCode/Code/test/test_interpolation.py                     LEGACY  imports factored_likelihood
+MonteCarloMarginalizeCode/Code/test/test_like_and_samp.py                     LEGACY  imports common_cl, factored_likelihood, lalsimutils, ourio, ourparams, xmlutils
+MonteCarloMarginalizeCode/Code/test/test_like_and_samp_margPsi.py             LEGACY  imports factored_likelihood, lalsimutils, ourio
+MonteCarloMarginalizeCode/Code/test/test_like_and_samp_noisydata.py           LEGACY  imports factored_likelihood, lalsimutils, ourio
+MonteCarloMarginalizeCode/Code/test/test_like_and_samp_noisydata_margPsi.py   LEGACY  imports factored_likelihood, lalsimutils, ourio
+MonteCarloMarginalizeCode/Code/test/test_like_and_samp_simplified.py          LEGACY  imports common_cl, factored_likelihood, lalsimutils, ourio, ourparams, xmlutils
+MonteCarloMarginalizeCode/Code/test/test_like_and_samp_singleifo.py           LEGACY  imports factored_likelihood, lalsimutils, ourio
+MonteCarloMarginalizeCode/Code/test/test_like_singletemplate_emcee.py         LEGACY  imports factored_likelihood, lalsimutils, ourio
+MonteCarloMarginalizeCode/Code/test/test_like_singletemplate_emcee_pt.py      LEGACY  imports factored_likelihood, lalsimutils, ourio
+MonteCarloMarginalizeCode/Code/test/test_likelihood.py                        LEGACY  imports factored_likelihood
+MonteCarloMarginalizeCode/Code/test/test_precompute.py                        LEGACY  imports factored_likelihood, lalsimutils
+MonteCarloMarginalizeCode/Code/test/test_precompute_noisydata.py              LEGACY  imports factored_likelihood
+MonteCarloMarginalizeCode/Code/test/test_precompute_singleifo.py              LEGACY  imports factored_likelihood
+MonteCarloMarginalizeCode/Code/test/test_profile_components.py                LEGACY  imports factored_likelihood, lalsimutils, ourparams
+MonteCarloMarginalizeCode/Code/test/test_psd_xml_io.py                        LEGACY  imports factored_likelihood, lalsimutils
+MonteCarloMarginalizeCode/Code/test/test_response_functions_Q.py              LEGACY  imports factored_likelihood, lalsimutils, ourio, ourparams, xmlutils
+MonteCarloMarginalizeCode/Code/test/test_rhotimeseries.py                     LEGACY  imports factored_likelihood
+MonteCarloMarginalizeCode/Code/test/test_runvar.py                            LEGACY  imports statutils
+MonteCarloMarginalizeCode/Code/test/test_sampler_visualization.py             LEGACY  imports lalsimutils
+MonteCarloMarginalizeCode/Code/test/integrators/test_mcsamp.py                LEGACY  imports statutils and mpl_toolkits.basemap
+MonteCarloMarginalizeCode/Code/test/integrators/test_mcsamp_pinned.py         LEGACY  imports statutils
+MonteCarloMarginalizeCode/Code/test/integrators/test_mcsampler_gpu.py         LEGACY  imports ourio
+
+# ---------------------------------------------------------------------------------------
+# HANDRUN -- quantitative studies with real internal gates, run by hand.
+#
+# These are NOT dead.  Each has a __main__, argparse, and its own pass/fail criterion (a
+# 4-sigma bias gate, an efficiency ratio, a bias-ordering assertion).  They collect ZERO items
+# under pytest and exit 5, so wiring them into a pytest job as they stand would report a green
+# tick over an empty run -- the exact trap .travis/test-slowrot.sh documents.
+#
+# The right move for the first six is to convert them the way the shape-recovery suite was
+# converted: a pytest wrapper under test/expensive_before_merging/, skipped unless
+# RIFT_RUN_EXPENSIVE=1, so the merge gate can invoke them and CI does not pay for them.  That
+# is a per-suite piece of work with a real cost, and it is not attempted here.
+MonteCarloMarginalizeCode/Code/test/integrators/test_AV_bootstrap.py                 HANDRUN  AV warm-start efficiency study with a 4-sigma bias gate; 0 collected, exit 5
+MonteCarloMarginalizeCode/Code/test/integrators/test_AV_warmstart_safety.py          HANDRUN  anti-bias guard for reusing a proposal across problems; 0 collected, exit 5
+MonteCarloMarginalizeCode/Code/test/integrators/test_portfolio_adaptive_alloc.py     HANDRUN  portfolio draw-allocation study vs standalone AV; 0 collected, exit 5
+MonteCarloMarginalizeCode/Code/test/integrators/test_portfolio_balance_heuristic.py  HANDRUN  portfolio safety under a decoy member; 0 collected, exit 5
+MonteCarloMarginalizeCode/Code/test/integrators/test_portfolio_oracle.py             HANDRUN  needle-target oracle study; 0 collected, exit 5
+MonteCarloMarginalizeCode/Code/test/integrators/test_mcsamplerEnsemble.py            HANDRUN  GMM-vs-mcsampler comparison demo, prints results; 0 collected, exit 5
+MonteCarloMarginalizeCode/Code/test/integrators/test_mcsamplerEnsemble_AdaptationDemo.py HANDRUN  adaptation demo, plots and prints; 0 collected, exit 5
+MonteCarloMarginalizeCode/Code/demo/rift/export_likelihoods/head_to_head/run_test.py HANDRUN  GP-vs-RF figure driver for a demo; --stage picks a stage; 0 collected, exit 5
+
+# ---------------------------------------------------------------------------------------
+# OPTDEP / GPU -- would run, but not in the environment CI builds.
+#
+# The two jax_gp files are the strongest candidates for promotion: jax-ile-check already
+# installs a CPU jax stack, so adding them there costs only optax and a raised EXPECTED_TESTS.
+# Not done here because that job's counts are pinned and this PR does not own them.
+MonteCarloMarginalizeCode/Code/RIFT/interpolators/jax_gp/test_coordinates.py   OPTDEP  needs jax; belongs in jax-ile-check, which already installs a CPU jax stack
+MonteCarloMarginalizeCode/Code/RIFT/interpolators/jax_gp/test_interpolators.py OPTDEP  needs jax and optax; 10 collected, all 10 error on ModuleNotFoundError optax rather than skipping
+# The two cupy parity legs are NOT listed here.  PR #242 landed, and its
+# .travis/test-q-window-stencil.sh names both in an EXCLUDED array -- with the same reason,
+# and with its own fail-closed check that an EXCLUDED path still exists and does not carry
+# the marker.  That is a better home than this file: the decision sits beside the gate it
+# belongs to.  Their roster lines were deleted when #242 merged, exactly as the census
+# demanded ("listed as GPU but IS now reachable -- delete it").
+MonteCarloMarginalizeCode/Code/test/backends/test_backends_lowlevel.py         OPTDEP  needs lscsoft-glue and htcondor; 15 collected, 15 pass where both are installed
+MonteCarloMarginalizeCode/Code/test/hyperpipe/tests/test_hydra_integration.py  OPTDEP  needs hydra and omegaconf; skips cleanly without them
+MonteCarloMarginalizeCode/Code/test/test_gwsignal_teob_mode_sign.py            OPTDEP  needs the gwsignal TEOB route; 2 collected, 1 passes and 1 skips without EOBRun_module
+MonteCarloMarginalizeCode/Code/test/test_gwsignal_teob_near_aligned.py         OPTDEP  needs EOBRun_module; 1 collected, skips without it
+MonteCarloMarginalizeCode/Code/test/test_teobresums_compat.py                  OPTDEP  TEOBResumS compat shim; 15 collected and all pass on CIT, unverified on a runner
+MonteCarloMarginalizeCode/Code/test/test_rimsky_integration.py                 OPTDEP  companion to test_rimsky_end_to_end.py; belongs in the rimsky-integration job, whose env this PR does not own
+MonteCarloMarginalizeCode/Code/test/integrators/test_NF_reuse.py               OPTDEP  needs nflows for the normalizing-flow store; collection errors without it
+MonteCarloMarginalizeCode/Code/test/integrators/test_mcsamp_vegas.py           OPTDEP  needs the vegas package, commented out of requirements.txt; NameError at import without it
+MonteCarloMarginalizeCode/Code/test/integrators/test_mcsampler_rosenbrock.py   HANDRUN  Rosenbrock sampler study; its docstring pairs it with plot_posterior_corner.py by hand
+MonteCarloMarginalizeCode/Code/test/test_eosmanager_misc.py                    OPTDEP  needs LALSIMULATION_DATADIR set; raises KeyError at import without it
+MonteCarloMarginalizeCode/Code/test/test_skysamp.py                            LEGACY  imports lalinference.bayestar.fits, removed upstream; cannot be imported
+MonteCarloMarginalizeCode/Code/test/test_mcsampler_foridiots.py                BROKEN  NameError int_vals at import; a plotting demo that no longer runs at all
+
+# ---------------------------------------------------------------------------------------
+# EXPENSIVE -- correctly gated already, by an env var rather than by CI membership.
+MonteCarloMarginalizeCode/Code/test/expensive_before_merging/integrators/test_shape_recovery.py          EXPENSIVE  4 collected, all skip unless RIFT_RUN_EXPENSIVE=1
+MonteCarloMarginalizeCode/Code/test/expensive_before_merging/integrators/test_escaped_mass_diagnostic.py EXPENSIVE  5 collected, all skip unless RIFT_RUN_EXPENSIVE=1
+
+# ---------------------------------------------------------------------------------------
+# BROKEN -- collects and FAILS on rift_O4d today.  Found only because this audit ran them.
+#
+# test_replica_pooling.py is the clearest argument for the census.  It loads six helpers out of
+# bin/integrate_likelihood_extrinsic_batchmode by REGEX and exec()s them into a synthetic
+# module.  The driver has since been refactored so that _lnZ_of_rvs and _kish_neff_of_rvs
+# delegate to a seventh helper, _lw_of, which the regex list does not extract.  Inside the
+# exec'd module _lw_of is undefined; the driver's own `except Exception: return None` swallows
+# the NameError, both helpers return None, and 10 of 15 tests die on `None - float`.  Adding
+# "_lw_of" to the slice list in the test is the immediate fix.  The reimplemented-harness shape
+# is the real problem and outlives that fix.
+MonteCarloMarginalizeCode/Code/test/integrators/test_replica_pooling.py  BROKEN  10 of 15 fail; its regex helper-slicer misses _lw_of, added to the driver after the test was written
+MonteCarloMarginalizeCode/Code/test/hyperpipe/tests/test_marg_list.py    BROKEN  2 of 3 fail; _stage_event_file writes event-N.net into base_dir while the test and assemble_marg_list's own run_dir docstring say run_dir

.travis/test-calmarg.sh

@@ -3,7 +3,10 @@
 # reduction and the per-realization self-term fix: precompute time-alignment +
 # identity-cal cross terms, the loop/fused reduction vs a brute-force reference
 # (incl. n_cal==1), the low-rank SVD self-term basis vs a direct band integral, and
-# the backtest of the cal reduction (default + distance-marginalization helpers).
+# the backtest of the cal reduction (default + distance-marginalization helpers),
+# and the driver's calibration OPTION-COMPATIBILITY gate (which flag combinations
+# are refused at startup instead of silently degrading to the zero-calibration
+# likelihood, and which must still be accepted).
 # Any nonzero exit fails the job (set -e).  GPU/CUDA paths are exercised separately
 # on hardware; here every check runs on the numpy backend.
 set -euo pipefail
@@ -13,6 +16,20 @@ command -v "$PY" >/dev/null 2>&1 || PY="$(command -v python3)"
 CODE="MonteCarloMarginalizeCode/Code"
 export OMP_NUM_THREADS=1
 
+# INVARIANT: this gate tests THIS CHECKOUT, never an installed build.  Must PREPEND --
+# appending lets a caller's PYTHONPATH win.
+#
+# Without it the first check below runs as `python <path>/test_precompute_alignment.py`, which
+# puts the SCRIPT'S directory on sys.path and not Code/, so bare `import RIFT` resolves to
+# whatever is installed.  In CI that happens to be the editable install of this checkout, so
+# it passes; on a plain checkout with a real RIFT in the environment it silently tests the
+# INSTALLED code, and reports its staleness as a failure of this branch -- observed on CIT,
+# where the CVMFS IGWN RIFT predates a kwarg the checkout added:
+#   PrecomputeLikelihoodTerms() got an unexpected keyword argument 'calibration_realizations'
+# The `python -m RIFT.calmarg.*` runs below are already safe (cwd is on sys.path under -m);
+# this makes the whole script safe the same way its sibling gates are.
+export PYTHONPATH="$PWD/$CODE${PYTHONPATH:+:$PYTHONPATH}"
+
 # precompute alignment + identity-cal self-term cross terms == baseline
 "$PY" "$CODE/RIFT/calmarg/test_precompute_alignment.py"
 
@@ -24,4 +41,19 @@ export OMP_NUM_THREADS=1
   && "$PY" -m RIFT.calmarg.backtest_calmarg --backend cpu --n-cal 8 --methods reference,in_loop_B \
   && "$PY" -m RIFT.calmarg.backtest_calmarg --backend cpu --n-cal 8 --loglikelihood distmarg --methods reference,in_loop_B )
 
+# Option-compatibility gate: which calibration option combinations the ILE driver
+# REFUSES at startup, and -- the load-bearing half -- which it must still ACCEPT.
+# pytest here, not a bare module run: these are parametrized accept/refuse pairs plus
+# real subprocess invocations of the driver, and `set -e` turns pytest's exit 5 ("no
+# tests ran") into a job failure rather than a silent green.  A collection floor is
+# asserted for the same reason: a file that stops collecting is a lost gate.
+_n=$( cd "$CODE" && "$PY" -m pytest -q --collect-only "RIFT/calmarg/test_option_compat.py" 2>/dev/null | grep -c '::' || true )
+if [ "${_n:-0}" -lt 20 ]; then
+  echo "test-calmarg.sh: option-compat suite collected ${_n:-0} tests, expected >= 20." >&2
+  echo "  Tests were renamed, removed, or the file stopped importing.  Fix it; as is," >&2
+  echo "  the gate covers less than it claims." >&2
+  exit 1
+fi
+( cd "$CODE" && PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}" "$PY" -m pytest -q "RIFT/calmarg/test_option_compat.py" )
+
 echo "calmarg CPU regression gate: PASS"