From 88588c17ecf722256dedbf19c9025f3650c056c8 Mon Sep 17 00:00:00 2001 From: seohyonkim Date: Tue, 5 Aug 2025 17:34:11 +0200 Subject: [PATCH 1/7] fill out config --- scripts/create_component/create_python_metric.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/create_component/create_python_metric.sh b/scripts/create_component/create_python_metric.sh index d36bc7a95..1da0b0be4 100755 --- a/scripts/create_component/create_python_metric.sh +++ b/scripts/create_component/create_python_metric.sh @@ -3,6 +3,6 @@ set -e common/scripts/create_component \ - --name my_python_metric \ + --name ksim \ --language python \ --type metric From 4f08836d1d32ba0699e390e3cb6847fd1888dc7a Mon Sep 17 00:00:00 2001 From: seohyonkim Date: Tue, 5 Aug 2025 18:06:34 +0200 Subject: [PATCH 2/7] wip --- src/metrics/ksim/config.vsh.yaml | 65 ++++++++++++++++++++++++++++++++ src/metrics/ksim/script.py | 58 ++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 src/metrics/ksim/config.vsh.yaml create mode 100644 src/metrics/ksim/script.py diff --git a/src/metrics/ksim/config.vsh.yaml b/src/metrics/ksim/config.vsh.yaml new file mode 100644 index 000000000..f85b3cc6e --- /dev/null +++ b/src/metrics/ksim/config.vsh.yaml @@ -0,0 +1,65 @@ +__merge__: ../../api/comp_metric.yaml +name: ksim +info: + metrics: + + - name: ksim + label: kSIM + summary: "The kSIM acceptance rate measures whether cells of the same pre-annotated cell type are still close to each other in the local neighborhoods after batch correction." + description: | + The kSIM acceptance rate requires ground truth cell type information and measures whether the neighbors of a cell have the same cell type as it does. If a method overcorrects the batch effects, it will have a low kSIM acceptance rate. We use the HNSW algorithm to find k-NNs (including the cell itself) for each cell i and denote the number of neighbors that have the same cell type as i as . In addition, we require at least β fraction of neighbors of cell i to have the same cell type as i in order to say cell i has a consistent neighborhood. + references: + doi: + - 10.1038/s41592-020-0905-x + links: + documentation: https://pegasus.readthedocs.io/en/stable/api/pegasus.calc_kSIM.html#pegasus.calc_kSIM + repository: https://github.com/lilab-bcb/pegasus + min: 0 + max: 1 + maximize: true + +arguments: + - name: "--rep" + type: "string" + default: "pca" + description: The embedding representation to consider. + - name: "--K" + type: "integer" + default: 24 + description: The number of nearest neighbors to be considered. + - name: "--min_rate" + type: "double" + default: 0.9 + description: Acceptance rate threshold. A cell is accepted if its kSIM rate is larger than or equal to min_rate. + - name: "--n_jobs" + type: "integer" + default: -1 + description: Number of threads used. If -1, use all physical CPU cores. + - name: "--random_state" + type: "integer" + default: 0 + description: Random seed set for reproducing results. + - name: "--use_cache" + type: "boolean" + default: True + description: If use cache results for kNN. + +resources: + - type: python_script + path: script.py + - path: /src/utils/read_anndata_partial.py + +engines: + - type: docker + image: openproblems/base_python:1.0.0 + setup: + - type: python + pypi: + - pegasuspy==1.10.2 + + +runners: + - type: executable + - type: nextflow + directives: + label: [midtime,midmem,midcpu] diff --git a/src/metrics/ksim/script.py b/src/metrics/ksim/script.py new file mode 100644 index 000000000..57d377586 --- /dev/null +++ b/src/metrics/ksim/script.py @@ -0,0 +1,58 @@ +import anndata as ad +import sys +import pegasus as pg + +## VIASH START +# Note: this section is auto-generated by viash at runtime. To edit it, make changes +# in config.vsh.yaml and then run `viash config inject config.vsh.yaml`. +par = { + 'input_integrated': 'resources_test/task_batch_integration/cxg_immune_cell_atlas/integrated_full.h5ad', + 'input_solution': 'resources_test/.../solution.h5ad', + 'output': 'output.h5ad', + "rep": "pca", + "K": 24, + "min_rate": 0.9, + "n_jobs": -1, + "random_state": 0, + "use_cache": True +} +meta = { + 'name': 'ksim' +} +## VIASH END + +sys.path.append(meta["resources_dir"]) +from read_anndata_partial import read_anndata + +print('Reading input files', flush=True) +adata = read_anndata(par['input_integrated'], obs='obs', obsm='obsm', uns='uns') +adata.obs = read_anndata(par['input_solution'], obs='obs').obs +adata.uns |= read_anndata(par['input_solution'], uns='uns').uns + +print('Compute metrics', flush=True) +score = pg.calc_kSIM( + adata, + attr='cell_type', + rep=par["rep"], + K=par["K"], + min_rate=par["min_rate"], + n_jobs=par["n_jobs"], + random_state=par["random_state"], + use_cache=par["use_cache"] +)[1] + +# TODO RETURNS A TOUPLE OF TWO THINGS: kSIM_mean (float) – Mean kSIM rate over all the cells., kSIM_accept_rate (float) – kSIM Acceptance rate of the sample + +print('Create output AnnData object', flush=True) +output = ad.AnnData( + uns={ + 'dataset_id': adata.uns['dataset_id'], + 'normalization_id': adata.uns['normalization_id'], + 'method_id': adata.uns['method_id'], + 'metric_ids': [ meta['name'] ], + 'metric_values': [ score ] + } +) + +print("Write output AnnData to file", flush=True) +output.write_h5ad(par['output'], compression='gzip') From 47882e3177cd239d27a1aae3d023a1a7551caa23 Mon Sep 17 00:00:00 2001 From: seohyonkim Date: Tue, 12 Aug 2025 13:32:15 +0200 Subject: [PATCH 3/7] working --- src/metrics/ksim/config.vsh.yaml | 4 ---- src/metrics/ksim/script.py | 34 +++++++++++--------------------- 2 files changed, 11 insertions(+), 27 deletions(-) diff --git a/src/metrics/ksim/config.vsh.yaml b/src/metrics/ksim/config.vsh.yaml index f85b3cc6e..4c7a9b1bf 100644 --- a/src/metrics/ksim/config.vsh.yaml +++ b/src/metrics/ksim/config.vsh.yaml @@ -19,10 +19,6 @@ info: maximize: true arguments: - - name: "--rep" - type: "string" - default: "pca" - description: The embedding representation to consider. - name: "--K" type: "integer" default: 24 diff --git a/src/metrics/ksim/script.py b/src/metrics/ksim/script.py index 57d377586..fa8a7fe7a 100644 --- a/src/metrics/ksim/script.py +++ b/src/metrics/ksim/script.py @@ -1,25 +1,9 @@ import anndata as ad import sys import pegasus as pg +import pegasusio +from scipy.sparse import csr_matrix -## VIASH START -# Note: this section is auto-generated by viash at runtime. To edit it, make changes -# in config.vsh.yaml and then run `viash config inject config.vsh.yaml`. -par = { - 'input_integrated': 'resources_test/task_batch_integration/cxg_immune_cell_atlas/integrated_full.h5ad', - 'input_solution': 'resources_test/.../solution.h5ad', - 'output': 'output.h5ad', - "rep": "pca", - "K": 24, - "min_rate": 0.9, - "n_jobs": -1, - "random_state": 0, - "use_cache": True -} -meta = { - 'name': 'ksim' -} -## VIASH END sys.path.append(meta["resources_dir"]) from read_anndata_partial import read_anndata @@ -28,20 +12,24 @@ adata = read_anndata(par['input_integrated'], obs='obs', obsm='obsm', uns='uns') adata.obs = read_anndata(par['input_solution'], obs='obs').obs adata.uns |= read_anndata(par['input_solution'], uns='uns').uns +print(adata) + +print('Convert to pegasusio.MultimodalData...', flush=True) +adata.X = csr_matrix(adata.shape) +mmdata = pegasusio.MultimodalData(adata) print('Compute metrics', flush=True) score = pg.calc_kSIM( - adata, + mmdata, attr='cell_type', - rep=par["rep"], + rep='emb', K=par["K"], min_rate=par["min_rate"], n_jobs=par["n_jobs"], random_state=par["random_state"], use_cache=par["use_cache"] -)[1] - -# TODO RETURNS A TOUPLE OF TWO THINGS: kSIM_mean (float) – Mean kSIM rate over all the cells., kSIM_accept_rate (float) – kSIM Acceptance rate of the sample +) +print("score:", score) print('Create output AnnData object', flush=True) output = ad.AnnData( From 74bd5c54b757612255213cd1fb19cb8015d9ca89 Mon Sep 17 00:00:00 2001 From: seo <159482645+seohyonkim@users.noreply.github.com> Date: Thu, 28 Aug 2025 14:59:18 +0200 Subject: [PATCH 4/7] Update scripts/create_component/create_python_metric.sh Co-authored-by: Robrecht Cannoodt --- scripts/create_component/create_python_metric.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/create_component/create_python_metric.sh b/scripts/create_component/create_python_metric.sh index 1da0b0be4..d36bc7a95 100755 --- a/scripts/create_component/create_python_metric.sh +++ b/scripts/create_component/create_python_metric.sh @@ -3,6 +3,6 @@ set -e common/scripts/create_component \ - --name ksim \ + --name my_python_metric \ --language python \ --type metric From a8814e29e020c3b8998d133035af6b5b84a23891 Mon Sep 17 00:00:00 2001 From: seohyonkim Date: Tue, 9 Sep 2025 18:00:14 +0200 Subject: [PATCH 5/7] add to changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09d672d08..9f9e1031e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ * Added `metrics/kbet_pg` and `metrics/kbet_pg_label` components (PR #52). +* Added `metircs/ksim` component (PR #75). + ## Minor changes * Un-pin the scPRINT version and update parameters (PR #51) From 86581248c00f87b0abfb95dcbbdce1111db00846 Mon Sep 17 00:00:00 2001 From: seohyonkim Date: Thu, 25 Sep 2025 03:20:56 +0200 Subject: [PATCH 6/7] rephrase metric infos --- src/metrics/ksim/config.vsh.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/metrics/ksim/config.vsh.yaml b/src/metrics/ksim/config.vsh.yaml index 4c7a9b1bf..b9ea44a48 100644 --- a/src/metrics/ksim/config.vsh.yaml +++ b/src/metrics/ksim/config.vsh.yaml @@ -5,9 +5,11 @@ info: - name: ksim label: kSIM - summary: "The kSIM acceptance rate measures whether cells of the same pre-annotated cell type are still close to each other in the local neighborhoods after batch correction." + summary: "The kSIM acceptance rate evaluates whether cells of the same known cell type remain grouped together in their neighborhoods after batch correction." description: | - The kSIM acceptance rate requires ground truth cell type information and measures whether the neighbors of a cell have the same cell type as it does. If a method overcorrects the batch effects, it will have a low kSIM acceptance rate. We use the HNSW algorithm to find k-NNs (including the cell itself) for each cell i and denote the number of neighbors that have the same cell type as i as . In addition, we require at least β fraction of neighbors of cell i to have the same cell type as i in order to say cell i has a consistent neighborhood. + The kSIM acceptance rate uses prior knowledge of cell type labels to assess local neighborhood consistency. For each cell, we look at its nearest neighbors (including itself) and check how many share the same cell type. + A cell is considered to have a consistent neighborhood if the majority of its neighbors still belong to its own type. The acceptance rate is the overall fraction of such cells in the dataset. + A high kSIM value means that cells of the same type remain locally clustered after correction, while a low value suggests that the correction has disrupted true biological structure—for example, by overcorrecting batch effects. references: doi: - 10.1038/s41592-020-0905-x From abd1ef63adc1d3ad921a22e76054e18ea05a2a8c Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Fri, 21 Aug 2026 17:31:26 +0200 Subject: [PATCH 7/7] fix ksim metric * Output both `ksim_mean` and `ksim_accept_rate` -- `calc_kSIM()` returns a tuple, so writing it as one value produced a 1x2 `metric_values` array * Add missing `metric_type: embedding`, without which `run_benchmark` filters the metric out * Register `metrics/ksim` in the `run_benchmark` workflow * Drop the tunable arguments in favour of pegasus' defaults (K=25, min_rate=0.9) * Use `meta["cpus"]` instead of every physical core on the node * Align the docker image and pypi dependencies with `metrics/kbet_pg` * Add the `## VIASH START` block, rephrase the descriptions in passive voice, fix the changelog typo --- CHANGELOG.md | 2 +- src/metrics/ksim/config.vsh.yaml | 95 ++++++++++++--------- src/metrics/ksim/script.py | 43 +++++++--- src/workflows/run_benchmark/config.vsh.yaml | 1 + src/workflows/run_benchmark/main.nf | 1 + 5 files changed, 88 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f9e1031e..f89fecbfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ * Added `metrics/kbet_pg` and `metrics/kbet_pg_label` components (PR #52). -* Added `metircs/ksim` component (PR #75). +* Added `metrics/ksim` component, reporting the `ksim_mean` and `ksim_accept_rate` metrics (PR #75). ## Minor changes diff --git a/src/metrics/ksim/config.vsh.yaml b/src/metrics/ksim/config.vsh.yaml index b9ea44a48..ad0e97941 100644 --- a/src/metrics/ksim/config.vsh.yaml +++ b/src/metrics/ksim/config.vsh.yaml @@ -1,63 +1,80 @@ -__merge__: ../../api/comp_metric.yaml +__merge__: /src/api/comp_metric.yaml name: ksim info: + metric_type: embedding metrics: - - - name: ksim - label: kSIM - summary: "The kSIM acceptance rate evaluates whether cells of the same known cell type remain grouped together in their neighborhoods after batch correction." + - name: ksim_mean + label: kSIM mean + summary: Mean fraction of each cell's nearest neighbours that share its cell type + after batch correction. description: | - The kSIM acceptance rate uses prior knowledge of cell type labels to assess local neighborhood consistency. For each cell, we look at its nearest neighbors (including itself) and check how many share the same cell type. - A cell is considered to have a consistent neighborhood if the majority of its neighbors still belong to its own type. The acceptance rate is the overall fraction of such cells in the dataset. - A high kSIM value means that cells of the same type remain locally clustered after correction, while a low value suggests that the correction has disrupted true biological structure—for example, by overcorrecting batch effects. + The kSIM metric uses prior knowledge of cell type labels to assess whether local + neighbourhoods remain biologically consistent after batch correction (Li et al., + Nat Methods 2020). For each cell, its K nearest neighbours in the integrated + embedding are retrieved (the cell itself counts as its own first neighbour) and the + fraction of those neighbours that carry the same cell type label is recorded. + kSIM mean is the average of these per-cell fractions over the whole dataset. + + A high value means that cells of the same type stay locally grouped after + correction, whereas a low value indicates that the correction has disrupted true + biological structure, for example by overcorrecting batch effects. This variant is + continuous and therefore separates methods across the whole range of the scale. + + This implementation uses the `pegasus.calc_kSIM` function with its default + neighbourhood size of K=25. references: - doi: + doi: - 10.1038/s41592-020-0905-x links: - documentation: https://pegasus.readthedocs.io/en/stable/api/pegasus.calc_kSIM.html#pegasus.calc_kSIM + homepage: https://pegasus.readthedocs.io/en/stable/ + documentation: https://pegasus.readthedocs.io/en/stable/api/pegasus.calc_kSIM.html repository: https://github.com/lilab-bcb/pegasus min: 0 max: 1 maximize: true + - name: ksim_accept_rate + label: kSIM acceptance rate + summary: Fraction of cells whose neighbourhood is almost entirely made up of their + own cell type after batch correction. + description: | + The kSIM acceptance rate is the thresholded companion of kSIM mean (Li et al., + Nat Methods 2020). The same per-cell fractions of same-cell-type nearest neighbours + are computed, but a cell is then counted as accepted only if that fraction reaches + at least `min_rate`. The acceptance rate is the proportion of accepted cells in the + dataset. -arguments: - - name: "--K" - type: "integer" - default: 24 - description: The number of nearest neighbors to be considered. - - name: "--min_rate" - type: "double" - default: 0.9 - description: Acceptance rate threshold. A cell is accepted if its kSIM rate is larger than or equal to min_rate. - - name: "--n_jobs" - type: "integer" - default: -1 - description: Number of threads used. If -1, use all physical CPU cores. - - name: "--random_state" - type: "integer" - default: 0 - description: Random seed set for reproducing results. - - name: "--use_cache" - type: "boolean" - default: True - description: If use cache results for kNN. + Because it is a thresholded quantity, this variant is more stringent than kSIM mean: + it reports how many cells retain an essentially pure neighbourhood rather than how + pure neighbourhoods are on average. It can saturate near 0 for poorly separated + integrations, so it is best read alongside kSIM mean. + This implementation uses the `pegasus.calc_kSIM` function with its defaults of + K=25 and min_rate=0.9, meaning that at least 23 of a cell's 25 neighbours must + share its cell type. + references: + doi: + - 10.1038/s41592-020-0905-x + links: + homepage: https://pegasus.readthedocs.io/en/stable/ + documentation: https://pegasus.readthedocs.io/en/stable/api/pegasus.calc_kSIM.html + repository: https://github.com/lilab-bcb/pegasus + min: 0 + max: 1 + maximize: true resources: - type: python_script path: script.py - path: /src/utils/read_anndata_partial.py - engines: - type: docker - image: openproblems/base_python:1.0.0 + image: openproblems/base_python:1 setup: - - type: python - pypi: - - pegasuspy==1.10.2 - - + - type: python + pypi: + - pegasuspy + - setuptools<81 runners: - type: executable - type: nextflow directives: - label: [midtime,midmem,midcpu] + label: [midtime, midmem, midcpu] diff --git a/src/metrics/ksim/script.py b/src/metrics/ksim/script.py index fa8a7fe7a..7134fde7f 100644 --- a/src/metrics/ksim/script.py +++ b/src/metrics/ksim/script.py @@ -1,35 +1,50 @@ -import anndata as ad import sys +import anndata as ad import pegasus as pg import pegasusio from scipy.sparse import csr_matrix +## VIASH START +par = { + 'input_integrated': 'resources_test/task_batch_integration/cxg_immune_cell_atlas/integrated_processed.h5ad', + 'input_solution': 'resources_test/task_batch_integration/cxg_immune_cell_atlas/solution.h5ad', + 'output': 'output.h5ad', +} + +meta = { + 'name': 'ksim', +} +## VIASH END sys.path.append(meta["resources_dir"]) from read_anndata_partial import read_anndata -print('Reading input files', flush=True) +n_threads = meta["cpus"] or -1 + +print('Read input...', flush=True) adata = read_anndata(par['input_integrated'], obs='obs', obsm='obsm', uns='uns') adata.obs = read_anndata(par['input_solution'], obs='obs').obs adata.uns |= read_anndata(par['input_solution'], uns='uns').uns -print(adata) +print(adata, flush=True) print('Convert to pegasusio.MultimodalData...', flush=True) adata.X = csr_matrix(adata.shape) mmdata = pegasusio.MultimodalData(adata) -print('Compute metrics', flush=True) -score = pg.calc_kSIM( +print('Compute kSIM...', flush=True) +# K and min_rate are pegasus' own defaults, spelled out here so the metric stays +# stable across pegasus releases. Note that pegasus caps K at sqrt(n_obs), so the +# effective neighbourhood is smaller for datasets with fewer than 625 cells. +ksim_mean, ksim_accept_rate = pg.calc_kSIM( mmdata, attr='cell_type', rep='emb', - K=par["K"], - min_rate=par["min_rate"], - n_jobs=par["n_jobs"], - random_state=par["random_state"], - use_cache=par["use_cache"] + K=25, + min_rate=0.9, + n_jobs=n_threads, ) -print("score:", score) +print('kSIM mean:', ksim_mean, flush=True) +print('kSIM acceptance rate:', ksim_accept_rate, flush=True) print('Create output AnnData object', flush=True) output = ad.AnnData( @@ -37,10 +52,10 @@ 'dataset_id': adata.uns['dataset_id'], 'normalization_id': adata.uns['normalization_id'], 'method_id': adata.uns['method_id'], - 'metric_ids': [ meta['name'] ], - 'metric_values': [ score ] + 'metric_ids': [ 'ksim_mean', 'ksim_accept_rate' ], + 'metric_values': [ ksim_mean, ksim_accept_rate ] } ) -print("Write output AnnData to file", flush=True) +print('Write data to file', flush=True) output.write_h5ad(par['output'], compression='gzip') diff --git a/src/workflows/run_benchmark/config.vsh.yaml b/src/workflows/run_benchmark/config.vsh.yaml index 09905ad0f..62b8ce4e7 100644 --- a/src/workflows/run_benchmark/config.vsh.yaml +++ b/src/workflows/run_benchmark/config.vsh.yaml @@ -119,6 +119,7 @@ dependencies: - name: metrics/kbet - name: metrics/kbet_pg - name: metrics/kbet_pg_label + - name: metrics/ksim - name: metrics/lisi - name: metrics/pcr # data processors diff --git a/src/workflows/run_benchmark/main.nf b/src/workflows/run_benchmark/main.nf index 6196f749b..64867fb23 100644 --- a/src/workflows/run_benchmark/main.nf +++ b/src/workflows/run_benchmark/main.nf @@ -58,6 +58,7 @@ metrics = [ kbet, kbet_pg, kbet_pg_label, + ksim, lisi, pcr ]