diff --git a/content/FLASHDeconv/FLASHDeconvWorkflow.py b/content/FLASHDeconv/FLASHDeconvWorkflow.py index 2d6e613..b098690 100644 --- a/content/FLASHDeconv/FLASHDeconvWorkflow.py +++ b/content/FLASHDeconv/FLASHDeconvWorkflow.py @@ -1,3 +1,5 @@ +import time + import pandas as pd import streamlit as st @@ -25,65 +27,62 @@ wf.show_execution_section() with t[3]: + # (file name suffix, name tag, prefix FLASHDeconv uses in its own output) + UPLOAD_FILE_TYPES = ( + ('deconv.mzML', 'out_deconv_mzML', 'out'), + ('annotated.mzML', 'anno_annotated_mzML', 'anno'), + ('spec1.tsv', 'spec1_tsv', ''), + ('spec2.tsv', 'spec2_tsv', ''), + ) + def process_uploaded_files(uploaded_files): - + + # FLASHDeconv names its output files 'out_deconv.mzML' and + # 'anno_annotated.mzML', so the part in front of the suffix ('out' + # and 'anno') is not an experiment name: file those under a common + # dataset, otherwise the two halves of a run never meet. + default_dataset = time.strftime('uploaded_%Y%m%d-%H%M%S') + # Store all uploaded files for file in uploaded_files: - if file.name.endswith("mzML"): - if file.name.endswith('_deconv.mzML'): - wf.file_manager.store_file( - file.name.split('_deconv.mzML')[0], 'out_deconv_mzML', file - ) - elif file.name.endswith('_annotated.mzML'): - wf.file_manager.store_file( - file.name.split('_annotated.mzML')[0], 'anno_annotated_mzML', file - ) - else: - st.warning(f'Invalid file : {file.name}') - elif file.name.endswith("tsv"): - if file.name.endswith('_spec1.tsv'): - wf.file_manager.store_file( - file.name.split('_spec1.tsv')[0], 'spec1_tsv', file - ) - elif file.name.endswith('_spec2.tsv'): - wf.file_manager.store_file( - file.name.split('_spec2.tsv')[0], 'spec2_tsv', file - ) - else: - st.warning(f'Invalid file : {file.name}') + for suffix, name_tag, tool_prefix in UPLOAD_FILE_TYPES: + if not file.name.endswith(suffix): + continue + experiment = file.name[:-len(suffix)].rstrip('_') + if experiment in ('', tool_prefix): + experiment = default_dataset + wf.file_manager.store_file(experiment, name_tag, file) + break else: st.warning(f'Invalid file : {file.name}') - + # Get the unparsed files input_files = set(wf.file_manager.get_results_list(['out_deconv_mzML', 'anno_annotated_mzML'])) parsed_files = set(wf.file_manager.get_results_list(['deconv_dfs', 'anno_dfs'])) unparsed_files = input_files - parsed_files - # Get the unpared tsv files - ms1_tsv_files = set(wf.file_manager.get_results_list(['spec1_tsv'])) - parsed_ms1_tsv_files = set(wf.file_manager.get_results_list(['parsed_tsv_file_ms1'])) - ms2_tsv_files = set(wf.file_manager.get_results_list(['spec2_tsv'])) - parsed_ms2_tsv_files = set(wf.file_manager.get_results_list(['parsed_tsv_file_ms2'])) - unparsed_tsv_files = ( - ( - (ms1_tsv_files - parsed_ms1_tsv_files) - | (ms2_tsv_files - parsed_ms2_tsv_files) - ) & input_files - ) - # Process unparsed datasets - for unparsed_dataset in (unparsed_files | unparsed_tsv_files): + for unparsed_dataset in unparsed_files: results = wf.file_manager.get_results( unparsed_dataset, ['out_deconv_mzML', 'anno_annotated_mzML', 'spec1_tsv', 'spec2_tsv'], partial=True ) - - parsed_data = parseDeconv(**results) - - for k, v in parsed_data.items(): - wf.file_manager.store_data(unparsed_dataset, k, v) + if not ('out_deconv_mzML' in results and 'anno_annotated_mzML' in results): + st.warning( + f"Experiment '{unparsed_dataset}' needs both the " + "deconvolved and the annotated mzML file." + ) + continue + + with st.spinner(f"Processing '{unparsed_dataset}'..."): + parseDeconv( + wf.file_manager, unparsed_dataset, + results['out_deconv_mzML'], results['anno_annotated_mzML'], + results.get('spec1_tsv'), results.get('spec2_tsv'), + logger=wf.logger + ) st.subheader("**Upload FLASHDeconv output files (\*_annotated.mzML & \*_deconv.mzML) or spec1/2 TSV files (Qscore Density Plot only)**") st.info( diff --git a/content/FLASHTnT/FLASHTnTWorkflow.py b/content/FLASHTnT/FLASHTnTWorkflow.py index c0f0d70..3ecbe3e 100644 --- a/content/FLASHTnT/FLASHTnTWorkflow.py +++ b/content/FLASHTnT/FLASHTnTWorkflow.py @@ -1,9 +1,12 @@ +import time + import pandas as pd import streamlit as st from pathlib import Path from src.parse.tnt import parseTnT +from src.parse.deconv import parseDeconv from src.Workflow import TagWorkflow from src.common.common import page_setup, save_params @@ -25,35 +28,43 @@ wf.show_execution_section() with t[3]: + # (file name suffix, name tag, prefix the tools use in their own output) + UPLOAD_FILE_TYPES = ( + ('deconv.mzML', 'out_deconv_mzML', 'out'), + ('annotated.mzML', 'anno_annotated_mzML', 'anno'), + ('tags.tsv', 'tags_tsv', ''), + ('tagged.tsv', 'tags_tsv', ''), + ('protein.tsv', 'protein_tsv', ''), + ) + + REQUIRED_FILES = ( + ('deconvolved mzML', 'out_deconv_mzML'), + ('annotated mzML', 'anno_annotated_mzML'), + ('tags.tsv', 'tags_tsv'), + ('protein.tsv', 'protein_tsv'), + ) + def process_uploaded_files(uploaded_files): - + + # FLASHDeconv and FLASHTnT name their output files 'out_deconv.mzML', + # 'anno_annotated.mzML', 'tags.tsv' and 'protein.tsv', so the part in + # front of the suffix is not an experiment name: file those under a + # common dataset, otherwise the files of a run never meet. + default_dataset = time.strftime('uploaded_%Y%m%d-%H%M%S') + # Store all uploaded files for file in uploaded_files: - if file.name.endswith("mzML"): - if file.name.endswith('_deconv.mzML'): - wf.file_manager.store_file( - file.name.split('_deconv.mzML')[0], 'out_deconv_mzML', file - ) - elif file.name.endswith('_annotated.mzML'): - wf.file_manager.store_file( - file.name.split('_annotated.mzML')[0], 'anno_annotated_mzML', file - ) - else: - st.warning(f'Invalid file : {file.name}') - elif file.name.endswith("tsv"): - if file.name.endswith('_tagged.tsv'): - wf.file_manager.store_file( - file.name.split('_tagged.tsv')[0], 'tags_tsv', file - ) - elif file.name.endswith('_protein.tsv'): - wf.file_manager.store_file( - file.name.split('_protein.tsv')[0], 'protein_tsv', file - ) - else: - st.warning(f'Invalid file : {file.name}') + for suffix, name_tag, tool_prefix in UPLOAD_FILE_TYPES: + if not file.name.endswith(suffix): + continue + experiment = file.name[:-len(suffix)].rstrip('_') + if experiment in ('', tool_prefix): + experiment = default_dataset + wf.file_manager.store_file(experiment, name_tag, file) + break else: st.warning(f'Invalid file : {file.name}') - + # Get the unparsed files input_files = set(wf.file_manager.get_results_list( ['out_deconv_mzML', 'anno_annotated_mzML', 'tags_tsv', 'protein_tsv'] @@ -64,22 +75,40 @@ def process_uploaded_files(uploaded_files): unparsed_files = input_files - parsed_files # Process unparsed datasets - for unparsed_dataset in (unparsed_files): + for unparsed_dataset in unparsed_files: results = wf.file_manager.get_results( unparsed_dataset, - ['out_deconv_mzML', 'anno_annotated_mzML', 'tags_tsv', 'protein_tsv'] + ['out_deconv_mzML', 'anno_annotated_mzML', 'tags_tsv', 'protein_tsv'], + partial=True ) - - parsed_data = parseTnT( - results['out_deconv_mzML'], results['anno_annotated_mzML'], - results['tags_tsv'], results['protein_tsv'] - ) - - for k, v in parsed_data.items(): - wf.file_manager.store_data(unparsed_dataset, k, v) + missing = [n for n, tag in REQUIRED_FILES if tag not in results] + if missing: + st.warning( + f"Experiment '{unparsed_dataset}' is missing the " + f"{', '.join(missing)} file(s)." + ) + continue + + with st.spinner(f"Processing '{unparsed_dataset}'..."): + # The tags are matched against the deconvolved masses, so the + # mzML files have to be parsed first + if not wf.file_manager.result_exists( + unparsed_dataset, 'deconv_tolerance' + ): + parseDeconv( + wf.file_manager, unparsed_dataset, + results['out_deconv_mzML'], results['anno_annotated_mzML'], + logger=wf.logger + ) + parseTnT( + wf.file_manager, unparsed_dataset, + results['out_deconv_mzML'], results['anno_annotated_mzML'], + results['tags_tsv'], results['protein_tsv'], + logger=wf.logger + ) # Upload files via upload widget - st.subheader("**Upload FLASHDeconv & FLASHTagger output files (\*_annotated.mzML, \*_deconv.mzML, \*_tagged.tsv & \*_protein.tsv)**") + st.subheader("**Upload FLASHDeconv & FLASHTnT output files (anno_annotated.mzML, out_deconv.mzML, tags.tsv & protein.tsv)**") # Display info how to upload files st.info( """ diff --git a/src/workflow/FileManager.py b/src/workflow/FileManager.py index 46227bb..a54543b 100644 --- a/src/workflow/FileManager.py +++ b/src/workflow/FileManager.py @@ -386,13 +386,19 @@ def store_file(self, dataset_id: str, name_tag: str, file: Path | BytesIO, file (Path of File-Like): The file that should be stored. remove (bool): Wether or not the file should be removed after copying it. - filetype (str): The file extension of the file. Only - neccessary if a file-like object is used as input. + file_name (str): The name to store the file under. Only + neccessary if the input has no file extension to derive + it from. """ # Define storage path if file_name is None: - file_name = f"{name_tag}{file.suffix}" + # File-like objects have no `suffix`; Streamlit's UploadedFile + # carries the original file name in `name` instead. + suffix = getattr(file, 'suffix', None) + if suffix is None: + suffix = Path(getattr(file, 'name', '')).suffix + file_name = f"{name_tag}{suffix}" target_path = Path( self.cache_path, 'files', dataset_id, file_name diff --git a/tests/test_manual_upload.py b/tests/test_manual_upload.py new file mode 100644 index 0000000..a7d5d59 --- /dev/null +++ b/tests/test_manual_upload.py @@ -0,0 +1,133 @@ +""" +Tests for the manual result upload of FLASHDeconv and FLASHTnT results. + +Uploading result files through the "Manual Result Upload" tab used to fail +with "AttributeError: 'UploadedFile' object has no attribute 'suffix'": +FileManager.store_file derived the stored file name from `file.suffix`, which +a Streamlit `UploadedFile` (a BytesIO subclass carrying the original name in +`name`) does not have. + +The dataset id the page derives from the file name is covered as well. +FLASHDeconv names its output files `out_deconv.mzML` and +`anno_annotated.mzML`, so taking the part in front of the suffix filed the two +halves of one run under the datasets "out" and "anno", where neither could be +parsed. The mappings are literals in the pages (which cannot be imported without a +Streamlit runtime), so they are reproduced here. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from io import BytesIO +from pathlib import Path + +import pytest + +from src.workflow.FileManager import FileManager + + +# content/FLASHDeconv/FLASHDeconvWorkflow.py :: UPLOAD_FILE_TYPES +UPLOAD_FILE_TYPES = ( + ('deconv.mzML', 'out_deconv_mzML', 'out'), + ('annotated.mzML', 'anno_annotated_mzML', 'anno'), + ('spec1.tsv', 'spec1_tsv', ''), + ('spec2.tsv', 'spec2_tsv', ''), +) + +# content/FLASHTnT/FLASHTnTWorkflow.py :: UPLOAD_FILE_TYPES +TNT_UPLOAD_FILE_TYPES = ( + ('deconv.mzML', 'out_deconv_mzML', 'out'), + ('annotated.mzML', 'anno_annotated_mzML', 'anno'), + ('tags.tsv', 'tags_tsv', ''), + ('tagged.tsv', 'tags_tsv', ''), + ('protein.tsv', 'protein_tsv', ''), +) + + +class FakeUploadedFile(BytesIO): + """Stand-in for Streamlit's UploadedFile: a BytesIO with a file name.""" + + def __init__(self, name, data=b'data'): + super().__init__(data) + self.name = name + + +@pytest.fixture +def file_manager(tmp_path): + return FileManager(tmp_path, Path(tmp_path, 'cache')) + + +def store_upload(file_manager, file_names, file_types=UPLOAD_FILE_TYPES, + default_dataset='uploaded'): + """The storing loop of the manual result upload tabs.""" + for file_name in file_names: + for suffix, name_tag, tool_prefix in file_types: + if not file_name.endswith(suffix): + continue + experiment = file_name[:-len(suffix)].rstrip('_') + if experiment in ('', tool_prefix): + experiment = default_dataset + file_manager.store_file( + experiment, name_tag, FakeUploadedFile(file_name) + ) + break + else: + raise AssertionError(f'unmatched file name: {file_name}') + + +def test_store_uploaded_file_keeps_extension(file_manager): + file_manager.store_file( + 'sample', 'out_deconv_mzML', FakeUploadedFile('out_deconv.mzML', b'mzML') + ) + + stored = file_manager.get_results('sample', ['out_deconv_mzML'])['out_deconv_mzML'] + assert stored.name == 'out_deconv_mzML.mzML' + assert stored.read_bytes() == b'mzML' + + +def test_store_path_input_still_works(file_manager, tmp_path): + source = Path(tmp_path, 'spec1.tsv') + source.write_text('a\tb\n') + + file_manager.store_file('sample', 'spec1_tsv', source, remove=False) + + assert file_manager.get_results('sample', ['spec1_tsv'])['spec1_tsv'].name == 'spec1_tsv.tsv' + + +def test_unchanged_output_names_land_in_one_dataset(file_manager): + store_upload(file_manager, ['out_deconv.mzML', 'anno_annotated.mzML', 'spec1.tsv']) + + assert file_manager.get_results_list( + ['out_deconv_mzML', 'anno_annotated_mzML', 'spec1_tsv'] + ) == ['uploaded'] + + +def test_renamed_experiments_stay_separate(file_manager): + store_upload( + file_manager, + ['a_deconv.mzML', 'a_annotated.mzML', 'b_deconv.mzML', 'b_annotated.mzML'] + ) + + assert sorted(file_manager.get_results_list( + ['out_deconv_mzML', 'anno_annotated_mzML'] + )) == ['a', 'b'] + + +def test_unchanged_tnt_output_names_land_in_one_dataset(file_manager): + store_upload( + file_manager, + ['out_deconv.mzML', 'anno_annotated.mzML', 'tags.tsv', 'protein.tsv'], + TNT_UPLOAD_FILE_TYPES + ) + + assert file_manager.get_results_list( + ['out_deconv_mzML', 'anno_annotated_mzML', 'tags_tsv', 'protein_tsv'] + ) == ['uploaded'] + + +def test_tnt_accepts_legacy_tag_file_name(file_manager): + store_upload(file_manager, ['sample_tagged.tsv'], TNT_UPLOAD_FILE_TYPES) + + assert file_manager.get_results_list(['tags_tsv']) == ['sample']