Derive observation type from the observation type's category

nirc_ehr/resources/queries/study/clinical_observations.js

@@ -45,6 +45,13 @@ function onUpsert(helper, scriptErrors, row, oldRow) {
             EHR.Server.Utils.addError(scriptErrors, 'remark', "You selected 'Yes' for " + row.category + ", please explain in the Remark", "WARN");
         }
 
+        // Always derive the type from the observation type's category rather than trusting the incoming value.
+        // The Observations form leaves it blank because it offers every type; the other forms set it explicitly,
+        // but their type pickers are filtered to the categories that agree with the value they set, so deriving
+        // here gives them the same answer. Deriving unconditionally also re-derives when a re-opened draft or a
+        // saved template carries a type left over from a different category.
+        row.type = triggerHelper.getObservationTypeCategory(row.category) === 'Behavior' ? 'Behavior' : 'Clinical';
+
         // Handle scheduled observations
         if (!helper.isValidateOnly() && row.scheduledDate) {
             var qc;

nirc_ehr/resources/web/nirc_ehr/model/sources/ObsDefaults.js

@@ -6,6 +6,13 @@
 EHR.model.DataModelManager.registerMetadata('ObsDefaults', {
     byQuery: {
         'study.clinical_observations': {
+            // This form offers every observation type, so it can't know the observation's type up front.
+            // Clearing the default inherited from ClinicalDefaults lets the trigger script derive it
+            // from the selected type's category.
+            type: {
+                hidden: true,
+                defaultValue: null
+            },
             category: {
                 lookup: {
                     columns: 'value,description',

nirc_ehr/src/org/labkey/nirc_ehr/dataentry/form/NIRCBehaviorRoundsFormType.java

@@ -46,7 +46,7 @@ public NIRCBehaviorRoundsFormType(DataEntryFormContext ctx, Module owner)
                 new NIRCAnimalDetailsFormSection(),
                 new NIRCCaseTemplateFormSection("Case Template", "Case Template", "nirc_ehr-casetemplatepanel", Arrays.asList(ClientDependency.supplierFromPath("nirc_ehr/panel/CaseTemplatePanel.js"))),
                 new NIRCCasesFormPanelSection("Behavior Case", ctx, true),
-                new NIRCClinicalObservationsFormSection(true, "cases"),
+                new NIRCClinicalObservationsFormSection(null, true, "cases"),
                 new NIRCTreatmentGivenFormSection(true, "cases")
         ));
 

nirc_ehr/src/org/labkey/nirc_ehr/dataentry/form/NIRCBehavioralCasesFormType.java

@@ -50,7 +50,7 @@ public NIRCBehavioralCasesFormType(DataEntryFormContext ctx, Module owner)
                 new NIRCCaseTemplateFormSection("Case Template", "Case Template", "nirc_ehr-casetemplatepanel", Arrays.asList(ClientDependency.supplierFromPath("nirc_ehr/panel/CaseTemplatePanel.js"))),
                 new NIRCCasesFormPanelSection("Behavior Case", ctx, true),
                 new NIRCClinicalRemarksFormPanelSection(true, "cases", "Behavior Assessment", ctx, true),
-                new NIRCClinicalObservationsFormSection(true, "cases"),
+                new NIRCClinicalObservationsFormSection(null, true, "cases"),
                 new NIRCObservationOrdersFormSection(null, true, "cases"),
                 new NIRCTreatmentGivenFormSection(true, "cases"),
                 new NIRCTreatmentOrderFormSection(true, "cases")

nirc_ehr/src/org/labkey/nirc_ehr/dataentry/form/NIRCBulkBehaviorFormType.java

@@ -45,8 +45,8 @@ public NIRCBulkBehaviorFormType(DataEntryFormContext ctx, Module owner)
                 new NIRCClinicalRemarksFormSection("Behavior Assessment", ctx.getContainer().hasPermission(ctx.getUser(), NIRCEHRVetTechPermission.class),
                         ctx.getContainer().hasPermission(ctx.getUser(), EHRVeterinarianPermission.class),
                         ctx.getContainer().hasPermission(ctx.getUser(), AdminPermission.class)),
-                new NIRCClinicalObservationsFormSection(false, null),
-                new NIRCObservationOrdersFormSection("NIRC_DAILY_CLINICAL_OBS_ORDERS", false, null),
+                new NIRCClinicalObservationsFormSection(null, false, null),
+                new NIRCObservationOrdersFormSection(null, false, null),
                 new NIRCTreatmentGivenFormSection(),
                 new NIRCTreatmentOrderFormSection()
         ));

nirc_ehr/src/org/labkey/nirc_ehr/dataentry/form/NIRCBulkClinicalFormType.java

@@ -51,7 +51,7 @@ public NIRCBulkClinicalFormType(DataEntryFormContext ctx, Module owner)
                         ctx.getContainer().hasPermission(ctx.getUser(), EHRVeterinarianPermission.class),
                         ctx.getContainer().hasPermission(ctx.getUser(), AdminPermission.class)),
                 new NIRCWeightFormSection(true, true),
-                new NIRCClinicalObservationsFormSection(false, null),
+                new NIRCClinicalObservationsFormSection("NIRC_DAILY_CLINICAL_OBS", false, null),
                 new NIRCObservationOrdersFormSection("NIRC_DAILY_CLINICAL_OBS_ORDERS", false, null),
                 new NIRCProcedureFormSection(),
                 new NIRCProcedureOrderFormSection(),

nirc_ehr/src/org/labkey/nirc_ehr/dataentry/form/NIRCCasesFormType.java

@@ -58,7 +58,7 @@ public NIRCCasesFormType(DataEntryFormContext ctx, Module owner)
                 new NIRCCasesFormPanelSection("Clinical Case", ctx, false),
                 new NIRCClinicalRemarksFormPanelSection(true, "cases", "Clinical Remarks", ctx, false),
                 new NIRCWeightFormSection(true, false, true, "cases"),
-                new NIRCClinicalObservationsFormSection(true, "cases"),
+                new NIRCClinicalObservationsFormSection("NIRC_DAILY_CLINICAL_OBS", true, "cases"),
                 new NIRCObservationOrdersFormSection(null, true, "cases"),
                 new NIRCProcedureFormSection(true, "cases"),
                 new NIRCProcedureOrderFormSection(true, "cases"),

nirc_ehr/src/org/labkey/nirc_ehr/dataentry/form/NIRCClinicalObservationsFormType.java

@@ -37,7 +37,7 @@ public NIRCClinicalObservationsFormType(DataEntryFormContext ctx, Module owner)
         super(ctx, owner, NAME, NAME, "Clinical", Arrays.asList(
                 new NIRCTaskFormSection(),
                 new NIRCAnimalDetailsFormSection(),
-                new NIRCClinicalObservationsFormSection(false, false),
+                new NIRCClinicalObservationsFormSection("NIRC_DAILY_CLINICAL_OBS", false),
                 new NIRCWeightFormSection(true, true)
         ));
 

nirc_ehr/src/org/labkey/nirc_ehr/dataentry/form/NIRCClinicalRoundsFormType.java

@@ -50,7 +50,7 @@ public NIRCClinicalRoundsFormType(DataEntryFormContext ctx, Module owner)
                 new NIRCCaseTemplateFormSection("Case Template", "Case Template", "nirc_ehr-casetemplatepanel", Arrays.asList(ClientDependency.supplierFromPath("nirc_ehr/panel/CaseTemplatePanel.js"))),
                 new NIRCCasesFormPanelSection("Clinical Case", ctx, false),
                 new NIRCWeightFormSection(true, false, true, "cases"),
-                new NIRCClinicalObservationsFormSection(true, "cases"),
+                new NIRCClinicalObservationsFormSection("NIRC_DAILY_CLINICAL_OBS", true, "cases"),
                 new NIRCProcedureFormSection(true, "cases"),
                 new NIRCTreatmentGivenFormSection(true, "cases"),
                 new NIRCVitalsFormSection(true, "cases"),

nirc_ehr/src/org/labkey/nirc_ehr/dataentry/section/NIRCClinicalObservationsFormSection.java

@@ -22,13 +22,13 @@
 public class NIRCClinicalObservationsFormSection extends BaseFormSection
 {
     public static final String LABEL = "Observations";
-    private boolean _autoPopulateDailyObs;
+    private final String _dailyObsOption;
 
-    public NIRCClinicalObservationsFormSection(boolean autoPopulateDailyObs, boolean initCollapsed)
+    public NIRCClinicalObservationsFormSection(String dailyObsOption, boolean initCollapsed)
     {
         super("study", "clinical_observations", LABEL, "ehr-clinicalobservationgridpanel", true, initCollapsed, true);
 
-        _autoPopulateDailyObs = autoPopulateDailyObs;
+        _dailyObsOption = dailyObsOption;
         addClientDependency(ClientDependency.supplierFromPath("ehr/plugin/ClinicalObservationsCellEditing.js"));
         addClientDependency(ClientDependency.supplierFromPath("nirc_ehr/data/ClinicalObservationClientStore.js"));
         addClientDependency(ClientDependency.supplierFromPath("ehr/grid/ClinicalObservationGridPanel.js"));
@@ -37,9 +37,9 @@ public NIRCClinicalObservationsFormSection(boolean autoPopulateDailyObs, boolean
         setClientStoreClass("NIRC_EHR.data.ClinicalObservationsClientStore");
     }
 
-    public NIRCClinicalObservationsFormSection(boolean isChild, String parentQueryName)
+    public NIRCClinicalObservationsFormSection(String dailyObsOption, boolean isChild, String parentQueryName)
     {
-        this(false, true);
+        this(dailyObsOption, true);
 
         if (isChild && null != parentQueryName)
         {
@@ -57,12 +57,9 @@ public List<String> getTbarButtons()
     {
         List<String> defaults = super.getTbarButtons();
 
-        if (_autoPopulateDailyObs)
+        if (_dailyObsOption != null)
         {
-            defaults.add("NIRC_AUTO_POPULATE_DAILY_OBS");
-        }
-        else {
-            defaults.add("NIRC_DAILY_CLINICAL_OBS");
+            defaults.add(_dailyObsOption);
         }
 
         return defaults;

nirc_ehr/src/org/labkey/nirc_ehr/query/NIRC_EHRTriggerHelper.java

@@ -83,6 +83,7 @@ public class NIRC_EHRTriggerHelper
     private User _user;
     private static final Logger _log = LogManager.getLogger(NIRC_EHRTriggerHelper.class);
     private final Map<String,Object> _cachedDrugFormulary = new HashMap<>();
+    private final Map<String,String> _cachedObservationTypeCategories = new HashMap<>();
 
     // Maps an originating observation order's taskid to the task its scheduled observations are grouped under,
     // for the duration of a single save batch (the same helper instance is reused across rows in the batch).
@@ -810,6 +811,26 @@ public void ensureDailyClinicalObservationOrders(String id, String caseid, final
         }
     }
 
+    /**
+     * Returns the category of an observation type from ehr.observation_types, or null when the type has no
+     * category or is not found. Cached for the life of the save batch.
+     */
+    public String getObservationTypeCategory(String observationType)
+    {
+        if (observationType == null)
+            return null;
+
+        if (!_cachedObservationTypeCategories.containsKey(observationType))
+        {
+            TableInfo ti = getTableInfo("ehr", "observation_types");
+            SimpleFilter filter = new SimpleFilter(FieldKey.fromString("value"), observationType);
+            List<String> categories = new TableSelector(ti, Collections.singleton("category"), filter, null).getArrayList(String.class);
+            _cachedObservationTypeCategories.put(observationType, categories.isEmpty() ? null : categories.get(0));
+        }
+
+        return _cachedObservationTypeCategories.get(observationType);
+    }
+
     // This helper function propagates clinical observations through clinical cases
     public Map<String, Object> handleScheduledObservations(Map<String, Object> row, String qcstate, String orderTasks) throws SQLException, BatchValidationException, QueryUpdateServiceException, DuplicateKeyException
     {

nirc_ehr/test/src/org.labkey.test/tests.nirc_ehr/NIRC_EHRTest.java

@@ -111,6 +111,9 @@ public class NIRC_EHRTest extends AbstractGenericEHRTest implements PostgresOnly
     // Dedicated animal for testScheduledObservationTaskGrouping; provisioned (alive, housed, assigned) in
     // createTestSubjects so the clinical case form raises no warnings that would keep the validation banner up.
     private static final String taskGroupAnimalId = "TESTGRP9090";
+    // Dedicated animal for testObservationTypeDerivedFromCategory; provisioned the same way so the Observations
+    // form can be submitted final in one step.
+    private static final String obsTypeAnimalId = "TESTOBSTYPE9191";
 
     private final String[] weightFields = {"Id", "date", "enddate", "project", "weight", FIELD_QCSTATELABEL, FIELD_OBJECTID, FIELD_LSID, "_recordid", "performedby"};
     private final Object[] weightData1 = {getExpectedAnimalIDCasing("TESTSUBJECT1"), EHRClientAPIHelper.DATE_SUBSTITUTION, null, null, "12", EHRQCState.IN_PROGRESS.label, null, null, "_recordID", 1004};
@@ -508,6 +511,32 @@ protected void createTestSubjects() throws Exception
         getApiHelper().deleteAllRecords("study", "Assignment", new Filter("Id", taskGroupAnimalId));
         getApiHelper().doSaveRows(DATA_ADMIN.getEmail(), insertCommand, getExtraContext());
 
+        // Fully provision the observation-type test animal for the same reason.
+        log("Creating observation type test subject");
+        fields = new String[]{"Id", "Species", "Birth", "Gender", "date", "calculated_status", "objectid", "performedby"};
+        data = new Object[][]{
+                {obsTypeAnimalId, "Rhesus", (new Date()).toString(), getMale(), new Date(), "Alive", UUID.randomUUID().toString(), 1004}
+        };
+        insertCommand = getApiHelper().prepareInsertCommand("study", "demographics", "lsid", fields, data);
+        getApiHelper().deleteAllRecords("study", "demographics", new Filter("Id", obsTypeAnimalId));
+        getApiHelper().doSaveRows(DATA_ADMIN.getEmail(), insertCommand, getExtraContext());
+
+        fields = new String[]{"Id", "date", "enddate", "room", "cage", "performedby"};
+        data = new Object[][]{
+                {obsTypeAnimalId, pastDate1, null, getRooms()[0], CAGES[1], 1004}
+        };
+        insertCommand = getApiHelper().prepareInsertCommand("study", "Housing", "lsid", fields, data);
+        getApiHelper().deleteAllRecords("study", "Housing", new Filter("Id", obsTypeAnimalId));
+        getApiHelper().doSaveRows(DATA_ADMIN.getEmail(), insertCommand, getExtraContext());
+
+        fields = new String[]{"Id", "date", "enddate", "project", "performedby"};
+        data = new Object[][]{
+                {obsTypeAnimalId, pastDate1, null, PROJECTS[0], 1004}
+        };
+        insertCommand = getApiHelper().prepareInsertCommand("study", "Assignment", "lsid", fields, data);
+        getApiHelper().deleteAllRecords("study", "Assignment", new Filter("Id", obsTypeAnimalId));
+        getApiHelper().doSaveRows(DATA_ADMIN.getEmail(), insertCommand, getExtraContext());
+
         primeCaches();
     }
 
@@ -857,7 +886,13 @@ public void testScheduledObservationTaskGrouping()
 
         Map<String, Integer> entriesPerCategory = new HashMap<>();
         for (Map<String, Object> row : getClinicalObservations(animalId))
+        {
             entriesPerCategory.merge(String.valueOf(row.get("category")), 1, Integer::sum);
+            // A scheduled observation takes its type from the originating order, which the daily clinical
+            // observation orders create as Clinical.
+            Assert.assertEquals("Scheduled observation for category " + row.get("category") + " should be Clinical",
+                    "Clinical", String.valueOf(row.get("type")));
+        }
         Assert.assertEquals("Expected the six daily observation categories", NIRC_DAILY_OBS_VALUES.size(), entriesPerCategory.size());
         entriesPerCategory.forEach((category, count) ->
                 Assert.assertEquals("Expected two entries (one per matching order) for category " + category, Integer.valueOf(2), count));
@@ -949,6 +984,56 @@ private int countObservationsForTask(String taskId)
         return executeSelectRowCommand("study", "clinical_observations", ContainerFilter.Current, "/" + getContainerPath(), List.of(new Filter("taskid", taskId))).getRowCount().intValue();
     }
 
+    // Two ehr.observation_types values on either side of the derivation: the first has no category, the second
+    // is categorized as Behavior. Both use a free-text Observation/Score editor, so neither depends on an
+    // ehr_lookups value list being populated.
+    private static final String UNCATEGORIZED_OBS_TYPE = "Mass";
+    private static final String BEHAVIOR_OBS_TYPE = "General Behavior Observation";
+
+    @Test
+    public void testObservationTypeDerivedFromCategory()
+    {
+        String animalId = obsTypeAnimalId;
+
+        // The Observations form offers every observation type, so it cannot set the observation's type up
+        // front; the trigger script derives it from the selected type's category. A type categorized as
+        // Behavior must be stored as a Behavior observation and everything else as Clinical, otherwise the
+        // entry drops out of the behavior views (study.behaviorObservations filters on type = 'Behavior').
+        log("Entering an uncategorized and a Behavior-categorized observation type on the Observations form");
+        gotoEnterData();
+        waitAndClickAndWait(Locator.linkWithText("Observations"));
+
+        Ext4GridRef observations = _helper.getExt4GridForFormSection("Observations");
+        addObservationRow(observations, animalId, UNCATEGORIZED_OBS_TYPE, "3 cm mass on left arm");
+        addObservationRow(observations, animalId, BEHAVIOR_OBS_TYPE, "Pacing observed");
+        submitForm("Submit Final", "Finalize");
+
+        Map<String, String> typeByCategory = new HashMap<>();
+        for (Map<String, Object> row : getClinicalObservations(animalId))
+            typeByCategory.put(String.valueOf(row.get("category")), String.valueOf(row.get("type")));
+
+        Assert.assertEquals("Expected exactly the two entered observations for " + animalId,
+                Set.of(UNCATEGORIZED_OBS_TYPE, BEHAVIOR_OBS_TYPE), typeByCategory.keySet());
+        Assert.assertEquals("An uncategorized observation type should be stored as a Clinical observation",
+                "Clinical", typeByCategory.get(UNCATEGORIZED_OBS_TYPE));
+        Assert.assertEquals("A Behavior-categorized observation type should be stored as a Behavior observation",
+                "Behavior", typeByCategory.get(BEHAVIOR_OBS_TYPE));
+    }
+
+    // Appends a row to an Observations grid and fills in the fields the trigger script needs to accept it: an
+    // animal, an observation type (the grid's "category"), and an Observation/Score plus remark, since an entry
+    // with neither raises a WARN that would disable Submit Final. The row index is read back from the grid
+    // rather than assumed, so this works whether or not the form starts with rows of its own.
+    private void addObservationRow(Ext4GridRef observations, String animalId, String category, String observation)
+    {
+        _helper.addRecordToGrid(observations);
+        int row = observations.getRowCount();
+        observations.setGridCell(row, "Id", animalId);
+        observations.setGridCell(row, "category", category);
+        observations.setGridCell(row, "observation", observation);
+        observations.setGridCellJS(row, "remark", "remark for " + category);
+    }
+
     @Test
     public void testObservationBulkEdit()
     {