Merge discvr-26.7 to develop

GenotypeAssays/src/org/labkey/genotypeassays/GenotypeAssaysController.java

@@ -20,25 +20,15 @@
 import org.json.JSONArray;
 import org.labkey.api.action.ApiResponse;
 import org.labkey.api.action.ApiSimpleResponse;
-import org.labkey.api.action.ConfirmAction;
 import org.labkey.api.action.MutatingApiAction;
 import org.labkey.api.action.SpringActionController;
-import org.labkey.api.data.DbSchema;
-import org.labkey.api.data.DbSchemaType;
-import org.labkey.api.data.SqlExecutor;
-import org.labkey.api.data.TableInfo;
+import org.labkey.api.data.ContainerType;
 import org.labkey.api.exp.api.ExpProtocol;
 import org.labkey.api.exp.api.ExperimentService;
 import org.labkey.api.security.RequiresPermission;
-import org.labkey.api.security.permissions.ReadPermission;
 import org.labkey.api.security.permissions.UpdatePermission;
-import org.labkey.api.util.HtmlString;
 import org.labkey.api.util.Pair;
-import org.labkey.api.util.URLHelper;
-import org.labkey.api.view.HtmlView;
 import org.springframework.validation.BindException;
-import org.springframework.validation.Errors;
-import org.springframework.web.servlet.ModelAndView;
 
 import java.util.Arrays;
 import java.util.HashMap;
@@ -54,66 +44,6 @@ public GenotypeAssaysController()
         setActionResolver(_actionResolver);
     }
 
-    @RequiresPermission(ReadPermission.class)
-    public static class MigrateLegacySSPAction extends ConfirmAction<Object>
-    {
-        @Override
-        public void validateCommand(Object form, Errors errors)
-        {
-
-        }
-
-        @Override
-        public ModelAndView getConfirmView(Object form, BindException errors) throws Exception
-        {
-            DbSchema schema = DbSchema.get("SSP_Assay", DbSchemaType.Module);
-            if (schema == null)
-                return new HtmlView(HtmlString.of("Either the legacy SSP module has not been installed, or it has already been removed"));
-            else
-                return new HtmlView(HtmlString.of("This allows an admin to copy any primers stored in the original SSP Assay module into the new genotyping module.  Any data has already been copied.  Do you want to continue?"));
-        }
-
-        @Override
-        public boolean handlePost(Object form, BindException errors) throws Exception
-        {
-            try
-            {
-                DbSchema schema = DbSchema.get("SSP_Assay", DbSchemaType.Module);
-                if (schema == null)
-                    return true; //module not installed
-
-                TableInfo primers = schema.getTable("primers");
-                if (primers == null)
-                    return true;
-
-                SqlExecutor sql = new SqlExecutor(schema);
-
-                sql.execute("INSERT INTO genotypeassays.primer_pairs (primerName, ref_nt_name, ref_nt_id, shortName, forwardPrimer, reversePrimer, createdBy, created, modifiedby, modified) " +
-                    "SELECT s.primerName, s.ref_nt_name, s.ref_nt_id, s.shortName, s.forwardPrimer, s.reversePrimer, s.createdBy, s.created, s.modifiedby, s.modified " +
-                    "FROM ssp_assay.primers s " +
-                    "LEFT JOIN genotypeassays.primer_pairs p ON (p.primerName = s.primerName) " +
-                    "WHERE p.primerName is null");
-
-                sql.execute("DROP TABLE ssp_assay.primers");
-                sql.execute("DROP TABLE ssp_assay.ssp_result_types");
-                sql.execute("DROP SCHEMA ssp_assay");
-
-                return true;
-            }
-            catch (Exception e)
-            {
-                errors.reject(ERROR_MSG, e.getMessage());
-                return false;
-            }
-        }
-
-        @Override
-        public URLHelper getSuccessURL(Object form)
-        {
-            return getContainer().getStartURL(getUser());
-        }
-    }
-
     @RequiresPermission(UpdatePermission.class)
     public static class CacheAnalysesAction extends MutatingApiAction<CacheAnalysesForm>
     {
@@ -134,6 +64,13 @@ public ApiResponse execute(CacheAnalysesForm form, BindException errors)
                         return null;
                     }
 
+                    if (!protocol.getContainer().getContainerFor(ContainerType.DataType.tabParent).equals(getContainer().getContainerFor(ContainerType.DataType.tabParent)))
+                    {
+                        errors.reject(ERROR_MSG, "Protocol is from the wrong container: " + form.getProtocolId());
+                        logger.error("CacheAnalysesAction targeted a protocol from the wrong container: {}, from {}, in the container: {}", form.getProtocolId(), protocol.getContainer().getPath(), getContainer().getPath());
+                        return null;
+                    }
+
                     String[] alleleNames = Arrays.stream(form.getAlleleNames()).map(StringEscapeUtils::unescapeHtml4).toArray(String[]::new);
                     Pair<List<Long>, List<Long>> ret = GenotypeAssaysManager.get().cacheAnalyses(getViewContext(), protocol, alleleNames);
                     resultProperties.put("runsCreated", ret.first);
@@ -214,6 +151,13 @@ public ApiResponse execute(CacheAnalysesForm form, BindException errors)
                         return null;
                     }
 
+                    if (!protocol.getContainer().getContainerFor(ContainerType.DataType.tabParent).equals(getContainer().getContainerFor(ContainerType.DataType.tabParent)))
+                    {
+                        errors.reject(ERROR_MSG, "Protocol is from the wrong container: " + form.getProtocolId());
+                        logger.error("CacheHaplotypesAction targeted a protocol from the wrong container: {}, from {}, in the container: {}", form.getProtocolId(), protocol.getContainer().getPath(), getContainer().getPath());
+                        return null;
+                    }
+
                     Pair<List<Long>, List<Long>> ret = GenotypeAssaysManager.get().cacheHaplotypes(getViewContext(), protocol, new JSONArray(form.getJson()));
                     resultProperties.put("runsCreated", ret.first);
                     resultProperties.put("runsDeleted", ret.second);

PMR/resources/data/birth_condition_raw.tsv

@@ -0,0 +1,3 @@
+value	category
+Born Dead/Not Born	false
+Live Birth	true

PMR/resources/data/birth_date_type.tsv

@@ -0,0 +1,4 @@
+value
+Estimated
+Actual
+Undetermined

PMR/resources/data/lookup_sets.tsv

@@ -53,6 +53,7 @@ behavior_types	Behavior Types	value
 fecal_smear_score	Fecal Smear Scores	value
 problem_list_subcategory	Problem List Subcategory	value
 customer_affiliation	Customer Affiliation	value
+birth_condition_raw	Birth Condition Field Values	value	value
 birth_date_type	Birth Date Type	value
 birth_type	Birth Type Field Values	value
 chemistry_method	Chemistry Method Field Values	value

PMR/resources/etls/prime-birth.xml

@@ -53,7 +53,7 @@
         </transform>
     </transforms>
     <incrementalFilter timestampColumnName="modified" pkColumnName="objectid" className="ModifiedSinceFilterStrategy" >
-        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" timestampColumnName="Created" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid">
+        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" timestampColumnName="Created" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="Id">
             <sourceFilters>
                 <sourceFilter column="datasetid/Name" operator="eq" value="birth" />
                 <sourceFilter column="Comment" operator="contains" value="Delete"/>

PMR/resources/etls/prime-demographics.xml

@@ -48,7 +48,7 @@
         </transform>
     </transforms>
     <incrementalFilter timestampColumnName="modified" pkColumnName="objectid" className="ModifiedSinceFilterStrategy" >
-        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" timestampColumnName="Created" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid">
+        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" timestampColumnName="Created" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="Id">
             <sourceFilters>
                 <sourceFilter column="datasetid/Name" operator="eq" value="demographics" />
                 <sourceFilter column="Comment" operator="contains" value="Delete"/>

PMR/resources/queries/ehr_lookups/birth_condition.sql

@@ -0,0 +1,10 @@
+-- This is a workaround to allow the lookup_sets system to provide an additional field beyond value/title:
+SELECT
+    value,
+    CASE
+        WHEN category = 'true' THEN true
+        WHEN category = 'false' THEN false
+        ELSE true
+    END as alive
+
+FROM ehr_lookups.birth_condition_raw

PMR/resources/queries/study/birth.query.xml

@@ -0,0 +1,18 @@
+<query xmlns="http://labkey.org/data/xml/query">
+    <metadata>
+        <tables xmlns="http://labkey.org/data/xml">
+            <table tableName="" tableDbType="NOT_IN_DB">
+                <columns>
+                    <column columnName="birth_condition">
+                        <fk>
+                            <fkDbSchema>ehr_lookups</fkDbSchema>
+                            <fkTable>birth_condition</fkTable>
+                            <fkColumnName>value</fkColumnName>
+                        </fk>
+                    </column>
+
+                </columns>
+            </table>
+        </tables>
+    </metadata>
+</query>

PMR/resources/views/populateData.html

@@ -236,6 +236,20 @@
             schemaName: 'ehr_lookups',
             queryName: 'geographic_origins',
             pk: 'rowid'
+        },{
+            label: 'Birth Condition Raw',
+            populateFn: 'populateFromFile',
+            moduleName: 'pmr',
+            schemaName: 'ehr_lookups',
+            queryName: 'birth_condition_raw',
+            pk: 'rowid'
+        },{
+            label: 'Birth Date Type',
+            populateFn: 'populateFromFile',
+            moduleName: 'pmr',
+            schemaName: 'ehr_lookups',
+            queryName: 'birth_date_type',
+            pk: 'rowid'
         },{
             label: 'Birth Type',
             populateFn: 'populateFromFile',

hivrc/src/org/labkey/hivrc/view/analysisHeader.jsp

@@ -1,9 +1,9 @@
-<%@ page import="org.apache.commons.lang3.StringUtils" %>
+<%@ page import="org.json.JSONArray" %>
+<%@ page import="org.labkey.api.util.JavaScriptFragment" %>
 <%@ page import="org.labkey.api.view.HttpView" %>
 <%@ page import="org.labkey.api.view.JspView" %>
 <%@ page import="org.labkey.api.view.template.ClientDependencies" %>
 <%@ page import="org.labkey.hivrc.query.AnalysisModel" %>
-<%@ page import="java.util.Arrays" %>
 <%@ page extends="org.labkey.api.jsp.JspBase" %>
 <%!
     @Override
@@ -53,7 +53,7 @@
             materials: <%=q(model.getMaterials())%>,
             methods: <%=q(model.getMethods())%>,
             results: <%=q(model.getResults())%>,
-            tags: <%=unsafe(model.getTags() == null || model.getTags().length == 0 ? "null" : "['" + unsafe(StringUtils.join(Arrays.asList(model.getTags()), "','")) + "']")%>
+            tags: <% if (model.getTags() == null || model.getTags().length == 0) { %><%=JavaScriptFragment.NULL%><% } else { %><%=new JSONArray(model.getTags())%><% } %>
         }).render(webpartId);
 
         if (LABKEY.Security.currentUser.canInsert) {

mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java

@@ -184,7 +184,7 @@ protected Integer getOrCreateOutputFile(Object dataFileUrl, Object folderName, S
     protected File getLocalSubdir(Object folderName) throws PipelineJobException
     {
         PipeRoot pr = PipelineService.get().getPipelineRootSetting(getContainerUser().getContainer());
-        File baseDir = new File(pr.getRootPath(), mGAPManager.DATA_DIR_NAME);
+        File baseDir = FileUtil.appendName(pr.getRootPath(), mGAPManager.DATA_DIR_NAME);
         if (!baseDir.exists())
         {
             baseDir.mkdirs();
@@ -196,7 +196,7 @@ protected File getLocalSubdir(Object folderName) throws PipelineJobException
             throw new PipelineJobException("Unable to find folderName");
         }
 
-        File subdir = new File(baseDir, folderNameString);
+        File subdir = FileUtil.appendName(baseDir, folderNameString);
         if (!subdir.exists())
         {
             subdir.mkdirs();
@@ -214,7 +214,7 @@ protected File doFileCopy(File f, File subdir, @Nullable String name) throws Pip
         }
 
         //Copy file locally, plus index if exists:
-        File localCopy = new File(subdir, name == null || f.getName().startsWith("mGap.v") ? f.getName() : FileUtil.makeLegalName(name).replaceAll(" ", "_") + ".vcf.gz");
+        File localCopy = FileUtil.appendName(subdir, name == null || f.getName().startsWith("mGap.v") ? f.getName() : FileUtil.makeLegalName(name).replaceAll(" ", "_") + ".vcf.gz");
         if (f.equals(localCopy))
         {
             return localCopy;
@@ -252,7 +252,7 @@ protected File doFileCopy(File f, File subdir, @Nullable String name) throws Pip
 
         if (doCopy)
         {
-            getStatusLogger().info("Creating symlink: " + f.getPath() + " / " + localCopy.getPath());
+            getStatusLogger().info("Creating local copy of file: " + f.getPath() + " / " + localCopy.getPath());
             try
             {
                 if (!Files.isReadable(f.toPath()))
@@ -265,11 +265,11 @@ protected File doFileCopy(File f, File subdir, @Nullable String name) throws Pip
                     throw new PipelineJobException("File should have been deleted: " + localCopy.getPath());
                 }
 
-                Files.createSymbolicLink(localCopy.toPath(), f.toPath());
+                Files.copy(f.toPath(), localCopy.toPath());
             }
             catch (IOException e)
             {
-                throw new PipelineJobException("Failed to create symlink: " + localCopy.getPath(), e);
+                throw new PipelineJobException("Failed to create local copy: " + localCopy.getPath(), e);
             }
         }
 
@@ -285,19 +285,19 @@ protected File doFileCopy(File f, File subdir, @Nullable String name) throws Pip
 
             if (!indexLocal.exists())
             {
-                getStatusLogger().info("Creating symlink copy of VCF index: " + index.getPath() + " / " + indexLocal.getPath());
+                getStatusLogger().info("Creating local copy of VCF index: " + index.getPath() + " / " + indexLocal.getPath());
                 try
                 {
-                    Files.createSymbolicLink(indexLocal.toPath(), index.toPath());
+                    Files.copy(index.toPath(), indexLocal.toPath());
                 }
                 catch (IOException e)
                 {
-                    getStatusLogger().error("Failed to create symlink: " + indexLocal.getPath(), e);
+                    getStatusLogger().error("Failed to create local copy: " + indexLocal.getPath(), e);
                 }
             }
             else
             {
-                getStatusLogger().info("Local index already exists: " + indexLocal.getPath());
+                getStatusLogger().info("Local copy of index already exists: " + indexLocal.getPath());
             }
         }
 

mGAP/src/org/labkey/mgap/mGAPController.java

@@ -222,7 +222,7 @@ public Object execute(RequestUserForm form, BindException errors) throws Excepti
                         Container c = mGAPManager.get().getMGapContainer();
                         if (c == null)
                         {
-                            _log.warn("mGAP container was not set, using: " + c.getPath());
+                            _log.warn("mGAP container was not set, using: " + getContainer().getPath());
                             c = getContainer();
                         }
 
@@ -376,9 +376,16 @@ public void validateForm(ApproveUserRequestsForm form, Errors errors)
             if (form.getRequestIds() == null || form.getRequestIds().length == 0)
             {
                 errors.reject(ERROR_MSG, "No request IDs provided");
+                return;
             }
 
-            TableInfo ti = mGAPSchema.getInstance().getSchema().getTable(mGAPSchema.TABLE_USER_REQUESTS);
+            if (!mGapContainer.hasPermission(getUser(), AdminPermission.class))
+            {
+                errors.reject(ERROR_MSG, "Admin permission on the mGAP container is required");
+                return;
+            }
+
+            TableInfo ti = QueryService.get().getUserSchema(getUser(), mGapContainer, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_USER_REQUESTS);
             for (int requestId : form.getRequestIds())
             {
                 TableSelector ts = new TableSelector(ti, PageFlowUtil.set("userId"), new SimpleFilter(FieldKey.fromString("rowId"), requestId), null);
@@ -387,14 +394,6 @@ public void validateForm(ApproveUserRequestsForm form, Errors errors)
                     errors.reject(ERROR_MSG, "No request found for request ID: " + requestId);
                     break;
                 }
-
-                //Note: if using LDAP, users will potentially get created automatically
-                //Integer userId = ts.getObject(Integer.class);
-                //if (userId != null)
-                //{
-                //    errors.reject(ERROR_MSG, "A user already exists for the request: " + requestId);
-                //    break;
-                //}
             }
         }
 
@@ -768,7 +767,7 @@ public ModelAndView getConfirmView(Object o, BindException errors) throws Except
         {
             setTitle("Update Update SnpEff Annotation");
 
-            return new HtmlView("Do you want to continue?");
+            return HtmlView.of("Do you want to continue?");
         }
 
         @Override
@@ -793,10 +792,14 @@ public boolean handlePost(Object o, BindException errors) throws Exception
             Sort sort = new Sort(FieldKey.fromString("contig"));
             sort.appendSortColumn(FieldKey.fromString("position"), Sort.SortDirection.ASC, false);
 
-            Integer outputFileId = new TableSelector(mGAPSchema.getInstance().getSchema().getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES), Collections.singleton("vcfId")).getObject(releaseRowId, Integer.class);
+            Integer outputFileId = new TableSelector(us.getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES), Collections.singleton("vcfId")).getObject(releaseRowId, Integer.class);
             ExpData data = SequenceOutputFile.getForId(outputFileId).getExpData();
-            File vcf = data.getFile();
+            if (!data.getContainer().hasPermission(getUser(), ReadPermission.class))
+            {
+                throw new UnauthorizedException("You don't have permission to read this resource");
+            }
 
+            File vcf = data.getFile();
             try (VCFFileReader reader = new VCFFileReader(vcf))
             {
                 try (CloseableIterator<VariantContext> it = reader.iterator())
@@ -1181,7 +1184,7 @@ public ModelAndView getConfirmView(Object o, BindException errors) throws Except
         {
             setTitle("Update Annotation Table");
 
-            HtmlView view = new HtmlView("This will update the annotation table using the VariantAnnotation github repo. Do you want to continue?");
+            HtmlView view = HtmlView.of("This will update the annotation table using the VariantAnnotation github repo. Do you want to continue?");
             return view;
         }
 
@@ -1237,6 +1240,11 @@ public boolean handlePost(Object o, BindException errors) throws Exception
                 }
             }
 
+            if (toAdd.isEmpty())
+            {
+                throw new IllegalStateException("Something went wrong downloading data");
+            }
+
             UserSchema us = QueryService.get().getUserSchema(getUser(), getContainer(), mGAPSchema.NAME);
             TableInfo ti = us.getTable(mGAPSchema.TABLE_VARIANT_ANNOTATIONS);
             ti.getUpdateService().truncateRows(getUser(), getContainer(), null, null);