diff --git a/frontend/src/__tests__/mocks/omezarrHelper.ts b/frontend/src/__tests__/mocks/omezarrHelper.ts index 92422468..ce7addc6 100644 --- a/frontend/src/__tests__/mocks/omezarrHelper.ts +++ b/frontend/src/__tests__/mocks/omezarrHelper.ts @@ -48,5 +48,6 @@ export const omezarrHelperMock = { generateNeuroglancerStateForOmeZarr: vi.fn(() => 'mock-state-ome-zarr'), determineLayerType: vi.fn(async () => 'image'), translateUnitToNeuroglancer: vi.fn((unit: string) => unit), - getResolvedScales: vi.fn(() => [1.0, 0.5, 0.5]) + getResolvedScales: vi.fn(() => [1.0, 0.5, 0.5]), + getDatasetWarnings: vi.fn(() => []) }; diff --git a/frontend/src/__tests__/unitTests/datasetWarnings.test.ts b/frontend/src/__tests__/unitTests/datasetWarnings.test.ts new file mode 100644 index 00000000..a86a25d8 --- /dev/null +++ b/frontend/src/__tests__/unitTests/datasetWarnings.test.ts @@ -0,0 +1,197 @@ +import { describe, it, expect } from 'vitest'; +import { getDatasetWarnings } from '@/omezarr-helper'; +import type { Metadata } from '@/omezarr-helper'; + +// Minimal stand-in for the parts of Metadata the checks read. Codec info is +// left out by default, which the chunk check treats as compressed. +const createMetadata = ( + chunks: number[], + dtype = 'uint16', + extra: Partial = {}, + shape: number[] = [8, 8, 8] +): Metadata => + ({ + arr: { chunks, dtype, shape }, + ...extra + }) as unknown as Metadata; + +const axes = (...names: string[]): Partial => ({ + multiscales: [ + { axes: names.map(name => ({ name, type: 'space' })), datasets: [{}, {}] } + ] as unknown as Metadata['multiscales'] +}); + +const levels = (count: number): Partial => ({ + multiscales: [ + { datasets: Array.from({ length: count }, () => ({})) } + ] as unknown as Metadata['multiscales'] +}); + +// zstd nested inside a sharding_indexed pipeline, as a sharded v3 array stores it. +const SHARDED_ZSTD: Partial = { + codecs: [ + { + name: 'sharding_indexed', + configuration: { codecs: [{ name: 'bytes' }, { name: 'zstd' }] } + } + ] +}; +const UNCOMPRESSED_V3: Partial = { + codecs: [{ name: 'bytes' }, { name: 'crc32c' }] +}; + +describe('getDatasetWarnings: chunk size', () => { + it('says nothing about reasonable chunks', () => { + expect(getDatasetWarnings(createMetadata([64, 64, 64]))).toEqual([]); + }); + + it('does not warn about a compressed 16 MB chunk', () => { + // raw/s2: 16 MB inner chunks that zstd takes to ~12 MB on disk. + expect( + getDatasetWarnings( + createMetadata([8, 128, 128, 128], 'uint8', SHARDED_ZSTD) + ) + ).toEqual([]); + }); + + it('holds an uncompressed array to the stricter limit', () => { + // The same 16 MB chunks, but stored raw, so 16 MB is what transfers. + for (const raw of [UNCOMPRESSED_V3, { compressor: null }]) { + expect( + getDatasetWarnings(createMetadata([8, 128, 128, 128], 'uint8', raw)) + ).toEqual([ + { case: 'zarr-large-chunks', size: '16 MB', compressed: false } + ]); + } + }); + + it('finds a compressor nested inside a sharding codec', () => { + // sharding_indexed is structural, so a flat scan would call this + // uncompressed and warn at 16 MB. + expect( + getDatasetWarnings( + createMetadata([8, 128, 128, 128], 'uint8', SHARDED_ZSTD) + ) + ).toEqual([]); + }); + + it('assumes compressed when codec metadata was never fetched', () => { + // Unknown lands on the permissive limit: a missed warning beats a false one. + expect( + getDatasetWarnings(createMetadata([8, 128, 128, 128], 'uint8')) + ).toEqual([]); + }); + + it('warns above the compressed limit', () => { + // seed151 img: 128 MB chunks. + expect( + getDatasetWarnings(createMetadata([256, 256, 256, 8], 'uint8')) + ).toEqual([ + { case: 'zarr-large-chunks', size: '128 MB', compressed: true } + ]); + }); + + it('accounts for the dtype width', () => { + expect(getDatasetWarnings(createMetadata([256, 256, 256]))).toEqual([]); + expect( + getDatasetWarnings(createMetadata([256, 256, 256], 'float64')) + ).toHaveLength(1); + }); +}); + +describe('getDatasetWarnings: resolution levels', () => { + const BIG = [3000, 3000, 1350, 8]; // 91 GB of uint8, the seed151 img extent + + it('warns when multiscales declares a single level for a large image', () => { + expect( + getDatasetWarnings(createMetadata([64, 64, 64], 'uint8', levels(1), BIG)) + ).toEqual([{ case: 'zarr-single-level', size: '91 GB' }]); + }); + + it('says nothing when the pyramid has levels', () => { + expect( + getDatasetWarnings(createMetadata([64, 64, 64], 'uint8', levels(5), BIG)) + ).toEqual([]); + }); + + it('says nothing about a small single-level image', () => { + expect( + getDatasetWarnings( + createMetadata([64, 64, 64], 'uint8', levels(1), [256, 256, 256]) + ) + ).toEqual([]); + }); + + it('never fires on a plain array, however large', () => { + // The bug that made this warn on raw/s2: a plain array also has one shape, + // but it declares no multiscales and so claims nothing. + expect( + getDatasetWarnings(createMetadata([64, 64, 64], 'uint8', {}, BIG)) + ).toEqual([]); + }); +}); + +describe('getDatasetWarnings: axis order', () => { + it('accepts spec order', () => { + for (const names of [ + ['t', 'c', 'z', 'y', 'x'], + ['c', 'z', 'y', 'x'], + ['z', 'y', 'x'], + ['y', 'x'] + ]) { + expect( + getDatasetWarnings( + createMetadata([64, 64, 64], 'uint16', axes(...names)) + ) + ).toEqual([]); + } + }); + + it('warns about the seed151 c,x,y,z order', () => { + expect( + getDatasetWarnings( + createMetadata([64, 64, 64, 8], 'uint8', axes('c', 'x', 'y', 'z')) + ) + ).toEqual([ + { + case: 'zarr-axis-order', + axisOrder: 'C, X, Y, Z', + expectedOrder: 'C, Z, Y, X' + } + ]); + }); + + it('warns when the channel axis trails the spatial axes', () => { + expect( + getDatasetWarnings( + createMetadata([64, 64, 64, 8], 'uint8', axes('x', 'y', 'z', 'c')) + ) + ).toEqual([ + { + case: 'zarr-axis-order', + axisOrder: 'X, Y, Z, C', + expectedOrder: 'C, Z, Y, X' + } + ]); + }); + + it('is case insensitive', () => { + expect( + getDatasetWarnings( + createMetadata([64, 64, 64], 'uint16', axes('Z', 'Y', 'X')) + ) + ).toEqual([]); + }); + + it('stays quiet about custom axes it cannot judge', () => { + expect( + getDatasetWarnings( + createMetadata([64, 64, 64], 'uint16', axes('c', 'angle', 'y', 'x')) + ) + ).toEqual([]); + }); + + it('stays quiet about a plain array with no axes', () => { + expect(getDatasetWarnings(createMetadata([64, 64, 64]))).toEqual([]); + }); +}); diff --git a/frontend/src/components/ui/BrowsePage/MetadataHint.tsx b/frontend/src/components/ui/BrowsePage/MetadataHint.tsx index 13fadf2f..0650026f 100644 --- a/frontend/src/components/ui/BrowsePage/MetadataHint.tsx +++ b/frontend/src/components/ui/BrowsePage/MetadataHint.tsx @@ -12,6 +12,10 @@ type MetadataHintVariant = | { case: 'zarr-v2-no-multiscales' } | { case: 'zarr-v3-no-multiscales' } | { case: 'zarr-query-error'; errorMessage?: string } + // Zarr - metadata is valid, but the layout will make viewing awkward + | { case: 'zarr-single-level'; size: string } + | { case: 'zarr-large-chunks'; size: string; compressed: boolean } + | { case: 'zarr-axis-order'; axisOrder: string; expectedOrder: string } // N5 - query never fired | { case: 'n5-has-s0-no-attrs' } | { case: 'n5-has-attrs-no-s0' } @@ -72,6 +76,24 @@ function getHintConfig(variant: MetadataHintVariant): HintConfig { ? `Could not read Zarr metadata. ${variant.errorMessage}` : 'Could not read Zarr metadata.' }; + case 'zarr-single-level': + return { + kind: 'warning', + title: 'Only one resolution level', + description: `This dataset declares multiscales but only supplies a single level, for ${variant.size} of data. Viewers must read full-resolution at every zoom level, so viewing the whole image is far more expensive than it needs to be. Generating a multiscale pyramid fixes this.` + }; + case 'zarr-axis-order': + return { + kind: 'warning', + title: 'Axes are not in the order OME-Zarr specifies', + description: `The axes are ordered ${variant.axisOrder}, but the spec requires T, C, Z, Y, X order - here that would be ${variant.expectedOrder}. Many tools take the last two axes to be the image plane, so they will show a cross-section rather than the expected view. Rewriting the dataset with the axes in spec order avoids this.` + }; + case 'zarr-large-chunks': + return { + kind: 'warning', + title: 'Chunks may be too large for efficient viewing', + description: `This dataset uses ${variant.size} chunks ${variant.compressed ? '(before compression)' : '(without compression)'}. Chunk files larger than the browser's cache limit are re-downloaded on every access, which makes viewing slow. A final chunk size of 1-10 MB works best.` + }; case 'n5-has-s0-no-attrs': logger.info( 'This folder has a .n5 extension but does not contain an attributes.json file required for N5 metadata preview.' diff --git a/frontend/src/components/ui/BrowsePage/ZarrPreview.tsx b/frontend/src/components/ui/BrowsePage/ZarrPreview.tsx index fac230b7..d7d0aa8f 100644 --- a/frontend/src/components/ui/BrowsePage/ZarrPreview.tsx +++ b/frontend/src/components/ui/BrowsePage/ZarrPreview.tsx @@ -6,13 +6,14 @@ import zarrLogo from '@/assets/zarr.jpg'; import ZarrMetadataTable from '@/components/ui/BrowsePage/ZarrMetadataTable'; import DataLinkDialog from '@/components/ui/Dialogs/DataLink'; import DataToolLinks from './DataToolLinks'; +import MetadataHint from './MetadataHint'; import type { OpenWithToolUrls, ZarrMetadata, PendingToolKey } from '@/hooks/useZarrMetadata'; import useDataToolLinks from '@/hooks/useDataToolLinks'; -import { Metadata } from '@/omezarr-helper'; +import { Metadata, getDatasetWarnings } from '@/omezarr-helper'; type ZarrPreviewProps = { readonly fspName: string; @@ -41,6 +42,12 @@ export default function ZarrPreview({ const [showDataLinkDialog, setShowDataLinkDialog] = useState(false); const [pendingToolKey, setPendingToolKey] = useState(null); + const metadata = zarrMetadataQuery.data?.metadata; + const warnings = + metadata && 'arr' in metadata + ? getDatasetWarnings(metadata as Metadata) + : []; + const { handleToolClick, handleDialogConfirm, @@ -55,6 +62,13 @@ export default function ZarrPreview({ return (
+ {warnings.length > 0 ? ( +
+ {warnings.map(warning => ( + + ))} +
+ ) : null}
diff --git a/frontend/src/omezarr-helper.ts b/frontend/src/omezarr-helper.ts index c34369b2..a135c310 100644 --- a/frontend/src/omezarr-helper.ts +++ b/frontend/src/omezarr-helper.ts @@ -1,6 +1,8 @@ import { default as log } from '@/logger'; +import { formatFileSize } from '@/utils'; import * as zarr from 'zarrita'; import * as omezarr from 'ome-zarr.js'; +import { classifyCodec } from '@bioimagetools/capability-manifest'; import type { OmeZarrMetadata, MultiscaleMetadata, @@ -22,6 +24,173 @@ export type Metadata = OmeZarrMetadata & { zarrVersion: 2 | 3; }; +/** + * Something about the dataset's layout that will make it awkward to view. + * Purely advisory - nothing is withheld on account of these. + */ +export type DatasetWarning = + | { case: 'zarr-single-level'; size: string } + | { case: 'zarr-large-chunks'; size: string; compressed: boolean } + | { case: 'zarr-axis-order'; axisOrder: string; expectedOrder: string }; + +/** + * Chunks above this defeat the browser cache. Applies when the array is stored + * without compression, so the size we compute is the size that transfers. + */ +export const MAX_CHUNK_BYTES = 10 * 1024 ** 2; +/** + * The same limit for compressed arrays, where all we can compute is the logical + * (uncompressed) extent and the real transfer is some unknowable fraction of it. + * Set high enough that a typical ratio cannot push a healthy dataset over. + */ +export const MAX_LOGICAL_CHUNK_BYTES = 32 * 1024 ** 2; +/** + * Below this, a single-level image is small enough that the missing pyramid + * costs nothing worth mentioning. + */ +export const MAX_SINGLE_LEVEL_BYTES = 1024 ** 3; + +/** The axis order OME-Zarr requires, minus any axes the dataset omits. */ +const CANONICAL_AXIS_ORDER = ['t', 'c', 'z', 'y', 'x']; + +/** + * Whether the array's chunks are compressed on disk. + * + * A codec pipeline can nest: a sharded v3 array lists only `sharding_indexed` + * at the top level and carries the real compressor in its configuration, so the + * pipeline has to be walked rather than scanned. Anything `classifyCodec` does + * not recognize counts as compression, and metadata we never fetched counts as + * compression too - both keep us on the permissive threshold, where the cost of + * being wrong is a missed warning instead of a false one. + */ +function hasCompressionCodec(codecs: NonNullable): boolean { + return codecs.some(codec => { + const nested = codec.configuration?.codecs; + if (Array.isArray(nested) && hasCompressionCodec(nested)) { + return true; + } + return classifyCodec(codec.name) !== 'structural'; + }); +} + +function isStoredCompressed(metadata: Metadata): boolean { + if (metadata.codecs) { + return hasCompressionCodec(metadata.codecs); + } + if (metadata.compressor !== undefined) { + return metadata.compressor !== null; + } + return true; +} + +/** + * Bytes per element for a zarrita dtype. Only numeric dtypes are sized; bool, + * string and object dtypes fall back to 1, which under-estimates rather than + * over-warns (they don't occur in imaging data). + */ +function getBytesPerElement(dtype: string): number { + const bits = Number(/^(?:u?int|float)(\d+)$/.exec(dtype)?.[1]); + return Number.isFinite(bits) ? bits / 8 : 1; +} + +function product(dims: number[]): number { + return dims.reduce((total, dim) => total * dim, 1); +} + +/** + * The axis names, lowercased, or null if any of them is not one of the five the + * spec defines. A dataset using custom axes is not something we can judge. + */ +function getCanonicalAxisNames(metadata: Metadata): string[] | null { + const axes = metadata.multiscales?.[0]?.axes ?? metadata.axes; + if (!axes?.length) { + return null; + } + const names = axes.map(axis => axis.name?.toLowerCase()); + return names.every(name => name && CANONICAL_AXIS_ORDER.includes(name)) + ? (names as string[]) + : null; +} + +/** Whether `names` appears in CANONICAL_AXIS_ORDER order, skipping absent axes. */ +function isCanonicallyOrdered(names: string[]): boolean { + let from = 0; + return names.every(name => { + const index = CANONICAL_AXIS_ORDER.indexOf(name, from); + if (index === -1) { + return false; + } + from = index + 1; + return true; + }); +} + +const formatAxes = (names: string[]) => + names.map(name => name.toUpperCase()).join(', '); + +/** + * Flag layout choices that make a dataset awkward to view. + * + * A multiscales group with a single dataset provides no downsampled data, so + * every zoom level reads full-resolution chunks - the root cause of the incident + * that prompted these checks. + * + * Chunks past the browser's cache entry limit are re-fetched on every access. + * Sizes are computed from the shape, so they are logical: exact for an + * uncompressed array and an upper bound for a compressed one, which is why the + * limit depends on whether a compressor is in play. + * + * Axis order matters because plenty of tools take the last two axes to be the + * image plane. OME-Zarr requires t, c, z, y, x order for exactly that reason, + * and a dataset that ignores it renders as a cross-section elsewhere. + */ +export function getDatasetWarnings(metadata: Metadata): DatasetWarning[] { + const { arr } = metadata; + if (!arr) { + return []; + } + + const bytesPerElement = getBytesPerElement(arr.dtype); + const warnings: DatasetWarning[] = []; + + // Declaring multiscales with one dataset is declaring a pyramid and supplying + // none. Keyed on the dataset count rather than the number of shapes, because + // a plain zarr array also has exactly one shape and is not making any such + // claim - `arr` is level 0, so its shape is the full resolution. + const levels = metadata.multiscales?.[0]?.datasets?.length; + const fullResBytes = product(arr.shape) * bytesPerElement; + if (levels === 1 && fullResBytes > MAX_SINGLE_LEVEL_BYTES) { + warnings.push({ + case: 'zarr-single-level', + size: formatFileSize(fullResBytes) + }); + } + + const compressed = isStoredCompressed(metadata); + const chunkBytes = product(arr.chunks) * bytesPerElement; + const chunkLimit = compressed ? MAX_LOGICAL_CHUNK_BYTES : MAX_CHUNK_BYTES; + if (chunkBytes > chunkLimit) { + warnings.push({ + case: 'zarr-large-chunks', + size: formatFileSize(chunkBytes), + compressed + }); + } + + const axisNames = getCanonicalAxisNames(metadata); + if (axisNames && !isCanonicallyOrdered(axisNames)) { + warnings.push({ + case: 'zarr-axis-order', + axisOrder: formatAxes(axisNames), + expectedOrder: formatAxes( + CANONICAL_AXIS_ORDER.filter(name => axisNames.includes(name)) + ) + }); + } + + return warnings; +} + type OmeZarrChannel = { name: string; color: string; diff --git a/frontend/src/queries/zarrQueries.ts b/frontend/src/queries/zarrQueries.ts index a574ffc0..d449a10a 100644 --- a/frontend/src/queries/zarrQueries.ts +++ b/frontend/src/queries/zarrQueries.ts @@ -196,7 +196,10 @@ async function fetchZarrMetadata({ scales: undefined, omero: undefined, labels: undefined, - zarrVersion: effectiveVersion + zarrVersion: effectiveVersion, + // This zarr.json is the array metadata, so the codec pipeline is + // already in hand - no second fetch needed. + codecs: (attrs as ZarrV3ArrayMetadata).codecs }, omeZarrUrl: null, availableZarrVersions, @@ -367,6 +370,21 @@ async function fetchZarrMetadata({ log.info('Getting Zarr array for', imageUrl, 'with Zarr version', 2); const arr = await getZarrArray(imageUrl, 2); const shapes = [arr.shape]; + + // Read the compressor so chunk-size warnings know whether the logical + // size is also the stored size. Left undefined on failure, which callers + // treat as compressed. + let compressor: ZarrV2ArrayMetadata['compressor']; + try { + const arrayMeta = (await fetchFileAsJson( + fspName, + zarrayFile.path + )) as ZarrV2ArrayMetadata; + compressor = arrayMeta.compressor; + } catch (error) { + log.trace('Could not fetch .zarray for compressor:', error); + } + return { metadata: { arr, @@ -375,7 +393,8 @@ async function fetchZarrMetadata({ scales: undefined, omero: undefined, labels: undefined, - zarrVersion: 2 + zarrVersion: 2, + compressor }, omeZarrUrl: null, availableZarrVersions,