Feat/quality based upsampling

config_files/data_preparation/quality/annealing_registry.yaml

@@ -0,0 +1,181 @@
+# Corpus registry for the annealing blend.
+#
+# Every join key below was verified by sampling real keys out of the JSONL and scanning
+# the propella `id` columns for them. Coverage measured 2026-08-17 against the caches
+# named in `annotation_root`:
+#
+#   finewiki (all 5 languages)                100%
+#   hplt-4-unfiltered (de/fr/it/es)           100%
+#   nemotron-cc/high-actual                   100%  (also covers the synthetic
+#                                                    high_diverse_qa_pairs subdirectory,
+#                                                    which shares warc_record_id)
+#   nemotron-climbmix                         100%  (English, and German via the source)
+#   finepdfs (eng/deu/fra/spa)                10-34%  only 1-5 shards fetched per split
+#
+# Datasets with no `annotation_split` have no propella corpus at all and can only be
+# shaped by their native metrics.
+
+annotation_root: /data/michael.fromm/hf-cache/datasets--openeurollm--propella-annotations/snapshots/e80fc1407801a15b956f40c642d3709b528abbc9/data/propella-1-4b
+extra_annotation_roots:
+  # The HPLT shards were fetched into a second cache; both are searched.
+  - /data/alex.jude/.cache/huggingface/datasets--openeurollm--propella-annotations/snapshots/9e9c5083f81dc4bbd2708b65816a4dc41f59b911/data/propella-1-4b
+
+datasets:
+  # ---------------------------------------------------------------- FineWiki
+  # Ids look like `dewiki/6851323` and are used verbatim on both sides.
+  - name: finewiki-en
+    jsonl_root: /data/annealing/english/Finewiki
+    annotation_split: finewiki
+    key: {kind: field, field: id}
+    native_metrics:
+      - {name: bytes_html, jq_pattern: .bytes_html}
+  - name: finewiki-de
+    jsonl_root: /data/annealing/german/Finewiki
+    annotation_split: finewiki
+    key: {kind: field, field: id}
+  - name: finewiki-fr
+    jsonl_root: /data/annealing/french/Finewiki
+    annotation_split: finewiki
+    key: {kind: field, field: id}
+  - name: finewiki-it
+    jsonl_root: /data/annealing/italian/Finewiki
+    annotation_split: finewiki
+    key: {kind: field, field: id}
+  - name: finewiki-es
+    jsonl_root: /data/annealing/spanish/Finewiki
+    annotation_split: finewiki
+    key: {kind: field, field: id}
+
+  # ---------------------------------------------------------------- HPLT v4
+  # 32-hex ids, used verbatim. No quality score of any kind in the records, so propella
+  # is the only quality signal available for these 14 TB.
+  - name: hplt-de
+    jsonl_root: /data/annealing/german/hpltv4
+    annotation_split: hplt-4-unfiltered/deu_Latn
+    key: {kind: field, field: id}
+    native_metrics:
+      - {name: lid_prob, jq_pattern: ".\"openlid-v3\".prob", aggregation: max}
+      - {name: cluster_size, jq_pattern: .cluster_size}
+  - name: hplt-fr
+    jsonl_root: /data/annealing/french/hpltv4
+    annotation_split: hplt-4-unfiltered/fra_Latn
+    key: {kind: field, field: id}
+    native_metrics:
+      - {name: lid_prob, jq_pattern: ".\"openlid-v3\".prob", aggregation: max}
+      - {name: cluster_size, jq_pattern: .cluster_size}
+  - name: hplt-it
+    jsonl_root: /data/annealing/italian/hpltv4
+    annotation_split: hplt-4-unfiltered/ita_Latn
+    key: {kind: field, field: id}
+    native_metrics:
+      - {name: lid_prob, jq_pattern: ".\"openlid-v3\".prob", aggregation: max}
+      - {name: cluster_size, jq_pattern: .cluster_size}
+  - name: hplt-es
+    jsonl_root: /data/annealing/spanish/hpltv4
+    annotation_split: hplt-4-unfiltered/spa_Latn
+    key: {kind: field, field: id}
+    native_metrics:
+      - {name: lid_prob, jq_pattern: ".\"openlid-v3\".prob", aggregation: max}
+      - {name: cluster_size, jq_pattern: .cluster_size}
+
+  # ---------------------------------------------------------------- Nemotron-CC
+  # No `id` field at all; `warc_record_id` is the annotation key. The annotation ids are
+  # not unique -- roughly 4% recur -- so the join keeps the first occurrence. This is not
+  # specific to this split: finewiki measured 868,586 duplicate keys in 43.1 M rows (2.0%)
+  # on a real run, so expect the join to report duplicates for most splits.
+  # The 2026-08 delivery renamed `warc_record_id` to `id`, keeping the same bare UUIDs.
+  # The old name silently produced a null join key for every document and 0% coverage.
+  # Both subdirectories -- high_actual and high_diverse_qa_pairs -- are covered by the
+  # high-actual annotation split; 400/400 sampled ids from each were found in it.
+  - name: nemotron-cc
+    jsonl_root: /data/annealing/english/Nemotron-CC
+    annotation_split: nemotron-cc/high-actual
+    key: {kind: field, field: id}
+
+  # ---------------------------------------------------------------- ClimbMix
+  # No identifier of any kind; the annotation key is the SHA-256 of the exact text.
+  - name: climbmix-en
+    jsonl_root: /data/annealing/english/Climbmix
+    annotation_split: nemotron-climbmix
+    key: {kind: sha256_text}
+
+  # German KletterMix is a translation of ClimbMix. Its own text is German, so hashing
+  # it matches nothing -- the annotation belongs to the English original, which the id
+  # points at as `<file>/<line>`, zero-indexed.
+  - name: klettermix-de
+    jsonl_root: /data/annealing/german/AIML-TUDA-KletterMix-filtered
+    glob: "*.jsonl"
+    annotation_split: nemotron-climbmix
+    key:
+      kind: source_pointer
+      field: id
+      source_root: /data/annealing_unused/Nemotron-ClimbMix
+      source_line_offset: 0
+    native_metrics:
+      - {name: proxy_score, jq_pattern: .proxy_score}
+      - {name: token_count, jq_pattern: .token_count}
+
+  # ---------------------------------------------------------------- FinePDFs
+  # Ids are UUIDs stored in two forms -- `<urn:uuid:...>` and bare -- mixed within
+  # single files on both sides, so both sides are normalised before comparing.
+  - name: finepdfs-en
+    jsonl_root: /data/annealing/english/Finepdfs
+    annotation_split: finepdfs/eng_Latn
+    key: {kind: urn_uuid_field, field: id}
+    note: "annotation coverage was 34% on 2026-08-17; finish the fetch before relying on a propella predicate"
+    native_metrics:
+      - {name: fw_edu, jq_pattern: .fw_edu_scores, aggregation: max}
+      - {name: dclm, jq_pattern: .dclm_scores, aggregation: max}
+      - {name: ocr_quality, jq_pattern: .ocr_quality_scores, aggregation: min}
+      - {name: full_doc_lid_score, jq_pattern: .full_doc_lid_score}
+      - {name: duplicate_count, jq_pattern: .duplicate_count}
+  - name: finepdfs-de
+    jsonl_root: /data/annealing/german/Finepdfs
+    annotation_split: finepdfs/deu_Latn
+    key: {kind: urn_uuid_field, field: id}
+    native_metrics:
+      - {name: fw_edu, jq_pattern: .fw_edu_scores, aggregation: max}
+      - {name: full_doc_lid_score, jq_pattern: .full_doc_lid_score}
+  - name: finepdfs-fr
+    jsonl_root: /data/annealing/french/Finepdfs
+    annotation_split: finepdfs/fra_Latn
+    key: {kind: urn_uuid_field, field: id}
+    native_metrics:
+      - {name: fw_edu, jq_pattern: .fw_edu_scores, aggregation: max}
+      - {name: full_doc_lid_score, jq_pattern: .full_doc_lid_score}
+  - name: finepdfs-es
+    jsonl_root: /data/annealing/spanish/Finepdfs
+    annotation_split: finepdfs/spa_Latn
+    key: {kind: urn_uuid_field, field: id}
+    native_metrics:
+      - {name: fw_edu, jq_pattern: .fw_edu_scores, aggregation: max}
+      - {name: full_doc_lid_score, jq_pattern: .full_doc_lid_score}
+
+  # `italian/Finepdfs` holds English data, byte-identical to `english/Finepdfs` in 578
+  # of its 580 files. Enabling it would count the same 5.4 TB of English documents
+  # twice, so it stays declared and disabled until that is resolved.
+  - name: finepdfs-it
+    jsonl_root: /data/annealing/italian/Finepdfs
+    annotation_split: finepdfs/eng_Latn
+    key: {kind: urn_uuid_field, field: id}
+    enabled: false
+    note: "English data in the Italian folder; duplicates finepdfs-en"
+
+  # -------------------------------------------- no propella corpus, native only
+  # The largest subset of the blend. Every candidate annotation key returns 0%, so it
+  # can only be shaped by its category directory and native fields.
+  - name: nemotron-cc-v2
+    jsonl_root: /data/annealing/english/Nemotron-CC-v2
+    native_metrics: []
+  - name: finephrase
+    jsonl_root: /data/annealing/english/Finephrase
+    native_metrics:
+      - {name: fw_edu, jq_pattern: .score}
+      - {name: language_score, jq_pattern: .language_score}
+      - {name: token_count, jq_pattern: .token_count}
+  - name: dolmino
+    jsonl_root: /data/annealing/english/Dolmino
+    note: "dclm_plus2 exists only under stem-heavy-crawl; it was null in all of 400 sampled sidecar parts, so the cube drops it"
+    native_metrics:
+      - {name: dclm_plus2, jq_pattern: '.metadata.dclm_plus2."__label__1"'}
+      - {name: len_cl100k_base, jq_pattern: .metadata.len_cl100k_base}

config_files/data_preparation/quality/annealing_selection.yaml

@@ -0,0 +1,145 @@
+# Example selection for the annealing blend.
+#
+# Each dataset states which documents to keep and how heavily to sample what survives.
+# `modalities quality preview` costs this in documents and tokens in seconds; nothing
+# is read or written until `modalities quality apply`.
+#
+# Predicates within a dataset are combined with AND. Ordinal levels come from the
+# declared scales in `selection.py`; note that `information_density` orders
+# `moderate` *below* `adequate`, which is easy to get backwards.
+
+# What to do with documents that carry no annotation. `keep` treats a propella
+# predicate as satisfied for them, which is the safe default while FinePDFs is only
+# partly downloaded -- `drop` there would silently discard two thirds of the corpus for
+# having no label rather than for failing the filter.
+missing_annotation: keep
+
+# What the training run will consume. It adjusts no ratio, but it is what makes the ratios
+# checkable: a ratio is per pass over the blend, so if the blend yields less than this the
+# loader wraps and every factor is multiplied by the number of passes.
+target_tokens: 400_000_000_000
+
+# Refuse to materialise if any dataset would be seen more often than this once wrapping is
+# counted. Set above the largest ratio below (3.0) so it does not fire on the intended
+# design, only on an accident -- a blend trimmed too far, or a ratio raised without
+# re-checking the yield. A dataset with an `upsampling` curve uses its own `max_factor`
+# instead, since that is already a declared cap. Pass --allow_overexposure to override.
+max_total_exposure: 4.0
+
+datasets:
+  # Web text carries the most junk, so it gets the strictest filter and is downsampled.
+  - name: hplt-de
+    ratio: 0.6
+    predicates:
+      - {field: educational_value, op: at_least, value: basic}
+      - {field: content_integrity, op: at_least, value: mostly_complete}
+      - {field: content_safety, op: at_least, value: mild_concerns}
+  - name: hplt-fr
+    ratio: 0.6
+    predicates:
+      - {field: educational_value, op: at_least, value: basic}
+      - {field: content_integrity, op: at_least, value: mostly_complete}
+      - {field: content_safety, op: at_least, value: mild_concerns}
+  - name: hplt-it
+    ratio: 0.8
+    predicates:
+      - {field: educational_value, op: at_least, value: basic}
+      - {field: content_integrity, op: at_least, value: mostly_complete}
+  - name: hplt-es
+    ratio: 0.6
+    predicates:
+      - {field: educational_value, op: at_least, value: basic}
+      - {field: content_integrity, op: at_least, value: mostly_complete}
+
+  # Already filtered upstream to a high-quality subset, so a light touch and a mild
+  # upsample.
+  #
+  # This is the natural place to try a quality-aware curve instead of the flat ratio: a
+  # single number repeats the barely-adequate documents exactly as often as the excellent
+  # ones. Replace `ratio` with the block below to make the repeat factor rise with quality.
+  # `preview` prints the resulting per-bucket factors, and `apply` then writes one index
+  # tree per bucket so each can carry its own factor into training.
+  #
+  #   upsampling:
+  #     quality_field: content_quality        # ordinal axis, worst to best
+  #     target_tokens: 900_000_000_000        # what this dataset should contribute
+  #     max_factor: 7.0                       # no bucket repeated more than this
+  #     discard_below_percentile: 40          # drop the weakest 40% of tokens
+  #
+  # Note that `ratio` and `upsampling` are mutually exclusive: the curve already determines how much
+  # is drawn, so setting both is refused rather than silently ignoring one.
+  - name: nemotron-cc
+    ratio: 1.2
+    predicates:
+      - {field: content_quality, op: at_least, value: adequate}
+      # `commercial_bias` is joined onto the sidecar but is not one of the seven fields
+      # the cube groups on by default, so thresholding it here would force the preview to
+      # read all 1.7 bn documents. To use it, rebuild this dataset's cube with
+      #   build-cube --only nemotron-cc --label_dimension commercial_bias ...
+      # naming the other seven too, since the flag replaces the default set.
+
+  - name: climbmix-en
+    ratio: 1.0
+    predicates:
+      - {field: educational_value, op: at_least, value: basic}
+      - {field: information_density, op: at_least, value: moderate}
+
+  # The German pool is small relative to the English one, so what survives is upsampled.
+  # Combines a propella predicate with the corpus's own proxy score.
+  - name: klettermix-de
+    ratio: 2.0
+    predicates:
+      - {field: educational_value, op: at_least, value: basic}
+      - {field: proxy_score, op: gte, value: 0.65}
+
+  # Reference text: keep all of it, upsample the smaller languages.
+  - name: finewiki-en
+    ratio: 1.0
+  - name: finewiki-de
+    ratio: 2.0
+  - name: finewiki-fr
+    ratio: 2.0
+  - name: finewiki-it
+    ratio: 3.0
+  - name: finewiki-es
+    ratio: 3.0
+
+  # Only 10-34% annotated. `missing_annotation: keep` above means the propella
+  # predicate applies to the annotated minority and the rest passes on the native
+  # score alone -- deliberate, and worth revisiting once the fetch is finished.
+  - name: finepdfs-en
+    ratio: 1.0
+    predicates:
+      - {field: fw_edu, op: gte, value: 2.0}
+      - {field: content_integrity, op: at_least, value: mostly_complete}
+  - name: finepdfs-de
+    ratio: 1.5
+    predicates:
+      - {field: fw_edu, op: gte, value: 1.5}
+  - name: finepdfs-fr
+    ratio: 1.5
+    predicates:
+      - {field: fw_edu, op: gte, value: 1.5}
+  - name: finepdfs-es
+    ratio: 1.5
+    predicates:
+      - {field: fw_edu, op: gte, value: 1.5}
+
+  # No propella corpus exists for these, so they are shaped by native metrics only.
+  - name: finephrase
+    ratio: 0.5
+    predicates:
+      - {field: fw_edu, op: gte, value: 2.5}
+      - {field: language_score, op: gte, value: 0.9}
+  - name: dolmino
+    ratio: 1.0
+    predicates:
+      # `dclm_plus2` only exists in the stem-heavy-crawl subdirectory -- it was null in
+      # every one of 400 sampled sidecar parts -- so it is dropped from the cube and
+      # cannot be thresholded. Length is the signal that is actually present here.
+      - {field: len_cl100k_base, op: gte, value: 200}
+
+  # Declared but excluded, so the reason is recorded rather than implied by absence.
+  - name: nemotron-cc-v2
+    ratio: 1.0
+    enabled: false

config_files/data_preparation/quality/annealing_tokenizer.yaml

@@ -0,0 +1,19 @@
+# The tokenizer the token estimates are measured with.
+#
+# Used by `modalities quality calibrate --tokenizer_config <this file>`, which measures
+# how many tokens each dataset's bytes are worth. Nothing here tokenizes the corpus: the
+# pipeline now exports JSONL and leaves tokenization to whatever consumes it. The estimate
+# is what makes `preview` able to cost a selection in tokens, so it should still name the
+# tokenizer the run actually trains with -- getting it wrong does not fail loudly, it
+# produces a plausible token budget for the wrong tokenizer.
+
+tokenizer:
+  component_key: tokenizer
+  variant_key: pretrained_hf_tokenizer
+  config:
+    # The tokenizer the long-context pipeline uses. The token audit under
+    # /data/michael.fromm used the Super-120B variant instead -- confirm which this run
+    # trains with before trusting any token figure.
+    pretrained_model_name_or_path: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16
+    padding: false
+    truncation: false