From 637aa9a71c96b9bb7228d18027242808a1ba1426 Mon Sep 17 00:00:00 2001 From: hjh0119 Date: Thu, 8 Oct 2026 10:27:57 +0800 Subject: [PATCH 1/5] feat(decision): add typed-decision (System-1) task_type for JEV/Clef/OmniJev Non-generative `decision` task_type: a single forward pass scores every option into calibrated per-option probabilities. transformers backend only. - model: ScoringModelLoader + JEV/Clef/OmniJev heads; OmniJev two-stage cached branch forward over the hybrid Qwen3.5 backbone + media compose - template: decision encode/collate (train right- / infer left-padding) - dataset: unified user schema, per-model preprocessor subclasses - loss: ScoringLoss + JEV/Clef/OmniJev heads (masked soft-CE + ordinal RPS) - trainer: ScoringTrainer re-activates and saves the fp32 head under LoRA - examples/train/decision/{jev,clef,omnijev}.sh + dataset plugin + data - tests/test_align: four-stage alignment harness vs each official oracle Aligned to each official inference path: encoding byte-exact, head math <=1e-7, full forward within bf16 tolerance, loss+grad vs re-derived formula. --- examples/train/decision/clef.sh | 43 + examples/train/decision/data/clef.jsonl | 4 + examples/train/decision/data/jev.jsonl | 4 + examples/train/decision/data/omnijev.jsonl | 4 + examples/train/decision/dataset.py | 56 + examples/train/decision/jev.sh | 54 + examples/train/decision/omnijev.sh | 69 ++ swift/arguments/base_args/model_args.py | 2 +- swift/dataset/__init__.py | 2 +- swift/dataset/preprocessor/__init__.py | 1 + swift/dataset/preprocessor/core.py | 17 + swift/dataset/preprocessor/decision.py | 641 ++++++++++ swift/infer_engine/transformers_engine.py | 7 + swift/loss/decision.py | 226 ++++ swift/loss/mapping.py | 6 + swift/model/constant.py | 5 + swift/model/decision_head.py | 1103 +++++++++++++++++ swift/model/models/__init__.py | 4 +- swift/model/models/decision.py | 391 ++++++ swift/model/omnijev_branch.py | 323 +++++ swift/model/omnijev_media.py | 261 ++++ swift/model/register.py | 4 +- swift/pipelines/train/tuner.py | 10 + swift/template/base.py | 175 ++- swift/template/constant.py | 5 + swift/template/templates/__init__.py | 6 +- swift/template/templates/decision.py | 527 ++++++++ swift/trainers/__init__.py | 2 + swift/trainers/decision_trainer.py | 103 ++ swift/trainers/trainer_factory.py | 2 + tests/test_align/test_decision_clef_align.py | 390 ++++++ tests/test_align/test_decision_jev_align.py | 526 ++++++++ .../test_align/test_decision_omnijev_align.py | 573 +++++++++ 33 files changed, 5535 insertions(+), 11 deletions(-) create mode 100644 examples/train/decision/clef.sh create mode 100644 examples/train/decision/data/clef.jsonl create mode 100644 examples/train/decision/data/jev.jsonl create mode 100644 examples/train/decision/data/omnijev.jsonl create mode 100644 examples/train/decision/dataset.py create mode 100644 examples/train/decision/jev.sh create mode 100644 examples/train/decision/omnijev.sh create mode 100644 swift/dataset/preprocessor/decision.py create mode 100644 swift/loss/decision.py create mode 100644 swift/model/decision_head.py create mode 100644 swift/model/models/decision.py create mode 100644 swift/model/omnijev_branch.py create mode 100644 swift/model/omnijev_media.py create mode 100644 swift/template/templates/decision.py create mode 100644 swift/trainers/decision_trainer.py create mode 100644 tests/test_align/test_decision_clef_align.py create mode 100644 tests/test_align/test_decision_jev_align.py create mode 100644 tests/test_align/test_decision_omnijev_align.py diff --git a/examples/train/decision/clef.sh b/examples/train/decision/clef.sh new file mode 100644 index 0000000000..5bedf4b524 --- /dev/null +++ b/examples/train/decision/clef.sh @@ -0,0 +1,43 @@ +# Typed-decision (System-1) fine-tuning of Cloudflare/clef. +# +# Clef decodes ALL of a record's questions jointly in ONE forward (the only multi-question-per-forward +# model of the three), scoring each option of every question and softmaxing within each question's own +# option set. `--task_type decision` routes to ScoringTrainer + the decision data collator; +# `--loss_type clef` is label-smoothing CE + Brier over each question's options. +# +# The scoring head (`scoring_head`) is the decision layer; ScoringTrainer re-activates it for joint +# training AUTOMATICALLY (the `--trainable_parameters scoring_head` flag is a no-op in the LoRA branch, +# so it is omitted). Clef ships a MERGED backbone that already carries the decision ability (no adapter +# to continue), so a fresh `--tuner_type lora --target_modules all-linear` is correct here -- unlike +# JEV/OmniJev, which continue-train their shipped adapter via `--adapters`. `--attn_impl eager` matches +# tests/test_align/test_decision_clef_align.py (removes sdpa/flash kernel divergence). +# +# sh examples/train/decision/clef.sh +USE_HF=1 \ +CUDA_VISIBLE_DEVICES=0 \ +swift sft \ + --external_plugins examples/train/decision/dataset.py \ + --model Cloudflare/clef \ + --model_type clef \ + --task_type decision \ + --loss_type clef \ + --dataset clef_decision \ + --tuner_type lora \ + --target_modules all-linear \ + --torch_dtype bfloat16 \ + --attn_impl eager \ + --num_train_epochs 1 \ + --per_device_train_batch_size 1 \ + --per_device_eval_batch_size 1 \ + --learning_rate 1e-4 \ + --lora_rank 8 \ + --lora_alpha 32 \ + --gradient_accumulation_steps 4 \ + --gradient_checkpointing true \ + --logging_steps 1 \ + --save_steps 50 \ + --save_total_limit 2 \ + --max_length 4096 \ + --split_dataset_ratio 0 \ + --output_dir output \ + --dataloader_num_workers 1 diff --git a/examples/train/decision/data/clef.jsonl b/examples/train/decision/data/clef.jsonl new file mode 100644 index 0000000000..155df8f370 --- /dev/null +++ b/examples/train/decision/data/clef.jsonl @@ -0,0 +1,4 @@ +{"id": "clef_noul", "state": {"text": "The sky is blue.", "lang": "en"}, "questions": {"is_blue": {"type": "noul", "instructions": "Is the sky blue?"}}, "answers": {"is_blue": true}} +{"id": "clef_choice", "state": "A review: the product arrived late and was damaged.", "questions": {"sentiment": {"type": "choice", "instructions": "Classify the sentiment.", "criteria": {"positive": "Good", "negative": "Bad", "neutral": "Mixed"}}}, "answers": {"sentiment": "negative"}} +{"id": "clef_score", "state": {"doc": "some text to rate"}, "questions": {"quality": {"type": "score", "instructions": "Rate quality.", "criteria": ["terrible", "poor", "ok", "good", "great"]}}, "answers": {"quality": 3}} +{"id": "clef_joint", "state": {"user": "alice", "msg": "hello there", "flag": true}, "questions": {"spam": {"type": "noul", "instructions": "Is this spam?"}, "topic": {"type": "choice", "instructions": "Pick a topic.", "criteria": {"b_tech": "Technology", "a_sport": "Sports", "c_food": "Food"}}, "polite": {"type": "score", "instructions": "Politeness level.", "criteria": ["rude", "neutral", "polite"]}}, "answers": {"spam": false, "topic": "a_sport", "polite": 2}} diff --git a/examples/train/decision/data/jev.jsonl b/examples/train/decision/data/jev.jsonl new file mode 100644 index 0000000000..5345efe11c --- /dev/null +++ b/examples/train/decision/data/jev.jsonl @@ -0,0 +1,4 @@ +{"state": "The sky is blue.", "questions": [{"kind": "noul", "question": "Is the sky blue?", "gold": 1}]} +{"state": "A review: the product arrived late and was damaged.", "questions": [{"kind": "choice", "question": "Classify the sentiment.", "options": ["positive", "negative", "neutral"], "gold": 1}]} +{"state": "some text to rate", "questions": [{"kind": "score", "question": "Rate quality.", "gold": [0.05, 0.1, 0.15, 0.4, 0.2, 0.1]}]} +{"state": "A joint record rendered one question per forward.", "questions": [{"kind": "noul", "question": "Is this spam?", "gold": 0}, {"kind": "choice", "question": "Pick a topic.", "options": ["Technology", "Sports", "Food"], "gold": 1}, {"kind": "score", "question": "Politeness level.", "gold": 4}]} diff --git a/examples/train/decision/data/omnijev.jsonl b/examples/train/decision/data/omnijev.jsonl new file mode 100644 index 0000000000..499a5c1f76 --- /dev/null +++ b/examples/train/decision/data/omnijev.jsonl @@ -0,0 +1,4 @@ +{"state": "Look at the image.", "images": ["http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png"], "questions": [{"type": "noul", "instructions": "Is there an animal in the image?", "gold": true}]} +{"images": ["http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png"], "questions": [{"type": "choice", "instructions": "What is in the image?", "options": [{"key": "cat", "text": "a cat"}, {"key": "dog", "text": "a dog"}, {"key": "bird", "text": "a bird"}], "gold": "cat"}]} +{"images": ["http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png"], "questions": [{"type": "score", "instructions": "How clear is the image?", "levels": ["blurry", "ok", "sharp"], "gold": "sharp"}]} +{"images": ["http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png"], "questions": [{"type": "choice", "instructions": "Which region contains the animal?", "options": [{"key": "left", "region": {"box": [0, 0, 100, 100]}}, {"key": "right", "region": {"box": [100, 0, 200, 100]}}, {"key": "none", "abstain": true}], "gold": "left"}]} diff --git a/examples/train/decision/dataset.py b/examples/train/decision/dataset.py new file mode 100644 index 0000000000..7827af3aa7 --- /dev/null +++ b/examples/train/decision/dataset.py @@ -0,0 +1,56 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""External plugin: register the three typed-decision mini datasets used by these examples. + +Each model has its own `ScoringPreprocessor` subclass (see `swift/dataset/preprocessor/decision.py`) +because the three corpora differ in raw shape: + * Clef -- ONE record -> ONE row carrying ALL its questions (joint decode); native + `{state, questions: {qid: {type, instructions, criteria}}, answers}` schema. + * JEV -- ONE record -> ONE row PER question (s1_pass scores one question per forward); unified + `{state, questions: [{kind, question, options, gold}]}` schema. + * OmniJev -- ONE record -> ONE row PER question, multimodal state (image / pre-composed video + mosaic); unified option blocks with region / abstain support. + +Pass this file to `swift sft --external_plugins examples/train/decision/dataset.py` and reference a +dataset by its registered name (`clef_decision` / `jev_decision` / `omnijev_decision`). +""" +import os + +from swift.dataset import DatasetMeta, register_dataset +from swift.dataset.preprocessor.decision import ClefPreprocessor, JevPreprocessor, OmniJevPreprocessor + +_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') + +register_dataset( + DatasetMeta( + dataset_name='clef_decision', + dataset_path=os.path.join(_DATA_DIR, 'clef.jsonl'), + preprocess_func=ClefPreprocessor(), + ), + exist_ok=True) + +register_dataset( + DatasetMeta( + dataset_name='jev_decision', + dataset_path=os.path.join(_DATA_DIR, 'jev.jsonl'), + preprocess_func=JevPreprocessor(), + ), + exist_ok=True) + +register_dataset( + DatasetMeta( + dataset_name='omnijev_decision', + dataset_path=os.path.join(_DATA_DIR, 'omnijev.jsonl'), + preprocess_func=OmniJevPreprocessor(), + ), + exist_ok=True) + +if __name__ == '__main__': + # Quick self-check: load each dataset and print the normalized decision row contract. + from swift.dataset import load_dataset + for name in ('clef_decision', 'jev_decision', 'omnijev_decision'): + ds = load_dataset([name], num_proc=1)[0] + print(f'==== {name}: {len(ds)} rows ====') + row = ds[0] + for k in ('messages', 'questions', 'kinds', 'options', 'target_probs', 'images'): + if k in row: + print(f' {k}: {row[k]}') diff --git a/examples/train/decision/jev.sh b/examples/train/decision/jev.sh new file mode 100644 index 0000000000..fd3e0becd2 --- /dev/null +++ b/examples/train/decision/jev.sh @@ -0,0 +1,54 @@ +# Typed-decision (System-1) DISTILLATION fine-tuning of autotrust/JEV-27B-VL. +# +# JEV scores ONE question per forward (serve_decide.py::s1_pass), so JevPreprocessor fans a record out +# into one row per question. It is a DISTILLATION model: `--loss_type jev_distill` is +# KL(target || model) on the activated option distribution (+ 0.5*RPS on `score`), and `target_probs` +# may be a full soft distribution (the teacher's) rather than a one-hot. `--task_type decision` routes +# to ScoringTrainer; `--attn_impl eager` matches tests/test_align/test_decision_jev_align.py. +# +# CONTINUE-TRAINING, not a fresh LoRA: JEV's decision ability lives in the shipped adapter +# `adapter_vllm/` (a backbone LoRA + the 24-slot verbalizer head packaged as an lm_head LoRA). So +# `--model_type jev` loads the base Qwen3.5-VL and builds the fp32 calibration head from +# `adapter_vllm/decision_head.json` (+ calibration.json), and `--adapters .../adapter_vllm` continues +# training THAT LoRA (is_trainable=True). A fresh `--tuner_type lora --target_modules all-linear` +# would DISCARD the trained lm_head-LoRA the head reads, so it is wrong for JEV. adapter_vllm ships no +# args.json, which is fine: `swift sft` defaults `--load_args false`, so nothing tries to read it. +# +# The head's fp32 `bias` is re-activated for joint training AUTOMATICALLY by ScoringTrainer (the +# `--trainable_parameters scoring_head` flag is a no-op in the adapter branch, so it is omitted). On +# save, ScoringTrainer._save writes the trained head beside the adapter as scoring_head.safetensors + +# scoring_head.json; the next `--adapters ` (or `swift infer`) reloads it via the guarded +# maybe_load_trained_scoring_head hook. +# +# sh examples/train/decision/jev.sh +# The decision LoRA lives in the model repo's `adapter_vllm/` subfolder. Resolve its local snapshot path +# with the SAME interpreter that runs `swift` (read from the swift entry-point shebang), so this works +# no matter which `python` happens to be first on PATH. USE_HF=1 -> huggingface_hub. +SWIFT_PY="$(sed -n '1s/^#!//p' "$(command -v swift)")" +JEV_ADAPTER="$("$SWIFT_PY" -c "import os; from huggingface_hub import snapshot_download; print(os.path.join(snapshot_download('autotrust/JEV-27B-VL'), 'adapter_vllm'))")" + +USE_HF=1 \ +CUDA_VISIBLE_DEVICES=0 \ +swift sft \ + --external_plugins examples/train/decision/dataset.py \ + --model autotrust/JEV-27B-VL \ + --model_type jev \ + --task_type decision \ + --adapters "$JEV_ADAPTER" \ + --loss_type jev_distill \ + --dataset jev_decision \ + --torch_dtype bfloat16 \ + --attn_impl eager \ + --num_train_epochs 1 \ + --per_device_train_batch_size 1 \ + --per_device_eval_batch_size 1 \ + --learning_rate 1e-4 \ + --gradient_accumulation_steps 4 \ + --gradient_checkpointing true \ + --logging_steps 1 \ + --save_steps 50 \ + --save_total_limit 2 \ + --max_length 4096 \ + --split_dataset_ratio 0 \ + --output_dir output \ + --dataloader_num_workers 1 diff --git a/examples/train/decision/omnijev.sh b/examples/train/decision/omnijev.sh new file mode 100644 index 0000000000..56fb0ecff6 --- /dev/null +++ b/examples/train/decision/omnijev.sh @@ -0,0 +1,69 @@ +# Typed-decision (System-1) fine-tuning of tinnel123/OmniJev. +# +# OmniJev scores ONE question per forward over a MULTIMODAL state (image, or a video pre-composed +# offline into a mosaic). OmniJevPreprocessor fans a record out into one row per question and carries +# the official option-BLOCK structure plus width-matched `omnijev_column_keys` (noul -> [yes,no]; +# choice -> keys + an extra learned `abstain` column; score -> the level labels). Because the hybrid +# Qwen3.5-4B backbone carries recurrent state, options are scored by the two-stage cached BRANCH +# (swift/model/omnijev_branch.py), not a block-diagonal mask. +# +# `--loss_type omnijev` uses PROPER scoring rules only (faithful to mso/head.py: no label smoothing). +# +# CONTINUE-TRAINING a SPLIT layout, not a fresh LoRA: OmniJev ships adapter-only (tinnel123/OmniJev = a +# Qwen/Qwen3.5-4B LoRA + the head artifacts head.pt / ord.pt / new_tok_emb.pt / head_meta.json, with NO +# base weights and NO config.json). So `--model Qwen/Qwen3.5-4B` supplies the hybrid base and +# `--adapters ` continues training THAT LoRA (is_trainable=True); a fresh `--tuner_type lora +# --target_modules all-linear` would discard it. The two live on DIFFERENT hubs: the base Qwen3.5-4B is +# distributed on ModelScope (so `USE_HF=0` -> swift resolves it there), while the tinnel123/OmniJev +# adapter is HF-only and is resolved by the explicit `huggingface_hub.snapshot_download` below (which is +# independent of `USE_HF`). Because the loader's model_dir is the BASE dir, the four +# head artifacts (which live in the adapter dir) are pointed at via OMNIJEV_* envs -- including +# new_tok_emb.pt, the ONLY source of the two option-token embeddings (modules_to_save=None, so they are +# not in the LoRA; a missing file must error, never silently mean-init). The head's fp32 params are +# re-activated for joint training AUTOMATICALLY by ScoringTrainer (--trainable_parameters is a no-op in +# the adapter branch, so it is omitted), and saved beside the adapter on checkpoint. +# +# MEMORY: the cached BRANCH is fundamentally incompatible with HF `--gradient_checkpointing` (that flag +# forces use_cache=False inside the backbone, but the branch's prefix pass MUST return a KV/recurrent +# cache to reuse per option row). `_omnijev_branch_forward` therefore auto-disables gradient_checkpointing +# around the branch, so `--gradient_checkpointing` is a NO-OP for OmniJev. The branch's own chunk-level +# activation recomputation (MSO_BRANCH_CKPT=1, swift/model/omnijev_branch.py) provides the memory savings +# instead -- set it for real runs; leave --gradient_checkpointing false to avoid the false impression. +# +# sh examples/train/decision/omnijev.sh +# Resolve the OmniJev adapter dir with the SAME interpreter that runs `swift` (its entry-point shebang), +# then point the OMNIJEV_* head-artifact envs at it (they live in the adapter dir, not the base dir). +SWIFT_PY="$(sed -n '1s/^#!//p' "$(command -v swift)")" +OMNIJEV_CKPT="$("$SWIFT_PY" -c "from huggingface_hub import snapshot_download; print(snapshot_download('tinnel123/OmniJev'))")" +export OMNIJEV_HEAD="$OMNIJEV_CKPT/head.pt" +export OMNIJEV_ORD="$OMNIJEV_CKPT/ord.pt" +export OMNIJEV_HEAD_META="$OMNIJEV_CKPT/head_meta.json" +export OMNIJEV_NEW_TOK_EMB="$OMNIJEV_CKPT/new_tok_emb.pt" +export MSO_BRANCH_CKPT=1 + +USE_HF=0 \ +CUDA_VISIBLE_DEVICES=0 \ +swift sft \ + --external_plugins examples/train/decision/dataset.py \ + --model Qwen/Qwen3.5-4B \ + --model_type omnijev \ + --task_type decision \ + --adapters "$OMNIJEV_CKPT" \ + --loss_type omnijev \ + --dataset omnijev_decision \ + --torch_dtype bfloat16 \ + --attn_impl eager \ + --freeze_vit true \ + --num_train_epochs 1 \ + --per_device_train_batch_size 1 \ + --per_device_eval_batch_size 1 \ + --learning_rate 1e-4 \ + --gradient_accumulation_steps 4 \ + --gradient_checkpointing false \ + --logging_steps 1 \ + --save_steps 50 \ + --save_total_limit 2 \ + --max_length 4096 \ + --split_dataset_ratio 0 \ + --output_dir output \ + --dataloader_num_workers 1 diff --git a/swift/arguments/base_args/model_args.py b/swift/arguments/base_args/model_args.py index cfd89e3bd3..22ec4785f1 100644 --- a/swift/arguments/base_args/model_args.py +++ b/swift/arguments/base_args/model_args.py @@ -69,7 +69,7 @@ class ModelArguments: model_type: Optional[str] = field( default=None, metadata={'help': f'model_type choices: {list(MODEL_MAPPING.keys())}'}) model_revision: Optional[str] = None - task_type: Literal['causal_lm', 'seq_cls', 'embedding', 'reranker', 'generative_reranker'] = None + task_type: Literal['causal_lm', 'seq_cls', 'embedding', 'reranker', 'generative_reranker', 'decision'] = None torch_dtype: Literal['bfloat16', 'float16', 'float32', None] = None # flash_attn: It will automatically convert names based on the model. diff --git a/swift/dataset/__init__.py b/swift/dataset/__init__.py index deb0ccd831..eb0d129a1d 100644 --- a/swift/dataset/__init__.py +++ b/swift/dataset/__init__.py @@ -7,7 +7,7 @@ from .media import MediaResource from .packing import IterablePackingDataset, PackingDataset from .preprocessor import (AlpacaPreprocessor, AnthropicMessagesPreprocessor, AutoPreprocessor, MessagesPreprocessor, - OpenAIMessagesPreprocessor, ResponsePreprocessor, RowPreprocessor) + OpenAIMessagesPreprocessor, ResponsePreprocessor, RowPreprocessor, ScoringPreprocessor) from .register import (DATASET_MAPPING, DatasetMeta, SubsetDataset, get_dataset_list, register_dataset, register_dataset_info) from .utils import (AddLengthPreprocessor, EncodePreprocessor, LazyLLMDataset, get_temporary_cache_files_directory, diff --git a/swift/dataset/preprocessor/__init__.py b/swift/dataset/preprocessor/__init__.py index 643e58abdb..f957174b22 100644 --- a/swift/dataset/preprocessor/__init__.py +++ b/swift/dataset/preprocessor/__init__.py @@ -1,4 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. from .core import (DATASET_TYPE, AlpacaPreprocessor, AnthropicMessagesPreprocessor, AutoPreprocessor, ClsPreprocessor, MessagesPreprocessor, OpenAIMessagesPreprocessor, ResponsePreprocessor, RowPreprocessor) +from .decision import ClefPreprocessor, JevPreprocessor, OmniJevPreprocessor, ScoringPreprocessor from .extra import ClsGenerationPreprocessor, GroundingMixin, TextGenerationPreprocessor diff --git a/swift/dataset/preprocessor/core.py b/swift/dataset/preprocessor/core.py index 565c7d1ad2..faed80ae93 100644 --- a/swift/dataset/preprocessor/core.py +++ b/swift/dataset/preprocessor/core.py @@ -38,6 +38,23 @@ class RowPreprocessor: # Qwen3-TTS 'ref_audios', 'audio_codes', + # `decision` task_type (typed-decision System-1 scoring). These are the + # row-contract fields emitted by the ScoringPreprocessor subclasses + # (see swift/dataset/preprocessor/decision.py) and consumed by the + # decision templates via StdTemplateInputs.extra_kwargs. Kept here so + # remove_useless_columns does not drop them before encoding; a no-op for + # any dataset whose features do not contain them. MUST stay in sync with + # the `_build_row` outputs in decision.py. + 'questions', + 'kinds', + 'options', + 'target_probs', + 'state', + 'question_ids', + 'criteria', + 'omnijev_options', + 'omnijev_column_keys', + 'omnijev_video', ] # Bump whenever the preprocessed-cache schema or semantics change (e.g. #9214 changed # messages[].loss from float64 to bool and added loss_scale). map_cache files are named diff --git a/swift/dataset/preprocessor/decision.py b/swift/dataset/preprocessor/decision.py new file mode 100644 index 0000000000..c9bf25427a --- /dev/null +++ b/swift/dataset/preprocessor/decision.py @@ -0,0 +1,641 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Preprocessor base for the `decision` task_type (typed-decision System-1 scoring models). + +A typed-decision record carries a `state` plus one or more questions; each question has a +`kind in {noul, choice, score}` and its own option set. `ScoringPreprocessor` normalizes a raw +dataset row into swift's `messages` plus the decision fields, which reach the Template through +`StdTemplateInputs.extra_kwargs` (non-standard keys are auto-collected by `from_dict`, so no schema +change is needed). The model template `_encode` renders the actual prompt from these fields and +records the per-question token spans; `target_probs` becomes the training labels. + +Output row contract (see also `swift/model/decision_head.py`): + messages: [{'role': 'user', 'content': state}] a valid message list (from_dict requires one); + the template re-renders the full decision layout from the fields below. + questions: List[str] per-question text + kinds: List[str] per-question 'noul' | 'choice' | 'score' + options: List[List[str]] per-question option texts (noul -> ['false','true'], score -> ['0'..'5']) + target_probs: List[List[float]] per-question gold distribution over its OWN options (training only) + +Subclasses override `parse_record` to turn a model-specific raw row into `(state, questions)`, +where each question is a dict `{'kind', 'question', 'options', 'gold'}` and `gold` is either an +option index (int) or a full distribution (sequence of floats, for distillation targets). +""" +from typing import Any, Dict, List, Optional, Sequence, Tuple + +import json + +from .core import RowPreprocessor + + +class ScoringPreprocessor(RowPreprocessor): + """Base: assemble the decision row contract; subclasses supply `parse_record`.""" + + NOUL_OPTIONS = ['false', 'true'] + SCORE_OPTIONS = ['0', '1', '2', '3', '4', '5'] + + def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]: + state, questions = self.parse_record(row) + if not questions: + return None + return self._build_row(row, state, questions) + + def _build_row(self, row: Dict[str, Any], state: Any, questions: List[Dict[str, Any]]) -> Dict[str, Any]: + """Assemble ONE decision row (the contract in the module docstring) from `(state, questions)`. + + Split out of `preprocess` so a subclass can fan a record out into several rows (JEV renders one + question per prompt) while reusing this row-building logic verbatim. + """ + kinds = [q['kind'] for q in questions] + options = [self._resolve_options(q) for q in questions] + out: Dict[str, Any] = { + 'messages': [{'role': 'user', 'content': state if state is not None else ''}], + 'questions': [q['question'] for q in questions], + 'kinds': kinds, + 'options': options, + } + for key in ('images', 'videos', 'audios'): + if row.get(key) is not None: + out[key] = row[key] + if self._has_gold(questions): + out['target_probs'] = [self.to_target_dist(q['gold'], len(opt)) for q, opt in zip(questions, options)] + return out + + def parse_record(self, row: Dict[str, Any]) -> Tuple[Any, List[Dict[str, Any]]]: + """Return `(state, questions)`; each question is `{'kind','question','options','gold'}`. + + `options` may be omitted for noul/score (defaults are filled by `_resolve_options`); `gold` + may be None at inference. Must be implemented by each model subclass. + """ + raise NotImplementedError + + def _resolve_options(self, question: Dict[str, Any]) -> List[str]: + options = question.get('options') + if options: + return list(options) + default = self.default_options(question['kind']) + if default is None: + raise ValueError(f"A 'choice' question must provide its options: {question}") + return default + + @classmethod + def default_options(cls, kind: str) -> Optional[List[str]]: + if kind == 'noul': + return list(cls.NOUL_OPTIONS) + if kind == 'score': + return list(cls.SCORE_OPTIONS) + return None + + @staticmethod + def _has_gold(questions: Sequence[Dict[str, Any]]) -> bool: + return any(q.get('gold') is not None for q in questions) + + @staticmethod + def to_target_dist(gold: Any, n_options: int) -> List[float]: + """Gold -> a distribution over the question's own `n_options` options. + + An int gold is the correct option index (one-hot); a sequence gold is already a full + distribution (JEV distillation), and must match `n_options`. + """ + if isinstance(gold, (list, tuple)): + probs = [float(x) for x in gold] + if len(probs) != n_options: + raise ValueError(f'target distribution length {len(probs)} != n_options {n_options}') + return probs + idx = int(gold) + if not 0 <= idx < n_options: + raise ValueError(f'gold index {idx} out of range for n_options {n_options}') + dist = [0.0] * n_options + dist[idx] = 1.0 + return dist + + @staticmethod + def _first(source: Dict[str, Any], keys: Sequence[str], default: Any = None) -> Any: + """Return the first non-None value among `keys` (a permissive column lookup shared by the + model subclasses, whose datasets use differing column names).""" + for key in keys: + value = source.get(key) + if value is not None: + return value + return default + + @staticmethod + def _as_option_dict(option: Any) -> Dict[str, Any]: + """Normalize ONE unified option element to a dict. + + The user-facing schema lets an option be either a plain string (sugar for `{'text': s}`) or + an object `{'key'?, 'text'?, 'abstain'?, 'region'?}`. Each subclass reads only the fields it + needs (JEV: text; Clef: key + text; OmniJev: key/text/abstain/region), so all three accept + the SAME option representation. + """ + if isinstance(option, str): + return {'text': option} + if isinstance(option, dict): + return option + raise ValueError(f'an option must be a string or an object, got {option!r}') + + @classmethod + def _option_text(cls, option: Any) -> str: + """The display text of a unified option: its `text`, else its `key`.""" + d = cls._as_option_dict(option) + text = d.get('text') + return str(text if text is not None else d.get('key', '')) + + @classmethod + def _option_key(cls, option: Any, index: int) -> str: + """The answer key of a unified option: its `key`, else its `text`, else `option_{index}`.""" + d = cls._as_option_dict(option) + key = d.get('key') or d.get('text') + return str(key) if key is not None else f'option_{index}' + + +class JevPreprocessor(ScoringPreprocessor): + """JEV distill-corpus preprocessor: fan a record out into ONE row per question. + + JEV's System-1 pass (`serve_decide.py::s1_pass`) scores a single question per forward, so a record + carrying several questions becomes several rows (`RowPreprocessor.batched_preprocess` accepts a + list return, core.py:214-220), each rendering one bare prompt and carrying `counts=[1]` after the + collator. This matches s1_pass exactly (one CompletionRequest per question). + + `parse_record` maps the corpus columns onto the shared question dict `{kind, question, options, + gold}`. `gold` is the teacher's soft distribution over the question's OWN options (JEV + distillation) or an option index; `to_target_dist` validates its length against the option count. + + NOTE(schema): `SargeDev/jev-distill-corpus-v3`'s exact column names could not be verified offline + (HF page/API unreachable in this environment), so the field lookup below is intentionally + permissive (it tries the natural serve_decide names plus common aliases). It MUST be re-checked + against the real dataset before a training run; a `columns={...}` remap can be passed to + `__init__` to rename the corpus columns onto the canonical names used here. + """ + + KIND_KEYS = ('kind', 'type', 'question_type') + QUESTION_KEYS = ('question', 'prompt', 'query') + OPTION_KEYS = ('options', 'opts', 'choices') + STATE_KEYS = ('state', 'context', 'passage', 'text') + GOLD_KEYS = ('target_probs', 'target', 'probs', 'dist', 'distribution', 'gold', 'label', 'answer') + # Optional: a record that already packs a list of question dicts under one column. + QUESTIONS_KEYS = ('questions',) + KIND_ALIASES = { + 'noul': 'noul', + 'yes_no': 'noul', + 'yn': 'noul', + 'bool': 'noul', + 'choice': 'choice', + 'multiple_choice': 'choice', + 'mcq': 'choice', + 'select': 'choice', + 'score': 'score', + 'rating': 'score', + 'ordinal': 'score', + 'likert': 'score', + } + + def preprocess(self, row: Dict[str, Any]) -> Optional[List[Dict[str, Any]]]: + state, questions = self.parse_record(row) + if not questions: + return None + # One row per question (JEV renders a single question per prompt). + return [self._build_row(row, state, [q]) for q in questions] + + def _resolve_options(self, question: Dict[str, Any]) -> List[str]: + """Force the canonical option set for noul/score; only `choice` uses caller-supplied options. + + JEV's System-1 head scores a FIXED per-kind verbalizer: noul always reads the two tokens for + ['false','true'] and score always reads the six digit tokens ['0'..'5'], regardless of any + option text a record carries. `serve_decide.py::decide()` mirrors this -- it forces + `['false','true']` / `[str(i) for i in range(6)]` and ignores caller options for those kinds + (only `choice` consumes them). Rendering custom noul/score text would desync the prompt from + the tokens the head actually scores and diverge from the oracle, so we force canonical here + (model-local override; the shared `ScoringPreprocessor` and Clef/OmniJev are unaffected). + """ + canonical = self.default_options(question['kind']) + if canonical is not None: # noul / score + return canonical + return super()._resolve_options(question) # choice: caller options are required + + def parse_record(self, row: Dict[str, Any]) -> Tuple[Any, List[Dict[str, Any]]]: + state = self._first(row, self.STATE_KEYS, default='') + return state, self._parse_questions(row) + + def _parse_questions(self, row: Dict[str, Any]) -> List[Dict[str, Any]]: + packed = self._first(row, self.QUESTIONS_KEYS, default=None) + if isinstance(packed, (list, tuple)) and packed and isinstance(packed[0], dict): + return [self._normalize_question(q) for q in packed] + kind = self._first(row, self.KIND_KEYS, default=None) + question = self._first(row, self.QUESTION_KEYS, default=None) + if kind is None or question is None: + return [] + return [self._normalize_question(row)] + + def _normalize_question(self, source: Dict[str, Any]) -> Dict[str, Any]: + kind = self._first(source, self.KIND_KEYS, default=None) + question = self._first(source, self.QUESTION_KEYS, default=None) + if kind is None or question is None: + raise ValueError(f'JEV question needs a kind and a question text; got {source!r}') + out: Dict[str, Any] = {'kind': self._normalize_kind(kind), 'question': question} + options = self._first(source, self.OPTION_KEYS, default=None) + if options is not None: + # Unified schema: an option may be a plain string or an object; JEV renders only the + # display text (no abstain/region), so collapse each element to its text. + out['options'] = [self._option_text(o) for o in options] + gold = self._first(source, self.GOLD_KEYS, default=None) + if gold is not None: + out['gold'] = gold + return out + + @classmethod + def _normalize_kind(cls, kind: Any) -> str: + key = str(kind).strip().lower() + if key not in cls.KIND_ALIASES: + raise ValueError(f'unknown JEV question kind {kind!r}; expected one of noul/choice/score (or an alias).') + return cls.KIND_ALIASES[key] + + +def _clef_option_ids(kind: str, criteria: Any) -> List[str]: + """Clef option ids in the exact order `joint_schema_model.py::question_options` emits them. + + MUST stay identical to `ClefTemplate._question_options` (both mirror the official ordering), so the + gold one-hot index computed here lines up with the logit column the head produces: + noul -> ['true','false'], choice -> criteria keys sorted as strings, score -> ['0','1',...]. + """ + if kind == 'noul': + return ['true', 'false'] + if kind == 'choice': + return [str(key) for key, _ in sorted((str(k), v) for k, v in (criteria or {}).items())] + return [str(index) for index in range(len(criteria or []))] + + +class ClefPreprocessor(ScoringPreprocessor): + """Clef joint-schema preprocessor: ONE record -> ONE row carrying ALL its questions. + + Clef decodes a record's questions jointly (decision-models-comparison.md §2: Clef is the only + multi-question-per-forward model), so unlike `JevPreprocessor` this does NOT fan out; the base + `preprocess` builds a single row whose `kinds` / `questions` / `options` / `target_probs` are + parallel lists over the record's questions, in the original `questions` order. + + A Clef record is `{id?, state, questions: {qid: {type, instructions, criteria}}, }` where + `criteria` is a dict {option_id: description} for `choice`, a list of level descriptions for + `score`, and optional for `noul`. `state` may be ANY JSON value (rendered verbatim by the + template), so the raw value is carried in a `state` key and `question_ids` / `criteria` are added + for `ClefTemplate` to rebuild the byte-exact SCHEMA FIELDS layout. + + NOTE(schema): Clef's training data is closed-source (decision-models-comparison.md §5), so the gold + column layout could not be verified offline; `_resolve_gold` is intentionally permissive (accepts + an option id, a bool for noul, an int index, or a full distribution, at question level or in a + record-level `answers` map) and MUST be re-checked against the real dataset before a training run. + """ + + STATE_KEYS = ('state', 'context', 'input') + QUESTIONS_KEYS = ('questions', 'fields', 'schema') + KIND_KEYS = ('type', 'kind', 'question_type') + INSTRUCTION_KEYS = ('instructions', 'instruction', 'question', 'prompt', 'text') + CRITERIA_KEYS = ('criteria', 'options', 'choices', 'levels') + GOLD_KEYS = ('answer', 'gold', 'label', 'target', 'value') + ANSWERS_KEYS = ('answers', 'labels', 'golds', 'targets') + ID_KEYS = ('id', 'record_id') + KIND_ALIASES = JevPreprocessor.KIND_ALIASES + + def parse_record(self, row: Dict[str, Any]) -> Tuple[Any, List[Dict[str, Any]]]: + state = self._first(row, self.STATE_KEYS, default='') + answers = self._first(row, self.ANSWERS_KEYS, default=None) + questions_raw = self._first(row, self.QUESTIONS_KEYS, default=None) + items = self._question_items(questions_raw) + questions: List[Dict[str, Any]] = [] + for qid, q in items: + if not isinstance(q, dict): + raise ValueError(f'Clef question {qid!r} must be a dict of type/instructions/criteria; got {q!r}') + kind = self._normalize_kind(self._first(q, self.KIND_KEYS, default=None)) + criteria = self._normalize_criteria(kind, self._first(q, self.CRITERIA_KEYS, default=None)) + instructions = self._first(q, self.INSTRUCTION_KEYS, default=None) + option_ids = _clef_option_ids(kind, criteria) + gold = self._resolve_gold(q, answers, qid, option_ids) + questions.append({ + 'kind': kind, + 'question': instructions if instructions is not None else str(qid), + 'options': option_ids, + 'question_id': str(qid), + 'criteria': criteria, + }) + if gold is not None: + questions[-1]['gold'] = gold + return state, questions + + def _build_row(self, row: Dict[str, Any], state: Any, questions: List[Dict[str, Any]]) -> Dict[str, Any]: + out = super()._build_row(row, state, questions) + # `state` may be a non-str JSON value; keep the message content a string and carry the raw + # value separately so ClefTemplate can render it byte-exactly (json for non-str). + out['messages'] = [{'role': 'user', 'content': self._state_to_text(state)}] + out['state'] = state + out['question_ids'] = [q['question_id'] for q in questions] + out['criteria'] = [q.get('criteria') for q in questions] + return out + + @staticmethod + def _state_to_text(state: Any) -> str: + if isinstance(state, str): + return state + if state is None: + return '' + return json.dumps(state, ensure_ascii=False, separators=(',', ':'), sort_keys=True) + + @staticmethod + def _question_items(questions_raw: Any) -> List[Tuple[Any, Any]]: + if isinstance(questions_raw, dict): + return list(questions_raw.items()) + if isinstance(questions_raw, (list, tuple)): + return list(enumerate(questions_raw)) + return [] + + @classmethod + def _normalize_criteria(cls, kind: str, raw: Any) -> Any: + """Coerce the unified `options` list into Clef's native `criteria` shape. + + Clef renders byte-exact against the official SCHEMA FIELDS layout, which needs a + `{option_id: description}` map for `choice` (columns sorted by id) and a list of level + descriptions for `score`. A native `criteria` dict/list is passed through; a unified + `options` list of strings / `{key,text}` objects is converted to that shape, so the SAME + user-facing schema works for Clef too. + """ + if raw is None or isinstance(raw, dict): + return raw # native criteria map (choice) or omitted (noul) + if not isinstance(raw, (list, tuple)): + return raw + if kind == 'score': + return [cls._option_text(o) for o in raw] # level descriptions + # choice (noul ignores criteria): {option_id: description}, ids from key/text/index. + return {cls._option_key(o, i): cls._option_text(o) for i, o in enumerate(raw)} + + def _resolve_gold(self, question: Dict[str, Any], answers: Any, qid: Any, + option_ids: List[str]) -> Optional[Any]: + """Resolve the correct answer to an option INDEX (or a full distribution), or None at inference. + + Accepts, at question level then in a record-level `answers` map: a distribution (list/tuple of + floats), a bool (noul), an option id string, or an int index. An unmatched value raises rather + than silently mis-indexing (which would poison the one-hot target). + """ + gold = self._first(question, self.GOLD_KEYS, default=None) + if gold is None and isinstance(answers, dict): + gold = answers.get(str(qid), answers.get(qid)) + if gold is None: + return None + if isinstance(gold, (list, tuple)): + return [float(x) for x in gold] # a full distribution; to_target_dist validates its length + if isinstance(gold, bool): + gold = 'true' if gold else 'false' + if isinstance(gold, int): + return gold + text = str(gold).strip() + for index, option_id in enumerate(option_ids): + if text == option_id or text.lower() == option_id.lower(): + return index + if text.lstrip('-').isdigit(): + return int(text) + raise ValueError(f'Clef gold {gold!r} for question {qid!r} matches no option id {option_ids}.') + + @classmethod + def _normalize_kind(cls, kind: Any) -> str: + key = str(kind).strip().lower() + if key not in cls.KIND_ALIASES: + raise ValueError(f'unknown Clef question kind {kind!r}; expected one of noul/choice/score (or an alias).') + return cls.KIND_ALIASES[key] + + +def _omnijev_render_option(o: Dict[str, Any]) -> str: + """Verbatim port of `mso/records.py::render_option`: how one option block is written into the + prompt. An abstain option renders as 'none of the above'; a region option as its integer box; + otherwise the option text truncated to 200 chars.""" + if o.get('abstain'): + return 'none of the above' + if 'region' in o: + box = o['region']['box'] + b = [int(round(float(v))) for v in box] + return 'region [{},{},{},{}]'.format(*b) + return str(o.get('text', ''))[:200] + + +def _omnijev_options(q: Dict[str, Any]) -> Tuple[List[Any], List[Dict[str, Any]]]: + """Verbatim port of `mso/infer.py::MSO1._options` -> `(keys, opts)`. + + `keys` are the answer keys (noul -> ['yes'], score -> the level labels, choice -> the option + keys); `opts` are the option-BLOCK dicts the template renders with `_omnijev_render_option` + (noul -> ONE block, score -> L blocks, choice -> K blocks, abstain excluded -- it is the head's + extra learned column, not a rendered block). Accepts both the Jev `criteria`-map shape and the + OmniJev `options`-list shape (whose entries may be region options). + """ + kind = q['kind'] + if kind == 'noul': + opts = [{'region': q['region']}] if q.get('region') else [{'text': ''}] + return ['yes'], opts + if kind == 'score': + lv = q.get('levels') or list((q.get('criteria') or {}).keys()) + if not lv and q.get('options') is not None: + # Unified schema: levels given as a plain `options` list (strings or {text/key}). + lv = [o if isinstance(o, str) else (o.get('text') or o.get('key')) for o in q['options']] + return list(lv), [{'text': str(x)} for x in lv] + if q.get('options') is not None: + keys: List[Any] = [] + opts = [] + for i, o in enumerate(q['options']): + if isinstance(o, str): # unified sugar: a plain string is {'text': s} + o = {'text': o} + if o.get('abstain'): + continue + keys.append(o.get('key') or o.get('text') or f'option_{i}') + opts.append(o) + return keys, opts + crit = q.get('criteria') or {} + keys = list(crit.keys()) + return keys, [{'text': k if crit[k] in (None, '') else f'{k}: {crit[k]}'} for k in keys] + + +def _omnijev_column_keys(kind: str, keys: Sequence[Any]) -> List[str]: + """The answer keys of each emitted logit COLUMN, width-matched to `OmniJevHead`'s layout: + noul -> ['yes','no'] (2 cols; col0 == P(yes)); choice -> keys + ['abstain'] (K+1 cols); score -> + the level labels (L cols). MUST stay identical to the head's column layout so the gold one-hot + index computed here lines up with the logit column the head produces.""" + if kind == 'noul': + return ['yes', 'no'] + if kind == 'choice': + return [str(k) for k in keys] + ['abstain'] + return [str(k) for k in keys] + + +class OmniJevPreprocessor(ScoringPreprocessor): + """OmniJev preprocessor: fan a record out into ONE row per question (like `JevPreprocessor`). + + OmniJev scores a single question per forward (`mso/infer.py::ask`/`ask_branch`; the K option + rows of decision G1's branch are expanded at forward time, not here), so a record carrying + several questions becomes several rows. Each row carries the official option-BLOCK structure + (`_omnijev_options`) plus the width-matched `column_keys`, and the gold as a distribution over + the COLUMN width (noul 2 / choice K+1 incl. the abstain column / score L) -- NOT the block count, + because for noul (1 block -> 2 cols) and choice (K blocks -> K+1 cols) they differ. + + The state is multimodal (`{'images': [...], 'video': {...}}` in official `system_one`); the raw + `images` / `videos` are passed through and the panel/mosaic composition (decision H) is applied + by `OmniJevTemplate`, not here. The per-question `instructions` become the message text. + + NOTE(schema): OmniJev's training corpus is not published (only the `system_one` request shape and + the infer CLI's `--questions` map are), so the field lookup below is intentionally permissive and + MUST be re-checked against the real dataset before a training run; a `columns={...}` remap can be + passed to `__init__` to rename corpus columns onto the canonical names used here. + """ + + STATE_KEYS = ('state', 'context', 'input') + QUESTIONS_KEYS = ('questions', 'fields') + KIND_KEYS = ('type', 'kind', 'question_type') + INSTRUCTION_KEYS = ('instructions', 'instruction', 'question', 'prompt', 'text') + LEVELS_KEYS = ('levels',) + CRITERIA_KEYS = ('criteria',) + OPTIONS_KEYS = ('options',) + REGION_KEYS = ('region',) + GOLD_KEYS = ('answer', 'gold', 'label', 'target', 'value') + ANSWERS_KEYS = ('answers', 'labels', 'golds', 'targets') + IMAGES_KEYS = ('images', 'image') + VIDEOS_KEYS = ('videos', 'video') + KIND_ALIASES = JevPreprocessor.KIND_ALIASES + + def preprocess(self, row: Dict[str, Any]) -> Optional[List[Dict[str, Any]]]: + state, questions = self.parse_record(row) + if not questions: + return None + return [self._build_row(row, state, [q]) for q in questions] + + def parse_record(self, row: Dict[str, Any]) -> Tuple[Any, List[Dict[str, Any]]]: + state = self._first(row, self.STATE_KEYS, default=None) + answers = self._first(row, self.ANSWERS_KEYS, default=None) + questions_raw = self._first(row, self.QUESTIONS_KEYS, default=None) + questions: List[Dict[str, Any]] = [] + for qid, q in self._question_items(questions_raw): + if not isinstance(q, dict): + raise ValueError(f'OmniJev question {qid!r} must be a dict of type/instructions/...; got {q!r}') + kind = self._normalize_kind(self._first(q, self.KIND_KEYS, default=None)) + instruction = self._first(q, self.INSTRUCTION_KEYS, default=None) + instruction = str(qid) if instruction is None else instruction + src: Dict[str, Any] = {'kind': kind} + for keys, dst in ((self.LEVELS_KEYS, 'levels'), (self.CRITERIA_KEYS, 'criteria'), + (self.OPTIONS_KEYS, 'options'), (self.REGION_KEYS, 'region')): + value = self._first(q, keys, default=None) + if value is not None: + src[dst] = value + keys, opts = _omnijev_options(src) + column_keys = _omnijev_column_keys(kind, keys) + gold = self._resolve_gold(q, answers, qid, kind, column_keys) + item: Dict[str, Any] = { + 'kind': kind, + 'question': instruction, + 'options': [_omnijev_render_option(o) for o in opts], + 'omnijev_options': opts, + 'column_keys': column_keys, + } + if gold is not None: + item['gold'] = gold + questions.append(item) + return state, questions + + def _build_row(self, row: Dict[str, Any], state: Any, questions: List[Dict[str, Any]]) -> Dict[str, Any]: + q = questions[0] # fan-out: exactly one question per row + column_keys = q['column_keys'] + out: Dict[str, Any] = { + 'messages': [{'role': 'user', 'content': self._content_text(state, q['question'])}], + 'questions': [q['question']], + 'kinds': [q['kind']], + 'options': [q['options']], + 'omnijev_options': [q['omnijev_options']], + 'omnijev_column_keys': [column_keys], + } + images, video = self._resolve_media(row, state) + if images is not None: + out['images'] = images + if video is not None: + # The official `video` DESCRIPTOR dict, NOT a swift video file: route it to extra_kwargs so + # OmniJevTemplate's `video_content` can crop the mosaic back into frames. Putting it in + # `videos` would make swift treat it as a clip list and prepend a spurious `