From 1bc277f99c79864c790d479d1bec0984b329f5e8 Mon Sep 17 00:00:00 2001 From: tinchen777 <59043279+tinchen777@users.noreply.github.com> Date: Sun, 14 Jun 2026 18:04:19 +0800 Subject: [PATCH 01/16] new: ignore whl --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e0254fa5d..36e14e6d6 100644 --- a/.gitignore +++ b/.gitignore @@ -180,4 +180,6 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ -.vscode/ \ No newline at end of file +.vscode/ + +*.whl From ac812804738082d3e31c81d038c66750d4450dc6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 07:35:29 +0000 Subject: [PATCH 02/16] Add usage demos and LoRA finetuning support - Add demos/ with finetune, eval, and LoRA examples plus a Chinese code walkthrough (config system + evaluation metrics). - Add optional, backward-compatible LoRA support to get_model via a peft_args block, plus a Llama-3.2-1B-Instruct-LoRA model config. --- configs/model/Llama-3.2-1B-Instruct-LoRA.yaml | 35 +++++ demos/1_finetune.sh | 37 +++++ demos/2_eval.sh | 35 +++++ demos/3_lora_finetune.sh | 39 ++++++ demos/README.md | 130 ++++++++++++++++++ src/model/__init__.py | 41 ++++++ 6 files changed, 317 insertions(+) create mode 100644 configs/model/Llama-3.2-1B-Instruct-LoRA.yaml create mode 100755 demos/1_finetune.sh create mode 100755 demos/2_eval.sh create mode 100755 demos/3_lora_finetune.sh create mode 100644 demos/README.md diff --git a/configs/model/Llama-3.2-1B-Instruct-LoRA.yaml b/configs/model/Llama-3.2-1B-Instruct-LoRA.yaml new file mode 100644 index 000000000..212df8df7 --- /dev/null +++ b/configs/model/Llama-3.2-1B-Instruct-LoRA.yaml @@ -0,0 +1,35 @@ +# LoRA variant of Llama-3.2-1B-Instruct. +# Identical to the base model config, but adds a `peft_args` block which +# `src/model/__init__.py:get_model` uses to wrap the base model with a LoRA +# adapter (via the `peft` library). Remove/omit `peft_args` to train full weights. +model_args: + pretrained_model_name_or_path: "meta-llama/Llama-3.2-1B-Instruct" + attn_implementation: 'flash_attention_2' + torch_dtype: bfloat16 +tokenizer_args: + pretrained_model_name_or_path: "meta-llama/Llama-3.2-1B-Instruct" +peft_args: + # mirrors peft.LoraConfig + r: 8 + lora_alpha: 32 + lora_dropout: 0.05 + bias: none + task_type: CAUSAL_LM + target_modules: + - q_proj + - k_proj + - v_proj + - o_proj + - gate_proj + - up_proj + - down_proj + # path: null # set to an existing adapter dir to resume / evaluate a LoRA checkpoint +template_args: + apply_chat_template: True + system_prompt: You are a helpful assistant. + system_prompt_with_special_tokens: "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nYou are a helpful assistant.<|eot_id|>" + user_start_tag: "<|start_header_id|>user<|end_header_id|>\n\n" + user_end_tag: "<|eot_id|>" + asst_start_tag: "<|start_header_id|>assistant<|end_header_id|>\n\n" + asst_end_tag: "<|eot_id|>" + date_string: 10 Apr 2025 diff --git a/demos/1_finetune.sh b/demos/1_finetune.sh new file mode 100755 index 000000000..e419103ca --- /dev/null +++ b/demos/1_finetune.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# ============================================================================= +# Demo 1: 直接（全参数）微调一个模型 +# ----------------------------------------------------------------------------- +# 入口: src/train.py (mode=train) +# 通过 Hydra 把以下配置组合在一起: +# experiment=finetune/tofu/default +# -> configs/experiment/finetune/tofu/default.yaml +# - model: Llama-3.2-1B-Instruct +# - trainer: finetune (handler=FinetuneTrainer, configs/trainer/finetune.yaml) +# - data: TOFU_QA_full (locuslab/TOFU 的 "full" split) +# - eval: tofu (训练过程中按 epoch 评测) +# +# 训练完的权重会保存到 paths.output_dir，默认 = saves// +# 即: saves/finetune/demo_finetune_full +# ============================================================================= +set -e +cd "$(dirname "$0")/.." # 切到仓库根目录 + +MODEL=Llama-3.2-1B-Instruct + +python src/train.py --config-name=train.yaml \ + experiment=finetune/tofu/default \ + model=${MODEL} \ + task_name=demo_finetune_full \ + trainer.args.num_train_epochs=5 \ + trainer.args.per_device_train_batch_size=4 \ + trainer.args.gradient_accumulation_steps=8 \ + trainer.args.learning_rate=1e-5 + +# 命令行上任意 key 都能覆盖 yaml，比如: +# trainer.args.num_train_epochs=10 +# data/datasets@data.train=TOFU_QA_retain \ +# data.train.TOFU_QA_retain.args.hf_args.name=retain90 # 改成训练 retain 模型 +# +# 多卡训练把 `python src/train.py` 换成: +# accelerate launch --config_file configs/accelerate/default_config.yaml src/train.py ... diff --git a/demos/2_eval.sh b/demos/2_eval.sh new file mode 100755 index 000000000..a7797ecf6 --- /dev/null +++ b/demos/2_eval.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# ============================================================================= +# Demo 2: 评测一个模型 (TOFU benchmark) +# ----------------------------------------------------------------------------- +# 入口: src/eval.py (mode=eval) +# 1) get_model() 根据 model 配置加载模型+tokenizer +# 2) get_evaluators() 根据 eval=tofu 构造 TOFUEvaluator +# 3) evaluator.evaluate() 逐个跑 configs/eval/tofu.yaml 里 default 列出的指标 +# +# 结果文件 (写入 paths.output_dir = saves/eval/): +# TOFU_EVAL.json 每条样本的细粒度分数 (value_by_index) +# TOFU_SUMMARY.json 每个指标的聚合值 (agg_value) +# +# retain_logs_path: 指向 "retain 参照模型" 的 EVAL.json。 +# forget_quality / privleak 等指标需要它来和参照模型做对比。 +# 需要先 `python setup_data.py --eval` 下载官方参照日志，或自己评一个 retain 模型。 +# ============================================================================= +set -e +cd "$(dirname "$0")/.." + +MODEL=Llama-3.2-1B-Instruct + +python src/eval.py --config-name=eval.yaml \ + experiment=eval/tofu/default \ + model=${MODEL} \ + model.model_args.pretrained_model_name_or_path=open-unlearning/tofu_${MODEL}_full \ + forget_split=forget10 \ + holdout_split=holdout10 \ + retain_logs_path=saves/eval/tofu_${MODEL}_retain90/TOFU_EVAL.json \ + task_name=demo_eval + +# 想评测 Demo 1 自己微调出来的模型, 把上面这行换成本地路径: +# model.model_args.pretrained_model_name_or_path=saves/finetune/demo_finetune_full +# +# 没有 retain 参照日志时, 去掉 retain_logs_path 即可 (forget_quality 会是 None)。 diff --git a/demos/3_lora_finetune.sh b/demos/3_lora_finetune.sh new file mode 100755 index 000000000..d53039066 --- /dev/null +++ b/demos/3_lora_finetune.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# ============================================================================= +# Demo 3: 用 LoRA 微调模型 +# ----------------------------------------------------------------------------- +# 注意: 原框架默认不支持 LoRA。本 demo 配套做了一处最小、非破坏性的扩展: +# - src/model/__init__.py: get_model() 在加载完基座模型后, 若 model 配置里有 +# `peft_args` 块, 就调用 get_peft_lora_model() 用 peft 包套一层 LoRA adapter。 +# - configs/model/Llama-3.2-1B-Instruct-LoRA.yaml: 在原模型配置上加了 peft_args。 +# +# 前置依赖: pip install peft +# +# 训练流程其余部分和 Demo 1 完全一样 (同样走 FinetuneTrainer)，只是把 +# model 换成带 peft_args 的 LoRA 配置。HF Trainer 会自动只保存 adapter 权重。 +# 输出: saves/finetune/demo_lora_finetune +# ============================================================================= +set -e +cd "$(dirname "$0")/.." + +python src/train.py --config-name=train.yaml \ + experiment=finetune/tofu/default \ + model=Llama-3.2-1B-Instruct-LoRA \ + task_name=demo_lora_finetune \ + trainer.args.num_train_epochs=5 \ + trainer.args.per_device_train_batch_size=4 \ + trainer.args.gradient_accumulation_steps=8 \ + trainer.args.learning_rate=1e-4 # LoRA 通常用比全参微调更大的学习率 + +# 也可以完全在命令行里临时指定 LoRA 超参 (无需改 yaml), 例如: +# model=Llama-3.2-1B-Instruct \ +# +model.peft_args.r=16 +model.peft_args.lora_alpha=32 \ +# +model.peft_args.task_type=CAUSAL_LM \ +# '+model.peft_args.target_modules=[q_proj,v_proj]' +# +# ---- 评测训练好的 LoRA adapter ---- +# 把 base 模型路径 + adapter 路径 (peft_args.path) 一起传给 eval: +# python src/eval.py experiment=eval/tofu/default \ +# model=Llama-3.2-1B-Instruct-LoRA \ +# +model.peft_args.path=saves/finetune/demo_lora_finetune \ +# task_name=demo_lora_eval diff --git a/demos/README.md b/demos/README.md new file mode 100644 index 000000000..f5feaf7a1 --- /dev/null +++ b/demos/README.md @@ -0,0 +1,130 @@ +# OpenUnlearning / TOFU 代码导读 + 使用 Demo + +本目录是对该仓库（TOFU 官方维护的新代码库 `open-unlearning`）的学习笔记和可运行示例。 + +## 一、配置参数是如何传递的？ —— Hydra 分层组合 + +整个项目用 [Hydra](https://hydra.cc) 管理配置，**完全不靠手写 argparse**。两个入口 +`src/train.py` / `src/eval.py` 都用 `@hydra.main` 装饰，启动时把一棵 `DictConfig` +配置树注入 `main(cfg)`。 + +### 1. 配置组合（defaults 列表） +顶层配置（如 `configs/train.yaml`）通过 `defaults:` 列表把各组件拼起来： + +```yaml +# configs/train.yaml +defaults: + - model: Llama-3.2-3B-Instruct # configs/model/*.yaml + - trainer: finetune # configs/trainer/*.yaml + - data: finetune # configs/data/*.yaml + - collator: DataCollatorForSupervisedDataset + - eval: tofu # configs/eval/*.yaml + - paths: default + - experiment: null # 可选实验包 +``` + +每个组件是 `configs/<组>/<名>.yaml`。这样模型、训练器、数据、评测彼此解耦。 + +### 2. experiment 包：一次性覆盖一整组默认值 +`configs/experiment/finetune/tofu/default.yaml` 顶部用 `# @package _global_` ++ `override`，把上面的默认值整组替换并设好实验超参： + +```yaml +# @package _global_ +defaults: + - override /model: Llama-3.2-1B-Instruct + - override /trainer: finetune + - override /data/datasets@data.train: TOFU_QA_full + - override /eval: tofu +trainer: + args: {learning_rate: 1e-5, num_train_epochs: 5} +``` + +### 3. 命令行覆盖（最常用） +任意叶子参数都能在命令行点号覆盖；加号 `+` 表示新增不存在的 key： + +```bash +python src/train.py experiment=finetune/tofu/default \ + model=Llama-3.2-1B-Instruct \ + trainer.args.num_train_epochs=10 \ + task_name=my_run \ + +model.peft_args.r=16 # + 表示新增字段 +``` + +### 4. 变量插值与 @package +- 插值 `${...}`：如 `eval.tofu.forget_split: ${forget_split}`，多处共享同一值。 +- `# @package eval.tofu.metrics.forget_quality`（写在指标 yaml 第一行）会把该文件 + 内容塞进配置树的指定位置 —— 这就是“在 `eval/tofu.yaml` 里 import 一个指标名， + 它的配置就自动出现在 `metrics` 下”的原理。 + +### 5. 配置 → 代码：handler 注册表模式 +配置里大量出现 `handler: XXX`。代码用「注册表」把字符串映射到具体类/函数： + +| 组件 | 注册表 | 加载函数 | +|------|--------|----------| +| 模型 | `MODEL_REGISTRY` | `src/model/__init__.py: get_model` | +| 训练器 | `TRAINER_REGISTRY` | `src/trainer/__init__.py: load_trainer` | +| 评测器 | `EVALUATOR_REGISTRY` | `src/evals/__init__.py: get_evaluators` | +| 指标 | `METRICS_REGISTRY` | `src/evals/metrics/__init__.py: get_metrics` | + +例如 `trainer.handler=GradDiff` → 找到 `GradDiff` 类并用 `trainer.args` / +`trainer.method_args` 实例化。新增方法只要写好类 + `_register_*` 即可。 + +--- + +## 二、实现了哪些评测指标？对应哪段代码 + +评测主流程：`src/evals/base.py: Evaluator.evaluate()` 遍历 `metrics`，每个指标是一个 +被 `@unlearning_metric` 装饰（`src/evals/metrics/base.py`）的函数。每个指标 yaml 用 +`defaults:` 声明它需要的 **数据集 / collator / 预计算指标 / 参照日志**，框架在 +`base.py: prepare_kwargs_evaluate_metric` 里自动准备好再调用函数。 + +| 指标 | 含义 | 实现函数 | 文件 | +|------|------|----------|------| +| **Verbatim Probability** (`forget/retain_Q_A_Prob`) | 标准答案的归一化条件概率 | `probability`, `probability_w_options` | `src/evals/metrics/memorization.py` | +| **Verbatim ROUGE** (`*_Q_A_ROUGE`) | 生成文本与标准答案的 ROUGE-L recall | `rouge` | `memorization.py` | +| **Truth Ratio** (`forget/retain_Truth_Ratio`) | 正确答案 vs 扰动错误答案的似然比 | `truth_ratio` | `memorization.py` | +| **Exact Memorization (EM)** | 教师强制下 argmax 预测命中目标 token 比例 | `exact_memorization` | `memorization.py` | +| **Extraction Strength (ES)** | 需要多少前缀才能逐字续写出剩余答案 | `extraction_strength` | `memorization.py` | +| **Forget Quality** | forget 模型 vs retain 模型 truth-ratio 分布的 KS 检验 p 值 | `ks_test` | `src/evals/metrics/privacy.py` | +| **Model Utility** | 多个 retain/real-author/world-fact 指标的调和平均 | `hm_aggregate` | `src/evals/metrics/utility.py` | +| **PrivLeak** | forget vs retain 的 MIA AUC 相对差 | `privleak`, `rel_diff` | `privacy.py` | +| **Classifier Prob / Gibberish** | 用分类器判断生成是否乱码 | `classifier_prob` | `utility.py` | +| **6 种 MIA 攻击** | LOSS / ZLib / Reference / GradNorm / MinK / MinK++ | `mia_*` | `src/evals/metrics/mia/*.py` | +| **lm-eval-harness** | MMLU/GSM8K 等通用基准 | `LMEvalEvaluator` | `src/evals/lm_eval.py` | + +底层公共算子（概率、生成相似度、批量推理）在 `src/evals/metrics/utils.py`： +`evaluate_probability` / `eval_text_similarity` / `run_batchwise_evals` / +`tokenwise_vocab_logprobs`。 + +TOFU 默认启用哪些指标，见 `configs/eval/tofu.yaml` 的 `defaults` 列表（其余被注释， +取消注释即可启用）。 + +**预计算依赖示例**：`model_utility` 的 yaml 里用 +`.@pre_compute.retain_Q_A_Prob: retain_Q_A_Prob` 声明它依赖 9 个子指标； +`forget_quality` 声明它依赖 `forget_Truth_Ratio`（pre_compute）和 retain 模型日志 +（`reference_logs` → `retain_logs_path`）。这正是评测时必须传 `retain_logs_path` 的原因。 + +--- + +## 三、三个使用 Demo + +| 脚本 | 作用 | +|------|------| +| `1_finetune.sh` | 全参数微调一个 TOFU 目标模型 | +| `2_eval.sh` | 在 TOFU benchmark 上评测一个模型 | +| `3_lora_finetune.sh` | 用 LoRA 微调（含对框架的最小扩展） | + +### 关于 LoRA 的说明 +原框架**默认不支持 LoRA**（见 `docs/components.md`）。本 demo 做了一处最小、向后兼容 +的扩展： + +1. `src/model/__init__.py` 的 `get_model()` 在加载基座模型后，若 model 配置含 + `peft_args`，则调用新增的 `get_peft_lora_model()` 用 `peft` 包套 LoRA adapter； + `peft_args.path` 可指向已有 adapter 用于续训/评测。 +2. 新增 `configs/model/Llama-3.2-1B-Instruct-LoRA.yaml`（在基座配置上加 `peft_args`）。 + +需先 `pip install peft`。没有 `peft_args` 的旧配置完全不受影响。 + +> 运行前请确认已 `pip install ".[lm-eval]"` 并 `python setup_data.py --eval` +> 下载评测所需的 retain 参照日志。 diff --git a/src/model/__init__.py b/src/model/__init__.py index 05add68be..f4a87e265 100644 --- a/src/model/__init__.py +++ b/src/model/__init__.py @@ -61,10 +61,51 @@ def get_model(model_cfg: DictConfig): raise ValueError( f"Error {e} while fetching model using {model_handler}.from_pretrained()." ) + # Optional: wrap the loaded base model with a PEFT/LoRA adapter. + # This is additive and only triggers when a `peft_args` block is present in + # the model config, so existing (non-LoRA) configs are unaffected. + peft_args = model_cfg.get("peft_args", None) + if peft_args is not None: + model = get_peft_lora_model(model, peft_args) + tokenizer = get_tokenizer(tokenizer_args) return model, tokenizer +def get_peft_lora_model(model, peft_args: DictConfig): + """Wrap a base causal LM with a LoRA adapter using the `peft` library. + + Args: + model: a freshly loaded base model (e.g. AutoModelForCausalLM). + peft_args (DictConfig): LoRA hyper-parameters. Recognised keys mirror + `peft.LoraConfig` (r, lora_alpha, lora_dropout, target_modules, + bias, task_type, ...). An optional `path` key can point to an + existing adapter checkpoint to resume/evaluate instead of creating + a fresh adapter. + """ + try: + from peft import LoraConfig, get_peft_model, PeftModel + except ImportError as e: + raise ImportError( + "LoRA finetuning requires the `peft` library. Install it with " + "`pip install peft`." + ) from e + + with open_dict(peft_args): + adapter_path = peft_args.pop("path", None) + + if adapter_path is not None: + # Load a previously trained LoRA adapter on top of the base model. + model = PeftModel.from_pretrained(model, adapter_path, is_trainable=True) + logger.info(f"Loaded existing LoRA adapter from {adapter_path}") + else: + lora_config = LoraConfig(**peft_args) + model = get_peft_model(model, lora_config) + logger.info("Created a new LoRA adapter on top of the base model.") + model.print_trainable_parameters() + return model + + def _add_or_replace_eos_token(tokenizer, eos_token: str) -> None: is_added = tokenizer.eos_token_id is None num_added_tokens = tokenizer.add_special_tokens({"eos_token": eos_token}) From 637821066ed45a54986fc9ec2f7724a16dd4b4f8 Mon Sep 17 00:00:00 2001 From: tinchen777 <59043279+tinchen777@users.noreply.github.com> Date: Mon, 15 Jun 2026 16:27:27 +0800 Subject: [PATCH 03/16] new: add ignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 36e14e6d6..52a2ddda8 100644 --- a/.gitignore +++ b/.gitignore @@ -181,5 +181,6 @@ cython_debug/ .idea/ .vscode/ +.history/ *.whl From fca38eb512b1e63ba5f0167d14fc47fa2eb8a3e2 Mon Sep 17 00:00:00 2001 From: tinchen777 <59043279+tinchen777@users.noreply.github.com> Date: Mon, 15 Jun 2026 22:03:55 +0800 Subject: [PATCH 04/16] new: change LLama-3.2-1B model --- configs/experiment/examples/tofu_eval.yaml | 2 +- configs/model/Llama-3.2-1B-Instruct-LoRA.yaml | 4 ++-- configs/model/Llama-3.2-1B-Instruct.yaml | 4 ++-- configs/unlearn.yaml | 2 +- demos/1_finetune.sh | 4 ++++ 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/configs/experiment/examples/tofu_eval.yaml b/configs/experiment/examples/tofu_eval.yaml index 0100d7921..7b2b0ae03 100644 --- a/configs/experiment/examples/tofu_eval.yaml +++ b/configs/experiment/examples/tofu_eval.yaml @@ -5,7 +5,7 @@ model: attn_implementation: flash_attention_2 torch_dtype: bfloat16 tokenizer_args: - pretrained_model_name_or_path: meta-llama/Llama-3.2-1B-Instruct + pretrained_model_name_or_path: unsloth/Llama-3.2-1B-Instruct template_args: apply_chat_template: true system_prompt: You are a helpful assistant. diff --git a/configs/model/Llama-3.2-1B-Instruct-LoRA.yaml b/configs/model/Llama-3.2-1B-Instruct-LoRA.yaml index 212df8df7..ad9a7a6df 100644 --- a/configs/model/Llama-3.2-1B-Instruct-LoRA.yaml +++ b/configs/model/Llama-3.2-1B-Instruct-LoRA.yaml @@ -3,11 +3,11 @@ # `src/model/__init__.py:get_model` uses to wrap the base model with a LoRA # adapter (via the `peft` library). Remove/omit `peft_args` to train full weights. model_args: - pretrained_model_name_or_path: "meta-llama/Llama-3.2-1B-Instruct" + pretrained_model_name_or_path: "unsloth/Llama-3.2-1B-Instruct" attn_implementation: 'flash_attention_2' torch_dtype: bfloat16 tokenizer_args: - pretrained_model_name_or_path: "meta-llama/Llama-3.2-1B-Instruct" + pretrained_model_name_or_path: "unsloth/Llama-3.2-1B-Instruct" peft_args: # mirrors peft.LoraConfig r: 8 diff --git a/configs/model/Llama-3.2-1B-Instruct.yaml b/configs/model/Llama-3.2-1B-Instruct.yaml index 1f14feaea..b77006136 100644 --- a/configs/model/Llama-3.2-1B-Instruct.yaml +++ b/configs/model/Llama-3.2-1B-Instruct.yaml @@ -1,9 +1,9 @@ model_args: - pretrained_model_name_or_path: "meta-llama/Llama-3.2-1B-Instruct" + pretrained_model_name_or_path: "unsloth/Llama-3.2-1B-Instruct" attn_implementation: 'flash_attention_2' torch_dtype: bfloat16 tokenizer_args: - pretrained_model_name_or_path: "meta-llama/Llama-3.2-1B-Instruct" + pretrained_model_name_or_path: "unsloth/Llama-3.2-1B-Instruct" template_args: apply_chat_template: True system_prompt: You are a helpful assistant. diff --git a/configs/unlearn.yaml b/configs/unlearn.yaml index 0ebde9770..e35e44a10 100644 --- a/configs/unlearn.yaml +++ b/configs/unlearn.yaml @@ -10,7 +10,7 @@ defaults: - _self_ trainer: - args: + args: remove_unused_columns: False mode: unlearn diff --git a/demos/1_finetune.sh b/demos/1_finetune.sh index e419103ca..990f0b54b 100755 --- a/demos/1_finetune.sh +++ b/demos/1_finetune.sh @@ -35,3 +35,7 @@ python src/train.py --config-name=train.yaml \ # # 多卡训练把 `python src/train.py` 换成: # accelerate launch --config_file configs/accelerate/default_config.yaml src/train.py ... + + +python src/train.py --config-name=unlearn.yaml experiment=unlearn/tofu/default \ + forget_split=forget10 retain_split=retain90 trainer=GradAscent task_name=SAMPLE_UNLEARN \ No newline at end of file From 94f9b2be1c5356d8e5a7a59e87a777c2d4eca5c9 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 14:21:38 +0000 Subject: [PATCH 05/16] Add unlearning demos and a custom unlearning trainer example - demos/4_unlearn.sh: run an existing method (GradDiff). - demos/5_custom_unlearn.sh + BoundedGradDiff trainer (class, registration, config) as a worked example of adding a custom unlearning algorithm. - Expand demos/README.md: unlearning flow, how to add your own method and how it gets called, config structure walkthrough, and eval output notes. --- configs/trainer/BoundedGradDiff.yaml | 9 ++++ demos/4_unlearn.sh | 40 +++++++++++++++ demos/5_custom_unlearn.sh | 36 +++++++++++++ demos/README.md | 64 ++++++++++++++++++++++++ src/trainer/__init__.py | 4 ++ src/trainer/unlearn/bounded_grad_diff.py | 56 +++++++++++++++++++++ 6 files changed, 209 insertions(+) create mode 100644 configs/trainer/BoundedGradDiff.yaml create mode 100755 demos/4_unlearn.sh create mode 100755 demos/5_custom_unlearn.sh create mode 100644 src/trainer/unlearn/bounded_grad_diff.py diff --git a/configs/trainer/BoundedGradDiff.yaml b/configs/trainer/BoundedGradDiff.yaml new file mode 100644 index 000000000..bc0602dff --- /dev/null +++ b/configs/trainer/BoundedGradDiff.yaml @@ -0,0 +1,9 @@ +defaults: + - finetune # 继承 configs/trainer/finetune.yaml 里的 HuggingFace TrainingArguments + +handler: BoundedGradDiff # 必须等于 src/trainer/unlearn/bounded_grad_diff.py 里的类名 +method_args: # 这些会作为 **kwargs 传给 BoundedGradDiff.__init__ + gamma: 1.0 # forget 项权重 (继承自 GradDiff) + alpha: 1.0 # retain 项权重 (继承自 GradDiff) + retain_loss_type: NLL # NLL 或 KL (继承自 GradDiff) + forget_loss_bound: 4.0 # 本算法新增: forget NLL 的上界 tau diff --git a/demos/4_unlearn.sh b/demos/4_unlearn.sh new file mode 100755 index 000000000..75d7918af --- /dev/null +++ b/demos/4_unlearn.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# ============================================================================= +# Demo 4: 用一个【已有的】遗忘方法做 unlearning (这里用 GradDiff) +# ----------------------------------------------------------------------------- +# 入口: src/train.py (mode=unlearn) +# experiment=unlearn/tofu/default -> configs/experiment/unlearn/tofu/default.yaml +# - model: 待遗忘的目标模型 (默认 open-unlearning/tofu_Llama-3.2-1B-Instruct_full) +# - data: unlearn -> 同时加载 forget 和 retain 两个数据集 +# collator 把每个 batch 组织成 {"forget": {...}, "retain": {...}} +# - trainer: 由命令行 trainer=GradDiff 指定 (handler=GradDiff) +# - eval: tofu (遗忘过程中/结束后评测) +# +# 遗忘方法的核心只有一个函数: trainer 的 compute_loss(model, inputs)。 +# - GradAscent: loss = -forget_loss +# - GradDiff: loss = gamma*(-forget_loss) + alpha*retain_loss +# - NPO/SimNPO/DPO/RMU/...: 各自不同的 compute_loss +# +# 输出: saves/unlearn/demo_unlearn_graddiff +# ============================================================================= +set -e +cd "$(dirname "$0")/.." + +MODEL=Llama-3.2-1B-Instruct + +python src/train.py --config-name=unlearn.yaml \ + experiment=unlearn/tofu/default \ + model=${MODEL} \ + trainer=GradDiff \ + trainer.method_args.gamma=1.0 \ + trainer.method_args.alpha=1.0 \ + trainer.method_args.retain_loss_type=NLL \ + forget_split=forget10 \ + retain_split=retain90 \ + holdout_split=holdout10 \ + retain_logs_path=saves/eval/tofu_${MODEL}_retain90/TOFU_EVAL.json \ + task_name=demo_unlearn_graddiff + +# 换方法只需改 trainer= : GradAscent / NPO / SimNPO / DPO / RMU / UNDIAL / WGA / CEU ... +# 对应方法的额外超参在 trainer.method_args.* 下覆盖, 例如 NPO: +# trainer=NPO trainer.method_args.beta=0.1 trainer.method_args.gamma=1.0 diff --git a/demos/5_custom_unlearn.sh b/demos/5_custom_unlearn.sh new file mode 100755 index 000000000..3cc9cd5ca --- /dev/null +++ b/demos/5_custom_unlearn.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# ============================================================================= +# Demo 5: 运行【自定义】遗忘算法 BoundedGradDiff +# ----------------------------------------------------------------------------- +# 把自己的算法接入本项目, 只需 3 步 (本 demo 已帮你做好): +# 1. 写 trainer 类: src/trainer/unlearn/bounded_grad_diff.py +# - 继承 GradDiff (或更底层的 UnlearnTrainer) +# - 只重写 compute_loss(model, inputs, ...) +# 2. 注册: src/trainer/__init__.py 里 _register_trainer(BoundedGradDiff) +# 3. 写配置: configs/trainer/BoundedGradDiff.yaml (handler + method_args) +# +# 之后用法和内置方法完全一样, 只是 trainer=BoundedGradDiff: +# 输出: saves/unlearn/demo_custom_unlearn +# ============================================================================= +set -e +cd "$(dirname "$0")/.." + +MODEL=Llama-3.2-1B-Instruct + +python src/train.py --config-name=unlearn.yaml \ + experiment=unlearn/tofu/default \ + model=${MODEL} \ + trainer=BoundedGradDiff \ + trainer.method_args.forget_loss_bound=4.0 \ + trainer.method_args.gamma=1.0 \ + trainer.method_args.alpha=1.0 \ + forget_split=forget10 \ + retain_split=retain90 \ + holdout_split=holdout10 \ + retain_logs_path=saves/eval/tofu_${MODEL}_retain90/TOFU_EVAL.json \ + task_name=demo_custom_unlearn + +# 调用链 (谁调用了你的 compute_loss): +# src/train.py -> load_trainer() 按 handler 从 TRAINER_REGISTRY 取出 BoundedGradDiff, +# 用 trainer.args(=TrainingArguments) 和 **method_args 实例化 -> trainer.train() +# -> HuggingFace Trainer 训练循环每个 step 调用 compute_loss(model, inputs)。 diff --git a/demos/README.md b/demos/README.md index f5feaf7a1..d9ac6d50b 100644 --- a/demos/README.md +++ b/demos/README.md @@ -114,6 +114,70 @@ TOFU 默认启用哪些指标，见 `configs/eval/tofu.yaml` 的 `defaults` 列 | `1_finetune.sh` | 全参数微调一个 TOFU 目标模型 | | `2_eval.sh` | 在 TOFU benchmark 上评测一个模型 | | `3_lora_finetune.sh` | 用 LoRA 微调（含对框架的最小扩展） | +| `4_unlearn.sh` | 用**已有**遗忘方法（GradDiff）做 unlearning | +| `5_custom_unlearn.sh` | 运行**自定义**遗忘算法 BoundedGradDiff | + +### 遗忘（unlearning）流程要点 +- 入口同样是 `src/train.py`（`mode=unlearn`），但数据用 `data=unlearn`，会同时加载 + `forget` + `retain` 两个数据集；collator 把每个 batch 组织成 + `{"forget": {...}, "retain": {...}}`（见 `src/data/unlearn.py`）。 +- 一个遗忘方法的本质 = 一个 trainer 类的 `compute_loss(model, inputs)`： + - GradAscent：`loss = -forget_loss` + - GradDiff：`loss = γ·(-forget_loss) + α·retain_loss` + - NPO/SimNPO/DPO/RMU…各自不同。 + +### 如何加入你自己的遗忘算法（官方推荐做法） +`docs/contributing.md` 明确**鼓励**贡献自有方法。标准三步（本仓库已用 +`BoundedGradDiff` 做了完整示例）： + +1. **实现 trainer 类**（`src/trainer/unlearn/bounded_grad_diff.py`）： + - 继承一个已有基类。继承 `GradDiff` 可白拿 `compute_retain_loss` / + `gamma` / `alpha` / `ref_model`；继承更底层的 `UnlearnTrainer` 则更自由。 + - **唯一必须实现的函数是 `compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None)`**。 + 需要额外超参就再写 `__init__`（额外参数来自配置的 `method_args`）。 + 其余（训练循环、保存、评测、deepspeed）都由父类 `UnlearnTrainer` / + HF `Trainer` 处理，一般无需改动。 +2. **注册**（`src/trainer/__init__.py`）：`_register_trainer(BoundedGradDiff)`。 +3. **写配置**（`configs/trainer/BoundedGradDiff.yaml`）：`handler` 写类名， + `method_args` 写你的超参。 + +**调用链**：`src/train.py` → `load_trainer()` 按 `handler` 从 `TRAINER_REGISTRY` +取出类，用 `trainer.args`(TrainingArguments) + `**method_args` 实例化 → +`trainer.train()` → HF Trainer 的训练循环每个 step 调用你的 `compute_loss`。 + +> 进一步（可选）：在 `community/methods/<你的方法>/` 放一个 `README.md` + `run.sh` +> 记录超参和复现命令，并把结果填到 `community/leaderboard.md`。 + +--- + +## 四、关于你的几个问题 + +**1. experiment 模块是不是"配好的配置清单"？不用它就得自己配每一部分吗？** +是的。`configs/experiment/**` 就是一份**预设好的整套实验清单**：它用 +`# @package _global_` + `override` 一次性把 model/trainer/data/eval 等都换成某次实验 +该用的值，并填好超参与变量插值（如 `forget_split` 联动到各处）。不用 experiment 也行， +但你就得在命令行/顶层 yaml 里**逐个**指定 `model=… trainer=… data=… eval=…` 以及所有 +联动参数，很繁琐。所以惯例：**用 experiment 打底，再用命令行覆盖个别参数**。 + +**2. config 的整体结构（每部分配什么）** + +| 配置组 | 路径 | 配的是什么 | +|--------|------|-----------| +| 顶层入口 | `configs/{train,unlearn,eval}.yaml` | `defaults:` 列表，决定加载哪些组件；`mode`、`task_name`、`seed` | +| model | `configs/model/*.yaml` | `model_args`(HF from_pretrained 参数、路径、dtype、attn)、`tokenizer_args`、`template_args`(对话模板)、(本仓库新增)`peft_args` | +| trainer | `configs/trainer/*.yaml` | `handler`(方法类名)、`args`(HF `TrainingArguments`：lr/epochs/batch/优化器/保存/eval 策略…)、`method_args`(方法自有超参) | +| data | `configs/data/*.yaml` + `data/datasets/*` | 用哪些数据集（finetune 只有 train；unlearn 有 forget+retain）、数据集 `handler`、`hf_args`(path/name/split)、question/answer_key、max_length | +| collator | `configs/collator/*.yaml` | 批处理拼接逻辑、padding 方式 | +| eval | `configs/eval/*.yaml` + `eval/_metrics/*` | `handler`(评测器)、启用哪些 `metrics`、各指标的数据集/`pre_compute`/参照日志、`forget/holdout_split`、`retain_logs_path`、`batch_size` | +| paths | `configs/paths/default.yaml` | 输出目录等路径 | +| hydra | `configs/hydra/*.yaml` | Hydra 运行时（日志、输出目录命名） | +| experiment | `configs/experiment/**` | 上面各组的**整套覆盖清单**，代表一次具体实验 | + +**3. eval 跑完会有图表吗？** +**不会。** 评测只产出两个 JSON：`_EVAL.json`（逐样本细分）和 +`_SUMMARY.json`（每个指标的聚合值），没有任何 matplotlib/绘图代码。 +要图的话：训练侧 `trainer.args.report_to=tensorboard`（默认）会写 TensorBoard 曲线， +可 `tensorboard --logdir saves/`；想要评测柱状图/对比图需自己读 SUMMARY.json 画。 ### 关于 LoRA 的说明 原框架**默认不支持 LoRA**（见 `docs/components.md`）。本 demo 做了一处最小、向后兼容 diff --git a/src/trainer/__init__.py b/src/trainer/__init__.py index 447b2d2dc..8fdf828d6 100644 --- a/src/trainer/__init__.py +++ b/src/trainer/__init__.py @@ -15,6 +15,7 @@ from trainer.unlearn.satimp import SatImp from trainer.unlearn.wga import WGA from trainer.unlearn.pdu import PDU +from trainer.unlearn.bounded_grad_diff import BoundedGradDiff import logging @@ -99,3 +100,6 @@ def load_trainer( _register_trainer(SatImp) _register_trainer(WGA) _register_trainer(PDU) + +# Register custom example unlearning trainer (see demos/) +_register_trainer(BoundedGradDiff) diff --git a/src/trainer/unlearn/bounded_grad_diff.py b/src/trainer/unlearn/bounded_grad_diff.py new file mode 100644 index 000000000..7d2c97942 --- /dev/null +++ b/src/trainer/unlearn/bounded_grad_diff.py @@ -0,0 +1,56 @@ +""" +一个【自定义遗忘算法】的示例: BoundedGradDiff (有界梯度差分)。 + +动机: 原版 GradDiff 的 forget 项是 `-forget_loss` (梯度上升)，会无上限地把 forget +样本的 loss 推高，常导致模型整体崩坏。这里给 forget 的 NLL 加一个上界 tau: +一旦模型对某个 forget 样本已经"足够不确定"(NLL >= tau)，该项梯度归零，从而在遗忘 +和保持模型可用性之间取得更稳的平衡。 + +这个例子展示了把自有算法接入本项目的最小写法: + 1. 继承一个已有的遗忘基类 (这里继承 GradDiff，白拿 retain 损失 / gamma / alpha / + ref_model 等); + 2. 只需重写 `compute_loss` 实现你自己的 loss; + 3. 在 src/trainer/__init__.py 里 `_register_trainer(BoundedGradDiff)` 注册; + 4. 加一个 configs/trainer/BoundedGradDiff.yaml 配置 (handler + method_args)。 + +也可以直接继承更底层的 `UnlearnTrainer` (见 base.py)，那样只需实现 `compute_loss`， +但 retain 损失等逻辑得自己写。 +""" + +import torch +from trainer.unlearn.grad_diff import GradDiff + + +class BoundedGradDiff(GradDiff): + def __init__(self, forget_loss_bound=4.0, *args, **kwargs): + # method_args 里的参数会作为关键字参数传进来 (见 load_trainer) + super().__init__(*args, **kwargs) + self.forget_loss_bound = forget_loss_bound # tau: forget NLL 的上界 + + def compute_loss( + self, model, inputs, return_outputs=False, num_items_in_batch=None + ): + # 1) forget 分支: collator 会把每个 batch 组织成 {"forget": {...}, "retain": {...}} + forget_inputs = inputs["forget"] + forget_inputs = { + "input_ids": forget_inputs["input_ids"], + "attention_mask": forget_inputs["attention_mask"], + "labels": forget_inputs["labels"], + } + forget_outputs = model(**forget_inputs) + # 有界梯度上升: NLL 一旦超过 tau，clamp 后梯度为 0，避免无限制崩坏 + bounded_nll = torch.clamp(forget_outputs.loss, max=self.forget_loss_bound) + forget_loss = -bounded_nll + + # 2) retain 分支: 直接复用父类 GradDiff 的实现 (支持 NLL 或 KL) + retain_inputs = inputs["retain"] + retain_inputs = { + "input_ids": retain_inputs["input_ids"], + "attention_mask": retain_inputs["attention_mask"], + "labels": retain_inputs["labels"], + } + retain_loss = self.compute_retain_loss(model=model, retain_inputs=retain_inputs) + + # 3) 合并: gamma / alpha 同样从父类继承 + loss = self.gamma * forget_loss + self.alpha * retain_loss + return (loss, forget_outputs) if return_outputs else loss From 75b4113c178998b03c6b993630d7ce48ac08b703 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 15:32:05 +0000 Subject: [PATCH 06/16] Fix bf16 numpy crash in eval and respect HF_HOME hub layout - evals/metrics/utils.py: cast losses/probs to float() before .numpy(), numpy has no bfloat16 so bf16 models crashed evaluate_probability (TypeError: Got unsupported ScalarType BFloat16). - model/__init__.py: stop passing cache_dir=$HF_HOME to from_pretrained so transformers uses the conventional $HF_HOME/hub location, matching where datasets are cached. --- src/evals/metrics/utils.py | 7 +++++-- src/model/__init__.py | 11 ++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/evals/metrics/utils.py b/src/evals/metrics/utils.py index 71684b7de..b7f7db47b 100644 --- a/src/evals/metrics/utils.py +++ b/src/evals/metrics/utils.py @@ -95,8 +95,11 @@ def evaluate_probability(model, batch): avg_losses = losses / num_token_gt normalized_probs = torch.exp(-avg_losses) - avg_losses = avg_losses.cpu().numpy().tolist() - normalized_probs = normalized_probs.cpu().numpy().tolist() + # .float() is required: numpy has no bfloat16, so converting a bf16 tensor + # (when the model runs in bfloat16) directly via .numpy() raises + # "TypeError: Got unsupported ScalarType BFloat16". + avg_losses = avg_losses.float().cpu().numpy().tolist() + normalized_probs = normalized_probs.float().cpu().numpy().tolist() return [ {"prob": prob, "avg_loss": avg_loss} for prob, avg_loss in zip(normalized_probs, avg_losses) diff --git a/src/model/__init__.py b/src/model/__init__.py index f4a87e265..822997e6d 100644 --- a/src/model/__init__.py +++ b/src/model/__init__.py @@ -1,13 +1,10 @@ from transformers import AutoModelForCausalLM, AutoTokenizer from omegaconf import DictConfig, open_dict from typing import Dict, Any -import os import torch import logging from model.probe import ProbedLlamaForCausalLM -hf_home = os.getenv("HF_HOME", default=None) - logger = logging.getLogger(__name__) MODEL_REGISTRY: Dict[str, Any] = {} @@ -50,11 +47,15 @@ def get_model(model_cfg: DictConfig): with open_dict(model_args): model_path = model_args.pop("pretrained_model_name_or_path", None) try: + # Note: we deliberately do NOT pass cache_dir here. Transformers resolves + # the cache from the HF_HOME / HF_HUB_CACHE env vars automatically, placing + # models under $HF_HOME/hub (the HF convention). Passing cache_dir=$HF_HOME + # would instead drop models directly under $HF_HOME (no /hub subdir) and + # diverge from where `datasets` caches data. model = model_cls.from_pretrained( pretrained_model_name_or_path=model_path, torch_dtype=torch_dtype, **model_args, - cache_dir=hf_home, ) except Exception as e: logger.warning(f"Model {model_path} requested with {model_cfg.model_args}") @@ -121,7 +122,7 @@ def _add_or_replace_eos_token(tokenizer, eos_token: str) -> None: def get_tokenizer(tokenizer_cfg: DictConfig): try: - tokenizer = AutoTokenizer.from_pretrained(**tokenizer_cfg, cache_dir=hf_home) + tokenizer = AutoTokenizer.from_pretrained(**tokenizer_cfg) except Exception as e: error_message = ( f"{'--' * 40}\n" From 0ab6b8cc4afdee606b942ecdf77b8556517795ce Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 15:36:38 +0000 Subject: [PATCH 07/16] docs: add runtime/troubleshooting section to demos README Covers prerequisites (setup_data), device selection (CPU/GPU, flash-attn caveats), HF_HOME cache layout, the two fixed bf16-related bugs, and task_name notes. --- demos/README.md | 57 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/demos/README.md b/demos/README.md index d9ac6d50b..6c03bb705 100644 --- a/demos/README.md +++ b/demos/README.md @@ -192,3 +192,60 @@ TOFU 默认启用哪些指标，见 `configs/eval/tofu.yaml` 的 `defaults` 列 > 运行前请确认已 `pip install ".[lm-eval]"` 并 `python setup_data.py --eval` > 下载评测所需的 retain 参照日志。 + +--- + +## 五、运行环境 / 常见坑（实跑前必看） + +### 1. 前置依赖 +```bash +pip install ".[lm-eval]" # 安装框架 + 评测依赖 +pip install peft # 仅 LoRA demo 需要 +python setup_data.py --eval # 下载 retain 参照日志到 saves/eval/ +``` +- `setup_data.py --eval` 下载的是各 retain 模型的评测日志（`saves/eval/.../TOFU_EVAL.json`）。 + `forget_quality` / `privleak` 这类指标要拿它和参照模型对比。**没有它不会崩**，但这些 + 指标会是 `None`。不需要时把命令里的 `retain_logs_path=...` 整行删掉即可。 + +### 2. 指定 device（CPU / 某号 GPU） +训练和评测的控制方式不同： + +| 场景 | 怎么指定 | +|------|----------| +| **评测** `src/eval.py` | 模型加载用 `model.model_args.device_map`（`eval.yaml` 默认 `cuda`）。可 `... model.model_args.device_map=cuda:0` 或 `device_map=cpu` | +| **训练** `src/train.py` | 配置里**没有** device_map，由 HF `Trainer`+`accelerate` 放卡。用环境变量控制 | + +```bash +CUDA_VISIBLE_DEVICES=2 python src/train.py ... # 单卡(推荐用环境变量锁物理卡) +CUDA_VISIBLE_DEVICES=0,1 accelerate launch \ + --config_file configs/accelerate/default_config.yaml src/train.py ... # 多卡 +python src/train.py ... trainer.args.use_cpu=true # CPU 训练(慢, 仅调试用) +``` + +> ⚠️ **CPU / 非 GPU 必看**：模型配置默认 `attn_implementation: flash_attention_2` +> + `torch_dtype: bfloat16`，flash-attn **只能在 GPU 跑**。CPU 时必须同时改： +> `model.model_args.attn_implementation=eager model.model_args.torch_dtype=float32`。 +> 另外训练时模型先在 CPU 初始化再搬到 GPU，会打印一句 "attempting to use Flash +> Attention 2.0 with a model not initialized on GPU" —— **这是无害警告**，可忽略； +> 想消掉就用 `attn_implementation=sdpa`（性能几乎一致且不挑设备）。 + +### 3. 缓存路径 HF_HOME +- 设 **`HF_HOME`** 一个变量即可，模型和数据集都会用它：模型 → `$HF_HOME/hub`， + 数据集 → `$HF_HOME/datasets`（HF 标准布局）。 +- 数据集是直接从 HF Hub 拉（`load_dataset("locuslab/TOFU", ...)`），**不需要**手动下到 + `./data/`；`configs/paths/default.yaml` 里的 `data_dir` 实际未被加载代码使用。 +- 首次运行会下载模型+数据，属正常；之后命中缓存复用。 + +> 注：早期版本 `src/model/__init__.py` 误把 `cache_dir=$HF_HOME` 传给 `from_pretrained`， +> 导致模型落在 `$HF_HOME/models--...` 而非 `$HF_HOME/hub/`。本仓库已修复，现在符合 HF 约定。 + +### 4. 已修复的两个框架 bug（bf16 相关） +本仓库分支已修复，跑前 `git pull` 即可： +- **`TypeError: Got unsupported ScalarType BFloat16`**：bf16 模型评测时 + `evaluate_probability` 直接 `.numpy()` 崩溃（numpy 无 bfloat16）。已在 + `src/evals/metrics/utils.py` 改为 `.float().cpu().numpy()`。 +- **HF_HOME 缓存层级**：见上条第 3 点。 + +### 5. task_name 注意 +`task_name` 仅决定输出目录 `saves///`，不影响算法逻辑。 +**不同实验务必用不同 `task_name`**，否则会写进同一目录互相覆盖。 From 54b69406c4edfee01577b1db65ac8db629cffd55 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 17:11:15 +0000 Subject: [PATCH 08/16] docs: correct setup_data flag to --eval_logs in demos README --- demos/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/demos/README.md b/demos/README.md index 6c03bb705..21ec50034 100644 --- a/demos/README.md +++ b/demos/README.md @@ -190,7 +190,7 @@ TOFU 默认启用哪些指标，见 `configs/eval/tofu.yaml` 的 `defaults` 列 需先 `pip install peft`。没有 `peft_args` 的旧配置完全不受影响。 -> 运行前请确认已 `pip install ".[lm-eval]"` 并 `python setup_data.py --eval` +> 运行前请确认已 `pip install ".[lm-eval]"` 并 `python setup_data.py --eval_logs` > 下载评测所需的 retain 参照日志。 --- @@ -201,9 +201,9 @@ TOFU 默认启用哪些指标，见 `configs/eval/tofu.yaml` 的 `defaults` 列 ```bash pip install ".[lm-eval]" # 安装框架 + 评测依赖 pip install peft # 仅 LoRA demo 需要 -python setup_data.py --eval # 下载 retain 参照日志到 saves/eval/ +python setup_data.py --eval_logs # 下载 retain 参照日志到 saves/eval/ ``` -- `setup_data.py --eval` 下载的是各 retain 模型的评测日志（`saves/eval/.../TOFU_EVAL.json`）。 +- `setup_data.py --eval_logs` 下载的是各 retain 模型的评测日志（`saves/eval/.../TOFU_EVAL.json`）。 `forget_quality` / `privleak` 这类指标要拿它和参照模型对比。**没有它不会崩**，但这些 指标会是 `None`。不需要时把命令里的 `retain_logs_path=...` 整行删掉即可。 From c383c61e968e4b5c097c6d3542cc7bd423d5dac7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 17:41:43 +0000 Subject: [PATCH 09/16] demos: pin a single GPU to avoid DataParallel OOM on shared clusters Plain 'python src/train.py' lets HF Trainer use nn.DataParallel across all visible GPUs; on a busy node that OOMs when replicating to an occupied card. Scripts now default CUDA_VISIBLE_DEVICES to a single card (overridable), and the README documents the failure mode. --- demos/1_finetune.sh | 5 +++++ demos/2_eval.sh | 4 ++++ demos/3_lora_finetune.sh | 5 +++++ demos/4_unlearn.sh | 5 +++++ demos/5_custom_unlearn.sh | 5 +++++ demos/README.md | 9 +++++++++ 6 files changed, 33 insertions(+) diff --git a/demos/1_finetune.sh b/demos/1_finetune.sh index 990f0b54b..84f05e3bb 100755 --- a/demos/1_finetune.sh +++ b/demos/1_finetune.sh @@ -17,6 +17,11 @@ set -e cd "$(dirname "$0")/.." # 切到仓库根目录 +# 共享集群必看: 只暴露一张【空闲】GPU, 否则 HF Trainer 会在所有可见卡上启用 +# DataParallel, 往被别人占满的卡复制模型而 CUDA OOM。先 `nvidia-smi` 选空闲卡, +# 或运行时 `CUDA_VISIBLE_DEVICES=3 bash demos/1_finetune.sh` 覆盖。 +export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0} + MODEL=Llama-3.2-1B-Instruct python src/train.py --config-name=train.yaml \ diff --git a/demos/2_eval.sh b/demos/2_eval.sh index a7797ecf6..b491d042d 100755 --- a/demos/2_eval.sh +++ b/demos/2_eval.sh @@ -18,6 +18,10 @@ set -e cd "$(dirname "$0")/.." +# 共享集群必看: 只暴露一张【空闲】GPU 评测。先 `nvidia-smi` 选空闲卡, +# 或运行时 `CUDA_VISIBLE_DEVICES=3 bash demos/2_eval.sh` 覆盖。 +export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0} + MODEL=Llama-3.2-1B-Instruct python src/eval.py --config-name=eval.yaml \ diff --git a/demos/3_lora_finetune.sh b/demos/3_lora_finetune.sh index d53039066..4c4fb4e08 100755 --- a/demos/3_lora_finetune.sh +++ b/demos/3_lora_finetune.sh @@ -16,6 +16,11 @@ set -e cd "$(dirname "$0")/.." +# 共享集群必看: 只暴露一张【空闲】GPU, 否则 HF Trainer 会在所有可见卡上启用 +# DataParallel, 往被别人占满的卡复制模型而 CUDA OOM。先 `nvidia-smi` 选空闲卡, +# 或运行时 `CUDA_VISIBLE_DEVICES=3 bash demos/3_lora_finetune.sh` 覆盖。 +export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0} + python src/train.py --config-name=train.yaml \ experiment=finetune/tofu/default \ model=Llama-3.2-1B-Instruct-LoRA \ diff --git a/demos/4_unlearn.sh b/demos/4_unlearn.sh index 75d7918af..8c0bc7575 100755 --- a/demos/4_unlearn.sh +++ b/demos/4_unlearn.sh @@ -20,6 +20,11 @@ set -e cd "$(dirname "$0")/.." +# 共享集群必看: 只暴露一张【空闲】GPU, 否则 HF Trainer 会在所有可见卡上启用 +# DataParallel, 往被别人占满的卡复制模型而 CUDA OOM。先 `nvidia-smi` 选一张空闲卡, +# 改下面的默认 0, 或运行时 `CUDA_VISIBLE_DEVICES=3 bash demos/4_unlearn.sh` 覆盖。 +export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0} + MODEL=Llama-3.2-1B-Instruct python src/train.py --config-name=unlearn.yaml \ diff --git a/demos/5_custom_unlearn.sh b/demos/5_custom_unlearn.sh index 3cc9cd5ca..fcdfeff66 100755 --- a/demos/5_custom_unlearn.sh +++ b/demos/5_custom_unlearn.sh @@ -15,6 +15,11 @@ set -e cd "$(dirname "$0")/.." +# 共享集群必看: 只暴露一张【空闲】GPU, 否则 HF Trainer 会在所有可见卡上启用 +# DataParallel, 往被别人占满的卡复制模型而 CUDA OOM。先 `nvidia-smi` 选一张空闲卡, +# 改下面的默认 0, 或运行时 `CUDA_VISIBLE_DEVICES=3 bash demos/5_custom_unlearn.sh` 覆盖。 +export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0} + MODEL=Llama-3.2-1B-Instruct python src/train.py --config-name=unlearn.yaml \ diff --git a/demos/README.md b/demos/README.md index 21ec50034..2071703b5 100644 --- a/demos/README.md +++ b/demos/README.md @@ -229,6 +229,15 @@ python src/train.py ... trainer.args.use_cpu=true # CPU 训练(慢, > Attention 2.0 with a model not initialized on GPU" —— **这是无害警告**，可忽略； > 想消掉就用 `attn_implementation=sdpa`（性能几乎一致且不挑设备）。 +> 🚨 **共享集群上的 DataParallel OOM**：用 `python src/train.py`（非 accelerate）启动时， +> 若**没设 `CUDA_VISIBLE_DEVICES`**，HF Trainer 会把所有可见 GPU 都拿来做 +> `nn.DataParallel`——只要其中一张被别人占满，复制模型时就 `CUDA out of memory` +> （报错栈里会出现 `torch/nn/parallel/_functions.py ... gather along dimension 0`）。 +> **务必先 `nvidia-smi` 选一张空闲卡并锁定单卡**：`CUDA_VISIBLE_DEVICES= bash demos/4_unlearn.sh`。 +> demos 里的训练/评测脚本已内置 `export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0}`， +> 默认锁 0 号卡，可在外部覆盖。注意：eval（step 0）能跑通、训练第一步才 OOM，正是 +> 因为评测单卡执行、训练才触发多卡复制。 + ### 3. 缓存路径 HF_HOME - 设 **`HF_HOME`** 一个变量即可，模型和数据集都会用它：模型 → `$HF_HOME/hub`， 数据集 → `$HF_HOME/datasets`（HF 标准布局）。 From cb3f9c34f3659033343f4ee04900dad04377a37a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 15:25:11 +0000 Subject: [PATCH 10/16] fix: apply_chat_template returns BatchEncoding on new transformers transformers flipped apply_chat_template's return_dict default to True, so tokenize=True now yields a BatchEncoding instead of list[int], breaking 'chat_ids += [eos]' with TypeError. Pass return_dict=False explicitly (backward compatible; old default was already False). --- src/data/utils.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/data/utils.py b/src/data/utils.py index 4a5df348b..433aca56f 100644 --- a/src/data/utils.py +++ b/src/data/utils.py @@ -59,14 +59,22 @@ def preprocess_chat_instance( date_str = template_config.get("date_string", None) date_info = {"date_string": date_str} if date_str is not None else {} chat_ids = tokenizer.apply_chat_template( - chat, tokenize=True, add_generation_prompt=False, **date_info + chat, + tokenize=True, + add_generation_prompt=False, + return_dict=False, + **date_info, ) # all except last response are in-context examples wrapped_prompt = tokenizer.apply_chat_template( chat[:-1], tokenize=False, add_generation_prompt=True, **date_info ) prompt_ids = tokenizer.apply_chat_template( - chat[:-1], tokenize=True, add_generation_prompt=True, **date_info + chat[:-1], + tokenize=True, + add_generation_prompt=True, + return_dict=False, + **date_info, ) else: wrapped_prompt = "" From 76741636187cf7d67e2a0acbb3f7e6d9f3f2bbbf Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 15:30:25 +0000 Subject: [PATCH 11/16] compat: adapt to transformers 5.x deprecations - from_pretrained: pass dtype= instead of deprecated torch_dtype= (model loading and MIA reference model). Requires transformers>=4.56. - Remove deprecated TrainingArguments.logging_dir (removed in v5.2); TensorBoard now defaults under output_dir/runs. Note: the YAML 'torch_dtype' key is the project's own config consumed by get_dtype() and is intentionally left unchanged. --- configs/experiment/examples/muse_unlearn.yaml | 2 +- configs/trainer/finetune.yaml | 3 ++- src/evals/metrics/mia/__init__.py | 2 +- src/model/__init__.py | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/configs/experiment/examples/muse_unlearn.yaml b/configs/experiment/examples/muse_unlearn.yaml index 0e6b2b6c3..1e6f3e46a 100644 --- a/configs/experiment/examples/muse_unlearn.yaml +++ b/configs/experiment/examples/muse_unlearn.yaml @@ -27,7 +27,7 @@ trainer: bf16_full_eval: true logging_steps: 5 output_dir: ${paths.output_dir} - logging_dir: ${trainer.args.output_dir}/logs + # logging_dir removed: deprecated in transformers (removed in v5.2). report_to: tensorboard ddp_find_unused_parameters: None gradient_checkpointing: false diff --git a/configs/trainer/finetune.yaml b/configs/trainer/finetune.yaml index eea5c42e6..c42490cdf 100644 --- a/configs/trainer/finetune.yaml +++ b/configs/trainer/finetune.yaml @@ -8,7 +8,8 @@ args: bf16_full_eval: True logging_steps: 5 output_dir: ${paths.output_dir} - logging_dir: ${trainer.args.output_dir}/logs + # logging_dir removed: deprecated in transformers (removed in v5.2). TensorBoard + # logs now default under ${output_dir}/runs. Set env TENSORBOARD_LOGGING_DIR to override. report_to: tensorboard ddp_find_unused_parameters: None gradient_checkpointing: False diff --git a/src/evals/metrics/mia/__init__.py b/src/evals/metrics/mia/__init__.py index 5ab869f6d..8ac8f3eb7 100644 --- a/src/evals/metrics/mia/__init__.py +++ b/src/evals/metrics/mia/__init__.py @@ -87,7 +87,7 @@ def mia_reference(model, **kwargs): logger.info(f"Loading reference model from {kwargs['reference_model_path']}") reference_model = AutoModelForCausalLM.from_pretrained( kwargs["reference_model_path"], - torch_dtype=model.dtype, + dtype=model.dtype, # transformers>=4.56 renamed `torch_dtype` -> `dtype` device_map={"": model.device}, ) return mia_auc( diff --git a/src/model/__init__.py b/src/model/__init__.py index 822997e6d..977dbd142 100644 --- a/src/model/__init__.py +++ b/src/model/__init__.py @@ -54,7 +54,7 @@ def get_model(model_cfg: DictConfig): # diverge from where `datasets` caches data. model = model_cls.from_pretrained( pretrained_model_name_or_path=model_path, - torch_dtype=torch_dtype, + dtype=torch_dtype, # transformers>=4.56 renamed `torch_dtype` -> `dtype` **model_args, ) except Exception as e: From 8bad8a3c8df7807cb49782509be49876747a0bcf Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 15:48:52 +0000 Subject: [PATCH 12/16] logging: add rich job_logging config and make it the default Hydra owns logging via logging.config.dictConfig (job_logging group), which overrode any user-set RichHandler. Add configs/hydra/job_logging/rich.yaml (rich.logging.RichHandler + file handler) and select it in hydra/default.yaml. Requires 'pip install rich'; revert per-run with hydra/job_logging=colorlog. --- configs/hydra/default.yaml | 3 ++- configs/hydra/job_logging/rich.yaml | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 configs/hydra/job_logging/rich.yaml diff --git a/configs/hydra/default.yaml b/configs/hydra/default.yaml index 2c52ad15f..96a368406 100644 --- a/configs/hydra/default.yaml +++ b/configs/hydra/default.yaml @@ -3,7 +3,8 @@ # enable color logging defaults: - override hydra_logging: colorlog - - override job_logging: colorlog + - override job_logging: rich # rich.logging.RichHandler (requires `pip install rich`). + # To revert to the old style: change to `colorlog`, or per-run `hydra/job_logging=colorlog`. # output directory, generated dynamically on each run run: diff --git a/configs/hydra/job_logging/rich.yaml b/configs/hydra/job_logging/rich.yaml new file mode 100644 index 000000000..94103eded --- /dev/null +++ b/configs/hydra/job_logging/rich.yaml @@ -0,0 +1,26 @@ +# Hydra job_logging config that routes logs through rich.logging.RichHandler. +# Select it via the defaults list (override job_logging: rich) in +# configs/hydra/default.yaml, or per-run on the CLI: `hydra/job_logging=rich`. +# Requires: pip install rich +version: 1 +formatters: + rich: + format: "%(name)s - %(message)s" + datefmt: "[%X]" +handlers: + rich: + class: rich.logging.RichHandler + formatter: rich + rich_tracebacks: true + show_time: true + show_level: true + show_path: false + markup: false # safer: don't parse [..] in log text (URLs/brackets) as rich markup + file: + class: logging.FileHandler + formatter: rich + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log +root: + level: INFO + handlers: [rich, file] +disable_existing_loggers: false From f3253d5e8256e9c62ecdcbfc6815b057ae713b67 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 15:52:22 +0000 Subject: [PATCH 13/16] logging: show logger name in rich format as [%(name)s] --- configs/hydra/job_logging/rich.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configs/hydra/job_logging/rich.yaml b/configs/hydra/job_logging/rich.yaml index 94103eded..afc813b8a 100644 --- a/configs/hydra/job_logging/rich.yaml +++ b/configs/hydra/job_logging/rich.yaml @@ -5,7 +5,9 @@ version: 1 formatters: rich: - format: "%(name)s - %(message)s" + # %(name)s = logger name (e.g. evaluator / metrics / __main__). RichHandler + # renders time+level itself; the logger name must be put in this format string. + format: "[%(name)s] %(message)s" datefmt: "[%X]" handlers: rich: From ec6991bbfb02814045e166c4b94152502993617e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 15:59:47 +0000 Subject: [PATCH 14/16] logging: give logger name its own color via custom Rich formatter Add src/log_utils.py:RichNameFormatter which wraps the logger name in a Rich style tag (default 'bold cyan') and escapes the message so brackets in log text aren't parsed as markup. rich.yaml uses it for the console handler (markup=true) and a separate plain formatter for the file handler. --- configs/hydra/job_logging/rich.yaml | 15 +++++++++------ src/log_utils.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 6 deletions(-) create mode 100644 src/log_utils.py diff --git a/configs/hydra/job_logging/rich.yaml b/configs/hydra/job_logging/rich.yaml index afc813b8a..32b030e5b 100644 --- a/configs/hydra/job_logging/rich.yaml +++ b/configs/hydra/job_logging/rich.yaml @@ -5,10 +5,13 @@ version: 1 formatters: rich: - # %(name)s = logger name (e.g. evaluator / metrics / __main__). RichHandler - # renders time+level itself; the logger name must be put in this format string. - format: "[%(name)s] %(message)s" - datefmt: "[%X]" + # Custom formatter (src/log_utils.py): colors the logger name and escapes + # the message so brackets in log text aren't parsed as Rich markup. + (): log_utils.RichNameFormatter + name_style: "bold cyan" # change to any Rich style, e.g. "magenta", "bold green" + plain: + # Plain text for the log file (no color markup written to disk). + format: "[%(asctime)s][%(name)s][%(levelname)s] - %(message)s" handlers: rich: class: rich.logging.RichHandler @@ -17,10 +20,10 @@ handlers: show_time: true show_level: true show_path: false - markup: false # safer: don't parse [..] in log text (URLs/brackets) as rich markup + markup: true # required so the name's [bold cyan]..[/] tags are colored file: class: logging.FileHandler - formatter: rich + formatter: plain filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log root: level: INFO diff --git a/src/log_utils.py b/src/log_utils.py new file mode 100644 index 000000000..5758fefd3 --- /dev/null +++ b/src/log_utils.py @@ -0,0 +1,28 @@ +import logging + +from rich.markup import escape + + +class RichNameFormatter(logging.Formatter): + """Formatter for use with RichHandler(markup=True). + + Wraps the logger name in a Rich style tag so it gets its own color, while + escaping the actual log message so any brackets in it (URLs, list reprs like + ``[q_proj, v_proj]``) are NOT parsed as Rich markup. + + Configured from Hydra's job_logging dictConfig via the ``()`` key, e.g.: + + formatters: + rich: + (): log_utils.RichNameFormatter + name_style: "bold cyan" + """ + + def __init__(self, name_style: str = "bold cyan", **kwargs): + super().__init__(**kwargs) + self.name_style = name_style + + def format(self, record: logging.LogRecord) -> str: + message = escape(record.getMessage()) + name = f"[{self.name_style}]{escape(record.name)}[/] " + return name + message From ca1d8fd0ff572e1ee913c35e7216d095f910814c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 16:08:42 +0000 Subject: [PATCH 15/16] logging: add rich config for hydra_logging group too The rich option only existed in the job_logging group, so selecting hydra_logging=rich failed with 'Could not find hydra/hydra_logging/rich'. Add configs/hydra/hydra_logging/rich.yaml and set both groups to rich in hydra/default.yaml. --- configs/hydra/default.yaml | 9 +++++---- configs/hydra/hydra_logging/rich.yaml | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 configs/hydra/hydra_logging/rich.yaml diff --git a/configs/hydra/default.yaml b/configs/hydra/default.yaml index 96a368406..8eb8b7630 100644 --- a/configs/hydra/default.yaml +++ b/configs/hydra/default.yaml @@ -1,10 +1,11 @@ # https://hydra.cc/docs/configure_hydra/intro/ -# enable color logging +# enable rich logging (requires `pip install rich`) defaults: - - override hydra_logging: colorlog - - override job_logging: rich # rich.logging.RichHandler (requires `pip install rich`). - # To revert to the old style: change to `colorlog`, or per-run `hydra/job_logging=colorlog`. + - override hydra_logging: rich # Hydra's own framework logs + - override job_logging: rich # your application logs (the ones you usually care about) + # To revert to the old style: change these to `colorlog`, or per-run e.g. + # `hydra/job_logging=colorlog hydra/hydra_logging=colorlog`. # output directory, generated dynamically on each run run: diff --git a/configs/hydra/hydra_logging/rich.yaml b/configs/hydra/hydra_logging/rich.yaml new file mode 100644 index 000000000..809329694 --- /dev/null +++ b/configs/hydra/hydra_logging/rich.yaml @@ -0,0 +1,21 @@ +# Hydra's *own* framework logs (the hydra_logging group), routed through Rich. +# Pairs with configs/hydra/job_logging/rich.yaml (your application logs). +# Requires: pip install rich +version: 1 +formatters: + rich: + (): log_utils.RichNameFormatter + name_style: "dim cyan" # dim, to visually separate Hydra's own logs from job logs +handlers: + rich: + class: rich.logging.RichHandler + formatter: rich + rich_tracebacks: true + show_time: true + show_level: true + show_path: false + markup: true +root: + level: INFO + handlers: [rich] +disable_existing_loggers: false From 21bad313d8b568bc1b0ed3ca6c73f6aa30df8797 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 16:22:41 +0000 Subject: [PATCH 16/16] docs: add Chinese Hydra guide with priority/compose mermaid diagram Covers config groups, defaults list and _self_ ordering, package directives, relocation syntax, override, interpolation, parameter precedence, CLI operators, an end-to-end example, a mermaid priority-chain/compose-flow diagram, and a common-errors cheat sheet. --- docs/hydra_cn.md | 258 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 258 insertions(+) create mode 100644 docs/hydra_cn.md diff --git a/docs/hydra_cn.md b/docs/hydra_cn.md new file mode 100644 index 000000000..ec3a4ec41 --- /dev/null +++ b/docs/hydra_cn.md @@ -0,0 +1,258 @@ +# Hydra 配置系统使用指南（中文） + +本文档讲解 [Hydra](https://hydra.cc) 在本项目（OpenUnlearning / TOFU）中的用法： +配置文件语法、参数优先级、命令行调用方式，并配一张「优先级链 + 合成流程」图。 + +--- + +## 0. Hydra 在本项目的角色 + +入口 `src/train.py` / `src/eval.py` 都用 `@hydra.main(config_path="../configs", config_name=...)` +装饰。Hydra 启动时做三件事，把整棵配置树 `cfg` 注入 `main(cfg)`： + +1. **Compose（组合）**：按 `defaults:` 列表把分散在 `configs/<组>/<名>.yaml` 的组件拼成一棵树； +2. **Override（覆盖）**：应用命令行参数； +3. **Interpolate（插值）**：解析 `${...}` 引用。 + +之后代码用「`handler` 字符串 → 注册表类」把配置变成对象 +（`MODEL_REGISTRY` / `TRAINER_REGISTRY` / `EVALUATOR_REGISTRY` / `METRICS_REGISTRY`）。 + +--- + +## 1. 配置目录 = config groups（配置组） + +`configs/` 下每个**子目录**是一个 **config group**，目录里每个 `.yaml` 是该组的一个**可选项**： + +``` +configs/ + train.yaml / unlearn.yaml / eval.yaml # 顶层入口配置 + model/ # 组 "model"：Llama-3.2-1B-Instruct / Qwen2.5-7B-Instruct / ... + trainer/ # 组 "trainer"：finetune / GradDiff / NPO / DPO / PDU / ... + data/ # 组 "data" + eval/ # 组 "eval"：tofu / muse / lm_eval + collator/ paths/ hydra/ experiment/ ... +``` + +「选一个选项」= 命令行 `model=Llama-3.2-1B-Instruct`，或在 `defaults` 里写 `- model: Llama-3.2-1B-Instruct`。 + +--- + +## 2. `defaults:` 列表 —— 组合的核心 + +以 `configs/unlearn.yaml` 为例： + +```yaml +defaults: + - model: Llama-3.2-3B-Instruct # 加载 configs/model/Llama-3.2-3B-Instruct.yaml -> cfg.model + - trainer: GradAscent + - data: unlearn + - collator: DataCollatorForSupervisedDataset + - eval: tofu + - hydra: default + - paths: default + - experiment: null # 占位：默认不选实验，可被命令行 experiment=... 填上 + - _self_ # 本文件自身正文在合成顺序中的位置 +``` + +要点： + +- 每个 `- 组: 选项` 把对应 yaml 合并进 `cfg.<组>`。 +- **`null`** = 该组默认不选。 +- **`_self_`** = **本文件自身正文**（`defaults:` 以外的键）在合成顺序里的**插入点**。 +- **合并是深合并（deep merge）**：后加载的配置只覆盖/新增重叠的叶子键，不会抹掉兄弟键。 + +### `_self_` 的位置很关键（项目里两种写法） + +- `unlearn.yaml`：`_self_` 在**最后** → 本文件正文最后合并，能盖过前面所有组（含 experiment）。 +- `train.yaml` / `eval.yaml`：`_self_` 在**最前** → 本文件正文先合并，会被后面的组覆盖。 + +> 规则：**defaults 从上到下依次合并，靠后覆盖靠前；`_self_` 表示"自己"在这条流水线里的插入点。** + +--- + +## 3. `# @package` 指令 —— 决定"内容放到树的哪个位置" + +写在文件**第一行**（不是注释）。默认按"被引入的路径"推导 package，`# @package` 显式指定目标。 + +### (a) `# @package _global_`（实验文件用） + +`configs/experiment/unlearn/tofu/default.yaml`： + +```yaml +# @package _global_ +defaults: + - override /model: Llama-3.2-1B-Instruct +forget_split: forget10 # 因为 _global_，落在 cfg 根，而非 cfg.experiment.* +``` + +`_global_` = 内容放到**配置树根**，所以实验文件能直接改 `cfg.model.*` / `cfg.trainer.*` / `cfg.forget_split` +—— 这就是 experiment 能当"整套实验清单"的原因。 + +### (b) `# @package eval.tofu.metrics.<名>`（指标文件用） + +`configs/eval/tofu_metrics/forget_quality.yaml` 第一行是 +`# @package eval.tofu.metrics.forget_quality`，把内容重定向到 `cfg.eval.tofu.metrics.forget_quality`， +对齐代码读取的 `eval.tofu.metrics`（`src/evals/base.py`）。这就是"在 defaults 里 import 一个指标名， +它的配置自动出现在 metrics 下"的原理。 + +### (c) defaults 里的 `@` —— 引入时重定位 + +`configs/eval/tofu_metrics/forget_Q_A_Prob.yaml`： + +```yaml +defaults: + - ../../data/datasets@datasets: TOFU_QA_forget # 加载该数据集，放到本配置的 datasets 子键 + - ../../collator@collators: DataCollatorForSupervisedDatasetwithIndex +``` + +`组路径@目标package: 选项` = 加载某组选项并放到 `@` 后指定位置。`.@` 里的 `.` 表示**当前目录组**： + +```yaml +# forget_quality.yaml：把当前目录(tofu_metrics)的 forget_Truth_Ratio 挂到 pre_compute 下 +defaults: + - .@pre_compute.forget_truth_ratio: forget_Truth_Ratio +``` + +命令行同样可用，例如换 forget 数据集：`data/datasets@data.forget=TOFU_QA_forget_idk`。 + +--- + +## 4. `- override /group: name`（实验文件里的覆盖） + +在 experiment（`# @package _global_`）里改默认组选择，用 `override` + **绝对组路径**（前导 `/`）： + +```yaml +defaults: + - override /model: Llama-3.2-1B-Instruct + - override /trainer: DPO + - override /data/datasets@data.forget: TOFU_QA_forget_idk +``` + +- 不加 `override`：该组在顶层已选过 → 报重复错误；`override` 表示"来替换已有选择"。 +- `/` 开头：从配置根算起的绝对组路径（实验文件在子目录，需绝对路径定位顶层组）。 + +--- + +## 5. 插值 `${...}` + +```yaml +forget_split: forget10 +eval: + tofu: + forget_split: ${forget_split} # 引用同树别处的值，一处改处处变 +``` + +`configs/paths/default.yaml` 用插值拼输出目录： +`output_dir: ${paths.root_dir}/saves/${mode}/${task_name}`。 +也支持环境变量插值 `${oc.env:HF_HOME}` 等。 + +--- + +## 6. 参数优先级（从低到高） + +最终值由下面这条链决定，**后者覆盖前者**： + +1. **defaults 列表各组配置**，按列表顺序（靠后 > 靠前）； +2. **`_self_`**（顶层文件正文）在它所处位置参与合并； +3. **experiment**（`# @package _global_`，通常排在 defaults 靠后）→ 覆盖前面各组； +4. **命令行参数** → **最高优先级**。 + +> 因此 `experiment=unlearn/tofu/default` 里 trainer=GradAscent，命令行 `trainer=GradDiff` 仍会生效。 + +⚠️ 叠加 **struct 模式**：cfg 默认"结构冻结"，**覆盖不存在的键会报错** +（如 `Key 'holdout_split' is not in struct`）。新增键用 `+`（见下）；代码里改结构用 `with open_dict(...)`。 + +--- + +## 7. 命令行调用方式 + +| 写法 | 含义 | 例子 | +|------|------|------| +| `group=option` | **选**配置组选项 | `model=Llama-3.2-1B-Instruct`、`trainer=PDU` | +| `key=value` | 覆盖**已存在**的键 | `trainer.args.learning_rate=1e-5` | +| `+key=value` | **新增**配置里没有的键 | `+model.peft_args.r=16` | +| `++key=value` | 新增**或**强制覆盖 | `++trainer.args.num_train_epochs=3` | +| `~key` | **删除**某键 | `~trainer.args.logging_dir` | +| `group@pkg=option` | 选选项并重定位 package | `data/datasets@data.forget=TOFU_QA_forget_idk` | +| `--config-name=X` | 选顶层入口配置 | `--config-name=unlearn.yaml` | + +调试 / 查看（建议先 dry-run 看合成结果，再真跑）： + +```bash +python src/train.py ... --cfg job # 打印 job 最终合成配置（不运行） +python src/train.py ... --cfg job --resolve # 连 ${...} 插值也算出来 +python src/train.py ... --cfg job --package trainer # 只看 trainer 子树 +python src/train.py ... --help # 看可用组和选项 +python src/train.py ... -m a=1,2 b=3,4 # multirun 扫参（笛卡尔积） +``` + +--- + +## 8. 端到端示例（DPO 命令） + +```bash +python src/train.py --config-name=unlearn.yaml \ + experiment=unlearn/tofu/idk \ + model=Llama-3.2-1B-Instruct \ + trainer.args.eval_on_start=False \ + forget_split=forget10 retain_split=retain90 \ + task_name=demo_unlearn_DPO +``` + +合成顺序： + +1. `--config-name=unlearn.yaml` → 顶层骨架（默认 model=3B、trainer=GradAscent、data=unlearn、eval=tofu…）； +2. `experiment=unlearn/tofu/idk` 填入 experiment 占位，其 `_global_`+`override` 把 trainer→DPO、 + data.forget→TOFU_QA_forget_idk，并设 forget/retain_split 等； +3. `_self_`（unlearn.yaml 正文）合并 `mode: unlearn` 等； +4. **命令行**最后覆盖：model→1B、eval_on_start=False、splits、task_name。 + +结果：DPO trainer + idk 数据 + 1B 模型，输出到 `saves/unlearn/demo_unlearn_DPO/`。 + +--- + +## 9. 「优先级链 + 合成流程」图 + +```mermaid +flowchart TB + subgraph CLI["命令行启动: python src/train.py --config-name=unlearn.yaml ..."] + A0["--config-name 选定顶层入口配置"] + end + + A0 --> B["读取顶层 defaults 列表"] + + subgraph COMPOSE["Compose: 按 defaults 顺序逐项深合并 (靠后覆盖靠前)"] + direction TB + D1["1) 各 config group 默认选项
model / trainer / data / collator / eval / paths / hydra"] + D2["2) _self_：顶层文件正文
(位置决定何时合并)"] + D3["3) experiment (# @package _global_ + override)
整组替换 + 设实验超参"] + D1 --> D2 --> D3 + end + + B --> COMPOSE + COMPOSE --> E["4) 命令行覆盖
group=opt / key=val / +key / ++key / ~key"] + E --> F["解析插值 ${...} 与 ${oc.env:...}"] + F --> G["最终 cfg (struct 锁定)"] + G --> H["注入 main(cfg)"] + H --> I["handler 字符串 → 注册表类
get_model / load_trainer / get_evaluators"] + + classDef low fill:#eef,stroke:#88a + classDef high fill:#fee,stroke:#a88 + class D1,D2 low + class D3,E high +``` + +**优先级一句话**：`各组默认 < _self_ < experiment < 命令行`，最后再做 `${...}` 插值，得到冻结的 `cfg`。 + +--- + +## 10. 常见报错速查 + +| 报错 | 原因 | 解决 | +|------|------|------| +| `Key 'X' is not in struct` | 覆盖了配置里不存在的键（struct 锁） | 用 `+X=...` 新增；或确认该实验是否定义了 X | +| `Could not override 'group'` / `Could not find 'group/opt'` | 选了不存在的组选项 | 检查 `configs/<组>/` 下有无该 yaml；用 `--help` 看可选项 | +| `Could not append to ...` | 用 `key=` 改不存在的键 | 改用 `+key=` | +| 多个 default 冲突 / 重复 | 同组被选两次未用 override | 实验里用 `- override /group: ...` | + +更多 Hydra 官方文档见：