diff --git a/configs/eval/tofu.yaml b/configs/eval/tofu.yaml index 5a895d192..29e05e488 100644 --- a/configs/eval/tofu.yaml +++ b/configs/eval/tofu.yaml @@ -4,6 +4,7 @@ defaults: # include all defined metrics files - tofu_metrics: # When you import a metric here, its configuration automatically populates the # metric key below, enabled by the @package directive at the top of each configuration file. + - forget_Truth_Ratio - forget_quality - forget_Q_A_Prob - forget_Q_A_ROUGE diff --git a/src/evals/metrics/memorization.py b/src/evals/metrics/memorization.py index d033c1ea0..c7bbe386c 100644 --- a/src/evals/metrics/memorization.py +++ b/src/evals/metrics/memorization.py @@ -119,10 +119,16 @@ def closer_to_1_better(arr): def true_better(arr): return np.mean(np.maximum(0, 1 - arr)) + # Extent of knowledge (as used in OpenUnlearning paper's meta-evaluation) uses tr=true/(true+false) + def prob_mean(arr): + return np.mean(arr) + if kwargs["aggregator"] == "closer_to_1_better": aggregator = closer_to_1_better elif kwargs["aggregator"] == "true_better": aggregator = true_better + elif kwargs["aggregator"] == "prob_mean": + aggregator = prob_mean else: raise ValueError(f"Invalid truth ratio aggregator: {kwargs['aggregator']}") @@ -153,7 +159,13 @@ def true_better(arr): correct_prob = np.exp(-correct_avg_losses) wrong_prob = np.exp(-wrong_avg_losses) - truth_ratios = wrong_prob / (correct_prob + 1e-10) + if kwargs["aggregator"] != "prob_mean": + # Original definition from TOFU: wrong / correct + truth_ratios = wrong_prob / (correct_prob + 1e-10) + else: + # New definition from OpenUnlearning: correct / (correct + wrong) + truth_ratios = correct_prob / (correct_prob + wrong_prob + 1e-10) + value_by_index = dict( zip(correct_indices, [{"score": val} for val in truth_ratios]) )