Spaces:
Sleeping
Sleeping
File size: 13,693 Bytes
ac3af28 6823e3f ac3af28 6823e3f ac3af28 6823e3f ac3af28 6823e3f ac3af28 6823e3f ac3af28 6823e3f ac3af28 6823e3f e8dc334 ac3af28 e8dc334 ac3af28 e8dc334 ac3af28 6823e3f ac3af28 5229d2d ac3af28 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 | """Future Oracle inference (self-contained port of nla.future_lens.eval's readout path).
A Future Oracle is a LoRA adapter on a Qwen3 base model that reads a single residual-stream
vector (block-`layer` output at token t) and verbalises the tokens the model will produce next.
Recipe (identical to training/eval in EasyNLA `nla/future_lens`):
1. activation h = hidden_states[layer + 1][t] of the bare model on the token ids (no prompt)
2. ground truth = the bare model's greedy continuation of ids[:t+1] (the oracle's label convention)
3. decoder prompt = template.format(layer, k, MARKER) + "\\n" (prompt_format "plain")
4. the marker token's *input embedding* is replaced by alpha_layer * alpha_mult * h / ||h||
(alpha_layer = p75 activation norm of that layer, from the collection sidecar)
5. greedy decode with the adapter on, cut at EOS, truncated to k tokens.
Registry: every `ckpts/<run>/iter_*/adapter_config.json` in the HF dataset repo is an oracle
(latest iter per run; sweep/ablation runs skipped). Alpha per layer + template come from the
matching `data*/train.parquet.nla_meta.yaml` sidecar (same base model, layers covered).
"""
from __future__ import annotations
import json
import os
import re
import threading
from dataclasses import dataclass, field
import torch
import yaml
from huggingface_hub import HfApi, hf_hub_download, snapshot_download
REPO = os.environ.get("FL_HF_REPO", "syvb/rl-future-lens-qwen3-8b")
TOKEN = os.environ.get("HF_TOKEN")
SKIP_RUN = re.compile(r"^(sweep_|ablation_|sft_filt|sft_unf)")
# base checkpoints served (the obsolete chat-model/text-label oracle would add a second 16 GB 8B)
BASES = [x for x in os.environ.get("FO_BASES", "Qwen/Qwen3-0.6B-Base,Qwen/Qwen3-1.7B-Base,Qwen/Qwen3-8B-Base").split(",") if x]
@dataclass
class Sidecar:
data_dir: str
base_model: str
template: str
prompt_format: str
marker: str
marker_id: int
left_id: int
right_id: int
alpha_by_layer: dict[int, float]
@dataclass
class Oracle:
run: str
iter_dir: str # repo path ckpts/<run>/iter_XXXXXXX
step: int
base_model: str
layers: list[int]
alpha_mult: float
injection: str
label: str
sidecar: Sidecar
local_dir: str | None = None
extra: dict = field(default_factory=dict)
@property
def adapter_name(self) -> str: # torch module names can't contain "."
return self.run.replace(".", "_")
@property
def size(self) -> str:
m = re.search(r"Qwen3-([\d.]+B)", self.base_model)
return m.group(1) if m else self.base_model
@property
def name(self) -> str:
kind = "RL" if self.run.startswith("rl_") else "SFT"
lay = f"layer {self.layers[0]}" if len(self.layers) == 1 else f"layers {','.join(map(str, self.layers))}"
return f"Future Oracle 路 Qwen3-{self.size} 路 {lay} 路 {kind} {self.run} @ {self.step}"
def _dl(path: str, local_dir: str) -> str:
return hf_hub_download(REPO, path, repo_type="dataset", token=TOKEN, local_dir=local_dir)
def load_sidecar(data_dir: str, cache: str) -> Sidecar | None:
for split in ("train", "eval"):
try:
p = _dl(f"{data_dir}/{split}.parquet.nla_meta.yaml", cache)
except Exception:
continue
m = yaml.safe_load(open(p))
fl, tk = m.get("future_lens", {}), m.get("tokens", {})
if not fl or not tk:
continue
pf = (fl.get("extra") or {}).get("prompt_format") or ("plain" if "Base" in m["extraction"]["base_model"] else "chat")
return Sidecar(
data_dir=data_dir, base_model=m["extraction"]["base_model"], template=fl["template"], prompt_format=pf,
marker=tk["injection_char"], marker_id=int(tk["injection_token_id"]),
left_id=int(tk["injection_left_neighbor_id"]), right_id=int(tk["injection_right_neighbor_id"]),
alpha_by_layer={int(k): float(v) for k, v in fl["injection_scale_by_layer"].items()},
)
return None
def scan_registry(cache: str = "/tmp/fo_meta") -> list[Oracle]:
"""Every finished oracle adapter in the repo, newest run first."""
api = HfApi(token=TOKEN)
files = [s.rfilename for s in api.dataset_info(REPO).siblings]
sidecars: list[Sidecar] = []
for d in sorted({f.split("/")[0] for f in files if f.startswith("data") and f.endswith(".nla_meta.yaml")}):
if d == "data_unf": # filter-ablation split: not a training split of any kept oracle
continue
sc = load_sidecar(d, cache)
if sc:
sidecars.append(sc)
iters: dict[str, list[tuple[int, str]]] = {}
for f in files:
m = re.match(r"ckpts/([^/]+)/iter_(\d+)/adapter_config\.json$", f)
if m and not SKIP_RUN.match(m.group(1)) and f"ckpts/{m.group(1)}/iter_{m.group(2)}/future_lens.json" in files:
iters.setdefault(m.group(1), []).append((int(m.group(2)), f"ckpts/{m.group(1)}/iter_{m.group(2)}"))
oracles: list[Oracle] = []
for run, its in iters.items():
step, idir = max(its)
cfg = json.load(open(_dl(f"{idir}/adapter_config.json", cache)))
fl = json.load(open(_dl(f"{idir}/future_lens.json", cache)))
base = cfg["base_model_name_or_path"]
if base not in BASES:
continue
layers = [int(x) for x in str(fl.get("layers", "")).split(",") if x]
cands = [s for s in sidecars if s.base_model == base and set(layers) <= set(s.alpha_by_layer)]
if not cands:
continue
sfx = re.search(r"_(\d+(?:\.\d+)?b)_L(\d+)$", run)
pref = f"data_{sfx.group(1)}_L{sfx.group(2)}" if sfx else None
sc = next((s for s in cands if s.data_dir == pref), None) or sorted(cands, key=lambda s: s.data_dir)[-1]
oracles.append(Oracle(run=run, iter_dir=idir, step=step, base_model=base, layers=layers,
alpha_mult=float(fl.get("alpha_mult", 1.0)), injection=fl.get("injection", "replace_embed"),
label=fl.get("label", "greedy"), sidecar=sc, extra=fl))
# small models first, then by run name
def key(o):
m = re.search(r"([\d.]+)B", o.size)
return (float(m.group(1)) if m else 99, o.run)
return sorted(oracles, key=key)
# ----------------------------------------------------------------------------
# models
# ----------------------------------------------------------------------------
class Bank:
"""Base models (bf16) with all oracle adapters attached as named PEFT adapters.
ZeroGPU: everything is assembled on CPU at import time and moved to CUDA once per base model
(`to_device`), which ZeroGPU packs for its workers. Attaching an adapter later (Refresh) moves
that base back to CPU, attaches, and moves it to CUDA again."""
def __init__(self, device: str = "cuda", dtype=torch.bfloat16):
self.device, self.dtype = device, dtype
self.tok = None
self.models: dict[str, torch.nn.Module] = {} # base id -> PeftModel (or bare base)
self.on_device: set[str] = set()
self.loaded: set[str] = set() # oracle runs attached
self.lock = threading.Lock()
def to_device(self):
for b, m in self.models.items():
if b not in self.on_device:
m.to(self.device)
self.on_device.add(b)
def tokenizer(self, base: str):
if self.tok is None:
from transformers import AutoTokenizer
self.tok = AutoTokenizer.from_pretrained(base) # Qwen3 sizes share one tokenizer
return self.tok
def base(self, base: str):
if base not in self.models:
self.tokenizer(base)
from transformers import AutoModelForCausalLM
m = AutoModelForCausalLM.from_pretrained(base, dtype=self.dtype, attn_implementation="sdpa")
self.models[base] = m.eval() # CPU until to_device()
return self.models[base]
def attach(self, o: Oracle):
if o.run in self.loaded:
return
with self.lock:
if o.run in self.loaded:
return
if o.local_dir is None:
o.local_dir = os.path.join(snapshot_download(REPO, repo_type="dataset", token=TOKEN,
allow_patterns=[f"{o.iter_dir}/*"],
local_dir="/tmp/fo_ckpts"), o.iter_dir)
from peft import PeftModel
m = self.base(o.base_model)
back = o.base_model in self.on_device
if back: # late attach: assemble on CPU, then re-pack
m.to("cpu"); self.on_device.discard(o.base_model)
# torch_device="cpu": PEFT otherwise infers "cuda" from ZeroGPU's patched availability flag
# and maps the safetensors there, which is forbidden outside a GPU call
if isinstance(m, PeftModel):
m.load_adapter(o.local_dir, adapter_name=o.adapter_name, torch_device="cpu")
else:
self.models[o.base_model] = PeftModel.from_pretrained(m, o.local_dir, adapter_name=o.adapter_name,
torch_device="cpu").eval()
self.loaded.add(o.run)
if back:
self.to_device()
def model(self, o: Oracle):
self.attach(o)
m = self.models[o.base_model]
m.set_adapter(o.adapter_name)
return m
def stop_ids(tok, model) -> set[int]:
ids = {tok.eos_token_id}
gc = getattr(getattr(model, "generation_config", None), "eos_token_id", None)
if gc is not None:
ids.update(gc if isinstance(gc, (list, tuple)) else [gc])
ids.discard(None)
return ids
@torch.no_grad()
def read_future(bank: Bank, o: Oracle, layer: int, ids: list[int], t: int, k: int,
control: str = "real", alpha_mult: float | None = None, shuffle_t: int | None = None) -> dict:
"""One oracle readout at token t of `ids`. control: real | none | shuffled (vector from
position shuffle_t of the same text) | wrong_layer (layer given, adapter untrained on it)."""
assert 0 <= t < len(ids)
dev = bank.device
model = bank.model(o)
tok = bank.tokenizer(o.base_model)
sc = o.sidecar
x = torch.tensor([ids], dtype=torch.long, device=dev)
# ---- target model: activation + its own greedy continuation (adapter OFF) ----
with model.disable_adapter():
out = model(input_ids=x, output_hidden_states=True, use_cache=False)
src_t = shuffle_t if control == "shuffled" else t
h = out.hidden_states[layer + 1][0, src_t].float()
top1 = int(out.logits[0, t].argmax())
gen = model.generate(input_ids=x[:, : t + 1], attention_mask=torch.ones_like(x[:, : t + 1]),
max_new_tokens=k, do_sample=False, pad_token_id=tok.pad_token_id or tok.eos_token_id)
greedy = gen[0, t + 1:].tolist()
p_next = torch.log_softmax(out.logits[0, t].float(), -1)
# ---- decoder prompt with the marker ----
prompt = sc.template.format(layer=int(layer), k=int(k), injection_char=sc.marker)
if sc.prompt_format == "plain":
text = prompt + "\n"
else:
text = tok.apply_chat_template([{"role": "user", "content": prompt}], tokenize=False,
add_generation_prompt=True, enable_thinking=False)
p_ids = tok.encode(text, add_special_tokens=False)
pos = [i for i in range(1, len(p_ids) - 1)
if p_ids[i] == sc.marker_id and p_ids[i - 1] == sc.left_id and p_ids[i + 1] == sc.right_id]
assert len(pos) == 1, f"marker not found exactly once in the decoder prompt: {pos}"
alpha = sc.alpha_by_layer[int(layer)] * (o.alpha_mult if alpha_mult is None else alpha_mult)
vec = None if control == "none" else (h / h.norm().clamp_min(1e-12)) * alpha
embed = model.get_input_embeddings()
def hook(module, args, kwargs, output):
if vec is None:
return output
inp = kwargs.get("input") if kwargs else None
if inp is None and args:
inp = args[0]
if inp is None or inp.dim() != 2 or inp.shape[1] < 2:
return output # KV-cached decode steps
out2 = output.clone()
out2[0, pos[0]] = vec.to(out2.dtype)
return out2
hd = embed.register_forward_hook(hook, with_kwargs=True)
try:
px = torch.tensor([p_ids], dtype=torch.long, device=dev)
g = model.generate(input_ids=px, attention_mask=torch.ones_like(px), max_new_tokens=k + 3, do_sample=False,
pad_token_id=tok.pad_token_id or tok.eos_token_id)
finally:
hd.remove()
resp = g[0, len(p_ids):].tolist()
eos = stop_ids(tok, model)
cut = next((i for i, x_ in enumerate(resp) if x_ in eos), len(resp))
readout = resp[:cut][:k]
actual = ids[t + 1: t + 1 + k]
return {
"oracle": o.name, "run": o.run, "size": o.size, "layer": int(layer), "t": t, "k": k, "control": control,
"alpha": alpha, "act_norm": float(h.norm()),
"readout": readout, "readout_text": tok.decode(readout),
"greedy": greedy, "greedy_text": tok.decode(greedy),
"actual": actual, "actual_text": tok.decode(actual),
"hits": [i < len(readout) and i < len(greedy) and readout[i] == greedy[i] for i in range(k)],
"top1_correct": (top1 == ids[t + 1]) if t + 1 < len(ids) else None, # None: no next token to compare
"p_actual_next": float(p_next[ids[t + 1]].exp()) if t + 1 < len(ids) else None,
}
|