Skip to content

Commit 910f14a

Browse files
flyingdoubleGcodelion
authored andcommitted
fix(entropy_decoding): Keep the nucleus head and apply min_p in _sample
- Top-p sorted the top-k probabilities in ascending order before the cumulative-sum test, so it removed the most likely tokens once the less likely ones exceeded top_p, instead of the least likely tail. Keep the descending top-k order. - min_p masked the logits, but sampling used the probabilities computed before, so it never removed anything. Mask the probabilities.
1 parent 990b5f9 commit 910f14a

2 files changed

Lines changed: 40 additions & 7 deletions

File tree

‎optillm/entropy_decoding.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -63,16 +63,16 @@ def _sample(logits: torch.Tensor, temperature=0.666, top_p=0.90, top_k=27, min_p
6363
probs = F.softmax(logit / temperature, dim=-1)
6464

6565
if min_p > 0.0:
66+
# Drop tokens below min_p times the most likely token's probability
6667
p_max = torch.max(probs, dim=-1, keepdim=True).values
67-
indices_to_remove = probs < (min_p * p_max)
68-
logit = torch.where(indices_to_remove, torch.full_like(logit, float('-inf')), logit)
68+
probs = probs.masked_fill(probs < (min_p * p_max), 0.0)
6969

70-
top_k_probs, top_k_indices = torch.topk(probs, k=min(top_k, probs.shape[-1]))
71-
probs_sort = torch.flip(top_k_probs, dims=[-1])
72-
probs_idx = torch.flip(top_k_indices, dims=[-1])
70+
# Nucleus over the top-k tokens in descending order: keep each token until
71+
# the probability of the tokens before it exceeds top_p, which always
72+
# keeps the most likely token and removes the least likely tail.
73+
probs_sort, probs_idx = torch.topk(probs, k=min(top_k, probs.shape[-1]))
7374
probs_sum = torch.cumsum(probs_sort, dim=-1)
74-
mask = torch.where(probs_sum - probs_sort > top_p, torch.tensor(1.0, device=device), torch.tensor(0.0, device=device))
75-
probs_sort = probs_sort * (1 - mask)
75+
probs_sort = probs_sort.masked_fill(probs_sum - probs_sort > top_p, 0.0)
7676
probs_sort = probs_sort / torch.sum(probs_sort, dim=-1, keepdim=True)
7777
next_token = torch.multinomial(probs_sort, 1, generator=generator)
7878
next_token_g = torch.gather(probs_idx, -1, next_token.reshape(bsz, 1).to(torch.int64))
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
"""Offline regression tests for top-p and min-p filtering in entropy decoding."""
2+
3+
import torch
4+
5+
from optillm.entropy_decoding import _sample
6+
7+
8+
def sampled_tokens(probs, num_samples=2000, **kwargs):
9+
logits = probs.log().view(1, 1, -1)
10+
generator = torch.Generator().manual_seed(0)
11+
return {
12+
_sample(logits, generator=generator, **kwargs).item()
13+
for _ in range(num_samples)
14+
}
15+
16+
17+
def test_top_p_keeps_the_most_likely_token():
18+
# The 39 less likely tokens together exceed top_p.
19+
probs = torch.tensor([0.05] + [0.95 / 39] * 39)
20+
21+
tokens = sampled_tokens(probs, temperature=1.0, top_p=0.9, top_k=40)
22+
23+
assert 0 in tokens
24+
# The nucleus keeps tokens until those before them exceed 0.9: 36 of 40.
25+
assert tokens == set(range(36))
26+
27+
28+
def test_min_p_removes_unlikely_tokens():
29+
probs = torch.tensor([0.5, 0.3] + [0.2 / 20] * 20)
30+
31+
tokens = sampled_tokens(probs, temperature=1.0, top_p=1.0, top_k=22, min_p=0.1)
32+
33+
assert tokens == {0, 1}

0 commit comments

Comments
 (0)