"""Token IDs model boundaries, not real-language tokenization or retrieval."""
import json
T=list(range(20));fact={6,7,8,9}
def windows(size,overlap):
 out=[];start=0
 while start<len(T):
  out.append(T[start:start+size])
  if start+size>=len(T):break
  start+=size-overlap
 return out
cases={'fixed8':windows(8,0),'overlap8_2':windows(8,2),'paragraph':[T[:6],T[6:12],T[12:]],'fixed12':windows(12,0)}
result={k:{'chunks':x,'tokens_stored':sum(map(len,x)),'fact_whole_in_one_chunk':any(fact<=set(c) for c in x)} for k,x in cases.items()}
assert not result['fixed8']['fact_whole_in_one_chunk']
assert result['overlap8_2']['fact_whole_in_one_chunk']
print(json.dumps({'label':'抽象token位置教学，不是嵌入/生成测试','results':result},ensure_ascii=False,indent=2))