import sys, os, json, glob, re, copy
from pathlib import Path
# Run from the repo root (move up if the notebook was opened inside notebooks/)
if Path.cwd().name == "notebooks": os.chdir("..")
sys.path.insert(0, "code")
import upstage_eval as ue
import metrics as M
from bs4 import BeautifulSoup
assert os.environ.get("UPSTAGE_API_KEY"), "export UPSTAGE_API_KEY first"
RES, FIG = Path("results"), Path("figures")
for d in [RES, RES/"parse", RES/"ie", RES/"report", FIG]: d.mkdir(parents=True, exist_ok=True)
FORCE = False  # set True to re-call the API (ignore cache)
def cached_parse(path, name, cats=("table",)):
    out = RES/"parse"/f"{name}.json"
    if out.exists() and not FORCE: return json.loads(out.read_text())
    resp = ue.document_parse(path, base64_categories=cats)
    out.write_text(json.dumps(resp, ensure_ascii=False)); return resp

# zero-shot extraction per schema → per-field accuracy/F1 vs. hand-made GT
ie_scores = {}
for sf in sorted(glob.glob("gt/exp2_schemas/ie_*.json")):
    spec = json.load(open(sf)); doc = Path(spec["doc"]).stem
    img = f"data/exp2_ie/{spec['doc']}"
    out = ue.ie_extract(img, spec["schema"], "document_schema")
    json.dump(out["parsed"], open(RES/"ie"/f"{doc}.json","w"), ensure_ascii=False, indent=2)
    gt = {k:v for k,v in (spec.get("gt") or {}).items() if not k.startswith("_")}
    if gt:
        pred_sub = {k: out["parsed"].get(k) if out["parsed"] else None for k in gt}
        s = M.field_scores(pred_sub, gt)
        wrong = [k for k in gt if not M._val_match(str((out["parsed"] or {}).get(k,"")), str(gt[k]))]
        ie_scores[doc] = {"accuracy":round(s["accuracy"],3),"f1":round(s["f1"],3),"wrong":wrong}
        print(f"{doc:22s} acc={s['accuracy']:.2f} f1={s['f1']:.2f} wrong={wrong}")
    else:
        print(f"{doc:22s} (no GT → skip scoring)  {str(out['parsed'])[:80]}")

# robustness probe: apply the hotel-receipt schema to the boarding pass → does it invent absent fields?
probe = json.load(open("gt/exp2_schemas/ie_receipt_hotel.json"))
pout = ue.ie_extract("data/exp2_ie/ie_boardingpass.jpeg", probe["schema"], "probe")
absent = ["supply_amount","vat","total_amount","business_registration_no"]
hall = {k:(pout["parsed"] or {}).get(k) for k in absent}
print("PROBE responses for absent fields:", hall, "\n -> 0/empty means no hallucination")

# defect experiment: print-shop receipt with a smudge over installment/card no.
#   default extraction vs. an instruction to "leave empty if not visible"
spec = json.load(open("gt/exp2_schemas/ie_receipt_cafe.json"))
img = "data/exp2_ie/ie_receipt_cafe.jpg"; DF=["card_number_masked","installment_months","card_issuer"]
A = ue.ie_extract(img, spec["schema"], "cafe")["parsed"]
sb = copy.deepcopy(spec["schema"])
for v in sb["properties"].values():
    if isinstance(v,dict) and v.get("type")=="string":
        v["description"]=v.get("description","")+" (if hidden/damaged and unreadable, return an empty string; do not guess)"
B = ue.ie_extract(img, sb, "cafe_null")["parsed"]
defect = {f:{"base":(A or {}).get(f),"null_instruct":(B or {}).get(f)} for f in DF}
json.dump({"base":A,"null_instruct":B}, open(RES/"ie"/"_defect_experiment.json","w"), ensure_ascii=False, indent=2)
for f,v in defect.items(): print(f"{f:20s} base={v['base']!r}  null_instruct={v['null_instruct']!r}")
print("Reading: installment (hidden) base='0' (guess) → '' under instruction proves 'default = hallucination'")

Model Test 2 — Universal Information Extract¶

1. Per-schema extraction + scoring¶

2. Robustness probe — does it hallucinate under a wrong schema?¶

3. Defect experiment — occluded fields: guess vs. honest under instruction¶