In-the-wild 2 — TOEIC mock-exam auto-grader (IE extraction → grading, with Solar-pro3)

Read a 6-page hand-filled answer sheet (photos) with IE (mode=enhanced) → grade against the key with rule/Solar-pro3. Reading is IE (vision); grading is rule/Solar.

• Note: notes/04-quiz-grader.md · implementation: code/grade_quiz.py · output: results/my_quiz_graded.json
• Constraint: only IE takes image input (Solar/Parse-OCR can't grade marks), so reading = IE.

import sys, os, json, glob, re, copy
from pathlib import Path
# Run from the repo root (move up if the notebook was opened inside notebooks/)
if Path.cwd().name == "notebooks": os.chdir("..")
sys.path.insert(0, "code")
import upstage_eval as ue
import metrics as M
from bs4 import BeautifulSoup
assert os.environ.get("UPSTAGE_API_KEY"), "export UPSTAGE_API_KEY first"
RES, FIG = Path("results"), Path("figures")
for d in [RES, RES/"parse", RES/"ie", RES/"report", FIG]: d.mkdir(parents=True, exist_ok=True)
FORCE = False  # set True to re-call the API (ignore cache)
def cached_parse(path, name, cats=("table",)):
    out = RES/"parse"/f"{name}.json"
    if out.exists() and not FORCE: return json.loads(out.read_text())
    resp = ue.document_parse(path, base64_categories=cats)
    out.write_text(json.dumps(resp, ensure_ascii=False)); return resp

Grading pipeline (photo → IE → rule/Solar grading)

import grade_quiz as gq
MY = Path("data/my_quiz"); KEYP = MY/"answer_key.json"
imgs = [str(p) for p in sorted(MY.glob("*")) if p.suffix.lower() in gq.IMG_EXT+(".pdf",)]
assert KEYP.exists() and imgs, "need images + answer_key.json under data/my_quiz/"
key = {str(k):v for k,v in json.load(open(KEYP)).items()}
pages = gq.expand_inputs(imgs); merged = {}
for path, label in pages:                       # per-page IE extraction (enhanced) → merge by question number
    ie = ue.ie_extract(path, gq.QUIZ_SCHEMA, "quiz", mode=gq.QUIZ_MODE)
    for q in (ie["parsed"] or {}).get("questions", []):
        try: n = int(q.get("question_number", 0))
        except Exception: continue
        if n and (n not in merged or (not merged[n].get("selected_answer") and q.get("selected_answer"))):
            merged[n] = q
questions = [merged[n] for n in sorted(merged)]
rg, sg = gq.rule_grade(questions, key), gq.solar_grade(questions, key)  # rule vs. Solar-pro3 grading
print(f"pages {len(pages)} → {len(questions)} questions")
for n in sorted(key, key=lambda x:int(x)):
    ext = next((q.get("selected_answer","") for q in questions if str(int(q['question_number']))==n), "")
    print(f"  Q{n} key={key[n]} extracted='{ext[:30]}' rule={rg.get(n)} solar={sg.get(n)}")
print(f"rule {sum(rg.values())}/{len(key)} | solar {sum(sg.values())}/{len(key)}")
json.dump({"key":key,"rule_grade":rg,"solar_grade":sg}, open(RES/"my_quiz_graded.json","w"), ensure_ascii=False, indent=2)