docs(ai): 논문용 실증 연구 — 통계 재분석·판정자 패널·지연 실험·양자화 연구

ai/scripts/llm_eval/analyze.py

@@ -6,14 +6,22 @@
 from __future__ import annotations
 
 import json
+import os
 import re
 import statistics
 import sys
 from collections import defaultdict
 from typing import Any
 
 sys.path.insert(0, __file__.rsplit("/", 1)[0])
-from cases import COACHING_CASES, FOLLOWUP_CASES, QUESTION_CASES  # noqa: E402
+import importlib as _il  # noqa: E402
+
+_C = _il.import_module(os.environ.get("LLM_EVAL_CASES", "cases"))  # noqa: E402
+COACHING_CASES, FOLLOWUP_CASES, QUESTION_CASES = (
+    _C.COACHING_CASES,
+    _C.FOLLOWUP_CASES,
+    _C.QUESTION_CASES,
+)
 
 HAN = re.compile(r"[一-鿿㐀-䶿]")
 KANA = re.compile(r"[぀-ヿ]")

ai/scripts/llm_eval/judge.py

@@ -21,7 +21,14 @@
 import httpx
 
 sys.path.insert(0, os.path.dirname(__file__))
-from cases import COACHING_CASES, FOLLOWUP_CASES, QUESTION_CASES  # noqa: E402
+import importlib as _il  # noqa: E402
+
+_C = _il.import_module(os.environ.get("LLM_EVAL_CASES", "cases"))  # noqa: E402
+COACHING_CASES, FOLLOWUP_CASES, QUESTION_CASES = (
+    _C.COACHING_CASES,
+    _C.FOLLOWUP_CASES,
+    _C.QUESTION_CASES,
+)
 
 JUDGES = ["gemini-3.1-pro-preview", "claude-opus-5"]
 
@@ -115,6 +122,8 @@ def case_brief(suite: str, cid: str) -> str:
             f"직군 {', '.join(c['job_categories'])} / 모드 {c['mode']} / 요청 질문 수 {c['max_questions']}\n"
             f"자기소개: {c.get('self_introduction') or '(없음)'}\n"
             f"최근 받은 질문(중복 금지): {c.get('recent_questions') or '(없음)'}\n"
+            f"타깃 회사/JD: {(c.get('target_company_name') or '') + ' ' + (c.get('target_job_description') or '(없음)')}\n"
+            f"집중 영역: {c.get('focus_areas') or '(없음)'}\n"
             f"지원자 자료:\n{_clip(c['context'], 6000)}"
         )
     c = C_BY_ID[cid]
@@ -137,16 +146,19 @@ def case_brief(suite: str, cid: str) -> str:
 
 
 async def judge_one(
-    client, base_url, api_key, judge, suite, cid, cands: dict[str, str], rng
+    client, base_url, api_key, judge, suite, cid, cands: dict[str, str], ordering=0
 ):
-    labels = list(cands)
-    rng.shuffle(labels)
+    # 재현 가능한 순서: (판정자, 과제, 케이스) 로 시드. ordering 1 은 같은 순서의 역순 → 위치 편향 상쇄.
+    labels = sorted(cands)
+    random.Random(f"{judge}|{suite}|{cid}").shuffle(labels)
+    if ordering % 2 == 1:
+        labels = labels[::-1]
     letters = list(string.ascii_uppercase[: len(labels)])
     mapping = dict(zip(letters, labels))
     blocks = "\n\n".join(f"### 후보 {L}\n{cands[mapping[L]]}" for L in letters)
     keys = KEYS[suite]
     schema = (
-        "{"
+        '{"rationale": "채점 근거 1~2문장 (점수보다 먼저)", '
         + ", ".join(f'"{k}": 1-5' for k in keys)
         + ', "issue": "가장 큰 문제 한 줄"}'
     )
@@ -180,6 +192,9 @@ async def judge_one(
                 "suite": suite,
                 "case_id": cid,
                 "ratings": {mapping[L]: data[L] for L in letters if L in data},
+                "positions": {mapping[L]: i for i, L in enumerate(letters)},
+                "ordering": ordering,
+                "output_chars": {mapping[L]: len(cands[mapping[L]]) for L in letters},
                 "n_candidates": len(letters),
             }
         except Exception as exc:  # noqa: BLE001
@@ -193,9 +208,19 @@ async def main() -> None:
     ap.add_argument("--out", required=True)
     ap.add_argument("--exclude", default="", help="쉼표로 구분한 label 제외")
     ap.add_argument("--concurrency", type=int, default=4)
+    ap.add_argument(
+        "--judges", default=",".join(JUDGES), help="쉼표로 구분한 판정 모델"
+    )
+    ap.add_argument(
+        "--orderings",
+        type=int,
+        default=1,
+        help="후보 제시 순서 수 (2 = 시드 순서 + 역순)",
+    )
     ap.add_argument("paths", nargs="+")
     args = ap.parse_args()
     exclude = {x for x in args.exclude.split(",") if x}
+    judges = [j for j in args.judges.split(",") if j]
 
     recs_by_label: dict[str, list[dict]] = defaultdict(list)
     for p in args.paths:
@@ -208,14 +233,13 @@ async def main() -> None:
     items = build_items(recs_by_label)
     base_url = os.environ["LLM_BASE_URL"].rstrip("/")
     api_key = os.environ["LLM_API_KEY"]
-    rng = random.Random(20260917)
     sem = asyncio.Semaphore(args.concurrency)
     async with httpx.AsyncClient() as client:
 
-        async def run(judge, key, cands):
+        async def run(judge, key, cands, ordering=0):
             async with sem:
                 res = await judge_one(
-                    client, base_url, api_key, judge, key[0], key[1], cands, rng
+                    client, base_url, api_key, judge, key[0], key[1], cands, ordering
                 )
                 print(
                     f"  {judge:<24} {key[0]:<9} {key[1]:<28} {'ERR ' + res['error'] if 'error' in res else 'ok'}",
@@ -224,9 +248,10 @@ async def run(judge, key, cands):
                 return res
 
         tasks = [
-            run(j, k, c)
+            run(j, k, c, o)
             for k, c in sorted(items.items())
-            for j in JUDGES
+            for j in judges
+            for o in range(args.orderings)
             if len(c) >= 2
         ]
         results = await asyncio.gather(*tasks)

ai/scripts/llm_eval/judge_bias.py

@@ -0,0 +1,77 @@
+"""판정자 계열 편향 혼합효과 분석 (두 라운드 통합).
+
+uv run --with numpy --with scipy --with pandas --with statsmodels python judge_bias.py
+저장소 루트에서 실행.
+"""
+
+import json
+from collections import defaultdict
+import statsmodels.formula.api as smf
+import pandas as pd
+
+rows = []
+for tag, path in [
+    ("round1", "docs/research/llm-eval-2026-09/eval-judge.jsonl"),
+    ("round2", "docs/research/local-llm-deep-dive-2026-09/data/judge.jsonl"),
+]:
+    sc = defaultdict(dict)
+    for line in open(path):
+        r = json.loads(line)
+        for m, s in (r.get("ratings") or {}).items():
+            try:
+                sc[(r["suite"], r["case_id"], m)][r["judge"]] = float(s["overall"])
+            except (KeyError, TypeError, ValueError):
+                pass
+    for (suite, cid, m), v in sc.items():
+        if len(v) == 2:
+            c, g = v["claude-opus-5"], v["gemini-3.1-pro-preview"]
+            rows.append(
+                dict(
+                    round=tag,
+                    suite=suite,
+                    case=f"{tag}:{suite}:{cid}",
+                    model=m,
+                    claude=c,
+                    gemini=g,
+                    diff=g - c,
+                    google=int(any(k in m.lower() for k in ("gemini", "gemma"))),
+                    big_google=int(
+                        m
+                        in (
+                            "gw-gemini-3.5-flash-lite",
+                            "gw-gemini-3.1-pro",
+                            "gw-gemma-4-31b",
+                        )
+                    ),
+                )
+            )
+df = pd.DataFrame(rows)
+df["quality"] = df["claude"]  # 다른 계열 판정자 점수를 품질 대리값으로
+df["qc"] = df["quality"] - df["quality"].mean()
+print("n pairs", len(df))
+for f in [
+    "diff ~ google",
+    "diff ~ qc",
+    "diff ~ google + qc",
+    "diff ~ big_google + qc",
+    "diff ~ google + qc + C(round)",
+]:
+    m = smf.mixedlm(f, df, groups=df["case"]).fit(reml=True, method="lbfgs")
+    print(f"\n== {f}  (mixed model, random intercept per case)")
+    for k in m.params.index:
+        if k == "Group Var":
+            continue
+        lo, hi = m.conf_int().loc[k]
+        print(
+            f"  {k:28s} {m.params[k]:+.3f} [{lo:+.3f}, {hi:+.3f}] p={m.pvalues[k]:.4f}"
+        )
+# 판정자별 점수 분산(척도 사용 폭)
+print(
+    "\nscore SD: claude", round(df.claude.std(), 3), "gemini", round(df.gemini.std(), 3)
+)
+print(
+    "slope gemini~claude (OLS):",
+    smf.ols("gemini ~ claude", df).fit().params.round(3).to_dict(),
+)
+# 그룹별 평균 차
+print(df.groupby(["round", "google"])["diff"].agg(["mean", "count"]).round(3))

ai/scripts/llm_eval/judge_panel.py

@@ -0,0 +1,149 @@
+"""3계열 판정자 패널 분석: 신뢰도(Krippendorff α)·계열 편향(혼합모형)·같은 계열 제외 패널 점수.
+
+uv run --with numpy --with scipy --with pandas --with statsmodels --with krippendorff \
+    python ai/scripts/llm_eval/judge_panel.py   (저장소 루트)
+"""
+
+from __future__ import annotations
+
+import json
+
+import krippendorff
+import numpy as np
+import pandas as pd
+import statsmodels.formula.api as smf
+
+ROUNDS = {
+    "round1": [
+        "docs/research/llm-eval-2026-09/eval-judge.jsonl",
+        "docs/research/thesis/judges/gpt55-r1.jsonl",
+    ],
+    "round2": [
+        "docs/research/local-llm-deep-dive-2026-09/data/judge.jsonl",
+        "docs/research/thesis/judges/gpt55-r2.jsonl",
+    ],
+}
+JUDGE_FAMILY = {
+    "gemini-3.1-pro-preview": "google",
+    "claude-opus-5": "anthropic",
+    "gpt-5.5": "openai",
+}
+
+
+def cand_family(model: str) -> str:
+    m = model.lower()
+    if "gemini" in m or "gemma" in m:
+        return "google"
+    if "gpt-oss" in m or "gptoss" in m:
+        return "openai"
+    return "other"
+
+
+rows = []
+for rnd, paths in ROUNDS.items():
+    for p in paths:
+        for line in open(p, encoding="utf-8"):
+            r = json.loads(line)
+            for model, sc in (r.get("ratings") or {}).items():
+                try:
+                    rows.append(
+                        dict(
+                            round=rnd,
+                            suite=r["suite"],
+                            case=f"{rnd}:{r['suite']}:{r['case_id']}",
+                            model=model,
+                            judge=r["judge"],
+                            score=float(sc["overall"]),
+                        )
+                    )
+                except (KeyError, TypeError, ValueError):
+                    pass
+df = pd.DataFrame(rows)
+df["same_family"] = [
+    int(JUDGE_FAMILY[j] == cand_family(m)) for j, m in zip(df.judge, df.model)
+]
+df["item"] = df["case"] + "|" + df["model"]
+print(
+    "rows", len(df), "items", df["item"].nunique(), "judges", sorted(df.judge.unique())
+)
+
+# 1) 신뢰도: 3판정자 Krippendorff α(ordinal), 판정자 쌍별 Spearman
+wide = df.pivot_table(index="item", columns="judge", values="score", aggfunc="mean")
+wide3 = wide.dropna()
+alpha = krippendorff.alpha(
+    reliability_data=wide3.T.values, level_of_measurement="ordinal"
+)
+print(
+    f"\n## 신뢰도 (완전 채점 {len(wide3)}문항)\nKrippendorff α (ordinal, 3 judges) = {alpha:.3f}"
+)
+js = list(wide3.columns)
+for i in range(len(js)):
+    for k in range(i + 1, len(js)):
+        a, b = wide3[js[i]], wide3[js[k]]
+        pa = krippendorff.alpha(
+            reliability_data=np.vstack([a.values, b.values]),
+            level_of_measurement="ordinal",
+        )
+        print(
+            f"  {js[i]} vs {js[k]}: Spearman {a.corr(b, method='spearman'):.3f}, α {pa:.3f}, mean diff {np.mean(a - b):+.3f}"
+        )
+
+# 2) 계열 편향 혼합모형: score ~ C(model) + C(judge) + same_family + (1|case)
+m = smf.mixedlm("score ~ C(model) + C(judge) + same_family", df, groups=df["case"]).fit(
+    reml=True, method="lbfgs"
+)
+lo, hi = m.conf_int().loc["same_family"]
+print(
+    f"\n## 같은 계열 편향 (모델·판정자 고정효과 통제, 케이스 무작위 절편)\nsame_family = {m.params['same_family']:+.3f} [{lo:+.3f}, {hi:+.3f}], p = {m.pvalues['same_family']:.4g}"
+)
+# 판정자별 자기 계열 효과
+for fam, judge in (("google", "gemini-3.1-pro-preview"), ("openai", "gpt-5.5")):
+    sub = df.copy()
+    sub["own"] = [
+        int(j == judge and cand_family(mm) == fam)
+        for j, mm in zip(sub.judge, sub.model)
+    ]
+    if sub["own"].sum() == 0:
+        continue
+    mm_ = smf.mixedlm("score ~ C(model) + C(judge) + own", sub, groups=sub["case"]).fit(
+        reml=True, method="lbfgs"
+    )
+    lo2, hi2 = mm_.conf_int().loc["own"]
+    print(
+        f"  {judge} → {fam} 계열 후보: {mm_.params['own']:+.3f} [{lo2:+.3f}, {hi2:+.3f}] p={mm_.pvalues['own']:.4g} (n own={int(sub['own'].sum())})"
+    )
+
+# 3) 같은 계열 판정자 제외 패널 평균 (라운드·과제별)
+df_loo = df[df.same_family == 0]
+panel = (
+    df_loo.groupby(["round", "suite", "model", "case"])["score"]
+    .mean()
+    .groupby(["round", "suite", "model"])
+    .agg(["mean", "count"])
+    .reset_index()
+)
+allj = (
+    df.groupby(["round", "suite", "model", "case"])["score"]
+    .mean()
+    .groupby(["round", "suite", "model"])
+    .mean()
+    .rename("all_judges")
+)
+by_judge = (
+    df.groupby(["round", "suite", "model", "judge"])["score"].mean().unstack("judge")
+)
+out = panel.merge(allj.reset_index(), on=["round", "suite", "model"]).merge(
+    by_judge.reset_index(), on=["round", "suite", "model"]
+)
+out = out.rename(columns={"mean": "panel_excl_same_family", "count": "cases"})
+pd.set_option("display.width", 250)
+pd.set_option("display.max_columns", 20)
+for (rnd, suite), g in out.groupby(["round", "suite"]):
+    print(f"\n## {rnd} · {suite} (같은 계열 제외 패널 평균 내림차순)")
+    print(
+        g.drop(columns=["round", "suite"])
+        .sort_values("panel_excl_same_family", ascending=False)
+        .round(2)
+        .to_string(index=False)
+    )
+out.round(3).to_csv("docs/research/thesis/judges/panel-scores.csv", index=False)