docs(ai): 로컬 LLM 전환 심층 조사 리포트(md·pdf) + 동시성 부하 도구

ai/scripts/llm_eval/README.md

@@ -10,6 +10,7 @@
 | `cases.py` | 라벨이 붙은 케이스: 질문 풀 5개(긴 문맥 p90 포함), 꼬리질문 14개(강한·약한·모름·재설명·확인형·사실 불일치 등), 코칭 3개 |
 | `run_eval.py` | 후보 1개를 돌려 원시 결과 JSONL 저장. `--latency` 는 콜드스타트, 코칭 15건×동시 5, 그 도중 꼬리질문 지연 측정 |
 | `analyze.py` | 규칙 기반 자동 지표 (태그 준수, 의도 정확도, 점수 라벨·사실대조 규칙, 근거 인용 검증, 중복, 외국 문자, 지연) |
+| `load_test.py` | 동시 사용자 부하: 동시 1/2/4/8명 꼬리질문 지연·첫 토큰·처리량, 코칭 15건×동시 5 도중 꼬리질문 지연 |
 | `judge.py` | 블라인드 비교 채점. 판정 모델 2개(gemini-3.1-pro-preview, claude-opus-5)로 자기 계열 선호 편향 상쇄 |
 
 ## 실행 (stackup-ai 컨테이너 안)
@@ -29,3 +30,12 @@ python judge.py --out judge.jsonl /tmp/eval/*.jsonl > judge-table.json
 
 주의: reasoning 을 끌 수 없는 모델(gpt-oss)은 운영 `LLM_FLASH_MAX_TOKENS=512` 에서 추론이 토큰을 다 써
 꼬리질문이 비어 나온다. 공정 비교가 필요하면 `--flash-max-tokens 2048`.
+
+JSON 스키마 강제(`response_format: json_schema`)로 질문 풀·코칭을 돌리려면 `run_eval.py --constrain-json`.
+소형 모델에서 파싱 성공률은 오르지만 내용 품질은 떨어질 수 있다 ([심층 조사 §4.5](../../../docs/research/local-llm-deep-dive-2026-09/local-llm-deep-dive.md)).
+
+```bash
+python load_test.py --label local-x --base-url http://<서버>:8080/v1 --api-key x --model <모델> \
+  --concurrency 1,2,4,8 --out /tmp/eval/load-local-x.jsonl
+```
+

ai/scripts/llm_eval/load_test.py

@@ -0,0 +1,153 @@
+"""동시 사용자 부하 테스트 — 운영 스트리밍 꼬리질문·코칭 체인으로 동시성 N 에서 지연 분포를 잰다.
+
+python load_test.py --label local-llamacpp-np4 --base-url http://host:port/v1 --api-key x \
+    --model qwen3-4b --concurrency 1,2,4,8 --requests-per-level 16 --out /tmp/eval/load.jsonl
+
+시나리오
+- followup@N : 꼬리질문 요청을 동시에 N 개씩 흘려 보냄 (서로 다른 케이스 순환). 요청별 총지연·첫 질문 토큰.
+- mixed      : 코칭 15건×동시 5 fan-out 을 돌리며 1.5초 간격으로 꼬리질문 6건 투입 → 꼬리질문 지연 분포.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import os
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(__file__))
+
+import run_eval as R  # noqa: E402
+from cases import COACHING_CASES, FOLLOWUP_CASES  # noqa: E402
+
+
+def pct(xs, q):
+    xs = sorted(x for x in xs if x is not None)
+    if not xs:
+        return None
+    return round(xs[min(len(xs) - 1, int(round(q * (len(xs) - 1))))], 2)
+
+
+async def level(settings, label, n, total, fh):
+    sem = asyncio.Semaphore(n)
+    cases = [FOLLOWUP_CASES[i % len(FOLLOWUP_CASES)] for i in range(total)]
+
+    async def one(i, c):
+        async with sem:
+            return await R.run_followup(settings, c, i, label, tag=f":load@{n}")
+
+    t0 = time.perf_counter()
+    recs = await asyncio.gather(*(one(i, c) for i, c in enumerate(cases)))
+    wall = time.perf_counter() - t0
+    for r in recs:
+        fh.write(json.dumps(r, ensure_ascii=False) + "\n")
+    ok = [r for r in recs if r["ok"]]
+    normal = [r for r in ok if r.get("answer_intent") == "NORMAL"]
+    summary = {
+        "label": label,
+        "suite": "load_summary",
+        "ok": True,
+        "concurrency": n,
+        "requests": total,
+        "success": len(ok),
+        "wall_sec": round(wall, 2),
+        "throughput_req_per_min": round(60 * len(ok) / wall, 1),
+        "latency_p50": pct([r["latency_sec"] for r in ok], 0.5),
+        "latency_p95": pct([r["latency_sec"] for r in ok], 0.95),
+        "ttft_p50": pct([r["ttft_sec"] for r in normal], 0.5),
+        "ttft_p95": pct([r["ttft_sec"] for r in normal], 0.95),
+        "over_10s": sum(1 for r in ok if r["latency_sec"] > 10),
+    }
+    fh.write(json.dumps(summary, ensure_ascii=False) + "\n")
+    fh.flush()
+    print(
+        "  ",
+        {k: v for k, v in summary.items() if k not in ("label", "suite", "ok")},
+        file=sys.stderr,
+    )
+
+
+async def mixed(settings, label, fh):
+    sem = asyncio.Semaphore(5)
+
+    async def coach(i):
+        async with sem:
+            return await R.run_coaching(
+                settings, COACHING_CASES[i % 3], i, label, tag=":mixed-fanout"
+            )
+
+    async def followups():
+        out = []
+        for i in range(6):
+            await asyncio.sleep(1.5)
+            out.append(
+                asyncio.create_task(
+                    R.run_followup(
+                        settings, FOLLOWUP_CASES[i], i, label, tag=":mixed-followup"
+                    )
+                )
+            )
+        return await asyncio.gather(*out)
+
+    t0 = time.perf_counter()
+    res = await asyncio.gather(*(coach(i) for i in range(15)), followups())
+    wall = time.perf_counter() - t0
+    coaches, fus = res[:15], res[15]
+    for r in list(coaches) + list(fus):
+        fh.write(json.dumps(r, ensure_ascii=False) + "\n")
+    summary = {
+        "label": label,
+        "suite": "mixed_summary",
+        "ok": True,
+        "fanout_wall_sec": round(wall, 2),
+        "coaching_success": sum(r["ok"] for r in coaches),
+        "followup_latency_p50": pct([r["latency_sec"] for r in fus if r["ok"]], 0.5),
+        "followup_latency_max": pct([r["latency_sec"] for r in fus if r["ok"]], 1.0),
+        "followup_ttft_p50": pct([r["ttft_sec"] for r in fus if r["ok"]], 0.5),
+    }
+    fh.write(json.dumps(summary, ensure_ascii=False) + "\n")
+    fh.flush()
+    print(
+        "   mixed",
+        {k: v for k, v in summary.items() if k not in ("label", "suite", "ok")},
+        file=sys.stderr,
+    )
+
+
+async def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--label", required=True)
+    ap.add_argument("--model", required=True)
+    ap.add_argument("--base-url", default="")
+    ap.add_argument("--api-key", default=None)
+    ap.add_argument("--concurrency", default="1,2,4,8")
+    ap.add_argument("--requests-per-level", type=int, default=16)
+    ap.add_argument("--timeout", type=float, default=300)
+    ap.add_argument("--flash-max-tokens", type=int, default=0)
+    ap.add_argument("--extra-body", default="")
+    ap.add_argument("--skip-mixed", action="store_true")
+    ap.add_argument("--out", required=True)
+    args = ap.parse_args()
+    args.latency = False
+    settings = R.make_settings(args)
+    if args.extra_body:
+        R.EXTRA_BODY.update(json.loads(args.extra_body))
+    os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True)
+    print(
+        f"== load {args.label} model={args.model} base={settings.llm_base_url}",
+        file=sys.stderr,
+    )
+    with open(args.out, "a", encoding="utf-8") as fh:
+        await R.run_followup(settings, FOLLOWUP_CASES[0], -1, args.label)  # 워밍업
+        for n in [int(x) for x in args.concurrency.split(",")]:
+            await level(
+                settings, args.label, n, max(args.requests_per_level, n * 2), fh
+            )
+        if not args.skip_mixed:
+            await mixed(settings, args.label, fh)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

ai/scripts/llm_eval/run_eval.py

@@ -66,6 +66,34 @@ async def on_llm_end(self, response, **kwargs: Any) -> None:  # noqa: ANN001
 
 
 EXTRA_BODY: dict[str, Any] = {}
+# --constrain-json: 질문 풀·코칭 체인에 JSON 스키마 강제(response_format json_schema) 적용
+CONSTRAIN_JSON = {"on": False}
+
+
+def _json_schema_body(model_cls: Any, name: str) -> dict[str, Any]:
+    return {
+        "response_format": {
+            "type": "json_schema",
+            "json_schema": {
+                "name": name,
+                "schema": model_cls.model_json_schema(by_alias=True),
+            },
+        }
+    }
+
+
+def _apply_body(node: Any, body: dict[str, Any]) -> None:
+    from langchain_openai import ChatOpenAI
+
+    if isinstance(node, ChatOpenAI):
+        node.extra_body = {**(node.extra_body or {}), **body}
+        return
+    for attr in ("steps", "first", "middle", "last", "bound"):
+        child = getattr(node, attr, None)
+        if child is None:
+            continue
+        for c in child if isinstance(child, (list, tuple)) else [child]:
+            _apply_body(c, body)
 
 
 def _apply_extra_body(node: Any) -> None:
@@ -169,6 +197,10 @@ async def run_questions(settings: Settings, case: dict, rep: int, label: str) ->
     cap = UsageCapture()
     base = build_question_generation_chain(settings)
     _apply_extra_body(base)
+    if CONSTRAIN_JSON["on"]:
+        from ai_server.chain.question_generation_chain import GeneratedQuestionPool
+
+        _apply_body(base, _json_schema_body(GeneratedQuestionPool, "question_pool"))
     chain = base.with_config(callbacks=[cap])
     gen = LlmQuestionGenerator(chain)
     t0 = time.perf_counter()
@@ -201,6 +233,10 @@ async def run_coaching(
     cap = UsageCapture()
     base = build_answer_coaching_chain(settings)
     _apply_extra_body(base)
+    if CONSTRAIN_JSON["on"]:
+        from ai_server.chain.feedback_generation_chain import CoachingResult
+
+        _apply_body(base, _json_schema_body(CoachingResult, "coaching"))
     chain = base.with_config(callbacks=[cap])
     coach = LlmAnswerCoach(chain)
     t0 = time.perf_counter()
@@ -299,13 +335,17 @@ async def main() -> int:
     ap.add_argument("--timeout", type=float, default=240.0)
     ap.add_argument("--flash-max-tokens", type=int, default=0)
     ap.add_argument("--latency", action="store_true")
+    ap.add_argument(
+        "--constrain-json", action="store_true", help="질문 풀·코칭에 JSON 스키마 강제"
+    )
     ap.add_argument(
         "--extra-body", default="", help='JSON, 예: \'{"reasoning_effort": "none"}\''
     )
     ap.add_argument("--out", required=True)
     args = ap.parse_args()
 
     settings = make_settings(args)
+    CONSTRAIN_JSON["on"] = args.constrain_json
     if args.extra_body:
         EXTRA_BODY.update(json.loads(args.extra_body))
     os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True)

docs/README.md

@@ -30,6 +30,7 @@
 
 ### 조사·의사결정 기록
 - [`research/llm-provider-evaluation-2026-09.md`](./research/llm-provider-evaluation-2026-09.md) — 로컬 LLM·오픈 모델 대안 실측 비교 (품질 블라인드 채점·지연·비용)
+- [`research/local-llm-deep-dive-2026-09/local-llm-deep-dive.md`](./research/local-llm-deep-dive-2026-09/local-llm-deep-dive.md) — 로컬 LLM 전환 심층 조사 (서빙 최적화·동시성·중형 MoE 품질·하드웨어·비용, PDF 동봉)
 
 ### 협업
 - [`coding-conventions.md`](./coding-conventions.md) — 언어별 공통 코딩 규약

docs/research/llm-provider-evaluation-2026-09.md

@@ -1,6 +1,7 @@
 # LLM 제공자 검토 — 로컬 LLM · 오픈 모델 대안 (2026-09-17)
 
 > 회의(2026-09-21) 안건용. 실험 하네스: [`ai/scripts/llm_eval/`](../../ai/scripts/llm_eval/README.md)
+> 로컬 LLM 전환 심층 조사(병렬 서빙·중형 MoE·하드웨어·비용): [`local-llm-deep-dive-2026-09/local-llm-deep-dive.md`](./local-llm-deep-dive-2026-09/local-llm-deep-dive.md)
 
 ## 1. 결론