o2o-negosium-original/lps/loadtest/monitor.py
민헌 89fe2c9805 feat(lps): 실시간 관측 대시보드 추가 — 큐 추이·코어별 CPU·프로세스 그룹 사용률
loadtest/monitor.py (:9700, 단일 파일·외부 인프라 없음) + run_monitor.sh(대화형).
e2e 부하(N=100 loadtest.py) 중 "프로세스가 잘 진행되는지, 코어가 전부 도는지,
병목이 어느 층인지"를 브라우저에서 2초 간격으로 본다:

- 큐 추이: /v1/lps/ops 폴링 — PENDING/RUNNING/DONE/DEAD 라인 + 처리량(개/분) 타일
- 프로세스 그룹 CPU: worker/api/chrome/postgres — 병목 층 판독
  (chrome 은 워커 자손만 집계해 사용자 브라우저와 분리, uvicorn spawn 자식은 부모로 api 귀속)
- 코어별 사용률 막대: 멀티코어 활용 확인
- 현재 스냅샷 표 + 호버 툴팁 + 라인 끝 직접 라벨(dataviz 팔레트 검증 통과, 다크 서피스)
- psutil 의존성 추가(로컬 관측 전용)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-10 09:50:14 +09:00

355 lines
18 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""LPS 실시간 모니터 — 부하/e2e 실행 중 큐 진행·CPU·병목을 브라우저에서 본다.
무엇을 보여주나 (2초 샘플링, 브라우저 :9700)
· 큐 흐름 : /v1/lps/ops 폴링 → PENDING/RUNNING/DONE/DEAD 추이 + 처리량(상품/분)
· CPU 코어 : 코어별 사용률 막대 — 멀티코어가 전부 도는지
· 프로세스 : worker/api/chrome/postgres 그룹별 CPU·메모리 — 병목이 어느 층인지
· 병목 판독 : RUNNING=동시성인데 코어가 놀면 I/O 바운드(크롤 대기=정상 병목),
chrome CPU 가 치솟으면 렌더 병목, postgres 가 치솟으면 DB 병목
실행 (Grafana 대체가 아니라 로컬 1회 측정용 — 외부 인프라 없음)
./run_monitor.sh # 대화형
BASE=http://localhost:9600 MONITOR_PORT=9700 python loadtest/monitor.py
"""
import asyncio
import os
import time
from collections import deque
from contextlib import asynccontextmanager
import httpx
import psutil
import uvicorn
from fastapi import FastAPI
from fastapi.responses import HTMLResponse, JSONResponse
BASE = os.environ.get("BASE", "http://localhost:9600")
PORT = int(os.environ.get("MONITOR_PORT", "9700"))
INTERVAL = float(os.environ.get("MONITOR_INTERVAL", "2"))
MAX_SAMPLES = 1800 # 2s × 1800 = 1시간 링버퍼
GROUPS = ("worker", "api", "chrome", "postgres")
SAMPLES: deque = deque(maxlen=MAX_SAMPLES)
_proc_cache: dict[int, psutil.Process] = {} # cpu_percent 는 이전 호출과의 간격으로 계산 → 객체 재사용 필수
def _classify_processes() -> dict[int, str]:
"""pid → 그룹. python 은 cmdline 으로 판별. uvicorn 멀티프로세스 자식(spawn, cmdline 에
파일명 없음)은 부모가 api 면 api 로. chrome 은 사용자의 브라우저와 섞이지 않게
**조상 체인에 워커가 있는 것(크롤 Chromium)만** 집계한다."""
cls: dict[int, str] = {}
api_parents: set[int] = set()
worker_pids: set[int] = set()
pythons: list[tuple[int, int]] = [] # (pid, ppid) — 2차 패스(부모 귀속)용
chromes: list[int] = []
parent_of: dict[int, int] = {}
for p in psutil.process_iter(attrs=["pid", "ppid", "name"]):
try:
pid, name = p.info["pid"], (p.info["name"] or "").lower()
parent_of[pid] = p.info["ppid"] or 0
if "postgres" in name:
cls[pid] = "postgres"
elif "chrom" in name: # chrome / chromium / helpers — 귀속은 2차 패스에서
chromes.append(pid)
elif "python" in name:
cmd = " ".join(p.cmdline())
if "worker_main.py" in cmd:
cls[pid] = "worker"
worker_pids.add(pid)
elif "web_main.py" in cmd or "router.router" in cmd:
cls[pid] = "api"
api_parents.add(pid)
elif "monitor.py" not in cmd and "locust" not in cmd:
pythons.append((pid, p.info["ppid"]))
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
continue
for pid, ppid in pythons:
if ppid in api_parents:
cls[pid] = "api"
for pid in chromes: # 워커(patchright→node→chromium)의 자손만 크롤 브라우저
cur, hops = pid, 0
while cur and hops < 12:
if cur in worker_pids:
cls[pid] = "chrome"
break
cur, hops = parent_of.get(cur, 0), hops + 1
return cls
def _sample_cpu() -> dict:
cores = psutil.cpu_percent(percpu=True)
groups = {g: {"cpu": 0.0, "mem": 0, "n": 0} for g in GROUPS}
cls = _classify_processes()
for pid, g in cls.items():
try:
proc = _proc_cache.get(pid)
if proc is None:
proc = _proc_cache[pid] = psutil.Process(pid)
groups[g]["cpu"] += proc.cpu_percent(None) # 코어 1개=100 기준(멀티코어면 100 초과 가능)
groups[g]["mem"] += proc.memory_info().rss
groups[g]["n"] += 1
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
continue
for pid in [pid for pid in _proc_cache if pid not in cls]: # 죽은 pid 정리
_proc_cache.pop(pid, None)
for g in GROUPS:
groups[g]["cpu"] = round(groups[g]["cpu"], 1)
groups[g]["mem"] = int(groups[g]["mem"] / 1048576) # MB
return {"cores": [round(c, 1) for c in cores], "groups": groups}
async def _sampler():
psutil.cpu_percent(percpu=True) # priming — 첫 유효 샘플부터 의미 있는 값
async with httpx.AsyncClient(timeout=3) as c:
while True:
snap = {"t": round(time.time(), 1), **_sample_cpu(), "ops": None}
try:
r = await c.get(f"{BASE}/v1/lps/ops")
if r.status_code == 200:
snap["ops"] = r.json()
except Exception:
pass # API 죽어 있어도 CPU 샘플은 계속
SAMPLES.append(snap)
await asyncio.sleep(INTERVAL)
@asynccontextmanager
async def _lifespan(app):
task = asyncio.create_task(_sampler())
yield
task.cancel()
app = FastAPI(lifespan=_lifespan)
@app.get("/api/series")
async def series(since: float = 0.0):
return JSONResponse({
"base": BASE, "interval": INTERVAL, "ncores": psutil.cpu_count(),
"samples": [s for s in SAMPLES if s["t"] > since],
})
@app.get("/")
async def index():
return HTMLResponse(PAGE)
# ──────────────────────────────────────────────────────────────────────────────
# 대시보드(단일 페이지, 외부 의존 없음). 다크 고정 — 로컬 관측 도구.
# 팔레트는 dataviz 검증 통과값(다크 서피스 #1a1a19 기준, 라인 끝 직접 라벨로 보조 인코딩).
PAGE = r"""<!doctype html>
<html lang="ko"><head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
<title>LPS 모니터</title>
<style>
:root {
--page:#0d0d0d; --surface:#1a1a19; --ink:#ffffff; --ink2:#c3c2b7; --muted:#898781;
--grid:#2c2c2a; --axis:#383835; --border:rgba(255,255,255,.10);
/* 큐 시리즈 */ --q-pending:#3987e5; --q-running:#c98500; --q-done:#199e70; --q-dead:#e66767;
/* 프로세스 그룹 */ --g-worker:#9085e9; --g-api:#008300; --g-chrome:#d95926; --g-postgres:#d55181;
--core:#3987e5;
}
* { box-sizing:border-box; margin:0 }
body { background:var(--page); color:var(--ink); font:14px/1.45 system-ui,-apple-system,"Segoe UI",sans-serif; padding:20px }
h1 { font-size:16px; font-weight:600 }
.sub { color:var(--muted); font-size:12px; margin:2px 0 16px }
.tiles { display:grid; grid-template-columns:repeat(auto-fit,minmax(120px,1fr)); gap:10px; margin-bottom:16px }
.tile { background:var(--surface); border:1px solid var(--border); border-radius:10px; padding:10px 12px }
.tile .k { color:var(--muted); font-size:11px }
.tile .v { font-size:22px; font-weight:600; margin-top:2px }
.tile .v small { font-size:12px; color:var(--ink2); font-weight:400 }
.cards { display:grid; grid-template-columns:1fr 1fr; gap:14px }
@media (max-width:1000px){ .cards { grid-template-columns:1fr } }
.card { background:var(--surface); border:1px solid var(--border); border-radius:10px; padding:14px }
.card h2 { font-size:13px; font-weight:600; color:var(--ink2); margin-bottom:2px }
.card .desc { color:var(--muted); font-size:11px; margin-bottom:8px }
.legend { display:flex; gap:14px; flex-wrap:wrap; font-size:12px; color:var(--ink2); margin-bottom:6px }
.legend i { display:inline-block; width:10px; height:10px; border-radius:3px; margin-right:5px; vertical-align:-1px }
.plot { position:relative }
canvas { width:100%; height:220px; display:block }
#cores-canvas { height:150px }
.tip { position:absolute; pointer-events:none; background:#242423; border:1px solid var(--border);
border-radius:8px; padding:8px 10px; font-size:12px; color:var(--ink2); display:none; z-index:5;
white-space:nowrap; box-shadow:0 4px 14px rgba(0,0,0,.4) }
.tip b { color:var(--ink); font-variant-numeric:tabular-nums }
table { width:100%; border-collapse:collapse; font-size:12px; margin-top:4px }
th { text-align:left; color:var(--muted); font-weight:500; padding:4px 8px; border-bottom:1px solid var(--axis) }
td { padding:4px 8px; color:var(--ink2); border-bottom:1px solid var(--grid); font-variant-numeric:tabular-nums }
td:first-child { color:var(--ink) }
.dot { display:inline-block; width:8px; height:8px; border-radius:50%; margin-right:6px }
.ok { background:#0ca30c } .bad { background:#d03b3b }
</style></head>
<body>
<h1>LPS 실시간 모니터</h1>
<div class="sub"><span id="conn" class="dot bad"></span>대상 <span id="base">…</span> · 2초 샘플링 · 최근 1시간 유지</div>
<div class="tiles" id="tiles"></div>
<div class="cards">
<div class="card">
<h2>큐 추이</h2><div class="desc">/v1/lps/ops — 잡 상태별 개수. DONE 이 계단처럼 오르면 정상 소화 중</div>
<div class="legend" id="lg-queue"></div>
<div class="plot"><canvas id="queue-canvas"></canvas><div class="tip" id="queue-tip"></div></div>
</div>
<div class="card">
<h2>프로세스 그룹 CPU</h2><div class="desc">코어 1개=100% 기준(멀티코어면 100 초과). 어느 층이 바쁜지 = 병목 후보</div>
<div class="legend" id="lg-cpu"></div>
<div class="plot"><canvas id="cpu-canvas"></canvas><div class="tip" id="cpu-tip"></div></div>
</div>
<div class="card">
<h2>코어별 사용률 (현재)</h2><div class="desc">막대가 고르게 차면 멀티코어 활용 중, 1~2개만 차면 단일 코어 병목</div>
<div class="plot"><canvas id="cores-canvas"></canvas><div class="tip" id="cores-tip"></div></div>
</div>
<div class="card">
<h2>현재 스냅샷 (표)</h2><div class="desc">그래프와 같은 데이터의 수치 뷰</div>
<table id="snap-table"><tbody></tbody></table>
</div>
</div>
<script>
const css = n => getComputedStyle(document.documentElement).getPropertyValue(n).trim();
const QUEUE = [ ["PENDING","--q-pending"], ["RUNNING","--q-running"], ["DONE","--q-done"], ["DEAD","--q-dead"] ];
const CPUG = [ ["worker","--g-worker"], ["api","--g-api"], ["chrome","--g-chrome"], ["postgres","--g-postgres"] ];
let samples = [], last = 0, ncores = 0;
function legend(el, defs){ el.innerHTML = defs.map(([n,v]) => `<span><i style="background:${css(v)}"></i>${n}</span>`).join(""); }
legend(document.getElementById("lg-queue"), QUEUE);
legend(document.getElementById("lg-cpu"), CPUG);
function fit(cv){ // DPR 스케일
const r = cv.getBoundingClientRect(), d = devicePixelRatio || 1;
if (cv.width !== r.width*d) { cv.width = r.width*d; cv.height = r.height*d; }
const ctx = cv.getContext("2d"); ctx.setTransform(d,0,0,d,0,0); return [ctx, r.width, r.height];
}
// ── 시계열 라인 차트(직접 라벨 + 크로스헤어 툴팁) ──────────────────────────
function lineChart(cvId, tipId, seriesDefs, getVal, fmt){
const cv = document.getElementById(cvId), tip = document.getElementById(tipId);
const PADL = 44, PADR = 78, PADT = 8, PADB = 20;
let hoverX = null;
function draw(){
const [ctx,W,H] = fit(cv); ctx.clearRect(0,0,W,H);
const pts = samples; if (pts.length < 2) return;
const t0 = pts[0].t, t1 = pts[pts.length-1].t;
let vmax = 1;
for (const s of pts) for (const [name] of seriesDefs){ const v = getVal(s,name); if (v != null && v > vmax) vmax = v; }
vmax *= 1.08;
const X = t => PADL + (W-PADL-PADR) * (t-t0) / Math.max(1,(t1-t0));
const Y = v => PADT + (H-PADT-PADB) * (1 - v/vmax);
ctx.strokeStyle = css("--grid"); ctx.lineWidth = 1; ctx.fillStyle = css("--muted"); ctx.font = "10px system-ui";
let prevLabel = null;
for (let i=0;i<=3;i++){ const v = vmax*i/3, y = Y(v), label = fmt(v, vmax);
ctx.beginPath(); ctx.moveTo(PADL,y); ctx.lineTo(W-PADR,y); ctx.stroke();
if (label !== prevLabel){ ctx.textAlign="right"; ctx.fillText(label, PADL-6, y+3); prevLabel = label; } }
ctx.textAlign="center";
for (let i=0;i<=3;i++){ const t = t0+(t1-t0)*i/3;
ctx.fillText(new Date(t*1000).toTimeString().slice(0,8), X(t), H-6); }
ctx.strokeStyle = css("--axis"); ctx.beginPath(); ctx.moveTo(PADL,Y(0)); ctx.lineTo(W-PADR,Y(0)); ctx.stroke();
const ends = []; // 라인 끝 직접 라벨(색 단독 의존 방지) — 겹치면 아래로 밀어 12px 간격 확보
for (const [name,varName] of seriesDefs){
ctx.strokeStyle = css(varName); ctx.lineWidth = 2; ctx.beginPath(); let started=false, lastY=null;
for (const s of pts){ const v = getVal(s,name); if (v==null) continue;
const x=X(s.t), y=Y(v); started ? ctx.lineTo(x,y) : ctx.moveTo(x,y); started=true; lastY=y; }
ctx.stroke();
if (lastY != null) ends.push({ name, y: lastY });
}
ends.sort((a,b) => a.y - b.y);
for (let i=1;i<ends.length;i++) ends[i].y = Math.max(ends[i].y, ends[i-1].y + 12);
if (ends.length){ // 바닥을 넘치면 위로 되밀기(겹침 방지 유지)
ends[ends.length-1].y = Math.min(ends[ends.length-1].y, H-PADB-2);
for (let i=ends.length-2;i>=0;i--) ends[i].y = Math.min(ends[i].y, ends[i+1].y - 12);
}
ctx.fillStyle = css("--ink2"); ctx.textAlign="left"; ctx.font="11px system-ui";
for (const e of ends) ctx.fillText(e.name, W-PADR+6, e.y+3);
if (hoverX != null){
let best=null, bd=1e9;
for (const s of pts){ const d = Math.abs(X(s.t)-hoverX); if (d<bd){bd=d;best=s;} }
if (best){ const x = X(best.t);
ctx.strokeStyle = css("--axis"); ctx.setLineDash([3,3]); ctx.beginPath();
ctx.moveTo(x,PADT); ctx.lineTo(x,H-PADB); ctx.stroke(); ctx.setLineDash([]);
tip.style.display="block";
tip.innerHTML = new Date(best.t*1000).toTimeString().slice(0,8) + "<br>" +
seriesDefs.map(([n,v]) => `<i class="dot" style="background:${css(v)}"></i>${n} <b>${fmt(getVal(best,n) ?? 0)}</b>`).join("<br>");
const r = cv.getBoundingClientRect();
tip.style.left = Math.min(x+12, r.width-tip.offsetWidth-4) + "px"; tip.style.top = "10px";
}
} else tip.style.display="none";
}
cv.addEventListener("mousemove", e => { hoverX = e.offsetX; draw(); });
cv.addEventListener("mouseleave", () => { hoverX = null; draw(); });
return draw;
}
const drawQueue = lineChart("queue-canvas","queue-tip", QUEUE,
(s,n) => s.ops ? s.ops[n.toLowerCase()] : null, v => Math.round(v));
const drawCpu = lineChart("cpu-canvas","cpu-tip", CPUG,
(s,n) => s.groups?.[n]?.cpu, (v, vmax) => (vmax ?? 100) < 10 ? v.toFixed(1)+"%" : Math.round(v)+"%");
// ── 코어별 막대(단일 시리즈 → 범례 없음, 값 직접 라벨) ─────────────────────
function drawCores(){
const cv = document.getElementById("cores-canvas");
const [ctx,W,H] = fit(cv); ctx.clearRect(0,0,W,H);
const s = samples[samples.length-1]; if (!s) return;
const cores = s.cores, n = cores.length, PADB = 18, PADT = 14;
const bw = Math.min(46, (W-16)/n - 6);
const X = i => 8 + i*( (W-16)/n ) + ((W-16)/n - bw)/2;
ctx.strokeStyle = css("--axis"); ctx.beginPath(); ctx.moveTo(4,H-PADB); ctx.lineTo(W-4,H-PADB); ctx.stroke();
cores.forEach((v,i) => {
const h = Math.max(2,(H-PADT-PADB) * v/100), x = X(i), y = H-PADB-h;
ctx.fillStyle = css("--core"); ctx.beginPath();
ctx.roundRect(x, y, bw, h, [4,4,0,0]); ctx.fill();
ctx.fillStyle = css("--muted"); ctx.font="10px system-ui"; ctx.textAlign="center";
ctx.fillText("c"+i, x+bw/2, H-5);
ctx.fillStyle = css("--ink2"); ctx.fillText(Math.round(v), x+bw/2, y-4);
});
}
// ── 타일 + 표 ────────────────────────────────────────────────────────────────
function tile(k,v,sub){ return `<div class="tile"><div class="k">${k}</div><div class="v">${v}${sub?` <small>${sub}</small>`:""}</div></div>`; }
function throughput(){ // 최근 60초 ΔDONE → 상품/분
const now = samples[samples.length-1], past = [...samples].reverse().find(s => s.ops && now.t - s.t >= 60);
if (!now?.ops || !past?.ops) return "–";
const d = now.ops.done - past.ops.done, dt = now.t - past.t;
return dt > 0 ? (d*60/dt).toFixed(1) : "–";
}
function render(){
const s = samples[samples.length-1]; if (!s) return;
const o = s.ops, totalCpu = s.cores.reduce((a,b)=>a+b,0) / s.cores.length;
document.getElementById("conn").className = "dot " + (o ? "ok" : "bad");
document.getElementById("tiles").innerHTML =
tile("DONE", o ? o.done : "–") + tile("처리량", throughput(), "개/분") +
tile("RUNNING", o ? o.running : "–") + tile("PENDING", o ? o.pending : "–") +
tile("큐 지연", o ? o.oldest_pending_sec : "–", "s") + tile("DEAD", o ? o.dead : "–") +
tile("차단(1h)", o ? o.blocks_1h : "–") + tile("CPU 평균", totalCpu.toFixed(0), "%");
const rows = [];
for (const [g] of CPUG){ const d = s.groups[g];
rows.push(`<tr><td><i class="dot" style="background:${css(CPUG.find(x=>x[0]===g)[1])}"></i>${g}</td><td>${d.cpu}%</td><td>${d.mem} MB</td><td>${d.n}개</td></tr>`); }
if (o) rows.push(`<tr><td>ops</td><td colspan="3">stuck_running ${o.stuck_running} · dead_1h ${o.dead_1h} · blocks_1h ${o.blocks_1h}</td></tr>`);
document.getElementById("snap-table").innerHTML =
"<tr><th>그룹</th><th>CPU</th><th>MEM</th><th>프로세스</th></tr>" + rows.join("");
drawQueue(); drawCpu(); drawCores();
}
async function poll(){
try {
const r = await (await fetch(`/api/series?since=${last}`)).json();
document.getElementById("base").textContent = r.base; ncores = r.ncores;
if (r.samples.length){ samples.push(...r.samples); last = samples[samples.length-1].t; }
const cut = (samples[samples.length-1]?.t ?? 0) - 3600;
while (samples.length && samples[0].t < cut) samples.shift();
render();
} catch (e) { document.getElementById("conn").className = "dot bad"; }
}
poll(); setInterval(poll, 2000);
addEventListener("resize", render);
</script>
</body></html>
"""
if __name__ == "__main__":
print(f"■ LPS 모니터: http://localhost:{PORT} (대상 API {BASE}, {INTERVAL:.0f}s 샘플링)")
uvicorn.run(app, host="0.0.0.0", port=PORT, log_level="warning")