277 lines
12 KiB
Python
277 lines
12 KiB
Python
"""⑦ render — i2v 클립을 1080x1920 배포본으로 합성한다.
|
|
|
|
QR은 프롬프트로 못 살린다. 원본 3071px → 출력 1248px 축소만으로도 데이터 모듈이 뭉개져
|
|
스캔이 안 되므로, 원본에서 오려 다시 얹는다. 기관 로고바도 같은 이유·같은 방법이고,
|
|
포스터 영역 안쪽에 뭉갠 채 남은 것들도 제자리에 덮는다.
|
|
|
|
루프는 핑퐁이 아니라 크로스페이드다. 생성형 불꽃은 터짐→소멸이라는 방향이 있는 사건이라
|
|
역재생이 눈에 보인다.
|
|
"""
|
|
import io
|
|
from collections import deque
|
|
from collections.abc import Iterator
|
|
from dataclasses import dataclass
|
|
from itertools import chain
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
from PIL import Image
|
|
|
|
from models.render import RenderResult
|
|
from models.tts import NarrationTimeline
|
|
from utils.audio import SAMPLE_RATE, apply_limiter, decode_audio, decode_mp3
|
|
from utils.image import bar_box, content_box, logo_bar, round_mask
|
|
from utils.qr import qr_box, qr_card
|
|
from utils.video import decode_frames, encode_mp4
|
|
|
|
OUT_W, OUT_H = 1080, 1920
|
|
FPS = 24
|
|
POSTER_W = 1000 # 9:16 안에서 포스터가 차지할 폭 (좌우 40px 여백)
|
|
TOP_PAD = 86
|
|
LOOP_OVERLAP = 16 # 겹침 프레임. 소재가 잔잔할수록 키워야 이음매가 죽는다
|
|
MIN_LOOP_SECONDS = 3
|
|
|
|
NARRATION_VOL, SFX_VOL, BGM_VOL = 1.45, 0.35, 0.20
|
|
BGM_FADE_IN = 1.0
|
|
BGM_FADE_OUT = 1.5
|
|
TAIL_FADE = 0.125
|
|
|
|
QR_DRIFT_RATIO = 0.04 # plate에서 찾은 QR이 이보다 어긋나면 패치를 포기한다
|
|
QR_SEARCH_PAD = 0.06
|
|
BAND_QR_MAX_H = 190
|
|
BAND_SIDE_PAD = 74
|
|
BAND_GAP = 60
|
|
|
|
|
|
@dataclass
|
|
class Layout:
|
|
"""0번 프레임에서 한 번 구해두고 모든 프레임에 그대로 적용한다."""
|
|
plate: tuple[int, int, int, int]
|
|
poster_size: tuple[int, int]
|
|
poster_top: int
|
|
background: tuple[int, ...]
|
|
band_y: int
|
|
band_h: int
|
|
bar_patch: tuple[Image.Image, tuple[int, int]] | None = None
|
|
qr_patch: tuple[Image.Image, tuple[int, int], Image.Image] | None = None
|
|
band_qr: tuple[Image.Image, tuple[int, int]] | None = None
|
|
band_bar: tuple[Image.Image, tuple[int, int]] | None = None
|
|
|
|
|
|
def norm_to_out(nx: float, ny: float, source: tuple[int, int], out_w: int) -> tuple[int, int]:
|
|
"""원본 정규좌표 → 출력 포스터 이미지 안의 픽셀 좌표."""
|
|
source_w, source_h = source
|
|
return round(nx * out_w), round(ny * out_w * source_h / source_w)
|
|
|
|
|
|
def band_colour(frame: Image.Image, plate: tuple[int, int, int, int]) -> tuple[int, ...]:
|
|
"""포스터 상단에서 뽑은 색은 채도가 높아 포스터와 경쟁하므로 밝기만 죽인다."""
|
|
left, top, right, bottom = plate
|
|
pixels = np.asarray(frame.convert("RGB")).astype(np.float32)
|
|
median = np.median(pixels[top:top + (bottom - top) // 5, left:right].reshape(-1, 3), axis=0)
|
|
return tuple(int(value * 0.34) for value in median)
|
|
|
|
|
|
def build_layout(frame: Image.Image, poster: Image.Image) -> Layout:
|
|
left, top, right, bottom = content_box(frame)
|
|
plate_w = right - left
|
|
source_w, source_h = poster.size
|
|
full_h = plate_w * source_h / source_w
|
|
|
|
source_bar = bar_box(poster)
|
|
source_qr = qr_box(poster)
|
|
has_band = bool(source_qr or source_bar)
|
|
|
|
if has_band:
|
|
poster_w = POSTER_W
|
|
poster_h = round(poster_w * (bottom - top) / plate_w)
|
|
poster_top = TOP_PAD
|
|
if TOP_PAD + poster_h >= OUT_H - 80:
|
|
raise RuntimeError(f"포스터가 너무 길다(밴드 {OUT_H - TOP_PAD - poster_h}px)")
|
|
else:
|
|
# 얹을 게 없으면 밴드는 존재 이유가 없다 — 포스터를 폭까지 키우고 상하 균등 여백
|
|
poster_w = OUT_W
|
|
poster_h = round(poster_w * (bottom - top) / plate_w)
|
|
if poster_h > OUT_H - 2 * TOP_PAD:
|
|
poster_h = OUT_H - 2 * TOP_PAD
|
|
poster_w = round(poster_h * plate_w / (bottom - top))
|
|
poster_top = (OUT_H - poster_h) // 2
|
|
|
|
layout = Layout(plate=(left, top, right, bottom), poster_size=(poster_w, poster_h),
|
|
poster_top=poster_top, background=band_colour(frame, (left, top, right, bottom)),
|
|
band_y=poster_top + poster_h, band_h=OUT_H - poster_top - poster_h)
|
|
|
|
if source_bar:
|
|
_, bar_y0 = norm_to_out(0.0, source_bar[1] / source_h, poster.size, poster_w)
|
|
_, bar_y1 = norm_to_out(0.0, source_bar[3] / source_h, poster.size, poster_w)
|
|
if 0 < bar_y0 < poster_h:
|
|
layout.bar_patch = (poster.crop(source_bar).resize((poster_w, bar_y1 - bar_y0),
|
|
Image.LANCZOS), (0, bar_y0))
|
|
|
|
if source_qr:
|
|
layout.qr_patch = inner_qr_patch(frame, poster, source_qr, layout, full_h)
|
|
|
|
if has_band:
|
|
layout.band_qr, layout.band_bar = band_assets(poster, layout)
|
|
return layout
|
|
|
|
|
|
def inner_qr_patch(frame: Image.Image, poster: Image.Image, source_qr: tuple[int, int, int, int],
|
|
layout: Layout, full_h: float):
|
|
"""포스터 안쪽 QR은 잘라낼 수 없으므로(그림 위에 얹혀 있다) 원본에서 오려 제자리에 덮는다."""
|
|
left, top, right, _ = layout.plate
|
|
plate_w = right - left
|
|
source_w, source_h = poster.size
|
|
poster_w, poster_h = layout.poster_size
|
|
x0, y0 = norm_to_out(source_qr[0] / source_w, source_qr[1] / source_h, poster.size, poster_w)
|
|
x1, y1 = norm_to_out(source_qr[2] / source_w, source_qr[3] / source_h, poster.size, poster_w)
|
|
|
|
# 네 귀퉁이를 다 뒤지면 엉뚱한 흰 덩어리를 찾았다고 오인해 drift 검사가 무의미해진다
|
|
window = (max(0.0, source_qr[0] / source_w - QR_SEARCH_PAD),
|
|
max(0.0, source_qr[1] / source_h - QR_SEARCH_PAD),
|
|
min(1.0, source_qr[2] / source_w + QR_SEARCH_PAD),
|
|
min(1.0, source_qr[3] / source_h + QR_SEARCH_PAD))
|
|
seen = qr_box(frame.crop((left, top, right, top + round(full_h))), strict=False, window=window)
|
|
if seen is None:
|
|
return None
|
|
drift = float(np.hypot(
|
|
(seen[0] + seen[2]) / 2 - (source_qr[0] + source_qr[2]) / 2 / source_w * plate_w,
|
|
(seen[1] + seen[3]) / 2 - (source_qr[1] + source_qr[3]) / 2 / source_h * full_h))
|
|
if drift > QR_DRIFT_RATIO * plate_w or y1 >= poster_h:
|
|
# 엉뚱한 자리에 QR을 찍는 것보다 안 찍는 게 낫다
|
|
return None
|
|
card = qr_card(poster, pad=0)
|
|
if card is None:
|
|
return None
|
|
patch = card.resize((x1 - x0, y1 - y0), Image.LANCZOS)
|
|
return patch, (x0, y0), round_mask(patch.size)
|
|
|
|
|
|
def band_assets(poster: Image.Image, layout: Layout):
|
|
"""밴드에 키워서 얹을 QR과 로고바. 밴드의 큰 QR이 스캔용이다."""
|
|
card = qr_card(poster)
|
|
band_qr = None
|
|
next_x = BAND_SIDE_PAD
|
|
if card:
|
|
height = min(BAND_QR_MAX_H, layout.band_h - 40)
|
|
resized = card.resize((round(card.width * height / card.height), height), Image.LANCZOS)
|
|
band_qr = (resized, (BAND_SIDE_PAD,
|
|
layout.band_y + (layout.band_h - resized.height) // 2))
|
|
next_x += resized.width + BAND_GAP
|
|
|
|
band_bar = None
|
|
bar = logo_bar(poster)
|
|
if bar:
|
|
width = OUT_W - next_x - BAND_SIDE_PAD
|
|
resized = bar.resize((width, round(bar.height * width / bar.width)), Image.LANCZOS)
|
|
band_bar = (resized, (next_x, layout.band_y + (layout.band_h - resized.height) // 2))
|
|
return band_qr, band_bar
|
|
|
|
|
|
def composite(frame: Image.Image, layout: Layout) -> Image.Image:
|
|
poster_area = frame.convert("RGB").crop(layout.plate).resize(layout.poster_size, Image.LANCZOS)
|
|
if layout.bar_patch:
|
|
poster_area.paste(*layout.bar_patch)
|
|
if layout.qr_patch:
|
|
patch, position, mask = layout.qr_patch
|
|
poster_area.paste(patch, position, mask)
|
|
|
|
canvas = Image.new("RGB", (OUT_W, OUT_H), layout.background)
|
|
canvas.paste(poster_area, ((OUT_W - layout.poster_size[0]) // 2, layout.poster_top))
|
|
if layout.band_qr:
|
|
canvas.paste(*layout.band_qr)
|
|
if layout.band_bar:
|
|
canvas.paste(*layout.band_bar)
|
|
return canvas
|
|
|
|
|
|
def loop_frames(clip: bytes, layout: Layout, tail: list[Image.Image],
|
|
loop_length: int, overlap: int) -> Iterator[Image.Image]:
|
|
"""머리에 꼬리를 녹여 마지막 프레임이 첫 프레임으로 자연스럽게 이어지게 한다."""
|
|
for index, frame in enumerate(decode_frames(clip)):
|
|
if index >= loop_length:
|
|
return
|
|
composited = composite(frame, layout)
|
|
if index < overlap:
|
|
weight = (index + 1) / (overlap + 1)
|
|
blended = (np.asarray(composited).astype(np.float32) * weight
|
|
+ np.asarray(tail[index]).astype(np.float32) * (1 - weight))
|
|
composited = Image.fromarray(np.clip(blended, 0, 255).astype(np.uint8))
|
|
yield composited
|
|
|
|
|
|
def fade(pcm: np.ndarray, seconds: float, out: bool) -> np.ndarray:
|
|
length = min(len(pcm), round(seconds * SAMPLE_RATE))
|
|
if length <= 0:
|
|
return pcm
|
|
ramp = np.linspace(0.0, 1.0, length, dtype=np.float32)
|
|
faded = pcm.astype(np.float32)
|
|
if out:
|
|
faded[len(pcm) - length:] *= ramp[::-1]
|
|
else:
|
|
faded[:length] *= ramp
|
|
return faded
|
|
|
|
|
|
def mix_audio(clip: bytes, narration: bytes | None, bgm: bytes | None,
|
|
duration: float) -> np.ndarray:
|
|
"""나레이션 1.45 / SFX 0.35 / BGM 0.20. 리미터가 컴프레서처럼 작동해 목소리가 또렷해진다."""
|
|
total = round(duration * SAMPLE_RATE)
|
|
track = np.zeros(total, dtype=np.float32)
|
|
|
|
sfx = decode_audio(clip)
|
|
if sfx is not None:
|
|
sfx = sfx[:total]
|
|
track[:len(sfx)] += sfx.astype(np.float32) * SFX_VOL
|
|
if narration:
|
|
voice = decode_mp3(narration)[:total]
|
|
track[:len(voice)] += voice.astype(np.float32) * NARRATION_VOL
|
|
if bgm:
|
|
music = decode_mp3(bgm)
|
|
if len(music) < total:
|
|
music = np.tile(music, total // len(music) + 1)
|
|
music = fade(fade(music[:total].astype(np.float32) * BGM_VOL, BGM_FADE_IN, out=False),
|
|
BGM_FADE_OUT, out=True)
|
|
track += music
|
|
|
|
track = fade(track, TAIL_FADE, out=True)
|
|
return apply_limiter(np.clip(track, -32768, 32767).astype(np.int16))
|
|
|
|
|
|
def render(clip: bytes, poster: Path | Image.Image, *,
|
|
narration: bytes | None = None, bgm: bytes | None = None,
|
|
timeline: NarrationTimeline | None = None,
|
|
overlap: int = LOOP_OVERLAP) -> RenderResult:
|
|
source = (Image.open(poster) if isinstance(poster, Path) else poster).convert("RGB")
|
|
|
|
# 1회차 — 레이아웃을 잡고 꼬리 프레임만 남긴다. 전부 들고 있으면 1GB를 넘는다.
|
|
layout, tail_frames, frame_count = None, deque(maxlen=overlap), 0
|
|
for frame in decode_frames(clip):
|
|
if layout is None:
|
|
layout = build_layout(frame, source)
|
|
tail_frames.append(frame)
|
|
frame_count += 1
|
|
if layout is None:
|
|
raise RuntimeError("클립에서 프레임을 못 뽑았다")
|
|
tail = [composite(frame, layout) for frame in tail_frames]
|
|
|
|
loop_length = frame_count - overlap
|
|
if loop_length < FPS * MIN_LOOP_SECONDS:
|
|
raise RuntimeError(f"루프 길이가 너무 짧다 ({loop_length}프레임) — overlap을 줄일 것")
|
|
duration = loop_length / FPS
|
|
|
|
if timeline and timeline.cues[-1].end > duration:
|
|
raise RuntimeError(f"나레이션이 {timeline.cues[-1].end:.2f}초로 영상 {duration:.2f}초를 "
|
|
"넘는다 — overlap을 줄이거나 나레이션을 짧게")
|
|
|
|
# 2회차 — 순서대로 합성해 인코더에 바로 밀어넣는다
|
|
frames = loop_frames(clip, layout, tail, loop_length, overlap)
|
|
first = next(frames)
|
|
thumbnail = io.BytesIO()
|
|
first.save(thumbnail, "JPEG", quality=92)
|
|
|
|
video = encode_mp4(chain([first], frames), (OUT_W, OUT_H),
|
|
mix_audio(clip, narration, bgm, duration), fps=FPS)
|
|
return RenderResult(video=video, thumbnail=thumbnail.getvalue(), duration=duration,
|
|
frames=loop_length, width=OUT_W, height=OUT_H)
|