312 lines
14 KiB
Python
312 lines
14 KiB
Python
"""⑩ hybrid — 생성 클립 아트워크 위에 원본 고정층을 다시 덮는다.
|
|
|
|
Kling 단독은 생동감이 있지만 제목대가 7~17 드리프트하고, 레이어 단독은 글자가 0px이지만
|
|
훅이 약하다. 그래서 Kling을 아트워크 층으로 쓰고 원본에서 오려낸 마스크를 위에 얹는다.
|
|
⑦ render의 QR·로고바 패치를 마스크 전체로 일반화한 것이고 추가 크레딧은 0이다.
|
|
"""
|
|
import io
|
|
import math
|
|
from collections.abc import Iterator
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
from PIL import Image, ImageDraw, ImageFilter
|
|
|
|
from models.hybrid import HybridResult, LayerConfig, Rect
|
|
from utils.video import decode_frames, encode_mp4
|
|
|
|
FPS = 24
|
|
DURATION = 8.0
|
|
HOLD_SECONDS = 0.5 # 첫 0.5초는 원본 정지 — 첫 프레임이 곧 썸네일이다
|
|
CRF = 17
|
|
|
|
FEATHER_PX = 30
|
|
SIL_DARK_MAX = 120
|
|
GLYPH_BRIGHT_MIN = 165
|
|
SWEEP_SECONDS = 0.8
|
|
SWEEP_GAIN = 120
|
|
GLOW_SECONDS = 0.9
|
|
GLOW_GAIN = 110
|
|
PROOF_COUNT = 8
|
|
PROOF_SIZE = (300, 400)
|
|
|
|
|
|
def ease_sine(t: float) -> float:
|
|
return 0.5 - 0.5 * math.cos(math.pi * min(max(t, 0.0), 1.0))
|
|
|
|
|
|
def punch_scale(t: float, peak: float) -> float:
|
|
"""1.0 → peak(0.22s) → 유지(0.12s) → 0.96(0.18s, 지나침) → 1.0. 오버슈트가 팝의 정체다."""
|
|
if t < 0 or t > 0.64:
|
|
return 1.0
|
|
if t < 0.22:
|
|
return 1.0 + (peak - 1.0) * (1 - (1 - t / 0.22) ** 3)
|
|
if t < 0.34:
|
|
return peak
|
|
if t < 0.52:
|
|
return peak + (0.96 - peak) * ease_sine((t - 0.34) / 0.18)
|
|
return 0.96 + 0.04 * ease_sine((t - 0.52) / 0.12)
|
|
|
|
|
|
def to_pixels(rect: Rect, size: tuple[int, int]) -> tuple[int, int, int, int]:
|
|
width, height = size
|
|
return (int(rect[0] * width), int(rect[1] * height),
|
|
int(rect[2] * width), int(rect[3] * height))
|
|
|
|
|
|
def feather_rect_mask(size: tuple[int, int], rect: Rect) -> Image.Image:
|
|
mask = Image.new("L", size, 0)
|
|
ImageDraw.Draw(mask).rectangle(to_pixels(rect, size), fill=255)
|
|
return mask.filter(ImageFilter.GaussianBlur(FEATHER_PX / 2))
|
|
|
|
|
|
def glyph_alpha(poster: Image.Image, rect: Rect, dark_ink: bool = True) -> Image.Image:
|
|
"""사각 안의 글자 픽셀만 알파로. 펀치 중 이웃을 원본에서 다시 얹을 때 쓴다.
|
|
|
|
dark_ink는 어두운 베벨·그림자도 글자로 본다(밝은 배경 포스터).
|
|
검정 포스터에서 켜면 배경 전체가 글자가 되어 검은 상자가 된다.
|
|
"""
|
|
width, height = poster.size
|
|
x0, y0, x1, y1 = to_pixels(rect, poster.size)
|
|
luminance = np.asarray(poster.convert("L")).astype(np.float32)
|
|
mask = np.zeros((height, width), np.uint8)
|
|
patch = luminance[y0:y1, x0:x1]
|
|
mask[y0:y1, x0:x1] = (((patch > GLYPH_BRIGHT_MIN) | (patch < 75)) * 255 if dark_ink
|
|
else (patch > 55) * 255)
|
|
return Image.fromarray(mask).filter(ImageFilter.MaxFilter(13)).filter(
|
|
ImageFilter.GaussianBlur(2))
|
|
|
|
|
|
def build_masks(poster: Image.Image, config: LayerConfig) -> dict:
|
|
width, height = poster.size
|
|
pixels = np.asarray(poster).astype(np.float32)
|
|
luminance = pixels.mean(axis=2)
|
|
|
|
fixed = np.zeros((height, width), np.float32)
|
|
for rect in config.fixed_rects:
|
|
fixed = np.maximum(fixed, np.asarray(feather_rect_mask(poster.size, rect)) / 255.0)
|
|
|
|
x0, y0, x1, y1 = to_pixels(config.sil_box, poster.size)
|
|
silhouette = np.zeros((height, width), np.uint8)
|
|
silhouette[y0:y1, x0:x1] = (pixels.max(axis=2) < SIL_DARK_MAX)[y0:y1, x0:x1] * 255
|
|
if config.sil_poly:
|
|
# 밝은 의상은 어두운 픽셀 임계로 안 잡혀서 다각형과 합친다
|
|
polygon = Image.new("L", poster.size, 0)
|
|
ImageDraw.Draw(polygon).polygon(
|
|
[(int(x * width), int(y * height)) for x, y in config.sil_poly], fill=255)
|
|
silhouette = np.maximum(silhouette, np.asarray(polygon))
|
|
silhouette = np.asarray(Image.fromarray(silhouette)
|
|
.filter(ImageFilter.MaxFilter(7))
|
|
.filter(ImageFilter.GaussianBlur(4))) / 255.0
|
|
|
|
glyphs = {}
|
|
for name, rect in config.glyphs.items():
|
|
gx0, gy0, gx1, gy1 = to_pixels(rect, poster.size)
|
|
patch = np.zeros((height, width), np.uint8)
|
|
patch[gy0:gy1, gx0:gx1] = (luminance[gy0:gy1, gx0:gx1] > GLYPH_BRIGHT_MIN) * 255
|
|
glyphs[name] = np.asarray(
|
|
Image.fromarray(patch).filter(ImageFilter.GaussianBlur(1.2))) / 255.0
|
|
|
|
glyph_union = (np.max(np.stack(list(glyphs.values())), axis=0) if glyphs
|
|
else np.zeros((height, width), np.float32))
|
|
hold = np.clip(fixed + silhouette + glyph_union, 0, 1)
|
|
|
|
if config.mask.mode == "glyph":
|
|
hold = glyph_hold_mask(poster, config)
|
|
silhouette = np.zeros_like(silhouette)
|
|
return {"hold": hold, "glyph": glyphs, "sil": silhouette}
|
|
|
|
|
|
def glyph_hold_mask(poster: Image.Image, config: LayerConfig) -> np.ndarray:
|
|
"""검정 포스터용. 사각 고정층이 검은 상자가 되므로 글자 픽셀만 얇게 잡는다."""
|
|
width, height = poster.size
|
|
spec = config.mask
|
|
luminance = np.asarray(poster.convert("L")).astype(np.float32)
|
|
thin = np.zeros((height, width), np.uint8)
|
|
thick = np.zeros((height, width), np.uint8)
|
|
for rect in config.fixed_rects:
|
|
x0, y0, x1, y1 = to_pixels(rect, poster.size)
|
|
target = thick if rect[1] >= spec.thick_from_y else thin
|
|
target[y0:y1, x0:x1] = (luminance[y0:y1, x0:x1] > spec.ink_min) * 255
|
|
thin_mask = Image.fromarray(thin).filter(
|
|
ImageFilter.MaxFilter(spec.thin_dilate)).filter(ImageFilter.GaussianBlur(2))
|
|
thick_mask = Image.fromarray(thick).filter(
|
|
ImageFilter.MaxFilter(spec.thick_dilate)).filter(ImageFilter.GaussianBlur(3))
|
|
return np.maximum(np.asarray(thin_mask), np.asarray(thick_mask)) / 255.0
|
|
|
|
|
|
def pop_rect(frame: Image.Image, poster: Image.Image, rect: Rect, scale: float) -> None:
|
|
"""원본의 사각 영역을 오려 중심 기준으로 배율을 먹여 덮는다."""
|
|
x0, y0, x1, y1 = to_pixels(rect, poster.size)
|
|
crop = poster.crop((x0, y0, x1, y1))
|
|
scaled_w, scaled_h = int(crop.width * scale), int(crop.height * scale)
|
|
crop = crop.resize((scaled_w, scaled_h), Image.LANCZOS)
|
|
mask = Image.new("L", crop.size, 0)
|
|
ImageDraw.Draw(mask).rectangle((6, 6, scaled_w - 7, scaled_h - 7), fill=255)
|
|
mask = mask.filter(ImageFilter.GaussianBlur(4))
|
|
frame.paste(crop, ((x0 + x1) // 2 - scaled_w // 2, (y0 + y1) // 2 - scaled_h // 2), mask)
|
|
|
|
|
|
def register(frame: np.ndarray, base: np.ndarray, weight: np.ndarray,
|
|
previous=(1.0, 0, 0)) -> tuple[np.ndarray, tuple, float]:
|
|
"""생성 클립을 원본에 정합한다.
|
|
|
|
모델이 프레임 전체를 2~3% 드리프트시키므로, 정합 없이 글자만 얇게 고정하면
|
|
클립의 글자가 옆에 비쳐 겹쳐 보인다. 고정층 영역에서 MAE가 최소인 스케일·이동을 찾는다.
|
|
"""
|
|
height, width = base.shape[:2]
|
|
step = 6 # 1/6 축소에서 탐색
|
|
small = (width // step, height // step)
|
|
base_small = np.asarray(Image.fromarray(base.astype(np.uint8)).resize(
|
|
small, Image.BILINEAR), np.float32)
|
|
frame_small = Image.fromarray(frame.astype(np.uint8)).resize(small, Image.BILINEAR)
|
|
weight_small = np.asarray(Image.fromarray((weight * 255).astype(np.uint8)).resize(
|
|
small, Image.BILINEAR), np.float32) / 255.0
|
|
weight_sum = weight_small.sum() + 1e-6
|
|
small_h, small_w = base_small.shape[:2]
|
|
|
|
def cost(scale, dx, dy):
|
|
inverse = 1 / scale
|
|
moved = frame_small.transform(
|
|
frame_small.size, Image.AFFINE,
|
|
(inverse, 0, small_w / 2 * (1 - inverse) - dx / step * inverse,
|
|
0, inverse, small_h / 2 * (1 - inverse) - dy / step * inverse), Image.BILINEAR)
|
|
diff = np.abs(np.asarray(moved, np.float32) - base_small).mean(axis=2)
|
|
return (diff * weight_small).sum() / weight_sum
|
|
|
|
best = (cost(*previous), *previous)
|
|
for scale in (previous[0] - 0.02, previous[0] - 0.01, previous[0],
|
|
previous[0] + 0.01, previous[0] + 0.02):
|
|
for dx in range(previous[1] - 24, previous[1] + 25, 8):
|
|
for dy in range(previous[2] - 24, previous[2] + 25, 8):
|
|
score = cost(scale, dx, dy)
|
|
if score < best[0]:
|
|
best = (score, scale, dx, dy)
|
|
_, scale, dx, dy = best
|
|
for fine_scale in (scale - 0.005, scale, scale + 0.005):
|
|
for fine_dx in range(dx - 4, dx + 5, 2):
|
|
for fine_dy in range(dy - 4, dy + 5, 2):
|
|
score = cost(fine_scale, fine_dx, fine_dy)
|
|
if score < best[0]:
|
|
best = (score, fine_scale, fine_dx, fine_dy)
|
|
_, scale, dx, dy = best
|
|
|
|
inverse = 1 / scale
|
|
warped = Image.fromarray(frame.astype(np.uint8)).transform(
|
|
(width, height), Image.AFFINE,
|
|
(inverse, 0, width / 2 * (1 - inverse) - dx * inverse,
|
|
0, inverse, height / 2 * (1 - inverse) - dy * inverse), Image.BICUBIC)
|
|
return np.asarray(warped, np.float32), (scale, dx, dy), best[0]
|
|
|
|
|
|
def clip_frame_reader(clip: bytes | Path, size: tuple[int, int], crop: str | None):
|
|
"""클립 프레임을 순서대로 하나씩만 메모리에 올린다. 인덱스는 단조 증가한다."""
|
|
frames = decode_frames(clip)
|
|
state = {"index": -1, "image": None}
|
|
|
|
def read(wanted: int) -> np.ndarray:
|
|
while state["index"] < wanted:
|
|
try:
|
|
frame = next(frames)
|
|
except StopIteration:
|
|
break
|
|
state["index"] += 1
|
|
state["image"] = frame
|
|
image = state["image"]
|
|
if crop:
|
|
x0, y0, w, h = (int(v) for v in crop.split(","))
|
|
image = image.crop((x0, y0, x0 + w, y0 + h))
|
|
return np.asarray(image.convert("RGB").resize(size, Image.LANCZOS)).astype(np.float32)
|
|
|
|
return read
|
|
|
|
|
|
def compose_frames(poster: Image.Image, config: LayerConfig, clip: bytes | Path,
|
|
masks: dict, do_register: bool) -> Iterator[Image.Image]:
|
|
width, height = poster.size
|
|
base = np.asarray(poster).astype(np.float32)
|
|
hold = masks["hold"][..., None]
|
|
read_clip = clip_frame_reader(clip, poster.size, config.hybrid.clip_crop)
|
|
|
|
xs = np.arange(width, dtype=np.float32)[None, :]
|
|
ys = np.arange(height, dtype=np.float32)[:, None]
|
|
hold_frames = int(HOLD_SECONDS * FPS)
|
|
total = int(DURATION * FPS)
|
|
glow_masks: dict = {}
|
|
# 펀치 안 하는 이웃도 항상 보호한다 — 펀치된 크롭의 배경이 이웃 글자를 덮는다
|
|
text_elements = [(rect, glyph_alpha(poster, rect, config.glyph_dark_ink))
|
|
for rect, _, _ in config.punches]
|
|
text_elements += [(rect, glyph_alpha(poster, rect, config.glyph_dark_ink))
|
|
for rect in config.protect]
|
|
registration = (1.0, 0, 0)
|
|
|
|
for index in range(total):
|
|
seconds = index / FPS
|
|
if index < hold_frames:
|
|
artwork = base
|
|
else:
|
|
artwork = read_clip(index - hold_frames)
|
|
if do_register:
|
|
artwork, registration, _ = register(artwork, base, masks["hold"], registration)
|
|
|
|
frame = artwork * (1 - hold) + base * hold
|
|
|
|
for name, rect in config.glyphs.items():
|
|
progress = (seconds - config.sweep_at.get(name, 0.0)) / SWEEP_SECONDS
|
|
if 0 <= progress <= 1:
|
|
gx0, gx1 = rect[0] * width, rect[2] * width
|
|
band_x = gx0 - 120 + (gx1 - gx0 + 240) * progress
|
|
band = np.exp(-((xs - 0.35 * (ys - rect[1] * height)) - band_x) ** 2
|
|
/ (2 * 55.0 ** 2))
|
|
frame = frame + SWEEP_GAIN * (band * masks["glyph"][name])[..., None]
|
|
|
|
for rect, at in config.glows:
|
|
progress = (seconds - at) / GLOW_SECONDS
|
|
if 0 <= progress <= 1:
|
|
glow = glow_masks.setdefault(
|
|
rect, np.asarray(glyph_alpha(poster, rect, config.glyph_dark_ink)) / 255.0)
|
|
frame = frame + GLOW_GAIN * math.sin(math.pi * progress) * glow[..., None]
|
|
|
|
image = Image.fromarray(np.clip(frame, 0, 255).astype(np.uint8))
|
|
active = []
|
|
for order, (rect, at, peak) in enumerate(config.punches):
|
|
scale = punch_scale(seconds - at, peak)
|
|
if scale != 1.0:
|
|
pop_rect(image, poster, rect, scale)
|
|
active.append(order)
|
|
if active:
|
|
for order, (_, alpha) in enumerate(text_elements):
|
|
if order not in active:
|
|
image.paste(poster, (0, 0), alpha)
|
|
yield image
|
|
|
|
|
|
def render_hybrid(poster: bytes | Path | Image.Image, clip: bytes | Path,
|
|
config: LayerConfig, *, preview: bool = False) -> HybridResult:
|
|
if isinstance(poster, Image.Image):
|
|
source = poster.convert("RGB")
|
|
else:
|
|
source = Image.open(poster if isinstance(poster, Path)
|
|
else io.BytesIO(poster)).convert("RGB")
|
|
masks = build_masks(source, config)
|
|
do_register = config.hybrid.register_frames or config.mask.mode == "glyph"
|
|
|
|
proof_at = {round(step * DURATION * FPS / PROOF_COUNT) for step in range(PROOF_COUNT)}
|
|
proof_tiles: list[Image.Image] = []
|
|
|
|
def frames() -> Iterator[Image.Image]:
|
|
for index, image in enumerate(compose_frames(source, config, clip, masks, do_register)):
|
|
if preview and index in proof_at:
|
|
proof_tiles.append(image.resize(PROOF_SIZE))
|
|
yield image
|
|
|
|
video = encode_mp4(frames(), source.size, fps=FPS, crf=CRF)
|
|
|
|
sheet = None
|
|
if proof_tiles:
|
|
sheet = Image.new("RGB", ((PROOF_SIZE[0] + 10) * len(proof_tiles), PROOF_SIZE[1]), "black")
|
|
for order, tile in enumerate(proof_tiles):
|
|
sheet.paste(tile, (order * (PROOF_SIZE[0] + 10), 0))
|
|
|
|
return HybridResult(video=video, frames=int(DURATION * FPS), duration=DURATION, proof=sheet)
|