ai-publish-lateast/o2o-plagiarism-ai
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: AI 표본 생성 재개 시 목표 건수 보존

Browse Source
This commit is contained in:
hbyang 2026-08-20 17:18:03 +09:00
parent 265a388389
commit 78e9f8648a
1 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
scripts/generate_ai_samples.py
View File

@ -264,6 +264,7 @@ def main() -> int:
done = load_done_ids(args.out) done = load_done_ids(args.out)
if done: if done:
logger.info("기존 출력 %d건을 건너뜁니다: %s", len(done), args.out) logger.info("기존 출력 %d건을 건너뜁니다: %s", len(done), args.out)
target_new = max(0, args.limit - len(done))
rng = random.Random(args.seed) rng = random.Random(args.seed)
# 소재 행을 결정적으로 섞는다. 같은 seed 면 같은 순서가 나온다. # 소재 행을 결정적으로 섞는다. 같은 seed 면 같은 순서가 나온다.
@ -335,7 +336,7 @@ def main() -> int:
with sink as sink: with sink as sink:
if args.dry_run: if args.dry_run:
for task in tasks: for task in tasks:
if written >= args.limit: if written >= target_new:
break break
_, _, _, target, _, model, prompt = task _, _, _, target, _, model, prompt = task
print(f"\n--- [{written + 1}] model={model} target={target}자 ---") print(f"\n--- [{written + 1}] model={model} target={target}자 ---")
@ -343,9 +344,9 @@ def main() -> int:
written += 1 written += 1
else: else:
with ThreadPoolExecutor(max_workers=args.concurrency) as executor: with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
while written < args.limit: while written < target_new:
batch = [] batch = []
for _ in range(min(args.concurrency, args.limit - written)): for _ in range(min(args.concurrency, target_new - written)):
task = next(tasks, None) task = next(tasks, None)
if task is not None: if task is not None:
batch.append(task) batch.append(task)
@ -381,7 +382,8 @@ def main() -> int:
if written % 50 == 0: if written % 50 == 0:
logger.info( logger.info(
"생성 %d/%d (길이 폐기 %d, 품질 폐기 %d, 실패 %d)", "생성 %d/%d (길이 폐기 %d, 품질 폐기 %d, 실패 %d)",
written, args.limit, skipped_len, skipped_quality, failed, len(done) + written, args.limit,
skipped_len, skipped_quality, failed,
) )
logger.info( logger.info(