feat: add scrape import functionality with status messages and logs
All checks were successful
Build Docker Image / build (push) Successful in 1m22s

This commit is contained in:
2026-03-03 23:44:03 +01:00
parent de2c8cdc77
commit 7c9aef1aee
2 changed files with 71 additions and 1 deletions

View File

@@ -1,6 +1,15 @@
import { db } from '$lib/server/db';
import { character, characterScrapeValidation } from '$lib/server/db/schema';
import { eq } from 'drizzle-orm';
import { exec } from 'node:child_process';
import { promisify } from 'node:util';
const execAsync = promisify(exec);
let isScrapeImportRunning = false;
const EXEC_OPTIONS = {
cwd: process.cwd(),
maxBuffer: 50 * 1024 * 1024
};
async function upsertCharacterFromScrapeValidation(characterId: string): Promise<boolean> {
const [scraped] = await db
@@ -147,6 +156,51 @@ export async function load() {
}
export const actions = {
runScrapeImport: async () => {
if (isScrapeImportRunning) {
return {
success: false,
message: 'A scrape is already running. Please wait for it to finish.',
logs: ''
};
}
isScrapeImportRunning = true;
try {
const scrapeResult = await execAsync('npm run scrape', EXEC_OPTIONS);
const importResult = await execAsync('npm run db:import', EXEC_OPTIONS);
const logs = [
'=== npm run scrape ===',
scrapeResult.stdout || '',
scrapeResult.stderr ? `\n[stderr]\n${scrapeResult.stderr}` : '',
'\n=== npm run db:import ===',
importResult.stdout || '',
importResult.stderr ? `\n[stderr]\n${importResult.stderr}` : ''
]
.filter(Boolean)
.join('\n');
return {
success: true,
message: 'Scrape and import completed successfully',
logs
};
} catch (error) {
const message = error instanceof Error ? error.message : 'Failed to run scripts';
const stdout = typeof error === 'object' && error && 'stdout' in error ? String((error as any).stdout || '') : '';
const stderr = typeof error === 'object' && error && 'stderr' in error ? String((error as any).stderr || '') : '';
const logs = [stdout, stderr ? `\n[stderr]\n${stderr}` : ''].filter(Boolean).join('\n');
return {
success: false,
message,
logs
};
} finally {
isScrapeImportRunning = false;
}
},
acceptOne: async ({ request }) => {
const formData = await request.formData();
const characterId = formData.get('characterId');

View File

@@ -1,7 +1,7 @@
<script lang="ts">
import { page } from '$app/stores';
let { data } = $props();
let { data, form } = $props();
const newCharacters = $derived(data.changes.filter((c: any) => c.type === 'new'));
const modifiedCharacters = $derived(data.changes.filter((c: any) => c.type === 'modified'));
@@ -40,6 +40,22 @@
<div>
<h1 class="text-3xl font-black uppercase tracking-[0.25em] text-amber-50 mb-2">Character Changes</h1>
<p class="text-gray-400">Total changes: {newCharacters.length} new, {modifiedCharacters.length} modified</p>
<form method="POST" action="?/runScrapeImport" class="mt-4">
<button
type="submit"
class="rounded-full border border-sky-300/40 bg-sky-500/20 px-4 py-2 text-sm font-semibold text-sky-100 transition hover:bg-sky-500/30"
>
🔄 Lancer scrape + import
</button>
</form>
{#if form?.message}
<p class={`mt-3 text-sm ${form.success ? 'text-emerald-300' : 'text-rose-300'}`}>
{form.message}
</p>
{/if}
{#if form?.logs}
<pre class="mt-3 max-h-72 overflow-auto rounded-lg border border-white/10 bg-slate-900/70 p-3 text-xs text-slate-200 whitespace-pre-wrap">{form.logs}</pre>
{/if}
{#if newCharacters.length + modifiedCharacters.length > 0}
<form method="POST" action="?/acceptAll" class="mt-4">
<button