feat: add scrape import functionality with status messages and logs
All checks were successful
Build Docker Image / build (push) Successful in 1m22s
All checks were successful
Build Docker Image / build (push) Successful in 1m22s
This commit is contained in:
@@ -1,6 +1,15 @@
|
||||
import { db } from '$lib/server/db';
|
||||
import { character, characterScrapeValidation } from '$lib/server/db/schema';
|
||||
import { eq } from 'drizzle-orm';
|
||||
import { exec } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
const execAsync = promisify(exec);
|
||||
let isScrapeImportRunning = false;
|
||||
const EXEC_OPTIONS = {
|
||||
cwd: process.cwd(),
|
||||
maxBuffer: 50 * 1024 * 1024
|
||||
};
|
||||
|
||||
async function upsertCharacterFromScrapeValidation(characterId: string): Promise<boolean> {
|
||||
const [scraped] = await db
|
||||
@@ -147,6 +156,51 @@ export async function load() {
|
||||
}
|
||||
|
||||
export const actions = {
|
||||
runScrapeImport: async () => {
|
||||
if (isScrapeImportRunning) {
|
||||
return {
|
||||
success: false,
|
||||
message: 'A scrape is already running. Please wait for it to finish.',
|
||||
logs: ''
|
||||
};
|
||||
}
|
||||
|
||||
isScrapeImportRunning = true;
|
||||
try {
|
||||
const scrapeResult = await execAsync('npm run scrape', EXEC_OPTIONS);
|
||||
const importResult = await execAsync('npm run db:import', EXEC_OPTIONS);
|
||||
|
||||
const logs = [
|
||||
'=== npm run scrape ===',
|
||||
scrapeResult.stdout || '',
|
||||
scrapeResult.stderr ? `\n[stderr]\n${scrapeResult.stderr}` : '',
|
||||
'\n=== npm run db:import ===',
|
||||
importResult.stdout || '',
|
||||
importResult.stderr ? `\n[stderr]\n${importResult.stderr}` : ''
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join('\n');
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: 'Scrape and import completed successfully',
|
||||
logs
|
||||
};
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Failed to run scripts';
|
||||
const stdout = typeof error === 'object' && error && 'stdout' in error ? String((error as any).stdout || '') : '';
|
||||
const stderr = typeof error === 'object' && error && 'stderr' in error ? String((error as any).stderr || '') : '';
|
||||
const logs = [stdout, stderr ? `\n[stderr]\n${stderr}` : ''].filter(Boolean).join('\n');
|
||||
return {
|
||||
success: false,
|
||||
message,
|
||||
logs
|
||||
};
|
||||
} finally {
|
||||
isScrapeImportRunning = false;
|
||||
}
|
||||
},
|
||||
|
||||
acceptOne: async ({ request }) => {
|
||||
const formData = await request.formData();
|
||||
const characterId = formData.get('characterId');
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<script lang="ts">
|
||||
import { page } from '$app/stores';
|
||||
|
||||
let { data } = $props();
|
||||
let { data, form } = $props();
|
||||
|
||||
const newCharacters = $derived(data.changes.filter((c: any) => c.type === 'new'));
|
||||
const modifiedCharacters = $derived(data.changes.filter((c: any) => c.type === 'modified'));
|
||||
@@ -40,6 +40,22 @@
|
||||
<div>
|
||||
<h1 class="text-3xl font-black uppercase tracking-[0.25em] text-amber-50 mb-2">Character Changes</h1>
|
||||
<p class="text-gray-400">Total changes: {newCharacters.length} new, {modifiedCharacters.length} modified</p>
|
||||
<form method="POST" action="?/runScrapeImport" class="mt-4">
|
||||
<button
|
||||
type="submit"
|
||||
class="rounded-full border border-sky-300/40 bg-sky-500/20 px-4 py-2 text-sm font-semibold text-sky-100 transition hover:bg-sky-500/30"
|
||||
>
|
||||
🔄 Lancer scrape + import
|
||||
</button>
|
||||
</form>
|
||||
{#if form?.message}
|
||||
<p class={`mt-3 text-sm ${form.success ? 'text-emerald-300' : 'text-rose-300'}`}>
|
||||
{form.message}
|
||||
</p>
|
||||
{/if}
|
||||
{#if form?.logs}
|
||||
<pre class="mt-3 max-h-72 overflow-auto rounded-lg border border-white/10 bg-slate-900/70 p-3 text-xs text-slate-200 whitespace-pre-wrap">{form.logs}</pre>
|
||||
{/if}
|
||||
{#if newCharacters.length + modifiedCharacters.length > 0}
|
||||
<form method="POST" action="?/acceptAll" class="mt-4">
|
||||
<button
|
||||
|
||||
Reference in New Issue
Block a user