import { db } from '$lib/server/db'; import { character, characterScrapeValidation } from '$lib/server/db/schema'; import { eq } from 'drizzle-orm'; import { exec } from 'node:child_process'; import { promisify } from 'node:util'; const execAsync = promisify(exec); let isScrapeImportRunning = false; const EXEC_OPTIONS = { cwd: process.cwd(), maxBuffer: 50 * 1024 * 1024 }; async function upsertCharacterFromScrapeValidation(characterId: string): Promise { const [scraped] = await db .select() .from(characterScrapeValidation) .where(eq(characterScrapeValidation.id, characterId)); if (!scraped) { return false; } await db .insert(character) .values({ id: scraped.id, name: scraped.name, frName: scraped.frName, gender: scraped.gender, age: scraped.age, affiliations: scraped.affiliations, devilFruitId: scraped.devilFruitId, hakiObservation: scraped.hakiObservation, hakiArmament: scraped.hakiArmament, hakiConqueror: scraped.hakiConqueror, bounty: scraped.bounty, height: scraped.height, origin: scraped.origin, frOrigin: scraped.frOrigin, firstAppearance: scraped.firstAppearance, pictureUrl: scraped.pictureUrl, epithets: scraped.epithets, frEpithets: scraped.frEpithets, status: scraped.status, arcId: scraped.arcId, url: scraped.url, frUrl: scraped.frUrl, }) .onConflictDoUpdate({ target: character.id, set: { name: scraped.name, frName: scraped.frName, gender: scraped.gender, age: scraped.age, affiliations: scraped.affiliations, devilFruitId: scraped.devilFruitId, hakiObservation: scraped.hakiObservation, hakiArmament: scraped.hakiArmament, hakiConqueror: scraped.hakiConqueror, bounty: scraped.bounty, height: scraped.height, origin: scraped.origin, frOrigin: scraped.frOrigin, firstAppearance: scraped.firstAppearance, pictureUrl: scraped.pictureUrl, epithets: scraped.epithets, frEpithets: scraped.frEpithets, status: scraped.status, arcId: scraped.arcId, url: scraped.url, frUrl: scraped.frUrl } }); return true; } export async function load() { // Get all characters from both tables const currentCharacters = await db.select().from(character); const scrapedCharacters = await db.select().from(characterScrapeValidation); // Create a map for quick lookup const currentCharMap = new Map(currentCharacters.map(c => [c.id, c])); // Compare and categorize changes const changes: { type: 'new' | 'modified'; id: string; scraped: (typeof scrapedCharacters)[0]; current?: (typeof currentCharacters)[0]; differences?: Record; }[] = []; for (const scraped of scrapedCharacters) { const current = currentCharMap.get(scraped.id); if (!current) { // New character changes.push({ type: 'new', id: scraped.id, scraped }); } else { // Check if different const differences: Record = {}; const fieldsToCompare = [ 'name', 'frName', 'gender', 'age', 'affiliations', 'devilFruitId', 'hakiObservation', 'hakiArmament', 'hakiConqueror', 'bounty', 'height', 'origin', 'frOrigin', 'firstAppearance', 'pictureUrl', 'epithets', 'frEpithets', 'status', 'arcId', 'url', 'frUrl' ]; for (const field of fieldsToCompare) { const currentValue = current[field as keyof typeof current]; const scrapedValue = scraped[field as keyof typeof scraped]; // Deep comparison for JSON fields if (JSON.stringify(currentValue) !== JSON.stringify(scrapedValue)) { differences[field] = { current: currentValue, scraped: scrapedValue }; } } if (Object.keys(differences).length > 0) { changes.push({ type: 'modified', id: scraped.id, scraped, current, differences }); } } } return { changes: changes.sort((a, b) => { // Show 'new' first, then 'modified' if (a.type !== b.type) { return a.type === 'new' ? -1 : 1; } return a.id.localeCompare(b.id); }) }; } export const actions = { runScrapeImport: async () => { if (isScrapeImportRunning) { return { success: false, message: 'A scrape is already running. Please wait for it to finish.', logs: '' }; } isScrapeImportRunning = true; try { const scrapeResult = await execAsync('npm run scrape', EXEC_OPTIONS); const importResult = await execAsync('npm run db:import', EXEC_OPTIONS); const logs = [ '=== npm run scrape ===', scrapeResult.stdout || '', scrapeResult.stderr ? `\n[stderr]\n${scrapeResult.stderr}` : '', '\n=== npm run db:import ===', importResult.stdout || '', importResult.stderr ? `\n[stderr]\n${importResult.stderr}` : '' ] .filter(Boolean) .join('\n'); return { success: true, message: 'Scrape and import completed successfully', logs }; } catch (error) { const message = error instanceof Error ? error.message : 'Failed to run scripts'; const stdout = typeof error === 'object' && error && 'stdout' in error ? String((error as any).stdout || '') : ''; const stderr = typeof error === 'object' && error && 'stderr' in error ? String((error as any).stderr || '') : ''; const logs = [stdout, stderr ? `\n[stderr]\n${stderr}` : ''].filter(Boolean).join('\n'); return { success: false, message, logs }; } finally { isScrapeImportRunning = false; } }, acceptOne: async ({ request }) => { const formData = await request.formData(); const characterId = formData.get('characterId'); if (!characterId || typeof characterId !== 'string') { return { success: false, message: 'characterId is required' }; } const applied = await upsertCharacterFromScrapeValidation(characterId); return { success: applied, message: applied ? 'Character applied successfully' : 'Character not found in scrape validation table' }; }, acceptAll: async () => { const scrapedCharacters = await db.select().from(characterScrapeValidation); let appliedCount = 0; for (const scraped of scrapedCharacters) { const applied = await upsertCharacterFromScrapeValidation(scraped.id); if (applied) { appliedCount++; } } return { success: true, appliedCount }; } };