import { createClient } from '@libsql/client'; import { drizzle } from 'drizzle-orm/libsql'; import { sql, eq, inArray } from 'drizzle-orm'; import fs from 'fs'; import { arc, character, devilFruit, characterScrapeValidation, type DevilFruitType } from '../src/lib/server/db/schema'; type Status = 'Alive' | 'Dead' | 'Unknown'; type ArcRecord = { id: string; name: string; frName?: string | null; startChapter: number; endChapter?: number | null; url?: string | null; }; type DevilFruitRecord = { id: string; name: string; type?: DevilFruitType | string | null; url?: string | null; }; type CharacterRecord = { id: string; name: string; frName?: string | null; gender?: string | null; age?: number | null; affiliations?: string[] | string | null; frAffiliations?: string[] | string | null; devilFruitId?: string | null; hakiObservation?: boolean; hakiArmament?: boolean; hakiConqueror?: boolean; bounty?: number | null; height?: number | null; origin?: string | null; frOrigin?: string | null; firstAppearance?: number; pictureUrl?: string | null; epithets?: string[] | string | null; frEpithets?: string[] | string | null; status?: Status | null; arcId?: string | null; url?: string | null; frUrl?: string | null; }; const DATABASE_URL = process.env.DATABASE_URL || 'file:local.db'; const client = createClient({ url: DATABASE_URL }); const db = drizzle(client); function readJsonFile(path: string): T[] | null { if (!fs.existsSync(path)) { return null; } const content = fs.readFileSync(path, 'utf-8'); return JSON.parse(content) as T[]; } function toNullable(value: T | undefined | null | ''): T | null { return value === undefined || value === null || value === '' ? null : value; } function toJsonArray(value: string[] | string | null | undefined): string[] | null { if (Array.isArray(value)) { return value.length > 0 ? value : null; } if (typeof value === 'string' && value.trim() !== '') { if (value.trim().startsWith('[')) { try { const parsed = JSON.parse(value); return Array.isArray(parsed) ? parsed : [value]; } catch { return [value]; } } const splitValues = value .split(',') .map((item) => item.trim()) .filter(Boolean); return splitValues.length > 0 ? splitValues : null; } return null; } function toDevilFruitType(value: DevilFruitType | string | null | undefined): DevilFruitType | null { if (!value) return null; if (value === 'Paramecia' || value === 'Zoan' || value === 'Logia' || value === 'Smile' || value === 'Unknown') { return value; } return 'Unknown'; } function toNumber(value: string | number | null | undefined): number | null { if (value === null || value === undefined || value === '') return null; const num = typeof value === 'string' ? parseFloat(value) : value; return isNaN(num) ? null : num; } function getErrorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } function logSqlOnError(statement: { sql: string; params: unknown[] } | null): void { if (!statement) return; console.error(` SQL: ${statement.sql}`); console.error(` Params: ${JSON.stringify(statement.params)}`); } function transformCharacterData(item: CharacterRecord) { return { id: item.id, name: item.name, frName: toNullable(item.frName), gender: toNullable(item.gender), age: toNullable(item.age), affiliations: toJsonArray(item.affiliations), frAffiliations: toJsonArray(item.frAffiliations), devilFruitId: toNullable(item.devilFruitId), hakiObservation: !!item.hakiObservation, hakiArmament: !!item.hakiArmament, hakiConqueror: !!item.hakiConqueror, bounty: item.bounty ?? 0, height: toNumber(item.height as string | number | null), origin: toNullable(item.origin), frOrigin: toNullable(item.frOrigin), firstAppearance: item.firstAppearance ?? 0, pictureUrl: toNullable(item.pictureUrl), epithets: toJsonArray(item.epithets), frEpithets: toJsonArray(item.frEpithets), status: toNullable(item.status), arcId: toNullable(item.arcId), url: toNullable(item.url), frUrl: toNullable(item.frUrl), isDeleted: false }; } async function isCharacterTableEmpty(): Promise { const result = await db.select({ count: sql`COUNT(*)` }).from(character); return result[0]?.count === 0; } async function importFromJson(): Promise { let totalSuccess = 0; let totalErrors = 0; try { const arcs = readJsonFile('./scraped-data/arcs.json'); if (arcs) { console.log('\n=== Importing Arcs ===\n'); console.log(`Found ${arcs.length} arcs\n`); let successCount = 0; let errorCount = 0; for (let i = 0; i < arcs.length; i++) { const item = arcs[i]; let lastSql: { sql: string; params: unknown[] } | null = null; try { const query = db .insert(arc) .values({ id: item.id, name: item.name, frName: toNullable(item.frName), startChapter: item.startChapter, endChapter: toNullable(item.endChapter), url: toNullable(item.url) }) .onConflictDoUpdate({ target: arc.id, set: { name: item.name, frName: toNullable(item.frName), startChapter: item.startChapter, endChapter: toNullable(item.endChapter), url: toNullable(item.url) } }); lastSql = query.toSQL(); await query; successCount++; process.stdout.write(`\rExecuted: ${successCount}/${arcs.length}`); } catch (error) { errorCount++; console.error(`\n✗ Error at arc ${i + 1}:`); console.error(` ID: ${item.id ?? 'N/A'}`); console.error(` Message: ${getErrorMessage(error)}`); logSqlOnError(lastSql); } } console.log(`\n\n✓ Arcs imported!`); console.log(` Success: ${successCount}`); console.log(` Errors: ${errorCount}`); totalSuccess += successCount; totalErrors += errorCount; } else { console.log('\n⚠️ No arcs.json found, skipping...\n'); } const fruits = readJsonFile('./scraped-data/devil-fruits.json'); if (fruits) { console.log('\n=== Importing Devil Fruits ===\n'); console.log(`Found ${fruits.length} devil fruits\n`); let successCount = 0; let errorCount = 0; for (let i = 0; i < fruits.length; i++) { const item = fruits[i]; let lastSql: { sql: string; params: unknown[] } | null = null; try { const query = db .insert(devilFruit) .values({ id: item.id, name: item.name, type: toDevilFruitType(item.type), url: toNullable(item.url) }) .onConflictDoUpdate({ target: devilFruit.id, set: { name: item.name, type: toDevilFruitType(item.type), url: toNullable(item.url) } }); lastSql = query.toSQL(); await query; successCount++; process.stdout.write(`\rExecuted: ${successCount}/${fruits.length}`); } catch (error) { errorCount++; console.error(`\n✗ Error at devil fruit ${i + 1}:`); console.error(` ID: ${item.id ?? 'N/A'}`); console.error(` Message: ${getErrorMessage(error)}`); logSqlOnError(lastSql); } } console.log(`\n\n✓ Devil Fruits imported!`); console.log(` Success: ${successCount}`); console.log(` Errors: ${errorCount}`); totalSuccess += successCount; totalErrors += errorCount; } else { console.log('\n⚠️ No devil-fruits.json found, skipping...\n'); } const characters = readJsonFile('./scraped-data/characters.json'); if (characters) { console.log('\n=== Importing Characters ===\n'); console.log(`Found ${characters.length} characters\n`); const isEmpty = await isCharacterTableEmpty(); let successCount = 0; let errorCount = 0; if (isEmpty) { // Populate empty character table console.log('Characters table is empty, populating...\n'); for (let i = 0; i < characters.length; i++) { const item = characters[i]; let lastSql: { sql: string; params: unknown[] } | null = null; try { const data = transformCharacterData(item); const query = db .insert(character) .values(data) .onConflictDoUpdate({ target: character.id, set: data }); lastSql = query.toSQL(); await query; successCount++; process.stdout.write(`\rExecuted: ${successCount}/${characters.length}`); } catch (error) { errorCount++; console.error(`\n✗ Error at character ${i + 1}:`); console.error(` ID: ${item.id ?? 'N/A'}`); console.error(` Message: ${getErrorMessage(error)}`); logSqlOnError(lastSql); } } } else { // Update scrapeValidation table console.log('Characters table not empty, updating scrapeValidation table for changes...\n'); const scrapedCharacterIds: string[] = []; for (let i = 0; i < characters.length; i++) { const item = characters[i]; let lastSql: { sql: string; params: unknown[] } | null = null; try { const selectQuery = db .select() .from(character) .where(eq(character.id, item.id)); lastSql = selectQuery.toSQL(); scrapedCharacterIds.push(item.id); const jsonData = transformCharacterData(item); const upsertQuery = db .insert(characterScrapeValidation) .values(jsonData) .onConflictDoUpdate({ target: characterScrapeValidation.id, set: jsonData }); lastSql = upsertQuery.toSQL(); await upsertQuery; successCount++; process.stdout.write(`\rProcessed: ${successCount}/${characters.length}`); } catch (error) { errorCount++; console.error(`\n✗ Error at character ${i + 1}:`); console.error(` ID: ${item.id ?? 'N/A'}`); console.error(` Message: ${getErrorMessage(error)}`); logSqlOnError(lastSql); } } // Fetch all characters from the character table and mark those absent from the // scrape as deleted in scrape validation. const allCharacters = await db.select({ id: character.id }).from(character); const scrapedSet = new Set(scrapedCharacterIds); const idsToMarkDeleted = allCharacters .map((c) => c.id) .filter((id) => !scrapedSet.has(id)); if (idsToMarkDeleted.length > 0) { console.log(`\n⚠️ Marking ${idsToMarkDeleted.length} character(s) as deleted in scrape validation...`); const deletedCharacterRows = await db .select() .from(character) .where(inArray(character.id, idsToMarkDeleted)); for (const row of deletedCharacterRows) { await db .insert(characterScrapeValidation) .values({ id: row.id, name: row.name, frName: row.frName, gender: row.gender, age: row.age, affiliations: row.affiliations, frAffiliations: row.frAffiliations, devilFruitId: row.devilFruitId, hakiObservation: row.hakiObservation, hakiArmament: row.hakiArmament, hakiConqueror: row.hakiConqueror, bounty: row.bounty, height: row.height, origin: row.origin, frOrigin: row.frOrigin, firstAppearance: row.firstAppearance, pictureUrl: row.pictureUrl, epithets: row.epithets, frEpithets: row.frEpithets, status: row.status, arcId: row.arcId, url: row.url, frUrl: row.frUrl, isDeleted: true }) .onConflictDoUpdate({ target: characterScrapeValidation.id, set: { isDeleted: true } }); } } } console.log(`\n\n✓ Characters imported!`); console.log(` Success: ${successCount}`); console.log(` Errors: ${errorCount}`); totalSuccess += successCount; totalErrors += errorCount; } else { console.log('\n⚠️ No characters.json found, skipping...\n'); } console.log(`\n=== Total Import Summary ===`); console.log(` Total Success: ${totalSuccess}`); console.log(` Total Errors: ${totalErrors}\n`); } catch (error) { console.error('✗ Import failed:', getErrorMessage(error)); process.exit(1); } finally { client.close(); } } importFromJson().catch((error) => { console.error(getErrorMessage(error)); process.exit(1); });