import { createClient } from '@libsql/client'; import { drizzle } from 'drizzle-orm/libsql'; import { sql, eq } from 'drizzle-orm'; import fs from 'fs'; import { arc, character, devilFruit, characterScrapeValidation, type DevilFruitType } from '../src/lib/server/db/schema'; type ArcRecord = { id: string; name: string; startChapter: number; endChapter?: number | null; url?: string | null; }; type DevilFruitRecord = { id: string; name: string; type?: DevilFruitType | string | null; url?: string | null; }; type CharacterRecord = { id: string; name: string; gender?: string | null; age?: number | null; affiliations?: string[] | string | null; devilFruitId?: string | null; hakiObservation?: boolean; hakiArmament?: boolean; hakiConqueror?: boolean; bounty?: number | null; height?: number | null; origin?: string | null; firstAppearance?: number; pictureUrl?: string | null; epithets?: string[] | string | null; status?: string | null; arcId?: string | null; url?: string | null; }; const DATABASE_URL = process.env.DATABASE_URL || 'file:local.db'; const client = createClient({ url: DATABASE_URL }); const db = drizzle(client); function readJsonFile(path: string): T[] | null { if (!fs.existsSync(path)) { return null; } const content = fs.readFileSync(path, 'utf-8'); return JSON.parse(content) as T[]; } function toNullable(value: T | undefined | null | ''): T | null { return value === undefined || value === null || value === '' ? null : value; } function toJsonArray(value: string[] | string | null | undefined): string[] | null { if (Array.isArray(value)) { return value.length > 0 ? value : null; } if (typeof value === 'string' && value.trim() !== '') { if (value.trim().startsWith('[')) { try { const parsed = JSON.parse(value); return Array.isArray(parsed) ? parsed : [value]; } catch { return [value]; } } const splitValues = value .split(',') .map((item) => item.trim()) .filter(Boolean); return splitValues.length > 0 ? splitValues : null; } return null; } function toDevilFruitType(value: DevilFruitType | string | null | undefined): DevilFruitType | null { if (!value) return null; if (value === 'Paramecia' || value === 'Zoan' || value === 'Logia' || value === 'Unknown') { return value; } return 'Unknown'; } function toNumber(value: string | number | null | undefined): number | null { if (value === null || value === undefined || value === '') return null; const num = typeof value === 'string' ? parseFloat(value) : value; return isNaN(num) ? null : num; } function getErrorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } function logSqlOnError(statement: { sql: string; params: unknown[] } | null): void { if (!statement) return; console.error(` SQL: ${statement.sql}`); console.error(` Params: ${JSON.stringify(statement.params)}`); } function transformCharacterData(item: CharacterRecord) { return { id: item.id, name: item.name, gender: toNullable(item.gender), age: toNullable(item.age), affiliations: toJsonArray(item.affiliations), devilFruitId: toNullable(item.devilFruitId), hakiObservation: !!item.hakiObservation, hakiArmament: !!item.hakiArmament, hakiConqueror: !!item.hakiConqueror, bounty: item.bounty ?? 0, height: toNumber(item.height as any), origin: toNullable(item.origin), firstAppearance: item.firstAppearance ?? 0, pictureUrl: toNullable(item.pictureUrl), epithets: toJsonArray(item.epithets), status: toNullable(item.status), arcId: toNullable(item.arcId), url: toNullable(item.url) }; } function hasChanged(jsonData: any, dbData: any): boolean { if (!dbData) return true; // Print any differences for debugging for (const key in jsonData) { const jsonValue = jsonData[key]; const dbValue = dbData[key]; const jsonString = typeof jsonValue === 'object' ? JSON.stringify(jsonValue) : String(jsonValue); const dbString = typeof dbValue === 'object' ? JSON.stringify(dbValue) : String(dbValue); if (jsonString !== dbString) { console.log(`\nField "${key}" changed for character ID ${jsonData.id}:`); console.log(` JSON: ${jsonString}`); console.log(` DB: ${dbString}`); } } // Compare each field return ( jsonData.name != dbData.name || jsonData.gender != dbData.gender || jsonData.age != dbData.age || JSON.stringify(jsonData.affiliations) != JSON.stringify(dbData.affiliations) || jsonData.devilFruitId != dbData.devilFruitId || jsonData.hakiObservation != dbData.hakiObservation || jsonData.hakiArmament != dbData.hakiArmament || jsonData.hakiConqueror != dbData.hakiConqueror || jsonData.bounty != dbData.bounty || jsonData.height != dbData.height || jsonData.origin != dbData.origin || jsonData.firstAppearance != dbData.firstAppearance || jsonData.pictureUrl != dbData.pictureUrl || JSON.stringify(jsonData.epithets) != JSON.stringify(dbData.epithets) || jsonData.status != dbData.status || jsonData.arcId != dbData.arcId || jsonData.url != dbData.url ); } async function isCharacterTableEmpty(): Promise { const result = await db.select({ count: sql`COUNT(*)` }).from(character); return result[0]?.count === 0; } async function importFromJson(): Promise { let totalSuccess = 0; let totalErrors = 0; try { const arcs = readJsonFile('./scraped-data/arcs.json'); if (arcs) { console.log('\n=== Importing Arcs ===\n'); console.log(`Found ${arcs.length} arcs\n`); let successCount = 0; let errorCount = 0; for (let i = 0; i < arcs.length; i++) { const item = arcs[i]; let lastSql: { sql: string; params: unknown[] } | null = null; try { const query = db .insert(arc) .values({ id: item.id, name: item.name, startChapter: item.startChapter, endChapter: toNullable(item.endChapter), url: toNullable(item.url) }) .onConflictDoUpdate({ target: arc.id, set: { name: item.name, startChapter: item.startChapter, endChapter: toNullable(item.endChapter), url: toNullable(item.url) } }); lastSql = query.toSQL(); await query; successCount++; process.stdout.write(`\rExecuted: ${successCount}/${arcs.length}`); } catch (error) { errorCount++; console.error(`\n✗ Error at arc ${i + 1}:`); console.error(` ID: ${item.id ?? 'N/A'}`); console.error(` Message: ${getErrorMessage(error)}`); logSqlOnError(lastSql); } } console.log(`\n\n✓ Arcs imported!`); console.log(` Success: ${successCount}`); console.log(` Errors: ${errorCount}`); totalSuccess += successCount; totalErrors += errorCount; } else { console.log('\n⚠️ No arcs.json found, skipping...\n'); } const fruits = readJsonFile('./scraped-data/devil-fruits.json'); if (fruits) { console.log('\n=== Importing Devil Fruits ===\n'); console.log(`Found ${fruits.length} devil fruits\n`); let successCount = 0; let errorCount = 0; for (let i = 0; i < fruits.length; i++) { const item = fruits[i]; let lastSql: { sql: string; params: unknown[] } | null = null; try { const query = db .insert(devilFruit) .values({ id: item.id, name: item.name, type: toDevilFruitType(item.type), url: toNullable(item.url) }) .onConflictDoUpdate({ target: devilFruit.id, set: { name: item.name, type: toDevilFruitType(item.type), url: toNullable(item.url) } }); lastSql = query.toSQL(); await query; successCount++; process.stdout.write(`\rExecuted: ${successCount}/${fruits.length}`); } catch (error) { errorCount++; console.error(`\n✗ Error at devil fruit ${i + 1}:`); console.error(` ID: ${item.id ?? 'N/A'}`); console.error(` Message: ${getErrorMessage(error)}`); logSqlOnError(lastSql); } } console.log(`\n\n✓ Devil Fruits imported!`); console.log(` Success: ${successCount}`); console.log(` Errors: ${errorCount}`); totalSuccess += successCount; totalErrors += errorCount; } else { console.log('\n⚠️ No devil-fruits.json found, skipping...\n'); } const characters = readJsonFile('./scraped-data/characters.json'); if (characters) { console.log('\n=== Importing Characters ===\n'); console.log(`Found ${characters.length} characters\n`); const isEmpty = await isCharacterTableEmpty(); let successCount = 0; let errorCount = 0; if (isEmpty) { // Populate empty character table console.log('Characters table is empty, populating...\n'); for (let i = 0; i < characters.length; i++) { const item = characters[i]; let lastSql: { sql: string; params: unknown[] } | null = null; try { const data = transformCharacterData(item); const query = db .insert(character) .values(data) .onConflictDoUpdate({ target: character.id, set: data }); lastSql = query.toSQL(); await query; successCount++; process.stdout.write(`\rExecuted: ${successCount}/${characters.length}`); } catch (error) { errorCount++; console.error(`\n✗ Error at character ${i + 1}:`); console.error(` ID: ${item.id ?? 'N/A'}`); console.error(` Message: ${getErrorMessage(error)}`); logSqlOnError(lastSql); } } } else { // Check for changes and update scrapeValidation table console.log('Characters table not empty, checking for changes...\n'); for (let i = 0; i < characters.length; i++) { const item = characters[i]; let lastSql: { sql: string; params: unknown[] } | null = null; try { const selectQuery = db .select() .from(character) .where(eq(character.id, item.id)); lastSql = selectQuery.toSQL(); const [dbCharacter] = await selectQuery; const jsonData = transformCharacterData(item); const changed = hasChanged(jsonData, dbCharacter); if (changed) { // Update scrapeValidation table with changes const upsertQuery = db .insert(characterScrapeValidation) .values(jsonData) .onConflictDoUpdate({ target: characterScrapeValidation.id, set: jsonData }); lastSql = upsertQuery.toSQL(); await upsertQuery; } else { // No changes, delete from scrapeValidation if it exists const deleteQuery = db .delete(characterScrapeValidation) .where(eq(characterScrapeValidation.id, item.id)); lastSql = deleteQuery.toSQL(); await deleteQuery; } successCount++; process.stdout.write(`\rProcessed: ${successCount}/${characters.length}`); } catch (error) { errorCount++; console.error(`\n✗ Error at character ${i + 1}:`); console.error(` ID: ${item.id ?? 'N/A'}`); console.error(` Message: ${getErrorMessage(error)}`); logSqlOnError(lastSql); } } } console.log(`\n\n✓ Characters imported!`); console.log(` Success: ${successCount}`); console.log(` Errors: ${errorCount}`); totalSuccess += successCount; totalErrors += errorCount; } else { console.log('\n⚠️ No characters.json found, skipping...\n'); } console.log(`\n=== Total Import Summary ===`); console.log(` Total Success: ${totalSuccess}`); console.log(` Total Errors: ${totalErrors}\n`); } catch (error) { console.error('✗ Import failed:', getErrorMessage(error)); process.exit(1); } finally { client.close(); } } importFromJson().catch((error) => { console.error(getErrorMessage(error)); process.exit(1); });