393 lines
11 KiB
TypeScript
393 lines
11 KiB
TypeScript
import { createClient } from '@libsql/client';
|
||
import { drizzle } from 'drizzle-orm/libsql';
|
||
import { sql, eq } from 'drizzle-orm';
|
||
import fs from 'fs';
|
||
import { arc, character, devilFruit, characterScrapeValidation, type DevilFruitType } from '../src/lib/server/db/schema';
|
||
|
||
type ArcRecord = {
|
||
id: string;
|
||
name: string;
|
||
startChapter: number;
|
||
endChapter?: number | null;
|
||
url?: string | null;
|
||
};
|
||
|
||
type DevilFruitRecord = {
|
||
id: string;
|
||
name: string;
|
||
type?: DevilFruitType | string | null;
|
||
url?: string | null;
|
||
};
|
||
|
||
type CharacterRecord = {
|
||
id: string;
|
||
name: string;
|
||
gender?: string | null;
|
||
age?: number | null;
|
||
affiliations?: string[] | string | null;
|
||
devilFruitId?: string | null;
|
||
hakiObservation?: boolean;
|
||
hakiArmament?: boolean;
|
||
hakiConqueror?: boolean;
|
||
bounty?: number | null;
|
||
height?: number | null;
|
||
origin?: string | null;
|
||
firstAppearance?: number;
|
||
pictureUrl?: string | null;
|
||
epithets?: string[] | string | null;
|
||
status?: string | null;
|
||
arcId?: string | null;
|
||
url?: string | null;
|
||
};
|
||
|
||
const DATABASE_URL = process.env.DATABASE_URL || 'file:local.db';
|
||
|
||
const client = createClient({ url: DATABASE_URL });
|
||
const db = drizzle(client);
|
||
|
||
function readJsonFile<T>(path: string): T[] | null {
|
||
if (!fs.existsSync(path)) {
|
||
return null;
|
||
}
|
||
|
||
const content = fs.readFileSync(path, 'utf-8');
|
||
return JSON.parse(content) as T[];
|
||
}
|
||
|
||
function toNullable<T>(value: T | undefined | null | ''): T | null {
|
||
return value === undefined || value === null || value === '' ? null : value;
|
||
}
|
||
|
||
function toJsonArray(value: string[] | string | null | undefined): string[] | null {
|
||
if (Array.isArray(value)) {
|
||
return value.length > 0 ? value : null;
|
||
}
|
||
|
||
if (typeof value === 'string' && value.trim() !== '') {
|
||
if (value.trim().startsWith('[')) {
|
||
try {
|
||
const parsed = JSON.parse(value);
|
||
return Array.isArray(parsed) ? parsed : [value];
|
||
} catch {
|
||
return [value];
|
||
}
|
||
}
|
||
|
||
const splitValues = value
|
||
.split(',')
|
||
.map((item) => item.trim())
|
||
.filter(Boolean);
|
||
|
||
return splitValues.length > 0 ? splitValues : null;
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
function toDevilFruitType(value: DevilFruitType | string | null | undefined): DevilFruitType | null {
|
||
if (!value) return null;
|
||
if (value === 'Paramecia' || value === 'Zoan' || value === 'Logia' || value === 'Unknown') {
|
||
return value;
|
||
}
|
||
return 'Unknown';
|
||
}
|
||
|
||
function toNumber(value: string | number | null | undefined): number | null {
|
||
if (value === null || value === undefined || value === '') return null;
|
||
const num = typeof value === 'string' ? parseFloat(value) : value;
|
||
return isNaN(num) ? null : num;
|
||
}
|
||
|
||
function getErrorMessage(error: unknown): string {
|
||
return error instanceof Error ? error.message : String(error);
|
||
}
|
||
|
||
function logSqlOnError(statement: { sql: string; params: unknown[] } | null): void {
|
||
if (!statement) return;
|
||
console.error(` SQL: ${statement.sql}`);
|
||
console.error(` Params: ${JSON.stringify(statement.params)}`);
|
||
}
|
||
|
||
function transformCharacterData(item: CharacterRecord) {
|
||
return {
|
||
id: item.id,
|
||
name: item.name,
|
||
gender: toNullable(item.gender),
|
||
age: toNullable(item.age),
|
||
affiliations: toJsonArray(item.affiliations),
|
||
devilFruitId: toNullable(item.devilFruitId),
|
||
hakiObservation: !!item.hakiObservation,
|
||
hakiArmament: !!item.hakiArmament,
|
||
hakiConqueror: !!item.hakiConqueror,
|
||
bounty: item.bounty ?? 0,
|
||
height: toNumber(item.height as any),
|
||
origin: toNullable(item.origin),
|
||
firstAppearance: item.firstAppearance ?? 0,
|
||
pictureUrl: toNullable(item.pictureUrl),
|
||
epithets: toJsonArray(item.epithets),
|
||
status: toNullable(item.status),
|
||
arcId: toNullable(item.arcId),
|
||
url: toNullable(item.url)
|
||
};
|
||
}
|
||
|
||
function hasChanged(jsonData: any, dbData: any): boolean {
|
||
if (!dbData) return true;
|
||
|
||
// Print any differences for debugging
|
||
for (const key in jsonData) {
|
||
const jsonValue = jsonData[key];
|
||
const dbValue = dbData[key];
|
||
const jsonString = typeof jsonValue === 'object' ? JSON.stringify(jsonValue) : String(jsonValue);
|
||
const dbString = typeof dbValue === 'object' ? JSON.stringify(dbValue) : String(dbValue);
|
||
if (jsonString !== dbString) {
|
||
console.log(`\nField "${key}" changed for character ID ${jsonData.id}:`);
|
||
console.log(` JSON: ${jsonString}`);
|
||
console.log(` DB: ${dbString}`);
|
||
} }
|
||
|
||
// Compare each field
|
||
return (
|
||
jsonData.name != dbData.name ||
|
||
jsonData.gender != dbData.gender ||
|
||
jsonData.age != dbData.age ||
|
||
JSON.stringify(jsonData.affiliations) != JSON.stringify(dbData.affiliations) ||
|
||
jsonData.devilFruitId != dbData.devilFruitId ||
|
||
jsonData.hakiObservation != dbData.hakiObservation ||
|
||
jsonData.hakiArmament != dbData.hakiArmament ||
|
||
jsonData.hakiConqueror != dbData.hakiConqueror ||
|
||
jsonData.bounty != dbData.bounty ||
|
||
jsonData.height != dbData.height ||
|
||
jsonData.origin != dbData.origin ||
|
||
jsonData.firstAppearance != dbData.firstAppearance ||
|
||
jsonData.pictureUrl != dbData.pictureUrl ||
|
||
JSON.stringify(jsonData.epithets) != JSON.stringify(dbData.epithets) ||
|
||
jsonData.status != dbData.status ||
|
||
jsonData.arcId != dbData.arcId ||
|
||
jsonData.url != dbData.url
|
||
);
|
||
}
|
||
|
||
async function isCharacterTableEmpty(): Promise<boolean> {
|
||
const result = await db.select({ count: sql<number>`COUNT(*)` }).from(character);
|
||
return result[0]?.count === 0;
|
||
}
|
||
|
||
async function importFromJson(): Promise<void> {
|
||
let totalSuccess = 0;
|
||
let totalErrors = 0;
|
||
|
||
try {
|
||
const arcs = readJsonFile<ArcRecord>('./scraped-data/arcs.json');
|
||
if (arcs) {
|
||
console.log('\n=== Importing Arcs ===\n');
|
||
console.log(`Found ${arcs.length} arcs\n`);
|
||
|
||
let successCount = 0;
|
||
let errorCount = 0;
|
||
|
||
for (let i = 0; i < arcs.length; i++) {
|
||
const item = arcs[i];
|
||
let lastSql: { sql: string; params: unknown[] } | null = null;
|
||
try {
|
||
const query = db
|
||
.insert(arc)
|
||
.values({
|
||
id: item.id,
|
||
name: item.name,
|
||
startChapter: item.startChapter,
|
||
endChapter: toNullable(item.endChapter),
|
||
url: toNullable(item.url)
|
||
})
|
||
.onConflictDoUpdate({
|
||
target: arc.id,
|
||
set: {
|
||
name: item.name,
|
||
startChapter: item.startChapter,
|
||
endChapter: toNullable(item.endChapter),
|
||
url: toNullable(item.url)
|
||
}
|
||
});
|
||
|
||
lastSql = query.toSQL();
|
||
await query;
|
||
|
||
successCount++;
|
||
process.stdout.write(`\rExecuted: ${successCount}/${arcs.length}`);
|
||
} catch (error) {
|
||
errorCount++;
|
||
console.error(`\n✗ Error at arc ${i + 1}:`);
|
||
console.error(` ID: ${item.id ?? 'N/A'}`);
|
||
console.error(` Message: ${getErrorMessage(error)}`);
|
||
logSqlOnError(lastSql);
|
||
}
|
||
}
|
||
|
||
console.log(`\n\n✓ Arcs imported!`);
|
||
console.log(` Success: ${successCount}`);
|
||
console.log(` Errors: ${errorCount}`);
|
||
|
||
totalSuccess += successCount;
|
||
totalErrors += errorCount;
|
||
} else {
|
||
console.log('\n⚠️ No arcs.json found, skipping...\n');
|
||
}
|
||
|
||
const fruits = readJsonFile<DevilFruitRecord>('./scraped-data/devil-fruits.json');
|
||
if (fruits) {
|
||
console.log('\n=== Importing Devil Fruits ===\n');
|
||
console.log(`Found ${fruits.length} devil fruits\n`);
|
||
|
||
let successCount = 0;
|
||
let errorCount = 0;
|
||
|
||
for (let i = 0; i < fruits.length; i++) {
|
||
const item = fruits[i];
|
||
let lastSql: { sql: string; params: unknown[] } | null = null;
|
||
try {
|
||
const query = db
|
||
.insert(devilFruit)
|
||
.values({
|
||
id: item.id,
|
||
name: item.name,
|
||
type: toDevilFruitType(item.type),
|
||
url: toNullable(item.url)
|
||
})
|
||
.onConflictDoUpdate({
|
||
target: devilFruit.id,
|
||
set: {
|
||
name: item.name,
|
||
type: toDevilFruitType(item.type),
|
||
url: toNullable(item.url)
|
||
}
|
||
});
|
||
|
||
lastSql = query.toSQL();
|
||
await query;
|
||
|
||
successCount++;
|
||
process.stdout.write(`\rExecuted: ${successCount}/${fruits.length}`);
|
||
} catch (error) {
|
||
errorCount++;
|
||
console.error(`\n✗ Error at devil fruit ${i + 1}:`);
|
||
console.error(` ID: ${item.id ?? 'N/A'}`);
|
||
console.error(` Message: ${getErrorMessage(error)}`);
|
||
logSqlOnError(lastSql);
|
||
}
|
||
}
|
||
|
||
console.log(`\n\n✓ Devil Fruits imported!`);
|
||
console.log(` Success: ${successCount}`);
|
||
console.log(` Errors: ${errorCount}`);
|
||
|
||
totalSuccess += successCount;
|
||
totalErrors += errorCount;
|
||
} else {
|
||
console.log('\n⚠️ No devil-fruits.json found, skipping...\n');
|
||
}
|
||
|
||
const characters = readJsonFile<CharacterRecord>('./scraped-data/characters.json');
|
||
if (characters) {
|
||
console.log('\n=== Importing Characters ===\n');
|
||
console.log(`Found ${characters.length} characters\n`);
|
||
|
||
const isEmpty = await isCharacterTableEmpty();
|
||
|
||
let successCount = 0;
|
||
let errorCount = 0;
|
||
|
||
if (isEmpty) {
|
||
// Populate empty character table
|
||
console.log('Characters table is empty, populating...\n');
|
||
|
||
for (let i = 0; i < characters.length; i++) {
|
||
const item = characters[i];
|
||
let lastSql: { sql: string; params: unknown[] } | null = null;
|
||
try {
|
||
const data = transformCharacterData(item);
|
||
const query = db
|
||
.insert(character)
|
||
.values(data)
|
||
.onConflictDoUpdate({
|
||
target: character.id,
|
||
set: data
|
||
});
|
||
|
||
lastSql = query.toSQL();
|
||
await query;
|
||
|
||
successCount++;
|
||
process.stdout.write(`\rExecuted: ${successCount}/${characters.length}`);
|
||
} catch (error) {
|
||
errorCount++;
|
||
console.error(`\n✗ Error at character ${i + 1}:`);
|
||
console.error(` ID: ${item.id ?? 'N/A'}`);
|
||
console.error(` Message: ${getErrorMessage(error)}`);
|
||
logSqlOnError(lastSql);
|
||
}
|
||
}
|
||
} else {
|
||
// Update scrapeValidation table
|
||
console.log('Characters table not empty, updating scrapeValidation table for changes...\n');
|
||
|
||
for (let i = 0; i < characters.length; i++) {
|
||
const item = characters[i];
|
||
let lastSql: { sql: string; params: unknown[] } | null = null;
|
||
try {
|
||
const selectQuery = db
|
||
.select()
|
||
.from(character)
|
||
.where(eq(character.id, item.id));
|
||
|
||
lastSql = selectQuery.toSQL();
|
||
|
||
const jsonData = transformCharacterData(item);
|
||
|
||
const upsertQuery = db
|
||
.insert(characterScrapeValidation)
|
||
.values(jsonData)
|
||
.onConflictDoUpdate({
|
||
target: characterScrapeValidation.id,
|
||
set: jsonData
|
||
});
|
||
|
||
lastSql = upsertQuery.toSQL();
|
||
await upsertQuery;
|
||
successCount++;
|
||
process.stdout.write(`\rProcessed: ${successCount}/${characters.length}`);
|
||
} catch (error) {
|
||
errorCount++;
|
||
console.error(`\n✗ Error at character ${i + 1}:`);
|
||
console.error(` ID: ${item.id ?? 'N/A'}`);
|
||
console.error(` Message: ${getErrorMessage(error)}`);
|
||
logSqlOnError(lastSql);
|
||
}
|
||
}
|
||
}
|
||
|
||
console.log(`\n\n✓ Characters imported!`);
|
||
console.log(` Success: ${successCount}`);
|
||
console.log(` Errors: ${errorCount}`);
|
||
|
||
totalSuccess += successCount;
|
||
totalErrors += errorCount;
|
||
} else {
|
||
console.log('\n⚠️ No characters.json found, skipping...\n');
|
||
}
|
||
|
||
console.log(`\n=== Total Import Summary ===`);
|
||
console.log(` Total Success: ${totalSuccess}`);
|
||
console.log(` Total Errors: ${totalErrors}\n`);
|
||
} catch (error) {
|
||
console.error('✗ Import failed:', getErrorMessage(error));
|
||
process.exit(1);
|
||
} finally {
|
||
client.close();
|
||
}
|
||
}
|
||
|
||
importFromJson().catch((error) => {
|
||
console.error(getErrorMessage(error));
|
||
process.exit(1);
|
||
});
|