OnePieceDle/scripts/import-json.ts

import { createClient } from '@libsql/client';
import { drizzle } from 'drizzle-orm/libsql';
import { sql, eq, inArray } from 'drizzle-orm';
import fs from 'fs';
import { arc, character, devilFruit, characterScrapeValidation, type DevilFruitType } from '../src/lib/server/db/schema';

type Status = 'Alive' | 'Dead' | 'Unknown';

type ArcRecord = {
	id: string;
	name: string;
	frName?: string | null;
	startChapter: number;
	endChapter?: number | null;
	url?: string | null;
};

type DevilFruitRecord = {
	id: string;
	name: string;
	type?: DevilFruitType | string | null;
	url?: string | null;
};

type CharacterRecord = {
	id: string;
	name: string;
	frName?: string | null;
	gender?: string | null;
	age?: number | null;
	affiliations?: string[] | string | null;
	frAffiliations?: string[] | string | null;
	devilFruitId?: string | null;
	hakiObservation?: boolean;
	hakiArmament?: boolean;
	hakiConqueror?: boolean;
	bounty?: number | null;
	height?: number | null;
	origin?: string | null;
	frOrigin?: string | null;
	firstAppearance?: number;
	pictureUrl?: string | null;
	epithets?: string[] | string | null;
	frEpithets?: string[] | string | null;
	status?: Status | null;
	arcId?: string | null;
	url?: string | null;
	frUrl?: string | null;
};

const DATABASE_URL = process.env.DATABASE_URL || 'file:local.db';

const client = createClient({ url: DATABASE_URL });
const db = drizzle(client);

function readJsonFile<T>(path: string): T[] | null {
	if (!fs.existsSync(path)) {
		return null;
	}

	const content = fs.readFileSync(path, 'utf-8');
	return JSON.parse(content) as T[];
}

function toNullable<T>(value: T | undefined | null | ''): T | null {
	return value === undefined || value === null || value === '' ? null : value;
}

function toJsonArray(value: string[] | string | null | undefined): string[] | null {
	if (Array.isArray(value)) {
		return value.length > 0 ? value : null;
	}

	if (typeof value === 'string' && value.trim() !== '') {
		if (value.trim().startsWith('[')) {
			try {
				const parsed = JSON.parse(value);
				return Array.isArray(parsed) ? parsed : [value];
			} catch {
				return [value];
			}
		}

		const splitValues = value
			.split(',')
			.map((item) => item.trim())
			.filter(Boolean);

		return splitValues.length > 0 ? splitValues : null;
	}

	return null;
}

function toDevilFruitType(value: DevilFruitType | string | null | undefined): DevilFruitType | null {
	if (!value) return null;
	if (value === 'Paramecia' || value === 'Zoan' || value === 'Logia' || value === 'Smile' || value === 'Unknown') {
		return value;
	}
	return 'Unknown';
}

function toNumber(value: string | number | null | undefined): number | null {
	if (value === null || value === undefined || value === '') return null;
	const num = typeof value === 'string' ? parseFloat(value) : value;
	return isNaN(num) ? null : num;
}

function getErrorMessage(error: unknown): string {
	return error instanceof Error ? error.message : String(error);
}

function logSqlOnError(statement: { sql: string; params: unknown[] } | null): void {
	if (!statement) return;
	console.error(`  SQL: ${statement.sql}`);
	console.error(`  Params: ${JSON.stringify(statement.params)}`);
}

function transformCharacterData(item: CharacterRecord) {
	return {
		id: item.id,
		name: item.name,
		frName: toNullable(item.frName),
		gender: toNullable(item.gender),
		age: toNullable(item.age),
		affiliations: toJsonArray(item.affiliations),
		frAffiliations: toJsonArray(item.frAffiliations),
		devilFruitId: toNullable(item.devilFruitId),
		hakiObservation: !!item.hakiObservation,
		hakiArmament: !!item.hakiArmament,
		hakiConqueror: !!item.hakiConqueror,
		bounty: item.bounty ?? 0,
		height: toNumber(item.height as string | number | null),
		origin: toNullable(item.origin),
		frOrigin: toNullable(item.frOrigin),
		firstAppearance: item.firstAppearance ?? 0,
		pictureUrl: toNullable(item.pictureUrl),
		epithets: toJsonArray(item.epithets),
		frEpithets: toJsonArray(item.frEpithets),
		status: toNullable(item.status),
		arcId: toNullable(item.arcId),
		url: toNullable(item.url),
		frUrl: toNullable(item.frUrl),
		isDeleted: false
	};
}

async function isCharacterTableEmpty(): Promise<boolean> {
	const result = await db.select({ count: sql<number>`COUNT(*)` }).from(character);
	return result[0]?.count === 0;
}

async function importFromJson(): Promise<void> {
	let totalSuccess = 0;
	let totalErrors = 0;

	try {
		const arcs = readJsonFile<ArcRecord>('./scraped-data/arcs.json');
		if (arcs) {
			console.log('\n=== Importing Arcs ===\n');
			console.log(`Found ${arcs.length} arcs\n`);

			let successCount = 0;
			let errorCount = 0;

			for (let i = 0; i < arcs.length; i++) {
				const item = arcs[i];
				let lastSql: { sql: string; params: unknown[] } | null = null;
				try {
					const query = db
						.insert(arc)
						.values({
							id: item.id,
							name: item.name,
							frName: toNullable(item.frName),
							startChapter: item.startChapter,
							endChapter: toNullable(item.endChapter),
							url: toNullable(item.url)
						})
						.onConflictDoUpdate({
							target: arc.id,
							set: {
								name: item.name,
								frName: toNullable(item.frName),
								startChapter: item.startChapter,
								endChapter: toNullable(item.endChapter),
								url: toNullable(item.url)
							}
						});

					lastSql = query.toSQL();
					await query;

					successCount++;
					process.stdout.write(`\rExecuted: ${successCount}/${arcs.length}`);
				} catch (error) {
					errorCount++;
					console.error(`\n✗ Error at arc ${i + 1}:`);
					console.error(`  ID: ${item.id ?? 'N/A'}`);
					console.error(`  Message: ${getErrorMessage(error)}`);
					logSqlOnError(lastSql);
				}
			}

			console.log(`\n\n✓ Arcs imported!`);
			console.log(`  Success: ${successCount}`);
			console.log(`  Errors: ${errorCount}`);

			totalSuccess += successCount;
			totalErrors += errorCount;
		} else {
			console.log('\n⚠️  No arcs.json found, skipping...\n');
		}

		const fruits = readJsonFile<DevilFruitRecord>('./scraped-data/devil-fruits.json');
		if (fruits) {
			console.log('\n=== Importing Devil Fruits ===\n');
			console.log(`Found ${fruits.length} devil fruits\n`);

			let successCount = 0;
			let errorCount = 0;

			for (let i = 0; i < fruits.length; i++) {
				const item = fruits[i];
				let lastSql: { sql: string; params: unknown[] } | null = null;
				try {
					const query = db
						.insert(devilFruit)
						.values({
							id: item.id,
							name: item.name,
							type: toDevilFruitType(item.type),
							url: toNullable(item.url)
						})
						.onConflictDoUpdate({
							target: devilFruit.id,
							set: {
								name: item.name,
								type: toDevilFruitType(item.type),
								url: toNullable(item.url)
							}
						});

					lastSql = query.toSQL();
					await query;

					successCount++;
					process.stdout.write(`\rExecuted: ${successCount}/${fruits.length}`);
				} catch (error) {
					errorCount++;
					console.error(`\n✗ Error at devil fruit ${i + 1}:`);
					console.error(`  ID: ${item.id ?? 'N/A'}`);
					console.error(`  Message: ${getErrorMessage(error)}`);
					logSqlOnError(lastSql);
				}
			}

			console.log(`\n\n✓ Devil Fruits imported!`);
			console.log(`  Success: ${successCount}`);
			console.log(`  Errors: ${errorCount}`);

			totalSuccess += successCount;
			totalErrors += errorCount;
		} else {
			console.log('\n⚠️  No devil-fruits.json found, skipping...\n');
		}

		const characters = readJsonFile<CharacterRecord>('./scraped-data/characters.json');
		if (characters) {
			console.log('\n=== Importing Characters ===\n');
			console.log(`Found ${characters.length} characters\n`);

			const isEmpty = await isCharacterTableEmpty();

			let successCount = 0;
			let errorCount = 0;

			if (isEmpty) {
				// Populate empty character table
				console.log('Characters table is empty, populating...\n');

				for (let i = 0; i < characters.length; i++) {
					const item = characters[i];
					let lastSql: { sql: string; params: unknown[] } | null = null;
					try {
						const data = transformCharacterData(item);
						const query = db
							.insert(character)
							.values(data)
							.onConflictDoUpdate({
								target: character.id,
								set: data
							});

						lastSql = query.toSQL();
						await query;

						successCount++;
						process.stdout.write(`\rExecuted: ${successCount}/${characters.length}`);
					} catch (error) {
						errorCount++;
						console.error(`\n✗ Error at character ${i + 1}:`);
						console.error(`  ID: ${item.id ?? 'N/A'}`);
						console.error(`  Message: ${getErrorMessage(error)}`);
						logSqlOnError(lastSql);
					}
				}
			} else {
				// Update scrapeValidation table
				console.log('Characters table not empty, updating scrapeValidation table for changes...\n');
				const scrapedCharacterIds: string[] = [];

				for (let i = 0; i < characters.length; i++) {
					const item = characters[i];
					let lastSql: { sql: string; params: unknown[] } | null = null;
					try {
						const selectQuery = db
							.select()
							.from(character)
							.where(eq(character.id, item.id));

						lastSql = selectQuery.toSQL();

						scrapedCharacterIds.push(item.id);
						const jsonData = transformCharacterData(item);

						const upsertQuery = db
							.insert(characterScrapeValidation)
							.values(jsonData)
							.onConflictDoUpdate({
								target: characterScrapeValidation.id,
								set: jsonData
							});

						lastSql = upsertQuery.toSQL();
						await upsertQuery;
						successCount++;
						process.stdout.write(`\rProcessed: ${successCount}/${characters.length}`);
					} catch (error) {
						errorCount++;
						console.error(`\n✗ Error at character ${i + 1}:`);
						console.error(`  ID: ${item.id ?? 'N/A'}`);
						console.error(`  Message: ${getErrorMessage(error)}`);
						logSqlOnError(lastSql);
					}
				}

				// Fetch all characters from the character table and mark those absent from the
				// scrape as deleted in scrape validation.
				const allCharacters = await db.select({ id: character.id }).from(character);
				const scrapedSet = new Set(scrapedCharacterIds);
				const idsToMarkDeleted = allCharacters
					.map((c) => c.id)
					.filter((id) => !scrapedSet.has(id));

				if (idsToMarkDeleted.length > 0) {
					console.log(`\n⚠️  Marking ${idsToMarkDeleted.length} character(s) as deleted in scrape validation...`);
					const deletedCharacterRows = await db
						.select()
						.from(character)
						.where(inArray(character.id, idsToMarkDeleted));

					for (const row of deletedCharacterRows) {
						await db
							.insert(characterScrapeValidation)
							.values({
								id: row.id,
								name: row.name,
								frName: row.frName,
								gender: row.gender,
								age: row.age,
								affiliations: row.affiliations,
								frAffiliations: row.frAffiliations,
								devilFruitId: row.devilFruitId,
								hakiObservation: row.hakiObservation,
								hakiArmament: row.hakiArmament,
								hakiConqueror: row.hakiConqueror,
								bounty: row.bounty,
								height: row.height,
								origin: row.origin,
								frOrigin: row.frOrigin,
								firstAppearance: row.firstAppearance,
								pictureUrl: row.pictureUrl,
								epithets: row.epithets,
								frEpithets: row.frEpithets,
								status: row.status,
								arcId: row.arcId,
								url: row.url,
								frUrl: row.frUrl,
								isDeleted: true
							})
							.onConflictDoUpdate({
								target: characterScrapeValidation.id,
								set: { isDeleted: true }
							});
					}
				}
			}

			console.log(`\n\n✓ Characters imported!`);
			console.log(`  Success: ${successCount}`);
			console.log(`  Errors: ${errorCount}`);

			totalSuccess += successCount;
			totalErrors += errorCount;
		} else {
			console.log('\n⚠️  No characters.json found, skipping...\n');
		}

		console.log(`\n=== Total Import Summary ===`);
		console.log(`  Total Success: ${totalSuccess}`);
		console.log(`  Total Errors: ${totalErrors}\n`);
	} catch (error) {
		console.error('✗ Import failed:', getErrorMessage(error));
		process.exit(1);
	} finally {
		client.close();
	}
}

importFromJson().catch((error) => {
	console.error(getErrorMessage(error));
	process.exit(1);
});