feat: enhance character scrape validation and management

- Added a new entry for "fuzzy_talisman" in the journal.
- Updated import-json script to handle character deletion and mark absent characters as deleted in the scrape validation.
- Modified schema to include an `isDeleted` field in the characterScrapeValidation table.
- Renamed function `upsertCharacterFromScrapeValidation` to `applyCharacterChangeFromScrapeValidation` for clarity.
- Enhanced character change loading to include deleted characters and updated UI to display them.
- Improved character change handling in the Svelte component to reflect new, modified, and deleted states.

drizzle/0001_fuzzy_talisman.sql

@@ -0,0 +1 @@
+ALTER TABLE `character_scrape_validation` ADD `is_deleted` integer DEFAULT false;

drizzle/meta/_journal.json

@@ -8,6 +8,13 @@
       "when": 1773602933375,
       "tag": "0000_huge_doctor_octopus",
       "breakpoints": true
+    },
+    {
+      "idx": 1,
+      "version": "6",
+      "when": 1773697753818,
+      "tag": "0001_fuzzy_talisman",
+      "breakpoints": true
     }
   ]
 }

scripts/import-json.ts

@@ -1,6 +1,6 @@
 import { createClient } from '@libsql/client';
 import { drizzle } from 'drizzle-orm/libsql';
-import { sql, eq } from 'drizzle-orm';
+import { sql, eq, inArray } from 'drizzle-orm';
 import fs from 'fs';
 import { arc, character, devilFruit, characterScrapeValidation, type DevilFruitType } from '../src/lib/server/db/schema';
 
@@ -140,7 +140,8 @@ function transformCharacterData(item: CharacterRecord) {
 		status: toNullable(item.status),
 		arcId: toNullable(item.arcId),
 		url: toNullable(item.url),
-		frUrl: toNullable(item.frUrl)
+		frUrl: toNullable(item.frUrl),
+		isDeleted: false
 	};
 }
 
@@ -307,6 +308,7 @@ async function importFromJson(): Promise<void> {
 			} else {
 				// Update scrapeValidation table
 				console.log('Characters table not empty, updating scrapeValidation table for changes...\n');
+				const scrapedCharacterIds: string[] = [];
 
 				for (let i = 0; i < characters.length; i++) {
 					const item = characters[i];
@@ -319,6 +321,7 @@ async function importFromJson(): Promise<void> {
 
 						lastSql = selectQuery.toSQL();
 
+						scrapedCharacterIds.push(item.id);
 						const jsonData = transformCharacterData(item);
 
 						const upsertQuery = db
@@ -341,6 +344,57 @@ async function importFromJson(): Promise<void> {
 						logSqlOnError(lastSql);
 					}
 				}
+
+				// Fetch all characters from the character table and mark those absent from the
+				// scrape as deleted in scrape validation.
+				const allCharacters = await db.select({ id: character.id }).from(character);
+				const scrapedSet = new Set(scrapedCharacterIds);
+				const idsToMarkDeleted = allCharacters
+					.map((c) => c.id)
+					.filter((id) => !scrapedSet.has(id));
+
+				if (idsToMarkDeleted.length > 0) {
+					console.log(`\n⚠️  Marking ${idsToMarkDeleted.length} character(s) as deleted in scrape validation...`);
+					const deletedCharacterRows = await db
+						.select()
+						.from(character)
+						.where(inArray(character.id, idsToMarkDeleted));
+
+					for (const row of deletedCharacterRows) {
+						await db
+							.insert(characterScrapeValidation)
+							.values({
+								id: row.id,
+								name: row.name,
+								frName: row.frName,
+								gender: row.gender,
+								age: row.age,
+								affiliations: row.affiliations,
+								frAffiliations: row.frAffiliations,
+								devilFruitId: row.devilFruitId,
+								hakiObservation: row.hakiObservation,
+								hakiArmament: row.hakiArmament,
+								hakiConqueror: row.hakiConqueror,
+								bounty: row.bounty,
+								height: row.height,
+								origin: row.origin,
+								frOrigin: row.frOrigin,
+								firstAppearance: row.firstAppearance,
+								pictureUrl: row.pictureUrl,
+								epithets: row.epithets,
+								frEpithets: row.frEpithets,
+								status: row.status,
+								arcId: row.arcId,
+								url: row.url,
+								frUrl: row.frUrl,
+								isDeleted: true
+							})
+							.onConflictDoUpdate({
+								target: characterScrapeValidation.id,
+								set: { isDeleted: true }
+							});
+					}
+				}
 			}
 
 			console.log(`\n\n✓ Characters imported!`);

src/lib/server/db/schema.ts

@@ -119,7 +119,8 @@ export const characterScrapeValidation = sqliteTable('character_scrape_validatio
 	status: text('status').$type<Status | null>(),
 	arcId: text('arc_id').references(() => arc.id, { onDelete: 'set null' }),
 	url: text('url'),
-	frUrl: text('fr_url')
+	frUrl: text('fr_url'),
+	isDeleted: integer('is_deleted', { mode: 'boolean' }).default(false),
 });
 
 export type CharacterScrapeValidation = InferSelectModel<typeof characterScrapeValidation>;

src/routes/(admin)/admin/character-changes/+page.server.ts

@@ -11,7 +11,7 @@ const EXEC_OPTIONS = {
 	maxBuffer: 50 * 1024 * 1024
 };
 
-async function upsertCharacterFromScrapeValidation(characterId: string): Promise<boolean> {
+async function applyCharacterChangeFromScrapeValidation(characterId: string): Promise<boolean> {
 	const [scraped] = await db
 		.select()
 		.from(characterScrapeValidation)
@@ -21,6 +21,11 @@ async function upsertCharacterFromScrapeValidation(characterId: string): Promise
 		return false;
 	}
 
+	if (scraped.isDeleted) {
+		await db.delete(character).where(eq(character.id, characterId));
+		return true;
+	}
+
 	await db
 		.insert(character)
 		.values({
@@ -87,7 +92,7 @@ export async function load() {
 
 	// Compare and categorize changes
 	const changes: {
-		type: 'new' | 'modified';
+		type: 'new' | 'modified' | 'deleted';
 		id: string;
 		scraped: (typeof scrapedCharacters)[0];
 		current?: (typeof currentCharacters)[0];
@@ -97,6 +102,18 @@ export async function load() {
 	for (const scraped of scrapedCharacters) {
 		const current = currentCharMap.get(scraped.id);
 
+		if (scraped.isDeleted) {
+			if (current) {
+				changes.push({
+					type: 'deleted',
+					id: scraped.id,
+					scraped,
+					current
+				});
+			}
+			continue;
+		}
+
 		if (!current) {
 			// New character
 			changes.push({
@@ -156,11 +173,16 @@ export async function load() {
 		}
 	}
 
+	const typeOrder: Record<'new' | 'modified' | 'deleted', number> = {
+		new: 0,
+		modified: 1,
+		deleted: 2
+	};
+
 	return {
 		changes: changes.sort((a, b) => {
-			// Show 'new' first, then 'modified'
 			if (a.type !== b.type) {
-				return a.type === 'new' ? -1 : 1;
+				return typeOrder[a.type] - typeOrder[b.type];
 			}
 			return a.id.localeCompare(b.id);
 		})
@@ -221,10 +243,10 @@ export const actions = {
 			return { success: false, message: 'characterId is required' };
 		}
 
-		const applied = await upsertCharacterFromScrapeValidation(characterId);
+		const applied = await applyCharacterChangeFromScrapeValidation(characterId);
 		return {
 			success: applied,
-			message: applied ? 'Character applied successfully' : 'Character not found in scrape validation table'
+			message: applied ? 'Character change applied successfully' : 'Character not found in scrape validation table'
 		};
 	},
 
@@ -233,7 +255,7 @@ export const actions = {
 		let appliedCount = 0;
 
 		for (const scraped of scrapedCharacters) {
-			const applied = await upsertCharacterFromScrapeValidation(scraped.id);
+			const applied = await applyCharacterChangeFromScrapeValidation(scraped.id);
 			if (applied) {
 				appliedCount++;
 			}

src/routes/(admin)/admin/character-changes/+page.svelte

@@ -1,10 +1,30 @@
 <script lang="ts">
+	type CharacterLike = {
+		name: string;
+		pictureUrl?: string | null;
+		url?: string | null;
+		status?: string | null;
+		gender?: string | null;
+		age?: number | null;
+		bounty?: number | null;
+		[key: string]: unknown;
+	};
+
+	type CharacterChange = {
+		type: 'new' | 'modified' | 'deleted';
+		id: string;
+		scraped: CharacterLike;
+		current?: CharacterLike;
+		differences?: Record<string, { current: unknown; scraped: unknown }>;
+	};
+
 	let { data, form } = $props();
 
-	const newCharacters = $derived(data.changes.filter((c: any) => c.type === 'new'));
+	const newCharacters = $derived((data.changes as CharacterChange[]).filter((c) => c.type === 'new'));
-	const modifiedCharacters = $derived(data.changes.filter((c: any) => c.type === 'modified'));
+	const modifiedCharacters = $derived((data.changes as CharacterChange[]).filter((c) => c.type === 'modified'));
+	const deletedCharacters = $derived((data.changes as CharacterChange[]).filter((c) => c.type === 'deleted'));
 
-	function formatValue(value: any): string {
+	function formatValue(value: unknown): string {
 		if (value === null || value === undefined) {
 			return '—';
 		}
@@ -25,7 +45,7 @@
 <div class="space-y-8">
 	<div>
 		<h1 class="text-3xl font-black uppercase tracking-[0.25em] text-amber-50 mb-2">Character Changes</h1>
-		<p class="text-gray-400">Total changes: {newCharacters.length} new, {modifiedCharacters.length} modified</p>
+		<p class="text-gray-400">Total changes: {newCharacters.length} new, {modifiedCharacters.length} modified, {deletedCharacters.length} deleted</p>
 		<form method="POST" action="?/runScrapeImport" class="mt-4">
 			<button
 				type="submit"
@@ -42,7 +62,7 @@
 		{#if form?.logs}
 			<pre class="mt-3 max-h-72 overflow-auto rounded-lg border border-white/10 bg-slate-900/70 p-3 text-xs text-slate-200 whitespace-pre-wrap">{form.logs}</pre>
 		{/if}
-		{#if newCharacters.length + modifiedCharacters.length > 0}
+		{#if newCharacters.length + modifiedCharacters.length + deletedCharacters.length > 0}
 			<form method="POST" action="?/acceptAll" class="mt-4">
 				<button
 					type="submit"
@@ -68,7 +88,7 @@
 								{#if change.scraped.pictureUrl}
 									<a href="https://onepiece.fandom.com/fr/wiki/{change.scraped.url}" target="_blank" rel="noopener noreferrer">
 										<img
-											src={change.scraped.pictureUrl}
+											src={change.scraped.pictureUrl ?? undefined}
 											alt={change.scraped.name}
 											class="w-12 h-12 rounded object-cover hover:opacity-80 transition"
 										/>
@@ -127,8 +147,8 @@
 								{#if change.current?.pictureUrl}
 									<a href="https://onepiece.fandom.com/fr/wiki/{change.current?.url ?? change.scraped.url}" target="_blank" rel="noopener noreferrer">
 										<img
-											src={change.current.pictureUrl}
+											src={change.current?.pictureUrl ?? undefined}
-											alt={change.current.name}
+											alt={change.current?.name ?? change.scraped.name}
 											class="w-12 h-12 rounded object-cover hover:opacity-80 transition"
 										/>
 									</a>
@@ -174,7 +194,49 @@
 		</section>
 	{/if}
 
-	{#if newCharacters.length === 0 && modifiedCharacters.length === 0}
+	<!-- Deleted Characters Section -->
+	{#if deletedCharacters.length > 0}
+		<section class="space-y-4">
+			<h2 class="text-xl font-bold text-rose-400 uppercase tracking-[0.15em]">
+				🗑️ Deleted Characters ({deletedCharacters.length})
+			</h2>
+			<div class="grid gap-4">
+				{#each deletedCharacters as change (change.id)}
+					<div class="rounded-lg border border-rose-500/30 bg-rose-500/5 p-4 space-y-3">
+						<div class="flex items-center justify-between gap-3">
+							<div class="flex items-center gap-3">
+								{#if change.current?.pictureUrl}
+									<a href="https://onepiece.fandom.com/fr/wiki/{change.current?.url ?? change.scraped.url}" target="_blank" rel="noopener noreferrer">
+										<img
+											src={change.current?.pictureUrl ?? undefined}
+											alt={change.current?.name ?? change.scraped.name}
+											class="w-12 h-12 rounded object-cover hover:opacity-80 transition"
+										/>
+									</a>
+								{/if}
+								<div>
+									<h3 class="font-bold text-rose-300">{change.current?.name ?? change.scraped.name}</h3>
+									<p class="text-sm text-gray-500">{change.id}</p>
+								</div>
+							</div>
+							<form method="POST" action="?/acceptOne">
+								<input type="hidden" name="characterId" value={change.id} />
+								<button
+									type="submit"
+									class="rounded-full border border-rose-300/40 bg-rose-500/20 px-3 py-1 text-xs font-semibold text-rose-100 transition hover:bg-rose-500/30"
+								>
+									Supprimer
+								</button>
+							</form>
+						</div>
+						<p class="text-sm text-rose-200/80">This character is no longer present in the latest scrape and will be removed if accepted.</p>
+					</div>
+				{/each}
+			</div>
+		</section>
+	{/if}
+
+	{#if newCharacters.length === 0 && modifiedCharacters.length === 0 && deletedCharacters.length === 0}
 		<div class="rounded-lg border border-white/10 bg-white/5 p-8 text-center">
 			<p class="text-gray-400">Aucun changement détecté. Les tables character et characterScrapeValidation sont synchronisées.</p>
 		</div>