Whidix/OnePieceDle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: enhance character scrape validation and management
All checks were successful
Build Docker Image / build (push) Successful in 1m14s
Details

Browse Source 
- Added a new entry for "fuzzy_talisman" in the journal.
- Updated import-json script to handle character deletion and mark absent characters as deleted in the scrape validation.
- Modified schema to include an `isDeleted` field in the characterScrapeValidation table.
- Renamed function `upsertCharacterFromScrapeValidation` to `applyCharacterChangeFromScrapeValidation` for clarity.
- Enhanced character change loading to include deleted characters and updated UI to display them.
- Improved character change handling in the Svelte component to reflect new, modified, and deleted states.
This commit is contained in:
whidix
2026-03-16 23:12:06 +01:00
parent 7760570365
commit 5ad0428420
7 changed files with 1562 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
drizzle/0001_fuzzy_talisman.sql Normal file
View File

@@ -0,0 +1 @@
ALTER TABLE `character_scrape_validation` ADD `is_deleted` integer DEFAULT false;

1396
drizzle/meta/0001_snapshot.json Normal file
View File

File diff suppressed because it is too large Load Diff

7
drizzle/meta/_journal.json
View File

@@ -8,6 +8,13 @@
"when": 1773602933375,
"tag": "0000_huge_doctor_octopus",
"breakpoints": true
},
{
"idx": 1,
"version": "6",
"when": 1773697753818,
"tag": "0001_fuzzy_talisman",
"breakpoints": true
}
]
}

58
scripts/import-json.ts
View File

@@ -1,6 +1,6 @@
import { createClient } from '@libsql/client';
import { drizzle } from 'drizzle-orm/libsql';
import { sql, eq } from 'drizzle-orm';
import { sql, eq, inArray } from 'drizzle-orm';
import fs from 'fs';
import { arc, character, devilFruit, characterScrapeValidation, type DevilFruitType } from '../src/lib/server/db/schema';
@@ -140,7 +140,8 @@ function transformCharacterData(item: CharacterRecord) {
status: toNullable(item.status),
arcId: toNullable(item.arcId),
url: toNullable(item.url),
frUrl: toNullable(item.frUrl)
frUrl: toNullable(item.frUrl),
isDeleted: false
};
}
@@ -307,6 +308,7 @@ async function importFromJson(): Promise<void> {
} else {
// Update scrapeValidation table
console.log('Characters table not empty, updating scrapeValidation table for changes...\n');
const scrapedCharacterIds: string[] = [];
for (let i = 0; i < characters.length; i++) {
const item = characters[i];
@@ -319,6 +321,7 @@ async function importFromJson(): Promise<void> {
lastSql = selectQuery.toSQL();
scrapedCharacterIds.push(item.id);
const jsonData = transformCharacterData(item);
const upsertQuery = db
@@ -341,6 +344,57 @@ async function importFromJson(): Promise<void> {
logSqlOnError(lastSql);
}
}
// Fetch all characters from the character table and mark those absent from the
// scrape as deleted in scrape validation.
const allCharacters = await db.select({ id: character.id }).from(character);
const scrapedSet = new Set(scrapedCharacterIds);
const idsToMarkDeleted = allCharacters
.map((c) => c.id)
.filter((id) => !scrapedSet.has(id));
if (idsToMarkDeleted.length > 0) {
console.log(`\n⚠️ Marking ${idsToMarkDeleted.length} character(s) as deleted in scrape validation...`);
const deletedCharacterRows = await db
.select()
.from(character)
.where(inArray(character.id, idsToMarkDeleted));
for (const row of deletedCharacterRows) {
await db
.insert(characterScrapeValidation)
.values({
id: row.id,
name: row.name,
frName: row.frName,
gender: row.gender,
age: row.age,
affiliations: row.affiliations,
frAffiliations: row.frAffiliations,
devilFruitId: row.devilFruitId,
hakiObservation: row.hakiObservation,
hakiArmament: row.hakiArmament,
hakiConqueror: row.hakiConqueror,
bounty: row.bounty,
height: row.height,
origin: row.origin,
frOrigin: row.frOrigin,
firstAppearance: row.firstAppearance,
pictureUrl: row.pictureUrl,
epithets: row.epithets,
frEpithets: row.frEpithets,
status: row.status,
arcId: row.arcId,
url: row.url,
frUrl: row.frUrl,
isDeleted: true
})
.onConflictDoUpdate({
target: characterScrapeValidation.id,
set: { isDeleted: true }
});
}
}
}
console.log(`\n\n✓ Characters imported!`);

3
src/lib/server/db/schema.ts
View File

@@ -119,7 +119,8 @@ export const characterScrapeValidation = sqliteTable('character_scrape_validatio
status: text('status').$type<Status | null>(),
arcId: text('arc_id').references(() => arc.id, { onDelete: 'set null' }),
url: text('url'),
frUrl: text('fr_url')
frUrl: text('fr_url'),
isDeleted: integer('is_deleted', { mode: 'boolean' }).default(false),
});
export type CharacterScrapeValidation = InferSelectModel<typeof characterScrapeValidation>;

36
src/routes/(admin)/admin/character-changes/+page.server.ts
View File

@@ -11,7 +11,7 @@ const EXEC_OPTIONS = {
maxBuffer: 50 * 1024 * 1024
};
async function upsertCharacterFromScrapeValidation(characterId: string): Promise<boolean> {
async function applyCharacterChangeFromScrapeValidation(characterId: string): Promise<boolean> {
const [scraped] = await db
.select()
.from(characterScrapeValidation)
@@ -21,6 +21,11 @@ async function upsertCharacterFromScrapeValidation(characterId: string): Promise
return false;
}
if (scraped.isDeleted) {
await db.delete(character).where(eq(character.id, characterId));
return true;
}
await db
.insert(character)
.values({
@@ -87,7 +92,7 @@ export async function load() {
// Compare and categorize changes
const changes: {
type: 'new' | 'modified';
type: 'new' | 'modified' | 'deleted';
id: string;
scraped: (typeof scrapedCharacters)[0];
current?: (typeof currentCharacters)[0];
@@ -97,6 +102,18 @@ export async function load() {
for (const scraped of scrapedCharacters) {
const current = currentCharMap.get(scraped.id);
if (scraped.isDeleted) {
if (current) {
changes.push({
type: 'deleted',
id: scraped.id,
scraped,
current
});
}
continue;
}
if (!current) {
// New character
changes.push({
@@ -156,11 +173,16 @@ export async function load() {
}
}
const typeOrder: Record<'new' | 'modified' | 'deleted', number> = {
new: 0,
modified: 1,
deleted: 2
};
return {
changes: changes.sort((a, b) => {
// Show 'new' first, then 'modified'
if (a.type !== b.type) {
return a.type === 'new' ? -1 : 1;
return typeOrder[a.type] - typeOrder[b.type];
}
return a.id.localeCompare(b.id);
})
@@ -221,10 +243,10 @@ export const actions = {
return { success: false, message: 'characterId is required' };
}
const applied = await upsertCharacterFromScrapeValidation(characterId);
const applied = await applyCharacterChangeFromScrapeValidation(characterId);
return {
success: applied,
message: applied ? 'Character applied successfully' : 'Character not found in scrape validation table'
message: applied ? 'Character change applied successfully' : 'Character not found in scrape validation table'
};
},
@@ -233,7 +255,7 @@ export const actions = {
let appliedCount = 0;
for (const scraped of scrapedCharacters) {
const applied = await upsertCharacterFromScrapeValidation(scraped.id);
const applied = await applyCharacterChangeFromScrapeValidation(scraped.id);
if (applied) {
appliedCount++;
}

80
src/routes/(admin)/admin/character-changes/+page.svelte
View File

@@ -1,10 +1,30 @@
<script lang="ts">
type CharacterLike = {
name: string;
pictureUrl?: string | null;
url?: string | null;
status?: string | null;
gender?: string | null;
age?: number | null;
bounty?: number | null;
[key: string]: unknown;
};
type CharacterChange = {
type: 'new' | 'modified' | 'deleted';
id: string;
scraped: CharacterLike;
current?: CharacterLike;
differences?: Record<string, { current: unknown; scraped: unknown }>;
};
let { data, form } = $props();
const newCharacters = $derived(data.changes.filter((c: any) => c.type === 'new'));
const modifiedCharacters = $derived(data.changes.filter((c: any) => c.type === 'modified'));
const newCharacters = $derived((data.changes as CharacterChange[]).filter((c) => c.type === 'new'));
const modifiedCharacters = $derived((data.changes as CharacterChange[]).filter((c) => c.type === 'modified'));
const deletedCharacters = $derived((data.changes as CharacterChange[]).filter((c) => c.type === 'deleted'));
function formatValue(value: any): string {
function formatValue(value: unknown): string {
if (value === null || value === undefined) {
return '—';
}
@@ -25,7 +45,7 @@
<div class="space-y-8">
<div>
<h1 class="text-3xl font-black uppercase tracking-[0.25em] text-amber-50 mb-2">Character Changes</h1>
<p class="text-gray-400">Total changes: {newCharacters.length} new, {modifiedCharacters.length} modified</p>
<p class="text-gray-400">Total changes: {newCharacters.length} new, {modifiedCharacters.length} modified, {deletedCharacters.length} deleted</p>
<form method="POST" action="?/runScrapeImport" class="mt-4">
<button
type="submit"
@@ -42,7 +62,7 @@
{#if form?.logs}
<pre class="mt-3 max-h-72 overflow-auto rounded-lg border border-white/10 bg-slate-900/70 p-3 text-xs text-slate-200 whitespace-pre-wrap">{form.logs}</pre>
{/if}
{#if newCharacters.length + modifiedCharacters.length > 0}
{#if newCharacters.length + modifiedCharacters.length + deletedCharacters.length > 0}
<form method="POST" action="?/acceptAll" class="mt-4">
<button
type="submit"
@@ -68,7 +88,7 @@
{#if change.scraped.pictureUrl}
<a href="https://onepiece.fandom.com/fr/wiki/{change.scraped.url}" target="_blank" rel="noopener noreferrer">
<img
src={change.scraped.pictureUrl}
src={change.scraped.pictureUrl ?? undefined}
alt={change.scraped.name}
class="w-12 h-12 rounded object-cover hover:opacity-80 transition"
/>
@@ -127,8 +147,8 @@
{#if change.current?.pictureUrl}
<a href="https://onepiece.fandom.com/fr/wiki/{change.current?.url ?? change.scraped.url}" target="_blank" rel="noopener noreferrer">
<img
src={change.current.pictureUrl}
alt={change.current.name}
src={change.current?.pictureUrl ?? undefined}
alt={change.current?.name ?? change.scraped.name}
class="w-12 h-12 rounded object-cover hover:opacity-80 transition"
/>
</a>
@@ -174,7 +194,49 @@
</section>
{/if}
{#if newCharacters.length === 0 && modifiedCharacters.length === 0}
<!-- Deleted Characters Section -->
{#if deletedCharacters.length > 0}
<section class="space-y-4">
<h2 class="text-xl font-bold text-rose-400 uppercase tracking-[0.15em]">
🗑️ Deleted Characters ({deletedCharacters.length})
</h2>
<div class="grid gap-4">
{#each deletedCharacters as change (change.id)}
<div class="rounded-lg border border-rose-500/30 bg-rose-500/5 p-4 space-y-3">
<div class="flex items-center justify-between gap-3">
<div class="flex items-center gap-3">
{#if change.current?.pictureUrl}
<a href="https://onepiece.fandom.com/fr/wiki/{change.current?.url ?? change.scraped.url}" target="_blank" rel="noopener noreferrer">
<img
src={change.current?.pictureUrl ?? undefined}
alt={change.current?.name ?? change.scraped.name}
class="w-12 h-12 rounded object-cover hover:opacity-80 transition"
/>
</a>
{/if}
<div>
<h3 class="font-bold text-rose-300">{change.current?.name ?? change.scraped.name}</h3>
<p class="text-sm text-gray-500">{change.id}</p>
</div>
</div>
<form method="POST" action="?/acceptOne">
<input type="hidden" name="characterId" value={change.id} />
<button
type="submit"
class="rounded-full border border-rose-300/40 bg-rose-500/20 px-3 py-1 text-xs font-semibold text-rose-100 transition hover:bg-rose-500/30"
>
Supprimer
</button>
</form>
</div>
<p class="text-sm text-rose-200/80">This character is no longer present in the latest scrape and will be removed if accepted.</p>
</div>
{/each}
</div>
</section>
{/if}
{#if newCharacters.length === 0 && modifiedCharacters.length === 0 && deletedCharacters.length === 0}
<div class="rounded-lg border border-white/10 bg-white/5 p-8 text-center">
<p class="text-gray-400">Aucun changement détecté. Les tables character et characterScrapeValidation sont synchronisées.</p>
</div>