From 5ad0428420ce57580d8e3a1adc0c3833f59b7f6b Mon Sep 17 00:00:00 2001 From: whidix Date: Mon, 16 Mar 2026 23:12:06 +0100 Subject: [PATCH] feat: enhance character scrape validation and management - Added a new entry for "fuzzy_talisman" in the journal. - Updated import-json script to handle character deletion and mark absent characters as deleted in the scrape validation. - Modified schema to include an `isDeleted` field in the characterScrapeValidation table. - Renamed function `upsertCharacterFromScrapeValidation` to `applyCharacterChangeFromScrapeValidation` for clarity. - Enhanced character change loading to include deleted characters and updated UI to display them. - Improved character change handling in the Svelte component to reflect new, modified, and deleted states. --- drizzle/0001_fuzzy_talisman.sql | 1 + drizzle/meta/0001_snapshot.json | 1396 +++++++++++++++++ drizzle/meta/_journal.json | 7 + scripts/import-json.ts | 58 +- src/lib/server/db/schema.ts | 3 +- .../admin/character-changes/+page.server.ts | 36 +- .../admin/character-changes/+page.svelte | 80 +- 7 files changed, 1562 insertions(+), 19 deletions(-) create mode 100644 drizzle/0001_fuzzy_talisman.sql create mode 100644 drizzle/meta/0001_snapshot.json diff --git a/drizzle/0001_fuzzy_talisman.sql b/drizzle/0001_fuzzy_talisman.sql new file mode 100644 index 0000000..dcefbb9 --- /dev/null +++ b/drizzle/0001_fuzzy_talisman.sql @@ -0,0 +1 @@ +ALTER TABLE `character_scrape_validation` ADD `is_deleted` integer DEFAULT false; \ No newline at end of file diff --git a/drizzle/meta/0001_snapshot.json b/drizzle/meta/0001_snapshot.json new file mode 100644 index 0000000..af9c040 --- /dev/null +++ b/drizzle/meta/0001_snapshot.json @@ -0,0 +1,1396 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "9a965dd1-d97c-4142-a795-0558214180a4", + "prevId": "4b4f14a1-b37b-44f4-aed3-7289bd8cb6a0", + "tables": { + "arc": { + "name": "arc", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fr_name": { + "name": "fr_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start_chapter": { + "name": "start_chapter", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "end_chapter": { + "name": "end_chapter", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "character": { + "name": "character", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fr_name": { + "name": "fr_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "gender": { + "name": "gender", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "age": { + "name": "age", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "affiliations": { + "name": "affiliations", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fr_affiliations": { + "name": "fr_affiliations", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "devil_fruit_id": { + "name": "devil_fruit_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "haki_observation": { + "name": "haki_observation", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "haki_armament": { + "name": "haki_armament", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "haki_conqueror": { + "name": "haki_conqueror", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "bounty": { + "name": "bounty", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "height": { + "name": "height", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fr_origin": { + "name": "fr_origin", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "first_appearance": { + "name": "first_appearance", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "picture_url": { + "name": "picture_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "epithets": { + "name": "epithets", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fr_epithets": { + "name": "fr_epithets", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "arc_id": { + "name": "arc_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fr_url": { + "name": "fr_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_in_daily_mode": { + "name": "is_in_daily_mode", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": { + "character_devil_fruit_id_devil_fruit_id_fk": { + "name": "character_devil_fruit_id_devil_fruit_id_fk", + "tableFrom": "character", + "tableTo": "devil_fruit", + "columnsFrom": [ + "devil_fruit_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "character_arc_id_arc_id_fk": { + "name": "character_arc_id_arc_id_fk", + "tableFrom": "character", + "tableTo": "arc", + "columnsFrom": [ + "arc_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "character_history": { + "name": "character_history", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "character_id": { + "name": "character_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "date": { + "name": "date", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "won": { + "name": "won", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "character_history_date_unique": { + "name": "character_history_date_unique", + "columns": [ + "date" + ], + "isUnique": true + } + }, + "foreignKeys": { + "character_history_character_id_character_id_fk": { + "name": "character_history_character_id_character_id_fk", + "tableFrom": "character_history", + "tableTo": "character", + "columnsFrom": [ + "character_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "character_override": { + "name": "character_override", + "columns": { + "character_id": { + "name": "character_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "gender": { + "name": "gender", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "age": { + "name": "age", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "affiliations": { + "name": "affiliations", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fr_affiliations": { + "name": "fr_affiliations", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "devil_fruit_id": { + "name": "devil_fruit_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "haki_observation": { + "name": "haki_observation", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "haki_armament": { + "name": "haki_armament", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "haki_conqueror": { + "name": "haki_conqueror", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bounty": { + "name": "bounty", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fr_origin": { + "name": "fr_origin", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "first_appearance": { + "name": "first_appearance", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "picture_url": { + "name": "picture_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "epithets": { + "name": "epithets", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fr_epithets": { + "name": "fr_epithets", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "arc_id": { + "name": "arc_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fr_url": { + "name": "fr_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "character_override_character_id_character_id_fk": { + "name": "character_override_character_id_character_id_fk", + "tableFrom": "character_override", + "tableTo": "character", + "columnsFrom": [ + "character_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "character_override_devil_fruit_id_devil_fruit_id_fk": { + "name": "character_override_devil_fruit_id_devil_fruit_id_fk", + "tableFrom": "character_override", + "tableTo": "devil_fruit", + "columnsFrom": [ + "devil_fruit_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "character_override_arc_id_arc_id_fk": { + "name": "character_override_arc_id_arc_id_fk", + "tableFrom": "character_override", + "tableTo": "arc", + "columnsFrom": [ + "arc_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "character_scrape_validation": { + "name": "character_scrape_validation", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fr_name": { + "name": "fr_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "gender": { + "name": "gender", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "age": { + "name": "age", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "affiliations": { + "name": "affiliations", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fr_affiliations": { + "name": "fr_affiliations", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "devil_fruit_id": { + "name": "devil_fruit_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "haki_observation": { + "name": "haki_observation", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "haki_armament": { + "name": "haki_armament", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "haki_conqueror": { + "name": "haki_conqueror", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "bounty": { + "name": "bounty", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fr_origin": { + "name": "fr_origin", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "first_appearance": { + "name": "first_appearance", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "picture_url": { + "name": "picture_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "epithets": { + "name": "epithets", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fr_epithets": { + "name": "fr_epithets", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "arc_id": { + "name": "arc_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fr_url": { + "name": "fr_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_deleted": { + "name": "is_deleted", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": { + "character_scrape_validation_devil_fruit_id_devil_fruit_id_fk": { + "name": "character_scrape_validation_devil_fruit_id_devil_fruit_id_fk", + "tableFrom": "character_scrape_validation", + "tableTo": "devil_fruit", + "columnsFrom": [ + "devil_fruit_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "character_scrape_validation_arc_id_arc_id_fk": { + "name": "character_scrape_validation_arc_id_arc_id_fk", + "tableFrom": "character_scrape_validation", + "tableTo": "arc", + "columnsFrom": [ + "arc_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "config": { + "name": "config", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "devil_fruit": { + "name": "devil_fruit", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "devil_fruit_name_unique": { + "name": "devil_fruit_name_unique", + "columns": [ + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "friendship": { + "name": "friendship", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "requester_id": { + "name": "requester_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "addressee_id": { + "name": "addressee_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "friendship_requester_id_addressee_id_unique": { + "name": "friendship_requester_id_addressee_id_unique", + "columns": [ + "requester_id", + "addressee_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "friendship_requester_id_user_id_fk": { + "name": "friendship_requester_id_user_id_fk", + "tableFrom": "friendship", + "tableTo": "user", + "columnsFrom": [ + "requester_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "friendship_addressee_id_user_id_fk": { + "name": "friendship_addressee_id_user_id_fk", + "tableFrom": "friendship", + "tableTo": "user", + "columnsFrom": [ + "addressee_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_character_history": { + "name": "user_character_history", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "character_history_id": { + "name": "character_history_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "try_count": { + "name": "try_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tried_character_ids": { + "name": "tried_character_ids", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "user_character_history_user_id_character_history_id_unique": { + "name": "user_character_history_user_id_character_history_id_unique", + "columns": [ + "user_id", + "character_history_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "user_character_history_user_id_user_id_fk": { + "name": "user_character_history_user_id_user_id_fk", + "tableFrom": "user_character_history", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_character_history_character_history_id_character_history_id_fk": { + "name": "user_character_history_character_history_id_character_history_id_fk", + "tableFrom": "user_character_history", + "tableTo": "character_history", + "columnsFrom": [ + "character_history_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_admin": { + "name": "is_admin", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "user_username_unique": { + "name": "user_username_unique", + "columns": [ + "username" + ], + "isUnique": true + }, + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index a8a9caf..0a68814 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1773602933375, "tag": "0000_huge_doctor_octopus", "breakpoints": true + }, + { + "idx": 1, + "version": "6", + "when": 1773697753818, + "tag": "0001_fuzzy_talisman", + "breakpoints": true } ] } \ No newline at end of file diff --git a/scripts/import-json.ts b/scripts/import-json.ts index 45082ea..3e96eeb 100644 --- a/scripts/import-json.ts +++ b/scripts/import-json.ts @@ -1,6 +1,6 @@ import { createClient } from '@libsql/client'; import { drizzle } from 'drizzle-orm/libsql'; -import { sql, eq } from 'drizzle-orm'; +import { sql, eq, inArray } from 'drizzle-orm'; import fs from 'fs'; import { arc, character, devilFruit, characterScrapeValidation, type DevilFruitType } from '../src/lib/server/db/schema'; @@ -140,7 +140,8 @@ function transformCharacterData(item: CharacterRecord) { status: toNullable(item.status), arcId: toNullable(item.arcId), url: toNullable(item.url), - frUrl: toNullable(item.frUrl) + frUrl: toNullable(item.frUrl), + isDeleted: false }; } @@ -307,6 +308,7 @@ async function importFromJson(): Promise { } else { // Update scrapeValidation table console.log('Characters table not empty, updating scrapeValidation table for changes...\n'); + const scrapedCharacterIds: string[] = []; for (let i = 0; i < characters.length; i++) { const item = characters[i]; @@ -319,6 +321,7 @@ async function importFromJson(): Promise { lastSql = selectQuery.toSQL(); + scrapedCharacterIds.push(item.id); const jsonData = transformCharacterData(item); const upsertQuery = db @@ -341,6 +344,57 @@ async function importFromJson(): Promise { logSqlOnError(lastSql); } } + + // Fetch all characters from the character table and mark those absent from the + // scrape as deleted in scrape validation. + const allCharacters = await db.select({ id: character.id }).from(character); + const scrapedSet = new Set(scrapedCharacterIds); + const idsToMarkDeleted = allCharacters + .map((c) => c.id) + .filter((id) => !scrapedSet.has(id)); + + if (idsToMarkDeleted.length > 0) { + console.log(`\n⚠️ Marking ${idsToMarkDeleted.length} character(s) as deleted in scrape validation...`); + const deletedCharacterRows = await db + .select() + .from(character) + .where(inArray(character.id, idsToMarkDeleted)); + + for (const row of deletedCharacterRows) { + await db + .insert(characterScrapeValidation) + .values({ + id: row.id, + name: row.name, + frName: row.frName, + gender: row.gender, + age: row.age, + affiliations: row.affiliations, + frAffiliations: row.frAffiliations, + devilFruitId: row.devilFruitId, + hakiObservation: row.hakiObservation, + hakiArmament: row.hakiArmament, + hakiConqueror: row.hakiConqueror, + bounty: row.bounty, + height: row.height, + origin: row.origin, + frOrigin: row.frOrigin, + firstAppearance: row.firstAppearance, + pictureUrl: row.pictureUrl, + epithets: row.epithets, + frEpithets: row.frEpithets, + status: row.status, + arcId: row.arcId, + url: row.url, + frUrl: row.frUrl, + isDeleted: true + }) + .onConflictDoUpdate({ + target: characterScrapeValidation.id, + set: { isDeleted: true } + }); + } + } } console.log(`\n\n✓ Characters imported!`); diff --git a/src/lib/server/db/schema.ts b/src/lib/server/db/schema.ts index 1abce0c..31dbde9 100644 --- a/src/lib/server/db/schema.ts +++ b/src/lib/server/db/schema.ts @@ -119,7 +119,8 @@ export const characterScrapeValidation = sqliteTable('character_scrape_validatio status: text('status').$type(), arcId: text('arc_id').references(() => arc.id, { onDelete: 'set null' }), url: text('url'), - frUrl: text('fr_url') + frUrl: text('fr_url'), + isDeleted: integer('is_deleted', { mode: 'boolean' }).default(false), }); export type CharacterScrapeValidation = InferSelectModel; diff --git a/src/routes/(admin)/admin/character-changes/+page.server.ts b/src/routes/(admin)/admin/character-changes/+page.server.ts index 2a9f72e..0be2b4d 100644 --- a/src/routes/(admin)/admin/character-changes/+page.server.ts +++ b/src/routes/(admin)/admin/character-changes/+page.server.ts @@ -11,7 +11,7 @@ const EXEC_OPTIONS = { maxBuffer: 50 * 1024 * 1024 }; -async function upsertCharacterFromScrapeValidation(characterId: string): Promise { +async function applyCharacterChangeFromScrapeValidation(characterId: string): Promise { const [scraped] = await db .select() .from(characterScrapeValidation) @@ -21,6 +21,11 @@ async function upsertCharacterFromScrapeValidation(characterId: string): Promise return false; } + if (scraped.isDeleted) { + await db.delete(character).where(eq(character.id, characterId)); + return true; + } + await db .insert(character) .values({ @@ -87,7 +92,7 @@ export async function load() { // Compare and categorize changes const changes: { - type: 'new' | 'modified'; + type: 'new' | 'modified' | 'deleted'; id: string; scraped: (typeof scrapedCharacters)[0]; current?: (typeof currentCharacters)[0]; @@ -97,6 +102,18 @@ export async function load() { for (const scraped of scrapedCharacters) { const current = currentCharMap.get(scraped.id); + if (scraped.isDeleted) { + if (current) { + changes.push({ + type: 'deleted', + id: scraped.id, + scraped, + current + }); + } + continue; + } + if (!current) { // New character changes.push({ @@ -156,11 +173,16 @@ export async function load() { } } + const typeOrder: Record<'new' | 'modified' | 'deleted', number> = { + new: 0, + modified: 1, + deleted: 2 + }; + return { changes: changes.sort((a, b) => { - // Show 'new' first, then 'modified' if (a.type !== b.type) { - return a.type === 'new' ? -1 : 1; + return typeOrder[a.type] - typeOrder[b.type]; } return a.id.localeCompare(b.id); }) @@ -221,10 +243,10 @@ export const actions = { return { success: false, message: 'characterId is required' }; } - const applied = await upsertCharacterFromScrapeValidation(characterId); + const applied = await applyCharacterChangeFromScrapeValidation(characterId); return { success: applied, - message: applied ? 'Character applied successfully' : 'Character not found in scrape validation table' + message: applied ? 'Character change applied successfully' : 'Character not found in scrape validation table' }; }, @@ -233,7 +255,7 @@ export const actions = { let appliedCount = 0; for (const scraped of scrapedCharacters) { - const applied = await upsertCharacterFromScrapeValidation(scraped.id); + const applied = await applyCharacterChangeFromScrapeValidation(scraped.id); if (applied) { appliedCount++; } diff --git a/src/routes/(admin)/admin/character-changes/+page.svelte b/src/routes/(admin)/admin/character-changes/+page.svelte index 1a1242c..7e90a8e 100644 --- a/src/routes/(admin)/admin/character-changes/+page.svelte +++ b/src/routes/(admin)/admin/character-changes/+page.svelte @@ -1,10 +1,30 @@