Whidix/OnePieceDle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: Whidix:a91b298ee5874057ef86fa6e57610d2fe5b91bb2
Branches Tags
Whidix:main
..
compare: Whidix:4e95abf09fe4af7b649ab3cb6c8109d3224c5d1c
Branches Tags
Whidix:main

3 Commits
a91b298ee5 ... 4e95abf09f

Author SHA1 Message Date
whidix
4e95abf09f fix: update devil fruit extraction to handle redirects and ensure correct title retrieval
Some checks failed
Build Docker Image / build (push) Has been cancelled
Details
2026-03-14 15:41:20 +01:00
whidix
66afda5101 Refactor code structure for improved readability and maintainability 2026-03-14 15:34:30 +01:00
whidix
a041a8caf5 Refactor database schema and update scraping logic for One Piece characters and arcs 
- Updated database schema to include French names and adjusted field names for consistency.
- Modified scraping script to fetch and store French names for arcs and characters.
- Improved API calls to handle redirects and fetch additional data for characters.
- Enhanced data extraction methods for character attributes and devil fruits.
- Cleaned up code for better readability and maintainability.
 2026-03-14 01:23:29 +01:00
25 changed files with 1475 additions and 11801 deletions
Expand all files Collapse all files

156
drizzle/0000_graceful_master_mold.sql
View File

@@ -1,156 +0,0 @@
CREATE TABLE `arc` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`startChapter` integer NOT NULL,
`endChapter` integer,
`url` text
);
--> statement-breakpoint
CREATE TABLE `character` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`gender` text,
`age` integer,
`affiliations` text,
`devilFruitId` text,
`hakiObservation` integer DEFAULT false,
`hakiArmament` integer DEFAULT false,
`hakiConqueror` integer DEFAULT false,
`bounty` integer DEFAULT 0,
`height` real,
`origin` text,
`firstAppearance` integer NOT NULL,
`pictureUrl` text,
`epithets` text,
`status` text,
`arcId` text,
`url` text,
`isInDailyMode` integer DEFAULT true,
FOREIGN KEY (`devilFruitId`) REFERENCES `devilFruit`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`arcId`) REFERENCES `arc`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
CREATE TABLE `characterHistory` (
`id` text PRIMARY KEY NOT NULL,
`characterId` text,
`date` text,
`won` integer DEFAULT 0 NOT NULL,
`createdAt` integer NOT NULL,
`updatedAt` integer NOT NULL,
FOREIGN KEY (`characterId`) REFERENCES `character`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
CREATE TABLE `characterOverride` (
`characterId` text PRIMARY KEY NOT NULL,
`name` text,
`gender` text,
`age` integer,
`affiliations` text,
`devilFruitId` text,
`hakiObservation` integer,
`hakiArmament` integer,
`hakiConqueror` integer,
`bounty` integer,
`height` real,
`origin` text,
`firstAppearance` integer NOT NULL,
`pictureUrl` text,
`epithets` text,
`status` text,
`arcId` text,
`url` text,
`notes` text,
FOREIGN KEY (`characterId`) REFERENCES `character`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`devilFruitId`) REFERENCES `devilFruit`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`arcId`) REFERENCES `arc`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
CREATE TABLE `characterScrapeValidation` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`gender` text,
`age` integer,
`affiliations` text,
`devilFruitId` text,
`hakiObservation` integer DEFAULT false,
`hakiArmament` integer DEFAULT false,
`hakiConqueror` integer DEFAULT false,
`bounty` integer,
`height` real,
`origin` text,
`firstAppearance` integer NOT NULL,
`pictureUrl` text,
`epithets` text,
`status` text,
`arcId` text,
`url` text,
FOREIGN KEY (`devilFruitId`) REFERENCES `devilFruit`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`arcId`) REFERENCES `arc`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
CREATE TABLE `config` (
`key` text PRIMARY KEY NOT NULL,
`value` text
);
--> statement-breakpoint
CREATE TABLE `devilFruit` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`type` text,
`url` text
);
--> statement-breakpoint
CREATE UNIQUE INDEX `devilFruit_name_unique` ON `devilFruit` (`name`);--> statement-breakpoint
CREATE TABLE `account` (
`id` text PRIMARY KEY NOT NULL,
`account_id` text NOT NULL,
`provider_id` text NOT NULL,
`user_id` text NOT NULL,
`access_token` text,
`refresh_token` text,
`id_token` text,
`access_token_expires_at` integer,
`refresh_token_expires_at` integer,
`scope` text,
`password` text,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
`updated_at` integer NOT NULL,
FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE INDEX `account_userId_idx` ON `account` (`user_id`);--> statement-breakpoint
CREATE TABLE `session` (
`id` text PRIMARY KEY NOT NULL,
`expires_at` integer NOT NULL,
`token` text NOT NULL,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
`updated_at` integer NOT NULL,
`ip_address` text,
`user_agent` text,
`user_id` text NOT NULL,
FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE UNIQUE INDEX `session_token_unique` ON `session` (`token`);--> statement-breakpoint
CREATE INDEX `session_userId_idx` ON `session` (`user_id`);--> statement-breakpoint
CREATE TABLE `user` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`email` text NOT NULL,
`email_verified` integer DEFAULT false NOT NULL,
`image` text,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
`updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL
);
--> statement-breakpoint
CREATE UNIQUE INDEX `user_email_unique` ON `user` (`email`);--> statement-breakpoint
CREATE TABLE `verification` (
`id` text PRIMARY KEY NOT NULL,
`identifier` text NOT NULL,
`value` text NOT NULL,
`expires_at` integer NOT NULL,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
`updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL
);
--> statement-breakpoint
CREATE INDEX `verification_identifier_idx` ON `verification` (`identifier`);

194
drizzle/0000_keen_rockslide.sql Normal file
View File

@@ -0,0 +1,194 @@
CREATE TABLE `arc` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`fr_name` text,
`start_chapter` integer NOT NULL,
`end_chapter` integer,
`url` text
);
--> statement-breakpoint
CREATE TABLE `character` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`fr_name` text,
`gender` text,
`age` integer,
`affiliations` text,
`devil_fruit_id` text,
`haki_observation` integer DEFAULT false,
`haki_armament` integer DEFAULT false,
`haki_conqueror` integer DEFAULT false,
`bounty` integer DEFAULT 0,
`height` real,
`origin` text,
`fr_origin` text,
`first_appearance` integer NOT NULL,
`picture_url` text,
`epithets` text,
`fr_epithets` text,
`status` text,
`arc_id` text,
`url` text,
`fr_url` text,
`is_in_daily_mode` integer DEFAULT false,
FOREIGN KEY (`devil_fruit_id`) REFERENCES `devil_fruit`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`arc_id`) REFERENCES `arc`(`id`) ON UPDATE no action ON DELETE set null
);
--> statement-breakpoint
CREATE TABLE `character_history` (
`id` text PRIMARY KEY NOT NULL,
`character_id` text,
`date` integer NOT NULL,
`won` integer DEFAULT 0 NOT NULL,
`created_at` integer NOT NULL,
`updated_at` integer NOT NULL,
FOREIGN KEY (`character_id`) REFERENCES `character`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE UNIQUE INDEX `character_history_date_unique` ON `character_history` (`date`);--> statement-breakpoint
CREATE TABLE `character_override` (
`character_id` text PRIMARY KEY NOT NULL,
`name` text,
`gender` text,
`age` integer,
`affiliations` text,
`devil_fruit_id` text,
`haki_observation` integer,
`haki_armament` integer,
`haki_conqueror` integer,
`bounty` integer,
`height` real,
`origin` text,
`first_appearance` integer,
`picture_url` text,
`epithets` text,
`status` text,
`arc_id` text,
`url` text,
`fr_url` text,
`notes` text,
FOREIGN KEY (`character_id`) REFERENCES `character`(`id`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (`devil_fruit_id`) REFERENCES `devil_fruit`(`id`) ON UPDATE no action ON DELETE set null,
FOREIGN KEY (`arc_id`) REFERENCES `arc`(`id`) ON UPDATE no action ON DELETE set null
);
--> statement-breakpoint
CREATE TABLE `character_scrape_validation` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`fr_name` text,
`gender` text,
`age` integer,
`affiliations` text,
`devil_fruit_id` text,
`haki_observation` integer DEFAULT false,
`haki_armament` integer DEFAULT false,
`haki_conqueror` integer DEFAULT false,
`bounty` integer,
`height` real,
`origin` text,
`fr_origin` text,
`first_appearance` integer NOT NULL,
`picture_url` text,
`epithets` text,
`fr_epithets` text,
`status` text,
`arc_id` text,
`url` text,
`fr_url` text,
FOREIGN KEY (`devil_fruit_id`) REFERENCES `devil_fruit`(`id`) ON UPDATE no action ON DELETE set null,
FOREIGN KEY (`arc_id`) REFERENCES `arc`(`id`) ON UPDATE no action ON DELETE set null
);
--> statement-breakpoint
CREATE TABLE `config` (
`key` text PRIMARY KEY NOT NULL,
`value` text
);
--> statement-breakpoint
CREATE TABLE `devil_fruit` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`type` text,
`url` text
);
--> statement-breakpoint
CREATE UNIQUE INDEX `devil_fruit_name_unique` ON `devil_fruit` (`name`);--> statement-breakpoint
CREATE TABLE `friendship` (
`id` text PRIMARY KEY NOT NULL,
`requester_id` text NOT NULL,
`addressee_id` text NOT NULL,
`status` text DEFAULT 'pending' NOT NULL,
`created_at` integer NOT NULL,
`updated_at` integer NOT NULL,
FOREIGN KEY (`requester_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (`addressee_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE UNIQUE INDEX `friendship_requester_id_addressee_id_unique` ON `friendship` (`requester_id`,`addressee_id`);--> statement-breakpoint
CREATE TABLE `user_character_history` (
`id` text PRIMARY KEY NOT NULL,
`user_id` text,
`character_history_id` text,
`try_count` integer NOT NULL,
`tried_character_ids` text,
`created_at` integer NOT NULL,
FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (`character_history_id`) REFERENCES `character_history`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE UNIQUE INDEX `user_character_history_user_id_character_history_id_unique` ON `user_character_history` (`user_id`,`character_history_id`);--> statement-breakpoint
CREATE TABLE `account` (
`id` text PRIMARY KEY NOT NULL,
`account_id` text NOT NULL,
`provider_id` text NOT NULL,
`user_id` text NOT NULL,
`access_token` text,
`refresh_token` text,
`id_token` text,
`access_token_expires_at` integer,
`refresh_token_expires_at` integer,
`scope` text,
`password` text,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
`updated_at` integer NOT NULL,
FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE INDEX `account_userId_idx` ON `account` (`user_id`);--> statement-breakpoint
CREATE TABLE `session` (
`id` text PRIMARY KEY NOT NULL,
`expires_at` integer NOT NULL,
`token` text NOT NULL,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
`updated_at` integer NOT NULL,
`ip_address` text,
`user_agent` text,
`user_id` text NOT NULL,
FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE UNIQUE INDEX `session_token_unique` ON `session` (`token`);--> statement-breakpoint
CREATE INDEX `session_userId_idx` ON `session` (`user_id`);--> statement-breakpoint
CREATE TABLE `user` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`username` text NOT NULL,
`email` text NOT NULL,
`email_verified` integer DEFAULT false NOT NULL,
`image` text,
`is_admin` integer DEFAULT false NOT NULL,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
`updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL
);
--> statement-breakpoint
CREATE UNIQUE INDEX `user_username_unique` ON `user` (`username`);--> statement-breakpoint
CREATE UNIQUE INDEX `user_email_unique` ON `user` (`email`);--> statement-breakpoint
CREATE TABLE `verification` (
`id` text PRIMARY KEY NOT NULL,
`identifier` text NOT NULL,
`value` text NOT NULL,
`expires_at` integer NOT NULL,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
`updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL
);
--> statement-breakpoint
CREATE INDEX `verification_identifier_idx` ON `verification` (`identifier`);

30
drizzle/0001_nostalgic_hercules.sql
View File

@@ -1,30 +0,0 @@
PRAGMA foreign_keys=OFF;--> statement-breakpoint
CREATE TABLE `__new_characterOverride` (
`characterId` text PRIMARY KEY NOT NULL,
`name` text,
`gender` text,
`age` integer,
`affiliations` text,
`devilFruitId` text,
`hakiObservation` integer,
`hakiArmament` integer,
`hakiConqueror` integer,
`bounty` integer,
`height` real,
`origin` text,
`firstAppearance` integer,
`pictureUrl` text,
`epithets` text,
`status` text,
`arcId` text,
`url` text,
`notes` text,
FOREIGN KEY (`characterId`) REFERENCES `character`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`devilFruitId`) REFERENCES `devilFruit`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`arcId`) REFERENCES `arc`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
INSERT INTO `__new_characterOverride`("characterId", "name", "gender", "age", "affiliations", "devilFruitId", "hakiObservation", "hakiArmament", "hakiConqueror", "bounty", "height", "origin", "firstAppearance", "pictureUrl", "epithets", "status", "arcId", "url", "notes") SELECT "characterId", "name", "gender", "age", "affiliations", "devilFruitId", "hakiObservation", "hakiArmament", "hakiConqueror", "bounty", "height", "origin", "firstAppearance", "pictureUrl", "epithets", "status", "arcId", "url", "notes" FROM `characterOverride`;--> statement-breakpoint
DROP TABLE `characterOverride`;--> statement-breakpoint
ALTER TABLE `__new_characterOverride` RENAME TO `characterOverride`;--> statement-breakpoint
PRAGMA foreign_keys=ON;

1
drizzle/0002_large_gwen_stacy.sql
View File

@@ -1 +0,0 @@
ALTER TABLE `user` ADD `is_admin` integer DEFAULT false NOT NULL;

16
drizzle/0003_wise_blonde_phantom.sql
View File

@@ -1,16 +0,0 @@
PRAGMA foreign_keys=OFF;--> statement-breakpoint
CREATE TABLE `__new_characterHistory` (
`id` text PRIMARY KEY NOT NULL,
`characterId` text,
`date` integer NOT NULL,
`won` integer DEFAULT 0 NOT NULL,
`createdAt` integer NOT NULL,
`updatedAt` integer NOT NULL,
FOREIGN KEY (`characterId`) REFERENCES `character`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
INSERT INTO `__new_characterHistory`("id", "characterId", "date", "won", "createdAt", "updatedAt") SELECT "id", "characterId", "date", "won", "createdAt", "updatedAt" FROM `characterHistory`;--> statement-breakpoint
DROP TABLE `characterHistory`;--> statement-breakpoint
ALTER TABLE `__new_characterHistory` RENAME TO `characterHistory`;--> statement-breakpoint
PRAGMA foreign_keys=ON;--> statement-breakpoint
CREATE UNIQUE INDEX `characterHistory_date_unique` ON `characterHistory` (`date`);

9
drizzle/0004_unique_lorna_dane.sql
View File

@@ -1,9 +0,0 @@
CREATE TABLE `userCharacterHistory` (
`id` text PRIMARY KEY NOT NULL,
`userId` text,
`characterId` text,
`tryCount` integer NOT NULL,
`createdAt` integer NOT NULL,
FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`characterId`) REFERENCES `character`(`id`) ON UPDATE no action ON DELETE no action
);

15
drizzle/0005_large_jane_foster.sql
View File

@@ -1,15 +0,0 @@
PRAGMA foreign_keys=OFF;--> statement-breakpoint
CREATE TABLE `__new_userCharacterHistory` (
`id` text PRIMARY KEY NOT NULL,
`userId` text,
`characterHistoryId` text,
`tryCount` integer NOT NULL,
`createdAt` integer NOT NULL,
FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`characterHistoryId`) REFERENCES `characterHistory`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
INSERT INTO `__new_userCharacterHistory`("id", "userId", "characterHistoryId", "tryCount", "createdAt") SELECT "id", "userId", "characterId", "tryCount", "createdAt" FROM `userCharacterHistory`;--> statement-breakpoint
DROP TABLE `userCharacterHistory`;--> statement-breakpoint
ALTER TABLE `__new_userCharacterHistory` RENAME TO `userCharacterHistory`;--> statement-breakpoint
PRAGMA foreign_keys=ON;

1
drizzle/0006_premium_mesmero.sql
View File

@@ -1 +0,0 @@
CREATE UNIQUE INDEX `userCharacterHistory_userId_characterHistoryId_unique` ON `userCharacterHistory` (`userId`,`characterHistoryId`);

29
drizzle/0007_gray_shinko_yamashiro.sql
View File

@@ -1,29 +0,0 @@
PRAGMA foreign_keys=OFF;--> statement-breakpoint
CREATE TABLE `__new_character` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`gender` text,
`age` integer,
`affiliations` text,
`devilFruitId` text,
`hakiObservation` integer DEFAULT false,
`hakiArmament` integer DEFAULT false,
`hakiConqueror` integer DEFAULT false,
`bounty` integer DEFAULT 0,
`height` real,
`origin` text,
`firstAppearance` integer NOT NULL,
`pictureUrl` text,
`epithets` text,
`status` text,
`arcId` text,
`url` text,
`isInDailyMode` integer DEFAULT false,
FOREIGN KEY (`devilFruitId`) REFERENCES `devilFruit`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`arcId`) REFERENCES `arc`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
INSERT INTO `__new_character`("id", "name", "gender", "age", "affiliations", "devilFruitId", "hakiObservation", "hakiArmament", "hakiConqueror", "bounty", "height", "origin", "firstAppearance", "pictureUrl", "epithets", "status", "arcId", "url", "isInDailyMode") SELECT "id", "name", "gender", "age", "affiliations", "devilFruitId", "hakiObservation", "hakiArmament", "hakiConqueror", "bounty", "height", "origin", "firstAppearance", "pictureUrl", "epithets", "status", "arcId", "url", "isInDailyMode" FROM `character`;--> statement-breakpoint
DROP TABLE `character`;--> statement-breakpoint
ALTER TABLE `__new_character` RENAME TO `character`;--> statement-breakpoint
PRAGMA foreign_keys=ON;

12
drizzle/0008_skinny_warpath.sql
View File

@@ -1,12 +0,0 @@
CREATE TABLE `friendship` (
`id` text PRIMARY KEY NOT NULL,
`requesterId` text NOT NULL,
`addresseeId` text NOT NULL,
`status` text DEFAULT 'pending' NOT NULL,
`createdAt` integer NOT NULL,
`updatedAt` integer NOT NULL,
FOREIGN KEY (`requesterId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (`addresseeId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE UNIQUE INDEX `friendship_requesterId_addresseeId_unique` ON `friendship` (`requesterId`,`addresseeId`);

51
drizzle/0009_true_gravity.sql
View File

@@ -1,51 +0,0 @@
PRAGMA foreign_keys=OFF;--> statement-breakpoint
ALTER TABLE `user` ADD `username` text;--> statement-breakpoint
UPDATE `user`
SET `username` = `name`
WHERE `username` IS NULL OR trim(`username`) = '';--> statement-breakpoint
UPDATE `user`
SET `username` = `username` || '_' || substr(`id`, 1, 6)
WHERE `id` IN (
SELECT u1.`id`
FROM `user` u1
JOIN `user` u2
ON lower(u1.`username`) = lower(u2.`username`)
AND u1.`id` > u2.`id`
);--> statement-breakpoint
CREATE TABLE `__new_user` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`username` text NOT NULL,
`email` text NOT NULL,
`email_verified` integer DEFAULT false NOT NULL,
`image` text,
`is_admin` integer DEFAULT false NOT NULL,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
`updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL
);--> statement-breakpoint
INSERT INTO `__new_user` (
`id`,
`name`,
`username`,
`email`,
`email_verified`,
`image`,
`is_admin`,
`created_at`,
`updated_at`
)
SELECT
`id`,
`name`,
`username`,
`email`,
`email_verified`,
`image`,
`is_admin`,
`created_at`,
`updated_at`
FROM `user`;--> statement-breakpoint
DROP TABLE `user`;--> statement-breakpoint
ALTER TABLE `__new_user` RENAME TO `user`;--> statement-breakpoint
CREATE UNIQUE INDEX `user_username_unique` ON `user` (`username`);--> statement-breakpoint
PRAGMA foreign_keys=ON;

501
drizzle/meta/0000_snapshot.json
View File

@@ -1,7 +1,7 @@
{
"version": "6",
"dialect": "sqlite",
"id": "d1237d76-8f1c-4721-b8dd-d31082ed7b9a",
"id": "8ffd14bd-bf33-410f-9778-92bc1abc8938",
"prevId": "00000000-0000-0000-0000-000000000000",
"tables": {
"arc": {
@@ -21,15 +21,22 @@
"notNull": true,
"autoincrement": false
},
"startChapter": {
"name": "startChapter",
"fr_name": {
"name": "fr_name",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"start_chapter": {
"name": "start_chapter",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"endChapter": {
"name": "endChapter",
"end_chapter": {
"name": "end_chapter",
"type": "integer",
"primaryKey": false,
"notNull": false,
@@ -66,6 +73,13 @@
"notNull": true,
"autoincrement": false
},
"fr_name": {
"name": "fr_name",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"gender": {
"name": "gender",
"type": "text",
@@ -87,31 +101,31 @@
"notNull": false,
"autoincrement": false
},
"devilFruitId": {
"name": "devilFruitId",
"devil_fruit_id": {
"name": "devil_fruit_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"hakiObservation": {
"name": "hakiObservation",
"haki_observation": {
"name": "haki_observation",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false,
"default": false
},
"hakiArmament": {
"name": "hakiArmament",
"haki_armament": {
"name": "haki_armament",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false,
"default": false
},
"hakiConqueror": {
"name": "hakiConqueror",
"haki_conqueror": {
"name": "haki_conqueror",
"type": "integer",
"primaryKey": false,
"notNull": false,
@@ -140,15 +154,22 @@
"notNull": false,
"autoincrement": false
},
"firstAppearance": {
"name": "firstAppearance",
"fr_origin": {
"name": "fr_origin",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"first_appearance": {
"name": "first_appearance",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"pictureUrl": {
"name": "pictureUrl",
"picture_url": {
"name": "picture_url",
"type": "text",
"primaryKey": false,
"notNull": false,
@@ -161,6 +182,13 @@
"notNull": false,
"autoincrement": false
},
"fr_epithets": {
"name": "fr_epithets",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"status": {
"name": "status",
"type": "text",
@@ -168,8 +196,8 @@
"notNull": false,
"autoincrement": false
},
"arcId": {
"name": "arcId",
"arc_id": {
"name": "arc_id",
"type": "text",
"primaryKey": false,
"notNull": false,
@@ -182,23 +210,30 @@
"notNull": false,
"autoincrement": false
},
"isInDailyMode": {
"name": "isInDailyMode",
"fr_url": {
"name": "fr_url",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"is_in_daily_mode": {
"name": "is_in_daily_mode",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false,
"default": true
"default": false
}
},
"indexes": {},
"foreignKeys": {
"character_devilFruitId_devilFruit_id_fk": {
"name": "character_devilFruitId_devilFruit_id_fk",
"character_devil_fruit_id_devil_fruit_id_fk": {
"name": "character_devil_fruit_id_devil_fruit_id_fk",
"tableFrom": "character",
"tableTo": "devilFruit",
"tableTo": "devil_fruit",
"columnsFrom": [
"devilFruitId"
"devil_fruit_id"
],
"columnsTo": [
"id"
@@ -206,17 +241,17 @@
"onDelete": "no action",
"onUpdate": "no action"
},
"character_arcId_arc_id_fk": {
"name": "character_arcId_arc_id_fk",
"character_arc_id_arc_id_fk": {
"name": "character_arc_id_arc_id_fk",
"tableFrom": "character",
"tableTo": "arc",
"columnsFrom": [
"arcId"
"arc_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onDelete": "set null",
"onUpdate": "no action"
}
},
@@ -224,8 +259,8 @@
"uniqueConstraints": {},
"checkConstraints": {}
},
"characterHistory": {
"name": "characterHistory",
"character_history": {
"name": "character_history",
"columns": {
"id": {
"name": "id",
@@ -234,8 +269,8 @@
"notNull": true,
"autoincrement": false
},
"characterId": {
"name": "characterId",
"character_id": {
"name": "character_id",
"type": "text",
"primaryKey": false,
"notNull": false,
@@ -243,9 +278,9 @@
},
"date": {
"name": "date",
"type": "text",
"type": "integer",
"primaryKey": false,
"notNull": false,
"notNull": true,
"autoincrement": false
},
"won": {
@@ -256,34 +291,42 @@
"autoincrement": false,
"default": 0
},
"createdAt": {
"name": "createdAt",
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"updatedAt": {
"name": "updatedAt",
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {},
"indexes": {
"character_history_date_unique": {
"name": "character_history_date_unique",
"columns": [
"date"
],
"isUnique": true
}
},
"foreignKeys": {
"characterHistory_characterId_character_id_fk": {
"name": "characterHistory_characterId_character_id_fk",
"tableFrom": "characterHistory",
"character_history_character_id_character_id_fk": {
"name": "character_history_character_id_character_id_fk",
"tableFrom": "character_history",
"tableTo": "character",
"columnsFrom": [
"characterId"
"character_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onDelete": "cascade",
"onUpdate": "no action"
}
},
@@ -291,11 +334,11 @@
"uniqueConstraints": {},
"checkConstraints": {}
},
"characterOverride": {
"name": "characterOverride",
"character_override": {
"name": "character_override",
"columns": {
"characterId": {
"name": "characterId",
"character_id": {
"name": "character_id",
"type": "text",
"primaryKey": true,
"notNull": true,
@@ -329,29 +372,29 @@
"notNull": false,
"autoincrement": false
},
"devilFruitId": {
"name": "devilFruitId",
"devil_fruit_id": {
"name": "devil_fruit_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"hakiObservation": {
"name": "hakiObservation",
"haki_observation": {
"name": "haki_observation",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"hakiArmament": {
"name": "hakiArmament",
"haki_armament": {
"name": "haki_armament",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"hakiConqueror": {
"name": "hakiConqueror",
"haki_conqueror": {
"name": "haki_conqueror",
"type": "integer",
"primaryKey": false,
"notNull": false,
@@ -378,15 +421,15 @@
"notNull": false,
"autoincrement": false
},
"firstAppearance": {
"name": "firstAppearance",
"first_appearance": {
"name": "first_appearance",
"type": "integer",
"primaryKey": false,
"notNull": true,
"notNull": false,
"autoincrement": false
},
"pictureUrl": {
"name": "pictureUrl",
"picture_url": {
"name": "picture_url",
"type": "text",
"primaryKey": false,
"notNull": false,
@@ -406,8 +449,8 @@
"notNull": false,
"autoincrement": false
},
"arcId": {
"name": "arcId",
"arc_id": {
"name": "arc_id",
"type": "text",
"primaryKey": false,
"notNull": false,
@@ -420,6 +463,13 @@
"notNull": false,
"autoincrement": false
},
"fr_url": {
"name": "fr_url",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"notes": {
"name": "notes",
"type": "text",
@@ -430,43 +480,43 @@
},
"indexes": {},
"foreignKeys": {
"characterOverride_characterId_character_id_fk": {
"name": "characterOverride_characterId_character_id_fk",
"tableFrom": "characterOverride",
"character_override_character_id_character_id_fk": {
"name": "character_override_character_id_character_id_fk",
"tableFrom": "character_override",
"tableTo": "character",
"columnsFrom": [
"characterId"
"character_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onDelete": "cascade",
"onUpdate": "no action"
},
"characterOverride_devilFruitId_devilFruit_id_fk": {
"name": "characterOverride_devilFruitId_devilFruit_id_fk",
"tableFrom": "characterOverride",
"tableTo": "devilFruit",
"character_override_devil_fruit_id_devil_fruit_id_fk": {
"name": "character_override_devil_fruit_id_devil_fruit_id_fk",
"tableFrom": "character_override",
"tableTo": "devil_fruit",
"columnsFrom": [
"devilFruitId"
"devil_fruit_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onDelete": "set null",
"onUpdate": "no action"
},
"characterOverride_arcId_arc_id_fk": {
"name": "characterOverride_arcId_arc_id_fk",
"tableFrom": "characterOverride",
"character_override_arc_id_arc_id_fk": {
"name": "character_override_arc_id_arc_id_fk",
"tableFrom": "character_override",
"tableTo": "arc",
"columnsFrom": [
"arcId"
"arc_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onDelete": "set null",
"onUpdate": "no action"
}
},
@@ -474,8 +524,8 @@
"uniqueConstraints": {},
"checkConstraints": {}
},
"characterScrapeValidation": {
"name": "characterScrapeValidation",
"character_scrape_validation": {
"name": "character_scrape_validation",
"columns": {
"id": {
"name": "id",
@@ -491,6 +541,13 @@
"notNull": true,
"autoincrement": false
},
"fr_name": {
"name": "fr_name",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"gender": {
"name": "gender",
"type": "text",
@@ -512,31 +569,31 @@
"notNull": false,
"autoincrement": false
},
"devilFruitId": {
"name": "devilFruitId",
"devil_fruit_id": {
"name": "devil_fruit_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"hakiObservation": {
"name": "hakiObservation",
"haki_observation": {
"name": "haki_observation",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false,
"default": false
},
"hakiArmament": {
"name": "hakiArmament",
"haki_armament": {
"name": "haki_armament",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false,
"default": false
},
"hakiConqueror": {
"name": "hakiConqueror",
"haki_conqueror": {
"name": "haki_conqueror",
"type": "integer",
"primaryKey": false,
"notNull": false,
@@ -564,15 +621,22 @@
"notNull": false,
"autoincrement": false
},
"firstAppearance": {
"name": "firstAppearance",
"fr_origin": {
"name": "fr_origin",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"first_appearance": {
"name": "first_appearance",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"pictureUrl": {
"name": "pictureUrl",
"picture_url": {
"name": "picture_url",
"type": "text",
"primaryKey": false,
"notNull": false,
@@ -585,6 +649,13 @@
"notNull": false,
"autoincrement": false
},
"fr_epithets": {
"name": "fr_epithets",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"status": {
"name": "status",
"type": "text",
@@ -592,8 +663,8 @@
"notNull": false,
"autoincrement": false
},
"arcId": {
"name": "arcId",
"arc_id": {
"name": "arc_id",
"type": "text",
"primaryKey": false,
"notNull": false,
@@ -605,34 +676,41 @@
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"fr_url": {
"name": "fr_url",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {
"characterScrapeValidation_devilFruitId_devilFruit_id_fk": {
"name": "characterScrapeValidation_devilFruitId_devilFruit_id_fk",
"tableFrom": "characterScrapeValidation",
"tableTo": "devilFruit",
"character_scrape_validation_devil_fruit_id_devil_fruit_id_fk": {
"name": "character_scrape_validation_devil_fruit_id_devil_fruit_id_fk",
"tableFrom": "character_scrape_validation",
"tableTo": "devil_fruit",
"columnsFrom": [
"devilFruitId"
"devil_fruit_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onDelete": "set null",
"onUpdate": "no action"
},
"characterScrapeValidation_arcId_arc_id_fk": {
"name": "characterScrapeValidation_arcId_arc_id_fk",
"tableFrom": "characterScrapeValidation",
"character_scrape_validation_arc_id_arc_id_fk": {
"name": "character_scrape_validation_arc_id_arc_id_fk",
"tableFrom": "character_scrape_validation",
"tableTo": "arc",
"columnsFrom": [
"arcId"
"arc_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onDelete": "set null",
"onUpdate": "no action"
}
},
@@ -664,8 +742,8 @@
"uniqueConstraints": {},
"checkConstraints": {}
},
"devilFruit": {
"name": "devilFruit",
"devil_fruit": {
"name": "devil_fruit",
"columns": {
"id": {
"name": "id",
@@ -697,8 +775,8 @@
}
},
"indexes": {
"devilFruit_name_unique": {
"name": "devilFruit_name_unique",
"devil_fruit_name_unique": {
"name": "devil_fruit_name_unique",
"columns": [
"name"
],
@@ -710,6 +788,183 @@
"uniqueConstraints": {},
"checkConstraints": {}
},
"friendship": {
"name": "friendship",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"requester_id": {
"name": "requester_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"addressee_id": {
"name": "addressee_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"status": {
"name": "status",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'pending'"
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"friendship_requester_id_addressee_id_unique": {
"name": "friendship_requester_id_addressee_id_unique",
"columns": [
"requester_id",
"addressee_id"
],
"isUnique": true
}
},
"foreignKeys": {
"friendship_requester_id_user_id_fk": {
"name": "friendship_requester_id_user_id_fk",
"tableFrom": "friendship",
"tableTo": "user",
"columnsFrom": [
"requester_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
},
"friendship_addressee_id_user_id_fk": {
"name": "friendship_addressee_id_user_id_fk",
"tableFrom": "friendship",
"tableTo": "user",
"columnsFrom": [
"addressee_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"user_character_history": {
"name": "user_character_history",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"user_id": {
"name": "user_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"character_history_id": {
"name": "character_history_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"try_count": {
"name": "try_count",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"tried_character_ids": {
"name": "tried_character_ids",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"user_character_history_user_id_character_history_id_unique": {
"name": "user_character_history_user_id_character_history_id_unique",
"columns": [
"user_id",
"character_history_id"
],
"isUnique": true
}
},
"foreignKeys": {
"user_character_history_user_id_user_id_fk": {
"name": "user_character_history_user_id_user_id_fk",
"tableFrom": "user_character_history",
"tableTo": "user",
"columnsFrom": [
"user_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
},
"user_character_history_character_history_id_character_history_id_fk": {
"name": "user_character_history_character_history_id_character_history_id_fk",
"tableFrom": "user_character_history",
"tableTo": "character_history",
"columnsFrom": [
"character_history_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"account": {
"name": "account",
"columns": {
@@ -947,6 +1202,13 @@
"notNull": true,
"autoincrement": false
},
"username": {
"name": "username",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"email": {
"name": "email",
"type": "text",
@@ -969,6 +1231,14 @@
"notNull": false,
"autoincrement": false
},
"is_admin": {
"name": "is_admin",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": false
},
"created_at": {
"name": "created_at",
"type": "integer",
@@ -987,6 +1257,13 @@
}
},
"indexes": {
"user_username_unique": {
"name": "user_username_unique",
"columns": [
"username"
],
"isUnique": true
},
"user_email_unique": {
"name": "user_email_unique",
"columns": [

1076
drizzle/meta/0001_snapshot.json
View File

File diff suppressed because it is too large Load Diff

1084
drizzle/meta/0002_snapshot.json
View File

File diff suppressed because it is too large Load Diff

1092
drizzle/meta/0003_snapshot.json
View File

File diff suppressed because it is too large Load Diff

1164
drizzle/meta/0004_snapshot.json
View File

File diff suppressed because it is too large Load Diff

1166
drizzle/meta/0005_snapshot.json
View File

File diff suppressed because it is too large Load Diff

1173
drizzle/meta/0006_snapshot.json
View File

File diff suppressed because it is too large Load Diff

1173
drizzle/meta/0007_snapshot.json
View File

File diff suppressed because it is too large Load Diff

1262
drizzle/meta/0008_snapshot.json
View File

File diff suppressed because it is too large Load Diff

1276
drizzle/meta/0009_snapshot.json
View File

File diff suppressed because it is too large Load Diff

67
drizzle/meta/_journal.json
View File

@@ -5,71 +5,8 @@
{
"idx": 0,
"version": "6",
"when": 1772325597983,
"tag": "0000_graceful_master_mold",
"breakpoints": true
},
{
"idx": 1,
"version": "6",
"when": 1772383366179,
"tag": "0001_nostalgic_hercules",
"breakpoints": true
},
{
"idx": 2,
"version": "6",
"when": 1772390182445,
"tag": "0002_large_gwen_stacy",
"breakpoints": true
},
{
"idx": 3,
"version": "6",
"when": 1772449624450,
"tag": "0003_wise_blonde_phantom",
"breakpoints": true
},
{
"idx": 4,
"version": "6",
"when": 1772480377099,
"tag": "0004_unique_lorna_dane",
"breakpoints": true
},
{
"idx": 5,
"version": "6",
"when": 1772562012631,
"tag": "0005_large_jane_foster",
"breakpoints": true
},
{
"idx": 6,
"version": "6",
"when": 1772562364830,
"tag": "0006_premium_mesmero",
"breakpoints": true
},
{
"idx": 7,
"version": "6",
"when": 1772735982970,
"tag": "0007_gray_shinko_yamashiro",
"breakpoints": true
},
{
"idx": 8,
"version": "6",
"when": 1772821532270,
"tag": "0008_skinny_warpath",
"breakpoints": true
},
{
"idx": 9,
"version": "6",
"when": 1772822823122,
"tag": "0009_true_gravity",
"when": 1773447741334,
"tag": "0000_keen_rockslide",
"breakpoints": true
}
]

53
scripts/import-json.ts
View File

@@ -4,6 +4,8 @@ import { sql, eq } from 'drizzle-orm';
import fs from 'fs';
import { arc, character, devilFruit, characterScrapeValidation, type DevilFruitType } from '../src/lib/server/db/schema';
type Status = 'Alive' | 'Dead' | 'Unknown';
type ArcRecord = {
id: string;
name: string;
@@ -22,9 +24,11 @@ type DevilFruitRecord = {
type CharacterRecord = {
id: string;
name: string;
frName?: string | null;
gender?: string | null;
age?: number | null;
affiliations?: string[] | string | null;
frAffiliations?: string[] | string | null;
devilFruitId?: string | null;
hakiObservation?: boolean;
hakiArmament?: boolean;
@@ -32,12 +36,15 @@ type CharacterRecord = {
bounty?: number | null;
height?: number | null;
origin?: string | null;
frOrigin?: string | null;
firstAppearance?: number;
pictureUrl?: string | null;
epithets?: string[] | string | null;
status?: string | null;
frEpithets?: string[] | string | null;
status?: Status | null;
arcId?: string | null;
url?: string | null;
frUrl?: string | null;
};
const DATABASE_URL = process.env.DATABASE_URL || 'file:local.db';
@@ -86,7 +93,7 @@ function toJsonArray(value: string[] | string | null | undefined): string[] | nu
function toDevilFruitType(value: DevilFruitType | string | null | undefined): DevilFruitType | null {
if (!value) return null;
if (value === 'Paramecia' || value === 'Zoan' || value === 'Logia' || value === 'Unknown') {
if (value === 'Paramecia' || value === 'Zoan' || value === 'Logia' || value === 'Smile' || value === 'Unknown') {
return value;
}
return 'Unknown';
@@ -115,59 +122,25 @@ function transformCharacterData(item: CharacterRecord) {
gender: toNullable(item.gender),
age: toNullable(item.age),
affiliations: toJsonArray(item.affiliations),
frAffiliations: toJsonArray(item.frAffiliations),
devilFruitId: toNullable(item.devilFruitId),
hakiObservation: !!item.hakiObservation,
hakiArmament: !!item.hakiArmament,
hakiConqueror: !!item.hakiConqueror,
bounty: item.bounty ?? 0,
height: toNumber(item.height as any),
height: toNumber(item.height as string | number | null),
origin: toNullable(item.origin),
frOrigin: toNullable(item.frOrigin),
firstAppearance: item.firstAppearance ?? 0,
pictureUrl: toNullable(item.pictureUrl),
epithets: toJsonArray(item.epithets),
frEpithets: toJsonArray(item.frEpithets),
status: toNullable(item.status),
arcId: toNullable(item.arcId),
url: toNullable(item.url)
};
}
function hasChanged(jsonData: any, dbData: any): boolean {
if (!dbData) return true;
// Print any differences for debugging
for (const key in jsonData) {
const jsonValue = jsonData[key];
const dbValue = dbData[key];
const jsonString = typeof jsonValue === 'object' ? JSON.stringify(jsonValue) : String(jsonValue);
const dbString = typeof dbValue === 'object' ? JSON.stringify(dbValue) : String(dbValue);
if (jsonString !== dbString) {
console.log(`\nField "${key}" changed for character ID ${jsonData.id}:`);
console.log(` JSON: ${jsonString}`);
console.log(` DB: ${dbString}`);
} }
// Compare each field
return (
jsonData.name != dbData.name ||
jsonData.gender != dbData.gender ||
jsonData.age != dbData.age ||
JSON.stringify(jsonData.affiliations) != JSON.stringify(dbData.affiliations) ||
jsonData.devilFruitId != dbData.devilFruitId ||
jsonData.hakiObservation != dbData.hakiObservation ||
jsonData.hakiArmament != dbData.hakiArmament ||
jsonData.hakiConqueror != dbData.hakiConqueror ||
jsonData.bounty != dbData.bounty ||
jsonData.height != dbData.height ||
jsonData.origin != dbData.origin ||
jsonData.firstAppearance != dbData.firstAppearance ||
jsonData.pictureUrl != dbData.pictureUrl ||
JSON.stringify(jsonData.epithets) != JSON.stringify(dbData.epithets) ||
jsonData.status != dbData.status ||
jsonData.arcId != dbData.arcId ||
jsonData.url != dbData.url
);
}
async function isCharacterTableEmpty(): Promise<boolean> {
const result = await db.select({ count: sql<number>`COUNT(*)` }).from(character);
return result[0]?.count === 0;

468
scripts/scrape-onepiece.ts
View File

@@ -6,6 +6,7 @@ import { createObjectCsvWriter } from 'csv-writer';
interface Arc {
id: string;
name: string;
frName: string | null;
startChapter: number;
endChapter: number | null;
url: string;
@@ -14,30 +15,34 @@ interface Arc {
interface Character {
id: string;
name: string;
frName: string | null;
gender: string | null;
age: number | null;
height: number | null;
origin: string | null;
frOrigin: string | null;
devilFruitId: string | null;
devilFruitUrl: string | null;
affiliations: string[];
frAffiliations: string[] | null;
bounty: number | null;
hakiObservation: boolean;
hakiArmament: boolean;
hakiConqueror: boolean;
epithets: string[];
frEpithets: string[] | null;
firstAppearance: number;
status: string | null;
pictureUrl: string | null;
url: string;
arcId?: string;
frUrl: string | null;
arcId: string;
}
interface CharacterListItem {
name: string;
url: string;
pictureUrl: string | null;
chapter: string;
chapter: number;
}
interface DevilFruitData {
@@ -52,31 +57,15 @@ interface DevilFruit {
url: string;
}
const FANDOM_API_BASE = 'https://onepiece.fandom.com/fr/api.php?action=parse&format=json&page=';
const FANDOM_API_BASE =
'https://onepiece.fandom.com/api.php?action=parse&redirects=true&format=json&page=';
const FR_FANDOM_API_BASE =
'https://onepiece.fandom.com/fr/api.php?action=parse&redirects=true&format=json&page=';
const OUTPUT_DIR = './scraped-data';
const MAX_RETRIES = 0; // Set to 0 to disable retries, can be increased if needed
const INITIAL_RETRY_DELAY = 1000;
const FETCH_CONCURRENCY = 50;
// Store cookies across requests (simulate browser behavior)
const cookies = new Map<string, string>();
function getCookieHeader(): string {
const cookieArray = Array.from(cookies.values()).map(c => c.split(';')[0]);
return cookieArray.length > 0 ? cookieArray.join('; ') : '';
}
function saveCookies(setCookieHeader: string | string[] | null): void {
if (setCookieHeader) {
const cookiesList = Array.isArray(setCookieHeader) ? setCookieHeader : [setCookieHeader];
cookiesList.forEach(cookie => {
const [nameValue] = cookie.split(';');
const [name] = nameValue.split('=');
if (name) cookies.set(name, cookie);
});
}
}
// Create output directory
if (!fs.existsSync(OUTPUT_DIR)) {
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
@@ -85,32 +74,24 @@ if (!fs.existsSync(OUTPUT_DIR)) {
/**
* Retry a fetch request with exponential backoff
*/
async function fetchWithRetry(url: string, options: RequestInit = {}, retries: number = 0): Promise<Response> {
async function fetchWithRetry(
url: string,
options: RequestInit = {},
retries: number = 0
): Promise<Response> {
try {
const headers: Record<string, string> = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:150.0) Firefox/150.0',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive',
Connection: 'keep-alive',
...((options.headers as Record<string, string>) || {})
};
// Add cookies from previous requests
const cookieHeader = getCookieHeader();
if (cookieHeader) {
headers['Cookie'] = cookieHeader;
}
const response = await fetch(url, {
headers,
...options
} as any);
// Save cookies from response
const setCookie = response.headers.get('set-cookie');
if (setCookie) {
saveCookies(setCookie);
}
});
// Check if response is OK (status 200-299)
if (response.ok) {
@@ -121,7 +102,7 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries: n
if (retries < MAX_RETRIES) {
const delay = INITIAL_RETRY_DELAY * Math.pow(2, retries);
console.log(`⚠️ HTTP ${response.status} for ${url}, retrying in ${delay}ms...`);
await new Promise(resolve => setTimeout(resolve, delay));
await new Promise((resolve) => setTimeout(resolve, delay));
return fetchWithRetry(url, options, retries + 1);
}
@@ -132,7 +113,7 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries: n
if (retries < MAX_RETRIES) {
const delay = INITIAL_RETRY_DELAY * Math.pow(2, retries);
console.log(`⚠️ Network error: ${(error as Error).message}, retrying in ${delay}ms...`);
await new Promise(resolve => setTimeout(resolve, delay));
await new Promise((resolve) => setTimeout(resolve, delay));
return fetchWithRetry(url, options, retries + 1);
}
@@ -141,6 +122,17 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries: n
}
}
/**
* Get the French link from the API response links array
*/
function getFrLink(links: { lang: string; ['*']: string; url: string }[]): { url: string } | null {
// Get french url by getting parse.langlinks where lang is "fr" and extract the name from there
const frLink = links.find(
(link: { lang: string; ['*']: string; url: string }) => link.lang === 'fr'
);
return frLink ? { url: frLink['url'] } : null;
}
/**
* Normalize string by decoding URI components, punctuation, and replacing spaces with underscores
@@ -148,7 +140,7 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries: n
function normalizeId(str: string): string {
return decodeURIComponent(str)
.normalize('NFD')
.replace(/[,:.\(\)]/g, '')
.replace(/[,:.()]/g, '')
.replace(/\s+/g, '_')
.toLowerCase();
}
@@ -158,10 +150,10 @@ function normalizeId(str: string): string {
*/
async function fetchAllArcs(): Promise<Arc[]> {
try {
const apiUrl = `${FANDOM_API_BASE}Chapitres_et_Tomes`;
const apiUrl = `${FANDOM_API_BASE}Chapters_and_Volumes`;
console.log('Fetching arcs list via API...');
const response = await fetchWithRetry(apiUrl);
const jsonData = await response.json() as any;
const jsonData = await response.json();
// Extract HTML from API response
const htmlContent = jsonData.parse?.text?.['*'];
@@ -172,40 +164,67 @@ async function fetchAllArcs(): Promise<Arc[]> {
const $ = cheerio.load(htmlContent);
const arcs: Arc[] = [];
// Find all arc links in the table
$('table.wikitable td a').each((index, element) => {
const text = $(element).text().trim();
const href = $(element).attr('href');
const seenArcUrls = new Set<string>();
// Check if it's an arc link (contains "Arc" and chapter info)
if (text.includes('Arc') && text.includes('Ch.') && href) {
// Extract arc name and chapter range
// Example text: "Arc Ville d'Orange(Ch.8 à 21)[T.1 à 3]"
console.log(`Processing arc link: ${text} (${href})`);
const nameMatch = text.match(/^(.*?Arc.*?)\s*\(Ch\.(\d+)(?:\s*à\s*(?:(\d+)|(?:...)))?\)/);
if (nameMatch) {
let arcName = nameMatch[1].trim();
// Remove "Arc " from the name
arcName = arcName.replace(/^Arc\s+/i, '');
// Arc rows are in table cells where first link points to a *_Arc page and text includes chapter range.
const arcCells = $('table.wikitable td').toArray();
for (const element of arcCells) {
const cell = $(element);
const firstLink = cell.find('a').first();
const href = firstLink.attr('href') || '';
let arcName = firstLink.text().trim();
const startChapter = parseInt(nameMatch[2]);
const endChapter = nameMatch[3] ? parseInt(nameMatch[3]) : null;
if (!href.startsWith('/wiki/') || !/_Arc$/i.test(href)) {
continue;
}
// Generate arc ID by normalizing the url
let arcId = normalizeId(href.replace('/fr/wiki/', ''));
// Remove "Arc_" from the id
arcId = arcId.replace(/^arc_/i, '');
if (!arcName || !/\bArc\b/i.test(arcName)) {
continue;
}
arcName = arcName.replace(/\bArc\b/i, '').trim();
const cleanUrl = href.replace('/wiki/', '');
if (seenArcUrls.has(cleanUrl)) {
continue;
}
const cellText = cell.text().replace(/\s+/g, ' ').trim();
const chapterMatch = cellText.match(/Chapters\s+(\d+)\s+to\s+(\d+|Current)/i);
if (!chapterMatch) {
continue;
}
const startChapter = parseInt(chapterMatch[1], 10);
const endChapter = /current/i.test(chapterMatch[2]) ? null : parseInt(chapterMatch[2], 10);
let arcId = normalizeId(cleanUrl);
arcId = arcId.replace(/_arc$/i, '');
// Query the href page via API to get the correct HTML content (in case of redirect) and extract the French name from there
const arcResponse = await fetchWithRetry(`${FANDOM_API_BASE}${cleanUrl}`);
const arcJsonData = await arcResponse.json();
let frArcName: string | null =
arcJsonData.parse?.langlinks.find(
(link: { lang: string; ['*']: string }) => link.lang === 'fr'
)?.['*'] || null;
// Remove "Arc" suffix from French name if present to keep it consistent with English names (e.g. "Arc de Luffy" becomes "Luffy")
if (frArcName && /\bArc\b/i.test(frArcName)) {
frArcName = frArcName.replace(/\bArc\b/i, '').trim();
}
arcs.push({
id: arcId,
name: arcName,
frName: frArcName,
startChapter,
endChapter,
url: href.replace('/fr/wiki/', '')
url: cleanUrl
});
seenArcUrls.add(cleanUrl);
}
}
});
console.log(`Found ${arcs.length} arcs.`);
return arcs;
@@ -234,10 +253,11 @@ async function saveArcsToCSV(arcs: Arc[]): Promise<void> {
header: [
{ id: 'id', title: 'ID' },
{ id: 'name', title: 'Name' },
{ id: 'frName', title: 'French Name' },
{ id: 'startChapter', title: 'Start Chapter' },
{ id: 'endChapter', title: 'End Chapter' },
{ id: 'url', title: 'URL' }
],
]
});
const records = arcs
@@ -245,6 +265,7 @@ async function saveArcsToCSV(arcs: Arc[]): Promise<void> {
.map((arc) => ({
id: arc.id || '',
name: arc.name || '',
frName: arc.frName || '',
startChapter: arc.startChapter || '',
endChapter: arc.endChapter || '',
url: arc.url || ''
@@ -259,10 +280,10 @@ async function saveArcsToCSV(arcs: Arc[]): Promise<void> {
*/
async function fetchAllCharactersUrl(): Promise<CharacterListItem[]> {
try {
const apiUrl = `${FANDOM_API_BASE}Liste_des_Personnages_Canon`;
const apiUrl = `${FANDOM_API_BASE}List_of_Canon_Characters`;
console.log('Fetching character list via API...');
const response = await fetchWithRetry(apiUrl);
const jsonData = await response.json() as any;
const jsonData = await response.json();
// Extract HTML from API response
const htmlContent = jsonData.parse?.text?.['*'];
@@ -272,11 +293,10 @@ async function fetchAllCharactersUrl(): Promise<CharacterListItem[]> {
const $ = cheerio.load(htmlContent);
const characters: CharacterListItem[] = [];
$('table.wikitable tbody tr').each((index, element) => {
$('table.fandom-table tbody tr').each((index, element) => {
if (index === 0) return; // Skip header row
let charpictureUrl = $(element).find('td:nth-child(1) a img').attr('data-src') || $(element).find('td:nth-child(1) a img').attr('src');
let charUrl = $(element).find('td:nth-child(2) a').attr('href');
let charName = $(element).find('td:nth-child(2) a').text().trim();
const charName = $(element).find('td:nth-child(2) a').text().trim();
let charChapter = $(element).find('td:nth-child(3)').text().trim();
// Remove parentheses and their content from chapter info (e.g. "1 (flashback)" becomes "1")
@@ -288,13 +308,16 @@ async function fetchAllCharactersUrl(): Promise<CharacterListItem[]> {
return;
}
if (parseInt(charChapter, 10) === 0) {
return;
}
if (charUrl) {
charUrl = charUrl.replace('/fr/wiki/', '');
charUrl = charUrl.replace('/wiki/', '');
characters.push({
name: charName,
url: charUrl,
pictureUrl: charpictureUrl || null,
chapter: charChapter,
chapter: parseInt(charChapter, 10)
});
}
});
@@ -312,26 +335,16 @@ async function fetchAllCharactersUrl(): Promise<CharacterListItem[]> {
async function fetchCharacter(
characterUrl: string,
characterName: string,
characterpictureUrl: string | null,
characterChapter: string
characterChapter: number,
arcsList: Arc[]
): Promise<Character | null> {
try {
console.log(`Fetching: ${characterName}...`);
// Use API to fetch character page
const apiUrl = `${FANDOM_API_BASE}${characterUrl}`;
let response = await fetchWithRetry(apiUrl);
let jsonData = await response.json() as any;
// Use final page name from API (if parse.limks contains one element, it means the original page was a redirect, so we use the the element 0 as the final URL, otherwise we use the original URL)
let finalCharacterUrl = characterUrl;
if (jsonData.parse?.links?.length === 1) {
finalCharacterUrl = jsonData.parse.links[0]['*'];
// Query the API again with the final URL to get the correct HTML content (in case of redirect)
response = await fetchWithRetry(`${FANDOM_API_BASE}${finalCharacterUrl}`);
jsonData = await response.json() as any;
}
const response = await fetchWithRetry(apiUrl);
const jsonData = await response.json();
const categories = jsonData.parse?.categories || [];
@@ -346,16 +359,16 @@ async function fetchCharacter(
const name = characterName;
// Generate character ID from URL + name combination
const finalCharacterId = normalizeId(finalCharacterUrl + '_' + name);
const finalCharacterId = normalizeId(characterUrl + '_' + name);
// Extract gender from JSON categories
let gender: string | null = null;
for (const cat of categories) {
const catName = cat['*'] || '';
if (catName === 'Personnages_Masculins') {
if (catName === 'Male_Characters') {
gender = 'Male';
break;
} else if (catName === 'Personnages_Féminins') {
} else if (catName === 'Female_Characters') {
gender = 'Female';
break;
}
@@ -365,7 +378,7 @@ async function fetchCharacter(
const age = extractAge($);
// Extract affiliations
const affiliations = extractAffiliations($);
const affiliations = await extractAffiliations($, 'en');
// Extract epithets
const epithets = extractEpithets($);
@@ -381,11 +394,11 @@ async function fetchCharacter(
let hakiConqueror = false;
for (const cat of categories) {
const catName = cat['*'] || '';
if (catName === 'Utilisateurs_du_Haki_de_l\'observation') {
if (catName === 'Observation_Haki_Users') {
hakiObservation = true;
} else if (catName === 'Utilisateurs_du_Haki_de_l\'armement') {
} else if (catName === 'Armament_Haki_Users') {
hakiArmament = true;
} else if (catName === 'Utilisateurs_du_Haki_des_rois') {
} else if (catName === 'Supreme_King_Haki_Users') {
hakiConqueror = true;
}
}
@@ -397,7 +410,7 @@ async function fetchCharacter(
const height = extractHeight($);
// Use chapter from character list, cast to int
let firstAppearance = parseInt(characterChapter);
const firstAppearance = characterChapter;
// Extract origin
const origin = extractOrigin($);
@@ -405,31 +418,66 @@ async function fetchCharacter(
// Extract status
const status = extractStatus($);
// Extract image URL and clean it
let pictureUrl = characterpictureUrl;
if (pictureUrl && pictureUrl.includes('Image_Non_Disponible')) {
pictureUrl = null;
let arcId = '';
const arc = arcsList.find(
(a) =>
a.startChapter <= firstAppearance &&
(a.endChapter === null || a.endChapter >= firstAppearance)
);
if (!arc) {
return null;
}
arcId = arc.id;
const frLink = getFrLink(jsonData.parse?.langlinks || []);
const frUrl = frLink ? frLink.url.replace('https://onepiece.fandom.com/fr/wiki/', '') : null;
const frjsonData = frUrl
? await fetchWithRetry(`${FR_FANDOM_API_BASE}${frUrl}`).then((res) => res.json())
: null;
let frName = frjsonData?.parse?.title || null;
const frAffiliations = frjsonData
? await extractAffiliations(cheerio.load(frjsonData.parse?.text?.['*'] || ''), 'fr')
: null;
const frEpithets = frjsonData
? extractEpithets(cheerio.load(frjsonData.parse?.text?.['*'] || ''))
: null;
const frOrigin = frjsonData
? extractOrigin(cheerio.load(frjsonData.parse?.text?.['*'] || ''))
: null;
if (name !== jsonData.parse?.title) {
frName = name;
}
return {
id: finalCharacterId,
name,
frName,
gender,
age,
height,
origin,
frOrigin,
devilFruitId,
devilFruitUrl,
affiliations,
frAffiliations,
bounty,
hakiObservation,
hakiArmament,
hakiConqueror,
epithets,
frEpithets,
firstAppearance,
arcId,
status,
pictureUrl,
url: finalCharacterUrl
pictureUrl: 'Image_Non_Disponible',
url: characterUrl,
frUrl
};
} catch (error) {
console.error(`Error fetching ${characterName}:`, (error as Error).message);
@@ -437,12 +485,11 @@ async function fetchCharacter(
}
}
/**
* Extract age from infobox
*/
function extractAge($: cheerio.CheerioAPI): number | null {
const div = $('[data-source="âge"] .pi-data-value');
const div = $('[data-source="age"] .pi-data-value');
if (div.length === 0) return null;
let text = div.html();
@@ -466,20 +513,41 @@ function extractAge($: cheerio.CheerioAPI): number | null {
/**
* Extract affiliations from infobox
*/
function extractAffiliations($: cheerio.CheerioAPI): string[] {
async function extractAffiliations($: cheerio.CheerioAPI, lang: string): Promise<string[]> {
const div = $('[data-source="affiliation"] .pi-data-value');
if (div.length === 0) return [];
const cleanedDiv = div.clone();
cleanedDiv.find('sup').remove();
let text = cleanedDiv.html();
const text = cleanedDiv.html();
if (!text) return [];
// Extract all link values
const linkValues = cleanedDiv.find('a').map((i, el) => $(el).text().trim()).get();
if (linkValues.length > 0) {
return linkValues;
// Resolve affiliations from linked page titles.
const links = cleanedDiv.find('a').toArray();
if (links.length > 0) {
const linkValues = await Promise.all(
links.map(async (el) => {
const href = $(el).attr('href') || '';
const resolvedTitle = await fetchWithRetry(
`${lang === 'fr' ? FR_FANDOM_API_BASE : FANDOM_API_BASE}${href.replace('/fr/wiki/', '').replace('/wiki/', '')}`
)
.then((res) => res.json())
.then((json) => json.parse?.title)
.catch(() => null);
if (resolvedTitle) {
return resolvedTitle;
}
return $(el).text().trim();
})
);
const uniqueLinks = Array.from(new Set(linkValues.filter(Boolean)));
if (uniqueLinks.length > 0) {
return uniqueLinks;
}
}
// Fallback to parsing text
@@ -490,28 +558,44 @@ function extractAffiliations($: cheerio.CheerioAPI): string[] {
/**
* Extract epithets from infobox
* Epithets are always between double quotes
* Handles both quoted and unquoted epithets, keeping only the main/latest readable values.
*/
function extractEpithets($: cheerio.CheerioAPI): string[] {
const div = $('[data-source="épithète"] .pi-data-value');
const div = $('[data-source="epithet"] .pi-data-value');
if (div.length === 0) return [];
const cleanedDiv = div.clone();
cleanedDiv.find('sup').remove();
let text = cleanedDiv.text();
if (!text) return [];
const html = cleanedDiv.html();
if (!html) return [];
// Extract all text between double quotes (both straight and curly quotes)
const matches = text.match(/["«"]([^"»"]+)["»"]/g);
if (!matches) return [];
const plainText = html.replace(/<br\s*\/?\s*>/gi, '\n').replace(/<[^>]*>/g, '');
// Remove the quotes and trim
const epithets = matches.map(match =>
match.replace(/^["«"]|["»"]$/g, '').trim()
).filter(Boolean);
const lines = plainText
.split('\n')
.map((line) => line.trim())
.filter(Boolean);
return epithets;
const epithets = lines
.map((line) => {
const normalized = line.replace(/\s+/g, ' ').trim();
// Prefer explicit quoted epithet if present.
const quotedMatch = normalized.match(/["«“](.*?)["»”]/);
if (quotedMatch?.[1]) {
return quotedMatch[1].trim();
}
// Otherwise keep only the base epithet text before extra notes/translations.
return normalized
.split(/[;(]/)[0]
.replace(/["'«»“”]/g, '')
.trim();
})
.filter(Boolean);
return Array.from(new Set(epithets));
}
/**
@@ -519,41 +603,22 @@ function extractEpithets($: cheerio.CheerioAPI): string[] {
* Returns both normalized ID and URL
*/
async function extractDevilFruit($: cheerio.CheerioAPI): Promise<DevilFruitData | null> {
const link = $('[data-source="dfnom"] .pi-data-value a').first();
const link = $('[data-source="dfname"] .pi-data-value a').first();
if (link.length === 0) return null;
const href = link.attr('href');
if (!href || !href.startsWith('/fr/wiki/')) return null;
if (!href || !href.startsWith('/wiki/')) return null;
const cleanUrl = href.replace('/fr/wiki/', '');
const cleanUrl = href.replace('/wiki/', '');
try {
// Fetch the page via API to follow redirects
const apiUrl = `${FANDOM_API_BASE}${decodeURIComponent(cleanUrl)}`;
const response = await fetchWithRetry(apiUrl);
const jsonData = await response.json() as any;
// Query the devil fruit page via API to get the correct HTML content (in case of redirect) and extract the type from there
const dfResponse = await fetchWithRetry(`${FANDOM_API_BASE}${cleanUrl}`);
const dfJsonData = await dfResponse.json();
const fruitTitle = dfJsonData.parse?.title || '';
// Use final page name from API (if parse.links contains one element, it means the original page was a redirect, so we use the the element 0 as the final URL, otherwise we use the original URL)
let finalPath = cleanUrl;
if (jsonData.parse?.links?.length === 1) {
finalPath = jsonData.parse.links[0]['*'];
}
if (finalPath) {
return {
devilFruitId: normalizeId(finalPath),
devilFruitUrl: finalPath
};
}
} catch (error) {
console.error(`Error fetching devil fruit page: ${(error as Error).message}`);
}
// Fallback to the original href
return {
devilFruitId: normalizeId(cleanUrl),
devilFruitUrl: cleanUrl
devilFruitId: normalizeId(fruitTitle),
devilFruitUrl: fruitTitle
};
}
@@ -561,7 +626,7 @@ async function extractDevilFruit($: cheerio.CheerioAPI): Promise<DevilFruitData
* Extract bounty from infobox
*/
function extractBounty($: cheerio.CheerioAPI): number | null {
const div = $('[data-source="prime"] .pi-data-value');
const div = $('[data-source="bounty"] .pi-data-value');
if (div.length === 0) return 0;
let text = div.html();
@@ -593,7 +658,7 @@ function extractBounty($: cheerio.CheerioAPI): number | null {
* Extract height from infobox
*/
function extractHeight($: cheerio.CheerioAPI): number | null {
const div = $('[data-source="taille"] .pi-data-value');
const div = $('[data-source="height"] .pi-data-value');
if (div.length === 0) return null;
let text = div.html();
@@ -602,43 +667,47 @@ function extractHeight($: cheerio.CheerioAPI): number | null {
// Remove all sup blocks (citations)
text = text.replace(/<sup[^>]*>.*?<\/sup>/gi, '');
// Check if there's a <p> tag - if yes, use content from <p>
let content;
const pMatch = text.match(/<p[^>]*>(.*?)<\/p>/i);
if (pMatch) {
// Extract content from the <p> tag
content = pMatch[1];
} else {
// Use the last value method (after any <br> tag)
content = text.split('<br>').pop();
}
// Convert line breaks to new lines so we can reliably pick the latest value.
const textWithNewLines = text.replace(/<br\s*\/?\s*>/gi, '\n');
const lines = textWithNewLines
.replace(/<[^>]*>/g, '')
.split('\n')
.map((line) => line.trim())
.filter(Boolean);
let cleanText = (content || '').replace(/<[^>]*>/g, '').trim();
// Keep only lines that look like a height value, then pick the latest one.
const heightLines = lines.filter((line) => /\d/.test(line) && /(cm|m)/i.test(line));
const latestLine =
heightLines.length > 0 ? heightLines[heightLines.length - 1] : lines[lines.length - 1];
if (!latestLine) return null;
// Remove content with parentheses
cleanText = cleanText.replace(/\([^)]*\)/g, '');
// Normalize units for meters or centimeters
// Remove descriptive suffixes like "(post-timeskip)".
const cleanText = latestLine.replace(/\([^)]*\)/g, '').trim();
const normalized = cleanText.toLowerCase().replace(/\s/g, '');
if (normalized.includes('cm')) {
const digitsOnly = normalized.replace(/\D/g, '');
const cm = parseFloat(digitsOnly);
return cm ? cm / 100 : null;
// Values are stored in meters in this dataset.
const cmMatch = normalized.match(/(\d+(?:[.,]\d+)?)cm/);
if (cmMatch) {
const cm = parseFloat(cmMatch[1].replace(',', '.'));
return Number.isFinite(cm) ? cm / 100 : null;
}
if (normalized.includes('m')) {
const parts = normalized.split('m').filter(Boolean);
return parts.length > 0 ? parseFloat(parts.join('.')) : null;
const mMatch = normalized.match(/(\d+(?:[.,]\d+)?)m/);
if (mMatch) {
const meters = parseFloat(mMatch[1].replace(',', '.'));
return Number.isFinite(meters) ? meters : null;
}
return normalized.length > 0 ? parseFloat(normalized.replace(/\D/g, '')) : null;
return null;
}
/**
* Extract origin from infobox
*/
function extractOrigin($: cheerio.CheerioAPI): string | null {
const div = $('[data-source="origine"] .pi-data-value');
const div = $(
'[data-source="origin"] .pi-data-value, [data-source="origine"] .pi-data-value'
).first();
if (div.length === 0) return null;
let text = div.html();
@@ -661,23 +730,22 @@ function extractOrigin($: cheerio.CheerioAPI): string | null {
* Extract status from infobox
*/
function extractStatus($: cheerio.CheerioAPI): string | null {
const div = $('[data-source="statut"] .pi-data-value');
const div = $('[data-source="status"] .pi-data-value');
if (div.length === 0) return null;
const statusText = div.text().trim().toLowerCase();
if (statusText.includes('vivant')) {
if (statusText.includes('Alive')) {
return 'Alive';
} else if (statusText.includes('décédé')) {
} else if (statusText.includes('Dead')) {
return 'Dead';
} else if (statusText.includes('inconnu')) {
} else if (statusText.includes('Unknown')) {
return 'Unknown';
}
return 'Alive';
}
/**
* Save data to JSON
*/
@@ -713,7 +781,7 @@ async function saveToCSV(characters: Character[]): Promise<void> {
{ id: 'arcId', title: 'Arc ID' },
{ id: 'pictureUrl', title: 'Image URL' },
{ id: 'url', title: 'Fandom URL' }
],
]
});
const records = characters
@@ -726,9 +794,11 @@ async function saveToCSV(characters: Character[]): Promise<void> {
height: c.height || '',
origin: c.origin || '',
status: c.status || '',
epithets: Array.isArray(c.epithets) ? c.epithets.join(', ') : (c.epithets || ''),
epithets: Array.isArray(c.epithets) ? c.epithets.join(', ') : c.epithets || '',
devilFruitId: c.devilFruitId || '',
affiliations: Array.isArray(c.affiliations) ? c.affiliations.join(', ') : (c.affiliations || ''),
affiliations: Array.isArray(c.affiliations)
? c.affiliations.join(', ')
: c.affiliations || '',
bounty: c.bounty ?? 0,
hakiObservation: c.hakiObservation ? 1 : 0,
hakiArmament: c.hakiArmament ? 1 : 0,
@@ -746,14 +816,17 @@ async function saveToCSV(characters: Character[]): Promise<void> {
/**
* Fetch devil fruit data from fandom using provided URL
*/
async function fetchDevilFruit(devilFruitUrl: string, devilFruitId: string): Promise<DevilFruit | null> {
async function fetchDevilFruit(
devilFruitUrl: string,
devilFruitId: string
): Promise<DevilFruit | null> {
try {
console.log(`Fetching devil fruit: ${devilFruitUrl}...`);
// Use API to fetch devil fruit page
const apiUrl = `${FANDOM_API_BASE}${devilFruitUrl}`;
const response = await fetchWithRetry(apiUrl);
const jsonData = await response.json() as any;
const jsonData = await response.json();
// Extract HTML from API response
const htmlContent = jsonData.parse?.text?.['*'];
@@ -766,8 +839,9 @@ async function fetchDevilFruit(devilFruitUrl: string, devilFruitId: string): Pro
let type: string | null = null;
// Determine type based on categories (if categories contain "Paramecia", "Zoan", "Logia" or "Smile")
if (jsonData.parse?.categories) {
const categories = jsonData.parse.categories
.map((cat: any) => String(cat['*'] || '').toLowerCase());
const categories = jsonData.parse.categories.map((cat: { ['*']: string }) =>
String(cat['*'] || '').toLowerCase()
);
if (categories.some((category: string) => category.includes('paramecia'))) {
type = 'Paramecia';
@@ -813,7 +887,7 @@ async function saveDevilFruitsToCSV(devilFruits: DevilFruit[]): Promise<void> {
{ id: 'name', title: 'Name' },
{ id: 'type', title: 'Type' },
{ id: 'url', title: 'URL' }
],
]
});
const records = devilFruits
@@ -847,6 +921,7 @@ async function main(): Promise<void> {
console.table({
ID: arc.id,
Name: arc.name,
FrenchName: arc.frName || '',
StartChapter: arc.startChapter,
EndChapter: arc.endChapter || 'Ongoing',
URL: arc.url
@@ -886,7 +961,7 @@ async function main(): Promise<void> {
const batch = failedCharacters.slice(i, i + FETCH_CONCURRENCY);
const batchResults = await Promise.all(
batch.map(async (char) => {
const data = await fetchCharacter(char.url, char.name, char.pictureUrl, char.chapter);
const data = await fetchCharacter(char.url, char.name, char.chapter, arcsList);
return { char, data };
})
);
@@ -918,13 +993,6 @@ async function main(): Promise<void> {
devilFruitUrls.add(data.devilFruitUrl);
}
if (data.firstAppearance) {
const arc = arcsList.find(a => a.startChapter <= data.firstAppearance && (a.endChapter === null || a.endChapter >= data.firstAppearance));
if (arc) {
data.arcId = arc.id;
}
}
characters.push(data);
} else {
nextFailedCharacters.push(char);
@@ -983,8 +1051,8 @@ async function main(): Promise<void> {
}
// Update characters with normalized devil fruit IDs
const devilFruitMap = new Map<string, string>(devilFruits.map(df => [df.id, df.id]));
characters.forEach(char => {
const devilFruitMap = new Map<string, string>(devilFruits.map((df) => [df.id, df.id]));
characters.forEach((char) => {
if (char.devilFruitUrl) {
const normalizedId = normalizeId(char.devilFruitUrl);
char.devilFruitId = devilFruitMap.get(normalizedId) || normalizedId;

97
src/lib/server/db/schema.ts
View File

@@ -17,13 +17,14 @@ export const config = sqliteTable('config', {
export const arc = sqliteTable('arc', {
id: text('id').primaryKey(),
name: text('name').notNull(),
startChapter: integer('startChapter').notNull(),
endChapter: integer('endChapter'),
frName: text('fr_name'),
startChapter: integer('start_chapter').notNull(),
endChapter: integer('end_chapter'),
url: text('url')
});
// Define the devil fruit table schema
export const devilFruit = sqliteTable('devilFruit', {
export const devilFruit = sqliteTable('devil_fruit', {
id: text('id').primaryKey(),
name: text('name').notNull().unique(),
type: text('type').$type<DevilFruitType>(),
@@ -34,91 +35,101 @@ export const devilFruit = sqliteTable('devilFruit', {
export const character = sqliteTable('character', {
id: text('id').primaryKey(),
name: text('name').notNull(),
frName: text('fr_name'),
gender: text('gender'),
age: integer('age'),
affiliations: text('affiliations', { mode: 'json' }).$type<string[]>(),
devilFruitId: text('devilFruitId').references(() => devilFruit.id),
hakiObservation: integer('hakiObservation', { mode: 'boolean' }).default(false),
hakiArmament: integer('hakiArmament', { mode: 'boolean' }).default(false),
hakiConqueror: integer('hakiConqueror', { mode: 'boolean' }).default(false),
devilFruitId: text('devil_fruit_id').references(() => devilFruit.id),
hakiObservation: integer('haki_observation', { mode: 'boolean' }).default(false),
hakiArmament: integer('haki_armament', { mode: 'boolean' }).default(false),
hakiConqueror: integer('haki_conqueror', { mode: 'boolean' }).default(false),
bounty: integer('bounty').default(0),
height: real('height'),
origin: text('origin'),
firstAppearance: integer('firstAppearance').notNull(),
pictureUrl: text('pictureUrl'),
frOrigin: text('fr_origin'),
firstAppearance: integer('first_appearance').notNull(),
pictureUrl: text('picture_url'),
epithets: text('epithets', { mode: 'json' }).$type<string[]>(),
frEpithets: text('fr_epithets', { mode: 'json' }).$type<string[]>(),
status: text('status').$type<Status | null>(),
arcId: text('arcId').references(() => arc.id),
arcId: text('arc_id').references(() => arc.id, { onDelete: 'set null' }),
url: text('url'),
isInDailyMode: integer('isInDailyMode', { mode: 'boolean' }).default(false)
frUrl: text('fr_url'),
isInDailyMode: integer('is_in_daily_mode', { mode: 'boolean' }).default(false)
});
// Define the character override table schema
export const characterOverride = sqliteTable('characterOverride', {
characterId: text('characterId').primaryKey().references(() => character.id),
export const characterOverride = sqliteTable('character_override', {
characterId: text('character_id').primaryKey().references(() => character.id, { onDelete: 'cascade' }),
name: text('name'),
gender: text('gender'),
age: integer('age'),
affiliations: text('affiliations', { mode: 'json' }).$type<string[]>(),
devilFruitId: text('devilFruitId').references(() => devilFruit.id),
hakiObservation: integer('hakiObservation', { mode: 'boolean' }),
hakiArmament: integer('hakiArmament', { mode: 'boolean' }),
hakiConqueror: integer('hakiConqueror', { mode: 'boolean' }),
devilFruitId: text('devil_fruit_id').references(() => devilFruit.id, { onDelete: 'set null' }),
hakiObservation: integer('haki_observation', { mode: 'boolean' }),
hakiArmament: integer('haki_armament', { mode: 'boolean' }),
hakiConqueror: integer('haki_conqueror', { mode: 'boolean' }),
bounty: integer('bounty'),
height: real('height'),
origin: text('origin'),
firstAppearance: integer('firstAppearance'),
pictureUrl: text('pictureUrl'),
firstAppearance: integer('first_appearance'),
pictureUrl: text('picture_url'),
epithets: text('epithets', { mode: 'json' }).$type<string[]>(),
status: text('status').$type<Status | null>(),
arcId: text('arcId').references(() => arc.id),
arcId: text('arc_id').references(() => arc.id, { onDelete: 'set null' }),
url: text('url'),
frUrl: text('fr_url'),
notes: text('notes')
});
// Define the character scrape validation table schema
export const characterScrapeValidation = sqliteTable('characterScrapeValidation', {
export const characterScrapeValidation = sqliteTable('character_scrape_validation', {
id: text('id').primaryKey(),
name: text('name').notNull(),
frName: text('fr_name'),
gender: text('gender'),
age: integer('age'),
affiliations: text('affiliations', { mode: 'json' }).$type<string[]>(),
devilFruitId: text('devilFruitId').references(() => devilFruit.id),
hakiObservation: integer('hakiObservation', { mode: 'boolean' }).default(false),
hakiArmament: integer('hakiArmament', { mode: 'boolean' }).default(false),
hakiConqueror: integer('hakiConqueror', { mode: 'boolean' }).default(false),
devilFruitId: text('devil_fruit_id').references(() => devilFruit.id, { onDelete: 'set null' }),
hakiObservation: integer('haki_observation', { mode: 'boolean' }).default(false),
hakiArmament: integer('haki_armament', { mode: 'boolean' }).default(false),
hakiConqueror: integer('haki_conqueror', { mode: 'boolean' }).default(false),
bounty: integer('bounty'),
height: real('height'),
origin: text('origin'),
firstAppearance: integer('firstAppearance').notNull(),
pictureUrl: text('pictureUrl'),
frOrigin: text('fr_origin'),
firstAppearance: integer('first_appearance').notNull(),
pictureUrl: text('picture_url'),
epithets: text('epithets', { mode: 'json' }).$type<string[]>(),
frEpithets: text('fr_epithets', { mode: 'json' }).$type<string[]>(),
status: text('status').$type<Status | null>(),
arcId: text('arcId').references(() => arc.id),
url: text('url')
arcId: text('arc_id').references(() => arc.id, { onDelete: 'set null' }),
url: text('url'),
frUrl: text('fr_url')
});
// Define the caracter history table schema
export const characterHistory = sqliteTable('characterHistory', {
// Define the character history table schema
export const characterHistory = sqliteTable('character_history', {
id: text('id')
.primaryKey()
.$defaultFn(() => crypto.randomUUID()),
characterId: text('characterId').references(() => character.id),
characterId: text('character_id').references(() => character.id, { onDelete: 'cascade' }),
date: integer('date').notNull().unique(),
won: integer('won').notNull().default(0),
createdAt: integer('createdAt').notNull().$default(() => Date.now()),
updatedAt: integer('updatedAt').notNull().$default(() => Date.now()),
createdAt: integer('created_at').notNull().$default(() => Date.now()),
updatedAt: integer('updated_at').notNull().$default(() => Date.now()),
});
// Define the user character history table schema
export const userCharacterHistory = sqliteTable('userCharacterHistory', {
export const userCharacterHistory = sqliteTable('user_character_history', {
id: text('id')
.primaryKey()
.$defaultFn(() => crypto.randomUUID()),
userId: text('userId').references(() => user.id),
characterHistoryId: text('characterHistoryId').references(() => characterHistory.id),
tryCount: integer('tryCount').notNull(),
createdAt: integer('createdAt').notNull().$default(() => Date.now())
userId: text('user_id').references(() => user.id, { onDelete: 'cascade' }),
characterHistoryId: text('character_history_id').references(() => characterHistory.id, { onDelete: 'cascade' }),
tryCount: integer('try_count').notNull(),
triedCharacterIds: text('tried_character_ids', { mode: 'json' }).$type<string[]>(),
createdAt: integer('created_at').notNull().$default(() => Date.now())
}, (table) => [
unique().on(table.userId, table.characterHistoryId)
]);
@@ -128,15 +139,15 @@ export const friendship = sqliteTable('friendship', {
id: text('id')
.primaryKey()
.$defaultFn(() => crypto.randomUUID()),
requesterId: text('requesterId')
requesterId: text('requester_id')
.notNull()
.references(() => user.id, { onDelete: 'cascade' }),
addresseeId: text('addresseeId')
addresseeId: text('addressee_id')
.notNull()
.references(() => user.id, { onDelete: 'cascade' }),
status: text('status').$type<FriendshipStatus>().notNull().default('pending'),
createdAt: integer('createdAt').notNull().$default(() => Date.now()),
updatedAt: integer('updatedAt').notNull().$default(() => Date.now()),
createdAt: integer('created_at').notNull().$default(() => Date.now()),
updatedAt: integer('updated_at').notNull().$default(() => Date.now()),
}, (table) => [
unique().on(table.requesterId, table.addresseeId)
]);