Whidix/OnePieceDle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor database schema and update scraping logic for One Piece characters and arcs

Browse Source 
- Updated database schema to include French names and adjusted field names for consistency.
- Modified scraping script to fetch and store French names for arcs and characters.
- Improved API calls to handle redirects and fetch additional data for characters.
- Enhanced data extraction methods for character attributes and devil fruits.
- Cleaned up code for better readability and maintainability.
This commit is contained in:
whidix
2026-03-14 01:23:29 +01:00
parent a91b298ee5
commit a041a8caf5
24 changed files with 844 additions and 11193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
drizzle/0000_graceful_master_mold.sql
View File

@@ -1,156 +0,0 @@
CREATE TABLE `arc` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`startChapter` integer NOT NULL,
`endChapter` integer,
`url` text
);
--> statement-breakpoint
CREATE TABLE `character` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`gender` text,
`age` integer,
`affiliations` text,
`devilFruitId` text,
`hakiObservation` integer DEFAULT false,
`hakiArmament` integer DEFAULT false,
`hakiConqueror` integer DEFAULT false,
`bounty` integer DEFAULT 0,
`height` real,
`origin` text,
`firstAppearance` integer NOT NULL,
`pictureUrl` text,
`epithets` text,
`status` text,
`arcId` text,
`url` text,
`isInDailyMode` integer DEFAULT true,
FOREIGN KEY (`devilFruitId`) REFERENCES `devilFruit`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`arcId`) REFERENCES `arc`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
CREATE TABLE `characterHistory` (
`id` text PRIMARY KEY NOT NULL,
`characterId` text,
`date` text,
`won` integer DEFAULT 0 NOT NULL,
`createdAt` integer NOT NULL,
`updatedAt` integer NOT NULL,
FOREIGN KEY (`characterId`) REFERENCES `character`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
CREATE TABLE `characterOverride` (
`characterId` text PRIMARY KEY NOT NULL,
`name` text,
`gender` text,
`age` integer,
`affiliations` text,
`devilFruitId` text,
`hakiObservation` integer,
`hakiArmament` integer,
`hakiConqueror` integer,
`bounty` integer,
`height` real,
`origin` text,
`firstAppearance` integer NOT NULL,
`pictureUrl` text,
`epithets` text,
`status` text,
`arcId` text,
`url` text,
`notes` text,
FOREIGN KEY (`characterId`) REFERENCES `character`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`devilFruitId`) REFERENCES `devilFruit`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`arcId`) REFERENCES `arc`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
CREATE TABLE `characterScrapeValidation` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`gender` text,
`age` integer,
`affiliations` text,
`devilFruitId` text,
`hakiObservation` integer DEFAULT false,
`hakiArmament` integer DEFAULT false,
`hakiConqueror` integer DEFAULT false,
`bounty` integer,
`height` real,
`origin` text,
`firstAppearance` integer NOT NULL,
`pictureUrl` text,
`epithets` text,
`status` text,
`arcId` text,
`url` text,
FOREIGN KEY (`devilFruitId`) REFERENCES `devilFruit`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`arcId`) REFERENCES `arc`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
CREATE TABLE `config` (
`key` text PRIMARY KEY NOT NULL,
`value` text
);
--> statement-breakpoint
CREATE TABLE `devilFruit` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`type` text,
`url` text
);
--> statement-breakpoint
CREATE UNIQUE INDEX `devilFruit_name_unique` ON `devilFruit` (`name`);--> statement-breakpoint
CREATE TABLE `account` (
`id` text PRIMARY KEY NOT NULL,
`account_id` text NOT NULL,
`provider_id` text NOT NULL,
`user_id` text NOT NULL,
`access_token` text,
`refresh_token` text,
`id_token` text,
`access_token_expires_at` integer,
`refresh_token_expires_at` integer,
`scope` text,
`password` text,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
`updated_at` integer NOT NULL,
FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE INDEX `account_userId_idx` ON `account` (`user_id`);--> statement-breakpoint
CREATE TABLE `session` (
`id` text PRIMARY KEY NOT NULL,
`expires_at` integer NOT NULL,
`token` text NOT NULL,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
`updated_at` integer NOT NULL,
`ip_address` text,
`user_agent` text,
`user_id` text NOT NULL,
FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE UNIQUE INDEX `session_token_unique` ON `session` (`token`);--> statement-breakpoint
CREATE INDEX `session_userId_idx` ON `session` (`user_id`);--> statement-breakpoint
CREATE TABLE `user` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`email` text NOT NULL,
`email_verified` integer DEFAULT false NOT NULL,
`image` text,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
`updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL
);
--> statement-breakpoint
CREATE UNIQUE INDEX `user_email_unique` ON `user` (`email`);--> statement-breakpoint
CREATE TABLE `verification` (
`id` text PRIMARY KEY NOT NULL,
`identifier` text NOT NULL,
`value` text NOT NULL,
`expires_at` integer NOT NULL,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
`updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL
);
--> statement-breakpoint
CREATE INDEX `verification_identifier_idx` ON `verification` (`identifier`);

194
drizzle/0000_keen_rockslide.sql Normal file
View File

@@ -0,0 +1,194 @@
CREATE TABLE `arc` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`fr_name` text,
`start_chapter` integer NOT NULL,
`end_chapter` integer,
`url` text
);
--> statement-breakpoint
CREATE TABLE `character` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`fr_name` text,
`gender` text,
`age` integer,
`affiliations` text,
`devil_fruit_id` text,
`haki_observation` integer DEFAULT false,
`haki_armament` integer DEFAULT false,
`haki_conqueror` integer DEFAULT false,
`bounty` integer DEFAULT 0,
`height` real,
`origin` text,
`fr_origin` text,
`first_appearance` integer NOT NULL,
`picture_url` text,
`epithets` text,
`fr_epithets` text,
`status` text,
`arc_id` text,
`url` text,
`fr_url` text,
`is_in_daily_mode` integer DEFAULT false,
FOREIGN KEY (`devil_fruit_id`) REFERENCES `devil_fruit`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`arc_id`) REFERENCES `arc`(`id`) ON UPDATE no action ON DELETE set null
);
--> statement-breakpoint
CREATE TABLE `character_history` (
`id` text PRIMARY KEY NOT NULL,
`character_id` text,
`date` integer NOT NULL,
`won` integer DEFAULT 0 NOT NULL,
`created_at` integer NOT NULL,
`updated_at` integer NOT NULL,
FOREIGN KEY (`character_id`) REFERENCES `character`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE UNIQUE INDEX `character_history_date_unique` ON `character_history` (`date`);--> statement-breakpoint
CREATE TABLE `character_override` (
`character_id` text PRIMARY KEY NOT NULL,
`name` text,
`gender` text,
`age` integer,
`affiliations` text,
`devil_fruit_id` text,
`haki_observation` integer,
`haki_armament` integer,
`haki_conqueror` integer,
`bounty` integer,
`height` real,
`origin` text,
`first_appearance` integer,
`picture_url` text,
`epithets` text,
`status` text,
`arc_id` text,
`url` text,
`fr_url` text,
`notes` text,
FOREIGN KEY (`character_id`) REFERENCES `character`(`id`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (`devil_fruit_id`) REFERENCES `devil_fruit`(`id`) ON UPDATE no action ON DELETE set null,
FOREIGN KEY (`arc_id`) REFERENCES `arc`(`id`) ON UPDATE no action ON DELETE set null
);
--> statement-breakpoint
CREATE TABLE `character_scrape_validation` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`fr_name` text,
`gender` text,
`age` integer,
`affiliations` text,
`devil_fruit_id` text,
`haki_observation` integer DEFAULT false,
`haki_armament` integer DEFAULT false,
`haki_conqueror` integer DEFAULT false,
`bounty` integer,
`height` real,
`origin` text,
`fr_origin` text,
`first_appearance` integer NOT NULL,
`picture_url` text,
`epithets` text,
`fr_epithets` text,
`status` text,
`arc_id` text,
`url` text,
`fr_url` text,
FOREIGN KEY (`devil_fruit_id`) REFERENCES `devil_fruit`(`id`) ON UPDATE no action ON DELETE set null,
FOREIGN KEY (`arc_id`) REFERENCES `arc`(`id`) ON UPDATE no action ON DELETE set null
);
--> statement-breakpoint
CREATE TABLE `config` (
`key` text PRIMARY KEY NOT NULL,
`value` text
);
--> statement-breakpoint
CREATE TABLE `devil_fruit` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`type` text,
`url` text
);
--> statement-breakpoint
CREATE UNIQUE INDEX `devil_fruit_name_unique` ON `devil_fruit` (`name`);--> statement-breakpoint
CREATE TABLE `friendship` (
`id` text PRIMARY KEY NOT NULL,
`requester_id` text NOT NULL,
`addressee_id` text NOT NULL,
`status` text DEFAULT 'pending' NOT NULL,
`created_at` integer NOT NULL,
`updated_at` integer NOT NULL,
FOREIGN KEY (`requester_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (`addressee_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE UNIQUE INDEX `friendship_requester_id_addressee_id_unique` ON `friendship` (`requester_id`,`addressee_id`);--> statement-breakpoint
CREATE TABLE `user_character_history` (
`id` text PRIMARY KEY NOT NULL,
`user_id` text,
`character_history_id` text,
`try_count` integer NOT NULL,
`tried_character_ids` text,
`created_at` integer NOT NULL,
FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (`character_history_id`) REFERENCES `character_history`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE UNIQUE INDEX `user_character_history_user_id_character_history_id_unique` ON `user_character_history` (`user_id`,`character_history_id`);--> statement-breakpoint
CREATE TABLE `account` (
`id` text PRIMARY KEY NOT NULL,
`account_id` text NOT NULL,
`provider_id` text NOT NULL,
`user_id` text NOT NULL,
`access_token` text,
`refresh_token` text,
`id_token` text,
`access_token_expires_at` integer,
`refresh_token_expires_at` integer,
`scope` text,
`password` text,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
`updated_at` integer NOT NULL,
FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE INDEX `account_userId_idx` ON `account` (`user_id`);--> statement-breakpoint
CREATE TABLE `session` (
`id` text PRIMARY KEY NOT NULL,
`expires_at` integer NOT NULL,
`token` text NOT NULL,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
`updated_at` integer NOT NULL,
`ip_address` text,
`user_agent` text,
`user_id` text NOT NULL,
FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE UNIQUE INDEX `session_token_unique` ON `session` (`token`);--> statement-breakpoint
CREATE INDEX `session_userId_idx` ON `session` (`user_id`);--> statement-breakpoint
CREATE TABLE `user` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`username` text NOT NULL,
`email` text NOT NULL,
`email_verified` integer DEFAULT false NOT NULL,
`image` text,
`is_admin` integer DEFAULT false NOT NULL,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
`updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL
);
--> statement-breakpoint
CREATE UNIQUE INDEX `user_username_unique` ON `user` (`username`);--> statement-breakpoint
CREATE UNIQUE INDEX `user_email_unique` ON `user` (`email`);--> statement-breakpoint
CREATE TABLE `verification` (
`id` text PRIMARY KEY NOT NULL,
`identifier` text NOT NULL,
`value` text NOT NULL,
`expires_at` integer NOT NULL,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
`updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL
);
--> statement-breakpoint
CREATE INDEX `verification_identifier_idx` ON `verification` (`identifier`);

30
drizzle/0001_nostalgic_hercules.sql
View File

@@ -1,30 +0,0 @@
PRAGMA foreign_keys=OFF;--> statement-breakpoint
CREATE TABLE `__new_characterOverride` (
`characterId` text PRIMARY KEY NOT NULL,
`name` text,
`gender` text,
`age` integer,
`affiliations` text,
`devilFruitId` text,
`hakiObservation` integer,
`hakiArmament` integer,
`hakiConqueror` integer,
`bounty` integer,
`height` real,
`origin` text,
`firstAppearance` integer,
`pictureUrl` text,
`epithets` text,
`status` text,
`arcId` text,
`url` text,
`notes` text,
FOREIGN KEY (`characterId`) REFERENCES `character`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`devilFruitId`) REFERENCES `devilFruit`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`arcId`) REFERENCES `arc`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
INSERT INTO `__new_characterOverride`("characterId", "name", "gender", "age", "affiliations", "devilFruitId", "hakiObservation", "hakiArmament", "hakiConqueror", "bounty", "height", "origin", "firstAppearance", "pictureUrl", "epithets", "status", "arcId", "url", "notes") SELECT "characterId", "name", "gender", "age", "affiliations", "devilFruitId", "hakiObservation", "hakiArmament", "hakiConqueror", "bounty", "height", "origin", "firstAppearance", "pictureUrl", "epithets", "status", "arcId", "url", "notes" FROM `characterOverride`;--> statement-breakpoint
DROP TABLE `characterOverride`;--> statement-breakpoint
ALTER TABLE `__new_characterOverride` RENAME TO `characterOverride`;--> statement-breakpoint
PRAGMA foreign_keys=ON;

1
drizzle/0002_large_gwen_stacy.sql
View File

@@ -1 +0,0 @@
ALTER TABLE `user` ADD `is_admin` integer DEFAULT false NOT NULL;

16
drizzle/0003_wise_blonde_phantom.sql
View File

@@ -1,16 +0,0 @@
PRAGMA foreign_keys=OFF;--> statement-breakpoint
CREATE TABLE `__new_characterHistory` (
`id` text PRIMARY KEY NOT NULL,
`characterId` text,
`date` integer NOT NULL,
`won` integer DEFAULT 0 NOT NULL,
`createdAt` integer NOT NULL,
`updatedAt` integer NOT NULL,
FOREIGN KEY (`characterId`) REFERENCES `character`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
INSERT INTO `__new_characterHistory`("id", "characterId", "date", "won", "createdAt", "updatedAt") SELECT "id", "characterId", "date", "won", "createdAt", "updatedAt" FROM `characterHistory`;--> statement-breakpoint
DROP TABLE `characterHistory`;--> statement-breakpoint
ALTER TABLE `__new_characterHistory` RENAME TO `characterHistory`;--> statement-breakpoint
PRAGMA foreign_keys=ON;--> statement-breakpoint
CREATE UNIQUE INDEX `characterHistory_date_unique` ON `characterHistory` (`date`);

9
drizzle/0004_unique_lorna_dane.sql
View File

@@ -1,9 +0,0 @@
CREATE TABLE `userCharacterHistory` (
`id` text PRIMARY KEY NOT NULL,
`userId` text,
`characterId` text,
`tryCount` integer NOT NULL,
`createdAt` integer NOT NULL,
FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`characterId`) REFERENCES `character`(`id`) ON UPDATE no action ON DELETE no action
);

15
drizzle/0005_large_jane_foster.sql
View File

@@ -1,15 +0,0 @@
PRAGMA foreign_keys=OFF;--> statement-breakpoint
CREATE TABLE `__new_userCharacterHistory` (
`id` text PRIMARY KEY NOT NULL,
`userId` text,
`characterHistoryId` text,
`tryCount` integer NOT NULL,
`createdAt` integer NOT NULL,
FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`characterHistoryId`) REFERENCES `characterHistory`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
INSERT INTO `__new_userCharacterHistory`("id", "userId", "characterHistoryId", "tryCount", "createdAt") SELECT "id", "userId", "characterId", "tryCount", "createdAt" FROM `userCharacterHistory`;--> statement-breakpoint
DROP TABLE `userCharacterHistory`;--> statement-breakpoint
ALTER TABLE `__new_userCharacterHistory` RENAME TO `userCharacterHistory`;--> statement-breakpoint
PRAGMA foreign_keys=ON;

1
drizzle/0006_premium_mesmero.sql
View File

@@ -1 +0,0 @@
CREATE UNIQUE INDEX `userCharacterHistory_userId_characterHistoryId_unique` ON `userCharacterHistory` (`userId`,`characterHistoryId`);

29
drizzle/0007_gray_shinko_yamashiro.sql
View File

@@ -1,29 +0,0 @@
PRAGMA foreign_keys=OFF;--> statement-breakpoint
CREATE TABLE `__new_character` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`gender` text,
`age` integer,
`affiliations` text,
`devilFruitId` text,
`hakiObservation` integer DEFAULT false,
`hakiArmament` integer DEFAULT false,
`hakiConqueror` integer DEFAULT false,
`bounty` integer DEFAULT 0,
`height` real,
`origin` text,
`firstAppearance` integer NOT NULL,
`pictureUrl` text,
`epithets` text,
`status` text,
`arcId` text,
`url` text,
`isInDailyMode` integer DEFAULT false,
FOREIGN KEY (`devilFruitId`) REFERENCES `devilFruit`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`arcId`) REFERENCES `arc`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
INSERT INTO `__new_character`("id", "name", "gender", "age", "affiliations", "devilFruitId", "hakiObservation", "hakiArmament", "hakiConqueror", "bounty", "height", "origin", "firstAppearance", "pictureUrl", "epithets", "status", "arcId", "url", "isInDailyMode") SELECT "id", "name", "gender", "age", "affiliations", "devilFruitId", "hakiObservation", "hakiArmament", "hakiConqueror", "bounty", "height", "origin", "firstAppearance", "pictureUrl", "epithets", "status", "arcId", "url", "isInDailyMode" FROM `character`;--> statement-breakpoint
DROP TABLE `character`;--> statement-breakpoint
ALTER TABLE `__new_character` RENAME TO `character`;--> statement-breakpoint
PRAGMA foreign_keys=ON;

12
drizzle/0008_skinny_warpath.sql
View File

@@ -1,12 +0,0 @@
CREATE TABLE `friendship` (
`id` text PRIMARY KEY NOT NULL,
`requesterId` text NOT NULL,
`addresseeId` text NOT NULL,
`status` text DEFAULT 'pending' NOT NULL,
`createdAt` integer NOT NULL,
`updatedAt` integer NOT NULL,
FOREIGN KEY (`requesterId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (`addresseeId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE UNIQUE INDEX `friendship_requesterId_addresseeId_unique` ON `friendship` (`requesterId`,`addresseeId`);

51
drizzle/0009_true_gravity.sql
View File

@@ -1,51 +0,0 @@
PRAGMA foreign_keys=OFF;--> statement-breakpoint
ALTER TABLE `user` ADD `username` text;--> statement-breakpoint
UPDATE `user`
SET `username` = `name`
WHERE `username` IS NULL OR trim(`username`) = '';--> statement-breakpoint
UPDATE `user`
SET `username` = `username` || '_' || substr(`id`, 1, 6)
WHERE `id` IN (
SELECT u1.`id`
FROM `user` u1
JOIN `user` u2
ON lower(u1.`username`) = lower(u2.`username`)
AND u1.`id` > u2.`id`
);--> statement-breakpoint
CREATE TABLE `__new_user` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`username` text NOT NULL,
`email` text NOT NULL,
`email_verified` integer DEFAULT false NOT NULL,
`image` text,
`is_admin` integer DEFAULT false NOT NULL,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
`updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL
);--> statement-breakpoint
INSERT INTO `__new_user` (
`id`,
`name`,
`username`,
`email`,
`email_verified`,
`image`,
`is_admin`,
`created_at`,
`updated_at`
)
SELECT
`id`,
`name`,
`username`,
`email`,
`email_verified`,
`image`,
`is_admin`,
`created_at`,
`updated_at`
FROM `user`;--> statement-breakpoint
DROP TABLE `user`;--> statement-breakpoint
ALTER TABLE `__new_user` RENAME TO `user`;--> statement-breakpoint
CREATE UNIQUE INDEX `user_username_unique` ON `user` (`username`);--> statement-breakpoint
PRAGMA foreign_keys=ON;

501
drizzle/meta/0000_snapshot.json
View File

@@ -1,7 +1,7 @@
{ {
"version": "6", "version": "6",
"dialect": "sqlite", "dialect": "sqlite",
"id": "d1237d76-8f1c-4721-b8dd-d31082ed7b9a", "id": "8ffd14bd-bf33-410f-9778-92bc1abc8938",
"prevId": "00000000-0000-0000-0000-000000000000", "prevId": "00000000-0000-0000-0000-000000000000",
"tables": { "tables": {
"arc": { "arc": {
@@ -21,15 +21,22 @@
"notNull": true, "notNull": true,
"autoincrement": false "autoincrement": false
}, },
"startChapter": { "fr_name": {
"name": "startChapter", "name": "fr_name",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"start_chapter": {
"name": "start_chapter",
"type": "integer", "type": "integer",
"primaryKey": false, "primaryKey": false,
"notNull": true, "notNull": true,
"autoincrement": false "autoincrement": false
}, },
"endChapter": { "end_chapter": {
"name": "endChapter", "name": "end_chapter",
"type": "integer", "type": "integer",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
@@ -66,6 +73,13 @@
"notNull": true, "notNull": true,
"autoincrement": false "autoincrement": false
}, },
"fr_name": {
"name": "fr_name",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"gender": { "gender": {
"name": "gender", "name": "gender",
"type": "text", "type": "text",
@@ -87,31 +101,31 @@
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"devilFruitId": { "devil_fruit_id": {
"name": "devilFruitId", "name": "devil_fruit_id",
"type": "text", "type": "text",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"hakiObservation": { "haki_observation": {
"name": "hakiObservation", "name": "haki_observation",
"type": "integer", "type": "integer",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
"autoincrement": false, "autoincrement": false,
"default": false "default": false
}, },
"hakiArmament": { "haki_armament": {
"name": "hakiArmament", "name": "haki_armament",
"type": "integer", "type": "integer",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
"autoincrement": false, "autoincrement": false,
"default": false "default": false
}, },
"hakiConqueror": { "haki_conqueror": {
"name": "hakiConqueror", "name": "haki_conqueror",
"type": "integer", "type": "integer",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
@@ -140,15 +154,22 @@
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"firstAppearance": { "fr_origin": {
"name": "firstAppearance", "name": "fr_origin",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"first_appearance": {
"name": "first_appearance",
"type": "integer", "type": "integer",
"primaryKey": false, "primaryKey": false,
"notNull": true, "notNull": true,
"autoincrement": false "autoincrement": false
}, },
"pictureUrl": { "picture_url": {
"name": "pictureUrl", "name": "picture_url",
"type": "text", "type": "text",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
@@ -161,6 +182,13 @@
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"fr_epithets": {
"name": "fr_epithets",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"status": { "status": {
"name": "status", "name": "status",
"type": "text", "type": "text",
@@ -168,8 +196,8 @@
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"arcId": { "arc_id": {
"name": "arcId", "name": "arc_id",
"type": "text", "type": "text",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
@@ -182,23 +210,30 @@
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"isInDailyMode": { "fr_url": {
"name": "isInDailyMode", "name": "fr_url",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"is_in_daily_mode": {
"name": "is_in_daily_mode",
"type": "integer", "type": "integer",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
"autoincrement": false, "autoincrement": false,
"default": true "default": false
} }
}, },
"indexes": {}, "indexes": {},
"foreignKeys": { "foreignKeys": {
"character_devilFruitId_devilFruit_id_fk": { "character_devil_fruit_id_devil_fruit_id_fk": {
"name": "character_devilFruitId_devilFruit_id_fk", "name": "character_devil_fruit_id_devil_fruit_id_fk",
"tableFrom": "character", "tableFrom": "character",
"tableTo": "devilFruit", "tableTo": "devil_fruit",
"columnsFrom": [ "columnsFrom": [
"devilFruitId" "devil_fruit_id"
], ],
"columnsTo": [ "columnsTo": [
"id" "id"
@@ -206,17 +241,17 @@
"onDelete": "no action", "onDelete": "no action",
"onUpdate": "no action" "onUpdate": "no action"
}, },
"character_arcId_arc_id_fk": { "character_arc_id_arc_id_fk": {
"name": "character_arcId_arc_id_fk", "name": "character_arc_id_arc_id_fk",
"tableFrom": "character", "tableFrom": "character",
"tableTo": "arc", "tableTo": "arc",
"columnsFrom": [ "columnsFrom": [
"arcId" "arc_id"
], ],
"columnsTo": [ "columnsTo": [
"id" "id"
], ],
"onDelete": "no action", "onDelete": "set null",
"onUpdate": "no action" "onUpdate": "no action"
} }
}, },
@@ -224,8 +259,8 @@
"uniqueConstraints": {}, "uniqueConstraints": {},
"checkConstraints": {} "checkConstraints": {}
}, },
"characterHistory": { "character_history": {
"name": "characterHistory", "name": "character_history",
"columns": { "columns": {
"id": { "id": {
"name": "id", "name": "id",
@@ -234,8 +269,8 @@
"notNull": true, "notNull": true,
"autoincrement": false "autoincrement": false
}, },
"characterId": { "character_id": {
"name": "characterId", "name": "character_id",
"type": "text", "type": "text",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
@@ -243,9 +278,9 @@
}, },
"date": { "date": {
"name": "date", "name": "date",
"type": "text", "type": "integer",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": true,
"autoincrement": false "autoincrement": false
}, },
"won": { "won": {
@@ -256,34 +291,42 @@
"autoincrement": false, "autoincrement": false,
"default": 0 "default": 0
}, },
"createdAt": { "created_at": {
"name": "createdAt", "name": "created_at",
"type": "integer", "type": "integer",
"primaryKey": false, "primaryKey": false,
"notNull": true, "notNull": true,
"autoincrement": false "autoincrement": false
}, },
"updatedAt": { "updated_at": {
"name": "updatedAt", "name": "updated_at",
"type": "integer", "type": "integer",
"primaryKey": false, "primaryKey": false,
"notNull": true, "notNull": true,
"autoincrement": false "autoincrement": false
} }
}, },
"indexes": {}, "indexes": {
"character_history_date_unique": {
"name": "character_history_date_unique",
"columns": [
"date"
],
"isUnique": true
}
},
"foreignKeys": { "foreignKeys": {
"characterHistory_characterId_character_id_fk": { "character_history_character_id_character_id_fk": {
"name": "characterHistory_characterId_character_id_fk", "name": "character_history_character_id_character_id_fk",
"tableFrom": "characterHistory", "tableFrom": "character_history",
"tableTo": "character", "tableTo": "character",
"columnsFrom": [ "columnsFrom": [
"characterId" "character_id"
], ],
"columnsTo": [ "columnsTo": [
"id" "id"
], ],
"onDelete": "no action", "onDelete": "cascade",
"onUpdate": "no action" "onUpdate": "no action"
} }
}, },
@@ -291,11 +334,11 @@
"uniqueConstraints": {}, "uniqueConstraints": {},
"checkConstraints": {} "checkConstraints": {}
}, },
"characterOverride": { "character_override": {
"name": "characterOverride", "name": "character_override",
"columns": { "columns": {
"characterId": { "character_id": {
"name": "characterId", "name": "character_id",
"type": "text", "type": "text",
"primaryKey": true, "primaryKey": true,
"notNull": true, "notNull": true,
@@ -329,29 +372,29 @@
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"devilFruitId": { "devil_fruit_id": {
"name": "devilFruitId", "name": "devil_fruit_id",
"type": "text", "type": "text",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"hakiObservation": { "haki_observation": {
"name": "hakiObservation", "name": "haki_observation",
"type": "integer", "type": "integer",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"hakiArmament": { "haki_armament": {
"name": "hakiArmament", "name": "haki_armament",
"type": "integer", "type": "integer",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"hakiConqueror": { "haki_conqueror": {
"name": "hakiConqueror", "name": "haki_conqueror",
"type": "integer", "type": "integer",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
@@ -378,15 +421,15 @@
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"firstAppearance": { "first_appearance": {
"name": "firstAppearance", "name": "first_appearance",
"type": "integer", "type": "integer",
"primaryKey": false, "primaryKey": false,
"notNull": true, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"pictureUrl": { "picture_url": {
"name": "pictureUrl", "name": "picture_url",
"type": "text", "type": "text",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
@@ -406,8 +449,8 @@
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"arcId": { "arc_id": {
"name": "arcId", "name": "arc_id",
"type": "text", "type": "text",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
@@ -420,6 +463,13 @@
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"fr_url": {
"name": "fr_url",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"notes": { "notes": {
"name": "notes", "name": "notes",
"type": "text", "type": "text",
@@ -430,43 +480,43 @@
}, },
"indexes": {}, "indexes": {},
"foreignKeys": { "foreignKeys": {
"characterOverride_characterId_character_id_fk": { "character_override_character_id_character_id_fk": {
"name": "characterOverride_characterId_character_id_fk", "name": "character_override_character_id_character_id_fk",
"tableFrom": "characterOverride", "tableFrom": "character_override",
"tableTo": "character", "tableTo": "character",
"columnsFrom": [ "columnsFrom": [
"characterId" "character_id"
], ],
"columnsTo": [ "columnsTo": [
"id" "id"
], ],
"onDelete": "no action", "onDelete": "cascade",
"onUpdate": "no action" "onUpdate": "no action"
}, },
"characterOverride_devilFruitId_devilFruit_id_fk": { "character_override_devil_fruit_id_devil_fruit_id_fk": {
"name": "characterOverride_devilFruitId_devilFruit_id_fk", "name": "character_override_devil_fruit_id_devil_fruit_id_fk",
"tableFrom": "characterOverride", "tableFrom": "character_override",
"tableTo": "devilFruit", "tableTo": "devil_fruit",
"columnsFrom": [ "columnsFrom": [
"devilFruitId" "devil_fruit_id"
], ],
"columnsTo": [ "columnsTo": [
"id" "id"
], ],
"onDelete": "no action", "onDelete": "set null",
"onUpdate": "no action" "onUpdate": "no action"
}, },
"characterOverride_arcId_arc_id_fk": { "character_override_arc_id_arc_id_fk": {
"name": "characterOverride_arcId_arc_id_fk", "name": "character_override_arc_id_arc_id_fk",
"tableFrom": "characterOverride", "tableFrom": "character_override",
"tableTo": "arc", "tableTo": "arc",
"columnsFrom": [ "columnsFrom": [
"arcId" "arc_id"
], ],
"columnsTo": [ "columnsTo": [
"id" "id"
], ],
"onDelete": "no action", "onDelete": "set null",
"onUpdate": "no action" "onUpdate": "no action"
} }
}, },
@@ -474,8 +524,8 @@
"uniqueConstraints": {}, "uniqueConstraints": {},
"checkConstraints": {} "checkConstraints": {}
}, },
"characterScrapeValidation": { "character_scrape_validation": {
"name": "characterScrapeValidation", "name": "character_scrape_validation",
"columns": { "columns": {
"id": { "id": {
"name": "id", "name": "id",
@@ -491,6 +541,13 @@
"notNull": true, "notNull": true,
"autoincrement": false "autoincrement": false
}, },
"fr_name": {
"name": "fr_name",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"gender": { "gender": {
"name": "gender", "name": "gender",
"type": "text", "type": "text",
@@ -512,31 +569,31 @@
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"devilFruitId": { "devil_fruit_id": {
"name": "devilFruitId", "name": "devil_fruit_id",
"type": "text", "type": "text",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"hakiObservation": { "haki_observation": {
"name": "hakiObservation", "name": "haki_observation",
"type": "integer", "type": "integer",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
"autoincrement": false, "autoincrement": false,
"default": false "default": false
}, },
"hakiArmament": { "haki_armament": {
"name": "hakiArmament", "name": "haki_armament",
"type": "integer", "type": "integer",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
"autoincrement": false, "autoincrement": false,
"default": false "default": false
}, },
"hakiConqueror": { "haki_conqueror": {
"name": "hakiConqueror", "name": "haki_conqueror",
"type": "integer", "type": "integer",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
@@ -564,15 +621,22 @@
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"firstAppearance": { "fr_origin": {
"name": "firstAppearance", "name": "fr_origin",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"first_appearance": {
"name": "first_appearance",
"type": "integer", "type": "integer",
"primaryKey": false, "primaryKey": false,
"notNull": true, "notNull": true,
"autoincrement": false "autoincrement": false
}, },
"pictureUrl": { "picture_url": {
"name": "pictureUrl", "name": "picture_url",
"type": "text", "type": "text",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
@@ -585,6 +649,13 @@
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"fr_epithets": {
"name": "fr_epithets",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"status": { "status": {
"name": "status", "name": "status",
"type": "text", "type": "text",
@@ -592,8 +663,8 @@
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"arcId": { "arc_id": {
"name": "arcId", "name": "arc_id",
"type": "text", "type": "text",
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
@@ -605,34 +676,41 @@
"primaryKey": false, "primaryKey": false,
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
},
"fr_url": {
"name": "fr_url",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
} }
}, },
"indexes": {}, "indexes": {},
"foreignKeys": { "foreignKeys": {
"characterScrapeValidation_devilFruitId_devilFruit_id_fk": { "character_scrape_validation_devil_fruit_id_devil_fruit_id_fk": {
"name": "characterScrapeValidation_devilFruitId_devilFruit_id_fk", "name": "character_scrape_validation_devil_fruit_id_devil_fruit_id_fk",
"tableFrom": "characterScrapeValidation", "tableFrom": "character_scrape_validation",
"tableTo": "devilFruit", "tableTo": "devil_fruit",
"columnsFrom": [ "columnsFrom": [
"devilFruitId" "devil_fruit_id"
], ],
"columnsTo": [ "columnsTo": [
"id" "id"
], ],
"onDelete": "no action", "onDelete": "set null",
"onUpdate": "no action" "onUpdate": "no action"
}, },
"characterScrapeValidation_arcId_arc_id_fk": { "character_scrape_validation_arc_id_arc_id_fk": {
"name": "characterScrapeValidation_arcId_arc_id_fk", "name": "character_scrape_validation_arc_id_arc_id_fk",
"tableFrom": "characterScrapeValidation", "tableFrom": "character_scrape_validation",
"tableTo": "arc", "tableTo": "arc",
"columnsFrom": [ "columnsFrom": [
"arcId" "arc_id"
], ],
"columnsTo": [ "columnsTo": [
"id" "id"
], ],
"onDelete": "no action", "onDelete": "set null",
"onUpdate": "no action" "onUpdate": "no action"
} }
}, },
@@ -664,8 +742,8 @@
"uniqueConstraints": {}, "uniqueConstraints": {},
"checkConstraints": {} "checkConstraints": {}
}, },
"devilFruit": { "devil_fruit": {
"name": "devilFruit", "name": "devil_fruit",
"columns": { "columns": {
"id": { "id": {
"name": "id", "name": "id",
@@ -697,8 +775,8 @@
} }
}, },
"indexes": { "indexes": {
"devilFruit_name_unique": { "devil_fruit_name_unique": {
"name": "devilFruit_name_unique", "name": "devil_fruit_name_unique",
"columns": [ "columns": [
"name" "name"
], ],
@@ -710,6 +788,183 @@
"uniqueConstraints": {}, "uniqueConstraints": {},
"checkConstraints": {} "checkConstraints": {}
}, },
"friendship": {
"name": "friendship",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"requester_id": {
"name": "requester_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"addressee_id": {
"name": "addressee_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"status": {
"name": "status",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'pending'"
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"friendship_requester_id_addressee_id_unique": {
"name": "friendship_requester_id_addressee_id_unique",
"columns": [
"requester_id",
"addressee_id"
],
"isUnique": true
}
},
"foreignKeys": {
"friendship_requester_id_user_id_fk": {
"name": "friendship_requester_id_user_id_fk",
"tableFrom": "friendship",
"tableTo": "user",
"columnsFrom": [
"requester_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
},
"friendship_addressee_id_user_id_fk": {
"name": "friendship_addressee_id_user_id_fk",
"tableFrom": "friendship",
"tableTo": "user",
"columnsFrom": [
"addressee_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"user_character_history": {
"name": "user_character_history",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"user_id": {
"name": "user_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"character_history_id": {
"name": "character_history_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"try_count": {
"name": "try_count",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"tried_character_ids": {
"name": "tried_character_ids",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"user_character_history_user_id_character_history_id_unique": {
"name": "user_character_history_user_id_character_history_id_unique",
"columns": [
"user_id",
"character_history_id"
],
"isUnique": true
}
},
"foreignKeys": {
"user_character_history_user_id_user_id_fk": {
"name": "user_character_history_user_id_user_id_fk",
"tableFrom": "user_character_history",
"tableTo": "user",
"columnsFrom": [
"user_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
},
"user_character_history_character_history_id_character_history_id_fk": {
"name": "user_character_history_character_history_id_character_history_id_fk",
"tableFrom": "user_character_history",
"tableTo": "character_history",
"columnsFrom": [
"character_history_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"account": { "account": {
"name": "account", "name": "account",
"columns": { "columns": {
@@ -947,6 +1202,13 @@
"notNull": true, "notNull": true,
"autoincrement": false "autoincrement": false
}, },
"username": {
"name": "username",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"email": { "email": {
"name": "email", "name": "email",
"type": "text", "type": "text",
@@ -969,6 +1231,14 @@
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"is_admin": {
"name": "is_admin",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": false
},
"created_at": { "created_at": {
"name": "created_at", "name": "created_at",
"type": "integer", "type": "integer",
@@ -987,6 +1257,13 @@
} }
}, },
"indexes": { "indexes": {
"user_username_unique": {
"name": "user_username_unique",
"columns": [
"username"
],
"isUnique": true
},
"user_email_unique": { "user_email_unique": {
"name": "user_email_unique", "name": "user_email_unique",
"columns": [ "columns": [

1076
drizzle/meta/0001_snapshot.json
View File

File diff suppressed because it is too large Load Diff

1084
drizzle/meta/0002_snapshot.json
View File

File diff suppressed because it is too large Load Diff

1092
drizzle/meta/0003_snapshot.json
View File

File diff suppressed because it is too large Load Diff

1164
drizzle/meta/0004_snapshot.json
View File

File diff suppressed because it is too large Load Diff

1166
drizzle/meta/0005_snapshot.json
View File

File diff suppressed because it is too large Load Diff

1173
drizzle/meta/0006_snapshot.json
View File

File diff suppressed because it is too large Load Diff

1173
drizzle/meta/0007_snapshot.json
View File

File diff suppressed because it is too large Load Diff

1262
drizzle/meta/0008_snapshot.json
View File

File diff suppressed because it is too large Load Diff

1276
drizzle/meta/0009_snapshot.json
View File

File diff suppressed because it is too large Load Diff

67
drizzle/meta/_journal.json
View File

@@ -5,71 +5,8 @@
{ {
"idx": 0, "idx": 0,
"version": "6", "version": "6",
"when": 1772325597983, "when": 1773447741334,
"tag": "0000_graceful_master_mold", "tag": "0000_keen_rockslide",
"breakpoints": true
},
{
"idx": 1,
"version": "6",
"when": 1772383366179,
"tag": "0001_nostalgic_hercules",
"breakpoints": true
},
{
"idx": 2,
"version": "6",
"when": 1772390182445,
"tag": "0002_large_gwen_stacy",
"breakpoints": true
},
{
"idx": 3,
"version": "6",
"when": 1772449624450,
"tag": "0003_wise_blonde_phantom",
"breakpoints": true
},
{
"idx": 4,
"version": "6",
"when": 1772480377099,
"tag": "0004_unique_lorna_dane",
"breakpoints": true
},
{
"idx": 5,
"version": "6",
"when": 1772562012631,
"tag": "0005_large_jane_foster",
"breakpoints": true
},
{
"idx": 6,
"version": "6",
"when": 1772562364830,
"tag": "0006_premium_mesmero",
"breakpoints": true
},
{
"idx": 7,
"version": "6",
"when": 1772735982970,
"tag": "0007_gray_shinko_yamashiro",
"breakpoints": true
},
{
"idx": 8,
"version": "6",
"when": 1772821532270,
"tag": "0008_skinny_warpath",
"breakpoints": true
},
{
"idx": 9,
"version": "6",
"when": 1772822823122,
"tag": "0009_true_gravity",
"breakpoints": true "breakpoints": true
} }
] ]

370
scripts/scrape-onepiece.ts
View File

@@ -6,6 +6,7 @@ import { createObjectCsvWriter } from 'csv-writer';
interface Arc { interface Arc {
id: string; id: string;
name: string; name: string;
frName: string | null;
startChapter: number; startChapter: number;
endChapter: number | null; endChapter: number | null;
url: string; url: string;
@@ -14,30 +15,34 @@ interface Arc {
interface Character { interface Character {
id: string; id: string;
name: string; name: string;
frName: string | null;
gender: string | null; gender: string | null;
age: number | null; age: number | null;
height: number | null; height: number | null;
origin: string | null; origin: string | null;
frOrigin: string | null;
devilFruitId: string | null; devilFruitId: string | null;
devilFruitUrl: string | null; devilFruitUrl: string | null;
affiliations: string[]; affiliations: string[];
frAffiliations: string[] | null;
bounty: number | null; bounty: number | null;
hakiObservation: boolean; hakiObservation: boolean;
hakiArmament: boolean; hakiArmament: boolean;
hakiConqueror: boolean; hakiConqueror: boolean;
epithets: string[]; epithets: string[];
frEpithets: string[] | null;
firstAppearance: number; firstAppearance: number;
status: string | null; status: string | null;
pictureUrl: string | null; pictureUrl: string | null;
url: string; url: string;
arcId?: string; frUrl: string | null;
arcId: string;
} }
interface CharacterListItem { interface CharacterListItem {
name: string; name: string;
url: string; url: string;
pictureUrl: string | null; chapter: number;
chapter: string;
} }
interface DevilFruitData { interface DevilFruitData {
@@ -52,30 +57,13 @@ interface DevilFruit {
url: string; url: string;
} }
const FANDOM_API_BASE = 'https://onepiece.fandom.com/fr/api.php?action=parse&format=json&page='; const FANDOM_API_BASE = 'https://onepiece.fandom.com/api.php?action=parse&redirects=true&format=json&page=';
const FR_FANDOM_API_BASE = 'https://onepiece.fandom.com/fr/api.php?action=parse&redirects=true&format=json&page=';
const OUTPUT_DIR = './scraped-data'; const OUTPUT_DIR = './scraped-data';
const MAX_RETRIES = 0; // Set to 0 to disable retries, can be increased if needed const MAX_RETRIES = 0; // Set to 0 to disable retries, can be increased if needed
const INITIAL_RETRY_DELAY = 1000; const INITIAL_RETRY_DELAY = 1000;
const FETCH_CONCURRENCY = 50; const FETCH_CONCURRENCY = 50;
// Store cookies across requests (simulate browser behavior)
const cookies = new Map<string, string>();
function getCookieHeader(): string {
const cookieArray = Array.from(cookies.values()).map(c => c.split(';')[0]);
return cookieArray.length > 0 ? cookieArray.join('; ') : '';
}
function saveCookies(setCookieHeader: string | string[] | null): void {
if (setCookieHeader) {
const cookiesList = Array.isArray(setCookieHeader) ? setCookieHeader : [setCookieHeader];
cookiesList.forEach(cookie => {
const [nameValue] = cookie.split(';');
const [name] = nameValue.split('=');
if (name) cookies.set(name, cookie);
});
}
}
// Create output directory // Create output directory
if (!fs.existsSync(OUTPUT_DIR)) { if (!fs.existsSync(OUTPUT_DIR)) {
@@ -95,22 +83,10 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries: n
...((options.headers as Record<string, string>) || {}) ...((options.headers as Record<string, string>) || {})
}; };
// Add cookies from previous requests
const cookieHeader = getCookieHeader();
if (cookieHeader) {
headers['Cookie'] = cookieHeader;
}
const response = await fetch(url, { const response = await fetch(url, {
headers, headers,
...options ...options
} as any); });
// Save cookies from response
const setCookie = response.headers.get('set-cookie');
if (setCookie) {
saveCookies(setCookie);
}
// Check if response is OK (status 200-299) // Check if response is OK (status 200-299)
if (response.ok) { if (response.ok) {
@@ -141,6 +117,16 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries: n
} }
} }
/**
* Get the French link from the API response links array
*/
function getFrLink(links: { lang: string; ['*']: string; url: string }[]): { url: string } | null {
// Get french url by getting parse.langlinks where lang is "fr" and extract the name from there
const frLink = links.find((link: { lang: string; ['*']: string; url: string }) => link.lang === 'fr');
return frLink ? { url: frLink['url'] } : null;
}
/** /**
* Normalize string by decoding URI components, punctuation, and replacing spaces with underscores * Normalize string by decoding URI components, punctuation, and replacing spaces with underscores
@@ -148,7 +134,7 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries: n
function normalizeId(str: string): string { function normalizeId(str: string): string {
return decodeURIComponent(str) return decodeURIComponent(str)
.normalize('NFD') .normalize('NFD')
.replace(/[,:.\(\)]/g, '') .replace(/[,:.()]/g, '')
.replace(/\s+/g, '_') .replace(/\s+/g, '_')
.toLowerCase(); .toLowerCase();
} }
@@ -158,10 +144,10 @@ function normalizeId(str: string): string {
*/ */
async function fetchAllArcs(): Promise<Arc[]> { async function fetchAllArcs(): Promise<Arc[]> {
try { try {
const apiUrl = `${FANDOM_API_BASE}Chapitres_et_Tomes`; const apiUrl = `${FANDOM_API_BASE}Chapters_and_Volumes`;
console.log('Fetching arcs list via API...'); console.log('Fetching arcs list via API...');
const response = await fetchWithRetry(apiUrl); const response = await fetchWithRetry(apiUrl);
const jsonData = await response.json() as any; const jsonData = await response.json();
// Extract HTML from API response // Extract HTML from API response
const htmlContent = jsonData.parse?.text?.['*']; const htmlContent = jsonData.parse?.text?.['*'];
@@ -172,40 +158,66 @@ async function fetchAllArcs(): Promise<Arc[]> {
const $ = cheerio.load(htmlContent); const $ = cheerio.load(htmlContent);
const arcs: Arc[] = []; const arcs: Arc[] = [];
// Find all arc links in the table const seenArcUrls = new Set<string>();
$('table.wikitable td a').each((index, element) => {
const text = $(element).text().trim();
const href = $(element).attr('href');
// Check if it's an arc link (contains "Arc" and chapter info) // Arc rows are in table cells where first link points to a *_Arc page and text includes chapter range.
if (text.includes('Arc') && text.includes('Ch.') && href) { const arcCells = $('table.wikitable td').toArray();
// Extract arc name and chapter range for (const element of arcCells) {
// Example text: "Arc Ville d'Orange(Ch.8 à 21)[T.1 à 3]" const cell = $(element);
console.log(`Processing arc link: ${text} (${href})`); const firstLink = cell.find('a').first();
const nameMatch = text.match(/^(.*?Arc.*?)\s*\(Ch\.(\d+)(?:\s*à\s*(?:(\d+)|(?:...)))?\)/); const href = firstLink.attr('href') || '';
if (nameMatch) { let arcName = firstLink.text().trim();
let arcName = nameMatch[1].trim();
// Remove "Arc " from the name
arcName = arcName.replace(/^Arc\s+/i, '');
const startChapter = parseInt(nameMatch[2]); if (!href.startsWith('/wiki/') || !/_Arc$/i.test(href)) {
const endChapter = nameMatch[3] ? parseInt(nameMatch[3]) : null; continue;
}
// Generate arc ID by normalizing the url if (!arcName || !/\bArc\b/i.test(arcName)) {
let arcId = normalizeId(href.replace('/fr/wiki/', '')); continue;
// Remove "Arc_" from the id }
arcId = arcId.replace(/^arc_/i, '');
arcName = arcName.replace(/\bArc\b/i, '').trim();
const cleanUrl = href.replace('/wiki/', '');
if (seenArcUrls.has(cleanUrl)) {
continue;
}
const cellText = cell.text().replace(/\s+/g, ' ').trim();
const chapterMatch = cellText.match(/Chapters\s+(\d+)\s+to\s+(\d+|Current)/i);
if (!chapterMatch) {
continue;
}
const startChapter = parseInt(chapterMatch[1], 10);
const endChapter = /current/i.test(chapterMatch[2])
? null
: parseInt(chapterMatch[2], 10);
let arcId = normalizeId(cleanUrl);
arcId = arcId.replace(/_arc$/i, '');
// Query the href page via API to get the correct HTML content (in case of redirect) and extract the French name from there
const arcResponse = await fetchWithRetry(`${FANDOM_API_BASE}${cleanUrl}`);
const arcJsonData = await arcResponse.json();
let frArcName: string | null = arcJsonData.parse?.langlinks.find((link: { lang: string; ['*']: string }) => link.lang === 'fr')?.['*'] || null;
// Remove "Arc" suffix from French name if present to keep it consistent with English names (e.g. "Arc de Luffy" becomes "Luffy")
if (frArcName && /\bArc\b/i.test(frArcName)) {
frArcName = frArcName.replace(/\bArc\b/i, '').trim();
}
arcs.push({ arcs.push({
id: arcId, id: arcId,
name: arcName, name: arcName,
frName: frArcName,
startChapter, startChapter,
endChapter, endChapter,
url: href.replace('/fr/wiki/', '') url: cleanUrl
}); });
seenArcUrls.add(cleanUrl);
} }
}
});
console.log(`Found ${arcs.length} arcs.`); console.log(`Found ${arcs.length} arcs.`);
return arcs; return arcs;
@@ -234,6 +246,7 @@ async function saveArcsToCSV(arcs: Arc[]): Promise<void> {
header: [ header: [
{ id: 'id', title: 'ID' }, { id: 'id', title: 'ID' },
{ id: 'name', title: 'Name' }, { id: 'name', title: 'Name' },
{ id: 'frName', title: 'French Name' },
{ id: 'startChapter', title: 'Start Chapter' }, { id: 'startChapter', title: 'Start Chapter' },
{ id: 'endChapter', title: 'End Chapter' }, { id: 'endChapter', title: 'End Chapter' },
{ id: 'url', title: 'URL' } { id: 'url', title: 'URL' }
@@ -245,6 +258,7 @@ async function saveArcsToCSV(arcs: Arc[]): Promise<void> {
.map((arc) => ({ .map((arc) => ({
id: arc.id || '', id: arc.id || '',
name: arc.name || '', name: arc.name || '',
frName: arc.frName || '',
startChapter: arc.startChapter || '', startChapter: arc.startChapter || '',
endChapter: arc.endChapter || '', endChapter: arc.endChapter || '',
url: arc.url || '' url: arc.url || ''
@@ -259,10 +273,10 @@ async function saveArcsToCSV(arcs: Arc[]): Promise<void> {
*/ */
async function fetchAllCharactersUrl(): Promise<CharacterListItem[]> { async function fetchAllCharactersUrl(): Promise<CharacterListItem[]> {
try { try {
const apiUrl = `${FANDOM_API_BASE}Liste_des_Personnages_Canon`; const apiUrl = `${FANDOM_API_BASE}List_of_Canon_Characters`;
console.log('Fetching character list via API...'); console.log('Fetching character list via API...');
const response = await fetchWithRetry(apiUrl); const response = await fetchWithRetry(apiUrl);
const jsonData = await response.json() as any; const jsonData = await response.json();
// Extract HTML from API response // Extract HTML from API response
const htmlContent = jsonData.parse?.text?.['*']; const htmlContent = jsonData.parse?.text?.['*'];
@@ -272,11 +286,10 @@ async function fetchAllCharactersUrl(): Promise<CharacterListItem[]> {
const $ = cheerio.load(htmlContent); const $ = cheerio.load(htmlContent);
const characters: CharacterListItem[] = []; const characters: CharacterListItem[] = [];
$('table.wikitable tbody tr').each((index, element) => { $('table.fandom-table tbody tr').each((index, element) => {
if (index === 0) return; // Skip header row if (index === 0) return; // Skip header row
let charpictureUrl = $(element).find('td:nth-child(1) a img').attr('data-src') || $(element).find('td:nth-child(1) a img').attr('src');
let charUrl = $(element).find('td:nth-child(2) a').attr('href'); let charUrl = $(element).find('td:nth-child(2) a').attr('href');
let charName = $(element).find('td:nth-child(2) a').text().trim(); const charName = $(element).find('td:nth-child(2) a').text().trim();
let charChapter = $(element).find('td:nth-child(3)').text().trim(); let charChapter = $(element).find('td:nth-child(3)').text().trim();
// Remove parentheses and their content from chapter info (e.g. "1 (flashback)" becomes "1") // Remove parentheses and their content from chapter info (e.g. "1 (flashback)" becomes "1")
@@ -288,13 +301,16 @@ async function fetchAllCharactersUrl(): Promise<CharacterListItem[]> {
return; return;
} }
if (parseInt(charChapter, 10) === 0) {
return;
}
if (charUrl) { if (charUrl) {
charUrl = charUrl.replace('/fr/wiki/', ''); charUrl = charUrl.replace('/wiki/', '');
characters.push({ characters.push({
name: charName, name: charName,
url: charUrl, url: charUrl,
pictureUrl: charpictureUrl || null, chapter: parseInt(charChapter, 10)
chapter: charChapter,
}); });
} }
}); });
@@ -312,26 +328,16 @@ async function fetchAllCharactersUrl(): Promise<CharacterListItem[]> {
async function fetchCharacter( async function fetchCharacter(
characterUrl: string, characterUrl: string,
characterName: string, characterName: string,
characterpictureUrl: string | null, characterChapter: number,
characterChapter: string arcsList: Arc[],
): Promise<Character | null> { ): Promise<Character | null> {
try { try {
console.log(`Fetching: ${characterName}...`); console.log(`Fetching: ${characterName}...`);
// Use API to fetch character page // Use API to fetch character page
const apiUrl = `${FANDOM_API_BASE}${characterUrl}`; const apiUrl = `${FANDOM_API_BASE}${characterUrl}`;
let response = await fetchWithRetry(apiUrl); const response = await fetchWithRetry(apiUrl);
const jsonData = await response.json();
let jsonData = await response.json() as any;
// Use final page name from API (if parse.limks contains one element, it means the original page was a redirect, so we use the the element 0 as the final URL, otherwise we use the original URL)
let finalCharacterUrl = characterUrl;
if (jsonData.parse?.links?.length === 1) {
finalCharacterUrl = jsonData.parse.links[0]['*'];
// Query the API again with the final URL to get the correct HTML content (in case of redirect)
response = await fetchWithRetry(`${FANDOM_API_BASE}${finalCharacterUrl}`);
jsonData = await response.json() as any;
}
const categories = jsonData.parse?.categories || []; const categories = jsonData.parse?.categories || [];
@@ -346,16 +352,16 @@ async function fetchCharacter(
const name = characterName; const name = characterName;
// Generate character ID from URL + name combination // Generate character ID from URL + name combination
const finalCharacterId = normalizeId(finalCharacterUrl + '_' + name); const finalCharacterId = normalizeId(characterUrl + '_' + name);
// Extract gender from JSON categories // Extract gender from JSON categories
let gender: string | null = null; let gender: string | null = null;
for (const cat of categories) { for (const cat of categories) {
const catName = cat['*'] || ''; const catName = cat['*'] || '';
if (catName === 'Personnages_Masculins') { if (catName === 'Male_Characters') {
gender = 'Male'; gender = 'Male';
break; break;
} else if (catName === 'Personnages_Féminins') { } else if (catName === 'Female_Characters') {
gender = 'Female'; gender = 'Female';
break; break;
} }
@@ -381,11 +387,11 @@ async function fetchCharacter(
let hakiConqueror = false; let hakiConqueror = false;
for (const cat of categories) { for (const cat of categories) {
const catName = cat['*'] || ''; const catName = cat['*'] || '';
if (catName === 'Utilisateurs_du_Haki_de_l\'observation') { if (catName === 'Observation_Haki_Users') {
hakiObservation = true; hakiObservation = true;
} else if (catName === 'Utilisateurs_du_Haki_de_l\'armement') { } else if (catName === 'Armament_Haki_Users') {
hakiArmament = true; hakiArmament = true;
} else if (catName === 'Utilisateurs_du_Haki_des_rois') { } else if (catName === 'Supreme_King_Haki_Users') {
hakiConqueror = true; hakiConqueror = true;
} }
} }
@@ -397,7 +403,7 @@ async function fetchCharacter(
const height = extractHeight($); const height = extractHeight($);
// Use chapter from character list, cast to int // Use chapter from character list, cast to int
let firstAppearance = parseInt(characterChapter); const firstAppearance = characterChapter;
// Extract origin // Extract origin
const origin = extractOrigin($); const origin = extractOrigin($);
@@ -405,31 +411,51 @@ async function fetchCharacter(
// Extract status // Extract status
const status = extractStatus($); const status = extractStatus($);
// Extract image URL and clean it let arcId = '';
let pictureUrl = characterpictureUrl; const arc = arcsList.find(a => a.startChapter <= firstAppearance && (a.endChapter === null || a.endChapter >= firstAppearance));
if (pictureUrl && pictureUrl.includes('Image_Non_Disponible')) { if (!arc) {
pictureUrl = null; return null;
} }
arcId = arc.id;
const frLink = getFrLink(jsonData.parse?.langlinks || []);
const frUrl = frLink ? frLink.url.replace('https://onepiece.fandom.com/fr/wiki/', '') : null;
const frjsonData = frUrl ? await fetchWithRetry(`${FR_FANDOM_API_BASE}${frUrl}`).then(res => res.json()) : null;
const frName = frjsonData?.parse?.title || null;
const frAffiliations = frjsonData ? extractAffiliations(cheerio.load(frjsonData.parse?.text?.['*'] || '')) : null;
const frEpithets = frjsonData ? extractEpithets(cheerio.load(frjsonData.parse?.text?.['*'] || '')) : null;
const frOrigin = frjsonData ? extractOrigin(cheerio.load(frjsonData.parse?.text?.['*'] || '')) : null;
return { return {
id: finalCharacterId, id: finalCharacterId,
name, name,
frName,
gender, gender,
age, age,
height, height,
origin, origin,
frOrigin,
devilFruitId, devilFruitId,
devilFruitUrl, devilFruitUrl,
affiliations, affiliations,
frAffiliations,
bounty, bounty,
hakiObservation, hakiObservation,
hakiArmament, hakiArmament,
hakiConqueror, hakiConqueror,
epithets, epithets,
frEpithets,
firstAppearance, firstAppearance,
arcId,
status, status,
pictureUrl, pictureUrl: "Image_Non_Disponible",
url: finalCharacterUrl url: characterUrl,
frUrl
}; };
} catch (error) { } catch (error) {
console.error(`Error fetching ${characterName}:`, (error as Error).message); console.error(`Error fetching ${characterName}:`, (error as Error).message);
@@ -442,7 +468,7 @@ async function fetchCharacter(
* Extract age from infobox * Extract age from infobox
*/ */
function extractAge($: cheerio.CheerioAPI): number | null { function extractAge($: cheerio.CheerioAPI): number | null {
const div = $('[data-source="âge"] .pi-data-value'); const div = $('[data-source="age"] .pi-data-value');
if (div.length === 0) return null; if (div.length === 0) return null;
let text = div.html(); let text = div.html();
@@ -473,11 +499,11 @@ function extractAffiliations($: cheerio.CheerioAPI): string[] {
const cleanedDiv = div.clone(); const cleanedDiv = div.clone();
cleanedDiv.find('sup').remove(); cleanedDiv.find('sup').remove();
let text = cleanedDiv.html(); const text = cleanedDiv.html();
if (!text) return []; if (!text) return [];
// Extract all link values // Extract all link values
const linkValues = cleanedDiv.find('a').map((i, el) => $(el).text().trim()).get(); const linkValues = cleanedDiv.find('a').map((_i, el) => $(el).text().trim()).get();
if (linkValues.length > 0) { if (linkValues.length > 0) {
return linkValues; return linkValues;
} }
@@ -490,28 +516,46 @@ function extractAffiliations($: cheerio.CheerioAPI): string[] {
/** /**
* Extract epithets from infobox * Extract epithets from infobox
* Epithets are always between double quotes * Handles both quoted and unquoted epithets, keeping only the main/latest readable values.
*/ */
function extractEpithets($: cheerio.CheerioAPI): string[] { function extractEpithets($: cheerio.CheerioAPI): string[] {
const div = $('[data-source="épithète"] .pi-data-value'); const div = $('[data-source="epithet"] .pi-data-value');
if (div.length === 0) return []; if (div.length === 0) return [];
const cleanedDiv = div.clone(); const cleanedDiv = div.clone();
cleanedDiv.find('sup').remove(); cleanedDiv.find('sup').remove();
let text = cleanedDiv.text(); const html = cleanedDiv.html();
if (!text) return []; if (!html) return [];
// Extract all text between double quotes (both straight and curly quotes) const plainText = html
const matches = text.match(/["«"]([^"»"]+)["»"]/g); .replace(/<br\s*\/?\s*>/gi, '\n')
if (!matches) return []; .replace(/<[^>]*>/g, '');
// Remove the quotes and trim const lines = plainText
const epithets = matches.map(match => .split('\n')
match.replace(/^["«"]|["»"]$/g, '').trim() .map((line) => line.trim())
).filter(Boolean); .filter(Boolean);
return epithets; const epithets = lines
.map((line) => {
const normalized = line.replace(/\s+/g, ' ').trim();
// Prefer explicit quoted epithet if present.
const quotedMatch = normalized.match(/["«“](.*?)["»”]/);
if (quotedMatch?.[1]) {
return quotedMatch[1].trim();
}
// Otherwise keep only the base epithet text before extra notes/translations.
return normalized
.split(/[;(]/)[0]
.replace(/["'«»“”]/g, '')
.trim();
})
.filter(Boolean);
return Array.from(new Set(epithets));
} }
/** /**
@@ -519,49 +563,27 @@ function extractEpithets($: cheerio.CheerioAPI): string[] {
* Returns both normalized ID and URL * Returns both normalized ID and URL
*/ */
async function extractDevilFruit($: cheerio.CheerioAPI): Promise<DevilFruitData | null> { async function extractDevilFruit($: cheerio.CheerioAPI): Promise<DevilFruitData | null> {
const link = $('[data-source="dfnom"] .pi-data-value a').first(); const link = $('[data-source="dfname"] .pi-data-value a').first();
if (link.length === 0) return null; if (link.length === 0) return null;
const href = link.attr('href'); const href = link.attr('href');
if (!href || !href.startsWith('/fr/wiki/')) return null; if (!href || !href.startsWith('/wiki/')) return null;
const cleanUrl = href.replace('/fr/wiki/', ''); const cleanUrl = href.replace('/wiki/', '');
try {
// Fetch the page via API to follow redirects
const apiUrl = `${FANDOM_API_BASE}${decodeURIComponent(cleanUrl)}`;
const response = await fetchWithRetry(apiUrl);
const jsonData = await response.json() as any;
// Use final page name from API (if parse.links contains one element, it means the original page was a redirect, so we use the the element 0 as the final URL, otherwise we use the original URL)
let finalPath = cleanUrl;
if (jsonData.parse?.links?.length === 1) {
finalPath = jsonData.parse.links[0]['*'];
}
if (finalPath) {
return {
devilFruitId: normalizeId(finalPath),
devilFruitUrl: finalPath
};
}
} catch (error) {
console.error(`Error fetching devil fruit page: ${(error as Error).message}`);
}
// Fallback to the original href
return { return {
devilFruitId: normalizeId(cleanUrl), devilFruitId: normalizeId(cleanUrl),
devilFruitUrl: cleanUrl devilFruitUrl: cleanUrl
}; };
} }
/** /**
* Extract bounty from infobox * Extract bounty from infobox
*/ */
function extractBounty($: cheerio.CheerioAPI): number | null { function extractBounty($: cheerio.CheerioAPI): number | null {
const div = $('[data-source="prime"] .pi-data-value'); const div = $('[data-source="bounty"] .pi-data-value');
if (div.length === 0) return 0; if (div.length === 0) return 0;
let text = div.html(); let text = div.html();
@@ -593,7 +615,7 @@ function extractBounty($: cheerio.CheerioAPI): number | null {
* Extract height from infobox * Extract height from infobox
*/ */
function extractHeight($: cheerio.CheerioAPI): number | null { function extractHeight($: cheerio.CheerioAPI): number | null {
const div = $('[data-source="taille"] .pi-data-value'); const div = $('[data-source="height"] .pi-data-value');
if (div.length === 0) return null; if (div.length === 0) return null;
let text = div.html(); let text = div.html();
@@ -602,43 +624,44 @@ function extractHeight($: cheerio.CheerioAPI): number | null {
// Remove all sup blocks (citations) // Remove all sup blocks (citations)
text = text.replace(/<sup[^>]*>.*?<\/sup>/gi, ''); text = text.replace(/<sup[^>]*>.*?<\/sup>/gi, '');
// Check if there's a <p> tag - if yes, use content from <p> // Convert line breaks to new lines so we can reliably pick the latest value.
let content; const textWithNewLines = text.replace(/<br\s*\/?\s*>/gi, '\n');
const pMatch = text.match(/<p[^>]*>(.*?)<\/p>/i); const lines = textWithNewLines
if (pMatch) { .replace(/<[^>]*>/g, '')
// Extract content from the <p> tag .split('\n')
content = pMatch[1]; .map((line) => line.trim())
} else { .filter(Boolean);
// Use the last value method (after any <br> tag)
content = text.split('<br>').pop();
}
let cleanText = (content || '').replace(/<[^>]*>/g, '').trim(); // Keep only lines that look like a height value, then pick the latest one.
const heightLines = lines.filter((line) => /\d/.test(line) && /(cm|m)/i.test(line));
const latestLine = heightLines.length > 0 ? heightLines[heightLines.length - 1] : lines[lines.length - 1];
if (!latestLine) return null;
// Remove content with parentheses // Remove descriptive suffixes like "(post-timeskip)".
cleanText = cleanText.replace(/\([^)]*\)/g, ''); const cleanText = latestLine.replace(/\([^)]*\)/g, '').trim();
// Normalize units for meters or centimeters
const normalized = cleanText.toLowerCase().replace(/\s/g, ''); const normalized = cleanText.toLowerCase().replace(/\s/g, '');
if (normalized.includes('cm')) {
const digitsOnly = normalized.replace(/\D/g, ''); // Values are stored in meters in this dataset.
const cm = parseFloat(digitsOnly); const cmMatch = normalized.match(/(\d+(?:[.,]\d+)?)cm/);
return cm ? cm / 100 : null; if (cmMatch) {
const cm = parseFloat(cmMatch[1].replace(',', '.'));
return Number.isFinite(cm) ? cm / 100 : null;
} }
if (normalized.includes('m')) { const mMatch = normalized.match(/(\d+(?:[.,]\d+)?)m/);
const parts = normalized.split('m').filter(Boolean); if (mMatch) {
return parts.length > 0 ? parseFloat(parts.join('.')) : null; const meters = parseFloat(mMatch[1].replace(',', '.'));
return Number.isFinite(meters) ? meters : null;
} }
return normalized.length > 0 ? parseFloat(normalized.replace(/\D/g, '')) : null; return null;
} }
/** /**
* Extract origin from infobox * Extract origin from infobox
*/ */
function extractOrigin($: cheerio.CheerioAPI): string | null { function extractOrigin($: cheerio.CheerioAPI): string | null {
const div = $('[data-source="origine"] .pi-data-value'); const div = $('[data-source="origin"] .pi-data-value, [data-source="origine"] .pi-data-value').first();
if (div.length === 0) return null; if (div.length === 0) return null;
let text = div.html(); let text = div.html();
@@ -661,16 +684,16 @@ function extractOrigin($: cheerio.CheerioAPI): string | null {
* Extract status from infobox * Extract status from infobox
*/ */
function extractStatus($: cheerio.CheerioAPI): string | null { function extractStatus($: cheerio.CheerioAPI): string | null {
const div = $('[data-source="statut"] .pi-data-value'); const div = $('[data-source="status"] .pi-data-value');
if (div.length === 0) return null; if (div.length === 0) return null;
const statusText = div.text().trim().toLowerCase(); const statusText = div.text().trim().toLowerCase();
if (statusText.includes('vivant')) { if (statusText.includes('Alive')) {
return 'Alive'; return 'Alive';
} else if (statusText.includes('décédé')) { } else if (statusText.includes('Dead')) {
return 'Dead'; return 'Dead';
} else if (statusText.includes('inconnu')) { } else if (statusText.includes('Unknown')) {
return 'Unknown'; return 'Unknown';
} }
@@ -753,7 +776,7 @@ async function fetchDevilFruit(devilFruitUrl: string, devilFruitId: string): Pro
// Use API to fetch devil fruit page // Use API to fetch devil fruit page
const apiUrl = `${FANDOM_API_BASE}${devilFruitUrl}`; const apiUrl = `${FANDOM_API_BASE}${devilFruitUrl}`;
const response = await fetchWithRetry(apiUrl); const response = await fetchWithRetry(apiUrl);
const jsonData = await response.json() as any; const jsonData = await response.json();
// Extract HTML from API response // Extract HTML from API response
const htmlContent = jsonData.parse?.text?.['*']; const htmlContent = jsonData.parse?.text?.['*'];
@@ -767,7 +790,7 @@ async function fetchDevilFruit(devilFruitUrl: string, devilFruitId: string): Pro
// Determine type based on categories (if categories contain "Paramecia", "Zoan", "Logia" or "Smile") // Determine type based on categories (if categories contain "Paramecia", "Zoan", "Logia" or "Smile")
if (jsonData.parse?.categories) { if (jsonData.parse?.categories) {
const categories = jsonData.parse.categories const categories = jsonData.parse.categories
.map((cat: any) => String(cat['*'] || '').toLowerCase()); .map((cat: { ['*']: string }) => String(cat['*'] || '').toLowerCase());
if (categories.some((category: string) => category.includes('paramecia'))) { if (categories.some((category: string) => category.includes('paramecia'))) {
type = 'Paramecia'; type = 'Paramecia';
@@ -847,6 +870,7 @@ async function main(): Promise<void> {
console.table({ console.table({
ID: arc.id, ID: arc.id,
Name: arc.name, Name: arc.name,
FrenchName: arc.frName || '',
StartChapter: arc.startChapter, StartChapter: arc.startChapter,
EndChapter: arc.endChapter || 'Ongoing', EndChapter: arc.endChapter || 'Ongoing',
URL: arc.url URL: arc.url
@@ -886,7 +910,7 @@ async function main(): Promise<void> {
const batch = failedCharacters.slice(i, i + FETCH_CONCURRENCY); const batch = failedCharacters.slice(i, i + FETCH_CONCURRENCY);
const batchResults = await Promise.all( const batchResults = await Promise.all(
batch.map(async (char) => { batch.map(async (char) => {
const data = await fetchCharacter(char.url, char.name, char.pictureUrl, char.chapter); const data = await fetchCharacter(char.url, char.name, char.chapter, arcsList);
return { char, data }; return { char, data };
}) })
); );
@@ -918,12 +942,6 @@ async function main(): Promise<void> {
devilFruitUrls.add(data.devilFruitUrl); devilFruitUrls.add(data.devilFruitUrl);
} }
if (data.firstAppearance) {
const arc = arcsList.find(a => a.startChapter <= data.firstAppearance && (a.endChapter === null || a.endChapter >= data.firstAppearance));
if (arc) {
data.arcId = arc.id;
}
}
characters.push(data); characters.push(data);
} else { } else {

97
src/lib/server/db/schema.ts
View File

@@ -17,13 +17,14 @@ export const config = sqliteTable('config', {
export const arc = sqliteTable('arc', { export const arc = sqliteTable('arc', {
id: text('id').primaryKey(), id: text('id').primaryKey(),
name: text('name').notNull(), name: text('name').notNull(),
startChapter: integer('startChapter').notNull(), frName: text('fr_name'),
endChapter: integer('endChapter'), startChapter: integer('start_chapter').notNull(),
endChapter: integer('end_chapter'),
url: text('url') url: text('url')
}); });
// Define the devil fruit table schema // Define the devil fruit table schema
export const devilFruit = sqliteTable('devilFruit', { export const devilFruit = sqliteTable('devil_fruit', {
id: text('id').primaryKey(), id: text('id').primaryKey(),
name: text('name').notNull().unique(), name: text('name').notNull().unique(),
type: text('type').$type<DevilFruitType>(), type: text('type').$type<DevilFruitType>(),
@@ -34,91 +35,101 @@ export const devilFruit = sqliteTable('devilFruit', {
export const character = sqliteTable('character', { export const character = sqliteTable('character', {
id: text('id').primaryKey(), id: text('id').primaryKey(),
name: text('name').notNull(), name: text('name').notNull(),
frName: text('fr_name'),
gender: text('gender'), gender: text('gender'),
age: integer('age'), age: integer('age'),
affiliations: text('affiliations', { mode: 'json' }).$type<string[]>(), affiliations: text('affiliations', { mode: 'json' }).$type<string[]>(),
devilFruitId: text('devilFruitId').references(() => devilFruit.id), devilFruitId: text('devil_fruit_id').references(() => devilFruit.id),
hakiObservation: integer('hakiObservation', { mode: 'boolean' }).default(false), hakiObservation: integer('haki_observation', { mode: 'boolean' }).default(false),
hakiArmament: integer('hakiArmament', { mode: 'boolean' }).default(false), hakiArmament: integer('haki_armament', { mode: 'boolean' }).default(false),
hakiConqueror: integer('hakiConqueror', { mode: 'boolean' }).default(false), hakiConqueror: integer('haki_conqueror', { mode: 'boolean' }).default(false),
bounty: integer('bounty').default(0), bounty: integer('bounty').default(0),
height: real('height'), height: real('height'),
origin: text('origin'), origin: text('origin'),
firstAppearance: integer('firstAppearance').notNull(), frOrigin: text('fr_origin'),
pictureUrl: text('pictureUrl'), firstAppearance: integer('first_appearance').notNull(),
pictureUrl: text('picture_url'),
epithets: text('epithets', { mode: 'json' }).$type<string[]>(), epithets: text('epithets', { mode: 'json' }).$type<string[]>(),
frEpithets: text('fr_epithets', { mode: 'json' }).$type<string[]>(),
status: text('status').$type<Status | null>(), status: text('status').$type<Status | null>(),
arcId: text('arcId').references(() => arc.id), arcId: text('arc_id').references(() => arc.id, { onDelete: 'set null' }),
url: text('url'), url: text('url'),
isInDailyMode: integer('isInDailyMode', { mode: 'boolean' }).default(false) frUrl: text('fr_url'),
isInDailyMode: integer('is_in_daily_mode', { mode: 'boolean' }).default(false)
}); });
// Define the character override table schema // Define the character override table schema
export const characterOverride = sqliteTable('characterOverride', { export const characterOverride = sqliteTable('character_override', {
characterId: text('characterId').primaryKey().references(() => character.id), characterId: text('character_id').primaryKey().references(() => character.id, { onDelete: 'cascade' }),
name: text('name'), name: text('name'),
gender: text('gender'), gender: text('gender'),
age: integer('age'), age: integer('age'),
affiliations: text('affiliations', { mode: 'json' }).$type<string[]>(), affiliations: text('affiliations', { mode: 'json' }).$type<string[]>(),
devilFruitId: text('devilFruitId').references(() => devilFruit.id), devilFruitId: text('devil_fruit_id').references(() => devilFruit.id, { onDelete: 'set null' }),
hakiObservation: integer('hakiObservation', { mode: 'boolean' }), hakiObservation: integer('haki_observation', { mode: 'boolean' }),
hakiArmament: integer('hakiArmament', { mode: 'boolean' }), hakiArmament: integer('haki_armament', { mode: 'boolean' }),
hakiConqueror: integer('hakiConqueror', { mode: 'boolean' }), hakiConqueror: integer('haki_conqueror', { mode: 'boolean' }),
bounty: integer('bounty'), bounty: integer('bounty'),
height: real('height'), height: real('height'),
origin: text('origin'), origin: text('origin'),
firstAppearance: integer('firstAppearance'), firstAppearance: integer('first_appearance'),
pictureUrl: text('pictureUrl'), pictureUrl: text('picture_url'),
epithets: text('epithets', { mode: 'json' }).$type<string[]>(), epithets: text('epithets', { mode: 'json' }).$type<string[]>(),
status: text('status').$type<Status | null>(), status: text('status').$type<Status | null>(),
arcId: text('arcId').references(() => arc.id), arcId: text('arc_id').references(() => arc.id, { onDelete: 'set null' }),
url: text('url'), url: text('url'),
frUrl: text('fr_url'),
notes: text('notes') notes: text('notes')
}); });
// Define the character scrape validation table schema // Define the character scrape validation table schema
export const characterScrapeValidation = sqliteTable('characterScrapeValidation', { export const characterScrapeValidation = sqliteTable('character_scrape_validation', {
id: text('id').primaryKey(), id: text('id').primaryKey(),
name: text('name').notNull(), name: text('name').notNull(),
frName: text('fr_name'),
gender: text('gender'), gender: text('gender'),
age: integer('age'), age: integer('age'),
affiliations: text('affiliations', { mode: 'json' }).$type<string[]>(), affiliations: text('affiliations', { mode: 'json' }).$type<string[]>(),
devilFruitId: text('devilFruitId').references(() => devilFruit.id), devilFruitId: text('devil_fruit_id').references(() => devilFruit.id, { onDelete: 'set null' }),
hakiObservation: integer('hakiObservation', { mode: 'boolean' }).default(false), hakiObservation: integer('haki_observation', { mode: 'boolean' }).default(false),
hakiArmament: integer('hakiArmament', { mode: 'boolean' }).default(false), hakiArmament: integer('haki_armament', { mode: 'boolean' }).default(false),
hakiConqueror: integer('hakiConqueror', { mode: 'boolean' }).default(false), hakiConqueror: integer('haki_conqueror', { mode: 'boolean' }).default(false),
bounty: integer('bounty'), bounty: integer('bounty'),
height: real('height'), height: real('height'),
origin: text('origin'), origin: text('origin'),
firstAppearance: integer('firstAppearance').notNull(), frOrigin: text('fr_origin'),
pictureUrl: text('pictureUrl'), firstAppearance: integer('first_appearance').notNull(),
pictureUrl: text('picture_url'),
epithets: text('epithets', { mode: 'json' }).$type<string[]>(), epithets: text('epithets', { mode: 'json' }).$type<string[]>(),
frEpithets: text('fr_epithets', { mode: 'json' }).$type<string[]>(),
status: text('status').$type<Status | null>(), status: text('status').$type<Status | null>(),
arcId: text('arcId').references(() => arc.id), arcId: text('arc_id').references(() => arc.id, { onDelete: 'set null' }),
url: text('url') url: text('url'),
frUrl: text('fr_url')
}); });
// Define the caracter history table schema // Define the character history table schema
export const characterHistory = sqliteTable('characterHistory', { export const characterHistory = sqliteTable('character_history', {
id: text('id') id: text('id')
.primaryKey() .primaryKey()
.$defaultFn(() => crypto.randomUUID()), .$defaultFn(() => crypto.randomUUID()),
characterId: text('characterId').references(() => character.id), characterId: text('character_id').references(() => character.id, { onDelete: 'cascade' }),
date: integer('date').notNull().unique(), date: integer('date').notNull().unique(),
won: integer('won').notNull().default(0), won: integer('won').notNull().default(0),
createdAt: integer('createdAt').notNull().$default(() => Date.now()), createdAt: integer('created_at').notNull().$default(() => Date.now()),
updatedAt: integer('updatedAt').notNull().$default(() => Date.now()), updatedAt: integer('updated_at').notNull().$default(() => Date.now()),
}); });
// Define the user character history table schema // Define the user character history table schema
export const userCharacterHistory = sqliteTable('userCharacterHistory', { export const userCharacterHistory = sqliteTable('user_character_history', {
id: text('id') id: text('id')
.primaryKey() .primaryKey()
.$defaultFn(() => crypto.randomUUID()), .$defaultFn(() => crypto.randomUUID()),
userId: text('userId').references(() => user.id), userId: text('user_id').references(() => user.id, { onDelete: 'cascade' }),
characterHistoryId: text('characterHistoryId').references(() => characterHistory.id), characterHistoryId: text('character_history_id').references(() => characterHistory.id, { onDelete: 'cascade' }),
tryCount: integer('tryCount').notNull(), tryCount: integer('try_count').notNull(),
createdAt: integer('createdAt').notNull().$default(() => Date.now()) triedCharacterIds: text('tried_character_ids', { mode: 'json' }).$type<string[]>(),
createdAt: integer('created_at').notNull().$default(() => Date.now())
}, (table) => [ }, (table) => [
unique().on(table.userId, table.characterHistoryId) unique().on(table.userId, table.characterHistoryId)
]); ]);
@@ -128,15 +139,15 @@ export const friendship = sqliteTable('friendship', {
id: text('id') id: text('id')
.primaryKey() .primaryKey()
.$defaultFn(() => crypto.randomUUID()), .$defaultFn(() => crypto.randomUUID()),
requesterId: text('requesterId') requesterId: text('requester_id')
.notNull() .notNull()
.references(() => user.id, { onDelete: 'cascade' }), .references(() => user.id, { onDelete: 'cascade' }),
addresseeId: text('addresseeId') addresseeId: text('addressee_id')
.notNull() .notNull()
.references(() => user.id, { onDelete: 'cascade' }), .references(() => user.id, { onDelete: 'cascade' }),
status: text('status').$type<FriendshipStatus>().notNull().default('pending'), status: text('status').$type<FriendshipStatus>().notNull().default('pending'),
createdAt: integer('createdAt').notNull().$default(() => Date.now()), createdAt: integer('created_at').notNull().$default(() => Date.now()),
updatedAt: integer('updatedAt').notNull().$default(() => Date.now()), updatedAt: integer('updated_at').notNull().$default(() => Date.now()),
}, (table) => [ }, (table) => [
unique().on(table.requesterId, table.addresseeId) unique().on(table.requesterId, table.addresseeId)
]); ]);