From 6f7bae2307a8f0c4622f41ebb1752af4adc8d389 Mon Sep 17 00:00:00 2001 From: whidix Date: Fri, 27 Feb 2026 01:14:44 +0100 Subject: [PATCH] feat(scraper): implement One Piece data scraper for devil fruits and characters - Added a new script to scrape devil fruits and characters from One Piece fandom. - Implemented functions to fetch, normalize, and save data in JSON, CSV, and SQL formats. - Created a structured output directory for scraped data. feat(database): update schema for devil fruits and characters - Defined new types for devil fruits and haki in the database schema. - Updated the character table to include fields for age, affiliations, devil fruit, haki, bounty, height, origin, first appearance, and picture URL. feat(ui): enhance main page and daily mode layout - Redesigned the main page with a new layout and styling for the OnePieceDle game. - Created a new daily mode page with sections for clues and user input for guesses. - Removed demo authentication routes and pages to streamline the application. --- .gitignore | 3 + SCRAPER.md | 130 ++++ drizzle/0000_dapper_sage.sql | 85 +++ drizzle/meta/0000_snapshot.json | 576 +++++++++++++++ drizzle/meta/_journal.json | 13 + package-lock.json | 648 ++++++++++++++++- package.json | 6 +- scripts/import-sql.js | 104 +++ scripts/scrape-onepiece.js | 672 ++++++++++++++++++ src/lib/server/db/schema.ts | 41 +- src/routes/+page.svelte | 65 +- src/routes/daily/+page.svelte | 87 +++ src/routes/demo/+page.svelte | 5 - src/routes/demo/better-auth/+page.server.ts | 20 - src/routes/demo/better-auth/+page.svelte | 14 - .../demo/better-auth/login/+page.server.ts | 61 -- .../demo/better-auth/login/+page.svelte | 42 -- 17 files changed, 2407 insertions(+), 165 deletions(-) create mode 100644 SCRAPER.md create mode 100644 drizzle/0000_dapper_sage.sql create mode 100644 drizzle/meta/0000_snapshot.json create mode 100644 drizzle/meta/_journal.json create mode 100644 scripts/import-sql.js create mode 100644 scripts/scrape-onepiece.js create mode 100644 src/routes/daily/+page.svelte delete mode 100644 src/routes/demo/+page.svelte delete mode 100644 src/routes/demo/better-auth/+page.server.ts delete mode 100644 src/routes/demo/better-auth/+page.svelte delete mode 100644 src/routes/demo/better-auth/login/+page.server.ts delete mode 100644 src/routes/demo/better-auth/login/+page.svelte diff --git a/.gitignore b/.gitignore index 23be534..9b12a10 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,6 @@ vite.config.js.timestamp-* vite.config.ts.timestamp-* # SQLite *.db + +# Script outputs +/scraped-data \ No newline at end of file diff --git a/SCRAPER.md b/SCRAPER.md new file mode 100644 index 0000000..299ca10 --- /dev/null +++ b/SCRAPER.md @@ -0,0 +1,130 @@ +# One Piece Scraper + +Script pour scraper les données des personnages de One Piece depuis le fandom wiki français. + +## Installation + +Installe les dépendances d'abord : + +```bash +npm install +``` + +## Utilisation + +```bash +# Scraper tous les formats (JSON, CSV, SQL) +npm run scrape + +# Ou spécifier un format +node scripts/scrape-onepiece.js json # JSON uniquement +node scripts/scrape-onepiece.js csv # CSV uniquement +node scripts/scrape-onepiece.js sql # SQL uniquement +node scripts/scrape-onepiece.js all # Tous les formats (défaut) +``` + +## Sortie + +Les données seront sauvegardées dans le dossier `scraped-data/` : + +- **characters.json** - Format JSON avec toutes les données structurées +- **characters.csv** - Format CSV pour importer dans Excel/Sheets +- **characters.sql** - Statements SQL avec gestion des conflits (upsert) + +## Données extraites + +Pour chaque personnage : +- 📝 **Nom** - Nom du personnage +- 👤 **Genre** - Masculin/Féminin (extrait des catégories) +- 🎂 **Âge** - Âge le plus récent (post-ellipse), chiffres uniquement +- 📏 **Taille** - Normalisée en mètres (format: "2.74" ou "174" pour cm) +- 🌍 **Origine** - Lieu d'origine (sans parenthèses) +- 😈 **Fruit du Démon** - Nom du fruit (si applicable) +- 👥 **Affiliations** - Liste des affiliations (équipages, organisations) +- 💰 **Prime** - Bounty la plus récente +- ⚡ **Haki** - Liste des types de Haki (Observation, Armament, Conqueror) +- 📖 **Première Apparition** - Numéro de chapitre +- 🖼️ **Image** - URL de l'image portrait nettoyée +- 🔗 **Fandom URL** - Lien vers la page wiki + +## Personnages scrapés + +Le script scrape tous les personnages canon de la liste officielle du Fandom wiki français. + +**Personnages actuellement filtrés** : Luffy et Moria (modifiable dans `fetchAllCharactersUrl`) + +Pour scraper tous les personnages, retire le filtre dans la fonction `fetchAllCharactersUrl`. + +## Fonctionnalités avancées + +### Requêtes parallèles +- Le scraper traite 5 personnages en parallèle pour plus d'efficacité +- Concurrency configurable dans le code + +### Nettoyage des données +- Suppression automatique des citations/références (`` tags) +- Âge : extraction du dernier âge (après ellipse), sans parenthèses +- Taille : normalisation m/cm et suppression des parenthèses +- Origine : suppression du contenu entre parenthèses +- Image : sélection automatique du portrait +- Première apparition : extraction du numéro de chapitre uniquement + +### SQL Upsert +- Le SQL généré utilise `INSERT ... ON CONFLICT(name) DO UPDATE` +- Met à jour les personnages existants au lieu de créer des doublons +- Compatible SQLite (utilisé par le projet) + +### Formats de données +- **Haki** : Stocké comme array JSON dans SQL : `["Observation","Armament"]` +- **Affiliations** : Liste dans JSON, comma-separated dans CSV/SQL +- Tous les champs nullable supportent `NULL` dans SQL + +## Configuration + +### Modifier les personnages filtrés + +Dans `scripts/scrape-onepiece.js`, fonction `fetchAllCharactersUrl` : + +```javascript +// Filtrer pour des personnages spécifiques +if (nameLower.includes('luffy') || nameLower.includes('moria')) { + characters.push({ name: charName, url: charLink }); +} +``` + +### Ajuster la concurrence + +Dans la fonction `main` : + +```javascript +const concurrency = 5; // Nombre de requêtes simultanées +``` + +## Importer les données SQL + +Après avoir généré le fichier SQL, importe-le dans la base de données : + +```bash +npm run db:import +``` + +Ce script : +- Lit automatiquement `scraped-data/characters.sql` +- Exécute chaque statement individuellement +- Affiche une barre de progression +- Gère les erreurs sans bloquer l'import complet +- Utilise le upsert pour éviter les doublons + +**Note** : Assure-toi d'avoir exécuté les migrations avant l'import : +```bash +npm run db:migrate +``` + +## Notes techniques + +- Source : `https://onepiece.fandom.com/fr/wiki` +- Parseur : Cheerio (DOM parsing) +- Traitement parallèle avec `Promise.all` +- User-Agent configuré pour éviter les blocages +- Pas de délai entre requêtes (batches parallèles) +- Gestion d'erreurs par personnage (ne bloque pas le scraping complet) diff --git a/drizzle/0000_dapper_sage.sql b/drizzle/0000_dapper_sage.sql new file mode 100644 index 0000000..92b1a68 --- /dev/null +++ b/drizzle/0000_dapper_sage.sql @@ -0,0 +1,85 @@ +CREATE TABLE `character` ( + `id` text PRIMARY KEY NOT NULL, + `name` text NOT NULL, + `gender` text, + `age` integer, + `affiliations` text, + `devilFruit` text, + `haki` text, + `bounty` integer, + `height` real, + `origin` text, + `firstAppearance` text, + `pictureUrl` text, + FOREIGN KEY (`devilFruit`) REFERENCES `devilFruit`(`id`) ON UPDATE no action ON DELETE no action +); +--> statement-breakpoint +CREATE TABLE `characterHistory` ( + `id` text PRIMARY KEY NOT NULL, + `characterId` text, + `date` integer, + `createdAt` integer NOT NULL, + `updatedAt` integer NOT NULL, + FOREIGN KEY (`characterId`) REFERENCES `character`(`id`) ON UPDATE no action ON DELETE no action +); +--> statement-breakpoint +CREATE TABLE `devilFruit` ( + `id` text PRIMARY KEY NOT NULL, + `name` text NOT NULL, + `type` text +); +--> statement-breakpoint +CREATE UNIQUE INDEX `devilFruit_name_unique` ON `devilFruit` (`name`);--> statement-breakpoint +CREATE TABLE `account` ( + `id` text PRIMARY KEY NOT NULL, + `account_id` text NOT NULL, + `provider_id` text NOT NULL, + `user_id` text NOT NULL, + `access_token` text, + `refresh_token` text, + `id_token` text, + `access_token_expires_at` integer, + `refresh_token_expires_at` integer, + `scope` text, + `password` text, + `created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL, + `updated_at` integer NOT NULL, + FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE INDEX `account_userId_idx` ON `account` (`user_id`);--> statement-breakpoint +CREATE TABLE `session` ( + `id` text PRIMARY KEY NOT NULL, + `expires_at` integer NOT NULL, + `token` text NOT NULL, + `created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL, + `updated_at` integer NOT NULL, + `ip_address` text, + `user_agent` text, + `user_id` text NOT NULL, + FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE UNIQUE INDEX `session_token_unique` ON `session` (`token`);--> statement-breakpoint +CREATE INDEX `session_userId_idx` ON `session` (`user_id`);--> statement-breakpoint +CREATE TABLE `user` ( + `id` text PRIMARY KEY NOT NULL, + `name` text NOT NULL, + `email` text NOT NULL, + `email_verified` integer DEFAULT false NOT NULL, + `image` text, + `created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL, + `updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX `user_email_unique` ON `user` (`email`);--> statement-breakpoint +CREATE TABLE `verification` ( + `id` text PRIMARY KEY NOT NULL, + `identifier` text NOT NULL, + `value` text NOT NULL, + `expires_at` integer NOT NULL, + `created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL, + `updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL +); +--> statement-breakpoint +CREATE INDEX `verification_identifier_idx` ON `verification` (`identifier`); \ No newline at end of file diff --git a/drizzle/meta/0000_snapshot.json b/drizzle/meta/0000_snapshot.json new file mode 100644 index 0000000..aabacff --- /dev/null +++ b/drizzle/meta/0000_snapshot.json @@ -0,0 +1,576 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "40edd98b-5a47-4a5e-a5b8-8a0f6eaaec76", + "prevId": "00000000-0000-0000-0000-000000000000", + "tables": { + "character": { + "name": "character", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "gender": { + "name": "gender", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "age": { + "name": "age", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "affiliations": { + "name": "affiliations", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "devilFruit": { + "name": "devilFruit", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "haki": { + "name": "haki", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bounty": { + "name": "bounty", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "firstAppearance": { + "name": "firstAppearance", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pictureUrl": { + "name": "pictureUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "character_devilFruit_devilFruit_id_fk": { + "name": "character_devilFruit_devilFruit_id_fk", + "tableFrom": "character", + "tableTo": "devilFruit", + "columnsFrom": [ + "devilFruit" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "characterHistory": { + "name": "characterHistory", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "characterId": { + "name": "characterId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "date": { + "name": "date", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updatedAt": { + "name": "updatedAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "characterHistory_characterId_character_id_fk": { + "name": "characterHistory_characterId_character_id_fk", + "tableFrom": "characterHistory", + "tableTo": "character", + "columnsFrom": [ + "characterId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "devilFruit": { + "name": "devilFruit", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "devilFruit_name_unique": { + "name": "devilFruit_name_unique", + "columns": [ + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json new file mode 100644 index 0000000..f37eb67 --- /dev/null +++ b/drizzle/meta/_journal.json @@ -0,0 +1,13 @@ +{ + "version": "7", + "dialect": "sqlite", + "entries": [ + { + "idx": 0, + "version": "6", + "when": 1772148571269, + "tag": "0000_dapper_sage", + "breakpoints": true + } + ] +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 8e0b139..c02433b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,10 @@ "@tailwindcss/typography": "^0.5.19", "@tailwindcss/vite": "^4.1.18", "@types/node": "^24", + "axios": "^1.6.0", "better-auth": "^1.4.18", + "cheerio": "^1.0.0-rc.12", + "csv-writer": "^1.6.0", "drizzle-kit": "^0.31.8", "drizzle-orm": "^0.45.1", "eslint": "^9.39.2", @@ -2695,6 +2698,25 @@ "node": ">= 0.4" } }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.13.5", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.5.tgz", + "integrity": "sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.11", + "form-data": "^4.0.5", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/axobject-query": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", @@ -2841,6 +2863,13 @@ "dev": true, "license": "MIT" }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "dev": true, + "license": "ISC" + }, "node_modules/brace-expansion": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", @@ -2859,6 +2888,20 @@ "dev": true, "license": "MIT" }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -2886,6 +2929,50 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/cheerio": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz", + "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "encoding-sniffer": "^0.2.1", + "htmlparser2": "^10.1.0", + "parse5": "^7.3.0", + "parse5-htmlparser2-tree-adapter": "^7.1.0", + "parse5-parser-stream": "^7.1.2", + "undici": "^7.19.0", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=20.18.1" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/chokidar": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", @@ -2932,6 +3019,19 @@ "dev": true, "license": "MIT" }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -2995,6 +3095,36 @@ "node": ">= 8" } }, + "node_modules/css-select": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", + "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", + "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/cssesc": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", @@ -3008,6 +3138,13 @@ "node": ">=4" } }, + "node_modules/csv-writer": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/csv-writer/-/csv-writer-1.6.0.tgz", + "integrity": "sha512-NOx7YDFWEsM/fTRAJjRpPp8t+MKRVvniAg9wQlUKx20MFrPs73WLJhFf5iteqrxNYnsy924K3Iroh3yNHeYd2g==", + "dev": true, + "license": "MIT" + }, "node_modules/data-uri-to-buffer": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", @@ -3060,6 +3197,16 @@ "dev": true, "license": "MIT" }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/detect-libc": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz", @@ -3077,6 +3224,65 @@ "dev": true, "license": "MIT" }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "dev": true, + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/drizzle-kit": { "version": "0.31.9", "resolved": "https://registry.npmjs.org/drizzle-kit/-/drizzle-kit-0.31.9.tgz", @@ -3219,6 +3425,35 @@ } } }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/encoding-sniffer": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz", + "integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==", + "dev": true, + "license": "MIT", + "dependencies": { + "iconv-lite": "^0.6.3", + "whatwg-encoding": "^3.1.1" + }, + "funding": { + "url": "https://github.com/fb55/encoding-sniffer?sponsor=1" + } + }, "node_modules/enhanced-resolve": { "version": "5.19.0", "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.19.0.tgz", @@ -3233,6 +3468,68 @@ "node": ">=10.13.0" } }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/esbuild": { "version": "0.25.12", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", @@ -3662,6 +3959,44 @@ "dev": true, "license": "ISC" }, + "node_modules/follow-redirects": { + "version": "1.15.11", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", + "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "dev": true, + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", @@ -3690,6 +4025,55 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/get-tsconfig": { "version": "4.13.6", "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.6.tgz", @@ -3729,6 +4113,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", @@ -3746,6 +4143,94 @@ "node": ">=8" } }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/htmlparser2": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", + "dev": true, + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "entities": "^7.0.1" + } + }, + "node_modules/htmlparser2/node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -4301,6 +4786,39 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/mini-svg-data-uri": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/mini-svg-data-uri/-/mini-svg-data-uri-1.4.4.tgz", @@ -4433,6 +4951,19 @@ "url": "https://opencollective.com/node-fetch" } }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, "node_modules/obug": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.1.tgz", @@ -4507,6 +5038,59 @@ "node": ">=6" } }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", + "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "domhandler": "^5.0.3", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-parser-stream": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz", + "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==", + "dev": true, + "license": "MIT", + "dependencies": { + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -4807,6 +5391,13 @@ "dev": true, "license": "ISC" }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "dev": true, + "license": "MIT" + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -4916,6 +5507,13 @@ "node": ">=6" } }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true, + "license": "MIT" + }, "node_modules/semver": { "version": "7.7.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", @@ -5247,6 +5845,16 @@ "typescript": ">=4.8.4 <6.0.0" } }, + "node_modules/undici": { + "version": "7.22.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.22.0.tgz", + "integrity": "sha512-RqslV2Us5BrllB+JeiZnK4peryVTndy9Dnqq62S3yYRRTj0tFQCwEniUy2167skdGOy3vqRzEvl1Dm4sV2ReDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, "node_modules/undici-types": { "version": "7.16.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", @@ -5867,6 +6475,30 @@ "dev": true, "license": "BSD-2-Clause" }, + "node_modules/whatwg-encoding": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation", + "dev": true, + "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-mimetype": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", @@ -5926,22 +6558,6 @@ } } }, - "node_modules/yaml": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", - "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", - "extraneous": true, - "license": "ISC", - "bin": { - "yaml": "bin.mjs" - }, - "engines": { - "node": ">= 14.6" - }, - "funding": { - "url": "https://github.com/sponsors/eemeli" - } - }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index f6e0d87..6d487b0 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,9 @@ "db:generate": "drizzle-kit generate", "db:migrate": "drizzle-kit migrate", "db:studio": "drizzle-kit studio", - "auth:schema": "npx @better-auth/cli generate --config src/lib/server/auth.ts --output src/lib/server/db/auth.schema.ts --yes" + "db:import": "node scripts/import-sql.js", + "auth:schema": "npx @better-auth/cli generate --config src/lib/server/auth.ts --output src/lib/server/db/auth.schema.ts --yes", + "scrape": "node scripts/scrape-onepiece.js" }, "devDependencies": { "@eslint/compat": "^2.0.2", @@ -30,6 +32,8 @@ "@tailwindcss/vite": "^4.1.18", "@types/node": "^24", "better-auth": "^1.4.18", + "cheerio": "^1.0.0-rc.12", + "csv-writer": "^1.6.0", "drizzle-kit": "^0.31.8", "drizzle-orm": "^0.45.1", "eslint": "^9.39.2", diff --git a/scripts/import-sql.js b/scripts/import-sql.js new file mode 100644 index 0000000..de5cb0e --- /dev/null +++ b/scripts/import-sql.js @@ -0,0 +1,104 @@ +import { createClient } from '@libsql/client'; +import fs from 'fs'; + +// Load environment variables +const DATABASE_URL = process.env.DATABASE_URL || 'file:local.db'; + +const client = createClient({ + url: DATABASE_URL +}); + +async function importSQL() { + try { + let totalSuccess = 0; + let totalErrors = 0; + + // Step 1: Import Devil Fruits + if (fs.existsSync('./scraped-data/devil-fruits.sql')) { + console.log('\n=== Importing Devil Fruits ===\n'); + const devilFruitsSql = fs.readFileSync('./scraped-data/devil-fruits.sql', 'utf-8'); + const dfStatements = devilFruitsSql.split(';\n\n').filter(s => s.trim()); + + console.log(`Found ${dfStatements.length} devil fruit statements\n`); + + let successCount = 0; + let errorCount = 0; + + for (let i = 0; i < dfStatements.length; i++) { + const statement = dfStatements[i]; + if (statement.trim()) { + try { + await client.execute(statement.trim() + ';'); + successCount++; + process.stdout.write(`\rExecuted: ${successCount}/${dfStatements.length}`); + } catch (error) { + errorCount++; + const valuesMatch = statement.match(/VALUES\s*\(([^)]+)\)/); + const values = valuesMatch ? valuesMatch[1] : 'N/A'; + console.error(`\n✗ Error at statement ${i + 1}:`); + console.error(` Values: ${values}`); + console.error(` Message: ${error.message}`); + } + } + } + + console.log(`\n\n✓ Devil Fruits imported!`); + console.log(` Success: ${successCount}`); + console.log(` Errors: ${errorCount}`); + + totalSuccess += successCount; + totalErrors += errorCount; + } else { + console.log('\n⚠️ No devil-fruits.sql found, skipping...\n'); + } + + // Step 2: Import Characters + if (fs.existsSync('./scraped-data/characters.sql')) { + console.log('\n=== Importing Characters ===\n'); + const charactersSql = fs.readFileSync('./scraped-data/characters.sql', 'utf-8'); + const charStatements = charactersSql.split(';\n\n').filter(s => s.trim()); + + console.log(`Found ${charStatements.length} character statements\n`); + + let successCount = 0; + let errorCount = 0; + + for (let i = 0; i < charStatements.length; i++) { + const statement = charStatements[i]; + if (statement.trim()) { + try { + await client.execute(statement.trim() + ';'); + successCount++; + process.stdout.write(`\rExecuted: ${successCount}/${charStatements.length}`); + } catch (error) { + errorCount++; + const valuesMatch = statement.match(/VALUES\s*\(([^)]+)\)/); + const values = valuesMatch ? valuesMatch[1] : 'N/A'; + console.error(`\n✗ Error at statement ${i + 1}:`); + console.error(` Values: ${values}`); + console.error(` Message: ${error.message}`); + } + } + } + + console.log(`\n\n✓ Characters imported!`); + console.log(` Success: ${successCount}`); + console.log(` Errors: ${errorCount}`); + + totalSuccess += successCount; + totalErrors += errorCount; + } else { + console.log('\n⚠️ No characters.sql found, skipping...\n'); + } + + console.log(`\n=== Total Import Summary ===`); + console.log(` Total Success: ${totalSuccess}`); + console.log(` Total Errors: ${totalErrors}\n`); + + } catch (error) { + console.error('✗ Import failed:', error.message); + process.exit(1); + } +} + +importSQL().catch(console.error); diff --git a/scripts/scrape-onepiece.js b/scripts/scrape-onepiece.js new file mode 100644 index 0000000..8127d10 --- /dev/null +++ b/scripts/scrape-onepiece.js @@ -0,0 +1,672 @@ +import * as cheerio from 'cheerio'; +import fs from 'fs'; +import { createObjectCsvWriter } from 'csv-writer'; + +const FANDOM_BASE_URL = 'https://onepiece.fandom.com/fr/wiki'; +const OUTPUT_DIR = './scraped-data'; +const DEVIL_FRUIT_CONCURRENCY = 5; +const CHARACTER_CONCURRENCY = 10; + +// Create output directory +if (!fs.existsSync(OUTPUT_DIR)) { + fs.mkdirSync(OUTPUT_DIR, { recursive: true }); +} + +/** + * Normalize string by removing accents and converting to lowercase + */ +function normalizeId(str) { + return decodeURIComponent(str) + .normalize('NFD') + .replace(/[\u0300-\u036f]/g, '') + .replace(/[,:]/g, '') + .toLowerCase(); +} + +/** + * Fetch all devil fruits URLs from One Piece fandom + */ +async function fetchAllDevilFruitsUrl() { + try { + const url = `${FANDOM_BASE_URL}/Fruits_du_Démon`; + console.log('Fetching devil fruits list...'); + const response = await fetch(url, { + headers: { + 'User-Agent': 'Mozilla/5.0 (compatible; MSIE 7.01; Windows NT 5.0)', + }, + }); + const data = await response.text(); + const $ = cheerio.load(data); + const devilFruits = []; + + // Find the main navibox table + $('table.navibox.toccolours').each((mainTableIndex, mainTable) => { + const mainHeader = $(mainTable).find('th[colspan="3"]').first().find('span').last().text().trim(); + if (mainHeader !== 'Fruits du Démon') return; + + $(mainTable).find('table.collapsible').each((typeTableIndex, typeTable) => { + const typeHeader = $(typeTable).find('th[colspan="3"]').first().text().trim(); + let type = null; + + if (typeHeader.includes('Paramecia')) type = 'Paramecia'; + else if (typeHeader.includes('Zoan')) type = 'Zoan'; + else if (typeHeader.includes('Logia')) type = 'Logia'; + else if (typeHeader.includes('Type Inconnu')) type = 'Unknown'; + + if (!type) return; + + $(typeTable).find('tr.navibox-row').each((rowIndex, row) => { + const categoryHeader = $(row).find('th').text().trim(); + + if (!categoryHeader.includes('Canon') && + !categoryHeader.includes('Standards') && + !categoryHeader.includes('Antiques') && + !categoryHeader.includes('Mythiques') && + !categoryHeader.includes('Hors-Série')) { + return; + } + + // Find all links in the row + $(row).find('td .hlist ul li a').each((linkIndex, link) => { + const name = $(link).text().trim(); + const href = $(link).attr('href'); + + if (name && href && href.startsWith('/fr/wiki/')) { + // Clean the URL + const cleanUrl = href.replace('/fr/wiki/', ''); + + // Skip classification pages and category pages + if (cleanUrl.includes('Classification') || + cleanUrl.includes('Catégorie:') || + cleanUrl === 'Fruits_du_Démon_Artificiels' || + cleanUrl === 'SMILE') { + return; + } + + devilFruits.push({ + id: normalizeId(cleanUrl), + name, + type, + url: cleanUrl, + }); + } + }); + }); + }); + }); + + console.log(`Found ${devilFruits.length} devil fruits.`); + return devilFruits; + } catch (error) { + console.error('Error fetching devil fruits list:', error.message); + return []; + } +} + +/** + * Fetch devil fruit data from fandom using provided URL + */ +async function fetchDevilFruit(devilFruitUrl, devilFruitId, devilFruitName, devilFruitType) { + try { + console.log(`Fetching: ${devilFruitName}...`); + + const response = await fetch(`${FANDOM_BASE_URL}/${devilFruitUrl}`, { + headers: { + 'User-Agent': 'Mozilla/5.0 (compatible; MSIE 7.01; Windows NT 5.0)', + }, + }); + const data = await response.text(); + const $ = cheerio.load(data); + + // Extract devil fruit name from page title if different + const name = $('h1.mw-page-title-main').text().trim() || devilFruitName; + + // Use the type from the list page + const type = devilFruitType; + + return { + id: devilFruitId, + name, + type + }; + } catch (error) { + console.error(`Error fetching ${devilFruitName}:`, error.message); + return null; + } +} + +/** + * Save devil fruits to JSON + */ +async function saveDevilFruitsToJSON(devilFruits) { + const filepath = `${OUTPUT_DIR}/devil-fruits.json`; + fs.writeFileSync(filepath, JSON.stringify(devilFruits, null, 2)); + console.log(`✓ Saved to ${filepath}`); +} + +/** + * Save devil fruits to SQL + */ +function saveDevilFruitsToSQL(devilFruits) { + const filepath = `${OUTPUT_DIR}/devil-fruits.sql`; + const escapeSql = (value) => (value ? `'${String(value).replace(/'/g, "''")}'` : 'NULL'); + + let sql = ''; + + devilFruits.forEach((df) => { + sql += `INSERT INTO devilFruit (id, name, type) \n`; + sql += `VALUES (${escapeSql(df.id)}, ${escapeSql(df.name)}, ${escapeSql(df.type)}) \n`; + sql += `ON CONFLICT(id) DO UPDATE SET \n`; + sql += ` name = excluded.name,\n`; + sql += ` type = excluded.type;\n\n`; + }); + + fs.writeFileSync(filepath, sql); + console.log(`✓ Saved to ${filepath}`); +} + +/** + * Fetch all cannon characters from One Piece fandom + */ +async function fetchAllCharactersUrl() { + try { + const url = `${FANDOM_BASE_URL}/Liste_des_Personnages_Canon`; + console.log('Fetching character list...'); + const response = await fetch(url, { + headers: { + 'User-Agent': 'Mozilla/5.0 (compatible; MSIE 7.01; Windows NT 5.0)', + }, + }); + const data = await response.text(); + const $ = cheerio.load(data); + const characters = []; + $('table.wikitable tbody tr').each((index, element) => { + if (index === 0) return; // Skip header row + const charpictureUrl = $(element).find('td:nth-child(1) a img').attr('data-src') || $(element).find('td:nth-child(1) a img').attr('src'); + const charLink = $(element).find('td:nth-child(2) a').attr('href'); + const charName = $(element).find('td:nth-child(2) a').text().trim(); + if (charLink) { + const cleanUrl = charLink.replace('/fr/wiki/', ''); + characters.push({ + id: normalizeId(cleanUrl), + name: charName, + url: cleanUrl, + pictureUrl: charpictureUrl, + }); + } + }); + console.log(`Found ${characters.length} characters.`); + return characters; + } catch (error) { + console.error('Error fetching character list:', error.message); + return []; + } +} + +/** + * Fetch character data from fandom using provided URL + */ +async function fetchCharacter(characterUrl, characterId, characterName, characterpictureUrl) { + try { + console.log(`Fetching: ${characterName}...`); + + const response = await fetch(`${FANDOM_BASE_URL}/${characterUrl}`, { + headers: { + 'User-Agent': 'Mozilla/5.0 (compatible; MSIE 7.01; Windows NT 5.0)', + }, + }); + // Log response status for debugging + const data = await response.text(); + + const $ = cheerio.load(data); + + // Extract character name + const name = $('h1.mw-page-title-main').text().trim() || characterName.replace(/_/g, ' '); + + // Extract gender from the specific categories link + let gender = null; + if ($('.page-header__categories a[title="Catégorie:Personnages Masculins"]').length > 0) { + gender = 'Male'; + } else if ($('.page-header__categories a[title="Catégorie:Personnages Féminins"]').length > 0) { + gender = 'Female'; + } + + // Extract age + const age = extractAge($); + + // Extract affiliations + const affiliations = extractAffiliations($); + + // Extract devil fruit + const devilFruit = await extractDevilFruit($); + + // Extract haki + let haki = []; + if ($('.page-header__categories a[title="Catégorie:Utilisateurs du Haki de l\'observation"]').length > 0) { + haki.push('Observation'); + } + if ($('.page-header__categories a[title="Catégorie:Utilisateurs du Haki de l\'armement"]').length > 0) { + haki.push('Armament'); + } + if ($('.page-header__categories a[title="Catégorie:Utilisateurs du Haki des rois"]').length > 0) { + haki.push('Conqueror'); + } + + // Extract bounty + const bounty = extractBounty($); + + // Extract height + const height = extractHeight($); + + // Extract first appearance + const firstAppearance = extractFirstAppearance($); + + // Extract origin + const origin = extractOrigin($); + + // Extract image URL and clean it + let pictureUrl = characterpictureUrl; + + return { + id: characterId, + name, + gender, + age, + height, + origin, + devilFruit, + affiliations, + bounty, + haki, + firstAppearance, + pictureUrl + }; + } catch (error) { + console.error(`Error fetching ${characterName}:`, error.message); + return null; + } +} + + +/** + * Extract age from infobox + */ +function extractAge($) { + const div = $('[data-source="âge"] .pi-data-value'); + if (div.length === 0) return null; + + let text = div.html(); + if (!text) return null; + + // Remove all sup blocks (citations) + text = text.replace(/]*>.*?<\/sup>/gi, ''); + + // Get the last element and extract only digits + const parts = text.split(']*>/g, '').trim(); + + // Remove content with parentheses + cleanText = cleanText.replace(/\([^)]*\)/g, ''); + + const digitsOnly = cleanText.replace(/\D/g, ''); + return digitsOnly || null; +} + +/** + * Extract affiliations from infobox + */ +function extractAffiliations($) { + const div = $('[data-source="affiliation"] .pi-data-value'); + if (div.length === 0) return []; + + const cleanedDiv = div.clone(); + cleanedDiv.find('sup').remove(); + + let text = cleanedDiv.html(); + if (!text) return []; + + // Extract all link values + const linkValues = cleanedDiv.find('a').map((i, el) => $(el).text().trim()).get(); + if (linkValues.length > 0) { + return linkValues; + } + + // Fallback to parsing text + const cleanText = text.replace(/<[^>]*>/g, '').trim(); + const parts = cleanText.split(/\s*\n\s*|\s*;\s*|\s*,\s*/).filter(Boolean); + return parts.length > 0 ? parts : []; +} + +/** + * Extract devil fruit from infobox + */ +async function extractDevilFruit($) { + const link = $('[data-source="dfnom"] .pi-data-value a').first(); + if (link.length === 0) return null; + + const href = link.attr('href'); + if (!href || !href.startsWith('/fr/wiki/')) return null; + + const cleanUrl = href.replace('/fr/wiki/', ''); + + try { + // Fetch the page to follow redirects + const response = await fetch(`${FANDOM_BASE_URL}/${cleanUrl}`, { + headers: { + 'User-Agent': 'Mozilla/5.0 (compatible; MSIE 7.01; Windows NT 5.0)', + }, + redirect: 'follow' // Explicitly follow redirects + }); + + // Check if response was a redirect (301, 302, etc.) + if (response.status === 301 || response.status === 302) { + // Use the final redirected URL + const finalUrl = new URL(response.url); + const pathname = finalUrl.pathname; + const finalPath = pathname.replace('/fr/wiki/', ''); + if (finalPath) { + return normalizeId(finalPath); + } + } else { + // Use the current URL if no redirect + const finalUrl = new URL(response.url); + const pathname = finalUrl.pathname; + const finalPath = pathname.replace('/fr/wiki/', ''); + if (finalPath) { + return normalizeId(finalPath); + } + } + } catch (error) { + console.error(`Error fetching devil fruit page: ${error.message}`); + } + + // Fallback to the original href + return normalizeId(cleanUrl); +} + +/** + * Extract bounty from infobox + */ +function extractBounty($) { + const div = $('[data-source="prime"] .pi-data-value'); + if (div.length === 0) return null; + + let text = div.html(); + if (!text) return null; + + // Remove all sup blocks (citations) + text = text.replace(/]*>.*?<\/sup>/gi, ''); + + // Extract the first value before any
tag + const firstValue = text.split(']*>/g, '').trim(); + + // Remove spaces and dots + cleanText = cleanText.replace(/[\s.]/g, ''); + + return cleanText || null; +} + +/** + * Extract height from infobox + */ +function extractHeight($) { + const div = $('[data-source="taille"] .pi-data-value'); + if (div.length === 0) return null; + + let text = div.html(); + if (!text) return null; + + // Remove all sup blocks (citations) + text = text.replace(/]*>.*?<\/sup>/gi, ''); + + // Extract the last value after any
tag + const lastValue = text.split('
').pop().trim(); + let cleanText = lastValue.replace(/<[^>]*>/g, '').trim(); + + // Remove content with parentheses + cleanText = cleanText.replace(/\([^)]*\)/g, ''); + + // Normalize units for meters or centimeters + const normalized = cleanText.toLowerCase().replace(/\s/g, ''); + if (normalized.includes('cm')) { + const digitsOnly = normalized.replace(/\D/g, ''); + return digitsOnly || null; + } + + if (normalized.includes('m')) { + const parts = normalized.split('m').filter(Boolean); + return parts.length > 0 ? parts.join('.') : null; + } + + return normalized.replace(/\D/g, '') || null; +} + +/** + * Extract first appearance from infobox + */ +function extractFirstAppearance($) { + const div = $('[data-source="première"] .pi-data-value'); + if (div.length === 0) return null; + + let text = div.html(); + if (!text) return null; + + // Remove all sup blocks (citations) + text = text.replace(/]*>.*?<\/sup>/gi, ''); + + // Extract digits after "Chapitre" + const cleanText = text.replace(/<[^>]*>/g, '').trim(); + const match = cleanText.match(/Chapitre\s+(\d+)/i); + return match ? match[1] : null; +} + +/** + * Extract origin from infobox + */ +function extractOrigin($) { + const div = $('[data-source="origine"] .pi-data-value'); + if (div.length === 0) return null; + + let text = div.html(); + if (!text) return null; + + // Remove all sup blocks (citations) + text = text.replace(/]*>.*?<\/sup>/gi, ''); + + // Extract the first value before any
tag + const firstValue = text.split(']*>/g, '').trim(); + + // Remove content with parentheses + cleanText = cleanText.replace(/\([^)]*\)/g, '').trim(); + + return cleanText || null; +} + + +/** + * Save data to JSON + */ +async function saveToJSON(characters) { + const filepath = `${OUTPUT_DIR}/characters.json`; + fs.writeFileSync(filepath, JSON.stringify(characters, null, 2)); + console.log(`✓ Saved to ${filepath}`); +} + +/** + * Save data to CSV + */ +async function saveToCSV(characters) { + const filepath = `${OUTPUT_DIR}/characters.csv`; + const csvWriter = createObjectCsvWriter({ + path: filepath, + header: [ + { id: 'id', title: 'ID' }, + { id: 'name', title: 'Name' }, + { id: 'gender', title: 'Gender' }, + { id: 'age', title: 'Age' }, + { id: 'height', title: 'Height' }, + { id: 'origin', title: 'Origin' }, + { id: 'devilFruit', title: 'Devil Fruit' }, + { id: 'affiliations', title: 'Affiliations' }, + { id: 'bounty', title: 'Bounty' }, + { id: 'haki', title: 'Haki' }, + { id: 'firstAppearance', title: 'First Appearance' }, + { id: 'pictureUrl', title: 'Image URL' } + ], + }); + + const records = characters + .filter((c) => c !== null) + .map((c) => ({ + id: c.id || '', + name: c.name || '', + gender: c.gender || '', + age: c.age || '', + height: c.height || '', + origin: c.origin || '', + devilFruit: c.devilFruit || '', + affiliations: Array.isArray(c.affiliations) ? c.affiliations.join(', ') : (c.affiliations || ''), + bounty: c.bounty || '', + haki: Array.isArray(c.haki) ? c.haki.join(', ') : (c.haki || ''), + firstAppearance: c.firstAppearance || '', + pictureUrl: c.pictureUrl || '' + })); + + await csvWriter.writeRecords(records); + console.log(`✓ Saved to ${filepath}`); +} + +/** + * Save data to SQL + */ +function saveToSQL(characters) { + const filepath = `${OUTPUT_DIR}/characters.sql`; + const escapeSql = (value) => (value ? `'${String(value).replace(/'/g, "''")}'` : 'NULL'); + + let sql = ''; + + characters + .filter((c) => c !== null) + .forEach((c) => { + const affiliations = Array.isArray(c.affiliations) ? c.affiliations.join(', ') : c.affiliations; + const hakiValue = Array.isArray(c.haki) && c.haki.length > 0 ? JSON.stringify(c.haki) : null; + + sql += `INSERT INTO character (id, name, gender, age, height, origin, devilFruit, affiliations, bounty, haki, firstAppearance, pictureUrl) \n`; + sql += `VALUES (${escapeSql(c.id)}, ${escapeSql(c.name)}, ${escapeSql(c.gender)}, ${escapeSql(c.age)}, ${escapeSql(c.height)}, ${escapeSql(c.origin)}, ${escapeSql(c.devilFruit)}, ${escapeSql(affiliations)}, ${escapeSql(c.bounty)}, ${escapeSql(hakiValue)}, ${escapeSql(c.firstAppearance)}, ${escapeSql(c.pictureUrl)}) \n`; + sql += `ON CONFLICT(id) DO UPDATE SET \n`; + sql += ` name = excluded.name,\n`; + sql += ` gender = excluded.gender,\n`; + sql += ` age = excluded.age,\n`; + sql += ` height = excluded.height,\n`; + sql += ` origin = excluded.origin,\n`; + sql += ` devilFruit = excluded.devilFruit,\n`; + sql += ` affiliations = excluded.affiliations,\n`; + sql += ` bounty = excluded.bounty,\n`; + sql += ` haki = excluded.haki,\n`; + sql += ` firstAppearance = excluded.firstAppearance,\n`; + sql += ` pictureUrl = excluded.pictureUrl;\n\n`; + }); + + fs.writeFileSync(filepath, sql); + console.log(`✓ Saved to ${filepath}`); +} + +/** + * Main execution + */ +async function main() { + const format = process.argv[2] || 'all'; // json, csv, sql, or all + + console.log(`\nOne Piece Scraper - Mode: ${format}\n`); + + // Step 1: Scraping Devil Fruits + console.log('=== Step 1: Scraping Devil Fruits ===\n'); + const devilFruitList = await fetchAllDevilFruitsUrl(); + + if (devilFruitList.length === 0) { + console.warn('No devil fruits found, continuing with characters...\n'); + } else { + const devilFruits = []; + + for (let i = 0; i < devilFruitList.length; i += DEVIL_FRUIT_CONCURRENCY) { + const batch = devilFruitList.slice(i, i + DEVIL_FRUIT_CONCURRENCY); + const results = await Promise.all( + batch.map((df) => fetchDevilFruit(df.url, df.id, df.name, df.type)) + ); + + results.filter(Boolean).forEach((data) => { + console.table({ + ID: data.id, + Name: data.name, + Type: data.type + }); + + devilFruits.push(data); + }); + } + + console.log(`\n✓ Scraped ${devilFruits.length} devil fruits\n`); + + if (format === 'json' || format === 'all') { + await saveDevilFruitsToJSON(devilFruits); + } + if (format === 'sql' || format === 'all') { + saveDevilFruitsToSQL(devilFruits); + } + } + + // Step 2: Scraping Characters + console.log('=== Step 2: Scraping Characters ===\n'); + const characterList = await fetchAllCharactersUrl(); + + if (characterList.length === 0) { + console.error('No characters found. Exiting.'); + return; + } + + const characters = []; + + for (let i = 0; i < characterList.length; i += CHARACTER_CONCURRENCY) { + const batch = characterList.slice(i, i + CHARACTER_CONCURRENCY); + const results = await Promise.all( + batch.map((char) => fetchCharacter(char.url, char.id, char.name, char.pictureUrl)) + ); + results.filter(Boolean).forEach((data) => { + console.table({ + ID: data.id, + Name: data.name, + Gender: data.gender, + Age: data.age, + Affiliations: data.affiliations.join(', '), + DevilFruit: data.devilFruit, + Haki: data.haki.join(', '), + Height: data.height, + Bounty: data.bounty, + Origin: data.origin, + FirstAppearance: data.firstAppearance, + pictureUrl: data.pictureUrl + }); + + characters.push(data); + }); + } + + console.log(`\n✓ Scraped ${characters.length} characters\n`); + + if (format === 'json' || format === 'all') { + await saveToJSON(characters); + } + if (format === 'csv' || format === 'all') { + await saveToCSV(characters); + } + if (format === 'sql' || format === 'all') { + saveToSQL(characters); + } + + console.log('\n✓ Done!\n'); +} + +main().catch(console.error); diff --git a/src/lib/server/db/schema.ts b/src/lib/server/db/schema.ts index dbc27a7..caca5d7 100644 --- a/src/lib/server/db/schema.ts +++ b/src/lib/server/db/schema.ts @@ -1,11 +1,44 @@ -import { integer, sqliteTable, text } from 'drizzle-orm/sqlite-core'; +import { integer, sqliteTable, text, real } from 'drizzle-orm/sqlite-core'; -export const task = sqliteTable('task', { +// Define haki types +export type HakiType = 'Observation' | 'Armament' | 'Conqueror'; + +// Define devil fruit types +export type DevilFruitType = 'Paramecia' | 'Zoan' | 'Logia' | 'Unknown'; + +// Define the devil fruit table schema +export const devilFruit = sqliteTable('devilFruit', { + id: text('id').primaryKey(), + name: text('name').notNull().unique(), + type: text('type').$type() +}); + +// Define the character table schema +export const character = sqliteTable('character', { + id: text('id').primaryKey(), + name: text('name').notNull(), + gender: text('gender'), + age: integer('age'), + affiliations: text('affiliations'), + devilFruit: text('devilFruit').references(() => devilFruit.id), + haki: text('haki', { mode: 'json' }).$type(), + bounty: integer('bounty'), + // height in meters as a float (e.g. 1.75) + height: real('height'), + origin: text('origin'), + firstAppearance: text('firstAppearance'), + pictureUrl: text('pictureUrl') +}); + +// Define the caracter history table schema +export const characterHistory = sqliteTable('characterHistory', { id: text('id') .primaryKey() .$defaultFn(() => crypto.randomUUID()), - title: text('title').notNull(), - priority: integer('priority').notNull().default(1) + characterId: text('characterId').references(() => character.id), + date: integer('date'), + createdAt: integer('createdAt').notNull().$default(() => Date.now()), + updatedAt: integer('updatedAt').notNull().$default(() => Date.now()), }); export * from './auth.schema'; diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index cc88df0..d4b7bd0 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -1,2 +1,63 @@ -

Welcome to SvelteKit

-

Visit svelte.dev/docs/kit to read the documentation

+ + OnePieceDle + + +
+
+
+ +
+
+
+ Jeu de devinettes Grand Line +
+
+

+ OnePieceDle +

+

+ Devine le personnage de l'equipage, des marines ou du vaste monde. Chaque indice te rapproche du tresor. +

+
+
+
+

Voyage du jour

+

Nouveau mystere toutes les 24 heures

+

Compare tes essais, debloque des indices et garde ta serie.

+ + Commencer + +
+
+

Partie libre

+

Entraine-toi avec des pirates legendaires

+

Choisis une epoque, regle la difficulte et vogue a ton rythme.

+ +
+
+
+
+
+ Photo +
+
+

Personnage de la veille

+

Placeholder

+

Le revele sera visible apres la partie du jour.

+
+ +
+
+
+
+
diff --git a/src/routes/daily/+page.svelte b/src/routes/daily/+page.svelte new file mode 100644 index 0000000..9d94edb --- /dev/null +++ b/src/routes/daily/+page.svelte @@ -0,0 +1,87 @@ + + OnePieceDle - Mode du jour + + +
+
+
+ +
+
+
+ Mode du jour +
+

+ Mystere du jour +

+

+ Devine le personnage. Chaque indice deblocque une nouvelle piste. +

+
+ +
+
+

Indices du jour

+
+
+

Indice 1

+

Origine: ???

+
+
+

Indice 2

+

Fruit du demon: ???

+
+
+

Indice 3

+

Affiliation: ???

+
+
+
+ +
+

Entrer une supposition

+
+ + +
+
+
+ +
+
+
+

Historique

+

Aucune tentative pour le moment.

+
+ +
+
+ +
+
+
+ Photo +
+
+

Personnage d'hier

+

Placeholder

+

Revele apres validation de ta tentative.

+
+ +
+
+
+
diff --git a/src/routes/demo/+page.svelte b/src/routes/demo/+page.svelte deleted file mode 100644 index 948d26f..0000000 --- a/src/routes/demo/+page.svelte +++ /dev/null @@ -1,5 +0,0 @@ - - -better-auth diff --git a/src/routes/demo/better-auth/+page.server.ts b/src/routes/demo/better-auth/+page.server.ts deleted file mode 100644 index 7c30835..0000000 --- a/src/routes/demo/better-auth/+page.server.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { redirect } from '@sveltejs/kit'; -import type { Actions } from './$types'; -import type { PageServerLoad } from './$types'; -import { auth } from '$lib/server/auth'; - -export const load: PageServerLoad = async (event) => { - if (!event.locals.user) { - return redirect(302, '/demo/better-auth/login'); - } - return { user: event.locals.user }; -}; - -export const actions: Actions = { - signOut: async (event) => { - await auth.api.signOut({ - headers: event.request.headers - }); - return redirect(302, '/demo/better-auth/login'); - } -}; diff --git a/src/routes/demo/better-auth/+page.svelte b/src/routes/demo/better-auth/+page.svelte deleted file mode 100644 index f25310a..0000000 --- a/src/routes/demo/better-auth/+page.svelte +++ /dev/null @@ -1,14 +0,0 @@ - - -

Hi, {data.user.name}!

-

Your user ID is {data.user.id}.

-
- -
diff --git a/src/routes/demo/better-auth/login/+page.server.ts b/src/routes/demo/better-auth/login/+page.server.ts deleted file mode 100644 index c610f54..0000000 --- a/src/routes/demo/better-auth/login/+page.server.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { fail, redirect } from '@sveltejs/kit'; -import type { Actions } from './$types'; -import type { PageServerLoad } from './$types'; -import { auth } from '$lib/server/auth'; -import { APIError } from 'better-auth/api'; - -export const load: PageServerLoad = async (event) => { - if (event.locals.user) { - return redirect(302, '/demo/better-auth'); - } - return {}; -}; - -export const actions: Actions = { - signInEmail: async (event) => { - const formData = await event.request.formData(); - const email = formData.get('email')?.toString() ?? ''; - const password = formData.get('password')?.toString() ?? ''; - - try { - await auth.api.signInEmail({ - body: { - email, - password, - callbackURL: '/auth/verification-success' - } - }); - } catch (error) { - if (error instanceof APIError) { - return fail(400, { message: error.message || 'Signin failed' }); - } - return fail(500, { message: 'Unexpected error' }); - } - - return redirect(302, '/demo/better-auth'); - }, - signUpEmail: async (event) => { - const formData = await event.request.formData(); - const email = formData.get('email')?.toString() ?? ''; - const password = formData.get('password')?.toString() ?? ''; - const name = formData.get('name')?.toString() ?? ''; - - try { - await auth.api.signUpEmail({ - body: { - email, - password, - name, - callbackURL: '/auth/verification-success' - } - }); - } catch (error) { - if (error instanceof APIError) { - return fail(400, { message: error.message || 'Registration failed' }); - } - return fail(500, { message: 'Unexpected error' }); - } - - return redirect(302, '/demo/better-auth'); - } -}; diff --git a/src/routes/demo/better-auth/login/+page.svelte b/src/routes/demo/better-auth/login/+page.svelte deleted file mode 100644 index 267b86a..0000000 --- a/src/routes/demo/better-auth/login/+page.svelte +++ /dev/null @@ -1,42 +0,0 @@ - - -

Login

-
- - - - - -
-

{form?.message ?? ''}