997 lines
33 KiB
TypeScript
997 lines
33 KiB
TypeScript
import * as cheerio from 'cheerio';
|
|
import fs from 'fs';
|
|
import { createObjectCsvWriter } from 'csv-writer';
|
|
|
|
// Type definitions
|
|
interface Arc {
|
|
id: string;
|
|
name: string;
|
|
startChapter: number;
|
|
endChapter: number | null;
|
|
url: string;
|
|
}
|
|
|
|
interface Character {
|
|
id: string;
|
|
name: string;
|
|
gender: string | null;
|
|
age: number | null;
|
|
height: number | null;
|
|
origin: string | null;
|
|
devilFruitId: string | null;
|
|
devilFruitUrl: string | null;
|
|
affiliations: string[];
|
|
bounty: number | null;
|
|
hakiObservation: boolean;
|
|
hakiArmament: boolean;
|
|
hakiConqueror: boolean;
|
|
epithets: string[];
|
|
firstAppearance: number;
|
|
status: string | null;
|
|
pictureUrl: string | null;
|
|
url: string;
|
|
arcId?: string;
|
|
}
|
|
|
|
interface CharacterListItem {
|
|
name: string;
|
|
url: string;
|
|
pictureUrl: string | null;
|
|
chapter: string;
|
|
}
|
|
|
|
interface DevilFruitData {
|
|
devilFruitId: string;
|
|
devilFruitUrl: string;
|
|
}
|
|
|
|
interface DevilFruit {
|
|
id: string;
|
|
name: string;
|
|
type: string | null;
|
|
url: string;
|
|
}
|
|
|
|
const FANDOM_API_BASE = 'https://onepiece.fandom.com/fr/api.php?action=parse&format=json&page=';
|
|
const OUTPUT_DIR = './scraped-data';
|
|
const MAX_RETRIES = 0; // Set to 0 to disable retries, can be increased if needed
|
|
const INITIAL_RETRY_DELAY = 1000;
|
|
const CHARACTER_FETCH_CONCURRENCY = 50;
|
|
|
|
// Store cookies across requests (simulate browser behavior)
|
|
const cookies = new Map<string, string>();
|
|
|
|
function getCookieHeader(): string {
|
|
const cookieArray = Array.from(cookies.values()).map(c => c.split(';')[0]);
|
|
return cookieArray.length > 0 ? cookieArray.join('; ') : '';
|
|
}
|
|
|
|
function saveCookies(setCookieHeader: string | string[] | null): void {
|
|
if (setCookieHeader) {
|
|
const cookiesList = Array.isArray(setCookieHeader) ? setCookieHeader : [setCookieHeader];
|
|
cookiesList.forEach(cookie => {
|
|
const [nameValue] = cookie.split(';');
|
|
const [name] = nameValue.split('=');
|
|
if (name) cookies.set(name, cookie);
|
|
});
|
|
}
|
|
}
|
|
|
|
// Create output directory
|
|
if (!fs.existsSync(OUTPUT_DIR)) {
|
|
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
|
|
}
|
|
|
|
/**
|
|
* Retry a fetch request with exponential backoff
|
|
*/
|
|
async function fetchWithRetry(url: string, options: RequestInit = {}, retries: number = 0): Promise<Response> {
|
|
try {
|
|
const headers: Record<string, string> = {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:150.0) Firefox/150.0',
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
|
'Connection': 'keep-alive',
|
|
...((options.headers as Record<string, string>) || {})
|
|
};
|
|
|
|
// Add cookies from previous requests
|
|
const cookieHeader = getCookieHeader();
|
|
if (cookieHeader) {
|
|
headers['Cookie'] = cookieHeader;
|
|
}
|
|
|
|
const response = await fetch(url, {
|
|
headers,
|
|
...options
|
|
} as any);
|
|
|
|
// Save cookies from response
|
|
const setCookie = response.headers.get('set-cookie');
|
|
if (setCookie) {
|
|
saveCookies(setCookie);
|
|
}
|
|
|
|
// Check if response is OK (status 200-299)
|
|
if (response.ok) {
|
|
return response;
|
|
}
|
|
|
|
// If not OK and we have retries left, retry
|
|
if (retries < MAX_RETRIES) {
|
|
const delay = INITIAL_RETRY_DELAY * Math.pow(2, retries);
|
|
console.log(`⚠️ HTTP ${response.status} for ${url}, retrying in ${delay}ms...`);
|
|
await new Promise(resolve => setTimeout(resolve, delay));
|
|
return fetchWithRetry(url, options, retries + 1);
|
|
}
|
|
|
|
// If we've exhausted retries, throw error
|
|
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
} catch (error) {
|
|
// If it's a network error and we have retries left, retry
|
|
if (retries < MAX_RETRIES) {
|
|
const delay = INITIAL_RETRY_DELAY * Math.pow(2, retries);
|
|
console.log(`⚠️ Network error: ${(error as Error).message}, retrying in ${delay}ms...`);
|
|
await new Promise(resolve => setTimeout(resolve, delay));
|
|
return fetchWithRetry(url, options, retries + 1);
|
|
}
|
|
|
|
// If we've exhausted retries, throw error
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
|
|
/**
|
|
* Normalize string by decoding URI components, punctuation, and replacing spaces with underscores
|
|
*/
|
|
function normalizeId(str: string): string {
|
|
return decodeURIComponent(str)
|
|
.normalize('NFD')
|
|
.replace(/[,:.\(\)]/g, '')
|
|
.replace(/\s+/g, '_')
|
|
.toLowerCase();
|
|
}
|
|
|
|
/**
|
|
* Fetch all arcs from One Piece fandom using API
|
|
*/
|
|
async function fetchAllArcs(): Promise<Arc[]> {
|
|
try {
|
|
const apiUrl = `${FANDOM_API_BASE}Chapitres_et_Tomes`;
|
|
console.log('Fetching arcs list via API...');
|
|
const response = await fetchWithRetry(apiUrl);
|
|
const jsonData = await response.json() as any;
|
|
|
|
// Extract HTML from API response
|
|
const htmlContent = jsonData.parse?.text?.['*'];
|
|
if (!htmlContent) {
|
|
throw new Error('Unable to extract HTML content from API response');
|
|
}
|
|
|
|
const $ = cheerio.load(htmlContent);
|
|
const arcs: Arc[] = [];
|
|
|
|
// Find all arc links in the table
|
|
$('table.wikitable td a').each((index, element) => {
|
|
const text = $(element).text().trim();
|
|
const href = $(element).attr('href');
|
|
|
|
// Check if it's an arc link (contains "Arc" and chapter info)
|
|
if (text.includes('Arc') && text.includes('Ch.') && href) {
|
|
// Extract arc name and chapter range
|
|
// Example text: "Arc Ville d'Orange(Ch.8 à 21)[T.1 à 3]"
|
|
console.log(`Processing arc link: ${text} (${href})`);
|
|
const nameMatch = text.match(/^(.*?Arc.*?)\s*\(Ch\.(\d+)(?:\s*à\s*(?:(\d+)|(?:...)))?\)/);
|
|
if (nameMatch) {
|
|
let arcName = nameMatch[1].trim();
|
|
// Remove "Arc " from the name
|
|
arcName = arcName.replace(/^Arc\s+/i, '');
|
|
|
|
const startChapter = parseInt(nameMatch[2]);
|
|
const endChapter = nameMatch[3] ? parseInt(nameMatch[3]) : null;
|
|
|
|
// Generate arc ID by normalizing the url
|
|
let arcId = normalizeId(href.replace('/fr/wiki/', ''));
|
|
// Remove "Arc_" from the id
|
|
arcId = arcId.replace(/^arc_/i, '');
|
|
|
|
arcs.push({
|
|
id: arcId,
|
|
name: arcName,
|
|
startChapter,
|
|
endChapter,
|
|
url: href.replace('/fr/wiki/', '')
|
|
});
|
|
}
|
|
}
|
|
});
|
|
|
|
console.log(`Found ${arcs.length} arcs.`);
|
|
return arcs;
|
|
} catch (error) {
|
|
console.error('Error fetching arcs list:', (error as Error).message);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Save arcs to JSON
|
|
*/
|
|
async function saveArcsToJSON(arcs: Arc[]): Promise<void> {
|
|
const filepath = `${OUTPUT_DIR}/arcs.json`;
|
|
fs.writeFileSync(filepath, JSON.stringify(arcs, null, 2));
|
|
console.log(`✓ Saved to ${filepath}`);
|
|
}
|
|
|
|
/**
|
|
* Save arcs to CSV
|
|
*/
|
|
async function saveArcsToCSV(arcs: Arc[]): Promise<void> {
|
|
const filepath = `${OUTPUT_DIR}/arcs.csv`;
|
|
const csvWriter = createObjectCsvWriter({
|
|
path: filepath,
|
|
header: [
|
|
{ id: 'id', title: 'ID' },
|
|
{ id: 'name', title: 'Name' },
|
|
{ id: 'startChapter', title: 'Start Chapter' },
|
|
{ id: 'endChapter', title: 'End Chapter' },
|
|
{ id: 'url', title: 'URL' }
|
|
],
|
|
});
|
|
|
|
const records = arcs
|
|
.filter((arc) => arc !== null)
|
|
.map((arc) => ({
|
|
id: arc.id || '',
|
|
name: arc.name || '',
|
|
startChapter: arc.startChapter || '',
|
|
endChapter: arc.endChapter || '',
|
|
url: arc.url || ''
|
|
}));
|
|
|
|
await csvWriter.writeRecords(records);
|
|
console.log(`✓ Saved to ${filepath}`);
|
|
}
|
|
|
|
/**
|
|
* Fetch all cannon characters from One Piece fandom using API
|
|
*/
|
|
async function fetchAllCharactersUrl(): Promise<CharacterListItem[]> {
|
|
try {
|
|
const apiUrl = `${FANDOM_API_BASE}Liste_des_Personnages_Canon`;
|
|
console.log('Fetching character list via API...');
|
|
const response = await fetchWithRetry(apiUrl);
|
|
const jsonData = await response.json() as any;
|
|
|
|
// Extract HTML from API response
|
|
const htmlContent = jsonData.parse?.text?.['*'];
|
|
if (!htmlContent) {
|
|
throw new Error('Unable to extract HTML content from API response');
|
|
}
|
|
|
|
const $ = cheerio.load(htmlContent);
|
|
const characters: CharacterListItem[] = [];
|
|
$('table.wikitable tbody tr').each((index, element) => {
|
|
if (index === 0) return; // Skip header row
|
|
let charpictureUrl = $(element).find('td:nth-child(1) a img').attr('data-src') || $(element).find('td:nth-child(1) a img').attr('src');
|
|
let charUrl = $(element).find('td:nth-child(2) a').attr('href');
|
|
let charName = $(element).find('td:nth-child(2) a').text().trim();
|
|
let charChapter = $(element).find('td:nth-child(3)').text().trim();
|
|
|
|
// Remove parentheses and their content from chapter info (e.g. "1 (flashback)" becomes "1")
|
|
charChapter = charChapter.replace(/\([^)]*\)/g, '');
|
|
charChapter = charChapter.replace(/\D/g, '');
|
|
|
|
// If charChapter is empty, skip the character as it means they don't have a proper page and are just mentioned in the list
|
|
if (!charChapter) {
|
|
return;
|
|
}
|
|
|
|
if (charUrl) {
|
|
charUrl = charUrl.replace('/fr/wiki/', '');
|
|
characters.push({
|
|
name: charName,
|
|
url: charUrl,
|
|
pictureUrl: charpictureUrl || null,
|
|
chapter: charChapter,
|
|
});
|
|
}
|
|
});
|
|
console.log(`Found ${characters.length} characters.`);
|
|
return characters;
|
|
} catch (error) {
|
|
console.error('Error fetching character list:', (error as Error).message);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Fetch character data from fandom using provided URL
|
|
*/
|
|
async function fetchCharacter(
|
|
characterUrl: string,
|
|
characterName: string,
|
|
characterpictureUrl: string | null,
|
|
characterChapter: string
|
|
): Promise<Character | null> {
|
|
try {
|
|
console.log(`Fetching: ${characterName}...`);
|
|
|
|
// Use API to fetch character page
|
|
const apiUrl = `${FANDOM_API_BASE}${characterUrl}`;
|
|
let response = await fetchWithRetry(apiUrl);
|
|
|
|
let jsonData = await response.json() as any;
|
|
|
|
// Use final page name from API (if parse.limks contains one element, it means the original page was a redirect, so we use the the element 0 as the final URL, otherwise we use the original URL)
|
|
let finalCharacterUrl = characterUrl;
|
|
if (jsonData.parse?.links?.length === 1) {
|
|
finalCharacterUrl = jsonData.parse.links[0]['*'];
|
|
// Query the API again with the final URL to get the correct HTML content (in case of redirect)
|
|
response = await fetchWithRetry(`${FANDOM_API_BASE}${finalCharacterUrl}`);
|
|
jsonData = await response.json() as any;
|
|
}
|
|
|
|
const categories = jsonData.parse?.categories || [];
|
|
|
|
// Extract HTML from API response
|
|
const htmlContent = jsonData.parse?.text?.['*'];
|
|
if (!htmlContent) {
|
|
throw new Error('Unable to extract HTML content from API response');
|
|
}
|
|
|
|
const $ = cheerio.load(htmlContent);
|
|
|
|
const name = characterName;
|
|
|
|
// Generate character ID from URL + name combination
|
|
const finalCharacterId = normalizeId(finalCharacterUrl + '_' + name);
|
|
|
|
// Extract gender from JSON categories
|
|
let gender: string | null = null;
|
|
for (const cat of categories) {
|
|
const catName = cat['*'] || '';
|
|
if (catName === 'Personnages_Masculins') {
|
|
gender = 'Male';
|
|
break;
|
|
} else if (catName === 'Personnages_Féminins') {
|
|
gender = 'Female';
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Extract age
|
|
const age = extractAge($);
|
|
|
|
// Extract affiliations
|
|
const affiliations = extractAffiliations($);
|
|
|
|
// Extract epithets
|
|
const epithets = extractEpithets($);
|
|
|
|
// Extract devil fruit
|
|
const devilFruitData = await extractDevilFruit($);
|
|
const devilFruitId = devilFruitData?.devilFruitId || null;
|
|
const devilFruitUrl = devilFruitData?.devilFruitUrl || null;
|
|
|
|
// Extract haki from JSON categories
|
|
let hakiObservation = false;
|
|
let hakiArmament = false;
|
|
let hakiConqueror = false;
|
|
for (const cat of categories) {
|
|
const catName = cat['*'] || '';
|
|
if (catName === 'Utilisateurs_du_Haki_de_l\'observation') {
|
|
hakiObservation = true;
|
|
} else if (catName === 'Utilisateurs_du_Haki_de_l\'armement') {
|
|
hakiArmament = true;
|
|
} else if (catName === 'Utilisateurs_du_Haki_des_rois') {
|
|
hakiConqueror = true;
|
|
}
|
|
}
|
|
|
|
// Extract bounty
|
|
const bounty = extractBounty($);
|
|
|
|
// Extract height
|
|
const height = extractHeight($);
|
|
|
|
// Use chapter from character list, cast to int
|
|
let firstAppearance = parseInt(characterChapter);
|
|
|
|
// Extract origin
|
|
const origin = extractOrigin($);
|
|
|
|
// Extract status
|
|
const status = extractStatus($);
|
|
|
|
// Extract image URL and clean it
|
|
let pictureUrl = characterpictureUrl;
|
|
if (pictureUrl && pictureUrl.includes('Image_Non_Disponible')) {
|
|
pictureUrl = null;
|
|
}
|
|
|
|
return {
|
|
id: finalCharacterId,
|
|
name,
|
|
gender,
|
|
age,
|
|
height,
|
|
origin,
|
|
devilFruitId,
|
|
devilFruitUrl,
|
|
affiliations,
|
|
bounty,
|
|
hakiObservation,
|
|
hakiArmament,
|
|
hakiConqueror,
|
|
epithets,
|
|
firstAppearance,
|
|
status,
|
|
pictureUrl,
|
|
url: finalCharacterUrl
|
|
};
|
|
} catch (error) {
|
|
console.error(`Error fetching ${characterName}:`, (error as Error).message);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
|
|
/**
|
|
* Extract age from infobox
|
|
*/
|
|
function extractAge($: cheerio.CheerioAPI): number | null {
|
|
const div = $('[data-source="âge"] .pi-data-value');
|
|
if (div.length === 0) return null;
|
|
|
|
let text = div.html();
|
|
if (!text) return null;
|
|
|
|
// Remove all sup blocks (citations)
|
|
text = text.replace(/<sup[^>]*>.*?<\/sup>/gi, '');
|
|
|
|
// Get the last element and extract only digits
|
|
const parts = text.split('<br');
|
|
const lastPart = parts[parts.length - 1];
|
|
let cleanText = lastPart.replace(/<[^>]*>/g, '').trim();
|
|
|
|
// Remove content with parentheses
|
|
cleanText = cleanText.replace(/\([^)]*\)/g, '');
|
|
|
|
const digitsOnly = cleanText.replace(/\D/g, '');
|
|
return parseInt(digitsOnly) || null;
|
|
}
|
|
|
|
/**
|
|
* Extract affiliations from infobox
|
|
*/
|
|
function extractAffiliations($: cheerio.CheerioAPI): string[] {
|
|
const div = $('[data-source="affiliation"] .pi-data-value');
|
|
if (div.length === 0) return [];
|
|
|
|
const cleanedDiv = div.clone();
|
|
cleanedDiv.find('sup').remove();
|
|
|
|
let text = cleanedDiv.html();
|
|
if (!text) return [];
|
|
|
|
// Extract all link values
|
|
const linkValues = cleanedDiv.find('a').map((i, el) => $(el).text().trim()).get();
|
|
if (linkValues.length > 0) {
|
|
return linkValues;
|
|
}
|
|
|
|
// Fallback to parsing text
|
|
const cleanText = text.replace(/<[^>]*>/g, '').trim();
|
|
const parts = cleanText.split(/\s*\n\s*|\s*;\s*|\s*,\s*/).filter(Boolean);
|
|
return parts.length > 0 ? parts : [];
|
|
}
|
|
|
|
/**
|
|
* Extract epithets from infobox
|
|
* Epithets are always between double quotes
|
|
*/
|
|
function extractEpithets($: cheerio.CheerioAPI): string[] {
|
|
const div = $('[data-source="épithète"] .pi-data-value');
|
|
if (div.length === 0) return [];
|
|
|
|
const cleanedDiv = div.clone();
|
|
cleanedDiv.find('sup').remove();
|
|
|
|
let text = cleanedDiv.text();
|
|
if (!text) return [];
|
|
|
|
// Extract all text between double quotes (both straight and curly quotes)
|
|
const matches = text.match(/["«"]([^"»"]+)["»"]/g);
|
|
if (!matches) return [];
|
|
|
|
// Remove the quotes and trim
|
|
const epithets = matches.map(match =>
|
|
match.replace(/^["«"]|["»"]$/g, '').trim()
|
|
).filter(Boolean);
|
|
|
|
return epithets;
|
|
}
|
|
|
|
/**
|
|
* Extract devil fruit from infobox
|
|
* Returns both normalized ID and URL
|
|
*/
|
|
async function extractDevilFruit($: cheerio.CheerioAPI): Promise<DevilFruitData | null> {
|
|
const link = $('[data-source="dfnom"] .pi-data-value a').first();
|
|
if (link.length === 0) return null;
|
|
|
|
const href = link.attr('href');
|
|
if (!href || !href.startsWith('/fr/wiki/')) return null;
|
|
|
|
const cleanUrl = href.replace('/fr/wiki/', '');
|
|
|
|
try {
|
|
// Fetch the page via API to follow redirects
|
|
const apiUrl = `${FANDOM_API_BASE}${decodeURIComponent(cleanUrl)}`;
|
|
const response = await fetchWithRetry(apiUrl);
|
|
const jsonData = await response.json() as any;
|
|
|
|
// Use final page name from API (if parse.links contains one element, it means the original page was a redirect, so we use the the element 0 as the final URL, otherwise we use the original URL)
|
|
let finalPath = cleanUrl;
|
|
if (jsonData.parse?.links?.length === 1) {
|
|
finalPath = jsonData.parse.links[0]['*'];
|
|
}
|
|
|
|
|
|
if (finalPath) {
|
|
return {
|
|
devilFruitId: normalizeId(finalPath),
|
|
devilFruitUrl: finalPath
|
|
};
|
|
}
|
|
} catch (error) {
|
|
console.error(`Error fetching devil fruit page: ${(error as Error).message}`);
|
|
}
|
|
|
|
// Fallback to the original href
|
|
return {
|
|
devilFruitId: normalizeId(cleanUrl),
|
|
devilFruitUrl: cleanUrl
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Extract bounty from infobox
|
|
*/
|
|
function extractBounty($: cheerio.CheerioAPI): number | null {
|
|
const div = $('[data-source="prime"] .pi-data-value');
|
|
if (div.length === 0) return 0;
|
|
|
|
let text = div.html();
|
|
if (!text) return 0;
|
|
|
|
// Remove all sup blocks (citations)
|
|
text = text.replace(/<sup[^>]*>.*?<\/sup>/gi, '');
|
|
|
|
// Extract the first value before any <br> tag
|
|
const firstValue = text.split('<br')[0].trim();
|
|
let cleanText = firstValue.replace(/<[^>]*>/g, '').trim();
|
|
|
|
// Check if cleanText contains digits
|
|
if (!/\d/.test(cleanText)) {
|
|
// If no digits, try second value after <br>
|
|
const secondValue = text.split('<br>')[1];
|
|
if (secondValue) {
|
|
cleanText = secondValue.replace(/<[^>]*>/g, '').trim();
|
|
}
|
|
}
|
|
|
|
// Remove all non-digits
|
|
cleanText = cleanText.replace(/\D/g, '');
|
|
|
|
return cleanText ? parseInt(cleanText) : 0;
|
|
}
|
|
|
|
/**
|
|
* Extract height from infobox
|
|
*/
|
|
function extractHeight($: cheerio.CheerioAPI): number | null {
|
|
const div = $('[data-source="taille"] .pi-data-value');
|
|
if (div.length === 0) return null;
|
|
|
|
let text = div.html();
|
|
if (!text) return null;
|
|
|
|
// Remove all sup blocks (citations)
|
|
text = text.replace(/<sup[^>]*>.*?<\/sup>/gi, '');
|
|
|
|
// Check if there's a <p> tag - if yes, use content from <p>
|
|
let content;
|
|
const pMatch = text.match(/<p[^>]*>(.*?)<\/p>/i);
|
|
if (pMatch) {
|
|
// Extract content from the <p> tag
|
|
content = pMatch[1];
|
|
} else {
|
|
// Use the last value method (after any <br> tag)
|
|
content = text.split('<br>').pop();
|
|
}
|
|
|
|
let cleanText = (content || '').replace(/<[^>]*>/g, '').trim();
|
|
|
|
// Remove content with parentheses
|
|
cleanText = cleanText.replace(/\([^)]*\)/g, '');
|
|
|
|
// Normalize units for meters or centimeters
|
|
const normalized = cleanText.toLowerCase().replace(/\s/g, '');
|
|
if (normalized.includes('cm')) {
|
|
const digitsOnly = normalized.replace(/\D/g, '');
|
|
return parseFloat(digitsOnly) || null;
|
|
}
|
|
|
|
if (normalized.includes('m')) {
|
|
const parts = normalized.split('m').filter(Boolean);
|
|
return parts.length > 0 ? parseFloat(parts.join('.')) : null;
|
|
}
|
|
|
|
return normalized.length > 0 ? parseFloat(normalized.replace(/\D/g, '')) : null;
|
|
}
|
|
|
|
/**
|
|
* Extract origin from infobox
|
|
*/
|
|
function extractOrigin($: cheerio.CheerioAPI): string | null {
|
|
const div = $('[data-source="origine"] .pi-data-value');
|
|
if (div.length === 0) return null;
|
|
|
|
let text = div.html();
|
|
if (!text) return null;
|
|
|
|
// Remove all sup blocks (citations)
|
|
text = text.replace(/<sup[^>]*>.*?<\/sup>/gi, '');
|
|
|
|
// Extract the first value before any <br> tag
|
|
const firstValue = text.split('<br')[0].trim();
|
|
let cleanText = firstValue.replace(/<[^>]*>/g, '').trim();
|
|
|
|
// Remove content with parentheses
|
|
cleanText = cleanText.replace(/\([^)]*\)/g, '').trim();
|
|
|
|
return cleanText || null;
|
|
}
|
|
|
|
/**
|
|
* Extract status from infobox
|
|
*/
|
|
function extractStatus($: cheerio.CheerioAPI): string {
|
|
const div = $('[data-source="statut"] .pi-data-value');
|
|
if (div.length === 0) return 'Alive';
|
|
|
|
const statusText = div.text().trim().toLowerCase();
|
|
|
|
if (statusText.includes('vivant')) {
|
|
return 'Alive';
|
|
} else if (statusText.includes('décédé')) {
|
|
return 'Dead';
|
|
}
|
|
|
|
return 'Alive';
|
|
}
|
|
|
|
|
|
/**
|
|
* Save data to JSON
|
|
*/
|
|
async function saveToJSON(characters: Character[]): Promise<void> {
|
|
const filepath = `${OUTPUT_DIR}/characters.json`;
|
|
fs.writeFileSync(filepath, JSON.stringify(characters, null, 2));
|
|
console.log(`✓ Saved to ${filepath}`);
|
|
}
|
|
|
|
/**
|
|
* Save data to CSV
|
|
*/
|
|
async function saveToCSV(characters: Character[]): Promise<void> {
|
|
const filepath = `${OUTPUT_DIR}/characters.csv`;
|
|
const csvWriter = createObjectCsvWriter({
|
|
path: filepath,
|
|
header: [
|
|
{ id: 'id', title: 'ID' },
|
|
{ id: 'name', title: 'Name' },
|
|
{ id: 'gender', title: 'Gender' },
|
|
{ id: 'age', title: 'Age' },
|
|
{ id: 'height', title: 'Height' },
|
|
{ id: 'origin', title: 'Origin' },
|
|
{ id: 'status', title: 'Status' },
|
|
{ id: 'epithets', title: 'Epithets' },
|
|
{ id: 'devilFruitId', title: 'Devil Fruit ID' },
|
|
{ id: 'affiliations', title: 'Affiliations' },
|
|
{ id: 'bounty', title: 'Bounty' },
|
|
{ id: 'hakiObservation', title: 'Haki Observation' },
|
|
{ id: 'hakiArmament', title: 'Haki Armament' },
|
|
{ id: 'hakiConqueror', title: 'Haki Conqueror' },
|
|
{ id: 'firstAppearance', title: 'First Appearance' },
|
|
{ id: 'arcId', title: 'Arc ID' },
|
|
{ id: 'pictureUrl', title: 'Image URL' },
|
|
{ id: 'url', title: 'Fandom URL' }
|
|
],
|
|
});
|
|
|
|
const records = characters
|
|
.filter((c) => c !== null)
|
|
.map((c) => ({
|
|
id: c.id || '',
|
|
name: c.name || '',
|
|
gender: c.gender || '',
|
|
age: c.age || '',
|
|
height: c.height || '',
|
|
origin: c.origin || '',
|
|
status: c.status || '',
|
|
epithets: Array.isArray(c.epithets) ? c.epithets.join(', ') : (c.epithets || ''),
|
|
devilFruitId: c.devilFruitId || '',
|
|
affiliations: Array.isArray(c.affiliations) ? c.affiliations.join(', ') : (c.affiliations || ''),
|
|
bounty: c.bounty ?? 0,
|
|
hakiObservation: c.hakiObservation ? 1 : 0,
|
|
hakiArmament: c.hakiArmament ? 1 : 0,
|
|
hakiConqueror: c.hakiConqueror ? 1 : 0,
|
|
firstAppearance: c.firstAppearance || '',
|
|
arcId: c.arcId || '',
|
|
pictureUrl: c.pictureUrl || '',
|
|
url: c.url || ''
|
|
}));
|
|
|
|
await csvWriter.writeRecords(records);
|
|
console.log(`✓ Saved to ${filepath}`);
|
|
}
|
|
|
|
/**
|
|
* Fetch devil fruit data from fandom using provided URL
|
|
*/
|
|
async function fetchDevilFruit(devilFruitUrl: string, devilFruitId: string): Promise<DevilFruit | null> {
|
|
try {
|
|
console.log(`Fetching devil fruit: ${devilFruitUrl}...`);
|
|
|
|
// Use API to fetch devil fruit page
|
|
const apiUrl = `${FANDOM_API_BASE}${devilFruitUrl}`;
|
|
const response = await fetchWithRetry(apiUrl);
|
|
const jsonData = await response.json() as any;
|
|
|
|
// Extract HTML from API response
|
|
const htmlContent = jsonData.parse?.text?.['*'];
|
|
if (!htmlContent) {
|
|
throw new Error('Unable to extract HTML content from API response');
|
|
}
|
|
|
|
const name = jsonData.parse?.title || devilFruitId.replace(/_/g, ' ');
|
|
|
|
let type: string | null = null;
|
|
// Determine type based on categories (if categories contain "Paramecia", "Zoan", "Logia" or "Smile")
|
|
if (jsonData.parse?.categories) {
|
|
const categories = jsonData.parse.categories
|
|
.map((cat: any) => String(cat['*'] || '').toLowerCase());
|
|
|
|
if (categories.some((category: string) => category.includes('paramecia'))) {
|
|
type = 'Paramecia';
|
|
} else if (categories.some((category: string) => category.includes('zoan'))) {
|
|
type = 'Zoan';
|
|
} else if (categories.some((category: string) => category.includes('logia'))) {
|
|
type = 'Logia';
|
|
} else if (categories.some((category: string) => category.includes('smile'))) {
|
|
type = 'Smile';
|
|
}
|
|
}
|
|
|
|
return {
|
|
id: devilFruitId,
|
|
name,
|
|
type,
|
|
url: devilFruitUrl
|
|
};
|
|
} catch (error) {
|
|
console.error(`Error fetching devil fruit ${devilFruitUrl}:`, (error as Error).message);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Save devil fruits to JSON
|
|
*/
|
|
async function saveDevilFruitsToJSON(devilFruits: DevilFruit[]): Promise<void> {
|
|
const filepath = `${OUTPUT_DIR}/devil-fruits.json`;
|
|
fs.writeFileSync(filepath, JSON.stringify(devilFruits, null, 2));
|
|
console.log(`✓ Saved to ${filepath}`);
|
|
}
|
|
|
|
/**
|
|
* Save devil fruits to CSV
|
|
*/
|
|
async function saveDevilFruitsToCSV(devilFruits: DevilFruit[]): Promise<void> {
|
|
const filepath = `${OUTPUT_DIR}/devil-fruits.csv`;
|
|
const csvWriter = createObjectCsvWriter({
|
|
path: filepath,
|
|
header: [
|
|
{ id: 'id', title: 'ID' },
|
|
{ id: 'name', title: 'Name' },
|
|
{ id: 'type', title: 'Type' },
|
|
{ id: 'url', title: 'URL' }
|
|
],
|
|
});
|
|
|
|
const records = devilFruits
|
|
.filter((df) => df !== null)
|
|
.map((df) => ({
|
|
id: df.id || '',
|
|
name: df.name || '',
|
|
type: df.type || '',
|
|
url: df.url || ''
|
|
}));
|
|
|
|
await csvWriter.writeRecords(records);
|
|
console.log(`✓ Saved to ${filepath}`);
|
|
}
|
|
|
|
/**
|
|
* Main execution
|
|
*/
|
|
async function main(): Promise<void> {
|
|
const format = process.argv[2] || 'all'; // json, csv, or all
|
|
|
|
console.log(`\nOne Piece Scraper - Mode: ${format}\n`);
|
|
|
|
// Step 1: Scraping Arcs
|
|
console.log('=== Step 1: Scraping Arcs ===\n');
|
|
const arcsList = await fetchAllArcs();
|
|
|
|
if (arcsList.length > 0) {
|
|
// Display arcs in table format
|
|
arcsList.forEach((arc) => {
|
|
console.table({
|
|
ID: arc.id,
|
|
Name: arc.name,
|
|
StartChapter: arc.startChapter,
|
|
EndChapter: arc.endChapter || 'Ongoing',
|
|
URL: arc.url
|
|
});
|
|
});
|
|
|
|
console.log(`\n✓ Found ${arcsList.length} arcs\n`);
|
|
|
|
if (format === 'json' || format === 'all') {
|
|
await saveArcsToJSON(arcsList);
|
|
}
|
|
if (format === 'csv' || format === 'all') {
|
|
await saveArcsToCSV(arcsList);
|
|
}
|
|
} else {
|
|
console.warn('No arcs found, continuing...\n');
|
|
}
|
|
|
|
// Step 2: Scraping Characters
|
|
console.log('=== Step 1: Scraping Characters ===\n');
|
|
const characterList = await fetchAllCharactersUrl();
|
|
|
|
if (characterList.length === 0) {
|
|
console.error('No characters found. Exiting.');
|
|
return;
|
|
}
|
|
|
|
const characters: Character[] = [];
|
|
const devilFruitUrls = new Set<string>();
|
|
let failedCharacters: CharacterListItem[] = [...characterList];
|
|
|
|
while (failedCharacters.length > 0) {
|
|
const nextFailedCharacters: CharacterListItem[] = [];
|
|
console.log(`\nFetching ${failedCharacters.length} characters...`);
|
|
|
|
for (let i = 0; i < failedCharacters.length; i += CHARACTER_FETCH_CONCURRENCY) {
|
|
const batch = failedCharacters.slice(i, i + CHARACTER_FETCH_CONCURRENCY);
|
|
const batchResults = await Promise.all(
|
|
batch.map(async (char) => {
|
|
const data = await fetchCharacter(char.url, char.name, char.pictureUrl, char.chapter);
|
|
return { char, data };
|
|
})
|
|
);
|
|
|
|
for (const { char, data } of batchResults) {
|
|
if (data) {
|
|
console.table({
|
|
ID: data.id,
|
|
Name: data.name,
|
|
Gender: data.gender,
|
|
Age: data.age,
|
|
Status: data.status,
|
|
Epithets: data.epithets.join(', '),
|
|
Affiliations: data.affiliations.join(', '),
|
|
DevilFruitId: data.devilFruitId,
|
|
DevilFruitUrl: data.devilFruitUrl,
|
|
HakiObservation: data.hakiObservation ? 'Yes' : 'No',
|
|
HakiArmament: data.hakiArmament ? 'Yes' : 'No',
|
|
HakiConqueror: data.hakiConqueror ? 'Yes' : 'No',
|
|
Height: data.height,
|
|
Bounty: data.bounty,
|
|
Origin: data.origin,
|
|
FirstAppearance: data.firstAppearance,
|
|
pictureUrl: data.pictureUrl,
|
|
FandomURL: data.url
|
|
});
|
|
|
|
if (data.devilFruitUrl) {
|
|
devilFruitUrls.add(data.devilFruitUrl);
|
|
}
|
|
|
|
if (data.firstAppearance) {
|
|
const arc = arcsList.find(a => a.startChapter <= data.firstAppearance && (a.endChapter === null || a.endChapter >= data.firstAppearance));
|
|
if (arc) {
|
|
data.arcId = arc.id;
|
|
}
|
|
}
|
|
|
|
characters.push(data);
|
|
} else {
|
|
nextFailedCharacters.push(char);
|
|
}
|
|
}
|
|
}
|
|
|
|
failedCharacters = nextFailedCharacters;
|
|
if (failedCharacters.length > 0) {
|
|
console.log(`⚠️ ${failedCharacters.length} characters failed. Retrying...`);
|
|
}
|
|
}
|
|
|
|
console.log(`\n✓ Scraped ${characters.length} characters\n`);
|
|
console.log(`✓ Found ${devilFruitUrls.size} unique devil fruits\n`);
|
|
|
|
// Step 3: Scraping Devil Fruits
|
|
console.log('=== Step 2: Scraping Devil Fruits ===\n');
|
|
|
|
if (devilFruitUrls.size === 0) {
|
|
console.warn('No devil fruits found from characters, skipping...\n');
|
|
} else {
|
|
const devilFruits: DevilFruit[] = [];
|
|
const devilFruitUrlArray = Array.from(devilFruitUrls);
|
|
|
|
for (let i = 0; i < devilFruitUrlArray.length; i++) {
|
|
const url = devilFruitUrlArray[i];
|
|
const data = await fetchDevilFruit(url, normalizeId(url));
|
|
|
|
if (data) {
|
|
console.table({
|
|
ID: data.id,
|
|
Name: data.name,
|
|
Type: data.type,
|
|
URL: data.url
|
|
});
|
|
|
|
devilFruits.push(data);
|
|
}
|
|
}
|
|
|
|
console.log(`\n✓ Scraped ${devilFruits.length} devil fruits\n`);
|
|
|
|
if (format === 'json' || format === 'all') {
|
|
await saveDevilFruitsToJSON(devilFruits);
|
|
}
|
|
if (format === 'csv' || format === 'all') {
|
|
await saveDevilFruitsToCSV(devilFruits);
|
|
}
|
|
|
|
// Update characters with normalized devil fruit IDs
|
|
const devilFruitMap = new Map<string, string>(devilFruits.map(df => [df.id, df.id]));
|
|
characters.forEach(char => {
|
|
if (char.devilFruitUrl) {
|
|
const normalizedId = normalizeId(char.devilFruitUrl);
|
|
char.devilFruitId = devilFruitMap.get(normalizedId) || normalizedId;
|
|
}
|
|
});
|
|
}
|
|
|
|
// Save characters after devil fruit IDs are updated
|
|
if (format === 'json' || format === 'all') {
|
|
await saveToJSON(characters);
|
|
}
|
|
if (format === 'csv' || format === 'all') {
|
|
await saveToCSV(characters);
|
|
}
|
|
|
|
console.log('\n✓ Done!\n');
|
|
}
|
|
|
|
main().catch(console.error);
|