From a77f84f595f2a88ceefc59b15fd59f86f20ddb6f Mon Sep 17 00:00:00 2001 From: whidix Date: Mon, 25 May 2026 19:03:12 +0200 Subject: [PATCH] feat: enhance character and devil fruit data structures with additional fields and parsing functions --- scripts/scrape-onepiece.ts | 235 ++++++++++++++++++++----------------- 1 file changed, 130 insertions(+), 105 deletions(-) diff --git a/scripts/scrape-onepiece.ts b/scripts/scrape-onepiece.ts index e0bb374..510b657 100644 --- a/scripts/scrape-onepiece.ts +++ b/scripts/scrape-onepiece.ts @@ -22,6 +22,7 @@ interface Character { origin: string | null; frOrigin: string | null; devilFruitId: string | null; + devilFruitName: string | null; devilFruitUrl: string | null; affiliation: string | null; frAffiliation: string | null; @@ -47,6 +48,7 @@ interface CharacterListItem { interface DevilFruitData { devilFruitId: string; + devilFruitTitle: string; devilFruitUrl: string; } @@ -66,6 +68,17 @@ const MAX_RETRIES = 0; // Set to 0 to disable retries, can be increased if neede const INITIAL_RETRY_DELAY = 1000; const FETCH_CONCURRENCY = 50; +type WikiParseData = { + parse?: { + text?: { + ['*']?: string; + }; + langlinks?: Array<{ lang: string; ['*']: string; url: string }>; + title?: string; + categories?: Array<{ ['*']: string }>; + }; +}; + // Create output directory if (!fs.existsSync(OUTPUT_DIR)) { fs.mkdirSync(OUTPUT_DIR, { recursive: true }); @@ -122,6 +135,44 @@ async function fetchWithRetry( } } +function getParsedHtml(jsonData: WikiParseData): string { + const htmlContent = jsonData.parse?.text?.['*']; + if (!htmlContent) { + throw new Error('Unable to extract HTML content from API response'); + } + + return htmlContent; +} + +function loadWikiPage(jsonData: WikiParseData): cheerio.CheerioAPI { + return cheerio.load(getParsedHtml(jsonData)); +} + +function getFrenchLink(links: Array<{ lang: string; ['*']: string; url: string }>): { url: string } | null { + const frenchLink = links.find((link) => link.lang === 'fr'); + return frenchLink ? { url: frenchLink.url } : null; +} + +function getFrenchLabel(links: Array<{ lang: string; ['*']: string; url: string }>): string | null { + return links.find((link) => link.lang === 'fr')?.['*'] || null; +} + +function removeSupBlocks(html: string): string { + return html.replace(/]*>.*?<\/sup>/gi, ''); +} + +function stripHtml(text: string): string { + return text.replace(/<[^>]*>/g, '').trim(); +} + +function firstTextLine(html: string): string { + return html.split(' { console.log('Fetching arcs list via API...'); const response = await fetchWithRetry(apiUrl); const jsonData = await response.json(); - - // Extract HTML from API response - const htmlContent = jsonData.parse?.text?.['*']; - if (!htmlContent) { - throw new Error('Unable to extract HTML content from API response'); - } - - const $ = cheerio.load(htmlContent); + const $ = loadWikiPage(jsonData); const arcs: Arc[] = []; const seenArcUrls = new Set(); @@ -204,10 +248,7 @@ async function fetchAllArcs(): Promise { // Query the href page via API to get the correct HTML content (in case of redirect) and extract the French name from there const arcResponse = await fetchWithRetry(`${FANDOM_API_BASE}${cleanUrl}`); const arcJsonData = await arcResponse.json(); - let frArcName: string | null = - arcJsonData.parse?.langlinks.find( - (link: { lang: string; ['*']: string }) => link.lang === 'fr' - )?.['*'] || null; + let frArcName: string | null = getFrenchLabel(arcJsonData.parse?.langlinks || []); // Remove "Arc" suffix from French name if present to keep it consistent with English names (e.g. "Arc de Luffy" becomes "Luffy") if (frArcName && /\bArc\b/i.test(frArcName)) { @@ -283,14 +324,7 @@ async function fetchAllCharacters(arcsList: Arc[]): Promise { console.log('Fetching character list via API...'); const response = await fetchWithRetry(`${FANDOM_API_BASE}List_of_Canon_Characters`); const jsonData = await response.json(); - - // Extract HTML from API response - const htmlContent = jsonData.parse?.text?.['*']; - if (!htmlContent) { - throw new Error('Unable to extract HTML content from API response'); - } - - const $ = cheerio.load(htmlContent); + const $ = loadWikiPage(jsonData); const characterList: CharacterListItem[] = []; $('table.fandom-table tbody tr').each((index, element) => { if (index === 0) return; // Skip header row @@ -326,24 +360,22 @@ async function fetchAllCharacters(arcsList: Arc[]): Promise { return []; } console.log(`Found ${characterList.length} characters.`); - - // Fetch the french character list to get the picture URLs - console.log('Fetching French character list via API...'); - const frResponse = await fetchWithRetry(`${FR_FANDOM_API_BASE}Liste_des_Personnages_Canon`); - const frJsonData = await frResponse.json(); - - // Create a map of character name to picture URL from the French list - const frHtmlContent = frJsonData.parse?.text?.['*']; - const fr$ = cheerio.load(frHtmlContent); - const frCharacterPictureMap: Record = {}; - fr$('table.wikitable tbody tr').each((index, element) => { - if (index === 0) return; // Skip header row - const charName = fr$(element).find('td:nth-child(2) a').text().trim(); - const pictureUrl = fr$(element).find('td:nth-child(1) img').attr('data-src') || fr$(element).find('td:nth-child(1) img').attr('src') || null; - if (charName && pictureUrl) { - frCharacterPictureMap[charName] = pictureUrl; - } - }); + console.log('Fetching French character list via API...'); + const frResponse = await fetchWithRetry(`${FR_FANDOM_API_BASE}Liste_des_Personnages_Canon`); + const frJsonData = await frResponse.json(); + const fr$ = loadWikiPage(frJsonData); + const frCharacterPictureMap: Record = {}; + fr$('table.wikitable tbody tr').each((index, element) => { + if (index === 0) return; // Skip header row + const charName = fr$(element).find('td:nth-child(2) a').text().trim(); + const pictureUrl = + fr$(element).find('td:nth-child(1) img').attr('data-src') || + fr$(element).find('td:nth-child(1) img').attr('src') || + null; + if (charName && pictureUrl) { + frCharacterPictureMap[charName] = pictureUrl; + } + }); const characters: Character[] = []; let failedCharacters: CharacterListItem[] = [...characterList]; @@ -423,14 +455,7 @@ async function fetchCharacter( const jsonData = await response.json(); const categories = jsonData.parse?.categories || []; - - // Extract HTML from API response - const htmlContent = jsonData.parse?.text?.['*']; - if (!htmlContent) { - throw new Error('Unable to extract HTML content from API response'); - } - - const $ = cheerio.load(htmlContent); + const $ = loadWikiPage(jsonData); const name = characterName; @@ -462,29 +487,19 @@ async function fetchCharacter( // Extract devil fruit const devilFruitData = await extractDevilFruit($); const devilFruitId = devilFruitData?.devilFruitId || null; + const devilFruitName = devilFruitData?.devilFruitTitle || null; const devilFruitUrl = devilFruitData?.devilFruitUrl || null; // Extract haki from JSON categories let hakiObservation = false; let hakiArmament = false; let hakiConqueror = false; - for (const cat of categories) { - const catName = cat['*'] || ''; - if (catName === 'Observation_Haki_Users') { - hakiObservation = true; - } else if (catName === 'Armament_Haki_Users') { - hakiArmament = true; - } else if (catName === 'Supreme_King_Haki_Users') { - hakiConqueror = true; - } - } // Extract bounty const bounty = extractBounty($); // Extract height const height = extractHeight($); - // Use chapter from character list, cast to int const firstAppearance = characterChapter; @@ -505,31 +520,26 @@ async function fetchCharacter( } arcId = arc.id; - const frLink = getFrLink(jsonData.parse?.langlinks || []); + const frLink = getFrenchLink(jsonData.parse?.langlinks || []); const frUrl = frLink ? frLink.url.replace('https://onepiece.fandom.com/fr/wiki/', '') : null; const frjsonData = frUrl ? await fetchWithRetry(`${FR_FANDOM_API_BASE}${frUrl}`).then((res) => res.json()) : null; + const frPage: cheerio.CheerioAPI | null = frjsonData ? loadWikiPage(frjsonData) : null; let frName = frjsonData?.parse?.title || null; - const frAffiliation = frjsonData - ? await extractAffiliations(cheerio.load(frjsonData.parse?.text?.['*'] || ''), 'fr') - : null; + const frAffiliation = frPage ? await extractAffiliations(frPage, 'fr') : null; - const frEpithets = frjsonData - ? extractEpithets(cheerio.load(frjsonData.parse?.text?.['*'] || '')) - : null; + const frEpithets = frPage ? extractEpithets(frPage) : null; - const frOrigin = frjsonData - ? extractOrigin(cheerio.load(frjsonData.parse?.text?.['*'] || '')) - : null; + const frOrigin = frPage ? extractOrigin(frPage) : null; if (name !== jsonData.parse?.title) { frName = name; } - const pictureUrl = frCharacterPictureMap[frName || ''] || null; + const pictureUrl = frCharacterPictureMap[frName || ''] || null; return { id: finalCharacterId, @@ -541,6 +551,7 @@ async function fetchCharacter( origin, frOrigin, devilFruitId, + devilFruitName, devilFruitUrl, affiliation, frAffiliation, @@ -573,16 +584,15 @@ function extractAge($: cheerio.CheerioAPI): number | null { let text = div.html(); if (!text) return null; - // Remove all sup blocks (citations) - text = text.replace(/]*>.*?<\/sup>/gi, ''); + text = removeSupBlocks(text); // Get the last element and extract only digits const parts = text.split(']*>/g, '').trim(); + let cleanText = stripHtml(lastPart); // Remove content with parentheses - cleanText = cleanText.replace(/\([^)]*\)/g, ''); + cleanText = removeParentheticalNotes(cleanText); const digitsOnly = cleanText.replace(/\D/g, ''); return parseInt(digitsOnly) || null; @@ -629,7 +639,7 @@ async function extractAffiliations($: cheerio.CheerioAPI, lang: string): Promise } // Fallback to parsing text - const cleanText = text.replace(/<[^>]*>/g, '').trim(); + const cleanText = stripHtml(text); const parts = cleanText.split(/\s*\n\s*|\s*;\s*|\s*,\s*/).filter(Boolean); return parts.length > 0 ? parts[0] : null; } @@ -683,22 +693,30 @@ function extractEpithets($: cheerio.CheerioAPI): string[] { * Returns both normalized ID and URL */ async function extractDevilFruit($: cheerio.CheerioAPI): Promise { - const link = $('[data-source="dfname"] .pi-data-value a').first(); + // dfname or dfename are used as data-source for devil fruit in the infobox, we check both to be safe + const fruit = $('[data-source="dfname"] .pi-data-value, [data-source="dfename"] .pi-data-value').first(); + const link = fruit.find('a').first(); if (link.length === 0) return null; + let fruitTitle = fruit.text().trim().toLowerCase().includes('smile') ? fruit.text().trim() : link.text().trim(); + if (!fruitTitle) return null; + const href = link.attr('href'); if (!href || !href.startsWith('/wiki/')) return null; const cleanUrl = href.replace('/wiki/', ''); - // Query the devil fruit page via API to get the correct HTML content (in case of redirect) and extract the type from there - const dfResponse = await fetchWithRetry(`${FANDOM_API_BASE}${cleanUrl}`); - const dfJsonData = await dfResponse.json(); - const fruitTitle = dfJsonData.parse?.title || ''; + const dfResponse = await fetchWithRetry(`${FANDOM_API_BASE}${cleanUrl}`); + const dfJsonData = await dfResponse.json(); + + if (!fruitTitle.toLowerCase().includes('smile')) { + fruitTitle = dfJsonData.parse?.title || ''; + } return { devilFruitId: normalizeId(fruitTitle), - devilFruitUrl: fruitTitle + devilFruitTitle: fruitTitle, + devilFruitUrl: cleanUrl }; } @@ -737,8 +755,7 @@ function extractHeight($: cheerio.CheerioAPI): number | null { let text = div.html(); if (!text) return null; - // Remove all sup blocks (citations) - text = text.replace(/]*>.*?<\/sup>/gi, ''); + text = removeSupBlocks(text); // Convert line breaks to new lines so we can reliably pick the latest value. const textWithNewLines = text.replace(//gi, '\n'); @@ -786,15 +803,14 @@ function extractOrigin($: cheerio.CheerioAPI): string | null { let text = div.html(); if (!text) return null; - // Remove all sup blocks (citations) - text = text.replace(/]*>.*?<\/sup>/gi, ''); + text = removeSupBlocks(text); // Extract the first value before any
tag - const firstValue = text.split(']*>/g, '').trim(); + const firstValue = firstTextLine(text); + let cleanText = stripHtml(firstValue); // Remove content with parentheses - cleanText = cleanText.replace(/\([^)]*\)/g, '').trim(); + cleanText = removeParentheticalNotes(cleanText); return cleanText || null; } @@ -890,6 +906,7 @@ async function saveToCSV(characters: Character[]): Promise { */ async function fetchDevilFruit( devilFruitUrl: string, + devilFruitTitle: string, devilFruitId: string ): Promise { try { @@ -903,13 +920,14 @@ async function fetchDevilFruit( // Extract HTML from API response const htmlContent = jsonData.parse?.text?.['*']; if (!htmlContent) { + // Error jsonData for debugging throw new Error('Unable to extract HTML content from API response'); } const name = jsonData.parse?.title || devilFruitId.replace(/_/g, ' '); let type: string | null = null; - // Determine type based on categories (if categories contain "Paramecia", "Zoan", "Logia" or "Smile") + // Determine type based on categories (if categories contain "Paramecia", "Zoan", "Logia") if (jsonData.parse?.categories) { const categories = jsonData.parse.categories.map((cat: { ['*']: string }) => String(cat['*'] || '').toLowerCase() @@ -921,14 +939,12 @@ async function fetchDevilFruit( type = 'Zoan'; } else if (categories.some((category: string) => category.includes('logia'))) { type = 'Logia'; - } else if (categories.some((category: string) => category.includes('smile'))) { - type = 'Smile'; } } return { id: devilFruitId, - name, + name: devilFruitTitle.toLowerCase().includes('smile') ? devilFruitTitle : name, type, url: devilFruitUrl }; @@ -1021,26 +1037,35 @@ async function main(): Promise { return; } - const devilFruitUrls = new Set( - characters.filter((c) => c.devilFruitUrl).map((c) => c.devilFruitUrl!) + const devilFruitEntries = Array.from( + new Map( + characters + .filter((c) => c.devilFruitUrl && c.devilFruitName) + .map((c) => { + const name = c.devilFruitName!; + const url = c.devilFruitUrl!; + const id = normalizeId(name); + return [`${normalizeId(url)}::${id}`, { url, name, id }] as const; + }) + ) + .values() ); - console.log(`✓ Found ${devilFruitUrls.size} unique devil fruits\n`); + console.log(`✓ Found ${devilFruitEntries.length} unique devil fruits\n`); // Step 3: Scraping Devil Fruits console.log('=== Step 2: Scraping Devil Fruits ===\n'); - if (devilFruitUrls.size === 0) { + if (devilFruitEntries.length === 0) { console.warn('No devil fruits found from characters, skipping...\n'); } else { const devilFruits: DevilFruit[] = []; - const devilFruitUrlArray = Array.from(devilFruitUrls); - for (let i = 0; i < devilFruitUrlArray.length; i += FETCH_CONCURRENCY) { - const batch = devilFruitUrlArray.slice(i, i + FETCH_CONCURRENCY); + for (let i = 0; i < devilFruitEntries.length; i += FETCH_CONCURRENCY) { + const batch = devilFruitEntries.slice(i, i + FETCH_CONCURRENCY); const batchResults = await Promise.all( - batch.map(async (url) => { - const data = await fetchDevilFruit(url, normalizeId(url)); - return { url, data }; + batch.map(async (entry) => { + const data = await fetchDevilFruit(entry.url, entry.name, entry.id); + return { entry, data }; }) ); @@ -1067,12 +1092,12 @@ async function main(): Promise { await saveDevilFruitsToCSV(devilFruits); } - // Update characters with normalized devil fruit IDs - const devilFruitMap = new Map(devilFruits.map((df) => [df.id, df.id])); + // Update characters with normalized devil fruit IDs derived from the character-page fruit name. characters.forEach((char) => { - if (char.devilFruitUrl) { - const normalizedId = normalizeId(char.devilFruitUrl); - char.devilFruitId = devilFruitMap.get(normalizedId) || normalizedId; + if (char.devilFruitName) { + char.devilFruitId = normalizeId(char.devilFruitName); + } else if (char.devilFruitUrl) { + char.devilFruitId = normalizeId(char.devilFruitUrl); } }); }