Whidix/OnePieceDle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: enhance character and devil fruit data structures with additional fields and parsing functions
Build Docker Image / build (push) Successful in 2m10s
Details

Browse Source
This commit is contained in:
Whidix
2026-05-25 19:03:12 +02:00
parent ef6bf9862e
commit a77f84f595
1 changed files with 130 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/scrape-onepiece.ts
+116 -91
View File
@@ -22,6 +22,7 @@ interface Character {
origin: string | null; origin: string | null;
frOrigin: string | null; frOrigin: string | null;
devilFruitId: string | null; devilFruitId: string | null;
devilFruitName: string | null;
devilFruitUrl: string | null; devilFruitUrl: string | null;
affiliation: string | null; affiliation: string | null;
frAffiliation: string | null; frAffiliation: string | null;
@@ -47,6 +48,7 @@ interface CharacterListItem {
interface DevilFruitData { interface DevilFruitData {
devilFruitId: string; devilFruitId: string;
devilFruitTitle: string;
devilFruitUrl: string; devilFruitUrl: string;
} }
@@ -66,6 +68,17 @@ const MAX_RETRIES = 0; // Set to 0 to disable retries, can be increased if neede
const INITIAL_RETRY_DELAY = 1000; const INITIAL_RETRY_DELAY = 1000;
const FETCH_CONCURRENCY = 50; const FETCH_CONCURRENCY = 50;
type WikiParseData = {
parse?: {
text?: {
['*']?: string;
};
langlinks?: Array<{ lang: string; ['*']: string; url: string }>;
title?: string;
categories?: Array<{ ['*']: string }>;
};
};
// Create output directory // Create output directory
if (!fs.existsSync(OUTPUT_DIR)) { if (!fs.existsSync(OUTPUT_DIR)) {
fs.mkdirSync(OUTPUT_DIR, { recursive: true }); fs.mkdirSync(OUTPUT_DIR, { recursive: true });
@@ -122,6 +135,44 @@ async function fetchWithRetry(
} }
} }
function getParsedHtml(jsonData: WikiParseData): string {
const htmlContent = jsonData.parse?.text?.['*'];
if (!htmlContent) {
throw new Error('Unable to extract HTML content from API response');
}
return htmlContent;
}
function loadWikiPage(jsonData: WikiParseData): cheerio.CheerioAPI {
return cheerio.load(getParsedHtml(jsonData));
}
function getFrenchLink(links: Array<{ lang: string; ['*']: string; url: string }>): { url: string } | null {
const frenchLink = links.find((link) => link.lang === 'fr');
return frenchLink ? { url: frenchLink.url } : null;
}
function getFrenchLabel(links: Array<{ lang: string; ['*']: string; url: string }>): string | null {
return links.find((link) => link.lang === 'fr')?.['*'] || null;
}
function removeSupBlocks(html: string): string {
return html.replace(/<sup[^>]*>.*?<\/sup>/gi, '');
}
function stripHtml(text: string): string {
return text.replace(/<[^>]*>/g, '').trim();
}
function firstTextLine(html: string): string {
return html.split('<br')[0].trim();
}
function removeParentheticalNotes(text: string): string {
return text.replace(/\([^)]*\)/g, '').trim();
}
/** /**
* Get the French link from the API response links array * Get the French link from the API response links array
*/ */
@@ -154,14 +205,7 @@ async function fetchAllArcs(): Promise<Arc[]> {
console.log('Fetching arcs list via API...'); console.log('Fetching arcs list via API...');
const response = await fetchWithRetry(apiUrl); const response = await fetchWithRetry(apiUrl);
const jsonData = await response.json(); const jsonData = await response.json();
const $ = loadWikiPage(jsonData);
// Extract HTML from API response
const htmlContent = jsonData.parse?.text?.['*'];
if (!htmlContent) {
throw new Error('Unable to extract HTML content from API response');
}
const $ = cheerio.load(htmlContent);
const arcs: Arc[] = []; const arcs: Arc[] = [];
const seenArcUrls = new Set<string>(); const seenArcUrls = new Set<string>();
@@ -204,10 +248,7 @@ async function fetchAllArcs(): Promise<Arc[]> {
// Query the href page via API to get the correct HTML content (in case of redirect) and extract the French name from there // Query the href page via API to get the correct HTML content (in case of redirect) and extract the French name from there
const arcResponse = await fetchWithRetry(`${FANDOM_API_BASE}${cleanUrl}`); const arcResponse = await fetchWithRetry(`${FANDOM_API_BASE}${cleanUrl}`);
const arcJsonData = await arcResponse.json(); const arcJsonData = await arcResponse.json();
let frArcName: string | null = let frArcName: string | null = getFrenchLabel(arcJsonData.parse?.langlinks || []);
arcJsonData.parse?.langlinks.find(
(link: { lang: string; ['*']: string }) => link.lang === 'fr'
)?.['*'] || null;
// Remove "Arc" suffix from French name if present to keep it consistent with English names (e.g. "Arc de Luffy" becomes "Luffy") // Remove "Arc" suffix from French name if present to keep it consistent with English names (e.g. "Arc de Luffy" becomes "Luffy")
if (frArcName && /\bArc\b/i.test(frArcName)) { if (frArcName && /\bArc\b/i.test(frArcName)) {
@@ -283,14 +324,7 @@ async function fetchAllCharacters(arcsList: Arc[]): Promise<Character[]> {
console.log('Fetching character list via API...'); console.log('Fetching character list via API...');
const response = await fetchWithRetry(`${FANDOM_API_BASE}List_of_Canon_Characters`); const response = await fetchWithRetry(`${FANDOM_API_BASE}List_of_Canon_Characters`);
const jsonData = await response.json(); const jsonData = await response.json();
const $ = loadWikiPage(jsonData);
// Extract HTML from API response
const htmlContent = jsonData.parse?.text?.['*'];
if (!htmlContent) {
throw new Error('Unable to extract HTML content from API response');
}
const $ = cheerio.load(htmlContent);
const characterList: CharacterListItem[] = []; const characterList: CharacterListItem[] = [];
$('table.fandom-table tbody tr').each((index, element) => { $('table.fandom-table tbody tr').each((index, element) => {
if (index === 0) return; // Skip header row if (index === 0) return; // Skip header row
@@ -326,20 +360,18 @@ async function fetchAllCharacters(arcsList: Arc[]): Promise<Character[]> {
return []; return [];
} }
console.log(`Found ${characterList.length} characters.`); console.log(`Found ${characterList.length} characters.`);
// Fetch the french character list to get the picture URLs
console.log('Fetching French character list via API...'); console.log('Fetching French character list via API...');
const frResponse = await fetchWithRetry(`${FR_FANDOM_API_BASE}Liste_des_Personnages_Canon`); const frResponse = await fetchWithRetry(`${FR_FANDOM_API_BASE}Liste_des_Personnages_Canon`);
const frJsonData = await frResponse.json(); const frJsonData = await frResponse.json();
const fr$ = loadWikiPage(frJsonData);
// Create a map of character name to picture URL from the French list
const frHtmlContent = frJsonData.parse?.text?.['*'];
const fr$ = cheerio.load(frHtmlContent);
const frCharacterPictureMap: Record<string, string> = {}; const frCharacterPictureMap: Record<string, string> = {};
fr$('table.wikitable tbody tr').each((index, element) => { fr$('table.wikitable tbody tr').each((index, element) => {
if (index === 0) return; // Skip header row if (index === 0) return; // Skip header row
const charName = fr$(element).find('td:nth-child(2) a').text().trim(); const charName = fr$(element).find('td:nth-child(2) a').text().trim();
const pictureUrl = fr$(element).find('td:nth-child(1) img').attr('data-src') || fr$(element).find('td:nth-child(1) img').attr('src') || null; const pictureUrl =
fr$(element).find('td:nth-child(1) img').attr('data-src') ||
fr$(element).find('td:nth-child(1) img').attr('src') ||
null;
if (charName && pictureUrl) { if (charName && pictureUrl) {
frCharacterPictureMap[charName] = pictureUrl; frCharacterPictureMap[charName] = pictureUrl;
} }
@@ -423,14 +455,7 @@ async function fetchCharacter(
const jsonData = await response.json(); const jsonData = await response.json();
const categories = jsonData.parse?.categories || []; const categories = jsonData.parse?.categories || [];
const $ = loadWikiPage(jsonData);
// Extract HTML from API response
const htmlContent = jsonData.parse?.text?.['*'];
if (!htmlContent) {
throw new Error('Unable to extract HTML content from API response');
}
const $ = cheerio.load(htmlContent);
const name = characterName; const name = characterName;
@@ -462,29 +487,19 @@ async function fetchCharacter(
// Extract devil fruit // Extract devil fruit
const devilFruitData = await extractDevilFruit($); const devilFruitData = await extractDevilFruit($);
const devilFruitId = devilFruitData?.devilFruitId || null; const devilFruitId = devilFruitData?.devilFruitId || null;
const devilFruitName = devilFruitData?.devilFruitTitle || null;
const devilFruitUrl = devilFruitData?.devilFruitUrl || null; const devilFruitUrl = devilFruitData?.devilFruitUrl || null;
// Extract haki from JSON categories // Extract haki from JSON categories
let hakiObservation = false; let hakiObservation = false;
let hakiArmament = false; let hakiArmament = false;
let hakiConqueror = false; let hakiConqueror = false;
for (const cat of categories) {
const catName = cat['*'] || '';
if (catName === 'Observation_Haki_Users') {
hakiObservation = true;
} else if (catName === 'Armament_Haki_Users') {
hakiArmament = true;
} else if (catName === 'Supreme_King_Haki_Users') {
hakiConqueror = true;
}
}
// Extract bounty // Extract bounty
const bounty = extractBounty($); const bounty = extractBounty($);
// Extract height // Extract height
const height = extractHeight($); const height = extractHeight($);
// Use chapter from character list, cast to int // Use chapter from character list, cast to int
const firstAppearance = characterChapter; const firstAppearance = characterChapter;
@@ -505,25 +520,20 @@ async function fetchCharacter(
} }
arcId = arc.id; arcId = arc.id;
const frLink = getFrLink(jsonData.parse?.langlinks || []); const frLink = getFrenchLink(jsonData.parse?.langlinks || []);
const frUrl = frLink ? frLink.url.replace('https://onepiece.fandom.com/fr/wiki/', '') : null; const frUrl = frLink ? frLink.url.replace('https://onepiece.fandom.com/fr/wiki/', '') : null;
const frjsonData = frUrl const frjsonData = frUrl
? await fetchWithRetry(`${FR_FANDOM_API_BASE}${frUrl}`).then((res) => res.json()) ? await fetchWithRetry(`${FR_FANDOM_API_BASE}${frUrl}`).then((res) => res.json())
: null; : null;
const frPage: cheerio.CheerioAPI | null = frjsonData ? loadWikiPage(frjsonData) : null;
let frName = frjsonData?.parse?.title || null; let frName = frjsonData?.parse?.title || null;
const frAffiliation = frjsonData const frAffiliation = frPage ? await extractAffiliations(frPage, 'fr') : null;
? await extractAffiliations(cheerio.load(frjsonData.parse?.text?.['*'] || ''), 'fr')
: null;
const frEpithets = frjsonData const frEpithets = frPage ? extractEpithets(frPage) : null;
? extractEpithets(cheerio.load(frjsonData.parse?.text?.['*'] || ''))
: null;
const frOrigin = frjsonData const frOrigin = frPage ? extractOrigin(frPage) : null;
? extractOrigin(cheerio.load(frjsonData.parse?.text?.['*'] || ''))
: null;
if (name !== jsonData.parse?.title) { if (name !== jsonData.parse?.title) {
frName = name; frName = name;
@@ -541,6 +551,7 @@ async function fetchCharacter(
origin, origin,
frOrigin, frOrigin,
devilFruitId, devilFruitId,
devilFruitName,
devilFruitUrl, devilFruitUrl,
affiliation, affiliation,
frAffiliation, frAffiliation,
@@ -573,16 +584,15 @@ function extractAge($: cheerio.CheerioAPI): number | null {
let text = div.html(); let text = div.html();
if (!text) return null; if (!text) return null;
// Remove all sup blocks (citations) text = removeSupBlocks(text);
text = text.replace(/<sup[^>]*>.*?<\/sup>/gi, '');
// Get the last element and extract only digits // Get the last element and extract only digits
const parts = text.split('<br'); const parts = text.split('<br');
const lastPart = parts[parts.length - 1]; const lastPart = parts[parts.length - 1];
let cleanText = lastPart.replace(/<[^>]*>/g, '').trim(); let cleanText = stripHtml(lastPart);
// Remove content with parentheses // Remove content with parentheses
cleanText = cleanText.replace(/\([^)]*\)/g, ''); cleanText = removeParentheticalNotes(cleanText);
const digitsOnly = cleanText.replace(/\D/g, ''); const digitsOnly = cleanText.replace(/\D/g, '');
return parseInt(digitsOnly) || null; return parseInt(digitsOnly) || null;
@@ -629,7 +639,7 @@ async function extractAffiliations($: cheerio.CheerioAPI, lang: string): Promise
} }
// Fallback to parsing text // Fallback to parsing text
const cleanText = text.replace(/<[^>]*>/g, '').trim(); const cleanText = stripHtml(text);
const parts = cleanText.split(/\s*\n\s*|\s*;\s*|\s*,\s*/).filter(Boolean); const parts = cleanText.split(/\s*\n\s*|\s*;\s*|\s*,\s*/).filter(Boolean);
return parts.length > 0 ? parts[0] : null; return parts.length > 0 ? parts[0] : null;
} }
@@ -683,22 +693,30 @@ function extractEpithets($: cheerio.CheerioAPI): string[] {
* Returns both normalized ID and URL * Returns both normalized ID and URL
*/ */
async function extractDevilFruit($: cheerio.CheerioAPI): Promise<DevilFruitData | null> { async function extractDevilFruit($: cheerio.CheerioAPI): Promise<DevilFruitData | null> {
const link = $('[data-source="dfname"] .pi-data-value a').first(); // dfname or dfename are used as data-source for devil fruit in the infobox, we check both to be safe
const fruit = $('[data-source="dfname"] .pi-data-value, [data-source="dfename"] .pi-data-value').first();
const link = fruit.find('a').first();
if (link.length === 0) return null; if (link.length === 0) return null;
let fruitTitle = fruit.text().trim().toLowerCase().includes('smile') ? fruit.text().trim() : link.text().trim();
if (!fruitTitle) return null;
const href = link.attr('href'); const href = link.attr('href');
if (!href || !href.startsWith('/wiki/')) return null; if (!href || !href.startsWith('/wiki/')) return null;
const cleanUrl = href.replace('/wiki/', ''); const cleanUrl = href.replace('/wiki/', '');
// Query the devil fruit page via API to get the correct HTML content (in case of redirect) and extract the type from there
const dfResponse = await fetchWithRetry(`${FANDOM_API_BASE}${cleanUrl}`); const dfResponse = await fetchWithRetry(`${FANDOM_API_BASE}${cleanUrl}`);
const dfJsonData = await dfResponse.json(); const dfJsonData = await dfResponse.json();
const fruitTitle = dfJsonData.parse?.title || '';
if (!fruitTitle.toLowerCase().includes('smile')) {
fruitTitle = dfJsonData.parse?.title || '';
}
return { return {
devilFruitId: normalizeId(fruitTitle), devilFruitId: normalizeId(fruitTitle),
devilFruitUrl: fruitTitle devilFruitTitle: fruitTitle,
devilFruitUrl: cleanUrl
}; };
} }
@@ -737,8 +755,7 @@ function extractHeight($: cheerio.CheerioAPI): number | null {
let text = div.html(); let text = div.html();
if (!text) return null; if (!text) return null;
// Remove all sup blocks (citations) text = removeSupBlocks(text);
text = text.replace(/<sup[^>]*>.*?<\/sup>/gi, '');
// Convert line breaks to new lines so we can reliably pick the latest value. // Convert line breaks to new lines so we can reliably pick the latest value.
const textWithNewLines = text.replace(/<br\s*\/?\s*>/gi, '\n'); const textWithNewLines = text.replace(/<br\s*\/?\s*>/gi, '\n');
@@ -786,15 +803,14 @@ function extractOrigin($: cheerio.CheerioAPI): string | null {
let text = div.html(); let text = div.html();
if (!text) return null; if (!text) return null;
// Remove all sup blocks (citations) text = removeSupBlocks(text);
text = text.replace(/<sup[^>]*>.*?<\/sup>/gi, '');
// Extract the first value before any <br> tag // Extract the first value before any <br> tag
const firstValue = text.split('<br')[0].trim(); const firstValue = firstTextLine(text);
let cleanText = firstValue.replace(/<[^>]*>/g, '').trim(); let cleanText = stripHtml(firstValue);
// Remove content with parentheses // Remove content with parentheses
cleanText = cleanText.replace(/\([^)]*\)/g, '').trim(); cleanText = removeParentheticalNotes(cleanText);
return cleanText || null; return cleanText || null;
} }
@@ -890,6 +906,7 @@ async function saveToCSV(characters: Character[]): Promise<void> {
*/ */
async function fetchDevilFruit( async function fetchDevilFruit(
devilFruitUrl: string, devilFruitUrl: string,
devilFruitTitle: string,
devilFruitId: string devilFruitId: string
): Promise<DevilFruit | null> { ): Promise<DevilFruit | null> {
try { try {
@@ -903,13 +920,14 @@ async function fetchDevilFruit(
// Extract HTML from API response // Extract HTML from API response
const htmlContent = jsonData.parse?.text?.['*']; const htmlContent = jsonData.parse?.text?.['*'];
if (!htmlContent) { if (!htmlContent) {
// Error jsonData for debugging
throw new Error('Unable to extract HTML content from API response'); throw new Error('Unable to extract HTML content from API response');
} }
const name = jsonData.parse?.title || devilFruitId.replace(/_/g, ' '); const name = jsonData.parse?.title || devilFruitId.replace(/_/g, ' ');
let type: string | null = null; let type: string | null = null;
// Determine type based on categories (if categories contain "Paramecia", "Zoan", "Logia" or "Smile") // Determine type based on categories (if categories contain "Paramecia", "Zoan", "Logia")
if (jsonData.parse?.categories) { if (jsonData.parse?.categories) {
const categories = jsonData.parse.categories.map((cat: { ['*']: string }) => const categories = jsonData.parse.categories.map((cat: { ['*']: string }) =>
String(cat['*'] || '').toLowerCase() String(cat['*'] || '').toLowerCase()
@@ -921,14 +939,12 @@ async function fetchDevilFruit(
type = 'Zoan'; type = 'Zoan';
} else if (categories.some((category: string) => category.includes('logia'))) { } else if (categories.some((category: string) => category.includes('logia'))) {
type = 'Logia'; type = 'Logia';
} else if (categories.some((category: string) => category.includes('smile'))) {
type = 'Smile';
} }
} }
return { return {
id: devilFruitId, id: devilFruitId,
name, name: devilFruitTitle.toLowerCase().includes('smile') ? devilFruitTitle : name,
type, type,
url: devilFruitUrl url: devilFruitUrl
}; };
@@ -1021,26 +1037,35 @@ async function main(): Promise<void> {
return; return;
} }
const devilFruitUrls = new Set<string>( const devilFruitEntries = Array.from(
characters.filter((c) => c.devilFruitUrl).map((c) => c.devilFruitUrl!) new Map(
characters
.filter((c) => c.devilFruitUrl && c.devilFruitName)
.map((c) => {
const name = c.devilFruitName!;
const url = c.devilFruitUrl!;
const id = normalizeId(name);
return [`${normalizeId(url)}::${id}`, { url, name, id }] as const;
})
)
.values()
); );
console.log(`✓ Found ${devilFruitUrls.size} unique devil fruits\n`); console.log(`✓ Found ${devilFruitEntries.length} unique devil fruits\n`);
// Step 3: Scraping Devil Fruits // Step 3: Scraping Devil Fruits
console.log('=== Step 2: Scraping Devil Fruits ===\n'); console.log('=== Step 2: Scraping Devil Fruits ===\n');
if (devilFruitUrls.size === 0) { if (devilFruitEntries.length === 0) {
console.warn('No devil fruits found from characters, skipping...\n'); console.warn('No devil fruits found from characters, skipping...\n');
} else { } else {
const devilFruits: DevilFruit[] = []; const devilFruits: DevilFruit[] = [];
const devilFruitUrlArray = Array.from(devilFruitUrls);
for (let i = 0; i < devilFruitUrlArray.length; i += FETCH_CONCURRENCY) { for (let i = 0; i < devilFruitEntries.length; i += FETCH_CONCURRENCY) {
const batch = devilFruitUrlArray.slice(i, i + FETCH_CONCURRENCY); const batch = devilFruitEntries.slice(i, i + FETCH_CONCURRENCY);
const batchResults = await Promise.all( const batchResults = await Promise.all(
batch.map(async (url) => { batch.map(async (entry) => {
const data = await fetchDevilFruit(url, normalizeId(url)); const data = await fetchDevilFruit(entry.url, entry.name, entry.id);
return { url, data }; return { entry, data };
}) })
); );
@@ -1067,12 +1092,12 @@ async function main(): Promise<void> {
await saveDevilFruitsToCSV(devilFruits); await saveDevilFruitsToCSV(devilFruits);
} }
// Update characters with normalized devil fruit IDs // Update characters with normalized devil fruit IDs derived from the character-page fruit name.
const devilFruitMap = new Map<string, string>(devilFruits.map((df) => [df.id, df.id]));
characters.forEach((char) => { characters.forEach((char) => {
if (char.devilFruitUrl) { if (char.devilFruitName) {
const normalizedId = normalizeId(char.devilFruitUrl); char.devilFruitId = normalizeId(char.devilFruitName);
char.devilFruitId = devilFruitMap.get(normalizedId) || normalizedId; } else if (char.devilFruitUrl) {
char.devilFruitId = normalizeId(char.devilFruitUrl);
} }
}); });
} }