Whidix/OnePieceDle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: enhance character and devil fruit data structures with additional fields and parsing functions
Build Docker Image / build (push) Successful in 2m10s
Details

Browse Source
This commit is contained in:
Whidix
2026-05-25 19:03:12 +02:00
parent ef6bf9862e
commit a77f84f595
1 changed files with 130 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/scrape-onepiece.ts
+130 -105
View File
@@ -22,6 +22,7 @@ interface Character {
origin: string | null;
frOrigin: string | null;
devilFruitId: string | null;
devilFruitName: string | null;
devilFruitUrl: string | null;
affiliation: string | null;
frAffiliation: string | null;
@@ -47,6 +48,7 @@ interface CharacterListItem {
interface DevilFruitData {
devilFruitId: string;
devilFruitTitle: string;
devilFruitUrl: string;
}
@@ -66,6 +68,17 @@ const MAX_RETRIES = 0; // Set to 0 to disable retries, can be increased if neede
const INITIAL_RETRY_DELAY = 1000;
const FETCH_CONCURRENCY = 50;
type WikiParseData = {
parse?: {
text?: {
['*']?: string;
};
langlinks?: Array<{ lang: string; ['*']: string; url: string }>;
title?: string;
categories?: Array<{ ['*']: string }>;
};
};
// Create output directory
if (!fs.existsSync(OUTPUT_DIR)) {
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
@@ -122,6 +135,44 @@ async function fetchWithRetry(
}
}
function getParsedHtml(jsonData: WikiParseData): string {
const htmlContent = jsonData.parse?.text?.['*'];
if (!htmlContent) {
throw new Error('Unable to extract HTML content from API response');
}
return htmlContent;
}
function loadWikiPage(jsonData: WikiParseData): cheerio.CheerioAPI {
return cheerio.load(getParsedHtml(jsonData));
}
function getFrenchLink(links: Array<{ lang: string; ['*']: string; url: string }>): { url: string } | null {
const frenchLink = links.find((link) => link.lang === 'fr');
return frenchLink ? { url: frenchLink.url } : null;
}
function getFrenchLabel(links: Array<{ lang: string; ['*']: string; url: string }>): string | null {
return links.find((link) => link.lang === 'fr')?.['*'] || null;
}
function removeSupBlocks(html: string): string {
return html.replace(/<sup[^>]*>.*?<\/sup>/gi, '');
}
function stripHtml(text: string): string {
return text.replace(/<[^>]*>/g, '').trim();
}
function firstTextLine(html: string): string {
return html.split('<br')[0].trim();
}
function removeParentheticalNotes(text: string): string {
return text.replace(/\([^)]*\)/g, '').trim();
}
/**
* Get the French link from the API response links array
*/
@@ -154,14 +205,7 @@ async function fetchAllArcs(): Promise<Arc[]> {
console.log('Fetching arcs list via API...');
const response = await fetchWithRetry(apiUrl);
const jsonData = await response.json();
// Extract HTML from API response
const htmlContent = jsonData.parse?.text?.['*'];
if (!htmlContent) {
throw new Error('Unable to extract HTML content from API response');
}
const $ = cheerio.load(htmlContent);
const $ = loadWikiPage(jsonData);
const arcs: Arc[] = [];
const seenArcUrls = new Set<string>();
@@ -204,10 +248,7 @@ async function fetchAllArcs(): Promise<Arc[]> {
// Query the href page via API to get the correct HTML content (in case of redirect) and extract the French name from there
const arcResponse = await fetchWithRetry(`${FANDOM_API_BASE}${cleanUrl}`);
const arcJsonData = await arcResponse.json();
let frArcName: string | null =
arcJsonData.parse?.langlinks.find(
(link: { lang: string; ['*']: string }) => link.lang === 'fr'
)?.['*'] || null;
let frArcName: string | null = getFrenchLabel(arcJsonData.parse?.langlinks || []);
// Remove "Arc" suffix from French name if present to keep it consistent with English names (e.g. "Arc de Luffy" becomes "Luffy")
if (frArcName && /\bArc\b/i.test(frArcName)) {
@@ -283,14 +324,7 @@ async function fetchAllCharacters(arcsList: Arc[]): Promise<Character[]> {
console.log('Fetching character list via API...');
const response = await fetchWithRetry(`${FANDOM_API_BASE}List_of_Canon_Characters`);
const jsonData = await response.json();
// Extract HTML from API response
const htmlContent = jsonData.parse?.text?.['*'];
if (!htmlContent) {
throw new Error('Unable to extract HTML content from API response');
}
const $ = cheerio.load(htmlContent);
const $ = loadWikiPage(jsonData);
const characterList: CharacterListItem[] = [];
$('table.fandom-table tbody tr').each((index, element) => {
if (index === 0) return; // Skip header row
@@ -326,24 +360,22 @@ async function fetchAllCharacters(arcsList: Arc[]): Promise<Character[]> {
return [];
}
console.log(`Found ${characterList.length} characters.`);
// Fetch the french character list to get the picture URLs
console.log('Fetching French character list via API...');
const frResponse = await fetchWithRetry(`${FR_FANDOM_API_BASE}Liste_des_Personnages_Canon`);
const frJsonData = await frResponse.json();
// Create a map of character name to picture URL from the French list
const frHtmlContent = frJsonData.parse?.text?.['*'];
const fr$ = cheerio.load(frHtmlContent);
const frCharacterPictureMap: Record<string, string> = {};
fr$('table.wikitable tbody tr').each((index, element) => {
if (index === 0) return; // Skip header row
const charName = fr$(element).find('td:nth-child(2) a').text().trim();
const pictureUrl = fr$(element).find('td:nth-child(1) img').attr('data-src') || fr$(element).find('td:nth-child(1) img').attr('src') || null;
if (charName && pictureUrl) {
frCharacterPictureMap[charName] = pictureUrl;
}
});
console.log('Fetching French character list via API...');
const frResponse = await fetchWithRetry(`${FR_FANDOM_API_BASE}Liste_des_Personnages_Canon`);
const frJsonData = await frResponse.json();
const fr$ = loadWikiPage(frJsonData);
const frCharacterPictureMap: Record<string, string> = {};
fr$('table.wikitable tbody tr').each((index, element) => {
if (index === 0) return; // Skip header row
const charName = fr$(element).find('td:nth-child(2) a').text().trim();
const pictureUrl =
fr$(element).find('td:nth-child(1) img').attr('data-src') ||
fr$(element).find('td:nth-child(1) img').attr('src') ||
null;
if (charName && pictureUrl) {
frCharacterPictureMap[charName] = pictureUrl;
}
});
const characters: Character[] = [];
let failedCharacters: CharacterListItem[] = [...characterList];
@@ -423,14 +455,7 @@ async function fetchCharacter(
const jsonData = await response.json();
const categories = jsonData.parse?.categories || [];
// Extract HTML from API response
const htmlContent = jsonData.parse?.text?.['*'];
if (!htmlContent) {
throw new Error('Unable to extract HTML content from API response');
}
const $ = cheerio.load(htmlContent);
const $ = loadWikiPage(jsonData);
const name = characterName;
@@ -462,29 +487,19 @@ async function fetchCharacter(
// Extract devil fruit
const devilFruitData = await extractDevilFruit($);
const devilFruitId = devilFruitData?.devilFruitId || null;
const devilFruitName = devilFruitData?.devilFruitTitle || null;
const devilFruitUrl = devilFruitData?.devilFruitUrl || null;
// Extract haki from JSON categories
let hakiObservation = false;
let hakiArmament = false;
let hakiConqueror = false;
for (const cat of categories) {
const catName = cat['*'] || '';
if (catName === 'Observation_Haki_Users') {
hakiObservation = true;
} else if (catName === 'Armament_Haki_Users') {
hakiArmament = true;
} else if (catName === 'Supreme_King_Haki_Users') {
hakiConqueror = true;
}
}
// Extract bounty
const bounty = extractBounty($);
// Extract height
const height = extractHeight($);
// Use chapter from character list, cast to int
const firstAppearance = characterChapter;
@@ -505,31 +520,26 @@ async function fetchCharacter(
}
arcId = arc.id;
const frLink = getFrLink(jsonData.parse?.langlinks || []);
const frLink = getFrenchLink(jsonData.parse?.langlinks || []);
const frUrl = frLink ? frLink.url.replace('https://onepiece.fandom.com/fr/wiki/', '') : null;
const frjsonData = frUrl
? await fetchWithRetry(`${FR_FANDOM_API_BASE}${frUrl}`).then((res) => res.json())
: null;
const frPage: cheerio.CheerioAPI | null = frjsonData ? loadWikiPage(frjsonData) : null;
let frName = frjsonData?.parse?.title || null;
const frAffiliation = frjsonData
? await extractAffiliations(cheerio.load(frjsonData.parse?.text?.['*'] || ''), 'fr')
: null;
const frAffiliation = frPage ? await extractAffiliations(frPage, 'fr') : null;
const frEpithets = frjsonData
? extractEpithets(cheerio.load(frjsonData.parse?.text?.['*'] || ''))
: null;
const frEpithets = frPage ? extractEpithets(frPage) : null;
const frOrigin = frjsonData
? extractOrigin(cheerio.load(frjsonData.parse?.text?.['*'] || ''))
: null;
const frOrigin = frPage ? extractOrigin(frPage) : null;
if (name !== jsonData.parse?.title) {
frName = name;
}
const pictureUrl = frCharacterPictureMap[frName || ''] || null;
const pictureUrl = frCharacterPictureMap[frName || ''] || null;
return {
id: finalCharacterId,
@@ -541,6 +551,7 @@ async function fetchCharacter(
origin,
frOrigin,
devilFruitId,
devilFruitName,
devilFruitUrl,
affiliation,
frAffiliation,
@@ -573,16 +584,15 @@ function extractAge($: cheerio.CheerioAPI): number | null {
let text = div.html();
if (!text) return null;
// Remove all sup blocks (citations)
text = text.replace(/<sup[^>]*>.*?<\/sup>/gi, '');
text = removeSupBlocks(text);
// Get the last element and extract only digits
const parts = text.split('<br');
const lastPart = parts[parts.length - 1];
let cleanText = lastPart.replace(/<[^>]*>/g, '').trim();
let cleanText = stripHtml(lastPart);
// Remove content with parentheses
cleanText = cleanText.replace(/\([^)]*\)/g, '');
cleanText = removeParentheticalNotes(cleanText);
const digitsOnly = cleanText.replace(/\D/g, '');
return parseInt(digitsOnly) || null;
@@ -629,7 +639,7 @@ async function extractAffiliations($: cheerio.CheerioAPI, lang: string): Promise
}
// Fallback to parsing text
const cleanText = text.replace(/<[^>]*>/g, '').trim();
const cleanText = stripHtml(text);
const parts = cleanText.split(/\s*\n\s*|\s*;\s*|\s*,\s*/).filter(Boolean);
return parts.length > 0 ? parts[0] : null;
}
@@ -683,22 +693,30 @@ function extractEpithets($: cheerio.CheerioAPI): string[] {
* Returns both normalized ID and URL
*/
async function extractDevilFruit($: cheerio.CheerioAPI): Promise<DevilFruitData | null> {
const link = $('[data-source="dfname"] .pi-data-value a').first();
// dfname or dfename are used as data-source for devil fruit in the infobox, we check both to be safe
const fruit = $('[data-source="dfname"] .pi-data-value, [data-source="dfename"] .pi-data-value').first();
const link = fruit.find('a').first();
if (link.length === 0) return null;
let fruitTitle = fruit.text().trim().toLowerCase().includes('smile') ? fruit.text().trim() : link.text().trim();
if (!fruitTitle) return null;
const href = link.attr('href');
if (!href || !href.startsWith('/wiki/')) return null;
const cleanUrl = href.replace('/wiki/', '');
// Query the devil fruit page via API to get the correct HTML content (in case of redirect) and extract the type from there
const dfResponse = await fetchWithRetry(`${FANDOM_API_BASE}${cleanUrl}`);
const dfJsonData = await dfResponse.json();
const fruitTitle = dfJsonData.parse?.title || '';
const dfResponse = await fetchWithRetry(`${FANDOM_API_BASE}${cleanUrl}`);
const dfJsonData = await dfResponse.json();
if (!fruitTitle.toLowerCase().includes('smile')) {
fruitTitle = dfJsonData.parse?.title || '';
}
return {
devilFruitId: normalizeId(fruitTitle),
devilFruitUrl: fruitTitle
devilFruitTitle: fruitTitle,
devilFruitUrl: cleanUrl
};
}
@@ -737,8 +755,7 @@ function extractHeight($: cheerio.CheerioAPI): number | null {
let text = div.html();
if (!text) return null;
// Remove all sup blocks (citations)
text = text.replace(/<sup[^>]*>.*?<\/sup>/gi, '');
text = removeSupBlocks(text);
// Convert line breaks to new lines so we can reliably pick the latest value.
const textWithNewLines = text.replace(/<br\s*\/?\s*>/gi, '\n');
@@ -786,15 +803,14 @@ function extractOrigin($: cheerio.CheerioAPI): string | null {
let text = div.html();
if (!text) return null;
// Remove all sup blocks (citations)
text = text.replace(/<sup[^>]*>.*?<\/sup>/gi, '');
text = removeSupBlocks(text);
// Extract the first value before any <br> tag
const firstValue = text.split('<br')[0].trim();
let cleanText = firstValue.replace(/<[^>]*>/g, '').trim();
const firstValue = firstTextLine(text);
let cleanText = stripHtml(firstValue);
// Remove content with parentheses
cleanText = cleanText.replace(/\([^)]*\)/g, '').trim();
cleanText = removeParentheticalNotes(cleanText);
return cleanText || null;
}
@@ -890,6 +906,7 @@ async function saveToCSV(characters: Character[]): Promise<void> {
*/
async function fetchDevilFruit(
devilFruitUrl: string,
devilFruitTitle: string,
devilFruitId: string
): Promise<DevilFruit | null> {
try {
@@ -903,13 +920,14 @@ async function fetchDevilFruit(
// Extract HTML from API response
const htmlContent = jsonData.parse?.text?.['*'];
if (!htmlContent) {
// Error jsonData for debugging
throw new Error('Unable to extract HTML content from API response');
}
const name = jsonData.parse?.title || devilFruitId.replace(/_/g, ' ');
let type: string | null = null;
// Determine type based on categories (if categories contain "Paramecia", "Zoan", "Logia" or "Smile")
// Determine type based on categories (if categories contain "Paramecia", "Zoan", "Logia")
if (jsonData.parse?.categories) {
const categories = jsonData.parse.categories.map((cat: { ['*']: string }) =>
String(cat['*'] || '').toLowerCase()
@@ -921,14 +939,12 @@ async function fetchDevilFruit(
type = 'Zoan';
} else if (categories.some((category: string) => category.includes('logia'))) {
type = 'Logia';
} else if (categories.some((category: string) => category.includes('smile'))) {
type = 'Smile';
}
}
return {
id: devilFruitId,
name,
name: devilFruitTitle.toLowerCase().includes('smile') ? devilFruitTitle : name,
type,
url: devilFruitUrl
};
@@ -1021,26 +1037,35 @@ async function main(): Promise<void> {
return;
}
const devilFruitUrls = new Set<string>(
characters.filter((c) => c.devilFruitUrl).map((c) => c.devilFruitUrl!)
const devilFruitEntries = Array.from(
new Map(
characters
.filter((c) => c.devilFruitUrl && c.devilFruitName)
.map((c) => {
const name = c.devilFruitName!;
const url = c.devilFruitUrl!;
const id = normalizeId(name);
return [`${normalizeId(url)}::${id}`, { url, name, id }] as const;
})
)
.values()
);
console.log(`✓ Found ${devilFruitUrls.size} unique devil fruits\n`);
console.log(`✓ Found ${devilFruitEntries.length} unique devil fruits\n`);
// Step 3: Scraping Devil Fruits
console.log('=== Step 2: Scraping Devil Fruits ===\n');
if (devilFruitUrls.size === 0) {
if (devilFruitEntries.length === 0) {
console.warn('No devil fruits found from characters, skipping...\n');
} else {
const devilFruits: DevilFruit[] = [];
const devilFruitUrlArray = Array.from(devilFruitUrls);
for (let i = 0; i < devilFruitUrlArray.length; i += FETCH_CONCURRENCY) {
const batch = devilFruitUrlArray.slice(i, i + FETCH_CONCURRENCY);
for (let i = 0; i < devilFruitEntries.length; i += FETCH_CONCURRENCY) {
const batch = devilFruitEntries.slice(i, i + FETCH_CONCURRENCY);
const batchResults = await Promise.all(
batch.map(async (url) => {
const data = await fetchDevilFruit(url, normalizeId(url));
return { url, data };
batch.map(async (entry) => {
const data = await fetchDevilFruit(entry.url, entry.name, entry.id);
return { entry, data };
})
);
@@ -1067,12 +1092,12 @@ async function main(): Promise<void> {
await saveDevilFruitsToCSV(devilFruits);
}
// Update characters with normalized devil fruit IDs
const devilFruitMap = new Map<string, string>(devilFruits.map((df) => [df.id, df.id]));
// Update characters with normalized devil fruit IDs derived from the character-page fruit name.
characters.forEach((char) => {
if (char.devilFruitUrl) {
const normalizedId = normalizeId(char.devilFruitUrl);
char.devilFruitId = devilFruitMap.get(normalizedId) || normalizedId;
if (char.devilFruitName) {
char.devilFruitId = normalizeId(char.devilFruitName);
} else if (char.devilFruitUrl) {
char.devilFruitId = normalizeId(char.devilFruitUrl);
}
});
}