diff --git a/scripts/scrape-onepiece.ts b/scripts/scrape-onepiece.ts index 296d709..14b1752 100644 --- a/scripts/scrape-onepiece.ts +++ b/scripts/scrape-onepiece.ts @@ -56,7 +56,7 @@ const FANDOM_API_BASE = 'https://onepiece.fandom.com/fr/api.php?action=parse&for const OUTPUT_DIR = './scraped-data'; const MAX_RETRIES = 0; // Set to 0 to disable retries, can be increased if needed const INITIAL_RETRY_DELAY = 1000; -const CHARACTER_FETCH_CONCURRENCY = 50; +const FETCH_CONCURRENCY = 50; // Store cookies across requests (simulate browser behavior) const cookies = new Map(); @@ -882,8 +882,8 @@ async function main(): Promise { const nextFailedCharacters: CharacterListItem[] = []; console.log(`\nFetching ${failedCharacters.length} characters...`); - for (let i = 0; i < failedCharacters.length; i += CHARACTER_FETCH_CONCURRENCY) { - const batch = failedCharacters.slice(i, i + CHARACTER_FETCH_CONCURRENCY); + for (let i = 0; i < failedCharacters.length; i += FETCH_CONCURRENCY) { + const batch = failedCharacters.slice(i, i + FETCH_CONCURRENCY); const batchResults = await Promise.all( batch.map(async (char) => { const data = await fetchCharacter(char.url, char.name, char.pictureUrl, char.chapter); @@ -950,19 +950,26 @@ async function main(): Promise { const devilFruits: DevilFruit[] = []; const devilFruitUrlArray = Array.from(devilFruitUrls); - for (let i = 0; i < devilFruitUrlArray.length; i++) { - const url = devilFruitUrlArray[i]; - const data = await fetchDevilFruit(url, normalizeId(url)); - - if (data) { - console.table({ - ID: data.id, - Name: data.name, - Type: data.type, - URL: data.url - }); + for (let i = 0; i < devilFruitUrlArray.length; i += FETCH_CONCURRENCY) { + const batch = devilFruitUrlArray.slice(i, i + FETCH_CONCURRENCY); + const batchResults = await Promise.all( + batch.map(async (url) => { + const data = await fetchDevilFruit(url, normalizeId(url)); + return { url, data }; + }) + ); - devilFruits.push(data); + for (const { data } of batchResults) { + if (data) { + console.table({ + ID: data.id, + Name: data.name, + Type: data.type, + URL: data.url + }); + + devilFruits.push(data); + } } }