Whidix/OnePieceDle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: unify fetch concurrency constant for character and devil fruit fetching
All checks were successful
Build Docker Image / build (push) Successful in 1m12s
Details

Browse Source
This commit is contained in:
whidix
2026-03-06 21:57:09 +01:00
parent d3e36e486f
commit a91b298ee5
1 changed files with 22 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
scripts/scrape-onepiece.ts
View File

@@ -56,7 +56,7 @@ const FANDOM_API_BASE = 'https://onepiece.fandom.com/fr/api.php?action=parse&for
const OUTPUT_DIR = './scraped-data'; const OUTPUT_DIR = './scraped-data';
const MAX_RETRIES = 0; // Set to 0 to disable retries, can be increased if needed const MAX_RETRIES = 0; // Set to 0 to disable retries, can be increased if needed
const INITIAL_RETRY_DELAY = 1000; const INITIAL_RETRY_DELAY = 1000;
const CHARACTER_FETCH_CONCURRENCY = 50; const FETCH_CONCURRENCY = 50;
// Store cookies across requests (simulate browser behavior) // Store cookies across requests (simulate browser behavior)
const cookies = new Map<string, string>(); const cookies = new Map<string, string>();
@@ -882,8 +882,8 @@ async function main(): Promise<void> {
const nextFailedCharacters: CharacterListItem[] = []; const nextFailedCharacters: CharacterListItem[] = [];
console.log(`\nFetching ${failedCharacters.length} characters...`); console.log(`\nFetching ${failedCharacters.length} characters...`);
for (let i = 0; i < failedCharacters.length; i += CHARACTER_FETCH_CONCURRENCY) { for (let i = 0; i < failedCharacters.length; i += FETCH_CONCURRENCY) {
const batch = failedCharacters.slice(i, i + CHARACTER_FETCH_CONCURRENCY); const batch = failedCharacters.slice(i, i + FETCH_CONCURRENCY);
const batchResults = await Promise.all( const batchResults = await Promise.all(
batch.map(async (char) => { batch.map(async (char) => {
const data = await fetchCharacter(char.url, char.name, char.pictureUrl, char.chapter); const data = await fetchCharacter(char.url, char.name, char.pictureUrl, char.chapter);
@@ -950,19 +950,26 @@ async function main(): Promise<void> {
const devilFruits: DevilFruit[] = []; const devilFruits: DevilFruit[] = [];
const devilFruitUrlArray = Array.from(devilFruitUrls); const devilFruitUrlArray = Array.from(devilFruitUrls);
for (let i = 0; i < devilFruitUrlArray.length; i++) { for (let i = 0; i < devilFruitUrlArray.length; i += FETCH_CONCURRENCY) {
const url = devilFruitUrlArray[i]; const batch = devilFruitUrlArray.slice(i, i + FETCH_CONCURRENCY);
const data = await fetchDevilFruit(url, normalizeId(url)); const batchResults = await Promise.all(
batch.map(async (url) => {
if (data) { const data = await fetchDevilFruit(url, normalizeId(url));
console.table({ return { url, data };
ID: data.id, })
Name: data.name, );
Type: data.type,
URL: data.url
});
devilFruits.push(data); for (const { data } of batchResults) {
if (data) {
console.table({
ID: data.id,
Name: data.name,
Type: data.type,
URL: data.url
});
devilFruits.push(data);
}
} }
} }