Whidix/OnePieceDle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: unify fetch concurrency constant for character and devil fruit fetching
All checks were successful
Build Docker Image / build (push) Successful in 1m12s
Details

Browse Source
This commit is contained in:
whidix
2026-03-06 21:57:09 +01:00
parent d3e36e486f
commit a91b298ee5
1 changed files with 22 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
scripts/scrape-onepiece.ts
View File

@@ -56,7 +56,7 @@ const FANDOM_API_BASE = 'https://onepiece.fandom.com/fr/api.php?action=parse&for
const OUTPUT_DIR = './scraped-data';
const MAX_RETRIES = 0; // Set to 0 to disable retries, can be increased if needed
const INITIAL_RETRY_DELAY = 1000;
const CHARACTER_FETCH_CONCURRENCY = 50;
const FETCH_CONCURRENCY = 50;
// Store cookies across requests (simulate browser behavior)
const cookies = new Map<string, string>();
@@ -882,8 +882,8 @@ async function main(): Promise<void> {
const nextFailedCharacters: CharacterListItem[] = [];
console.log(`\nFetching ${failedCharacters.length} characters...`);
for (let i = 0; i < failedCharacters.length; i += CHARACTER_FETCH_CONCURRENCY) {
const batch = failedCharacters.slice(i, i + CHARACTER_FETCH_CONCURRENCY);
for (let i = 0; i < failedCharacters.length; i += FETCH_CONCURRENCY) {
const batch = failedCharacters.slice(i, i + FETCH_CONCURRENCY);
const batchResults = await Promise.all(
batch.map(async (char) => {
const data = await fetchCharacter(char.url, char.name, char.pictureUrl, char.chapter);
@@ -950,10 +950,16 @@ async function main(): Promise<void> {
const devilFruits: DevilFruit[] = [];
const devilFruitUrlArray = Array.from(devilFruitUrls);
for (let i = 0; i < devilFruitUrlArray.length; i++) {
const url = devilFruitUrlArray[i];
for (let i = 0; i < devilFruitUrlArray.length; i += FETCH_CONCURRENCY) {
const batch = devilFruitUrlArray.slice(i, i + FETCH_CONCURRENCY);
const batchResults = await Promise.all(
batch.map(async (url) => {
const data = await fetchDevilFruit(url, normalizeId(url));
return { url, data };
})
);
for (const { data } of batchResults) {
if (data) {
console.table({
ID: data.id,
@@ -965,6 +971,7 @@ async function main(): Promise<void> {
devilFruits.push(data);
}
}
}
console.log(`\n✓ Scraped ${devilFruits.length} devil fruits\n`);