Download all card's art from Bandori wiki

scrape.sh

#!/usr/bin/env bash

# Bandori card art URL 
BASE_URL="https://bandori.fandom.com/wiki/Category:Card_Art"

# Image's URL pattern
IMAGE_PATTERN="https:\/\/static\.wikia\.nocookie\.net\/bandori\/images\/.\/.{2}\/[^\/]+\.png"

# Next page URL pattern
NEXT_PAGE_PATTERN="https://bandori\.fandom\.com/wiki/Category:Card_Art\?from=[^\/]+\.png"

# Download images from a page
download_page() {
    local page_url="$1"
    echo "[*] Downloading images from: $page_url"

    # Extract image URLs and download them
    curl -s "$page_url" | \
    rg -o "$IMAGE_PATTERN" | \
    rg -v "_T\.png" | \
    uniq | \
    while read -r url; do
        local filename=$(basename "$url")
        # echo "[+] Downloading: $filename"
        aria2c -q -x 4 -s 4 -j 4 --conditional-get -o "$filename" "$url"
        if [ $? -eq 0 ]; then
            echo "[*] Downloaded: $filename"
        else
            echo "[-] Failed to download: $filename"
        fi
    done
}

# Fetch the next page URL
fetch_next_page_url() {
    local current_url="$1"
    curl -s "$current_url" | \
    rg -o "$NEXT_PAGE_PATTERN" | \
    sed -n '2p'
}

main() {
    local next_page_url="$BASE_URL"
    while [ -n "$next_page_url" ]; do
        # Download images from the current page
        download_page "$next_page_url"
        next_page_url=$(fetch_next_page_url "$next_page_url")
        if [ -n "$next_page_url" ]; then
            echo "[*] Next page found: $next_page_url"
        else
            echo "[-] No more pages found."
        fi
    done
}

main
echo "[*] Done :D"