Created
February 29, 2024 18:08
-
-
Save ramonrails/8348f93972fc4890371e57128b25a06a to your computer and use it in GitHub Desktop.
wget wrapper function to fetch a website
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# fetch entire website | |
# | |
# Example: | |
# fetch-website https://www.bharat.in | |
# | |
function fetch-website { | |
# Check if wget is available | |
if ! command -v wget &>/dev/null; then | |
# If not installed, install using brew (assuming macOS) | |
if command -v brew &>/dev/null; then | |
echo "wget is not installed. Installing using brew..." | |
brew install wget | |
else | |
echo "wget is not installed and brew is not found. Please install wget manually." | |
# exit 1 | |
fi | |
# wget installed, let's run | |
else | |
# Check for required arguments | |
if [ $# -lt 1 ]; then | |
echo "Usage: fetch-website https://www.bharat.in" | |
else | |
# --show-progress display the progress bar in any verbosity mode | |
# -r, --recursive specify recursive download | |
# -c, --continue resume getting a partially-downloaded file | |
# -E, --adjust-extension save HTML/CSS documents with proper extensions | |
# -k, --convert-links make links in downloaded HTML or CSS point to local files | |
# -l, --level=NUMBER maximum recursion depth (inf or 0 for infinite) | |
# -L, --relative follow relative links only (severely limits content) | |
# -np, --no-parent don't ascend to the parent directory | |
# -p, --page-requisites get all images, etc. needed to display HTML page | |
# -t, --tries=NUMBER set number of retries to NUMBER (0 unlimits) | |
# | |
wget --recursive --level=2 --convert-links --no-parent --page-requisites --adjust-extension --show-progress --continue --tries=2 "$1" | |
fi | |
fi | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment