@@ -78,6 +78,7 @@ grep -PR '<a .*?href=' |
7878 cut -d' #' |
7979 sort -u -t$' \t ' |
8080 sort -u |
81+ tee >( cat 1>&2 ) |
8182 python -c '
8283import sys
8384from urllib.parse import urljoin
@@ -90,9 +91,14 @@ for line in sys.stdin.readlines():
9091 grep -v $' \t ' ' $' |
9192 while read -r line; do
9293 while IFS=$' \t ' read -r file url; do
93- url=$( python -c ' import html, sys; print(html.unescape(sys.argv[-1]))' " $url " )
94- [ -f " $url " ||
95- curl --silent --fail --retry 5 --retry-delay 5 --user-agent ' Mozilla/5.0 Firefox 61' " $url " > /dev/null 2>&1 ||
94+ target_file=" $( python -c '
95+ import html, sys # fixes &
96+ from urllib.parse import unquote # fixes %20
97+ print(html.unescape(unquote(sys.argv[-1])))' " $url " ) "
98+ if [ -f " $target_file " ; then continue ; fi
99+ 100+ url=" ${url// /% 20} "
101+ curl --silent --fail --retry 5 --retry-delay 5 --user-agent ' Mozilla/5.0 Firefox 101' " $url " > /dev/null 2>&1 ||
96102 die " broken link in $file : $url "
97103 done
98104 done
0 commit comments