| Paste number 8821: | bash robots.txt parser (untested) |
| Pasted by: | sbp |
| When: | 6 years, 11 months ago |
| Share: | Tweet this! | http://paste.lisp.org/+6T1 |
| Channel: | #swhack |
| Paste contents: |
function robots() {
BASES=$(grep Disallow: $DOCUMENT_ROOT/robots.txt | sed 's/Disallow: //')
while true
do read LINE || break
for BASE in $BASES
do LINE=$(echo $LINE | grep -v $BASE); [[ -z $LINE ]] && break
done
if [[ -n $LINE ]]
then echo $LINE
fi
done
}
Annotations for this paste:
| Annotation number 1: | same thing, better syntax, and confirmedly works |
| Pasted by: | sbp |
| When: | 6 years, 11 months ago |
| Share: | Tweet this! | http://paste.lisp.org/+6T1/1 |
| Paste contents: |
function robots() {
ROBOTS=$DOCUMENT_ROOT/robots.txt
BASES=$(grep Disallow: $ROBOTS | sed 's/Disallow: //')
while true
do read LINE || break
for BASE in $BASES
do LINE=$(echo $LINE | grep -v $BASE)
[[ -z $LINE ]] && break
done
[[ -n $LINE ]] && echo $LINE
done
}