diff --git a/ipset/get_combined.sh b/ipset/get_combined.sh index 7f670fc..1e6c0e3 100755 --- a/ipset/get_combined.sh +++ b/ipset/get_combined.sh @@ -7,37 +7,44 @@ EXEDIR=$(dirname $SCRIPT) . "$EXEDIR/def.sh" ZREESTR=$TMPDIR/reestr.txt -ZURL_REESTR=https://reestr.rublacklist.net/api/current +#ZURL_REESTR=https://reestr.rublacklist.net/api/current +ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv ZAZ=$TMPDIR/zapret-ip.txt ZURL_AZ=http://antizapret.prostovpn.org/iplist.txt getuser # assume all https banned by ip -curl -k --fail --max-time 300 --max-filesize 41943040 "$ZURL_REESTR" -o $ZREESTR +curl -k --fail --max-time 300 --max-filesize 41943040 "$ZURL_REESTR" -o $ZREESTR || +{ + echo reestr list download failed + exit 2 +} dlsize=$(wc -c "$ZREESTR" | cut -f 1 -d ' ') if test $dlsize -lt 1048576; then echo reestr ip list is too small. can be bad. exit 2 fi -sed -i 's/\\n/\r\n/g' $ZREESTR +#sed -i 's/\\n/\r\n/g' $ZREESTR # find entries with https or without domain name - they should be banned by IP -(grep "https://" $ZREESTR ; grep -vE ";[a-z|*|_|0-9|\-]+\.[a-z|*|_|0-9|\-]+" $ZREESTR ) | +(grep -a "https://" $ZREESTR ; grep -avE "^.*;[^;:/]+\.[^;:/]+;" $ZREESTR ) | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(/[0-9]+)?' | cut_local | sort -u >$ZIPLIST_IPBAN - + rm -f $ZREESTR -curl --fail --max-time 120 --max-filesize 10485760 -k -L "$ZURL_AZ" | cut_local >$ZAZ && +curl --fail --max-time 120 --max-filesize 10485760 -k -L "$ZURL_AZ" | cut_local >$ZAZ || { - dlsize=$(wc -c "$ZAZ" | cut -f 1 -d ' ') - if test $dlsize -lt 204800; then - echo antizapret list file is too small. can be bad. - exit 2 - fi - # do not include hosts banned by ip - grep -xvFf $ZIPLIST_IPBAN $ZAZ >$ZIPLIST - rm -f $ZAZ - "$EXEDIR/create_ipset.sh" + echo antizapret list download failed + exit 2 } +dlsize=$(wc -c "$ZAZ" | cut -f 1 -d ' ') +if test $dlsize -lt 204800; then + echo antizapret list file is too small. can be bad. + exit 2 +fi +# do not include hosts banned by ip +grep -xvFf $ZIPLIST_IPBAN $ZAZ >$ZIPLIST +rm -f $ZAZ +"$EXEDIR/create_ipset.sh" diff --git a/ipset/get_reestr.sh b/ipset/get_reestr.sh index 6077106..2d2e00d 100755 --- a/ipset/get_reestr.sh +++ b/ipset/get_reestr.sh @@ -9,26 +9,33 @@ EXEDIR=$(dirname $SCRIPT) ZREESTR=$TMPDIR/zapret.txt ZDIG=$TMPDIR/zapret-dig.txt ZIPLISTTMP=$TMPDIR/zapret-ip.txt -ZURL=https://reestr.rublacklist.net/api/current +#ZURL=https://reestr.rublacklist.net/api/current +ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv getuser -curl -k --fail --max-time 300 --max-filesize 41943040 "$ZURL" >$ZREESTR && { - dlsize=$(wc -c "$ZREESTR" | cut -f 1 -d ' ') - if test $dlsize -lt 204800; then - echo list file is too small. can be bad. - exit 2 - fi - sed -i 's/\\n/\r\n/g' $ZREESTR - sed -nre 's/^[^;]*;([^;|\\]{4,250})\;.*/\1/p' $ZREESTR | sort | uniq >$ZDIG - rm -f $ZREESTR - echo digging started ... - dig A +short +time=8 +tries=2 -f $ZDIG | grep -E '^[^;].*[^\.]$' | grep -vE '^192\.168\.[0-9]+\.[0-9]+$' | grep -vE '^127\.[0-9]+\.[0-9]+\.[0-9]+$' | grep -vE '^10\.[0-9]+\.[0-9]+\.[0-9]+$' >$ZIPLISTTMP || { - rm -f $ZDIG - exit 1 - } - rm -f $ZDIG $ZIPLIST - sort $ZIPLISTTMP | uniq >$ZIPLIST - rm -f $ZIPLISTTMP - "$EXEDIR/create_ipset.sh" +curl -k --fail --max-time 300 --max-filesize 41943040 "$ZURL" >$ZREESTR || +{ + echo reestr list download failed + exit 2 } +dlsize=$(wc -c "$ZREESTR" | cut -f 1 -d ' ') +if test $dlsize -lt 204800; then + echo list file is too small. can be bad. + exit 2 +fi +echo preparing dig list .. +#sed -i 's/\\n/\r\n/g' $ZREESTR +#sed -nre 's/^[^;]*;([^;|\\]{4,250})\;.*$/\1/p' $ZREESTR | sort | uniq >$ZDIG +cut -f2 -d';' $ZREESTR | grep -avE '^$|\*|:' >$ZDIG +rm -f $ZREESTR +echo digging started ... +dig A +short +time=8 +tries=2 -f $ZDIG | grep -E '^[^;].*[^\.]$' | grep -vE '^192\.168\.[0-9]+\.[0-9]+$' | grep -vE '^127\.[0-9]+\.[0-9]+\.[0-9]+$' | grep -vE '^10\.[0-9]+\.[0-9]+\.[0-9]+$' >$ZIPLISTTMP || { + rm -f $ZDIG + exit 1 +} +rm -f $ZDIG $ZIPLIST +sort $ZIPLISTTMP | uniq >$ZIPLIST +rm -f $ZIPLISTTMP +"$EXEDIR/create_ipset.sh" +