diff --git a/ipset/def.sh b/ipset/def.sh index fbacf87..9d1dc06 100644 --- a/ipset/def.sh +++ b/ipset/def.sh @@ -1,8 +1,3 @@ -[ -n "$IPSET_DIR" ] || { - IPSET_DIR="$(dirname "$0")" - IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" -} - . "$IPSET_DIR/../config" [ -z "$TMPDIR" ] && TMPDIR=/tmp @@ -52,15 +47,12 @@ exists() # MacoS in cron does not include /usr/local/bin to PATH if [ -x /usr/local/bin/ggrep ] ; then GREP=/usr/local/bin/ggrep -elif [ -x /usr/local/bin/grep ] ; then - GREP=/usr/local/bin/grep elif exists ggrep; then GREP=$(which ggrep) else GREP=$(which grep) fi - grep_supports_b() { # \b does not work with BSD grep @@ -126,32 +118,19 @@ zzsize() } digger() -{ - # $1 - family (4|6) - # $2 - s=enable mdig stats - if [ -x "$MDIG" ]; then - local cmd - [ "$2" == "s" ] && cmd=--stats=1000 - "$MDIG" --family=$1 --threads=$MDIG_THREADS $1 - else - local A=A - [ "$1" = "6" ] && A=AAAA - dig $A +short +time=8 +tries=2 -f - | $GREP -E '^[^;].*[^\.]$' - fi -} -filedigger() { # $1 - hostlist # $2 - family (4|6) >&2 echo digging $(wc -l <"$1" | xargs) ipv$2 domains : "$1" - zzcat "$1" | digger $2 s -} -dnstest() -{ - local ip=$(echo w3.org | digger 46) - [ -n "$ip" ] -} + if [ -x "$MDIG" ]; then + zzcat "$1" | "$MDIG" --family=$2 --threads=$MDIG_THREADS --stats=1000 + else + local A=A + [ "$2" = "6" ] && A=AAAA + zzcat "$1" | dig $A +short +time=8 +tries=2 -f - | $GREP -E '^[^;].*[^\.]$' + fi +} cut_local() { @@ -173,29 +152,24 @@ oom_adjust_high() getexclude() { oom_adjust_high - dnstest || { - echo "! DNS is not working. list processing aborted." - return 1 - } + [ -f "$ZUSERLIST_EXCLUDE" ] && { - [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST_EXCLUDE" 4 | sort -u > "$ZIPLIST_EXCLUDE" - [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST_EXCLUDE" 6 | sort -u > "$ZIPLIST_EXCLUDE6" + [ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST_EXCLUDE" 4 | sort -u > "$ZIPLIST_EXCLUDE" + [ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST_EXCLUDE" 6 | sort -u > "$ZIPLIST_EXCLUDE6" } - return 0 } getuser() { - getexclude || return + getexclude [ -f "$ZUSERLIST" ] && { - [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST" 4 | cut_local | sort -u > "$ZIPLIST_USER" - [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST" 6 | cut_local6 | sort -u > "$ZIPLIST_USER6" + [ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST" 4 | cut_local | sort -u > "$ZIPLIST_USER" + [ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST" 6 | cut_local6 | sort -u > "$ZIPLIST_USER6" } [ -f "$ZUSERLIST_IPBAN" ] && { - [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST_IPBAN" 4 | cut_local | sort -u > "$ZIPLIST_USER_IPBAN" - [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST_IPBAN" 6 | cut_local6 | sort -u > "$ZIPLIST_USER_IPBAN6" + [ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST_IPBAN" 4 | cut_local | sort -u > "$ZIPLIST_USER_IPBAN" + [ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST_IPBAN" 6 | cut_local6 | sort -u > "$ZIPLIST_USER_IPBAN6" } - return 0 } hup_zapret_daemons() diff --git a/ipset/get_antifilter_ip.sh b/ipset/get_antifilter_ip.sh index 91eda04..1628890 100755 --- a/ipset/get_antifilter_ip.sh +++ b/ipset/get_antifilter_ip.sh @@ -5,9 +5,10 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" . "$IPSET_DIR/def.sh" -getuser && { - . "$IPSET_DIR/antifilter.helper" - get_antifilter https://antifilter.network/download/ip.lst "$ZIPLIST" -} +getuser + +. "$IPSET_DIR/antifilter.helper" + +get_antifilter https://antifilter.network/download/ip.lst "$ZIPLIST" "$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_antifilter_ipsmart.sh b/ipset/get_antifilter_ipsmart.sh index 9f0d671..b71298f 100755 --- a/ipset/get_antifilter_ipsmart.sh +++ b/ipset/get_antifilter_ipsmart.sh @@ -5,9 +5,10 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" . "$IPSET_DIR/def.sh" -getuser && { - . "$IPSET_DIR/antifilter.helper" - get_antifilter https://antifilter.network/download/ipsmart.lst "$ZIPLIST" -} +getuser + +. "$IPSET_DIR/antifilter.helper" + +get_antifilter https://antifilter.network/download/ipsmart.lst "$ZIPLIST" "$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_antifilter_ipsum.sh b/ipset/get_antifilter_ipsum.sh index 71759de..3fc2673 100755 --- a/ipset/get_antifilter_ipsum.sh +++ b/ipset/get_antifilter_ipsum.sh @@ -5,9 +5,10 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" . "$IPSET_DIR/def.sh" -getuser && { - . "$IPSET_DIR/antifilter.helper" - get_antifilter https://antifilter.network/download/ipsum.lst "$ZIPLIST" -} +getuser + +. "$IPSET_DIR/antifilter.helper" + +get_antifilter https://antifilter.network/download/ipsum.lst "$ZIPLIST" "$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_reestr_combined.sh b/ipset/get_reestr_combined.sh index 1a5569e..675d37d 100755 --- a/ipset/get_reestr_combined.sh +++ b/ipset/get_reestr_combined.sh @@ -9,6 +9,7 @@ ZREESTR="$TMPDIR/reestr.txt" #ZURL_REESTR=https://reestr.rublacklist.net/api/current ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv +getuser dig_reestr() { @@ -35,31 +36,29 @@ dig_reestr() rm -f "$TMP" } -getuser && { - curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" || - { - echo reestr list download failed - exit 2 - } - dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ') - if test $dlsize -lt 1048576; then - echo reestr ip list is too small. can be bad. - exit 2 - fi - #sed -i 's/\\n/\r\n/g' $ZREESTR +curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" || +{ + echo reestr list download failed + exit 2 +} +dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ') +if test $dlsize -lt 1048576; then + echo reestr ip list is too small. can be bad. + exit 2 +fi +#sed -i 's/\\n/\r\n/g' $ZREESTR - get_ip_regex +get_ip_regex - [ "$DISABLE_IPV4" != "1" ] && { - dig_reestr "$REG_IPV4" "$ZIPLIST" "$ZIPLIST_IPBAN" 4 - } - - [ "$DISABLE_IPV6" != "1" ] && { - dig_reestr "$REG_IPV6" "$ZIPLIST6" "$ZIPLIST_IPBAN6" 6 - } - - rm -f "$ZREESTR" +[ "$DISABLE_IPV4" != "1" ] && { + dig_reestr "$REG_IPV4" "$ZIPLIST" "$ZIPLIST_IPBAN" 4 } +[ "$DISABLE_IPV6" != "1" ] && { + dig_reestr "$REG_IPV6" "$ZIPLIST6" "$ZIPLIST_IPBAN6" 6 +} + +rm -f "$ZREESTR" + "$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_reestr_hostlist.sh b/ipset/get_reestr_hostlist.sh index 3900434..d204d27 100755 --- a/ipset/get_reestr_hostlist.sh +++ b/ipset/get_reestr_hostlist.sh @@ -6,10 +6,8 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" . "$IPSET_DIR/def.sh" # useful in case ipban set is used in custom scripts -FAIL= -getuser || FAIL=1 +getuser "$IPSET_DIR/create_ipset.sh" -[ -n "$FAIL" ] && exit ZREESTR="$TMPDIR/zapret.txt" #ZURL=https://reestr.rublacklist.net/api/current diff --git a/ipset/get_reestr_ip.sh b/ipset/get_reestr_ip.sh index 7ab8f27..0eeec80 100755 --- a/ipset/get_reestr_ip.sh +++ b/ipset/get_reestr_ip.sh @@ -9,6 +9,7 @@ ZREESTR="$TMPDIR/reestr.txt" #ZURL_REESTR=https://reestr.rublacklist.net/api/current ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv +getuser dig_reestr() { @@ -22,31 +23,29 @@ dig_reestr() } -getuser && { - # assume all https banned by ip - curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" || - { - echo reestr list download failed - exit 2 - } - dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ') - if test $dlsize -lt 1048576; then - echo reestr ip list is too small. can be bad. - exit 2 - fi - #sed -i 's/\\n/\r\n/g' $ZREESTR +# assume all https banned by ip +curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" || +{ + echo reestr list download failed + exit 2 +} +dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ') +if test $dlsize -lt 1048576; then + echo reestr ip list is too small. can be bad. + exit 2 +fi +#sed -i 's/\\n/\r\n/g' $ZREESTR - get_ip_regex +get_ip_regex - [ "$DISABLE_IPV4" != "1" ] && { - dig_reestr "$REG_IPV4" "$ZIPLIST" 4 - } - - [ "$DISABLE_IPV6" != "1" ] && { - dig_reestr "$REG_IPV6" "$ZIPLIST6" 6 - } - - rm -f "$ZREESTR" +[ "$DISABLE_IPV4" != "1" ] && { + dig_reestr "$REG_IPV4" "$ZIPLIST" 4 } +[ "$DISABLE_IPV6" != "1" ] && { + dig_reestr "$REG_IPV6" "$ZIPLIST6" 6 +} + +rm -f "$ZREESTR" + "$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_reestr_resolve.sh b/ipset/get_reestr_resolve.sh index ae986ed..c6913ad 100755 --- a/ipset/get_reestr_resolve.sh +++ b/ipset/get_reestr_resolve.sh @@ -11,45 +11,44 @@ ZIPLISTTMP="$TMPDIR/zapret-ip.txt" #ZURL=https://reestr.rublacklist.net/api/current ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv -getuser && { - # both disabled - [ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0 +getuser - curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL" >"$ZREESTR" || - { - echo reestr list download failed - exit 2 - } +# both disabled +[ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0 - dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ') - if test $dlsize -lt 204800; then - echo list file is too small. can be bad. - exit 2 - fi - - echo preparing dig list .. - LANG=C cut -f2 -d ';' "$ZREESTR" | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG" - rm -f "$ZREESTR" - - echo digging started. this can take long ... - - [ "$DISABLE_IPV4" != "1" ] && { - filedigger "$ZDIG" 4 | cut_local >"$ZIPLISTTMP" || { - rm -f "$ZDIG" - exit 1 - } - ip2net4 <"$ZIPLISTTMP" | zz "$ZIPLIST" - rm -f "$ZIPLISTTMP" - } - [ "$DISABLE_IPV6" != "1" ] && { - filedigger "$ZDIG" 6 | cut_local6 >"$ZIPLISTTMP" || { - rm -f "$ZDIG" - exit 1 - } - ip2net6 <"$ZIPLISTTMP" | zz "$ZIPLIST6" - rm -f "$ZIPLISTTMP" - } - rm -f "$ZDIG" +curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL" >"$ZREESTR" || +{ + echo reestr list download failed + exit 2 } +dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ') +if test $dlsize -lt 204800; then + echo list file is too small. can be bad. + exit 2 +fi + +echo preparing dig list .. +LANG=C cut -f2 -d ';' "$ZREESTR" | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG" +rm -f "$ZREESTR" + +echo digging started. this can take long ... + +[ "$DISABLE_IPV4" != "1" ] && { + digger "$ZDIG" 4 | cut_local >"$ZIPLISTTMP" || { + rm -f "$ZDIG" + exit 1 + } + ip2net4 <"$ZIPLISTTMP" | zz "$ZIPLIST" + rm -f "$ZIPLISTTMP" +} +[ "$DISABLE_IPV6" != "1" ] && { + digger "$ZDIG" 6 | cut_local6 >"$ZIPLISTTMP" || { + rm -f "$ZDIG" + exit 1 + } + ip2net6 <"$ZIPLISTTMP" | zz "$ZIPLIST6" + rm -f "$ZIPLISTTMP" +} +rm -f "$ZDIG" "$IPSET_DIR/create_ipset.sh"