mirror of
https://github.com/bol-van/zapret.git
synced 2024-11-26 20:20:53 +03:00
ipset: check dns before processing lists
This commit is contained in:
bol-van
parent
cc85c1b489
commit
e9b125e7ef
60
ipset/def.sh
60
ipset/def.sh
@ -1,3 +1,8 @@
|
||||
[ -n "$IPSET_DIR" ] || {
|
||||
IPSET_DIR="$(dirname "$0")"
|
||||
IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
|
||||
}
|
||||
|
||||
. "$IPSET_DIR/../config"
|
||||
|
||||
[ -z "$TMPDIR" ] && TMPDIR=/tmp
|
||||
@ -47,12 +52,15 @@ exists()
|
||||
# MacoS in cron does not include /usr/local/bin to PATH
|
||||
if [ -x /usr/local/bin/ggrep ] ; then
|
||||
GREP=/usr/local/bin/ggrep
|
||||
elif [ -x /usr/local/bin/grep ] ; then
|
||||
GREP=/usr/local/bin/grep
|
||||
elif exists ggrep; then
|
||||
GREP=$(which ggrep)
|
||||
else
|
||||
GREP=$(which grep)
|
||||
fi
|
||||
|
||||
|
||||
grep_supports_b()
|
||||
{
|
||||
# \b does not work with BSD grep
|
||||
@ -118,19 +126,32 @@ zzsize()
|
||||
}
|
||||
|
||||
digger()
|
||||
{
|
||||
# $1 - family (4|6)
|
||||
# $2 - s=enable mdig stats
|
||||
if [ -x "$MDIG" ]; then
|
||||
local cmd
|
||||
[ "$2" == "s" ] && cmd=--stats=1000
|
||||
"$MDIG" --family=$1 --threads=$MDIG_THREADS $1
|
||||
else
|
||||
local A=A
|
||||
[ "$1" = "6" ] && A=AAAA
|
||||
dig $A +short +time=8 +tries=2 -f - | $GREP -E '^[^;].*[^\.]$'
|
||||
fi
|
||||
}
|
||||
filedigger()
|
||||
{
|
||||
# $1 - hostlist
|
||||
# $2 - family (4|6)
|
||||
>&2 echo digging $(wc -l <"$1" | xargs) ipv$2 domains : "$1"
|
||||
|
||||
if [ -x "$MDIG" ]; then
|
||||
zzcat "$1" | "$MDIG" --family=$2 --threads=$MDIG_THREADS --stats=1000
|
||||
else
|
||||
local A=A
|
||||
[ "$2" = "6" ] && A=AAAA
|
||||
zzcat "$1" | dig $A +short +time=8 +tries=2 -f - | $GREP -E '^[^;].*[^\.]$'
|
||||
fi
|
||||
zzcat "$1" | digger $2 s
|
||||
}
|
||||
dnstest()
|
||||
{
|
||||
local ip=$(echo w3.org | digger 46)
|
||||
[ -n "$ip" ]
|
||||
}
|
||||
|
||||
|
||||
cut_local()
|
||||
{
|
||||
@ -152,24 +173,29 @@ oom_adjust_high()
|
||||
getexclude()
|
||||
{
|
||||
oom_adjust_high
|
||||
|
||||
[ -f "$ZUSERLIST_EXCLUDE" ] && {
|
||||
[ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST_EXCLUDE" 4 | sort -u > "$ZIPLIST_EXCLUDE"
|
||||
[ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST_EXCLUDE" 6 | sort -u > "$ZIPLIST_EXCLUDE6"
|
||||
dnstest || {
|
||||
echo "! DNS is not working. list processing aborted."
|
||||
return 1
|
||||
}
|
||||
[ -f "$ZUSERLIST_EXCLUDE" ] && {
|
||||
[ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST_EXCLUDE" 4 | sort -u > "$ZIPLIST_EXCLUDE"
|
||||
[ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST_EXCLUDE" 6 | sort -u > "$ZIPLIST_EXCLUDE6"
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
getuser()
|
||||
{
|
||||
getexclude
|
||||
getexclude || return
|
||||
[ -f "$ZUSERLIST" ] && {
|
||||
[ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST" 4 | cut_local | sort -u > "$ZIPLIST_USER"
|
||||
[ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST" 6 | cut_local6 | sort -u > "$ZIPLIST_USER6"
|
||||
[ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST" 4 | cut_local | sort -u > "$ZIPLIST_USER"
|
||||
[ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST" 6 | cut_local6 | sort -u > "$ZIPLIST_USER6"
|
||||
}
|
||||
[ -f "$ZUSERLIST_IPBAN" ] && {
|
||||
[ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST_IPBAN" 4 | cut_local | sort -u > "$ZIPLIST_USER_IPBAN"
|
||||
[ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST_IPBAN" 6 | cut_local6 | sort -u > "$ZIPLIST_USER_IPBAN6"
|
||||
[ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST_IPBAN" 4 | cut_local | sort -u > "$ZIPLIST_USER_IPBAN"
|
||||
[ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST_IPBAN" 6 | cut_local6 | sort -u > "$ZIPLIST_USER_IPBAN6"
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
hup_zapret_daemons()
|
||||
|
||||
@ -5,10 +5,9 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
|
||||
|
||||
. "$IPSET_DIR/def.sh"
|
||||
|
||||
getuser
|
||||
|
||||
. "$IPSET_DIR/antifilter.helper"
|
||||
|
||||
get_antifilter https://antifilter.network/download/ip.lst "$ZIPLIST"
|
||||
getuser && {
|
||||
. "$IPSET_DIR/antifilter.helper"
|
||||
get_antifilter https://antifilter.network/download/ip.lst "$ZIPLIST"
|
||||
}
|
||||
|
||||
"$IPSET_DIR/create_ipset.sh"
|
||||
|
||||
@ -5,10 +5,9 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
|
||||
|
||||
. "$IPSET_DIR/def.sh"
|
||||
|
||||
getuser
|
||||
|
||||
. "$IPSET_DIR/antifilter.helper"
|
||||
|
||||
get_antifilter https://antifilter.network/download/ipsmart.lst "$ZIPLIST"
|
||||
getuser && {
|
||||
. "$IPSET_DIR/antifilter.helper"
|
||||
get_antifilter https://antifilter.network/download/ipsmart.lst "$ZIPLIST"
|
||||
}
|
||||
|
||||
"$IPSET_DIR/create_ipset.sh"
|
||||
|
||||
@ -5,10 +5,9 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
|
||||
|
||||
. "$IPSET_DIR/def.sh"
|
||||
|
||||
getuser
|
||||
|
||||
. "$IPSET_DIR/antifilter.helper"
|
||||
|
||||
get_antifilter https://antifilter.network/download/ipsum.lst "$ZIPLIST"
|
||||
getuser && {
|
||||
. "$IPSET_DIR/antifilter.helper"
|
||||
get_antifilter https://antifilter.network/download/ipsum.lst "$ZIPLIST"
|
||||
}
|
||||
|
||||
"$IPSET_DIR/create_ipset.sh"
|
||||
|
||||
@ -9,7 +9,6 @@ ZREESTR="$TMPDIR/reestr.txt"
|
||||
#ZURL_REESTR=https://reestr.rublacklist.net/api/current
|
||||
ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
|
||||
|
||||
getuser
|
||||
|
||||
dig_reestr()
|
||||
{
|
||||
@ -36,29 +35,31 @@ dig_reestr()
|
||||
rm -f "$TMP"
|
||||
}
|
||||
|
||||
getuser && {
|
||||
|
||||
curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
|
||||
{
|
||||
echo reestr list download failed
|
||||
exit 2
|
||||
curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
|
||||
{
|
||||
echo reestr list download failed
|
||||
exit 2
|
||||
}
|
||||
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
|
||||
if test $dlsize -lt 1048576; then
|
||||
echo reestr ip list is too small. can be bad.
|
||||
exit 2
|
||||
fi
|
||||
#sed -i 's/\\n/\r\n/g' $ZREESTR
|
||||
|
||||
get_ip_regex
|
||||
|
||||
[ "$DISABLE_IPV4" != "1" ] && {
|
||||
dig_reestr "$REG_IPV4" "$ZIPLIST" "$ZIPLIST_IPBAN" 4
|
||||
}
|
||||
|
||||
[ "$DISABLE_IPV6" != "1" ] && {
|
||||
dig_reestr "$REG_IPV6" "$ZIPLIST6" "$ZIPLIST_IPBAN6" 6
|
||||
}
|
||||
|
||||
rm -f "$ZREESTR"
|
||||
}
|
||||
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
|
||||
if test $dlsize -lt 1048576; then
|
||||
echo reestr ip list is too small. can be bad.
|
||||
exit 2
|
||||
fi
|
||||
#sed -i 's/\\n/\r\n/g' $ZREESTR
|
||||
|
||||
get_ip_regex
|
||||
|
||||
[ "$DISABLE_IPV4" != "1" ] && {
|
||||
dig_reestr "$REG_IPV4" "$ZIPLIST" "$ZIPLIST_IPBAN" 4
|
||||
}
|
||||
|
||||
[ "$DISABLE_IPV6" != "1" ] && {
|
||||
dig_reestr "$REG_IPV6" "$ZIPLIST6" "$ZIPLIST_IPBAN6" 6
|
||||
}
|
||||
|
||||
rm -f "$ZREESTR"
|
||||
|
||||
"$IPSET_DIR/create_ipset.sh"
|
||||
|
||||
@ -6,8 +6,10 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
|
||||
. "$IPSET_DIR/def.sh"
|
||||
|
||||
# useful in case ipban set is used in custom scripts
|
||||
getuser
|
||||
FAIL=
|
||||
getuser || FAIL=1
|
||||
"$IPSET_DIR/create_ipset.sh"
|
||||
[ -n "$FAIL" ] && exit
|
||||
|
||||
ZREESTR="$TMPDIR/zapret.txt"
|
||||
#ZURL=https://reestr.rublacklist.net/api/current
|
||||
|
||||
@ -9,7 +9,6 @@ ZREESTR="$TMPDIR/reestr.txt"
|
||||
#ZURL_REESTR=https://reestr.rublacklist.net/api/current
|
||||
ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
|
||||
|
||||
getuser
|
||||
|
||||
dig_reestr()
|
||||
{
|
||||
@ -23,29 +22,31 @@ dig_reestr()
|
||||
}
|
||||
|
||||
|
||||
# assume all https banned by ip
|
||||
curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
|
||||
{
|
||||
echo reestr list download failed
|
||||
exit 2
|
||||
getuser && {
|
||||
# assume all https banned by ip
|
||||
curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
|
||||
{
|
||||
echo reestr list download failed
|
||||
exit 2
|
||||
}
|
||||
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
|
||||
if test $dlsize -lt 1048576; then
|
||||
echo reestr ip list is too small. can be bad.
|
||||
exit 2
|
||||
fi
|
||||
#sed -i 's/\\n/\r\n/g' $ZREESTR
|
||||
|
||||
get_ip_regex
|
||||
|
||||
[ "$DISABLE_IPV4" != "1" ] && {
|
||||
dig_reestr "$REG_IPV4" "$ZIPLIST" 4
|
||||
}
|
||||
|
||||
[ "$DISABLE_IPV6" != "1" ] && {
|
||||
dig_reestr "$REG_IPV6" "$ZIPLIST6" 6
|
||||
}
|
||||
|
||||
rm -f "$ZREESTR"
|
||||
}
|
||||
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
|
||||
if test $dlsize -lt 1048576; then
|
||||
echo reestr ip list is too small. can be bad.
|
||||
exit 2
|
||||
fi
|
||||
#sed -i 's/\\n/\r\n/g' $ZREESTR
|
||||
|
||||
get_ip_regex
|
||||
|
||||
[ "$DISABLE_IPV4" != "1" ] && {
|
||||
dig_reestr "$REG_IPV4" "$ZIPLIST" 4
|
||||
}
|
||||
|
||||
[ "$DISABLE_IPV6" != "1" ] && {
|
||||
dig_reestr "$REG_IPV6" "$ZIPLIST6" 6
|
||||
}
|
||||
|
||||
rm -f "$ZREESTR"
|
||||
|
||||
"$IPSET_DIR/create_ipset.sh"
|
||||
|
||||
@ -11,44 +11,45 @@ ZIPLISTTMP="$TMPDIR/zapret-ip.txt"
|
||||
#ZURL=https://reestr.rublacklist.net/api/current
|
||||
ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
|
||||
|
||||
getuser
|
||||
getuser && {
|
||||
# both disabled
|
||||
[ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0
|
||||
|
||||
# both disabled
|
||||
[ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0
|
||||
|
||||
curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL" >"$ZREESTR" ||
|
||||
{
|
||||
echo reestr list download failed
|
||||
exit 2
|
||||
}
|
||||
|
||||
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
|
||||
if test $dlsize -lt 204800; then
|
||||
echo list file is too small. can be bad.
|
||||
exit 2
|
||||
fi
|
||||
|
||||
echo preparing dig list ..
|
||||
LANG=C cut -f2 -d ';' "$ZREESTR" | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG"
|
||||
rm -f "$ZREESTR"
|
||||
|
||||
echo digging started. this can take long ...
|
||||
|
||||
[ "$DISABLE_IPV4" != "1" ] && {
|
||||
digger "$ZDIG" 4 | cut_local >"$ZIPLISTTMP" || {
|
||||
rm -f "$ZDIG"
|
||||
exit 1
|
||||
curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL" >"$ZREESTR" ||
|
||||
{
|
||||
echo reestr list download failed
|
||||
exit 2
|
||||
}
|
||||
ip2net4 <"$ZIPLISTTMP" | zz "$ZIPLIST"
|
||||
rm -f "$ZIPLISTTMP"
|
||||
}
|
||||
[ "$DISABLE_IPV6" != "1" ] && {
|
||||
digger "$ZDIG" 6 | cut_local6 >"$ZIPLISTTMP" || {
|
||||
rm -f "$ZDIG"
|
||||
exit 1
|
||||
|
||||
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
|
||||
if test $dlsize -lt 204800; then
|
||||
echo list file is too small. can be bad.
|
||||
exit 2
|
||||
fi
|
||||
|
||||
echo preparing dig list ..
|
||||
LANG=C cut -f2 -d ';' "$ZREESTR" | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG"
|
||||
rm -f "$ZREESTR"
|
||||
|
||||
echo digging started. this can take long ...
|
||||
|
||||
[ "$DISABLE_IPV4" != "1" ] && {
|
||||
filedigger "$ZDIG" 4 | cut_local >"$ZIPLISTTMP" || {
|
||||
rm -f "$ZDIG"
|
||||
exit 1
|
||||
}
|
||||
ip2net4 <"$ZIPLISTTMP" | zz "$ZIPLIST"
|
||||
rm -f "$ZIPLISTTMP"
|
||||
}
|
||||
ip2net6 <"$ZIPLISTTMP" | zz "$ZIPLIST6"
|
||||
rm -f "$ZIPLISTTMP"
|
||||
[ "$DISABLE_IPV6" != "1" ] && {
|
||||
filedigger "$ZDIG" 6 | cut_local6 >"$ZIPLISTTMP" || {
|
||||
rm -f "$ZDIG"
|
||||
exit 1
|
||||
}
|
||||
ip2net6 <"$ZIPLISTTMP" | zz "$ZIPLIST6"
|
||||
rm -f "$ZIPLISTTMP"
|
||||
}
|
||||
rm -f "$ZDIG"
|
||||
}
|
||||
rm -f "$ZDIG"
|
||||
|
||||
"$IPSET_DIR/create_ipset.sh"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user