drygdryg/zapret
1
0
Fork 0
mirror of https://github.com/bol-van/zapret.git synced 2024-11-30 05:50:53 +03:00
Code Issues Packages Projects Releases Wiki Activity

Revert "ipset: check dns before processing lists"

Browse Source 
This reverts commit 033def82ffec51155bb44daf7cef408647e5b8cd.
This commit is contained in:
bol-van 2021-10-19 11:22:25 +03:00
parent e9b125e7ef
commit 7355b20544
8 changed files with 111 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
ipset/def.sh
View File

@ -1,8 +1,3 @@
[ -n "$IPSET_DIR" ] || {
IPSET_DIR="$(dirname "$0")"
IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
}
. "$IPSET_DIR/../config"
[ -z "$TMPDIR" ] && TMPDIR=/tmp
@ -52,15 +47,12 @@ exists()
# MacoS in cron does not include /usr/local/bin to PATH
if [ -x /usr/local/bin/ggrep ] ; then
GREP=/usr/local/bin/ggrep
elif [ -x /usr/local/bin/grep ] ; then
GREP=/usr/local/bin/grep
elif exists ggrep; then
GREP=$(which ggrep)
else
GREP=$(which grep)
fi
grep_supports_b()
{
# \b does not work with BSD grep
@ -126,32 +118,19 @@ zzsize()
}
digger()
{
# $1 - family (4|6)
# $2 - s=enable mdig stats
if [ -x "$MDIG" ]; then
local cmd
[ "$2" == "s" ] && cmd=--stats=1000
"$MDIG" --family=$1 --threads=$MDIG_THREADS $1
else
local A=A
[ "$1" = "6" ] && A=AAAA
dig $A +short +time=8 +tries=2 -f - | $GREP -E '^[^;].*[^\.]$'
fi
}
filedigger()
{
# $1 - hostlist
# $2 - family (4|6)
>&2 echo digging $(wc -l <"$1" | xargs) ipv$2 domains : "$1"
zzcat "$1" | digger $2 s
}
dnstest()
{
local ip=$(echo w3.org | digger 46)
[ -n "$ip" ]
}
if [ -x "$MDIG" ]; then
zzcat "$1" | "$MDIG" --family=$2 --threads=$MDIG_THREADS --stats=1000
else
local A=A
[ "$2" = "6" ] && A=AAAA
zzcat "$1" | dig $A +short +time=8 +tries=2 -f - | $GREP -E '^[^;].*[^\.]$'
fi
}
cut_local()
{
@ -173,29 +152,24 @@ oom_adjust_high()
getexclude()
{
oom_adjust_high
dnstest || {
echo "! DNS is not working. list processing aborted."
return 1
}
[ -f "$ZUSERLIST_EXCLUDE" ] && {
[ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST_EXCLUDE" 4 | sort -u > "$ZIPLIST_EXCLUDE"
[ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST_EXCLUDE" 6 | sort -u > "$ZIPLIST_EXCLUDE6"
[ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST_EXCLUDE" 4 | sort -u > "$ZIPLIST_EXCLUDE"
[ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST_EXCLUDE" 6 | sort -u > "$ZIPLIST_EXCLUDE6"
}
return 0
}
getuser()
{
getexclude || return
getexclude
[ -f "$ZUSERLIST" ] && {
[ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST" 4 | cut_local | sort -u > "$ZIPLIST_USER"
[ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST" 6 | cut_local6 | sort -u > "$ZIPLIST_USER6"
[ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST" 4 | cut_local | sort -u > "$ZIPLIST_USER"
[ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST" 6 | cut_local6 | sort -u > "$ZIPLIST_USER6"
}
[ -f "$ZUSERLIST_IPBAN" ] && {
[ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST_IPBAN" 4 | cut_local | sort -u > "$ZIPLIST_USER_IPBAN"
[ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST_IPBAN" 6 | cut_local6 | sort -u > "$ZIPLIST_USER_IPBAN6"
[ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST_IPBAN" 4 | cut_local | sort -u > "$ZIPLIST_USER_IPBAN"
[ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST_IPBAN" 6 | cut_local6 | sort -u > "$ZIPLIST_USER_IPBAN6"
}
return 0
}
hup_zapret_daemons()

9
ipset/get_antifilter_ip.sh
View File

@ -5,9 +5,10 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
. "$IPSET_DIR/def.sh"
getuser && {
. "$IPSET_DIR/antifilter.helper"
get_antifilter https://antifilter.network/download/ip.lst "$ZIPLIST"
}
getuser
. "$IPSET_DIR/antifilter.helper"
get_antifilter https://antifilter.network/download/ip.lst "$ZIPLIST"
"$IPSET_DIR/create_ipset.sh"

9
ipset/get_antifilter_ipsmart.sh
View File

@ -5,9 +5,10 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
. "$IPSET_DIR/def.sh"
getuser && {
. "$IPSET_DIR/antifilter.helper"
get_antifilter https://antifilter.network/download/ipsmart.lst "$ZIPLIST"
}
getuser
. "$IPSET_DIR/antifilter.helper"
get_antifilter https://antifilter.network/download/ipsmart.lst "$ZIPLIST"
"$IPSET_DIR/create_ipset.sh"

9
ipset/get_antifilter_ipsum.sh
View File

@ -5,9 +5,10 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
. "$IPSET_DIR/def.sh"
getuser && {
. "$IPSET_DIR/antifilter.helper"
get_antifilter https://antifilter.network/download/ipsum.lst "$ZIPLIST"
}
getuser
. "$IPSET_DIR/antifilter.helper"
get_antifilter https://antifilter.network/download/ipsum.lst "$ZIPLIST"
"$IPSET_DIR/create_ipset.sh"

33
ipset/get_reestr_combined.sh
View File

@ -9,6 +9,7 @@ ZREESTR="$TMPDIR/reestr.txt"
#ZURL_REESTR=https://reestr.rublacklist.net/api/current
ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
getuser
dig_reestr()
{
@ -35,31 +36,29 @@ dig_reestr()
rm -f "$TMP"
}
getuser && {
curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
{
curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
{
echo reestr list download failed
exit 2
}
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 1048576; then
}
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 1048576; then
echo reestr ip list is too small. can be bad.
exit 2
fi
#sed -i 's/\\n/\r\n/g' $ZREESTR
fi
#sed -i 's/\\n/\r\n/g' $ZREESTR
get_ip_regex
get_ip_regex
[ "$DISABLE_IPV4" != "1" ] && {
[ "$DISABLE_IPV4" != "1" ] && {
dig_reestr "$REG_IPV4" "$ZIPLIST" "$ZIPLIST_IPBAN" 4
}
[ "$DISABLE_IPV6" != "1" ] && {
dig_reestr "$REG_IPV6" "$ZIPLIST6" "$ZIPLIST_IPBAN6" 6
}
rm -f "$ZREESTR"
}
[ "$DISABLE_IPV6" != "1" ] && {
dig_reestr "$REG_IPV6" "$ZIPLIST6" "$ZIPLIST_IPBAN6" 6
}
rm -f "$ZREESTR"
"$IPSET_DIR/create_ipset.sh"

4
ipset/get_reestr_hostlist.sh
View File

@ -6,10 +6,8 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
. "$IPSET_DIR/def.sh"
# useful in case ipban set is used in custom scripts
FAIL=
getuser || FAIL=1
getuser
"$IPSET_DIR/create_ipset.sh"
[ -n "$FAIL" ] && exit
ZREESTR="$TMPDIR/zapret.txt"
#ZURL=https://reestr.rublacklist.net/api/current

35
ipset/get_reestr_ip.sh
View File

@ -9,6 +9,7 @@ ZREESTR="$TMPDIR/reestr.txt"
#ZURL_REESTR=https://reestr.rublacklist.net/api/current
ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
getuser
dig_reestr()
{
@ -22,31 +23,29 @@ dig_reestr()
}
getuser && {
# assume all https banned by ip
curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
{
# assume all https banned by ip
curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
{
echo reestr list download failed
exit 2
}
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 1048576; then
}
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 1048576; then
echo reestr ip list is too small. can be bad.
exit 2
fi
#sed -i 's/\\n/\r\n/g' $ZREESTR
fi
#sed -i 's/\\n/\r\n/g' $ZREESTR
get_ip_regex
get_ip_regex
[ "$DISABLE_IPV4" != "1" ] && {
[ "$DISABLE_IPV4" != "1" ] && {
dig_reestr "$REG_IPV4" "$ZIPLIST" 4
}
[ "$DISABLE_IPV6" != "1" ] && {
dig_reestr "$REG_IPV6" "$ZIPLIST6" 6
}
rm -f "$ZREESTR"
}
[ "$DISABLE_IPV6" != "1" ] && {
dig_reestr "$REG_IPV6" "$ZIPLIST6" 6
}
rm -f "$ZREESTR"
"$IPSET_DIR/create_ipset.sh"

41
ipset/get_reestr_resolve.sh
View File

@ -11,45 +11,44 @@ ZIPLISTTMP="$TMPDIR/zapret-ip.txt"
#ZURL=https://reestr.rublacklist.net/api/current
ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
getuser && {
# both disabled
[ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0
getuser
curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL" >"$ZREESTR" ||
{
# both disabled
[ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0
curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL" >"$ZREESTR" ||
{
echo reestr list download failed
exit 2
}
}
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 204800; then
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 204800; then
echo list file is too small. can be bad.
exit 2
fi
fi
echo preparing dig list ..
LANG=C cut -f2 -d ';' "$ZREESTR" | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG"
rm -f "$ZREESTR"
echo preparing dig list ..
LANG=C cut -f2 -d ';' "$ZREESTR" | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG"
rm -f "$ZREESTR"
echo digging started. this can take long ...
echo digging started. this can take long ...
[ "$DISABLE_IPV4" != "1" ] && {
filedigger "$ZDIG" 4 | cut_local >"$ZIPLISTTMP" || {
[ "$DISABLE_IPV4" != "1" ] && {
digger "$ZDIG" 4 | cut_local >"$ZIPLISTTMP" || {
rm -f "$ZDIG"
exit 1
}
ip2net4 <"$ZIPLISTTMP" | zz "$ZIPLIST"
rm -f "$ZIPLISTTMP"
}
[ "$DISABLE_IPV6" != "1" ] && {
filedigger "$ZDIG" 6 | cut_local6 >"$ZIPLISTTMP" || {
}
[ "$DISABLE_IPV6" != "1" ] && {
digger "$ZDIG" 6 | cut_local6 >"$ZIPLISTTMP" || {
rm -f "$ZDIG"
exit 1
}
ip2net6 <"$ZIPLISTTMP" | zz "$ZIPLIST6"
rm -f "$ZIPLISTTMP"
}
rm -f "$ZDIG"
}
rm -f "$ZDIG"
"$IPSET_DIR/create_ipset.sh"