reestr lists: support ipban

ipset/get_reestr_hostlist.sh

@@ -5,37 +5,60 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 . "$IPSET_DIR/def.sh"
 
-# useful in case ipban set is used in custom scripts
-FAIL=
-getipban || FAIL=1
-"$IPSET_DIR/create_ipset.sh"
-[ -n "$FAIL" ] && exit
-
 ZREESTR="$TMPDIR/zapret.txt"
-#ZURL=https://reestr.rublacklist.net/api/current
+IPB="$TMPDIR/ipb.txt"
 ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
 
-curl -H "Accept-Encoding: gzip" -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" | gunzip - >"$ZREESTR" ||
+dl_checked()
 {
- echo reestr list download failed   
+  # $1 - url
- exit 2
+  # $2 - file
+  # $3 - minsize
+  # $4 - maxsize
+  # $5 - maxtime
+  curl -H "Accept-Encoding: gzip" -k --fail --max-time $5 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$2" ||
+  {
+   echo list download failed : $1
+   return 2
+  }
+  dlsize=$(LANG=C wc -c "$2" | xargs | cut -f 1 -d ' ')
+  if test $dlsize -lt $3; then
+   echo list is too small : $dlsize bytes. can be bad.
+   return 2
+  fi
+  return 0
 }
 
 reestr_list()
 {
  LANG=C cut -s -f2 -d';' "$ZREESTR" | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' | $AWK '{ print tolower($0) }'
 }
+reestr_extract_ip()
+{
+ LANG=C nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' "$ZREESTR" | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
+}
 
-dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
+ipban_fin()
-if test $dlsize -lt 204800; then
+{
- echo list file is too small. can be bad.
+ getipban
+ "$IPSET_DIR/create_ipset.sh"
+}
+
+dl_checked "$ZURL_REESTR" "$ZREESTR" 204800 251658240 600 || {
+ ipban_fin
  exit 2
-fi
+}
 
 reestr_list | sort -u | zz "$ZHOSTLIST"
 
+reestr_extract_ip <"$ZREESTR" >"$IPB"
 rm -f "$ZREESTR"
+$AWK '/^([0-9]{1,3}\.){3}[0-9]{1,3}$/' "$IPB" | ip2net4 | zz "$ZIPLIST_IPBAN"
+$AWK '/^([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}$/' "$IPB" | ip2net6 | zz "$ZIPLIST_IPBAN6"
+rm -f "$IPB"
 
 hup_zapret_daemons
 
+ipban_fin
+
 exit 0

ipset/get_reestr_preresolved.sh

@@ -5,24 +5,27 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 . "$IPSET_DIR/def.sh"
 
-
+TMPLIST="$TMPDIR/list.txt"
-TMPLIST="$TMPDIR/list_nethub.txt"
 
 BASEURL="https://raw.githubusercontent.com/bol-van/rulist/main"
 URL4="$BASEURL/reestr_resolved4.txt"
 URL6="$BASEURL/reestr_resolved6.txt"
+IPB4="$BASEURL/reestr_ipban4.txt"
+IPB6="$BASEURL/reestr_ipban6.txt"
 
 dl()
 {
   # $1 - url
   # $2 - file
-  curl -H "Accept-Encoding: gzip" -k --fail --max-time 180 --connect-timeout 10 --retry 4 --max-filesize 33554432 "$1" | gunzip - >"$TMPLIST" ||
+  # $3 - minsize
+  # $4 - maxsize
+  curl -H "Accept-Encoding: gzip" -k --fail --max-time 120 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$TMPLIST" ||
   {
    echo list download failed : $1
    exit 2
   }
   dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
-  if test $dlsize -lt 32768; then
+  if test $dlsize -lt $3; then
    echo list is too small : $dlsize bytes. can be bad.
    exit 2
   fi
@@ -31,8 +34,14 @@ dl()
 }
 
 getuser && {
- [ "$DISABLE_IPV4" != "1" ] && dl "$URL4" "$ZIPLIST"
+ [ "$DISABLE_IPV4" != "1" ] && {
- [ "$DISABLE_IPV6" != "1" ] && dl "$URL6" "$ZIPLIST6"
+ 	dl "$URL4" "$ZIPLIST" 32768 4194304
+ 	dl "$IPB4" "$ZIPLIST_IPBAN" 8192 1048576
+ }
+ [ "$DISABLE_IPV6" != "1" ] && {
+ 	dl "$URL6" "$ZIPLIST6" 8192 4194304
+ 	dl "$IPB6" "$ZIPLIST_IPBAN6" 128 1048576
+ }
 }
 
 "$IPSET_DIR/create_ipset.sh"

ipset/get_reestr_preresolved_smart.sh

@@ -5,24 +5,27 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 . "$IPSET_DIR/def.sh"
 
-
+TMPLIST="$TMPDIR/list.txt"
-TMPLIST="$TMPDIR/list_nethub.txt"
 
 BASEURL="https://raw.githubusercontent.com/bol-van/rulist/main"
 URL4="$BASEURL/reestr_smart4.txt"
 URL6="$BASEURL/reestr_smart6.txt"
+IPB4="$BASEURL/reestr_ipban4.txt"
+IPB6="$BASEURL/reestr_ipban6.txt"
 
 dl()
 {
   # $1 - url
   # $2 - file
-  curl -H "Accept-Encoding: gzip" -k --fail --max-time 180 --connect-timeout 10 --retry 4 --max-filesize 33554432 "$1" | gunzip - >"$TMPLIST" ||
+  # $3 - minsize
+  # $4 - maxsize
+  curl -H "Accept-Encoding: gzip" -k --fail --max-time 120 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$TMPLIST" ||
   {
    echo list download failed : $1
    exit 2
   }
   dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
-  if test $dlsize -lt 32768; then
+  if test $dlsize -lt $3; then
    echo list is too small : $dlsize bytes. can be bad.
    exit 2
   fi
@@ -31,8 +34,14 @@ dl()
 }
 
 getuser && {
- [ "$DISABLE_IPV4" != "1" ] && dl "$URL4" "$ZIPLIST"
+ [ "$DISABLE_IPV4" != "1" ] && {
- [ "$DISABLE_IPV6" != "1" ] && dl "$URL6" "$ZIPLIST6"
+ 	dl "$URL4" "$ZIPLIST" 32768 4194304
+ 	dl "$IPB4" "$ZIPLIST_IPBAN" 8192 1048576
+ }
+ [ "$DISABLE_IPV6" != "1" ] && {
+ 	dl "$URL6" "$ZIPLIST6" 8192 4194304
+ 	dl "$IPB6" "$ZIPLIST_IPBAN6" 128 1048576
+ }
 }
 
 "$IPSET_DIR/create_ipset.sh"

ipset/get_reestr_resolvable_domains.sh

@@ -9,18 +9,22 @@ TMPLIST="$TMPDIR/list_nethub.txt"
 
 BASEURL="https://raw.githubusercontent.com/bol-van/rulist/main"
 URL="$BASEURL/reestr_hostname_resolvable.txt"
+IPB4="$BASEURL/reestr_ipban4.txt"
+IPB6="$BASEURL/reestr_ipban6.txt"
 
 dl()
 {
   # $1 - url
   # $2 - file
-  curl -H "Accept-Encoding: gzip" -k --fail --max-time 180 --connect-timeout 10 --retry 4 --max-filesize 33554432 "$1" | gunzip - >"$TMPLIST" ||
+  # $3 - minsize
+  # $4 - maxsize
+  curl -H "Accept-Encoding: gzip" -k --fail --max-time 120 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$TMPLIST" ||
   {
    echo list download failed : $1
    exit 2
   }
   dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
-  if test $dlsize -lt 65536; then
+  if test $dlsize -lt $3; then
    echo list is too small : $dlsize bytes. can be bad.
    exit 2
   fi
@@ -28,8 +32,14 @@ dl()
   rm -f "$TMPLIST"
 }
 
-dl "$URL" "$ZHOSTLIST"
+dl "$URL" "$ZHOSTLIST" 65536 67108864
 
 hup_zapret_daemons
 
+[ "$DISABLE_IPV4" != "1" ] && dl "$IPB4" "$ZIPLIST_IPBAN" 8192 1048576
+[ "$DISABLE_IPV6" != "1" ] && dl "$IPB6" "$ZIPLIST_IPBAN6" 128 1048576
+
+getipban
+"$IPSET_DIR/create_ipset.sh"
+
 exit 0

ipset/get_reestr_resolve.sh

@@ -7,28 +7,56 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 ZREESTR="$TMPDIR/zapret.txt"
 ZDIG="$TMPDIR/zapret-dig.txt"
+IPB="$TMPDIR/ipb.txt"
 ZIPLISTTMP="$TMPDIR/zapret-ip.txt"
 #ZURL=https://reestr.rublacklist.net/api/current
 ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
 
+dl_checked()
+{
+  # $1 - url
+  # $2 - file
+  # $3 - minsize
+  # $4 - maxsize
+  # $5 - maxtime
+  curl -H "Accept-Encoding: gzip" -k --fail --max-time $5 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$2" ||
+  {
+   echo list download failed : $1
+   return 2
+  }
+  dlsize=$(LANG=C wc -c "$2" | xargs | cut -f 1 -d ' ')
+  if test $dlsize -lt $3; then
+   echo list is too small : $dlsize bytes. can be bad.
+   return 2
+  fi
+  return 0
+}
+
+reestr_list()
+{
+ LANG=C cut -s -f2 -d';' "$ZREESTR" | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p'
+}
+reestr_extract_ip()
+{
+ LANG=C nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' "$ZREESTR" | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
+}
+
 getuser && {
  # both disabled
  [ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0
 
- curl -H "Accept-Encoding: gzip" -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" | gunzip - >"$ZREESTR" ||
+ dl_checked "$ZURL_REESTR" "$ZREESTR" 204800 251658240 600 || exit 2
- {
+ 
-  echo reestr list download failed   
+ echo preparing ipban list ..
-  exit 2
+ 
- }
+ reestr_extract_ip <"$ZREESTR" >"$IPB"
-
+ $AWK '/^([0-9]{1,3}\.){3}[0-9]{1,3}$/' "$IPB" | ip2net4 | zz "$ZIPLIST_IPBAN"
- dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
+ $AWK '/^([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}$/' "$IPB" | ip2net6 | zz "$ZIPLIST_IPBAN6"
- if test $dlsize -lt 204800; then
+ rm -f "$IPB"
-  echo list file is too small. can be bad.
-  exit 2
- fi
 
  echo preparing dig list ..
- LANG=C cut -f2 -d ';' "$ZREESTR"  | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG"
+ reestr_list | sort -u >"$ZDIG"
+
  rm -f "$ZREESTR"
 
  echo digging started. this can take long ...