From e7169717395b9f564f113d819879a48dc804291f Mon Sep 17 00:00:00 2001
From: bol-van <none@none.none>
Date: Sun, 26 Nov 2023 14:31:37 +0300
Subject: [PATCH] reestr lists: support ipban

---
 ipset/get_reestr_hostlist.sh           | 51 ++++++++++++++++++-------
 ipset/get_reestr_preresolved.sh        | 21 ++++++++---
 ipset/get_reestr_preresolved_smart.sh  | 21 ++++++++---
 ipset/get_reestr_resolvable_domains.sh | 16 ++++++--
 ipset/get_reestr_resolve.sh            | 52 ++++++++++++++++++++------
 5 files changed, 120 insertions(+), 41 deletions(-)

diff --git a/ipset/get_reestr_hostlist.sh b/ipset/get_reestr_hostlist.sh
index f0883e1..aa5ad18 100755
--- a/ipset/get_reestr_hostlist.sh
+++ b/ipset/get_reestr_hostlist.sh
@@ -5,37 +5,60 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 . "$IPSET_DIR/def.sh"
 
-# useful in case ipban set is used in custom scripts
-FAIL=
-getipban || FAIL=1
-"$IPSET_DIR/create_ipset.sh"
-[ -n "$FAIL" ] && exit
-
 ZREESTR="$TMPDIR/zapret.txt"
-#ZURL=https://reestr.rublacklist.net/api/current
+IPB="$TMPDIR/ipb.txt"
 ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
 
-curl -H "Accept-Encoding: gzip" -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" | gunzip - >"$ZREESTR" ||
+dl_checked()
 {
- echo reestr list download failed   
- exit 2
+  # $1 - url
+  # $2 - file
+  # $3 - minsize
+  # $4 - maxsize
+  # $5 - maxtime
+  curl -H "Accept-Encoding: gzip" -k --fail --max-time $5 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$2" ||
+  {
+   echo list download failed : $1
+   return 2
+  }
+  dlsize=$(LANG=C wc -c "$2" | xargs | cut -f 1 -d ' ')
+  if test $dlsize -lt $3; then
+   echo list is too small : $dlsize bytes. can be bad.
+   return 2
+  fi
+  return 0
 }
 
 reestr_list()
 {
  LANG=C cut -s -f2 -d';' "$ZREESTR" | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' | $AWK '{ print tolower($0) }'
 }
+reestr_extract_ip()
+{
+ LANG=C nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' "$ZREESTR" | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
+}
 
-dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
-if test $dlsize -lt 204800; then
- echo list file is too small. can be bad.
+ipban_fin()
+{
+ getipban
+ "$IPSET_DIR/create_ipset.sh"
+}
+
+dl_checked "$ZURL_REESTR" "$ZREESTR" 204800 251658240 600 || {
+ ipban_fin
  exit 2
-fi
+}
 
 reestr_list | sort -u | zz "$ZHOSTLIST"
 
+reestr_extract_ip <"$ZREESTR" >"$IPB"
 rm -f "$ZREESTR"
+$AWK '/^([0-9]{1,3}\.){3}[0-9]{1,3}$/' "$IPB" | ip2net4 | zz "$ZIPLIST_IPBAN"
+$AWK '/^([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}$/' "$IPB" | ip2net6 | zz "$ZIPLIST_IPBAN6"
+rm -f "$IPB"
 
 hup_zapret_daemons
 
+ipban_fin
+
 exit 0
diff --git a/ipset/get_reestr_preresolved.sh b/ipset/get_reestr_preresolved.sh
index 984035a..db6f288 100755
--- a/ipset/get_reestr_preresolved.sh
+++ b/ipset/get_reestr_preresolved.sh
@@ -5,24 +5,27 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 . "$IPSET_DIR/def.sh"
 
-
-TMPLIST="$TMPDIR/list_nethub.txt"
+TMPLIST="$TMPDIR/list.txt"
 
 BASEURL="https://raw.githubusercontent.com/bol-van/rulist/main"
 URL4="$BASEURL/reestr_resolved4.txt"
 URL6="$BASEURL/reestr_resolved6.txt"
+IPB4="$BASEURL/reestr_ipban4.txt"
+IPB6="$BASEURL/reestr_ipban6.txt"
 
 dl()
 {
   # $1 - url
   # $2 - file
-  curl -H "Accept-Encoding: gzip" -k --fail --max-time 180 --connect-timeout 10 --retry 4 --max-filesize 33554432 "$1" | gunzip - >"$TMPLIST" ||
+  # $3 - minsize
+  # $4 - maxsize
+  curl -H "Accept-Encoding: gzip" -k --fail --max-time 120 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$TMPLIST" ||
   {
    echo list download failed : $1
    exit 2
   }
   dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
-  if test $dlsize -lt 32768; then
+  if test $dlsize -lt $3; then
    echo list is too small : $dlsize bytes. can be bad.
    exit 2
   fi
@@ -31,8 +34,14 @@ dl()
 }
 
 getuser && {
- [ "$DISABLE_IPV4" != "1" ] && dl "$URL4" "$ZIPLIST"
- [ "$DISABLE_IPV6" != "1" ] && dl "$URL6" "$ZIPLIST6"
+ [ "$DISABLE_IPV4" != "1" ] && {
+ 	dl "$URL4" "$ZIPLIST" 32768 4194304
+ 	dl "$IPB4" "$ZIPLIST_IPBAN" 8192 1048576
+ }
+ [ "$DISABLE_IPV6" != "1" ] && {
+ 	dl "$URL6" "$ZIPLIST6" 8192 4194304
+ 	dl "$IPB6" "$ZIPLIST_IPBAN6" 128 1048576
+ }
 }
 
 "$IPSET_DIR/create_ipset.sh"
diff --git a/ipset/get_reestr_preresolved_smart.sh b/ipset/get_reestr_preresolved_smart.sh
index e8a70ae..f4a1395 100755
--- a/ipset/get_reestr_preresolved_smart.sh
+++ b/ipset/get_reestr_preresolved_smart.sh
@@ -5,24 +5,27 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 . "$IPSET_DIR/def.sh"
 
-
-TMPLIST="$TMPDIR/list_nethub.txt"
+TMPLIST="$TMPDIR/list.txt"
 
 BASEURL="https://raw.githubusercontent.com/bol-van/rulist/main"
 URL4="$BASEURL/reestr_smart4.txt"
 URL6="$BASEURL/reestr_smart6.txt"
+IPB4="$BASEURL/reestr_ipban4.txt"
+IPB6="$BASEURL/reestr_ipban6.txt"
 
 dl()
 {
   # $1 - url
   # $2 - file
-  curl -H "Accept-Encoding: gzip" -k --fail --max-time 180 --connect-timeout 10 --retry 4 --max-filesize 33554432 "$1" | gunzip - >"$TMPLIST" ||
+  # $3 - minsize
+  # $4 - maxsize
+  curl -H "Accept-Encoding: gzip" -k --fail --max-time 120 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$TMPLIST" ||
   {
    echo list download failed : $1
    exit 2
   }
   dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
-  if test $dlsize -lt 32768; then
+  if test $dlsize -lt $3; then
    echo list is too small : $dlsize bytes. can be bad.
    exit 2
   fi
@@ -31,8 +34,14 @@ dl()
 }
 
 getuser && {
- [ "$DISABLE_IPV4" != "1" ] && dl "$URL4" "$ZIPLIST"
- [ "$DISABLE_IPV6" != "1" ] && dl "$URL6" "$ZIPLIST6"
+ [ "$DISABLE_IPV4" != "1" ] && {
+ 	dl "$URL4" "$ZIPLIST" 32768 4194304
+ 	dl "$IPB4" "$ZIPLIST_IPBAN" 8192 1048576
+ }
+ [ "$DISABLE_IPV6" != "1" ] && {
+ 	dl "$URL6" "$ZIPLIST6" 8192 4194304
+ 	dl "$IPB6" "$ZIPLIST_IPBAN6" 128 1048576
+ }
 }
 
 "$IPSET_DIR/create_ipset.sh"
diff --git a/ipset/get_reestr_resolvable_domains.sh b/ipset/get_reestr_resolvable_domains.sh
index 010b598..fa4837f 100755
--- a/ipset/get_reestr_resolvable_domains.sh
+++ b/ipset/get_reestr_resolvable_domains.sh
@@ -9,18 +9,22 @@ TMPLIST="$TMPDIR/list_nethub.txt"
 
 BASEURL="https://raw.githubusercontent.com/bol-van/rulist/main"
 URL="$BASEURL/reestr_hostname_resolvable.txt"
+IPB4="$BASEURL/reestr_ipban4.txt"
+IPB6="$BASEURL/reestr_ipban6.txt"
 
 dl()
 {
   # $1 - url
   # $2 - file
-  curl -H "Accept-Encoding: gzip" -k --fail --max-time 180 --connect-timeout 10 --retry 4 --max-filesize 33554432 "$1" | gunzip - >"$TMPLIST" ||
+  # $3 - minsize
+  # $4 - maxsize
+  curl -H "Accept-Encoding: gzip" -k --fail --max-time 120 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$TMPLIST" ||
   {
    echo list download failed : $1
    exit 2
   }
   dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
-  if test $dlsize -lt 65536; then
+  if test $dlsize -lt $3; then
    echo list is too small : $dlsize bytes. can be bad.
    exit 2
   fi
@@ -28,8 +32,14 @@ dl()
   rm -f "$TMPLIST"
 }
 
-dl "$URL" "$ZHOSTLIST"
+dl "$URL" "$ZHOSTLIST" 65536 67108864
 
 hup_zapret_daemons
 
+[ "$DISABLE_IPV4" != "1" ] && dl "$IPB4" "$ZIPLIST_IPBAN" 8192 1048576
+[ "$DISABLE_IPV6" != "1" ] && dl "$IPB6" "$ZIPLIST_IPBAN6" 128 1048576
+
+getipban
+"$IPSET_DIR/create_ipset.sh"
+
 exit 0
diff --git a/ipset/get_reestr_resolve.sh b/ipset/get_reestr_resolve.sh
index 59bad01..74b7201 100755
--- a/ipset/get_reestr_resolve.sh
+++ b/ipset/get_reestr_resolve.sh
@@ -7,28 +7,56 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 ZREESTR="$TMPDIR/zapret.txt"
 ZDIG="$TMPDIR/zapret-dig.txt"
+IPB="$TMPDIR/ipb.txt"
 ZIPLISTTMP="$TMPDIR/zapret-ip.txt"
 #ZURL=https://reestr.rublacklist.net/api/current
 ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
 
+dl_checked()
+{
+  # $1 - url
+  # $2 - file
+  # $3 - minsize
+  # $4 - maxsize
+  # $5 - maxtime
+  curl -H "Accept-Encoding: gzip" -k --fail --max-time $5 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$2" ||
+  {
+   echo list download failed : $1
+   return 2
+  }
+  dlsize=$(LANG=C wc -c "$2" | xargs | cut -f 1 -d ' ')
+  if test $dlsize -lt $3; then
+   echo list is too small : $dlsize bytes. can be bad.
+   return 2
+  fi
+  return 0
+}
+
+reestr_list()
+{
+ LANG=C cut -s -f2 -d';' "$ZREESTR" | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p'
+}
+reestr_extract_ip()
+{
+ LANG=C nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' "$ZREESTR" | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
+}
+
 getuser && {
  # both disabled
  [ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0
 
- curl -H "Accept-Encoding: gzip" -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" | gunzip - >"$ZREESTR" ||
- {
-  echo reestr list download failed   
-  exit 2
- }
-
- dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
- if test $dlsize -lt 204800; then
-  echo list file is too small. can be bad.
-  exit 2
- fi
+ dl_checked "$ZURL_REESTR" "$ZREESTR" 204800 251658240 600 || exit 2
+ 
+ echo preparing ipban list ..
+ 
+ reestr_extract_ip <"$ZREESTR" >"$IPB"
+ $AWK '/^([0-9]{1,3}\.){3}[0-9]{1,3}$/' "$IPB" | ip2net4 | zz "$ZIPLIST_IPBAN"
+ $AWK '/^([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}$/' "$IPB" | ip2net6 | zz "$ZIPLIST_IPBAN6"
+ rm -f "$IPB"
 
  echo preparing dig list ..
- LANG=C cut -f2 -d ';' "$ZREESTR"  | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG"
+ reestr_list | sort -u >"$ZDIG"
+
  rm -f "$ZREESTR"
 
  echo digging started. this can take long ...