history purge

ipset/antifilter.helper

@@ -0,0 +1,19 @@
+get_antifilter()
+{
+ # $1 - list url
+ # $2 - target file
+ local ZIPLISTTMP="$TMPDIR/zapret-ip.txt"
+
+ [ "$DISABLE_IPV4" != "1" ] && {
+  curl --fail --max-time 150 --connect-timeout 20 --max-filesize 41943040 -k -L "$1" | cut_local >"$ZIPLISTTMP" &&
+  {
+   dlsize=$(LANG=C wc -c "$ZIPLISTTMP" | xargs | cut -f 1 -d ' ')
+   if [ $dlsize -lt 204800 ]; then
+    echo list file is too small. can be bad.
+    exit 2
+   fi
+   ip2net4 <"$ZIPLISTTMP" | zz "$2"
+   rm -f "$ZIPLISTTMP"
+  }
+ }
+}

ipset/clear_lists.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+IPSET_DIR="$(dirname "$0")"
+IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
+
+. "$IPSET_DIR/def.sh"
+
+rm -f "$ZIPLIST"* "$ZIPLIST6"* "$ZIPLIST_USER" "$ZIPLIST_USER6" "$ZIPLIST_IPBAN"* "$ZIPLIST_IPBAN6"* "$ZIPLIST_USER_IPBAN" "$ZIPLIST_USER_IPBAN6" "$ZIPLIST_EXCLUDE" "$ZIPLIST_EXCLUDE6" "$ZHOSTLIST"*

ipset/create_ipset.sh

@@ -0,0 +1,201 @@
+#!/bin/sh
+
+# create ipset or ipfw table from resolved ip's
+# $1=no-update    - do not update ipset, only create if its absent
+
+IPSET_DIR="$(dirname "$0")"
+IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
+
+. "$IPSET_DIR/def.sh"
+
+IPSET_CMD="$TMPDIR/ipset_cmd.txt"
+IPSET_SAVERAM_CHUNK_SIZE=20000
+IPSET_SAVERAM_MIN_FILESIZE=131072
+
+
+while [ -n "$1" ]; do
+	[ "$1" = "no-update" ] && NO_UPDATE=1
+	[ "$1" = "clear" ] && DO_CLEAR=1
+	shift
+done
+
+
+file_extract_lines()
+{
+ # $1 - filename
+ # $2 - from line (starting with 0)
+ # $3 - line count
+ # awk "{ err=1 } NR < $(($2+1)) { next } { print; err=0 } NR == $(($2+$3)) { exit err } END {exit err}" "$1"
+ awk "NR < $(($2+1)) { next } { print } NR == $(($2+$3)) { exit }" "$1"
+}
+ipset_restore_chunked()
+{
+ # $1 - filename
+ # $2 - chunk size
+ local pos lines
+ [ -f "$1" ] || return
+ lines=$(wc -l <"$1")
+ pos=$lines
+ while [ "$pos" -gt "0" ]; do
+    pos=$((pos-$2))
+    [ "$pos" -lt "0" ] && pos=0
+    file_extract_lines "$1" $pos $2 | ipset -! restore
+    sed -i "$(($pos+1)),$ d" "$1"
+ done
+}
+
+
+ipset_get_script()
+{
+ # $1 - filename
+ # $2 - ipset name
+ zzcat "$1" | sort -u | sed -nEe "s/^.+$/add $2 &/p"
+}
+
+ipset_restore()
+{
+ # $1 - filename
+ # $2 - ipset name
+ # $3 - "6" = ipv6
+ zzexist "$1" || return
+ local fsize=$(zzsize "$1")
+ local svram=0
+ # do not saveram small files. file can also be gzipped
+ [ "$SAVERAM" = "1" ] && [ "$fsize" -ge "$IPSET_SAVERAM_MIN_FILESIZE" ] && svram=1
+
+ local T="Adding to ipset $2 ($IPSTYPE"
+ [ "$svram" = "1" ] && T="$T, saveram"
+ T="$T) : $f"
+ echo $T
+
+ if [ "$svram" = "1" ]; then
+  ipset_get_script "$1" "$2" >"$IPSET_CMD"
+  ipset_restore_chunked "$IPSET_CMD" $IPSET_SAVERAM_CHUNK_SIZE
+  rm -f "$IPSET_CMD"
+ else
+  ipset_get_script "$1" "$2" | ipset -! restore
+ fi
+}
+
+create_ipset()
+{
+ if [ "$1" -eq "6" ]; then
+  FAMILY=inet6
+ else
+  FAMILY=inet
+ fi
+ ipset create $2 $3 $4 family $FAMILY 2>/dev/null || {
+  [ "$NO_UPDATE" = "1" ] && return
+ }
+ ipset flush $2
+ [ "$DO_CLEAR" = "1" ] || {
+  for f in "$5" "$6" ; do
+   ipset_restore "$f" "$2" $1
+  done
+ }
+ return 0
+}
+
+
+add_ipfw_table()
+{
+ # $1 - table name
+ sed -nEe "s/^.+$/table $1 add &/p" | ipfw -q /dev/stdin
+}
+populate_ipfw_table()
+{
+ # $1 - table name
+ # $2 - ip list file
+ zzexist "$2" || return
+ zzcat "$2" | sort -u | add_ipfw_table $1
+}
+create_ipfw_table()
+{
+ # $1 - table name
+ # $2 - table options
+ # $3,$4, ... - ip list files. can be v4,v6 or mixed
+
+ local name=$1
+ ipfw table "$name" create $2 2>/dev/null || {
+  [ "$NO_UPDATE" = "1" ] && return
+ }
+ ipfw -q table $1 flush
+ shift
+ shift
+ [ "$DO_CLEAR" = "1" ] || {
+  while [ -n "$1" ]; do
+   populate_ipfw_table $name "$1"
+   shift
+  done
+ }
+}
+
+print_reloading_backend()
+{
+ # $1 - backend name
+ local s="reloading $1 backend"
+ if [ "$NO_UPDATE" = 1 ]; then
+  s="$s (no-update)"
+ else
+  s="$s (forced-update)"
+ fi
+ echo $s
+}
+
+
+oom_adjust_high
+
+if [ -n "$LISTS_RELOAD" ] ; then
+ if [ "$LISTS_RELOAD" = "-" ] ; then
+  echo not reloading ip list backend
+  true
+ else
+  echo executing custom ip list reload command : $LISTS_RELOAD
+  $LISTS_RELOAD
+ fi
+elif exists ipset; then
+ # ipset seem to buffer the whole script to memory
+ # on low RAM system this can cause oom errors
+ # in SAVERAM mode we feed script lines in portions starting from the end, while truncating source file to free /tmp space
+ # only /tmp is considered tmpfs. other locations mean tmpdir was redirected to a disk
+ SAVERAM=0
+ [ "$TMPDIR" = "/tmp" ] && {
+  RAMSIZE=$($GREP MemTotal /proc/meminfo | awk '{print $2}')
+  [ "$RAMSIZE" -lt "110000" ] && SAVERAM=1
+ }
+ print_reloading_backend ipset
+ [ "$DISABLE_IPV4" != "1" ] && {
+   create_ipset 4 $ZIPSET hash:net "$IPSET_OPT" "$ZIPLIST" "$ZIPLIST_USER"
+   create_ipset 4 $ZIPSET_IPBAN hash:net "$IPSET_OPT" "$ZIPLIST_IPBAN" "$ZIPLIST_USER_IPBAN"
+   create_ipset 4 $ZIPSET_EXCLUDE hash:net "$IPSET_OPT_EXCLUDE" "$ZIPLIST_EXCLUDE"
+ }
+ [ "$DISABLE_IPV6" != "1" ] && {
+   create_ipset 6 $ZIPSET6 hash:net "$IPSET_OPT" "$ZIPLIST6" "$ZIPLIST_USER6"
+   create_ipset 6 $ZIPSET_IPBAN6 hash:net "$IPSET_OPT" "$ZIPLIST_IPBAN6" "$ZIPLIST_USER_IPBAN6"
+   create_ipset 6 $ZIPSET_EXCLUDE6 hash:net "$IPSET_OPT_EXCLUDE" "$ZIPLIST_EXCLUDE6"
+ }
+ true
+elif exists ipfw; then
+ print_reloading_backend "ipfw table"
+ if [ "$DISABLE_IPV4" != "1" ] && [ "$DISABLE_IPV6" != "1" ]; then
+  create_ipfw_table $ZIPSET "$IPFW_TABLE_OPT" "$ZIPLIST" "$ZIPLIST_USER" "$ZIPLIST6" "$ZIPLIST_USER6"
+  create_ipfw_table $ZIPSET_IPBAN "$IPFW_TABLE_OPT" "$ZIPLIST_IPBAN" "$ZIPLIST_USER_IPBAN" "$ZIPLIST_IPBAN6" "$ZIPLIST_USER_IPBAN6"
+  create_ipfw_table $ZIPSET_EXCLUDE "$IPFW_TABLE_OPT_EXCLUDE" "$ZIPLIST_EXCLUDE" "$ZIPLIST_EXCLUDE6"
+ elif [ "$DISABLE_IPV4" != "1" ]; then
+  create_ipfw_table $ZIPSET "$IPFW_TABLE_OPT" "$ZIPLIST" "$ZIPLIST_USER"
+  create_ipfw_table $ZIPSET_IPBAN "$IPFW_TABLE_OPT" "$ZIPLIST_IPBAN" "$ZIPLIST_USER_IPBAN"
+  create_ipfw_table $ZIPSET_EXCLUDE "$IPFW_TABLE_OPT_EXCLUDE" "$ZIPLIST_EXCLUDE"
+ elif [ "$DISABLE_IPV6" != "1" ]; then
+  create_ipfw_table $ZIPSET "$IPFW_TABLE_OPT" "$ZIPLIST6" "$ZIPLIST_USER6"
+  create_ipfw_table $ZIPSET_IPBAN "$IPFW_TABLE_OPT" "$ZIPLIST_IPBAN6" "$ZIPLIST_USER_IPBAN6"
+  create_ipfw_table $ZIPSET_EXCLUDE "$IPFW_TABLE_OPT_EXCLUDE" "$ZIPLIST_EXCLUDE6"
+ else
+  create_ipfw_table $ZIPSET "$IPFW_TABLE_OPT"
+  create_ipfw_table $ZIPSET_IPBAN "$IPFW_TABLE_OPT"
+  create_ipfw_table $ZIPSET_EXCLUDE "$IPFW_TABLE_OPT_EXCLUDE"
+ fi
+ true
+else
+ echo no supported ip list backend found
+ true
+fi

ipset/def.sh

@@ -0,0 +1,186 @@
+. "$IPSET_DIR/../config"
+
+[ -z "$TMPDIR" ] && TMPDIR=/tmp
+[ -z "$GZIP_LISTS" ] && GZIP_LISTS=1
+
+[ -z "$IPSET_OPT" ] && IPSET_OPT="hashsize 262144 maxelem 2097152"
+[ -z "$IPSET_OPT_EXCLUDE" ] && IPSET_OPT_EXCLUDE="hashsize 1024 maxelem 65536"
+
+[ -z "$IPFW_TABLE_OPT" ] && IPFW_TABLE_OPT="algo addr:radix"
+[ -z "$IPFW_TABLE_OPT_EXCLUDE" ] && IPFW_TABLE_OPT_EXCLUDE="algo addr:radix"
+
+
+ZIPSET=zapret
+ZIPSET6=zapret6
+ZIPSET_EXCLUDE=nozapret
+ZIPSET_EXCLUDE6=nozapret6
+ZIPLIST="$IPSET_DIR/zapret-ip.txt"
+ZIPLIST6="$IPSET_DIR/zapret-ip6.txt"
+ZIPLIST_EXCLUDE="$IPSET_DIR/zapret-ip-exclude.txt"
+ZIPLIST_EXCLUDE6="$IPSET_DIR/zapret-ip-exclude6.txt"
+ZIPLIST_USER="$IPSET_DIR/zapret-ip-user.txt"
+ZIPLIST_USER6="$IPSET_DIR/zapret-ip-user6.txt"
+ZUSERLIST="$IPSET_DIR/zapret-hosts-user.txt"
+ZHOSTLIST="$IPSET_DIR/zapret-hosts.txt"
+
+ZIPSET_IPBAN=ipban
+ZIPSET_IPBAN6=ipban6
+ZIPLIST_IPBAN="$IPSET_DIR/zapret-ip-ipban.txt"
+ZIPLIST_IPBAN6="$IPSET_DIR/zapret-ip-ipban6.txt"
+ZIPLIST_USER_IPBAN="$IPSET_DIR/zapret-ip-user-ipban.txt"
+ZIPLIST_USER_IPBAN6="$IPSET_DIR/zapret-ip-user-ipban6.txt"
+ZUSERLIST_IPBAN="$IPSET_DIR/zapret-hosts-user-ipban.txt"
+ZUSERLIST_EXCLUDE="$IPSET_DIR/zapret-hosts-user-exclude.txt"
+
+
+IP2NET="$IPSET_DIR/../ip2net/ip2net"
+MDIG="$IPSET_DIR/../mdig/mdig"
+[ -z "$MDIG_THREADS" ] && MDIG_THREADS=30
+
+
+exists()
+{
+ which "$1" >/dev/null 2>/dev/null
+}
+
+# BSD grep is damn slow with -f option. prefer GNU grep (ggrep) if present
+# MacoS in cron does not include /usr/local/bin to PATH
+if [ -x /usr/local/bin/ggrep ] ; then
+ GREP=/usr/local/bin/ggrep
+elif exists ggrep; then
+ GREP=$(which ggrep)
+else
+ GREP=$(which grep)
+fi
+
+grep_supports_b()
+{
+ # \b does not work with BSD grep
+ $GREP --version 2>&1 | $GREP -qE "BusyBox|GNU"
+}
+get_ip_regex()
+{
+ REG_IPV4='((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(/([0-9]|[12][0-9]|3[012]))?'
+ REG_IPV6='[0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}|:)+(/([0-9][0-9]?|1[01][0-9]|12[0-8]))?'
+ # good but too slow
+ # REG_IPV6='([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}(/[0-9]+)?|([0-9a-fA-F]{1,4}:){1,7}:(/[0-9]+)?|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}(/[0-9]+)?|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}(/[0-9]+)?|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}(/[0-9]+)?|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}(/[0-9]+)?|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}(/[0-9]+)?|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})(/[0-9]+)?|:((:[0-9a-fA-F]{1,4}){1,7}|:)(/([0-9][0-9]?|1[01][0-9]|12[0-8]))?'
+ grep_supports_b && {
+  REG_IPV4="\b$REG_IPV4\b"
+  REG_IPV6="\b$REG_IPV6\b"
+ }
+}
+
+
+ip2net4()
+{
+ if [ -x "$IP2NET" ]; then
+  "$IP2NET" -4 $IP2NET_OPT4
+ else
+  sort -u
+ fi
+}
+ip2net6()
+{
+ if [ -x "$IP2NET" ]; then
+  "$IP2NET" -6 $IP2NET_OPT6
+ else
+  sort -u
+ fi
+}
+
+zzexist()
+{
+ [ -f "$1.gz" ] || [ -f "$1" ]
+}
+zzcat()
+{
+ if [ -f "$1.gz" ]; then
+ 	gunzip -c "$1.gz"
+ else
+ 	cat "$1"
+ fi
+}
+zz()
+{
+ if [ "$GZIP_LISTS" = "1" ]; then
+  gzip -c >"$1.gz"
+  rm -f "$1"
+ else
+  cat >"$1"
+  rm -f "$1.gz"
+ fi
+}
+zzsize()
+{
+ local f="$1"
+ [ -f "$1.gz" ] && f="$1.gz"
+ wc -c <"$f" | xargs
+}
+
+digger()
+{
+ # $1 - hostlist
+ # $2 - family (4|6)
+ >&2 echo digging $(wc -l <"$1" | xargs) ipv$2 domains : "$1"
+
+ if [ -x "$MDIG" ]; then
+  zzcat "$1" | "$MDIG" --family=$2 --threads=$MDIG_THREADS --stats=1000
+ else
+  local A=A
+  [ "$2" = "6" ] && A=AAAA
+  zzcat "$1" | dig $A +short +time=8 +tries=2 -f - | $GREP -E '^[^;].*[^\.]$'
+ fi
+}
+
+cut_local()
+{
+  $GREP -vE '^192\.168\.|^127\.|^10\.'
+}
+cut_local6()
+{
+  $GREP -vE '^::|^fc..:|^fd..:'
+}
+
+oom_adjust_high()
+{
+	[ -f /proc/$$/oom_score_adj ] && {
+		echo setting high oom kill priority
+		echo -n 100 >/proc/$$/oom_score_adj
+	}
+}
+
+getexclude()
+{
+ oom_adjust_high
+
+ [ -f "$ZUSERLIST_EXCLUDE" ] && {
+  [ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST_EXCLUDE" 4 | sort -u > "$ZIPLIST_EXCLUDE"
+  [ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST_EXCLUDE" 6 | sort -u > "$ZIPLIST_EXCLUDE6"
+ }
+}
+
+getuser()
+{
+ getexclude
+ [ -f "$ZUSERLIST" ] && {
+  [ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST" 4 | cut_local | sort -u > "$ZIPLIST_USER"
+  [ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST" 6 | cut_local6 | sort -u > "$ZIPLIST_USER6"
+ }
+ [ -f "$ZUSERLIST_IPBAN" ] && {
+  [ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST_IPBAN" 4 | cut_local | sort -u > "$ZIPLIST_USER_IPBAN"
+  [ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST_IPBAN" 6 | cut_local6 | sort -u > "$ZIPLIST_USER_IPBAN6"
+ }
+}
+
+hup_zapret_daemons()
+{
+ echo forcing zapret daemons to reload their hostlist
+ if exists killall; then
+  kcmd=killall
+  killall -HUP tpws nfqws dvtws 2>/dev/null
+ elif exists pkill; then
+  pkill -HUP ^tpws$ ^nfqws$ ^dvtws$
+ else
+  echo no mass killer available ! cant HUP zapret daemons
+ fi
+}

ipset/get_antifilter_ip.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+IPSET_DIR="$(dirname "$0")"
+IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
+
+. "$IPSET_DIR/def.sh"
+
+getuser
+
+. "$IPSET_DIR/antifilter.helper"
+
+get_antifilter https://antifilter.network/download/ip.lst "$ZIPLIST"
+
+"$IPSET_DIR/create_ipset.sh"

ipset/get_antifilter_ipsmart.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+IPSET_DIR="$(dirname "$0")"
+IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
+
+. "$IPSET_DIR/def.sh"
+
+getuser
+
+. "$IPSET_DIR/antifilter.helper"
+
+get_antifilter https://antifilter.network/download/ipsmart.lst "$ZIPLIST"
+
+"$IPSET_DIR/create_ipset.sh"

ipset/get_antifilter_ipsum.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+IPSET_DIR="$(dirname "$0")"
+IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
+
+. "$IPSET_DIR/def.sh"
+
+getuser
+
+. "$IPSET_DIR/antifilter.helper"
+
+get_antifilter https://antifilter.network/download/ipsum.lst "$ZIPLIST"
+
+"$IPSET_DIR/create_ipset.sh"

ipset/get_config.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+# run script specified in config
+
+IPSET_DIR="$(dirname "$0")"
+IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
+
+. "$IPSET_DIR/../config"
+
+[ -z "$GETLIST" ] && GETLIST=get_exclude.sh
+[ -x "$IPSET_DIR/$GETLIST" ] && exec "$IPSET_DIR/$GETLIST"

ipset/get_exclude.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+# resolve user host list
+
+IPSET_DIR="$(dirname "$0")"
+IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
+
+. "$IPSET_DIR/def.sh"
+
+getexclude
+
+"$IPSET_DIR/create_ipset.sh"

ipset/get_reestr_combined.sh

@@ -0,0 +1,64 @@
+#!/bin/sh
+
+IPSET_DIR="$(dirname "$0")"
+IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
+
+. "$IPSET_DIR/def.sh"
+
+ZREESTR="$TMPDIR/reestr.txt"
+#ZURL_REESTR=https://reestr.rublacklist.net/api/current
+ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
+
+getuser
+
+dig_reestr()
+{
+ # $1 - grep ipmask
+ # $2 - iplist
+ # $3 - ipban list
+ # $4 - ip version : 4,6
+
+ local DOMMASK='^.*;[^ ;:/]+\.[^ ;:/]+;'
+ local TMP="$TMPDIR/tmp.txt"
+
+ echo processing reestr lists $2 $3
+
+ # find entries with https or without domain name - they should be banned by IP
+ # 2971-18 is TELEGRAM. lots of proxy IPs banned, list grows very large
+ (nice -n 5 $GREP -avE "$DOMMASK" "$ZREESTR" ; $GREP -a "https://" "$ZREESTR") |
+  nice -n 5 $GREP -oE "$1" | cut_local | sort -u >$TMP
+
+ ip2net$4 <"$TMP" | zz "$3" 
+
+ # other IPs go to regular zapret list
+ tail -n +2 "$ZREESTR"  |  nice -n 5 $GREP -oE "$1" | cut_local | nice -n 5 $GREP -xvFf "$TMP" | ip2net$4 | zz "$2"
+
+ rm -f "$TMP"
+}
+
+
+curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
+{
+ echo reestr list download failed
+ exit 2
+}
+dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
+if test $dlsize -lt 1048576; then
+ echo reestr ip list is too small. can be bad.
+ exit 2
+fi
+#sed -i 's/\\n/\r\n/g' $ZREESTR
+
+get_ip_regex
+
+[ "$DISABLE_IPV4" != "1" ] && {
+ dig_reestr "$REG_IPV4" "$ZIPLIST" "$ZIPLIST_IPBAN" 4
+}
+
+[ "$DISABLE_IPV6" != "1" ] && {
+ dig_reestr "$REG_IPV6" "$ZIPLIST6" "$ZIPLIST_IPBAN6" 6
+}
+
+rm -f "$ZREESTR"
+
+"$IPSET_DIR/create_ipset.sh"

ipset/get_reestr_hostlist.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+IPSET_DIR="$(dirname "$0")"
+IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
+
+. "$IPSET_DIR/def.sh"
+
+# useful in case ipban set is used in custom scripts
+getuser
+"$IPSET_DIR/create_ipset.sh"
+
+ZREESTR="$TMPDIR/zapret.txt"
+#ZURL=https://reestr.rublacklist.net/api/current
+ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
+
+curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL" >"$ZREESTR" ||
+{
+ echo reestr list download failed   
+ exit 2
+}
+dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
+if test $dlsize -lt 204800; then
+ echo list file is too small. can be bad.
+ exit 2
+fi
+(LANG=C cut -s -f2 -d';' "$ZREESTR" | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' | awk '{ print tolower($0) }' ; cat "$ZUSERLIST" ) | sort -u | zz "$ZHOSTLIST"
+rm -f "$ZREESTR"
+
+hup_zapret_daemons
+
+exit 0

ipset/get_reestr_ip.sh

@@ -0,0 +1,51 @@
+#!/bin/sh
+
+IPSET_DIR="$(dirname "$0")"
+IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
+
+. "$IPSET_DIR/def.sh"
+
+ZREESTR="$TMPDIR/reestr.txt"
+#ZURL_REESTR=https://reestr.rublacklist.net/api/current
+ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
+
+getuser
+
+dig_reestr()
+{
+ # $1 - grep ipmask
+ # $2 - iplist
+ # $3 - ip version : 4,6
+
+ echo processing reestr list $2
+
+ tail -n +2 "$ZREESTR" | nice -n 5 $GREP -oE "$1" | cut_local | ip2net$3 | zz "$2"
+}
+
+
+# assume all https banned by ip
+curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
+{
+ echo reestr list download failed
+ exit 2
+}
+dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
+if test $dlsize -lt 1048576; then
+ echo reestr ip list is too small. can be bad.
+ exit 2
+fi
+#sed -i 's/\\n/\r\n/g' $ZREESTR
+
+get_ip_regex
+
+[ "$DISABLE_IPV4" != "1" ] && {
+ dig_reestr "$REG_IPV4" "$ZIPLIST" 4
+}
+
+[ "$DISABLE_IPV6" != "1" ] && {
+ dig_reestr "$REG_IPV6" "$ZIPLIST6" 6
+}
+
+rm -f "$ZREESTR"
+
+"$IPSET_DIR/create_ipset.sh"

ipset/get_reestr_resolve.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+
+IPSET_DIR="$(dirname "$0")"
+IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
+
+. "$IPSET_DIR/def.sh"
+
+ZREESTR="$TMPDIR/zapret.txt"
+ZDIG="$TMPDIR/zapret-dig.txt"
+ZIPLISTTMP="$TMPDIR/zapret-ip.txt"
+#ZURL=https://reestr.rublacklist.net/api/current
+ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
+
+getuser
+
+# both disabled
+[ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0
+
+curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL" >"$ZREESTR" ||
+{
+ echo reestr list download failed   
+ exit 2
+}
+
+dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
+if test $dlsize -lt 204800; then
+ echo list file is too small. can be bad.
+ exit 2
+fi
+
+echo preparing dig list ..
+LANG=C cut -f2 -d ';' "$ZREESTR"  | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG"
+rm -f "$ZREESTR"
+
+echo digging started. this can take long ...
+
+[ "$DISABLE_IPV4" != "1" ] && {
+ digger "$ZDIG" 4 | cut_local >"$ZIPLISTTMP" || {
+  rm -f "$ZDIG"
+  exit 1
+ }
+ ip2net4 <"$ZIPLISTTMP" | zz "$ZIPLIST"
+ rm -f "$ZIPLISTTMP"
+}
+[ "$DISABLE_IPV6" != "1" ] && {
+ digger "$ZDIG" 6 | cut_local6 >"$ZIPLISTTMP" || {
+  rm -f "$ZDIG"
+  exit 1
+ }
+ ip2net6 <"$ZIPLISTTMP" | zz "$ZIPLIST6"
+ rm -f "$ZIPLISTTMP"
+}
+rm -f "$ZDIG"
+"$IPSET_DIR/create_ipset.sh"

ipset/get_user.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+# resolve user host list
+
+IPSET_DIR="$(dirname "$0")"
+IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
+
+. "$IPSET_DIR/def.sh"
+
+getuser
+
+"$IPSET_DIR/create_ipset.sh"

ipset/zapret-hosts-user-exclude.txt

@@ -0,0 +1,6 @@
+10.0.0.0/8
+172.16.0.0/12
+192.168.0.0/16
+169.254.0.0/16
+fc00::/7
+fe80::/10

ipset/zapret-hosts-user-ipban.txt

@@ -0,0 +1 @@
+pornhub.com

ipset/zapret-hosts-user.txt

@@ -0,0 +1 @@
+st.kinozal.tv