From 81dbfc327a8d382053065c9d8186e4dc81abc487 Mon Sep 17 00:00:00 2001
From: bolvan <k@vodka.home.kg>
Date: Sat, 6 May 2017 09:47:30 +0300
Subject: [PATCH] get_combined.sh

---
 ipset/def.sh          |  4 ++--
 ipset/get_combined.sh | 41 +++++++++++++++++++++++++++++++++++++++++
 ipset/get_reestr.sh   |  2 +-
 3 files changed, 44 insertions(+), 3 deletions(-)
 create mode 100755 ipset/get_combined.sh

diff --git a/ipset/def.sh b/ipset/def.sh
index 473adf9..f771501 100755
--- a/ipset/def.sh
+++ b/ipset/def.sh
@@ -12,9 +12,9 @@ ZUSERLIST_IPBAN=$EXEDIR/zapret-hosts-user-ipban.txt
 getuser()
 {
  [ -f $ZUSERLIST ] && {
-  dig A +short +time=8 +tries=2 -f $ZUSERLIST | grep -E '^[^;].*[^.]$' | grep -vE '^192.168.[0-9]*.[0-9]*$' | grep -vE '^127.[0-9]*.[0-9]*.[0-9]*$' | grep -vE '^10.[0-9]*.[0-9]*.[0-9]*$' | sort | uniq >$ZIPLIST_USER
+  dig A +short +time=8 +tries=2 -f $ZUSERLIST | grep -E '^[^;].*[^.]$' | grep -vE '^192\.168\.[0-9]+.[0-9]+$' | grep -vE '^127\.[0-9]+\.[0-9]+\.[0-9]+$' | grep -vE '^10\.[0-9]+\.[0-9]+\.[0-9]+$' | sort -u >$ZIPLIST_USER
  }
  [ -f $ZUSERLIST_IPBAN ] && {
-  dig A +short +time=8 +tries=2 -f $ZUSERLIST_IPBAN | grep -E '^[^;].*[^.]$' | grep -vE '^192.168.[0-9]*.[0-9]*$' | grep -vE '^127.[0-9]*.[0-9]*.[0-9]*$' | grep -vE '^10.[0-9]*.[0-9]*.[0-9]*$' | sort | uniq >$ZIPLIST_USER_IPBAN
+  dig A +short +time=8 +tries=2 -f $ZUSERLIST_IPBAN | grep -E '^[^;].*[^.]$' | grep -vE '^192\.168\.[0-9]+\.[0-9]+$' | grep -vE '^127\.[0-9]+\.[0-9]+\.[0-9]+$' | grep -vE '^10\.[0-9]+\.[0-9]+\.[0-9]+$' | sort -u >$ZIPLIST_USER_IPBAN
  }
 }
\ No newline at end of file
diff --git a/ipset/get_combined.sh b/ipset/get_combined.sh
new file mode 100755
index 0000000..adf7746
--- /dev/null
+++ b/ipset/get_combined.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+# get rublacklist and resolve it
+
+SCRIPT=$(readlink -f $0)
+EXEDIR=$(dirname $SCRIPT)
+
+. "$EXEDIR/def.sh"
+
+ZREESTR=/tmp/reestr.txt
+ZANTIZAPRET=/tmp/antizapret.txt
+ZURL_REESTR=http://reestr.rublacklist.net/api/current
+ZURL_ANTIZAPRET=http://antizapret.prostovpn.org/iplist.txt
+
+getuser
+
+curl --fail --max-time 300 --max-filesize 41943040 "$ZURL_REESTR" |
+  grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' |
+  grep -vE '^192\.168\.[0-9]+\.[0-9]+$' |
+  grep -vE '^127\.[0-9]+\.[0-9]+\.[0-9]+$' |
+  grep -vE '^10\.[0-9]+\.[0-9]+\.[0-9]+$' |
+  sort -u >$ZREESTR
+dlsize=$(wc -c "$ZREESTR" | cut -f 1 -d ' ')
+if test $dlsize -lt 10240; then
+ echo reestr ip list is too small. can be bad.
+ exit 2
+fi
+curl --fail --max-time 300 --max-filesize 33554432 -k -L "$ZURL_ANTIZAPRET" |
+ grep -vE '^192\.168\.[0-9]+\.[0-9]+$' |
+ grep -vE '^127\.[0-9]+\.[0-9]+\.[0-9]+$' |
+ grep -vE '^10\.[0-9]+\.[0-9]+\.[0-9]+$' >$ZANTIZAPRET
+dlsize=$(wc -c "$ZANTIZAPRET" | cut -f 1 -d ' ')
+if test $dlsize -lt 10240; then
+ echo reestr ip list is too small. can be bad.
+ exit 2
+fi
+
+grep -vFf $ZREESTR $ZANTIZAPRET >$ZIPLIST
+mv -f $ZREESTR $ZIPLIST_IPBAN
+rm -f $ZANTIZAPRET 
+
+"$EXEDIR/create_ipset.sh"
diff --git a/ipset/get_reestr.sh b/ipset/get_reestr.sh
index 9c2eb28..ecc8e95 100755
--- a/ipset/get_reestr.sh
+++ b/ipset/get_reestr.sh
@@ -13,7 +13,7 @@ ZURL=http://reestr.rublacklist.net/api/current
 
 getuser
 
-curl --fail --max-time 300 --max-filesize 20971520 "$ZURL" >$ZREESTR && {
+curl --fail --max-time 300 --max-filesize 41943040 "$ZURL" >$ZREESTR && {
  dlsize=$(wc -c "$ZREESTR" | cut -f 1 -d ' ')
  if test $dlsize -lt 204800; then
   echo list file is too small. can be bad.