ip2net iplist groupper

changes.txt

@@ -95,3 +95,7 @@ tpws,nfqws : added hostnospace option
 v19
 
 tpws : added hostlist option
+
+v20
+
+added ip2net. ip2net groups ips from iplist into subnets and reduces ipset size twice

compile/build_howto_openwrt.txt

@@ -17,22 +17,25 @@ How to compile native programs for use in openwrt
 2) ./scripts/feeds update -a
    ./scripts/feeds install -a
 
-3) <add zapret packages to build root>
-   <copy package descriptions>
+3) #add zapret packages to build root
+   #copy package descriptions
    copy compile/openwrt/* to ~/openwrt
-   <copy source code of tpws>
+   #copy source code of tpws
    copy tpws to ~/openwrt/package/zapret/tpws
-   <copy source code of nfq>
+   #copy source code of nfq
    copy nfq to ~/openwrt/package/zapret/nfq
+   #copy source code of ip2net
+   copy ip2net to ~/openwrt/package/zapret/ip2net
 
 4) make menuconfig
-   <select your target architecture>
-   <select packages Network/Zapret/* as "M">
+   #select your target architecture
+   #select packages Network/Zapret/* as "M"
 
 5) make toolchain/compile
 
 6) make package/tpws/compile
    make package/nfqws/compile
+   make package/ip2net/compile
 
 7) find bin -name tpws*.ipk
-   <take your tpws*.ipk and nfqws*.ipk from there>
+   #take your tpws*.ipk , nfqws*.ipk , ip2net*.ipk from there

compile/openwrt/package/zapret/ip2net/Makefile

@@ -0,0 +1,32 @@
+#
+
+include $(TOPDIR)/rules.mk
+
+PKG_NAME:=ip2net
+PKG_RELEASE:=1
+
+include $(INCLUDE_DIR)/package.mk
+
+define Package/ip2net
+	SECTION:=net
+	CATEGORY:=Network
+	TITLE:=ip2net
+	SUBMENU:=Zapret
+endef
+
+define Build/Prepare
+	mkdir -p $(PKG_BUILD_DIR)
+	$(CP) ./ip2net/* $(PKG_BUILD_DIR)/
+endef
+
+define Build/Compile
+	$(MAKE) -C $(PKG_BUILD_DIR) $(TARGET_CONFIGURE_OPTS)
+endef
+
+define Package/ip2net/install
+	$(INSTALL_DIR) $(1)/opt/zapret/ip2net
+	$(INSTALL_BIN) $(PKG_BUILD_DIR)/ip2net $(1)/opt/zapret/ip2net
+endef
+
+$(eval $(call BuildPackage,ip2net))
+

compile/openwrt/package/zapret/ip2net/readme.txt

@@ -0,0 +1 @@
+Copy "ip2net" folder here !

ip2net/Makefile

@@ -0,0 +1,12 @@
+CC ?= gcc
+CFLAGS += -s
+LIBS = 
+SRC_FILES = *.c
+
+all: ip2net
+
+ip2net: $(SRC_FILES)
+	$(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS)
+
+clean:
+	rm -f ip2net *.o

ip2net/ip2net.c

@@ -0,0 +1,93 @@
+// group ip list from stdout into subnets
+// ip list must be pre-uniqued
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "qsort.h"
+
+#define ALLOC_STEP 16384
+#define PCTMULT 3/4
+
+typedef unsigned int uint;
+typedef unsigned char uchar;
+
+int ucmp (const void * a,const void * b, void *arg)
+{
+   if (*(uint*)a < *(uint*)b)
+    return -1;
+   else if (*(uint*)a > *(uint*)b)
+    return 1;
+   else
+    return 0;
+}
+
+uint mask_from_bitcount(uint zct)
+{
+ return ~((1<<zct)-1);
+}
+
+int main()
+{
+ uint u1,u2,u3,u4,ip;
+ uint ipct=0,iplist_size=0,*iplist=NULL,*iplist_new;
+ uint pos=0,p;
+ uint i,zct,subnet_ct,end_ip;
+ 
+ while (!feof(stdin))
+  if (scanf("%u.%u.%u.%u",&u1,&u2,&u3,&u4)==4 && !(u1 & 0xFFFFFF00) && !(u2 & 0xFFFFFF00) && !(u3 & 0xFFFFFF00) && !(u4 & 0xFFFFFF00))
+  {
+   ip = u1<<24 | u2<<16 | u3<<8 | u4;
+   if (ipct>=iplist_size)
+   {
+     iplist_size += ALLOC_STEP;
+     iplist_new = (uint*)(iplist ? realloc(iplist,sizeof(*iplist)*iplist_size) : malloc(sizeof(*iplist)*iplist_size));
+     if (!iplist_new)
+     {
+       free(iplist);
+       fprintf(stderr,"out of memory\n");
+       return 100;
+     }
+     iplist = iplist_new;
+   }
+   iplist[ipct++]= ip;
+  }
+
+  gnu_quicksort(iplist,ipct,sizeof(*iplist),ucmp,NULL);
+
+  while(pos<ipct)
+  {
+   uchar subnet_ok=0;
+   uint mask,ip_start,ip_end,ip_ct,subnet_ct,pos_end;
+
+   for(zct=10, pos_end=pos+1 ; zct>=2 ; zct--)
+   {
+    mask = mask_from_bitcount(zct);
+    ip_start = iplist[pos] & mask;
+    subnet_ct = ~mask+1;
+    if (iplist[pos]>(ip_start+subnet_ct*PCTMULT)) continue;
+    ip_end = ip_start | ~mask;
+    for(p=pos, ip_ct=0 ; p<ipct && iplist[p]<=ip_end; p++) ip_ct++;
+    if (ip_ct>=(subnet_ct*PCTMULT))
+    {
+    	subnet_ok=1;
+    	pos_end = p;
+    	break;
+    }
+   }
+   if (!subnet_ok) zct=0,ip_start=iplist[pos];
+
+   u1 = ip_start>>24;
+   u2 = (ip_start>>16) & 0xFF;
+   u3 = (ip_start>>8) & 0xFF;
+   u4 = ip_start & 0xFF;
+   if (zct)
+    printf("%u.%u.%u.%u/%u\n",u1,u2,u3,u4,32-zct);
+   else
+    printf("%u.%u.%u.%u\n",u1,u2,u3,u4);
+
+   pos = pos_end;
+  }
+
+  free(iplist);
+  return 0;
+}

ip2net/qsort.c

@@ -0,0 +1,250 @@
+/* Copyright (C) 1991-2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Written by Douglas C. Schmidt (schmidt@ics.uci.edu).
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* If you consider tuning this algorithm, you should consult first:
+   Engineering a sort function; Jon Bentley and M. Douglas McIlroy;
+   Software - Practice and Experience; Vol. 23 (11), 1249-1265, 1993.  */
+
+//#include <alloca.h>
+#include <limits.h>
+#include <stdlib.h>
+//#include <string.h>
+#include "qsort.h"
+
+/* Byte-wise swap two items of size SIZE. */
+#define SWAP(a, b, size)						      \
+  do									      \
+    {									      \
+      size_t __size = (size);						      \
+      char *__a = (a), *__b = (b);					      \
+      do								      \
+	{								      \
+	  char __tmp = *__a;						      \
+	  *__a++ = *__b;						      \
+	  *__b++ = __tmp;						      \
+	} while (--__size > 0);						      \
+    } while (0)
+
+/* Discontinue quicksort algorithm when partition gets below this size.
+   This particular magic number was chosen to work best on a Sun 4/260. */
+#define MAX_THRESH 4
+
+/* Stack node declarations used to store unfulfilled partition obligations. */
+typedef struct
+  {
+    char *lo;
+    char *hi;
+  } stack_node;
+
+/* The next 4 #defines implement a very fast in-line stack abstraction. */
+/* The stack needs log (total_elements) entries (we could even subtract
+   log(MAX_THRESH)).  Since total_elements has type size_t, we get as
+   upper bound for log (total_elements):
+   bits per byte (CHAR_BIT) * sizeof(size_t).  */
+#define STACK_SIZE	(CHAR_BIT * sizeof(size_t))
+#define PUSH(low, high)	((void) ((top->lo = (low)), (top->hi = (high)), ++top))
+#define	POP(low, high)	((void) (--top, (low = top->lo), (high = top->hi)))
+#define	STACK_NOT_EMPTY	(stack < top)
+
+
+/* Order size using quicksort.  This implementation incorporates
+   four optimizations discussed in Sedgewick:
+
+   1. Non-recursive, using an explicit stack of pointer that store the
+      next array partition to sort.  To save time, this maximum amount
+      of space required to store an array of SIZE_MAX is allocated on the
+      stack.  Assuming a 32-bit (64 bit) integer for size_t, this needs
+      only 32 * sizeof(stack_node) == 256 bytes (for 64 bit: 1024 bytes).
+      Pretty cheap, actually.
+
+   2. Chose the pivot element using a median-of-three decision tree.
+      This reduces the probability of selecting a bad pivot value and
+      eliminates certain extraneous comparisons.
+
+   3. Only quicksorts TOTAL_ELEMS / MAX_THRESH partitions, leaving
+      insertion sort to order the MAX_THRESH items within each partition.
+      This is a big win, since insertion sort is faster for small, mostly
+      sorted array segments.
+
+   4. The larger of the two sub-partitions is always pushed onto the
+      stack first, with the algorithm then concentrating on the
+      smaller partition.  This *guarantees* no more than log (total_elems)
+      stack size is needed (actually O(1) in this case)!  */
+
+void
+gnu_quicksort (void *const pbase, size_t total_elems, size_t size,
+	    __gnu_compar_d_fn_t cmp, void *arg)
+{
+  char *base_ptr = (char *) pbase;
+
+  const size_t max_thresh = MAX_THRESH * size;
+
+  if (total_elems == 0)
+    /* Avoid lossage with unsigned arithmetic below.  */
+    return;
+
+  if (total_elems > MAX_THRESH)
+    {
+      char *lo = base_ptr;
+      char *hi = &lo[size * (total_elems - 1)];
+      stack_node stack[STACK_SIZE];
+      stack_node *top = stack;
+
+      PUSH (NULL, NULL);
+
+      while (STACK_NOT_EMPTY)
+        {
+          char *left_ptr;
+          char *right_ptr;
+
+	  /* Select median value from among LO, MID, and HI. Rearrange
+	     LO and HI so the three values are sorted. This lowers the
+	     probability of picking a pathological pivot value and
+	     skips a comparison for both the LEFT_PTR and RIGHT_PTR in
+	     the while loops. */
+
+	  char *mid = lo + size * ((hi - lo) / size >> 1);
+
+	  if ((*cmp) ((void *) mid, (void *) lo, arg) < 0)
+	    SWAP (mid, lo, size);
+	  if ((*cmp) ((void *) hi, (void *) mid, arg) < 0)
+	    SWAP (mid, hi, size);
+	  else
+	    goto jump_over;
+	  if ((*cmp) ((void *) mid, (void *) lo, arg) < 0)
+	    SWAP (mid, lo, size);
+	jump_over:;
+
+	  left_ptr  = lo + size;
+	  right_ptr = hi - size;
+
+	  /* Here's the famous ``collapse the walls'' section of quicksort.
+	     Gotta like those tight inner loops!  They are the main reason
+	     that this algorithm runs much faster than others. */
+	  do
+	    {
+	      while ((*cmp) ((void *) left_ptr, (void *) mid, arg) < 0)
+		left_ptr += size;
+
+	      while ((*cmp) ((void *) mid, (void *) right_ptr, arg) < 0)
+		right_ptr -= size;
+
+	      if (left_ptr < right_ptr)
+		{
+		  SWAP (left_ptr, right_ptr, size);
+		  if (mid == left_ptr)
+		    mid = right_ptr;
+		  else if (mid == right_ptr)
+		    mid = left_ptr;
+		  left_ptr += size;
+		  right_ptr -= size;
+		}
+	      else if (left_ptr == right_ptr)
+		{
+		  left_ptr += size;
+		  right_ptr -= size;
+		  break;
+		}
+	    }
+	  while (left_ptr <= right_ptr);
+
+          /* Set up pointers for next iteration.  First determine whether
+             left and right partitions are below the threshold size.  If so,
+             ignore one or both.  Otherwise, push the larger partition's
+             bounds on the stack and continue sorting the smaller one. */
+
+          if ((size_t) (right_ptr - lo) <= max_thresh)
+            {
+              if ((size_t) (hi - left_ptr) <= max_thresh)
+		/* Ignore both small partitions. */
+                POP (lo, hi);
+              else
+		/* Ignore small left partition. */
+                lo = left_ptr;
+            }
+          else if ((size_t) (hi - left_ptr) <= max_thresh)
+	    /* Ignore small right partition. */
+            hi = right_ptr;
+          else if ((right_ptr - lo) > (hi - left_ptr))
+            {
+	      /* Push larger left partition indices. */
+              PUSH (lo, right_ptr);
+              lo = left_ptr;
+            }
+          else
+            {
+	      /* Push larger right partition indices. */
+              PUSH (left_ptr, hi);
+              hi = right_ptr;
+            }
+        }
+    }
+
+  /* Once the BASE_PTR array is partially sorted by quicksort the rest
+     is completely sorted using insertion sort, since this is efficient
+     for partitions below MAX_THRESH size. BASE_PTR points to the beginning
+     of the array to sort, and END_PTR points at the very last element in
+     the array (*not* one beyond it!). */
+
+#define min(x, y) ((x) < (y) ? (x) : (y))
+
+  {
+    char *const end_ptr = &base_ptr[size * (total_elems - 1)];
+    char *tmp_ptr = base_ptr;
+    char *thresh = min(end_ptr, base_ptr + max_thresh);
+    char *run_ptr;
+
+    /* Find smallest element in first threshold and place it at the
+       array's beginning.  This is the smallest array element,
+       and the operation speeds up insertion sort's inner loop. */
+
+    for (run_ptr = tmp_ptr + size; run_ptr <= thresh; run_ptr += size)
+      if ((*cmp) ((void *) run_ptr, (void *) tmp_ptr, arg) < 0)
+        tmp_ptr = run_ptr;
+
+    if (tmp_ptr != base_ptr)
+      SWAP (tmp_ptr, base_ptr, size);
+
+    /* Insertion sort, running from left-hand-side up to right-hand-side.  */
+
+    run_ptr = base_ptr + size;
+    while ((run_ptr += size) <= end_ptr)
+      {
+	tmp_ptr = run_ptr - size;
+	while ((*cmp) ((void *) run_ptr, (void *) tmp_ptr, arg) < 0)
+	  tmp_ptr -= size;
+
+	tmp_ptr += size;
+        if (tmp_ptr != run_ptr)
+          {
+            char *trav;
+
+	    trav = run_ptr + size;
+	    while (--trav >= run_ptr)
+              {
+                char c = *trav;
+                char *hi, *lo;
+
+                for (hi = lo = trav; (lo -= size) >= tmp_ptr; hi = lo)
+                  *hi = *lo;
+                *hi = c;
+              }
+          }
+      }
+  }
+}

ip2net/qsort.h

@@ -0,0 +1,6 @@
+#pragma once
+
+// GNU qsort is 2x faster than musl
+
+typedef int (*__gnu_compar_d_fn_t) (const void *, const void *, void *);
+void gnu_quicksort (void *const pbase, size_t total_elems, size_t size, __gnu_compar_d_fn_t cmp, void *arg);

ipset/create_ipset.sh

@@ -4,20 +4,36 @@
 SCRIPT=$(readlink -f $0)
 EXEDIR=$(dirname $SCRIPT)
 IPSET_OPT="hashsize 131072 maxelem 524288"
+IP2NET=$EXEDIR/../ip2net/ip2net
 
 . "$EXEDIR/def.sh"
 
 create_ipset()
 {
-ipset flush $2 2>/dev/null || ipset create $2 $1 $IPSET_OPT
+local IPSTYPE
+if [ -x $IP2NET ]; then
+ IPSTYPE=hash:net
+else
+ IPSTYPE=$1
+fi
+ipset flush $2 2>/dev/null || ipset create $2 $IPSTYPE $IPSET_OPT
 for f in "$3" "$4"
 do
  [ -f "$f" ] && {
-  echo Adding to ipset $2 \($1\) : $f
-  if [ -f "$ZIPLIST_EXCLUDE" ] ; then
-   grep -vxFf $ZIPLIST_EXCLUDE "$f" | sort -u | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore
+  if [ -x $IP2NET ]; then
+   echo Adding to ipset $2 \($IPSTYPE , ip2net\) : $f
+   if [ -f "$ZIPLIST_EXCLUDE" ] ; then
+    grep -vxFf $ZIPLIST_EXCLUDE "$f" | sort -u | $IP2NET | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore
+   else
+    sort -u "$f" | $IP2NET | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore
+   fi
   else
-   sort -u "$f" | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore
+   echo Adding to ipset $2 \($IPSTYPE\) : $f
+   if [ -f "$ZIPLIST_EXCLUDE" ] ; then
+    grep -vxFf $ZIPLIST_EXCLUDE "$f" | sort -u | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore
+   else
+    sort -u "$f" | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore
+   fi
   fi
  }
 done

readme.txt

@@ -1,4 +1,4 @@
-zapret v.19
+zapret v.20
 
 Для чего это надо
 -----------------
@@ -241,6 +241,9 @@ Debian 7 изначально содержит ядро 3.2. Оно не уме
 Собрать tpws :
  cd /opt/zapret/tpws
  make
+Собрать ip2net :
+ cd /opt/zapret/ip2net
+ make
 Скопировать /opt/zapret/init.d/debian7/zapret в /etc/init.d.
 В /etc/init.d/zapret выбрать пераметр "ISP". В зависимости от него будут применены нужные правила.
 Там же выбрать параметр SLAVE_ETH, соответствующий названию внутреннего сетевого интерфейса.
@@ -320,7 +323,7 @@ opkg install iptables-mod-extra iptables-mod-nfqueue iptables-mod-filter iptable
 Скорее всего таковой найдется. Если нет - вам придется собирать самостоятельно.
 
 Скопировать директорию "zapret" в /opt на роутер.
-Скопировать работающий бинарик nfqws в /opt/zapret/nfq, tpws в /opt/zapret/tpws.
+Скопировать работающий бинарик nfqws в /opt/zapret/nfq, tpws в /opt/zapret/tpws, ip2net в /opt/zapret/ip2net.
 Скопировать /opt/zapret/init.d/zapret в /etc/init.d.
 В /etc/init.d/zapret выбрать пераметр "ISP". В зависимости от него будут применены нужные правила.
 /etc/init.d/zapret enable