diff --git a/binaries/armhf/ip2net b/binaries/armhf/ip2net new file mode 100755 index 0000000..7550f35 Binary files /dev/null and b/binaries/armhf/ip2net differ diff --git a/binaries/mips32r1-lsb/ip2net b/binaries/mips32r1-lsb/ip2net new file mode 100755 index 0000000..005eb40 Binary files /dev/null and b/binaries/mips32r1-lsb/ip2net differ diff --git a/binaries/mips32r1-msb/ip2net b/binaries/mips32r1-msb/ip2net new file mode 100755 index 0000000..3652611 Binary files /dev/null and b/binaries/mips32r1-msb/ip2net differ diff --git a/binaries/x86/ip2net b/binaries/x86/ip2net new file mode 100755 index 0000000..fe164c6 Binary files /dev/null and b/binaries/x86/ip2net differ diff --git a/binaries/x86_64/ip2net b/binaries/x86_64/ip2net new file mode 100755 index 0000000..cec11bd Binary files /dev/null and b/binaries/x86_64/ip2net differ diff --git a/changes.txt b/changes.txt index be1cb25..3967670 100644 --- a/changes.txt +++ b/changes.txt @@ -95,3 +95,7 @@ tpws,nfqws : added hostnospace option v19 tpws : added hostlist option + +v20 + +added ip2net. ip2net groups ips from iplist into subnets and reduces ipset size twice diff --git a/compile/build_howto_openwrt.txt b/compile/build_howto_openwrt.txt index 94c856c..c334587 100644 --- a/compile/build_howto_openwrt.txt +++ b/compile/build_howto_openwrt.txt @@ -17,22 +17,25 @@ How to compile native programs for use in openwrt 2) ./scripts/feeds update -a ./scripts/feeds install -a -3) - +3) #add zapret packages to build root + #copy package descriptions copy compile/openwrt/* to ~/openwrt - + #copy source code of tpws copy tpws to ~/openwrt/package/zapret/tpws - + #copy source code of nfq copy nfq to ~/openwrt/package/zapret/nfq + #copy source code of ip2net + copy ip2net to ~/openwrt/package/zapret/ip2net 4) make menuconfig - + #select your target architecture + #select packages Network/Zapret/* as "M" 5) make toolchain/compile 6) make package/tpws/compile make package/nfqws/compile + make package/ip2net/compile 7) find bin -name tpws*.ipk - + #take your tpws*.ipk , nfqws*.ipk , ip2net*.ipk from there diff --git a/compile/openwrt/package/zapret/ip2net/Makefile b/compile/openwrt/package/zapret/ip2net/Makefile new file mode 100644 index 0000000..4564675 --- /dev/null +++ b/compile/openwrt/package/zapret/ip2net/Makefile @@ -0,0 +1,32 @@ +# + +include $(TOPDIR)/rules.mk + +PKG_NAME:=ip2net +PKG_RELEASE:=1 + +include $(INCLUDE_DIR)/package.mk + +define Package/ip2net + SECTION:=net + CATEGORY:=Network + TITLE:=ip2net + SUBMENU:=Zapret +endef + +define Build/Prepare + mkdir -p $(PKG_BUILD_DIR) + $(CP) ./ip2net/* $(PKG_BUILD_DIR)/ +endef + +define Build/Compile + $(MAKE) -C $(PKG_BUILD_DIR) $(TARGET_CONFIGURE_OPTS) +endef + +define Package/ip2net/install + $(INSTALL_DIR) $(1)/opt/zapret/ip2net + $(INSTALL_BIN) $(PKG_BUILD_DIR)/ip2net $(1)/opt/zapret/ip2net +endef + +$(eval $(call BuildPackage,ip2net)) + diff --git a/compile/openwrt/package/zapret/ip2net/readme.txt b/compile/openwrt/package/zapret/ip2net/readme.txt new file mode 100644 index 0000000..abf7acd --- /dev/null +++ b/compile/openwrt/package/zapret/ip2net/readme.txt @@ -0,0 +1 @@ +Copy "ip2net" folder here ! diff --git a/ip2net/Makefile b/ip2net/Makefile new file mode 100644 index 0000000..38628fd --- /dev/null +++ b/ip2net/Makefile @@ -0,0 +1,12 @@ +CC ?= gcc +CFLAGS += -s +LIBS = +SRC_FILES = *.c + +all: ip2net + +ip2net: $(SRC_FILES) + $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) + +clean: + rm -f ip2net *.o diff --git a/ip2net/ip2net.c b/ip2net/ip2net.c new file mode 100644 index 0000000..8bbdb5a --- /dev/null +++ b/ip2net/ip2net.c @@ -0,0 +1,93 @@ +// group ip list from stdout into subnets +// ip list must be pre-uniqued + +#include +#include +#include "qsort.h" + +#define ALLOC_STEP 16384 +#define PCTMULT 3/4 + +typedef unsigned int uint; +typedef unsigned char uchar; + +int ucmp (const void * a,const void * b, void *arg) +{ + if (*(uint*)a < *(uint*)b) + return -1; + else if (*(uint*)a > *(uint*)b) + return 1; + else + return 0; +} + +uint mask_from_bitcount(uint zct) +{ + return ~((1<=iplist_size) + { + iplist_size += ALLOC_STEP; + iplist_new = (uint*)(iplist ? realloc(iplist,sizeof(*iplist)*iplist_size) : malloc(sizeof(*iplist)*iplist_size)); + if (!iplist_new) + { + free(iplist); + fprintf(stderr,"out of memory\n"); + return 100; + } + iplist = iplist_new; + } + iplist[ipct++]= ip; + } + + gnu_quicksort(iplist,ipct,sizeof(*iplist),ucmp,NULL); + + while(pos=2 ; zct--) + { + mask = mask_from_bitcount(zct); + ip_start = iplist[pos] & mask; + subnet_ct = ~mask+1; + if (iplist[pos]>(ip_start+subnet_ct*PCTMULT)) continue; + ip_end = ip_start | ~mask; + for(p=pos, ip_ct=0 ; p=(subnet_ct*PCTMULT)) + { + subnet_ok=1; + pos_end = p; + break; + } + } + if (!subnet_ok) zct=0,ip_start=iplist[pos]; + + u1 = ip_start>>24; + u2 = (ip_start>>16) & 0xFF; + u3 = (ip_start>>8) & 0xFF; + u4 = ip_start & 0xFF; + if (zct) + printf("%u.%u.%u.%u/%u\n",u1,u2,u3,u4,32-zct); + else + printf("%u.%u.%u.%u\n",u1,u2,u3,u4); + + pos = pos_end; + } + + free(iplist); + return 0; +} diff --git a/ip2net/qsort.c b/ip2net/qsort.c new file mode 100644 index 0000000..2ee1185 --- /dev/null +++ b/ip2net/qsort.c @@ -0,0 +1,250 @@ +/* Copyright (C) 1991-2018 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Written by Douglas C. Schmidt (schmidt@ics.uci.edu). + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +/* If you consider tuning this algorithm, you should consult first: + Engineering a sort function; Jon Bentley and M. Douglas McIlroy; + Software - Practice and Experience; Vol. 23 (11), 1249-1265, 1993. */ + +//#include +#include +#include +//#include +#include "qsort.h" + +/* Byte-wise swap two items of size SIZE. */ +#define SWAP(a, b, size) \ + do \ + { \ + size_t __size = (size); \ + char *__a = (a), *__b = (b); \ + do \ + { \ + char __tmp = *__a; \ + *__a++ = *__b; \ + *__b++ = __tmp; \ + } while (--__size > 0); \ + } while (0) + +/* Discontinue quicksort algorithm when partition gets below this size. + This particular magic number was chosen to work best on a Sun 4/260. */ +#define MAX_THRESH 4 + +/* Stack node declarations used to store unfulfilled partition obligations. */ +typedef struct + { + char *lo; + char *hi; + } stack_node; + +/* The next 4 #defines implement a very fast in-line stack abstraction. */ +/* The stack needs log (total_elements) entries (we could even subtract + log(MAX_THRESH)). Since total_elements has type size_t, we get as + upper bound for log (total_elements): + bits per byte (CHAR_BIT) * sizeof(size_t). */ +#define STACK_SIZE (CHAR_BIT * sizeof(size_t)) +#define PUSH(low, high) ((void) ((top->lo = (low)), (top->hi = (high)), ++top)) +#define POP(low, high) ((void) (--top, (low = top->lo), (high = top->hi))) +#define STACK_NOT_EMPTY (stack < top) + + +/* Order size using quicksort. This implementation incorporates + four optimizations discussed in Sedgewick: + + 1. Non-recursive, using an explicit stack of pointer that store the + next array partition to sort. To save time, this maximum amount + of space required to store an array of SIZE_MAX is allocated on the + stack. Assuming a 32-bit (64 bit) integer for size_t, this needs + only 32 * sizeof(stack_node) == 256 bytes (for 64 bit: 1024 bytes). + Pretty cheap, actually. + + 2. Chose the pivot element using a median-of-three decision tree. + This reduces the probability of selecting a bad pivot value and + eliminates certain extraneous comparisons. + + 3. Only quicksorts TOTAL_ELEMS / MAX_THRESH partitions, leaving + insertion sort to order the MAX_THRESH items within each partition. + This is a big win, since insertion sort is faster for small, mostly + sorted array segments. + + 4. The larger of the two sub-partitions is always pushed onto the + stack first, with the algorithm then concentrating on the + smaller partition. This *guarantees* no more than log (total_elems) + stack size is needed (actually O(1) in this case)! */ + +void +gnu_quicksort (void *const pbase, size_t total_elems, size_t size, + __gnu_compar_d_fn_t cmp, void *arg) +{ + char *base_ptr = (char *) pbase; + + const size_t max_thresh = MAX_THRESH * size; + + if (total_elems == 0) + /* Avoid lossage with unsigned arithmetic below. */ + return; + + if (total_elems > MAX_THRESH) + { + char *lo = base_ptr; + char *hi = &lo[size * (total_elems - 1)]; + stack_node stack[STACK_SIZE]; + stack_node *top = stack; + + PUSH (NULL, NULL); + + while (STACK_NOT_EMPTY) + { + char *left_ptr; + char *right_ptr; + + /* Select median value from among LO, MID, and HI. Rearrange + LO and HI so the three values are sorted. This lowers the + probability of picking a pathological pivot value and + skips a comparison for both the LEFT_PTR and RIGHT_PTR in + the while loops. */ + + char *mid = lo + size * ((hi - lo) / size >> 1); + + if ((*cmp) ((void *) mid, (void *) lo, arg) < 0) + SWAP (mid, lo, size); + if ((*cmp) ((void *) hi, (void *) mid, arg) < 0) + SWAP (mid, hi, size); + else + goto jump_over; + if ((*cmp) ((void *) mid, (void *) lo, arg) < 0) + SWAP (mid, lo, size); + jump_over:; + + left_ptr = lo + size; + right_ptr = hi - size; + + /* Here's the famous ``collapse the walls'' section of quicksort. + Gotta like those tight inner loops! They are the main reason + that this algorithm runs much faster than others. */ + do + { + while ((*cmp) ((void *) left_ptr, (void *) mid, arg) < 0) + left_ptr += size; + + while ((*cmp) ((void *) mid, (void *) right_ptr, arg) < 0) + right_ptr -= size; + + if (left_ptr < right_ptr) + { + SWAP (left_ptr, right_ptr, size); + if (mid == left_ptr) + mid = right_ptr; + else if (mid == right_ptr) + mid = left_ptr; + left_ptr += size; + right_ptr -= size; + } + else if (left_ptr == right_ptr) + { + left_ptr += size; + right_ptr -= size; + break; + } + } + while (left_ptr <= right_ptr); + + /* Set up pointers for next iteration. First determine whether + left and right partitions are below the threshold size. If so, + ignore one or both. Otherwise, push the larger partition's + bounds on the stack and continue sorting the smaller one. */ + + if ((size_t) (right_ptr - lo) <= max_thresh) + { + if ((size_t) (hi - left_ptr) <= max_thresh) + /* Ignore both small partitions. */ + POP (lo, hi); + else + /* Ignore small left partition. */ + lo = left_ptr; + } + else if ((size_t) (hi - left_ptr) <= max_thresh) + /* Ignore small right partition. */ + hi = right_ptr; + else if ((right_ptr - lo) > (hi - left_ptr)) + { + /* Push larger left partition indices. */ + PUSH (lo, right_ptr); + lo = left_ptr; + } + else + { + /* Push larger right partition indices. */ + PUSH (left_ptr, hi); + hi = right_ptr; + } + } + } + + /* Once the BASE_PTR array is partially sorted by quicksort the rest + is completely sorted using insertion sort, since this is efficient + for partitions below MAX_THRESH size. BASE_PTR points to the beginning + of the array to sort, and END_PTR points at the very last element in + the array (*not* one beyond it!). */ + +#define min(x, y) ((x) < (y) ? (x) : (y)) + + { + char *const end_ptr = &base_ptr[size * (total_elems - 1)]; + char *tmp_ptr = base_ptr; + char *thresh = min(end_ptr, base_ptr + max_thresh); + char *run_ptr; + + /* Find smallest element in first threshold and place it at the + array's beginning. This is the smallest array element, + and the operation speeds up insertion sort's inner loop. */ + + for (run_ptr = tmp_ptr + size; run_ptr <= thresh; run_ptr += size) + if ((*cmp) ((void *) run_ptr, (void *) tmp_ptr, arg) < 0) + tmp_ptr = run_ptr; + + if (tmp_ptr != base_ptr) + SWAP (tmp_ptr, base_ptr, size); + + /* Insertion sort, running from left-hand-side up to right-hand-side. */ + + run_ptr = base_ptr + size; + while ((run_ptr += size) <= end_ptr) + { + tmp_ptr = run_ptr - size; + while ((*cmp) ((void *) run_ptr, (void *) tmp_ptr, arg) < 0) + tmp_ptr -= size; + + tmp_ptr += size; + if (tmp_ptr != run_ptr) + { + char *trav; + + trav = run_ptr + size; + while (--trav >= run_ptr) + { + char c = *trav; + char *hi, *lo; + + for (hi = lo = trav; (lo -= size) >= tmp_ptr; hi = lo) + *hi = *lo; + *hi = c; + } + } + } + } +} diff --git a/ip2net/qsort.h b/ip2net/qsort.h new file mode 100644 index 0000000..f537ab7 --- /dev/null +++ b/ip2net/qsort.h @@ -0,0 +1,6 @@ +#pragma once + +// GNU qsort is 2x faster than musl + +typedef int (*__gnu_compar_d_fn_t) (const void *, const void *, void *); +void gnu_quicksort (void *const pbase, size_t total_elems, size_t size, __gnu_compar_d_fn_t cmp, void *arg); diff --git a/ipset/create_ipset.sh b/ipset/create_ipset.sh index 67a1f1f..386effc 100755 --- a/ipset/create_ipset.sh +++ b/ipset/create_ipset.sh @@ -4,20 +4,36 @@ SCRIPT=$(readlink -f $0) EXEDIR=$(dirname $SCRIPT) IPSET_OPT="hashsize 131072 maxelem 524288" +IP2NET=$EXEDIR/../ip2net/ip2net . "$EXEDIR/def.sh" create_ipset() { -ipset flush $2 2>/dev/null || ipset create $2 $1 $IPSET_OPT +local IPSTYPE +if [ -x $IP2NET ]; then + IPSTYPE=hash:net +else + IPSTYPE=$1 +fi +ipset flush $2 2>/dev/null || ipset create $2 $IPSTYPE $IPSET_OPT for f in "$3" "$4" do [ -f "$f" ] && { - echo Adding to ipset $2 \($1\) : $f - if [ -f "$ZIPLIST_EXCLUDE" ] ; then - grep -vxFf $ZIPLIST_EXCLUDE "$f" | sort -u | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore + if [ -x $IP2NET ]; then + echo Adding to ipset $2 \($IPSTYPE , ip2net\) : $f + if [ -f "$ZIPLIST_EXCLUDE" ] ; then + grep -vxFf $ZIPLIST_EXCLUDE "$f" | sort -u | $IP2NET | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore + else + sort -u "$f" | $IP2NET | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore + fi else - sort -u "$f" | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore + echo Adding to ipset $2 \($IPSTYPE\) : $f + if [ -f "$ZIPLIST_EXCLUDE" ] ; then + grep -vxFf $ZIPLIST_EXCLUDE "$f" | sort -u | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore + else + sort -u "$f" | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore + fi fi } done diff --git a/readme.txt b/readme.txt index 6384c2e..0943ecf 100644 --- a/readme.txt +++ b/readme.txt @@ -1,4 +1,4 @@ -zapret v.19 +zapret v.20 Для чего это надо ----------------- @@ -241,6 +241,9 @@ Debian 7 изначально содержит ядро 3.2. Оно не уме Собрать tpws : cd /opt/zapret/tpws make +Собрать ip2net : + cd /opt/zapret/ip2net + make Скопировать /opt/zapret/init.d/debian7/zapret в /etc/init.d. В /etc/init.d/zapret выбрать пераметр "ISP". В зависимости от него будут применены нужные правила. Там же выбрать параметр SLAVE_ETH, соответствующий названию внутреннего сетевого интерфейса. @@ -320,7 +323,7 @@ opkg install iptables-mod-extra iptables-mod-nfqueue iptables-mod-filter iptable Скорее всего таковой найдется. Если нет - вам придется собирать самостоятельно. Скопировать директорию "zapret" в /opt на роутер. -Скопировать работающий бинарик nfqws в /opt/zapret/nfq, tpws в /opt/zapret/tpws. +Скопировать работающий бинарик nfqws в /opt/zapret/nfq, tpws в /opt/zapret/tpws, ip2net в /opt/zapret/ip2net. Скопировать /opt/zapret/init.d/zapret в /etc/init.d. В /etc/init.d/zapret выбрать пераметр "ISP". В зависимости от него будут применены нужные правила. /etc/init.d/zapret enable