readme: add get_refilter*.sh info

docs/readme.txt

@@ -981,10 +981,13 @@ Cкрипты с названием get_antifilter_* оперируют спис
 9) ipset/get_antifilter_allyouneed.sh. получает лист https://antifilter.download/list/allyouneed.lst.
 Суммарный список префиксов, созданный из ipsum.lst и subnet.lst.
 
-Все варианты рассмотренных скриптов автоматически создают и заполняют ipset.
+10) ipset/get_refilter_ipsum.sh.
-Варианты 2-9 дополнительно вызывают вариант 1.
+Список берется отсюда : https://github.com/1andrevich/Re-filter-lists
 
-10) ipset/get_config.sh. этот скрипт вызывает то, что прописано в переменной GETLIST из файла config
+Все варианты рассмотренных скриптов автоматически создают и заполняют ipset.
+Варианты 2-10 дополнительно вызывают вариант 1.
+
+11) ipset/get_config.sh. этот скрипт вызывает то, что прописано в переменной GETLIST из файла config
 Если переменная не определена, то ресолвятся лишь листы для ipset nozapret/nozapret6.
 
 Листы РКН все время изменяются. Возникают новые тенденции. Требования к RAM могут меняться.
@@ -1111,7 +1114,11 @@ ipset/zapret-hosts-users-exclude.txt.gz или ipset/zapret-hosts-users-exclude.
 
 Поддомены учитываются автоматически. Например, строчка "ru" вносит в список "*.ru". Строчка "*.ru" в списке не сработает.
 
-Список доменов РКН может быть получен скриптами ipset/get_reestr_hostlist.sh или ipset/get_antizapret_domains.sh
+Список доменов РКН может быть получен скриптами
+ipset/get_reestr_hostlist.sh
+ipset/get_antizapret_domains.sh
+ipset/get_reestr_resolvable_domains.sh
+ipset/get_refilter_domains.sh
 - кладется в ipset/zapret-hosts.txt.gz.
 
 Чтобы обновить списки, перезапускать nfqws или tpws не нужно. Обновляете файлы, затем даете сигнал HUP.

install_easy.sh

@@ -219,10 +219,10 @@ select_getlist()
 		echo
 		if ask_yes_no $D "do you want to auto download ip/host list"; then
 			if [ "$MODE_FILTER" = "hostlist" ] ; then
-				GETLISTS="get_antizapret_domains.sh get_reestr_resolvable_domains.sh get_reestr_hostlist.sh"
+				GETLISTS="get_refilter_domains.sh get_antizapret_domains.sh get_reestr_resolvable_domains.sh get_reestr_hostlist.sh"
 				GETLIST_DEF="get_antizapret_domains.sh"
 			else
-				GETLISTS="get_user.sh get_antifilter_ip.sh get_antifilter_ipsmart.sh get_antifilter_ipsum.sh get_antifilter_ipresolve.sh get_antifilter_allyouneed.sh get_reestr_resolve.sh get_reestr_preresolved.sh get_reestr_preresolved_smart.sh"
+				GETLISTS="get_user.sh get_refilter_ipsum.sh get_antifilter_ip.sh get_antifilter_ipsmart.sh get_antifilter_ipsum.sh get_antifilter_ipresolve.sh get_antifilter_allyouneed.sh get_reestr_resolve.sh get_reestr_preresolved.sh get_reestr_preresolved_smart.sh"
 				GETLIST_DEF="get_antifilter_allyouneed.sh"
 			fi
 			ask_list GETLIST "$GETLISTS" "$GETLIST_DEF" && write_config_var GETLIST
@@ -462,6 +462,8 @@ files/huawei/E8372/zapret \
 files/huawei/E8372/run-zapret-ip \
 ipset/get_exclude.sh \
 ipset/clear_lists.sh \
+ipset/get_refilter_domains.sh \
+ipset/get_refilter_ipsum.sh \
 ipset/get_antifilter_ipresolve.sh \
 ipset/get_reestr_resolvable_domains.sh \
 ipset/get_config.sh \

ipset/get_refilter_domains.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+
+IPSET_DIR="$(dirname "$0")"
+IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
+
+. "$IPSET_DIR/def.sh"
+
+TMPLIST="$TMPDIR/list.txt"
+
+URL="https://github.com/1andrevich/Re-filter-lists/releases/latest/download/domains_all.lst"
+
+dl()
+{
+  # $1 - url
+  # $2 - file
+  # $3 - minsize
+  # $4 - maxsize
+  curl -L -H "Accept-Encoding: gzip" -k --fail --max-time 60 --connect-timeout 10 --retry 4 --max-filesize $4 -o "$TMPLIST" "$1" ||
+  {
+   echo list download failed : $1
+   exit 2
+  }
+  dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
+  if test $dlsize -lt $3; then
+   echo list is too small : $dlsize bytes. can be bad.
+   exit 2
+  fi
+  zzcat "$TMPLIST" | tr -d '\015' | zz "$2"
+  rm -f "$TMPLIST"
+}
+
+# useful in case ipban set is used in custom scripts
+FAIL=
+getipban || FAIL=1
+"$IPSET_DIR/create_ipset.sh"
+[ -n "$FAIL" ] && exit
+
+dl "$URL" "$ZHOSTLIST" 32768 4194304
+
+hup_zapret_daemons
+
+exit 0

ipset/get_refilter_ipsum.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+IPSET_DIR="$(dirname "$0")"
+IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
+
+. "$IPSET_DIR/def.sh"
+
+TMPLIST="$TMPDIR/list.txt"
+
+URL="https://github.com/1andrevich/Re-filter-lists/releases/latest/download/ipsum.lst"
+
+dl()
+{
+  # $1 - url
+  # $2 - file
+  # $3 - minsize
+  # $4 - maxsize
+  curl -L -H "Accept-Encoding: gzip" -k --fail --max-time 60 --connect-timeout 10 --retry 4 --max-filesize $4 -o "$TMPLIST" "$1" ||
+  {
+   echo list download failed : $1
+   exit 2
+  }
+  dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
+  if test $dlsize -lt $3; then
+   echo list is too small : $dlsize bytes. can be bad.
+   exit 2
+  fi
+  # remove DOS EOL \r
+  zzcat "$TMPLIST" | tr -d '\015' | zz "$2"
+  rm -f "$TMPLIST"
+}
+
+getuser && {
+ [ "$DISABLE_IPV4" != "1" ] && {
+ 	dl "$URL" "$ZIPLIST" 32768 4194304
+ }
+}
+
+"$IPSET_DIR/create_ipset.sh"

tpws/params.h

@@ -14,10 +14,6 @@
 #define HOSTLIST_AUTO_FAIL_THRESHOLD_DEFAULT	3
 #define	HOSTLIST_AUTO_FAIL_TIME_DEFAULT 	60
 
-#define L7_PROTO_HTTP	1
-#define L7_PROTO_TLS	2
-#define L7_PROTO_UNKNOWN 0x80000000
-
 enum bindll { unwanted=0, no, prefer, force };
 
 #define MAX_BINDS	32

tpws/tamper.c

@@ -38,7 +38,7 @@ static bool dp_impossible(struct desync_profile *dp, const char *hostname, t_l7p
 static bool dp_match(struct desync_profile *dp, const struct sockaddr *dest, const char *hostname, t_l7proto l7proto)
 {
 	// impossible case, hard filter
-	// impossible check avoid relatively slow ipset search
+	// impossible check avoids relatively slow ipset search
 	if (!dp_impossible(dp,hostname,l7proto) && dp_match_l3l4(dp,dest))
 	{
 		// soft filter
@@ -375,7 +375,7 @@ void tamper_out(t_ctrack *ctrack, const struct sockaddr *dest, uint8_t *segment,
 	if (ctrack->dp->oob) *split_flags |= SPLIT_FLAG_OOB;
 }
 
-static void auto_hostlist_reset_fail_counter(struct desync_profile *dp, const char *hostname)
+static void auto_hostlist_reset_fail_counter(struct desync_profile *dp, const char *hostname, const char *client_ip_port, t_l7proto l7proto)
 {
 	if (hostname)
 	{
@@ -386,12 +386,12 @@ static void auto_hostlist_reset_fail_counter(struct desync_profile *dp, const ch
 		{
 			HostFailPoolDel(&dp->hostlist_auto_fail_counters, fail_counter);
 			VPRINT("auto hostlist (profile %d) : %s : fail counter reset. website is working.\n", dp->n, hostname);
-			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : fail counter reset. website is working.", hostname, dp->n);
+			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : fail counter reset. website is working.", hostname, dp->n, client_ip_port, l7proto_str(l7proto));
 		}
 	}
 }
 
-static void auto_hostlist_failed(struct desync_profile *dp, const char *hostname)
+static void auto_hostlist_failed(struct desync_profile *dp, const char *hostname, const char *client_ip_port, t_l7proto l7proto)
 {
 	hostfail_pool *fail_counter;
 
@@ -407,7 +407,7 @@ static void auto_hostlist_failed(struct desync_profile *dp, const char *hostname
 	}
 	fail_counter->counter++;
 	VPRINT("auto hostlist (profile %d) : %s : fail counter %d/%d\n", dp->n , hostname, fail_counter->counter, dp->hostlist_auto_fail_threshold);
-	HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : fail counter %d/%d", hostname, dp->n, fail_counter->counter, dp->hostlist_auto_fail_threshold);
+	HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : fail counter %d/%d", hostname, dp->n, client_ip_port, l7proto_str(l7proto), fail_counter->counter, dp->hostlist_auto_fail_threshold);
 	if (fail_counter->counter >= dp->hostlist_auto_fail_threshold)
 	{
 		VPRINT("auto hostlist (profile %d) : fail threshold reached. adding %s to auto hostlist\n", dp->n , hostname);
@@ -418,7 +418,7 @@ static void auto_hostlist_failed(struct desync_profile *dp, const char *hostname
 		if (!HostlistCheck(dp, hostname, &bExcluded) && !bExcluded)
 		{
 			VPRINT("auto hostlist (profile %d) : adding %s to %s\n", dp->n, hostname, dp->hostlist_auto_filename);
-			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : adding to %s", hostname, dp->n, dp->hostlist_auto_filename);
+			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : adding to %s", hostname, dp->n, client_ip_port, l7proto_str(l7proto), dp->hostlist_auto_filename);
 			if (!StrPoolAddStr(&dp->hostlist, hostname))
 			{
 				DLOG_ERR("StrPoolAddStr out of memory\n");
@@ -434,16 +434,22 @@ static void auto_hostlist_failed(struct desync_profile *dp, const char *hostname
 		else
 		{
 			VPRINT("auto hostlist (profile %d) : NOT adding %s\n", dp->n, hostname);
-			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : NOT adding, duplicate detected", hostname, dp->n);
+			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : NOT adding, duplicate detected", hostname, dp->n, client_ip_port, l7proto_str(l7proto));
 		}
 	}
 }
 
-void tamper_in(t_ctrack *ctrack, uint8_t *segment,size_t segment_buffer_size,size_t *size)
+void tamper_in(t_ctrack *ctrack, const struct sockaddr *client, uint8_t *segment,size_t segment_buffer_size,size_t *size)
 {
+	DBGPRINT("tamper_in hostname=%s\n", ctrack->hostname);
+
 	bool bFail=false;
 
-	DBGPRINT("tamper_in hostname=%s\n", ctrack->hostname);
+	char client_ip_port[48];
+	if (*params.hostlist_auto_debuglog)
+		ntop46_port((struct sockaddr*)client,client_ip_port,sizeof(client_ip_port));
+	else
+		*client_ip_port=0;
 
 	if (ctrack->dp && ctrack->b_ah_check)
 	{
@@ -458,7 +464,7 @@ void tamper_in(t_ctrack *ctrack, uint8_t *segment,size_t segment_buffer_size,siz
 				if (bFail)
 				{
 					VPRINT("redirect to another domain detected. possibly DPI redirect.\n");
-					HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : redirect to another domain", ctrack->hostname, ctrack->dp->n);
+					HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : redirect to another domain", ctrack->hostname, ctrack->dp->n, client_ip_port, l7proto_str(ctrack->l7proto));
 				}
 				else
 					VPRINT("local or in-domain redirect detected. it's not a DPI redirect.\n");
@@ -468,17 +474,23 @@ void tamper_in(t_ctrack *ctrack, uint8_t *segment,size_t segment_buffer_size,siz
 				// received not http reply. do not monitor this connection anymore
 				VPRINT("incoming unknown HTTP data detected for hostname %s\n", ctrack->hostname);
 			}
-			if (bFail) auto_hostlist_failed(ctrack->dp, ctrack->hostname);
+			if (bFail) auto_hostlist_failed(ctrack->dp, ctrack->hostname, client_ip_port, ctrack->l7proto);
 		}
-		if (!bFail) auto_hostlist_reset_fail_counter(ctrack->dp, ctrack->hostname);
+		if (!bFail) auto_hostlist_reset_fail_counter(ctrack->dp, ctrack->hostname, client_ip_port, ctrack->l7proto);
 	}
 	ctrack->bTamperInCutoff = true;
 }
 
-void rst_in(t_ctrack *ctrack)
+void rst_in(t_ctrack *ctrack, const struct sockaddr *client)
 {
 	DBGPRINT("rst_in hostname=%s\n", ctrack->hostname);
 
+	char client_ip_port[48];
+	if (*params.hostlist_auto_debuglog)
+		ntop46_port((struct sockaddr*)client,client_ip_port,sizeof(client_ip_port));
+	else
+		*client_ip_port=0;
+
 	if (ctrack->dp && ctrack->b_ah_check)
 	{
 		HostFailPoolPurgeRateLimited(&ctrack->dp->hostlist_auto_fail_counters);
@@ -486,15 +498,21 @@ void rst_in(t_ctrack *ctrack)
 		if (!ctrack->bTamperInCutoff && ctrack->hostname)
 		{
 			VPRINT("incoming RST detected for hostname %s\n", ctrack->hostname);
-			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : incoming RST", ctrack->hostname, ctrack->dp->n);
+			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : incoming RST", ctrack->hostname, ctrack->dp->n, client_ip_port, l7proto_str(ctrack->l7proto));
-			auto_hostlist_failed(ctrack->dp, ctrack->hostname);
+			auto_hostlist_failed(ctrack->dp, ctrack->hostname, client_ip_port, ctrack->l7proto);
 		}
 	}
 }
-void hup_out(t_ctrack *ctrack)
+void hup_out(t_ctrack *ctrack, const struct sockaddr *client)
 {
 	DBGPRINT("hup_out hostname=%s\n", ctrack->hostname);
 	
+	char client_ip_port[48];
+	if (*params.hostlist_auto_debuglog)
+		ntop46_port((struct sockaddr*)client,client_ip_port,sizeof(client_ip_port));
+	else
+		*client_ip_port=0;
+
 	if (ctrack->dp && ctrack->b_ah_check)
 	{
 		HostFailPoolPurgeRateLimited(&ctrack->dp->hostlist_auto_fail_counters);
@@ -503,8 +521,8 @@ void hup_out(t_ctrack *ctrack)
 		{
 			// local leg dropped connection after first request. probably due to timeout.
 			VPRINT("local leg closed connection after first request (timeout ?). hostname: %s\n", ctrack->hostname);
-			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client closed connection without server reply", ctrack->hostname, ctrack->dp->n);
+			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : client closed connection without server reply", ctrack->hostname, ctrack->dp->n, client_ip_port, l7proto_str(ctrack->l7proto));
-			auto_hostlist_failed(ctrack->dp, ctrack->hostname);
+			auto_hostlist_failed(ctrack->dp, ctrack->hostname, client_ip_port, ctrack->l7proto);
 		}
 	}
 }

tpws/tamper.h

@@ -10,12 +10,15 @@
 #define SPLIT_FLAG_OOB		0x02
 
 typedef enum {UNKNOWN=0, HTTP, TLS} t_l7proto;
+#define L7_PROTO_HTTP	1
+#define L7_PROTO_TLS	2
+#define L7_PROTO_UNKNOWN 0x80000000
 const char *l7proto_str(t_l7proto l7);
+
 typedef struct
 {
 	// common state
 	t_l7proto l7proto;
-	bool bFirstReplyChecked;
 	bool bTamperInCutoff;
 	bool b_ah_check;
 	bool b_not_act;
@@ -26,8 +29,8 @@ typedef struct
 void apply_desync_profile(t_ctrack *ctrack, const struct sockaddr *dest);
 
 void tamper_out(t_ctrack *ctrack, const struct sockaddr *dest, uint8_t *segment,size_t segment_buffer_size,size_t *size, size_t *split_pos, uint8_t *split_flags);
-void tamper_in(t_ctrack *ctrack, uint8_t *segment,size_t segment_buffer_size,size_t *size);
+void tamper_in(t_ctrack *ctrack, const struct sockaddr *client, uint8_t *segment,size_t segment_buffer_size,size_t *size);
 // connection reset by remote leg
-void rst_in(t_ctrack *ctrack);
+void rst_in(t_ctrack *ctrack, const struct sockaddr *client);
 // local leg closed connection (timeout waiting response ?)
-void hup_out(t_ctrack *ctrack);
+void hup_out(t_ctrack *ctrack, const struct sockaddr *client);

tpws/tpws_conn.c

@@ -569,7 +569,6 @@ static tproxy_conn_t* add_tcp_connection(int efd, struct tailhead *conn_list,int
 {
 	struct sockaddr_storage orig_dst;
 	tproxy_conn_t *conn;
-	int remote_fd=0;
 
 	if (proxy_type==CONN_TYPE_TRANSPARENT)
 	{
@@ -597,19 +596,8 @@ static tproxy_conn_t* add_tcp_connection(int efd, struct tailhead *conn_list,int
 		return 0;
 	}
 
-	if (proxy_type==CONN_TYPE_TRANSPARENT)
-	{
-		if ((remote_fd = connect_remote((struct sockaddr *)&orig_dst, 0)) < 0)
-		{
-			DLOG_ERR("Failed to connect\n");
-			close(local_fd);
-			return NULL;
-		}
-	}
-
 	if(!(conn = new_conn(local_fd, false)))
 	{
-		if (remote_fd) close(remote_fd);
 		close(local_fd);
 		return NULL;
 	}
@@ -617,18 +605,33 @@ static tproxy_conn_t* add_tcp_connection(int efd, struct tailhead *conn_list,int
 	conn->state = CONN_AVAILABLE; // accepted connection is immediately available
 	conn->efd = efd;
 
+	socklen_t salen=sizeof(conn->client);
+	getpeername(conn->fd,(struct sockaddr *)&conn->client,&salen);
+
 	if (proxy_type==CONN_TYPE_TRANSPARENT)
 	{
 		sa46copy(&conn->dest, (struct sockaddr *)&orig_dst);
 
-		if(!(conn->partner = new_conn(remote_fd, true)))
+		if(!(conn->partner = new_conn(0, true)))
 		{
 			free_conn(conn);
-			close(remote_fd);
 			return NULL;
 		}
+
 		conn->partner->partner = conn;
 		conn->partner->efd = efd;
+		conn->partner->client = conn->client;
+		conn->partner->dest = conn->dest;
+
+		apply_desync_profile(&conn->track, (struct sockaddr *)&conn->dest);
+
+		if ((conn->partner->fd = connect_remote((struct sockaddr *)&orig_dst, conn->track.dp ? conn->track.dp->mss : 0)) < 0)
+		{
+			DLOG_ERR("Failed to connect\n");
+			free_conn(conn->partner);
+			free_conn(conn);
+			return NULL;
+		}
 
 		//remote_fd is connecting. Non-blocking connects are signaled as done by
 		//socket being marked as ready for writing
@@ -663,9 +666,6 @@ static tproxy_conn_t* add_tcp_connection(int efd, struct tailhead *conn_list,int
 		legs_remote++;
 	}
 
-	if (proxy_type==CONN_TYPE_TRANSPARENT)
-		apply_desync_profile(&conn->track, (struct sockaddr *)&conn->dest);
-
 	return conn;
 } 
 
@@ -815,6 +815,8 @@ static bool proxy_mode_connect_remote(tproxy_conn_t *conn, struct tailhead *conn
 	}
 	conn->partner->partner = conn;
 	conn->partner->efd = conn->efd;
+	conn->partner->client = conn->client;
+	conn->partner->dest = conn->dest;
 	if (!epoll_set(conn->partner, EPOLLOUT))
 	{
 		DLOG_ERR("socks epoll_set error %d\n", errno);
@@ -1074,7 +1076,7 @@ static void tamper(tproxy_conn_t *conn, uint8_t *segment, size_t segment_buffer_
 		if (conn->remote)
 		{
 			if (conn_partner_alive(conn) && !conn->partner->track.bTamperInCutoff)
-				tamper_in(&conn->partner->track,segment,segment_buffer_size,segment_size);
+				tamper_in(&conn->partner->track,(struct sockaddr*)&conn->partner->client,segment,segment_buffer_size,segment_size);
 		}
 		else
 		{
@@ -1524,18 +1526,11 @@ int event_loop(const int *listen_fd, size_t listen_fd_ct)
 				else
 				{	
 					print_legs();
-					
+
 					if (params.debug>=1)
 					{
-						struct sockaddr_storage sa;
-						socklen_t salen=sizeof(sa);
 						char ip_port[48];
-
+						ntop46_port((struct sockaddr*)&conn->client,ip_port,sizeof(ip_port));
-						if (getpeername(conn->fd,(struct sockaddr *)&sa,&salen))
-							*ip_port=0;
-						else
-							ntop46_port((struct sockaddr*)&sa,ip_port,sizeof(ip_port));
-
 						VPRINT("Socket fd=%d (local) connected from %s\n", conn->fd, ip_port);
 					}
 					set_user_timeout(conn->fd, params.tcp_user_timeout_local);
@@ -1563,7 +1558,7 @@ int event_loop(const int *listen_fd, size_t listen_fd_ct)
 						read_all_and_buffer(conn,3);
 						if (errn==ECONNRESET && conn_partner_alive(conn))
 						{
-							if (conn->remote && params.tamper) rst_in(&conn->partner->track);
+							if (conn->remote && params.tamper) rst_in(&conn->partner->track,(struct sockaddr*)&conn->partner->client);
 
 							struct linger lin;
 							lin.l_onoff=1;
@@ -1588,7 +1583,7 @@ int event_loop(const int *listen_fd, size_t listen_fd_ct)
 					{
 						DBGPRINT("EPOLLRDHUP\n");
 						read_all_and_buffer(conn,2);
-						if (!conn->remote && params.tamper) hup_out(&conn->track);
+						if (!conn->remote && params.tamper) hup_out(&conn->track,(struct sockaddr*)&conn->client);
 
 						conn->state = CONN_RDHUP; // only writes. do not receive RDHUP anymore
 						if (conn_has_unsent(conn))

tpws/tpws_conn.h

@@ -54,7 +54,7 @@ struct tproxy_conn
 	int splice_pipe[2];
 	conn_state_t state;
 	conn_type_t conn_type;
-	sockaddr_in46 src, dest;
+	sockaddr_in46 client, dest; // ip:port of client, ip:port of target
 
 	struct tproxy_conn *partner; // other leg
 	time_t orphan_since;