tpws: fix regression

binaries/win64/zapret-winws/list-youtube.txt

@@ -1,3 +1,4 @@
 googlevideo.com
 youtubei.googleapis.com
 i.ytimg.com
+youtube.com

binaries/win64/zapret-winws/preset_russia.cmd

@@ -3,5 +3,5 @@ start "zapret: http,https,quic" /min "%~dp0winws.exe" ^
 --filter-udp=443 --hostlist="%~dp0list-youtube.txt" --dpi-desync=fake --dpi-desync-repeats=11 --dpi-desync-fake-quic="%~dp0quic_initial_www_google_com.bin" --new ^
 --filter-udp=443 --dpi-desync=fake --dpi-desync-repeats=11 --new ^
 --filter-tcp=80 --dpi-desync=fake,split2 --dpi-desync-autottl=2 --dpi-desync-fooling=md5sig --new ^
---filter-tcp=443 --hostlist="%~dp0list-youtube.txt" --dpi-desync=fake,split2 --dpi-desync-autottl=2 --dpi-desync-fooling=md5sig --dpi-desync-fake-tls="%~dp0tls_clienthello_www_google_com.bin" --new ^
+--filter-tcp=443 --hostlist="%~dp0list-youtube.txt" --dpi-desync=fake,split2 --dpi-desync-repeats=11 --dpi-desync-autottl=2 --dpi-desync-fooling=md5sig --dpi-desync-fake-tls="%~dp0tls_clienthello_www_google_com.bin" --new ^
 --dpi-desync=fake,disorder2 --dpi-desync-autottl=2 --dpi-desync-fooling=md5sig

binaries/win64/zapret-winws/preset_russia_autohostlist.cmd

@@ -3,5 +3,5 @@ start "zapret: http,https,quic" /min "%~dp0winws.exe" ^
 --filter-udp=443 --hostlist="%~dp0list-youtube.txt" --dpi-desync=fake --dpi-desync-repeats=11 --dpi-desync-fake-quic="%~dp0quic_initial_www_google_com.bin" --new ^
 --filter-udp=443 --dpi-desync=fake --dpi-desync-repeats=11 --new ^
 --filter-tcp=80 --dpi-desync=fake,split2 --dpi-desync-autottl=2 --dpi-desync-fooling=md5sig --hostlist-auto="%~dp0autohostlist.txt" --new ^
---filter-tcp=443 --hostlist="%~dp0list-youtube.txt" --dpi-desync=fake,split2 --dpi-desync-autottl=2 --dpi-desync-fooling=md5sig --dpi-desync-fake-tls="%~dp0tls_clienthello_www_google_com.bin" --new ^
+--filter-tcp=443 --hostlist="%~dp0list-youtube.txt" --dpi-desync=fake,split2 --dpi-desync-repeats=11 --dpi-desync-autottl=2 --dpi-desync-fooling=md5sig --dpi-desync-fake-tls="%~dp0tls_clienthello_www_google_com.bin" --new ^
 --dpi-desync=fake,disorder2 --dpi-desync-autottl=2 --dpi-desync-fooling=md5sig --hostlist-auto="%~dp0autohostlist.txt"

tpws/helpers.c

@@ -11,6 +11,12 @@
 #include <time.h>
 #include <sys/stat.h>
 
+void rtrim(char *s)
+{
+	if (s)
+		for (char *p = s + strlen(s) - 1; p >= s && (*p == '\n' || *p == '\r'); p--) *p = '\0';
+}
+
 char *strncasestr(const char *s,const char *find, size_t slen)
 {
 	char c, sc;
@@ -81,7 +87,6 @@ void print_sockaddr(const struct sockaddr *sa)
 	printf("%s",ip_port);
 }
 
-
 // -1 = error,  0 = not local, 1 = local
 bool check_local_ip(const struct sockaddr *saddr)
 {
@@ -287,3 +292,101 @@ bool pf_is_empty(const port_filter *pf)
 {
 	return !pf->neg && !pf->from && !pf->to;
 }
+
+
+static void mask_from_preflen6_make(uint8_t plen, struct in6_addr *a)
+{
+	if (plen >= 128)
+		memset(a->s6_addr,0xFF,16);
+	else
+	{
+		uint8_t n = plen >> 3;
+		memset(a->s6_addr,0xFF,n);
+		memset(a->s6_addr+n,0x00,16-n);
+		a->s6_addr[n] = (uint8_t)(0xFF00 >> (plen & 7));
+	}
+}
+struct in6_addr ip6_mask[129];
+void mask_from_preflen6_prepare(void)
+{
+	for (int plen=0;plen<=128;plen++) mask_from_preflen6_make(plen, ip6_mask+plen);
+}
+
+#if defined(__GNUC__) && !defined(__llvm__)
+__attribute__((optimize ("no-strict-aliasing")))
+#endif
+void ip6_and(const struct in6_addr * restrict a, const struct in6_addr * restrict b, struct in6_addr * restrict result)
+{
+#ifdef __SIZEOF_INT128__
+	// gcc and clang have 128 bit int types on some 64-bit archs. take some advantage
+	*((unsigned __int128*)result->s6_addr) = *((unsigned __int128*)a->s6_addr) & *((unsigned __int128*)b->s6_addr);
+#else
+	((uint64_t*)result->s6_addr)[0] = ((uint64_t*)a->s6_addr)[0] & ((uint64_t*)b->s6_addr)[0];
+	((uint64_t*)result->s6_addr)[1] = ((uint64_t*)a->s6_addr)[1] & ((uint64_t*)b->s6_addr)[1];
+#endif
+}
+
+void str_cidr4(char *s, size_t s_len, const struct cidr4 *cidr)
+{
+	char s_ip[16];
+	*s_ip=0;
+	inet_ntop(AF_INET, &cidr->addr, s_ip, sizeof(s_ip));
+	snprintf(s,s_len,cidr->preflen<32 ? "%s/%u" : "%s", s_ip, cidr->preflen);
+}
+void print_cidr4(const struct cidr4 *cidr)
+{
+	char s[19];
+	str_cidr4(s,sizeof(s),cidr);
+	printf("%s",s);
+}
+void str_cidr6(char *s, size_t s_len, const struct cidr6 *cidr)
+{
+	char s_ip[40];
+	*s_ip=0;
+	inet_ntop(AF_INET6, &cidr->addr, s_ip, sizeof(s_ip));
+	snprintf(s,s_len,cidr->preflen<128 ? "%s/%u" : "%s", s_ip, cidr->preflen);
+}
+void print_cidr6(const struct cidr6 *cidr)
+{
+	char s[44];
+	str_cidr6(s,sizeof(s),cidr);
+	printf("%s",s);
+}
+bool parse_cidr4(char *s, struct cidr4 *cidr)
+{
+	char *p,d;
+	bool b;
+	unsigned int plen;
+
+	if ((p = strchr(s, '/')))
+	{
+		if (sscanf(p + 1, "%u", &plen)!=1 || plen>32)
+			return false;
+		cidr->preflen = (uint8_t)plen;
+		d=*p; *p=0; // backup char
+	}
+	else
+		cidr->preflen = 32;
+	b = (inet_pton(AF_INET, s, &cidr->addr)==1);
+	if (p) *p=d; // restore char
+	return b;
+}
+bool parse_cidr6(char *s, struct cidr6 *cidr)
+{
+	char *p,d;
+	bool b;
+	unsigned int plen;
+
+	if ((p = strchr(s, '/')))
+	{
+		if (sscanf(p + 1, "%u", &plen)!=1 || plen>128)
+			return false;
+		cidr->preflen = (uint8_t)plen;
+		d=*p; *p=0; // backup char
+	}
+	else
+		cidr->preflen = 128;
+	b = (inet_pton(AF_INET6, s, &cidr->addr)==1);
+	if (p) *p=d; // restore char
+	return b;
+}

tpws/helpers.h

@@ -8,6 +8,7 @@
 #include <stdio.h>
 #include <time.h>
 
+void rtrim(char *s);
 char *strncasestr(const char *s,const char *find, size_t slen);
 
 bool append_to_list_file(const char *filename, const char *s);
@@ -71,3 +72,33 @@ bool pf_is_empty(const port_filter *pf);
 #else
 #define IN6_EXTRACT_MAP4(a)	(((const uint32_t *) (a))[3])
 #endif
+
+
+struct cidr4
+{
+	struct in_addr addr;
+	uint8_t	preflen;
+};
+struct cidr6
+{
+	struct in6_addr addr;
+	uint8_t	preflen;
+};
+void str_cidr4(char *s, size_t s_len, const struct cidr4 *cidr);
+void print_cidr4(const struct cidr4 *cidr);
+void str_cidr6(char *s, size_t s_len, const struct cidr6 *cidr);
+void print_cidr6(const struct cidr6 *cidr);
+bool parse_cidr4(char *s, struct cidr4 *cidr);
+bool parse_cidr6(char *s, struct cidr6 *cidr);
+
+static inline uint32_t mask_from_preflen(uint32_t preflen)
+{
+	return preflen ? preflen<32 ? ~((1 << (32-preflen)) - 1) : 0xFFFFFFFF : 0;
+}
+void ip6_and(const struct in6_addr * restrict a, const struct in6_addr * restrict b, struct in6_addr * restrict result);
+extern struct in6_addr ip6_mask[129];
+void mask_from_preflen6_prepare(void);
+static inline const struct in6_addr *mask_from_preflen6(uint8_t preflen)
+{
+	return ip6_mask+preflen;
+}

tpws/hostlist.c

@@ -172,10 +172,9 @@ static bool LoadIncludeHostListsForProfile(struct desync_profile *dp)
 	return true;
 }
 
-// return : true = apply fooling, false = do not apply
 bool HostlistCheck(struct desync_profile *dp, const char *host, bool *excluded)
 {
-	VPRINT("* Hostlist check for profile %d\n",dp->n);
+	VPRINT("* hostlist check for profile %d\n",dp->n);
 	if (*dp->hostlist_auto_filename)
 	{
 		time_t t = file_mod_time(dp->hostlist_auto_filename);

tpws/ipset.c

@@ -0,0 +1,195 @@
+#include <stdio.h>
+#include "ipset.h"
+#include "gzip.h"
+#include "helpers.h"
+
+// inplace tolower() and add to pool
+static bool addpool(ipset *ips, char **s, const char *end, int *ct)
+{
+	char *p, cidr[128];
+	size_t l;
+	struct cidr4 c4;
+	struct cidr6 c6;
+
+	// advance until eol
+	for (p=*s; p<end && *p && *p!='\r' && *p != '\n'; p++);
+
+	// comment line
+	if (!(**s == '#' || **s == ';' || **s == '/' || **s == '\r' || **s == '\n' ))
+	{
+		l = p-*s;
+		if (l>=sizeof(cidr)) l=sizeof(cidr)-1;
+		memcpy(cidr,*s,l);
+		cidr[l]=0;
+		rtrim(cidr);
+
+		if (parse_cidr4(cidr,&c4))
+		{
+			if (!ipset4AddCidr(&ips->ips4, &c4))
+			{
+				ipsetDestroy(ips);
+				return false;
+			}
+			(*ct)++;
+		}
+		else if (parse_cidr6(cidr,&c6))
+		{
+			if (!ipset6AddCidr(&ips->ips6, &c6))
+			{
+				ipsetDestroy(ips);
+				return false;
+			}
+			(*ct)++;
+		}
+		else
+			DLOG_ERR("bad ip or subnet : %s\n",cidr);
+	}
+
+	// advance to the next line
+	for (; p<end && (!*p || *p=='\r' || *p=='\n') ; p++);
+	*s = p;
+	return true;
+
+}
+
+static bool AppendIpset(ipset *ips, const char *filename)
+{
+	char *p, *e, s[256], *zbuf;
+	size_t zsize;
+	int ct = 0;
+	FILE *F;
+	int r;
+
+	DLOG_CONDUP("Loading ipset %s\n",filename);
+
+	if (!(F = fopen(filename, "rb")))
+	{
+		DLOG_ERR("Could not open %s\n", filename);
+		return false;
+	}
+
+	if (is_gzip(F))
+	{
+		r = z_readfile(F,&zbuf,&zsize);
+		fclose(F);
+		if (r==Z_OK)
+		{
+			DLOG_CONDUP("zlib compression detected. uncompressed size : %zu\n", zsize);
+			
+			p = zbuf;
+			e = zbuf + zsize;
+			while(p<e)
+			{
+				if (!addpool(ips,&p,e,&ct))
+				{
+					DLOG_ERR("Not enough memory to store ipset : %s\n", filename);
+					free(zbuf);
+					return false;
+				}
+			}
+			free(zbuf);
+		}
+		else
+		{
+			DLOG_ERR("zlib decompression failed : result %d\n",r);
+			return false;
+		}
+	}
+	else
+	{
+		DLOG_CONDUP("loading plain text list\n");
+		
+		while (fgets(s, sizeof(s)-1, F))
+		{
+			p = s;
+			if (!addpool(ips,&p,p+strlen(p),&ct))
+			{
+				DLOG_ERR("Not enough memory to store ipset : %s\n", filename);
+				fclose(F);
+				return false;
+			}
+		}
+		fclose(F);
+	}
+
+	DLOG_CONDUP("Loaded %d ip/subnets from %s\n", ct, filename);
+	return true;
+}
+
+static bool LoadIpsets(ipset *ips, struct str_list_head *file_list)
+{
+	struct str_list *file;
+
+	ipsetDestroy(ips);
+
+	LIST_FOREACH(file, file_list, next)
+	{
+		if (!AppendIpset(ips, file->str)) return false;
+	}
+	return true;
+}
+
+bool LoadIncludeIpsets()
+{
+	struct desync_profile_list *dpl;
+	LIST_FOREACH(dpl, &params.desync_profiles, next)
+		if (!LoadIpsets(&dpl->dp.ips, &dpl->dp.ipset_files))
+			return false;
+	return true;
+}
+bool LoadExcludeIpsets()
+{
+	struct desync_profile_list *dpl;
+	LIST_FOREACH(dpl, &params.desync_profiles, next)
+		if (!LoadIpsets(&dpl->dp.ips_exclude, &dpl->dp.ipset_exclude_files))
+			return false;
+	return true;
+}
+
+bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_addr *ipv6)
+{
+	char s_ip[40];
+	bool bInSet=false;
+
+	if (!!ipv4 != !!ipv6)
+	{
+		*s_ip=0;
+		if (ipv4)
+		{
+			if (params.debug) inet_ntop(AF_INET, ipv4, s_ip, sizeof(s_ip));
+			if (ips->ips4) bInSet = ipset4Check(ips->ips4, ipv4, 32);
+		}
+		if (ipv6)
+		{
+			if (params.debug) inet_ntop(AF_INET6, ipv6, s_ip, sizeof(s_ip));
+			if (ips->ips6) bInSet = ipset6Check(ips->ips6, ipv6, 128);
+		}
+		VPRINT("ipset check for %s : %s\n", s_ip, bInSet ? "positive" : "negative");
+	}
+	else
+		// ipv4 and ipv6 are both empty or non-empty
+		VPRINT("ipset check error !!!!!!!! ipv4=%p ipv6=%p\n",ipv4,ipv6);
+	return bInSet;
+}
+
+static bool IpsetCheck_(const ipset *ips, const ipset *ips_exclude, const struct in_addr *ipv4, const struct in6_addr *ipv6)
+{
+	if (!IPSET_EMPTY(ips_exclude))
+	{
+		VPRINT("exclude ");
+		if (SearchIpset(ips_exclude, ipv4, ipv6))
+			return false;
+	}
+	if (!IPSET_EMPTY(ips))
+	{
+		VPRINT("include ");
+		return SearchIpset(ips, ipv4, ipv6);
+	}
+	return true;
+}
+
+bool IpsetCheck(struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6)
+{
+	if (!PROFILE_IPSETS_EMPTY(dp)) VPRINT("* ipset check for profile %d\n",dp->n);
+	return IpsetCheck_(&dp->ips,&dp->ips_exclude,ipv4,ipv6);
+}

tpws/ipset.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <stdbool.h>
+#include <arpa/inet.h>
+#include "params.h"
+#include "pools.h"
+
+bool LoadIncludeIpsets();
+bool LoadExcludeIpsets();
+bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_addr *ipv6);
+bool IpsetCheck(struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6);

tpws/params.c

@@ -169,8 +169,12 @@ static void dp_entry_destroy(struct desync_profile_list *entry)
 {
 	strlist_destroy(&entry->dp.hostlist_files);
 	strlist_destroy(&entry->dp.hostlist_exclude_files);
+	strlist_destroy(&entry->dp.ipset_files);
+	strlist_destroy(&entry->dp.ipset_exclude_files);
 	StrPoolDestroy(&entry->dp.hostlist_exclude);
 	StrPoolDestroy(&entry->dp.hostlist);
+	ipsetDestroy(&entry->dp.ips);
+	ipsetDestroy(&entry->dp.ips_exclude);
 	HostFailPoolDestroy(&entry->dp.hostlist_auto_fail_counters);
 	free(entry);
 }

tpws/params.h

@@ -14,6 +14,10 @@
 #define HOSTLIST_AUTO_FAIL_THRESHOLD_DEFAULT	3
 #define	HOSTLIST_AUTO_FAIL_TIME_DEFAULT 	60
 
+#define L7_PROTO_HTTP	1
+#define L7_PROTO_TLS	2
+#define L7_PROTO_UNKNOWN 0x80000000
+
 enum bindll { unwanted=0, no, prefer, force };
 
 #define MAX_BINDS	32
@@ -51,6 +55,9 @@ struct desync_profile
 	
 	bool filter_ipv4,filter_ipv6;
 	port_filter pf_tcp;
+	uint32_t filter_l7;	// L7_PROTO_* bits
+	ipset ips,ips_exclude;
+	struct str_list_head ipset_files, ipset_exclude_files;
 
 	strpool *hostlist, *hostlist_exclude;
 	struct str_list_head hostlist_files, hostlist_exclude_files;
@@ -60,6 +67,8 @@ struct desync_profile
 	hostfail_pool *hostlist_auto_fail_counters;
 };
 
+#define PROFILE_IPSETS_EMPTY(dp) (IPSET_EMPTY(&dp->ips) && IPSET_EMPTY(&dp->ips_exclude))
+
 struct desync_profile_list {
 	struct desync_profile dp;
 	LIST_ENTRY(desync_profile_list) next;

tpws/pools.c

@@ -52,6 +52,12 @@ bool StrPoolAddStr(strpool **pp, const char *s)
 {
 	return StrPoolAddStrLen(pp, s, strlen(s));
 }
+bool StrPoolAddUniqueStr(strpool **pp,const char *s)
+{
+	if (StrPoolCheckStr(*pp,s))
+		return true;
+	return StrPoolAddStr(pp,s);
+}
 
 bool StrPoolCheckStr(strpool *p, const char *s)
 {
@@ -151,3 +157,127 @@ void strlist_destroy(struct str_list_head *head)
 		strlist_entry_destroy(entry);
 	}
 }
+
+
+
+void ipset4Destroy(ipset4 **ipset)
+{
+	ipset4 *elem, *tmp;
+	HASH_ITER(hh, *ipset, elem, tmp)
+	{
+		HASH_DEL(*ipset, elem);
+		free(elem);
+	}
+}
+bool ipset4Check(ipset4 *ipset, const struct in_addr *a, uint8_t preflen)
+{
+	uint32_t ip = ntohl(a->s_addr);
+	struct cidr4 cidr;
+	ipset4 *ips_found;
+
+	// zero alignment bytes
+	memset(&cidr,0,sizeof(cidr));
+	cidr.preflen = preflen+1;
+	do
+	{
+		cidr.preflen--;
+		cidr.addr.s_addr = htonl(ip & mask_from_preflen(cidr.preflen));
+		HASH_FIND(hh, ipset, &cidr, sizeof(cidr), ips_found);
+		if (ips_found) return true;
+	} while(cidr.preflen);
+
+	return false;
+}
+bool ipset4Add(ipset4 **ipset, const struct in_addr *a, uint8_t preflen)
+{
+	if (preflen>32) return false;
+
+	// avoid dups
+	if (ipset4Check(*ipset, a, preflen)) return true; // already included
+
+	struct ipset4 *entry = calloc(1,sizeof(ipset4));
+	if (!entry) return false;
+
+	entry->cidr.addr.s_addr = htonl(ntohl(a->s_addr) & mask_from_preflen(preflen));
+	entry->cidr.preflen = preflen;
+	oom = false;
+	HASH_ADD(hh, *ipset, cidr, sizeof(entry->cidr), entry);
+	if (oom) { free(entry); return false; }
+
+	return true;
+}
+void ipset4Print(ipset4 *ipset)
+{
+	ipset4 *ips, *tmp;
+	HASH_ITER(hh, ipset , ips, tmp)
+	{
+		print_cidr4(&ips->cidr);
+		printf("\n");
+	}
+}
+
+void ipset6Destroy(ipset6 **ipset)
+{
+	ipset6 *elem, *tmp;
+	HASH_ITER(hh, *ipset, elem, tmp)
+	{
+		HASH_DEL(*ipset, elem);
+		free(elem);
+	}
+}
+bool ipset6Check(ipset6 *ipset, const struct in6_addr *a, uint8_t preflen)
+{
+	struct cidr6 cidr;
+	ipset6 *ips_found;
+
+	// zero alignment bytes
+	memset(&cidr,0,sizeof(cidr));
+	cidr.preflen = preflen+1;
+	do
+	{
+		cidr.preflen--;
+		ip6_and(a, mask_from_preflen6(cidr.preflen), &cidr.addr);
+		HASH_FIND(hh, ipset, &cidr, sizeof(cidr), ips_found);
+		if (ips_found) return true;
+	} while(cidr.preflen);
+
+	return false;
+}
+bool ipset6Add(ipset6 **ipset, const struct in6_addr *a, uint8_t preflen)
+{
+	if (preflen>128) return false;
+
+	// avoid dups
+	if (ipset6Check(*ipset, a, preflen)) return true; // already included
+
+	struct ipset6 *entry = calloc(1,sizeof(ipset6));
+	if (!entry) return false;
+
+	ip6_and(a, mask_from_preflen6(preflen), &entry->cidr.addr);
+	entry->cidr.preflen = preflen;
+	oom = false;
+	HASH_ADD(hh, *ipset, cidr, sizeof(entry->cidr), entry);
+	if (oom) { free(entry); return false; }
+
+	return true;
+}
+void ipset6Print(ipset6 *ipset)
+{
+	ipset6 *ips, *tmp;
+	HASH_ITER(hh, ipset , ips, tmp)
+	{
+		print_cidr6(&ips->cidr);
+		printf("\n");
+	}
+}
+
+void ipsetDestroy(ipset *ipset)
+{
+	ipset4Destroy(&ipset->ips4);
+	ipset6Destroy(&ipset->ips6);
+}
+void ipsetPrint(ipset *ipset)
+{
+	ipset4Print(ipset->ips4);
+	ipset6Print(ipset->ips6);
+}

tpws/pools.h

@@ -5,6 +5,8 @@
 #include <sys/queue.h>
 #include <time.h>
 
+#include "helpers.h"
+
 //#define HASH_BLOOM 20
 #define HASH_NONFATAL_OOM 1
 #define HASH_FUNCTION HASH_BER
@@ -17,6 +19,7 @@ typedef struct strpool {
 
 void StrPoolDestroy(strpool **pp);
 bool StrPoolAddStr(strpool **pp,const char *s);
+bool StrPoolAddUniqueStr(strpool **pp,const char *s);
 bool StrPoolAddStrLen(strpool **pp,const char *s,size_t slen);
 bool StrPoolCheckStr(strpool *p,const char *s);
 
@@ -44,3 +47,41 @@ void HostFailPoolDump(hostfail_pool *p);
 
 bool strlist_add(struct str_list_head *head, const char *filename);
 void strlist_destroy(struct str_list_head *head);
+
+
+typedef struct ipset4 {
+	struct cidr4 cidr;	/* key */
+	UT_hash_handle hh;	/* makes this structure hashable */
+} ipset4;
+typedef struct ipset6 {
+	struct cidr6 cidr;	/* key */
+	UT_hash_handle hh;	/* makes this structure hashable */
+} ipset6;
+// combined ipset ipv4 and ipv6
+typedef struct ipset {
+	ipset4 *ips4;
+	ipset6 *ips6;
+} ipset;
+
+#define IPSET_EMPTY(ips) (!(ips)->ips4 && !(ips)->ips6)
+
+void ipset4Destroy(ipset4 **ipset);
+bool ipset4Add(ipset4 **ipset, const struct in_addr *a, uint8_t preflen);
+static inline bool ipset4AddCidr(ipset4 **ipset, const struct cidr4 *cidr)
+{
+	return ipset4Add(ipset,&cidr->addr,cidr->preflen);
+}
+bool ipset4Check(ipset4 *ipset, const struct in_addr *a, uint8_t preflen);
+void ipset4Print(ipset4 *ipset);
+
+void ipset6Destroy(ipset6 **ipset);
+bool ipset6Add(ipset6 **ipset, const struct in6_addr *a, uint8_t preflen);
+static inline bool ipset6AddCidr(ipset6 **ipset, const struct cidr6 *cidr)
+{
+	return ipset6Add(ipset,&cidr->addr,cidr->preflen);
+}
+bool ipset6Check(ipset6 *ipset, const struct in6_addr *a, uint8_t preflen);
+void ipset6Print(ipset6 *ipset);
+
+void ipsetDestroy(ipset *ipset);
+void ipsetPrint(ipset *ipset);

tpws/tamper.c

@@ -1,22 +1,50 @@
 #define _GNU_SOURCE
 
-#include "tamper.h"
-#include "hostlist.h"
-#include "protocol.h"
-#include "helpers.h"
 #include <string.h>
 #include <stdio.h>
+#include "tamper.h"
+#include "hostlist.h"
+#include "ipset.h"
+#include "protocol.h"
+#include "helpers.h"
 
-static bool dp_match_l3l4(struct desync_profile *dp, bool ipv6, uint16_t tcp_port)
+const char *l7proto_str(t_l7proto l7)
 {
-	return \
-		((!ipv6 && dp->filter_ipv4) || (ipv6 && dp->filter_ipv6)) &&
-		(!tcp_port || pf_in_range(tcp_port,&dp->pf_tcp));
-}
-static bool dp_match(struct desync_profile *dp, bool ipv6, uint16_t tcp_port, const char *hostname)
-{
-	if (dp_match_l3l4(dp,ipv6,tcp_port))
+	switch(l7)
 	{
+		case HTTP: return "http";
+		case TLS: return "tls";
+		default: return "unknown";
+	}
+}
+static bool l7_proto_match(t_l7proto l7proto, uint32_t filter_l7)
+{
+	return  (l7proto==UNKNOWN && (filter_l7 & L7_PROTO_UNKNOWN)) ||
+		(l7proto==HTTP && (filter_l7 & L7_PROTO_HTTP)) ||
+		(l7proto==TLS && (filter_l7 & L7_PROTO_TLS));
+}
+
+static bool dp_match_l3l4(struct desync_profile *dp, const struct sockaddr *dest)
+{
+	return  ((dest->sa_family==AF_INET && dp->filter_ipv4) || (dest->sa_family==AF_INET6 && dp->filter_ipv6)) &&
+		pf_in_range(saport(dest), &dp->pf_tcp) &&
+		IpsetCheck(dp, dest->sa_family==AF_INET ? &((struct sockaddr_in*)dest)->sin_addr : NULL, dest->sa_family==AF_INET6 ? &((struct sockaddr_in6*)dest)->sin6_addr : NULL);
+}
+static bool dp_impossible(struct desync_profile *dp, const char *hostname, t_l7proto l7proto)
+{
+	return	!PROFILE_IPSETS_EMPTY(dp) &&
+		((dp->filter_l7 && !l7_proto_match(l7proto, dp->filter_l7)) || (!*dp->hostlist_auto_filename && !hostname && (dp->hostlist || dp->hostlist_exclude)));
+}
+static bool dp_match(struct desync_profile *dp, const struct sockaddr *dest, const char *hostname, t_l7proto l7proto)
+{
+	// impossible case, hard filter
+	// impossible check avoid relatively slow ipset search
+	if (!dp_impossible(dp,hostname,l7proto) && dp_match_l3l4(dp,dest))
+	{
+		// soft filter
+		if (dp->filter_l7 && !l7_proto_match(l7proto, dp->filter_l7))
+			return false;
+
 		// autohostlist profile matching l3/l4 filter always win
 		if (*dp->hostlist_auto_filename) return true;
 
@@ -32,13 +60,18 @@ static bool dp_match(struct desync_profile *dp, bool ipv6, uint16_t tcp_port, co
 	}
 	return false;
 }
-static struct desync_profile *dp_find(struct desync_profile_list_head *head, bool ipv6, uint16_t tcp_port, const char *hostname)
+static struct desync_profile *dp_find(struct desync_profile_list_head *head, const struct sockaddr *dest, const char *hostname, t_l7proto l7proto)
 {
 	struct desync_profile_list *dpl;
-	VPRINT("desync profile search for hostname='%s' ipv6=%u tcp_port=%u\n", hostname ? hostname : "", ipv6, tcp_port);
+	if (params.debug)
+	{
+		char ip_port[48];
+		ntop46_port(dest, ip_port,sizeof(ip_port));
+		VPRINT("desync profile search for tcp target=%s l7proto=%s hostname='%s'\n", ip_port, l7proto_str(l7proto), hostname ? hostname : "");
+	}
 	LIST_FOREACH(dpl, head, next)
 	{
-		if (dp_match(&dpl->dp,ipv6,tcp_port,hostname))
+		if (dp_match(&dpl->dp,dest,hostname,l7proto))
 		{
 			VPRINT("desync profile %d matches\n",dpl->dp.n);
 			return &dpl->dp;
@@ -49,7 +82,7 @@ static struct desync_profile *dp_find(struct desync_profile_list_head *head, boo
 }
 void apply_desync_profile(t_ctrack *ctrack, const struct sockaddr *dest)
 {
-	ctrack->dp = dp_find(&params.desync_profiles, dest->sa_family==AF_INET6, saport(dest), ctrack->hostname);
+	ctrack->dp = dp_find(&params.desync_profiles, dest, ctrack->hostname, ctrack->l7proto);
 }
 
 
@@ -114,33 +147,49 @@ void tamper_out(t_ctrack *ctrack, const struct sockaddr *dest, uint8_t *segment,
 		l7proto = UNKNOWN;
 	}
 
-	if (ctrack->l7proto==UNKNOWN) ctrack->l7proto=l7proto;
-
 	if (bHaveHost)
-	{
 		VPRINT("request hostname: %s\n", Host);
-		if (!ctrack->hostname)
-		{
-			if (!(ctrack->hostname=strdup(Host)))
-			{
-				DLOG_ERR("strdup hostname : out of memory\n");
-				return;
-			}
+	if (ctrack->b_not_act)
+	{
+		VPRINT("Not acting on this request\n");
+		return;
+	}
 
-			struct desync_profile *dp_prev = ctrack->dp;
-			apply_desync_profile(ctrack, dest);
-			if (ctrack->dp!=dp_prev)
-				VPRINT("desync profile changed by revealed hostname !\n");
-			else if (*ctrack->dp->hostlist_auto_filename)
-			{
-				bool bHostExcluded;
-				if (!HostlistCheck(ctrack->dp, Host, &bHostExcluded))
-				{
-					ctrack->b_ah_check = !bHostExcluded;
-					VPRINT("Not acting on this request\n");
-					return;
-				}
-			}
+	bool bDiscoveredL7 = ctrack->l7proto==UNKNOWN && l7proto!=UNKNOWN;
+	if (bDiscoveredL7)
+	{
+		VPRINT("discovered l7 protocol\n");
+		ctrack->l7proto=l7proto;
+	}
+
+	bool bDiscoveredHostname = bHaveHost && !ctrack->hostname;
+	if (bDiscoveredHostname)
+	{
+		VPRINT("discovered hostname\n");
+		if (!(ctrack->hostname=strdup(Host)))
+		{
+			DLOG_ERR("strdup hostname : out of memory\n");
+			return;
+		}
+	}
+
+	if (bDiscoveredL7 || bDiscoveredHostname)
+	{
+		struct desync_profile *dp_prev = ctrack->dp;
+		apply_desync_profile(ctrack, dest);
+		if (ctrack->dp!=dp_prev)
+			VPRINT("desync profile changed by revealed l7 protocol or hostname !\n");
+	}
+
+	if (bDiscoveredHostname && *ctrack->dp->hostlist_auto_filename)
+	{
+		bool bHostExcluded;
+		if (!HostlistCheck(ctrack->dp, Host, &bHostExcluded))
+		{
+			ctrack->b_ah_check = !bHostExcluded;
+			VPRINT("Not acting on this request\n");
+			ctrack->b_not_act = true;
+			return;
 		}
 	}
 

tpws/tamper.h

@@ -10,6 +10,7 @@
 #define SPLIT_FLAG_OOB		0x02
 
 typedef enum {UNKNOWN=0, HTTP, TLS} t_l7proto;
+const char *l7proto_str(t_l7proto l7);
 typedef struct
 {
 	// common state
@@ -17,6 +18,7 @@ typedef struct
 	bool bFirstReplyChecked;
 	bool bTamperInCutoff;
 	bool b_ah_check;
+	bool b_not_act;
 	char *hostname;
 	struct desync_profile *dp;		// desync profile cache
 } t_ctrack;

tpws/tpws.c

@@ -33,6 +33,7 @@
 
 #include "tpws_conn.h"
 #include "hostlist.h"
+#include "ipset.h"
 #include "params.h"
 #include "sec.h"
 #include "redirect.h"
@@ -46,7 +47,7 @@ bool bHup = false;
 static void onhup(int sig)
 {
 	printf("HUP received !\n");
-	printf("Will reload hostlist on next request (if any)\n");
+	printf("Will reload hostlists and ipsets on next request (if any)\n");
 	bHup = true;
 }
 // should be called in normal execution
@@ -54,9 +55,9 @@ void dohup(void)
 {
 	if (bHup)
 	{
-		if (!LoadIncludeHostLists() || !LoadExcludeHostLists())
+		if (!LoadIncludeHostLists() || !LoadExcludeHostLists() || !LoadIncludeIpsets() || !LoadExcludeIpsets())
 		{
-			// what will we do without hostlist ?? sure, gonna die
+			// what will we do without hostlist or ipset ?? sure, gonna die
 			exit(1);
 		}
 		bHup = false;
@@ -180,6 +181,9 @@ static void exithelp(void)
 		" --new\t\t\t\t\t; begin new strategy\n"
 		" --filter-l3=ipv4|ipv6\t\t\t; L3 protocol filter. multiple comma separated values allowed.\n"
 		" --filter-tcp=[~]port1[-port2]\t\t; TCP port filter. ~ means negation\n"
+		" --filter-l7=[http|tls|unknown]\t\t; L6-L7 protocol filter. multiple comma separated values allowed.\n"
+		" --ipset=<filename>\t\t\t; ipset include filter (one ip/CIDR per line, ipv4 and ipv6 accepted, gzip supported, multiple ipsets allowed)\n"
+		" --ipset-exclude=<filename>\t\t; ipset exclude filter (one ip/CIDR per line, ipv4 and ipv6 accepted, gzip supported, multiple ipsets allowed)\n"
 		"\nHOSTLIST FILTER:\n"
 		" --hostlist=<filename>\t\t\t; only act on hosts in the list (one host per line, subdomains auto apply, gzip supported, multiple hostlists allowed)\n"
 		" --hostlist-exclude=<filename>\t\t; do not act on hosts in the list (one host per line, subdomains auto apply, gzip supported, multiple hostlists allowed)\n"
@@ -317,6 +321,35 @@ static bool wf_make_l3(char *opt, bool *ipv4, bool *ipv6)
 	return true;
 }
 
+static bool parse_l7_list(char *opt, uint32_t *l7)
+{
+	char *e,*p,c;
+
+	for (p=opt,*l7=0 ; p ; )
+	{
+		if ((e = strchr(p,',')))
+		{
+			c=*e;
+			*e=0;
+		}
+
+		if (!strcmp(p,"http"))
+			*l7 |= L7_PROTO_HTTP;
+		else if (!strcmp(p,"tls"))
+			*l7 |= L7_PROTO_TLS;
+		else if (!strcmp(p,"unknown"))
+			*l7 |= L7_PROTO_UNKNOWN;
+		else return false;
+
+		if (e)
+		{
+			*e++=c;
+		}
+		p = e;
+	}
+	return true;
+}
+
 void parse_params(int argc, char *argv[])
 {
 	int option_index = 0;
@@ -409,21 +442,24 @@ void parse_params(int argc, char *argv[])
 		{ "tamper-cutoff",required_argument,0,0 },// optidx=54
 		{ "connect-bind-addr",required_argument,0,0 },// optidx=55
 
-		{ "new",no_argument,0,0 },		// optidx=56
-		{ "filter-l3",required_argument,0,0 },	// optidx=57
-		{ "filter-tcp",required_argument,0,0 },	// optidx=58
+		{ "new",no_argument,0,0 },				// optidx=56
+		{ "filter-l3",required_argument,0,0 },			// optidx=57
+		{ "filter-tcp",required_argument,0,0 },			// optidx=58
+		{ "filter-l7",required_argument,0,0 },			// optidx=59
+		{ "ipset",required_argument,0,0 },			// optidx=60
+		{ "ipset-exclude",required_argument,0,0 },		// optidx=61
 
 #if defined(__FreeBSD__)
-		{ "enable-pf",no_argument,0,0 },// optidx=59
+		{ "enable-pf",no_argument,0,0 },// optidx=62
 #elif defined(__APPLE__)
-		{ "local-tcp-user-timeout",required_argument,0,0 },// optidx=59
-		{ "remote-tcp-user-timeout",required_argument,0,0 },// optidx=60
+		{ "local-tcp-user-timeout",required_argument,0,0 },	// optidx=62
+		{ "remote-tcp-user-timeout",required_argument,0,0 },	// optidx=63
 #elif defined(__linux__)
-		{ "local-tcp-user-timeout",required_argument,0,0 },// optidx=59
-		{ "remote-tcp-user-timeout",required_argument,0,0 },// optidx=60
-		{ "mss",required_argument,0,0 },// optidx=61
+		{ "local-tcp-user-timeout",required_argument,0,0 },	// optidx=62
+		{ "remote-tcp-user-timeout",required_argument,0,0 },	// optidx=63
+		{ "mss",required_argument,0,0 },			// optidx=64
 #ifdef SPLICE_PRESENT
-		{ "nosplice",no_argument,0,0 },// optidx=62
+		{ "nosplice",no_argument,0,0 },				// optidx=65
 #endif
 #endif
 		{ "hostlist-auto-retrans-threshold",optional_argument,0,0}, // ignored. for nfqws command line compatibility
@@ -935,13 +971,36 @@ void parse_params(int argc, char *argv[])
 				exit_clean(1);
 			}
 			break;
+		case 59: /* filter-l7 */
+			if (!parse_l7_list(optarg,&dp->filter_l7))
+			{
+				DLOG_ERR("Invalid l7 filter : %s\n",optarg);
+				exit_clean(1);
+			}
+			break;
+		case 60: /* ipset */
+			if (!strlist_add(&dp->ipset_files, optarg))
+			{
+				DLOG_ERR("strlist_add failed\n");
+				exit_clean(1);
+			}
+			params.tamper = true;
+			break;
+		case 61: /* ipset-exclude */
+			if (!strlist_add(&dp->ipset_exclude_files, optarg))
+			{
+				DLOG_ERR("strlist_add failed\n");
+				exit_clean(1);
+			}
+			params.tamper = true;
+			break;
 			
 #if defined(__FreeBSD__)
-		case 59: /* enable-pf */
+		case 62: /* enable-pf */
 			params.pf_enable = true;
 			break;
 #elif defined(__linux__) || defined(__APPLE__)
-		case 59: /* local-tcp-user-timeout */
+		case 62: /* local-tcp-user-timeout */
 			params.tcp_user_timeout_local = atoi(optarg);
 			if (params.tcp_user_timeout_local<0 || params.tcp_user_timeout_local>86400)
 			{
@@ -949,7 +1008,7 @@ void parse_params(int argc, char *argv[])
 				exit_clean(1);
 			}
 			break;
-		case 60: /* remote-tcp-user-timeout */
+		case 63: /* remote-tcp-user-timeout */
 			params.tcp_user_timeout_remote = atoi(optarg);
 			if (params.tcp_user_timeout_remote<0 || params.tcp_user_timeout_remote>86400)
 			{
@@ -960,7 +1019,7 @@ void parse_params(int argc, char *argv[])
 #endif
 
 #if defined(__linux__)
-		case 61: /* mss */
+		case 64: /* mss */
 			// this option does not work in any BSD and MacOS. OS may accept but it changes nothing
 			dp->mss = atoi(optarg);
 			if (dp->mss<88 || dp->mss>32767)
@@ -970,7 +1029,7 @@ void parse_params(int argc, char *argv[])
 			}
 			break;
 #ifdef SPLICE_PRESENT
-		case 62: /* nosplice */
+		case 65: /* nosplice */
 			params.nosplice = true;
 			break;
 #endif
@@ -1021,6 +1080,16 @@ void parse_params(int argc, char *argv[])
 		DLOG_ERR("Exclude hostlist load failed\n");
 		exit_clean(1);
 	}
+	if (!LoadIncludeIpsets())
+	{
+		DLOG_ERR("Include ipset load failed\n");
+		exit_clean(1);
+	}
+	if (!LoadExcludeIpsets())
+	{
+		DLOG_ERR("Exclude ipset load failed\n");
+		exit_clean(1);
+	}
 }
 
 
@@ -1087,7 +1156,7 @@ static bool read_system_maxfiles(rlim_t *maxfile)
 		return false;
 	n=fscanf(F,"%ju",&um);
 	fclose(F);
-	if (n != 1)	return false;
+	if (n != 1) return false;
 	*maxfile = (rlim_t)um;
 	return true;
 #elif defined(BSD)
@@ -1177,6 +1246,8 @@ int main(int argc, char *argv[])
 	char ip_port[48];
 
 	srand(time(NULL));
+	mask_from_preflen6_prepare();
+
 	parse_params(argc, argv);
 
 	if (params.daemon) daemonize();