nfqws,tpws: multiple hostlist support, exclude hostlist support

tpws/hostlist.c

@@ -3,6 +3,7 @@
 #include "gzip.h"
 #include "params.h"
 
+
 static bool addpool(strpool **hostlist, char **s, const char *end)
 {
 	char *p;
@@ -22,20 +23,16 @@ static bool addpool(strpool **hostlist, char **s, const char *end)
 }
 
 
-bool LoadHostList(strpool **hostlist, char *filename)
+bool AppendHostList(strpool **hostlist, char *filename)
 {
 	char *p, *e, s[256], *zbuf;
 	size_t zsize;
 	int ct = 0;
 	FILE *F;
 	int r;
-	
-	if (*hostlist)
-	{
-		StrPoolDestroy(hostlist);
-		*hostlist = NULL;
-	}
-	
+
+	printf("Loading hostlist %s\n",filename);
+
 	if (!(F = fopen(filename, "rb")))
 	{
 		fprintf(stderr, "Could not open %s\n", filename);
@@ -92,8 +89,25 @@ bool LoadHostList(strpool **hostlist, char *filename)
 	return true;
 }
 
+bool LoadHostLists(strpool **hostlist, struct str_list_head *file_list)
+{
+	struct str_list *file;
 
-bool SearchHostList(strpool *hostlist, const char *host, bool debug)
+	if (*hostlist)
+	{
+		StrPoolDestroy(hostlist);
+		*hostlist = NULL;
+	}
+
+	LIST_FOREACH(file, file_list, next)
+	{
+		if (!AppendHostList(hostlist, file->str)) return false;
+	}
+	return true;
+}
+
+
+bool SearchHostList(strpool *hostlist, const char *host)
 {
 	if (hostlist)
 	{
@@ -102,7 +116,7 @@ bool SearchHostList(strpool *hostlist, const char *host, bool debug)
 		while (p)
 		{
 			bInHostList = StrPoolCheckStr(hostlist, p);
-			if (debug) VPRINT("Hostlist check for %s : %s", p, bInHostList ? "positive" : "negative")
+			if (params.debug) printf("Hostlist check for %s : %s\n", p, bInHostList ? "positive" : "negative");
 			if (bInHostList) return true;
 			p = strchr(p, '.');
 			if (p) p++;
@@ -110,3 +124,19 @@ bool SearchHostList(strpool *hostlist, const char *host, bool debug)
 	}
 	return false;
 }
+
+// return : true = apply fooling, false = do not apply
+bool HostlistCheck(strpool *hostlist, strpool *hostlist_exclude, const char *host)
+{
+	if (hostlist_exclude)
+	{
+		if (params.debug) printf("Checking exclude hostlist\n");
+		if (SearchHostList(hostlist_exclude, host)) return false;
+	}
+	if (hostlist)
+	{
+		if (params.debug) printf("Checking include hostlist\n");
+		return SearchHostList(hostlist, host);
+	}
+	return true;
+}

tpws/hostlist.h

@@ -3,5 +3,8 @@
 #include <stdbool.h>
 #include "strpool.h"
 
-bool LoadHostList(strpool **hostlist, char *filename);
-bool SearchHostList(strpool *hostlist, const char *host, bool debug);
+bool AppendHostList(strpool **hostlist, char *filename);
+bool LoadHostLists(strpool **hostlist, struct str_list_head *file_list);
+bool SearchHostList(strpool *hostlist, const char *host);
+// return : true = apply fooling, false = do not apply
+bool HostlistCheck(strpool *hostlist, strpool *hostlist_exclude, const char *host);

tpws/params.h

@@ -4,6 +4,7 @@
 #include <stdbool.h>
 #include <stdint.h>
 #include <sys/param.h>
+#include <sys/queue.h>
 #include "strpool.h"
 
 enum splithttpreq { split_none = 0, split_method, split_host };
@@ -42,9 +43,11 @@ struct params_s
 	enum splithttpreq split_http_req;
 	bool split_any_protocol;
 	int split_pos;
-	char hostfile[256];
+
 	char pidfile[256];
-	strpool *hostlist;
+
+	strpool *hostlist, *hostlist_exclude;
+	struct str_list_head hostlist_files, hostlist_exclude_files;
 
 	int debug;
 

tpws/strpool.c

@@ -6,71 +6,102 @@
 #undef uthash_nonfatal_oom
 #define uthash_nonfatal_oom(elt) ut_oom_recover(elt)
 
-static bool oom=false;
+static bool oom = false;
 static void ut_oom_recover(strpool *elem)
 {
- oom=true;
+	oom = true;
 }
 
 // for zero terminated strings
-bool StrPoolAddStr(strpool **pp,const char *s)
+bool StrPoolAddStr(strpool **pp, const char *s)
 {
- strpool *elem;
- if (!(elem = (strpool*)malloc(sizeof(strpool))))
-  return false;
- if (!(elem->str = strdup(s)))
- {
-  free(elem);
-  return false;
- }
- oom = false;
- HASH_ADD_KEYPTR( hh, *pp, elem->str, strlen(elem->str), elem );
- if (oom)
- {
-  free(elem->str);
-  free(elem);
-  return false;
- }
- return true;
+	strpool *elem;
+	if (!(elem = (strpool*)malloc(sizeof(strpool))))
+		return false;
+	if (!(elem->str = strdup(s)))
+	{
+		free(elem);
+		return false;
+	}
+	oom = false;
+	HASH_ADD_KEYPTR(hh, *pp, elem->str, strlen(elem->str), elem);
+	if (oom)
+	{
+		free(elem->str);
+		free(elem);
+		return false;
+	}
+	return true;
 }
 // for not zero terminated strings
-bool StrPoolAddStrLen(strpool **pp,const char *s,size_t slen)
+bool StrPoolAddStrLen(strpool **pp, const char *s, size_t slen)
 {
- strpool *elem;
- if (!(elem = (strpool*)malloc(sizeof(strpool))))
-  return false;
- if (!(elem->str = malloc(slen+1)))
- {
-  free(elem);
-  return false;
- }
- memcpy(elem->str,s,slen);
- elem->str[slen]=0;
- oom = false;
- HASH_ADD_KEYPTR( hh, *pp, elem->str, strlen(elem->str), elem );
- if (oom)
- {
-  free(elem->str);
-  free(elem);
-  return false;
- }
- return true;
+	strpool *elem;
+	if (!(elem = (strpool*)malloc(sizeof(strpool))))
+		return false;
+	if (!(elem->str = malloc(slen + 1)))
+	{
+		free(elem);
+		return false;
+	}
+	memcpy(elem->str, s, slen);
+	elem->str[slen] = 0;
+	oom = false;
+	HASH_ADD_KEYPTR(hh, *pp, elem->str, strlen(elem->str), elem);
+	if (oom)
+	{
+		free(elem->str);
+		free(elem);
+		return false;
+	}
+	return true;
 }
 
-bool StrPoolCheckStr(strpool *p,const char *s)
+bool StrPoolCheckStr(strpool *p, const char *s)
 {
- strpool *elem;
- HASH_FIND_STR( p, s, elem);
- return elem!=NULL;
+	strpool *elem;
+	HASH_FIND_STR(p, s, elem);
+	return elem != NULL;
 }
 
 void StrPoolDestroy(strpool **p)
 {
- strpool *elem,*tmp;
- HASH_ITER(hh, *p, elem, tmp) {
-   free(elem->str);
-   HASH_DEL(*p, elem);
-   free(elem);
- }
- *p = NULL;
+	strpool *elem, *tmp;
+	HASH_ITER(hh, *p, elem, tmp) {
+		free(elem->str);
+		HASH_DEL(*p, elem);
+		free(elem);
+	}
+	*p = NULL;
+}
+
+
+
+
+bool strlist_add(struct str_list_head *head, const char *filename)
+{
+	struct str_list *entry = malloc(sizeof(struct str_list));
+	if (!entry) return false;
+	entry->str = strdup(filename);
+	if (!entry->str)
+	{
+		free(entry);
+		return false;
+	}
+	LIST_INSERT_HEAD(head, entry, next);
+	return true;
+}
+static void strlist_entry_destroy(struct str_list *entry)
+{
+	if (entry->str)	free(entry->str);
+	free(entry);
+}
+void strlist_destroy(struct str_list_head *head)
+{
+	struct str_list *entry;
+	while (entry = LIST_FIRST(head))
+	{
+		LIST_REMOVE(entry, next);
+		strlist_entry_destroy(entry);
+	}
 }

tpws/strpool.h

@@ -2,6 +2,7 @@
 
 #include <stdbool.h>
 #include <ctype.h>
+#include <sys/queue.h>
 
 //#define HASH_BLOOM 20
 #define HASH_NONFATAL_OOM 1
@@ -17,3 +18,12 @@ void StrPoolDestroy(strpool **p);
 bool StrPoolAddStr(strpool **pp,const char *s);
 bool StrPoolAddStrLen(strpool **pp,const char *s,size_t slen);
 bool StrPoolCheckStr(strpool *p,const char *s);
+
+struct str_list {
+	char *str;
+	LIST_ENTRY(str_list) next;
+};
+LIST_HEAD(str_list_head, str_list);
+
+bool strlist_add(struct str_list_head *head, const char *filename);
+void strlist_destroy(struct str_list_head *head);

tpws/tamper.c

@@ -48,7 +48,7 @@ void modify_tcp_segment(char *segment,size_t segment_buffer_size,size_t *size,si
 	{
 		VPRINT("Data block looks like http request start : %s", *method)
 		// cpu saving : we search host only if and when required. we do not research host every time we need its position
-		if (params.hostlist && find_host(&pHost,segment,*size))
+		if ((params.hostlist || params.hostlist_exclude) && find_host(&pHost,segment,*size))
 		{
 			p = pHost + 5;
 			while (p < (segment + *size) && (*p == ' ' || *p == '\t')) p++;
@@ -58,7 +58,7 @@ void modify_tcp_segment(char *segment,size_t segment_buffer_size,size_t *size,si
 			Host[pp - p] = '\0';
 			VPRINT("Requested Host is : %s", Host)
 			for(p = Host; *p; p++) *p=tolower(*p);
-			bBypass = !SearchHostList(params.hostlist,Host,!!params.debug);
+			bBypass = !HostlistCheck(params.hostlist, params.hostlist_exclude, Host);
 		}
 		if (!bBypass)
 		{
@@ -218,10 +218,10 @@ void modify_tcp_segment(char *segment,size_t segment_buffer_size,size_t *size,si
 
 			VPRINT("packet contains TLS ClientHello")
 			// we need host only if hostlist is present
-			if (params.hostlist && TLSHelloExtractHost((uint8_t*)segment,*size,host,sizeof(host)))
+			if ((params.hostlist || params.hostlist_exclude) && TLSHelloExtractHost((uint8_t*)segment,*size,host,sizeof(host)))
 			{
 				VPRINT("hostname: %s",host)
-				if (!SearchHostList(params.hostlist,host,!!params.debug))
+				if (!HostlistCheck(params.hostlist, params.hostlist_exclude, host))
 				{
 					VPRINT("Not acting on this request")
 					return;

tpws/tpws.c

@@ -43,8 +43,8 @@ bool bHup = false;
 static void onhup(int sig)
 {
 	printf("HUP received !\n");
-	if (params.hostlist)
-		printf("Will reload hostlist on next request\n");
+	if (params.hostlist || params.hostlist_exclude)
+		printf("Will reload hostlists on next request\n");
 	bHup = true;
 }
 // should be called in normal execution
@@ -52,13 +52,11 @@ void dohup()
 {
 	if (bHup)
 	{
-		if (params.hostlist)
+		if (!LoadHostLists(&params.hostlist, &params.hostlist_files) ||
+			!LoadHostLists(&params.hostlist_exclude, &params.hostlist_exclude_files))
 		{
-			if (!LoadHostList(&params.hostlist, params.hostfile))
-			{
-				// what will we do without hostlist ?? sure, gonna die
-				exit(1);
-			}
+			// what will we do without hostlist ?? sure, gonna die
+			exit(1);
 		}
 		bHup = false;
 	}
@@ -144,8 +142,9 @@ static void exithelp()
 #endif
 		" --debug=0|1|2\t\t\t; 0(default)=silent 1=verbose 2=debug\n"
 		"\nTAMPERING:\n"
-		" --hostlist=<filename>\t\t; only act on host in the list (one host per line, subdomains auto apply)\n"
-		" --split-http-req=method|host\n"
+		" --hostlist=<filename>\t\t; only act on hosts in the list (one host per line, subdomains auto apply, gzip supported, multiple hostlists allowed)\n"
+		" --hostlist-exclude=<filename>\t; do not act on hosts in the list (one host per line, subdomains auto apply, gzip supported, multiple hostlists allowed)\n"
+		" --split-http-req=method|host\t; split at specified logical part of plain http request\n"
 		" --split-pos=<numeric_offset>\t; split at specified pos. split-http-req takes precedence for http.\n"
 		" --split-any-protocol\t\t; split not only http and https\n"
 		" --hostcase\t\t\t; change Host: => host:\n"
@@ -163,6 +162,13 @@ static void exithelp()
 }
 static void cleanup_params()
 {
+	strlist_destroy(&params.hostlist_files);
+	strlist_destroy(&params.hostlist_exclude_files);
+	if (params.hostlist_exclude)
+	{
+		StrPoolDestroy(&params.hostlist_exclude);
+		params.hostlist_exclude = NULL;
+	}
 	if (params.hostlist)
 	{
 		StrPoolDestroy(&params.hostlist);
@@ -208,6 +214,9 @@ void parse_params(int argc, char *argv[])
 	params.maxconn = DEFAULT_MAX_CONN;
 	params.max_orphan_time = DEFAULT_MAX_ORPHAN_TIME;
 	params.binds_last = -1;
+	LIST_INIT(&params.hostlist_files);
+	LIST_INIT(&params.hostlist_exclude_files);
+
 #if defined(__OpenBSD__) || defined(__APPLE__)
 	params.pf_enable = true; // OpenBSD and MacOS have no other choice
 #endif
@@ -249,17 +258,18 @@ void parse_params(int argc, char *argv[])
 		{ "hosttab",no_argument,0,0 },// optidx=28
 		{ "unixeol",no_argument,0,0 },// optidx=29
 		{ "hostlist",required_argument,0,0 },// optidx=30
-		{ "pidfile",required_argument,0,0 },// optidx=31
-		{ "debug",optional_argument,0,0 },// optidx=32
-		{ "local-rcvbuf",required_argument,0,0 },// optidx=33
-		{ "local-sndbuf",required_argument,0,0 },// optidx=34
-		{ "remote-rcvbuf",required_argument,0,0 },// optidx=35
-		{ "remote-sndbuf",required_argument,0,0 },// optidx=36
-		{ "socks",no_argument,0,0 },// optidx=37
-		{ "no-resolve",no_argument,0,0 },// optidx=38
-		{ "skip-nodelay",no_argument,0,0 },// optidx=39
+		{ "hostlist-exclude",required_argument,0,0 },// optidx=31
+		{ "pidfile",required_argument,0,0 },// optidx=32
+		{ "debug",optional_argument,0,0 },// optidx=33
+		{ "local-rcvbuf",required_argument,0,0 },// optidx=34
+		{ "local-sndbuf",required_argument,0,0 },// optidx=35
+		{ "remote-rcvbuf",required_argument,0,0 },// optidx=36
+		{ "remote-sndbuf",required_argument,0,0 },// optidx=37
+		{ "socks",no_argument,0,0 },// optidx=38
+		{ "no-resolve",no_argument,0,0 },// optidx=39
+		{ "skip-nodelay",no_argument,0,0 },// optidx=40
 #if defined(BSD) && !defined(__OpenBSD__) && !defined(__APPLE__)
-		{ "enable-pf",no_argument,0,0 },// optidx=40
+		{ "enable-pf",no_argument,0,0 },// optidx=41
 #endif
 		{ NULL,0,NULL,0 }
 	};
@@ -460,42 +470,51 @@ void parse_params(int argc, char *argv[])
 			params.tamper = true;
 			break;
 		case 30: /* hostlist */
-			if (!LoadHostList(&params.hostlist, optarg))
+			if (!strlist_add(&params.hostlist_files, optarg))
+			{
+				fprintf(stderr, "strlist_add failed\n");
 				exit_clean(1);
-			strncpy(params.hostfile,optarg,sizeof(params.hostfile));
-			params.hostfile[sizeof(params.hostfile)-1]='\0';
+			}
 			params.tamper = true;
 			break;
-		case 31: /* pidfile */
+		case 31: /* hostlist-exclude */
+			if (!strlist_add(&params.hostlist_exclude_files, optarg))
+			{
+				fprintf(stderr, "strlist_add failed\n");
+				exit_clean(1);
+			}
+			params.tamper = true;
+			break;
+		case 32: /* pidfile */
 			strncpy(params.pidfile,optarg,sizeof(params.pidfile));
 			params.pidfile[sizeof(params.pidfile)-1]='\0';
 			break;
-		case 32:
+		case 33:
 			params.debug = optarg ? atoi(optarg) : 1;
 			break;
-		case 33: /* local-rcvbuf */
+		case 34: /* local-rcvbuf */
 			params.local_rcvbuf = atoi(optarg)/2;
 			break;
-		case 34: /* local-sndbuf */
+		case 35: /* local-sndbuf */
 			params.local_sndbuf = atoi(optarg)/2;
 			break;
-		case 35: /* remote-rcvbuf */
+		case 36: /* remote-rcvbuf */
 			params.remote_rcvbuf = atoi(optarg)/2;
 			break;
-		case 36: /* remote-sndbuf */
+		case 37: /* remote-sndbuf */
 			params.remote_sndbuf = atoi(optarg)/2;
 			break;
-		case 37: /* socks */
+		case 38: /* socks */
 			params.proxy_type = CONN_TYPE_SOCKS;
 			break;
-		case 38: /* no-resolve */
+		case 39: /* no-resolve */
 			params.no_resolve = true;
 			break;
-		case 39: /* skip-nodelay */
+		case 40: /* skip-nodelay */
 			params.skip_nodelay = true;
 			break;
 #if defined(BSD) && !defined(__OpenBSD__) && !defined(__APPLE__)
-		case 40: /* enable-pf */
+		case 41: /* enable-pf */
 			params.pf_enable = true;
 			break;
 #endif
@@ -515,6 +534,17 @@ void parse_params(int argc, char *argv[])
 		fprintf(stderr, "Cannot split with --skip-nodelay\n");
 		exit_clean(1);
 	}
+
+	if (!LoadHostLists(&params.hostlist, &params.hostlist_files))
+	{
+		fprintf(stderr, "Include hostlist load failed\n");
+		exit_clean(1);
+	}
+	if (!LoadHostLists(&params.hostlist_exclude, &params.hostlist_exclude_files))
+	{
+		fprintf(stderr, "Exclude hostlist load failed\n");
+		exit_clean(1);
+	}
 }