tpws: oob and disorder protocol filters

docs/changes.txt

@@ -262,3 +262,12 @@ tpws: out of band send when splitting (--oob)
 nfqws: autottl
 nfqws: datanoack fooling
 nftables: use POSTNAT path for tcp redirections to allow NAT-breaking strategies. use additional mark bit DESYNC_MARK_POSTNAT.
+
+v55
+
+tpws: incompatible oob parameter change. it doesn't take oob byte anymore. instead it takes optional protocol filter - http or tls.
+  the same is done with disorder. oob byte can be specified in parameter --oob-data.
+blockcheck: quick mode, strategy order optimizations, QUIC protocol support
+nfqws: syndata desync mode
+
+   á

docs/readme.eng.md

@@ -576,8 +576,9 @@ tpws is transparent proxy.
  --split-http-req=method|host	; split http request at specified logical position.
  --split-pos=<numeric_offset>   ; split at specified pos. split-http-req takes precedence over split-pos for http reqs.
  --split-any-protocol		; split not only http and https
- --disorder                     ; when splitting simulate sending second fragment first
+ --disorder[=http|tls]          ; when splitting simulate sending second fragment first
- --oob[=<char>|0xHEX]		; when splitting send out of band byte. default is HEX 0x00.
+ --oob[=http|tls]               ; when splitting send out of band byte. default is HEX 0x00.
+ --oob-data=<char>|0xHEX        ; override default 0x00 OOB byte.
  --hostcase                     ; change Host: => host:
  --hostspell                    ; exact spelling of "Host" header. must be 4 chars. default is "host"
  --hostdot                      ; add "." after Host: name
@@ -796,10 +797,6 @@ may start to break the website. This situation can only be controlled manually.
 Remove undesired domain from the autohostlist file, restart nfqws/tpws or send them SIGHUP.
 Use exclude hostlist to prevent further auto additions.
 
-It's possible to use one auto hostlist with multiple processes. All processes check for file modification time.
-If a process modified autohostlist, all others will reread it automatically.
-All processes must run with the same uid.
-
 If zapret scripts are used then autohostlist is `ipset/zapret-hosts-auto.txt`
 and exlude list is `ipset/zapret-hosts-user-exclude.txt`. autohostlist mode
 includes hostlist mode. You can use `ipset/zapret-hosts-user.txt`.

docs/readme.txt

@@ -1,4 +1,4 @@
-zapret v.54
+zapret v.55
 
 English
 -------
@@ -657,6 +657,9 @@ tpws - это transparent proxy.
  --split-http-req=method|host	; способ разделения http запросов на сегменты : около метода (GET,POST) или около заголовка Host
  --split-pos=<offset>	; делить все посылы на сегменты в указанной позиции. единственная опция, работающая на не-http. при указании split-http-req он имеет преимущество на http.
  --split-any-protocol	; применять split-pos к любым пакетам. по умолчанию - только к http и TLS ClientHello
+ --disorder[=http|tls]  ; путем манипуляций с сокетом вынуждает отправлять первым второй сегмент разделенного запроса
+ --oob[=http|tls]       ; отправить байт out-of-band data (OOB) в конце первой части сплита
+ --oob-data=<char>|0xHEX ; переопределить байт OOB. по умолчанию 0x00.
  --disorder		; путем манипуляций с сокетом вынуждает отправлять первым второй сегмент разделенного запроса
  --oob[=<char>|0xHEX]	; отправить байт out-of-band data (OOB) в конце первой части сплита
  --hostcase             ; менять регистр заголовка "Host:". по умолчанию на "host:".

tpws/params.h

@@ -50,7 +50,8 @@ struct params_s
 	int tlsrec_pos;
 	bool split_any_protocol;
 	int split_pos;
-	bool disorder, oob;
+	bool disorder, disorder_http, disorder_tls;
+	bool oob, oob_http, oob_tls;
 	uint8_t oob_byte;
 	int ttl_default;
 

tpws/tamper.c

@@ -25,7 +25,7 @@ bool find_host(uint8_t **pHost,uint8_t *buf,size_t bs)
 
 static const char *http_methods[] = { "GET /","POST /","HEAD /","OPTIONS /","PUT /","DELETE /","CONNECT /","TRACE /",NULL };
 // segment buffer has at least 5 extra bytes to extend data block
-void tamper_out(t_ctrack *ctrack, uint8_t *segment,size_t segment_buffer_size,size_t *size, size_t *split_pos)
+void tamper_out(t_ctrack *ctrack, uint8_t *segment,size_t segment_buffer_size,size_t *size, size_t *split_pos, uint8_t *split_flags)
 {
 	uint8_t *p, *pp, *pHost = NULL;
 	size_t method_len = 0, pos;
@@ -37,6 +37,7 @@ void tamper_out(t_ctrack *ctrack, uint8_t *segment,size_t segment_buffer_size,si
 	DBGPRINT("tamper_out")
 
 	*split_pos=0;
+	*split_flags=0;
 
 	for (method = http_methods; *method; method++)
 	{
@@ -209,6 +210,8 @@ void tamper_out(t_ctrack *ctrack, uint8_t *segment,size_t segment_buffer_size,si
 				default:
 					if (params.split_pos < *size) *split_pos = params.split_pos;
 			}
+			if (params.disorder_http) *split_flags |= SPLIT_FLAG_DISORDER;
+			if (params.oob_http) *split_flags |= SPLIT_FLAG_OOB;
 		}
 		else
 		{
@@ -269,6 +272,9 @@ void tamper_out(t_ctrack *ctrack, uint8_t *segment,size_t segment_buffer_size,si
 		
 			if (params.split_pos < *size)
 				*split_pos = params.split_pos;
+
+			if (params.disorder_tls) *split_flags |= SPLIT_FLAG_DISORDER;
+			if (params.oob_tls) *split_flags |= SPLIT_FLAG_OOB;
 		}
 	}
 	else if (params.split_any_protocol && params.split_pos < *size)
@@ -279,7 +285,8 @@ void tamper_out(t_ctrack *ctrack, uint8_t *segment,size_t segment_buffer_size,si
 		DBGPRINT("tamper_out put hostname : %s", Host)
 		ctrack->hostname=strdup(Host);
 	}
-
+	if (params.disorder) *split_flags |= SPLIT_FLAG_DISORDER;
+	if (params.oob) *split_flags |= SPLIT_FLAG_OOB;
 }
 
 

tpws/tamper.h

@@ -4,6 +4,9 @@
 #include <stdint.h>
 #include <sys/types.h>
 
+#define SPLIT_FLAG_DISORDER	0x01
+#define SPLIT_FLAG_OOB		0x02
+
 typedef enum {UNKNOWN=0, HTTP, TLS} t_l7proto;
 typedef struct
 {
@@ -14,7 +17,7 @@ typedef struct
 	char *hostname;
 } t_ctrack;
 
-void tamper_out(t_ctrack *ctrack, uint8_t *segment,size_t segment_buffer_size,size_t *size, size_t *split_pos);
+void tamper_out(t_ctrack *ctrack, uint8_t *segment,size_t segment_buffer_size,size_t *size, size_t *split_pos, uint8_t *split_flags);
 void tamper_in(t_ctrack *ctrack, uint8_t *segment,size_t segment_buffer_size,size_t *size);
 // connection reset by remote leg
 void rst_in(t_ctrack *ctrack);

tpws/tpws.c

@@ -171,11 +171,12 @@ static void exithelp(void)
 		" --split-pos=<numeric_offset>\t\t; split at specified pos. split-http-req takes precedence for http.\n"
 		" --split-any-protocol\t\t\t; split not only http and https\n"
 #if defined(BSD) && !defined(__APPLE__)
-		" --disorder\t\t\t\t; when splitting simulate sending second fragment first (BSD sends entire message instead of first fragment, this is not good)\n"
+		" --disorder[=http|tls]\t\t\t; when splitting simulate sending second fragment first (BSD sends entire message instead of first fragment, this is not good)\n"
 #else
-		" --disorder\t\t\t\t; when splitting simulate sending second fragment first\n"
+		" --disorder[=http|tls]\t\t\t; when splitting simulate sending second fragment first\n"
 #endif
-		" --oob[=<char>|0xHEX]\t\t\t; when splitting send out of band byte. default is HEX 0x00.\n"
+		" --oob[=http|tls]\t\t\t; when splitting send out of band byte. default is HEX 0x00.\n"
+		" --oob-data=<char>|0xHEX\t\t; override default 0x00 OOB byte.\n"
 		" --hostcase\t\t\t\t; change Host: => host:\n"
 		" --hostspell\t\t\t\t; exact spelling of \"Host\" header. must be 4 chars. default is \"host\"\n"
 		" --hostdot\t\t\t\t; add \".\" after Host: name\n"
@@ -295,33 +296,34 @@ void parse_params(int argc, char *argv[])
 		{ "split-http-req",required_argument,0,0 },// optidx=23
 		{ "split-pos",required_argument,0,0 },// optidx=24
 		{ "split-any-protocol",optional_argument,0,0},// optidx=25
-		{ "disorder",no_argument,0,0 },// optidx=26
+		{ "disorder",optional_argument,0,0 },// optidx=26
 		{ "oob",optional_argument,0,0 },// optidx=27
-		{ "methodspace",no_argument,0,0 },// optidx=28
+		{ "oob-data",required_argument,0,0 },// optidx=28
-		{ "methodeol",no_argument,0,0 },// optidx=29
+		{ "methodspace",no_argument,0,0 },// optidx=29
-		{ "hosttab",no_argument,0,0 },// optidx=30
+		{ "methodeol",no_argument,0,0 },// optidx=30
-		{ "unixeol",no_argument,0,0 },// optidx=31
+		{ "hosttab",no_argument,0,0 },// optidx=31
-		{ "tlsrec",required_argument,0,0 },// optidx=32
+		{ "unixeol",no_argument,0,0 },// optidx=32
-		{ "tlsrec-pos",required_argument,0,0 },// optidx=33
+		{ "tlsrec",required_argument,0,0 },// optidx=33
-		{ "hostlist",required_argument,0,0 },// optidx=34
+		{ "tlsrec-pos",required_argument,0,0 },// optidx=34
-		{ "hostlist-exclude",required_argument,0,0 },// optidx=35
+		{ "hostlist",required_argument,0,0 },// optidx=35
-		{ "hostlist-auto",required_argument,0,0}, // optidx=36
+		{ "hostlist-exclude",required_argument,0,0 },// optidx=36
-		{ "hostlist-auto-fail-threshold",required_argument,0,0}, // optidx=37
+		{ "hostlist-auto",required_argument,0,0}, // optidx=37
-		{ "hostlist-auto-fail-time",required_argument,0,0},	// optidx=38
+		{ "hostlist-auto-fail-threshold",required_argument,0,0}, // optidx=38
-		{ "hostlist-auto-debug",required_argument,0,0}, // optidx=39
+		{ "hostlist-auto-fail-time",required_argument,0,0},	// optidx=39
-		{ "pidfile",required_argument,0,0 },// optidx=40
+		{ "hostlist-auto-debug",required_argument,0,0}, // optidx=40
-		{ "debug",optional_argument,0,0 },// optidx=41
+		{ "pidfile",required_argument,0,0 },// optidx=41
-		{ "local-rcvbuf",required_argument,0,0 },// optidx=42
+		{ "debug",optional_argument,0,0 },// optidx=42
-		{ "local-sndbuf",required_argument,0,0 },// optidx=43
+		{ "local-rcvbuf",required_argument,0,0 },// optidx=43
-		{ "remote-rcvbuf",required_argument,0,0 },// optidx=44
+		{ "local-sndbuf",required_argument,0,0 },// optidx=44
-		{ "remote-sndbuf",required_argument,0,0 },// optidx=45
+		{ "remote-rcvbuf",required_argument,0,0 },// optidx=45
-		{ "socks",no_argument,0,0 },// optidx=46
+		{ "remote-sndbuf",required_argument,0,0 },// optidx=46
-		{ "no-resolve",no_argument,0,0 },// optidx=47
+		{ "socks",no_argument,0,0 },// optidx=47
-		{ "skip-nodelay",no_argument,0,0 },// optidx=48
+		{ "no-resolve",no_argument,0,0 },// optidx=48
-		{ "tamper-start",required_argument,0,0 },// optidx=49
+		{ "skip-nodelay",no_argument,0,0 },// optidx=49
-		{ "tamper-cutoff",required_argument,0,0 },// optidx=50
+		{ "tamper-start",required_argument,0,0 },// optidx=50
+		{ "tamper-cutoff",required_argument,0,0 },// optidx=51
 #if defined(BSD) && !defined(__OpenBSD__) && !defined(__APPLE__)
-		{ "enable-pf",no_argument,0,0 },// optidx=51
+		{ "enable-pf",no_argument,0,0 },// optidx=52
 #endif
 		{ "hostlist-auto-retrans-threshold",optional_argument,0,0}, // ignored. for nfqws command line compatibility
 		{ NULL,0,NULL,0 }
@@ -507,41 +509,64 @@ void parse_params(int argc, char *argv[])
 			params.split_any_protocol = true;
 			break;
 		case 26: /* disorder */
-			params.disorder = true;
+			if (optarg)
+			{
+				if (!strcmp(optarg,"http")) params.disorder_http=true;
+				else if (!strcmp(optarg,"tls")) params.disorder_tls=true;
+				else
+				{
+					fprintf(stderr, "Invalid argument for disorder\n");
+					exit_clean(1);
+				}
+			}
+			else
+				params.disorder = true;
 			save_default_ttl();
 			break;
 		case 27: /* oob */
 			if (optarg)
+			{
+				if (!strcmp(optarg,"http")) params.oob_http=true;
+				else if (!strcmp(optarg,"tls")) params.oob_tls=true;
+				else
+				{
+					fprintf(stderr, "Invalid argument for oob\n");
+					exit_clean(1);
+				}
+			}
+			else
+				params.oob = true;
+			break;
+		case 28: /* oob-data */
 			{
 				size_t l = strlen(optarg);
 				unsigned int bt;
 				if (l==1) params.oob_byte = (uint8_t)*optarg;
 				else if (l!=4 || sscanf(optarg,"0x%02X",&bt)!=1)
 				{
-					fprintf(stderr, "Invalid argument for oob\n");
+					fprintf(stderr, "Invalid argument for oob-data\n");
 					exit_clean(1);
 				}
 				else params.oob_byte = (uint8_t)bt;
 			}
-			params.oob = true;
 			break;
-		case 28: /* methodspace */
+		case 29: /* methodspace */
 			params.methodspace = true;
 			params.tamper = true;
 			break;
-		case 29: /* methodeol */
+		case 30: /* methodeol */
 			params.methodeol = true;
 			params.tamper = true;
 			break;
-		case 30: /* hosttab */
+		case 31: /* hosttab */
 			params.hosttab = true;
 			params.tamper = true;
 			break;
-		case 31: /* unixeol */
+		case 32: /* unixeol */
 			params.unixeol = true;
 			params.tamper = true;
 			break;
-		case 32: /* tlsrec */
+		case 33: /* tlsrec */
 			if (!strcmp(optarg, "sni"))
 				params.tlsrec = tlsrec_sni;
 			else
@@ -551,7 +576,7 @@ void parse_params(int argc, char *argv[])
 			}
 			params.tamper = true;
 			break;
-		case 33: /* tlsrec-pos */
+		case 34: /* tlsrec-pos */
 			if ((params.tlsrec_pos = atoi(optarg))>0)
 				params.tlsrec = tlsrec_pos;
 			else
@@ -561,7 +586,7 @@ void parse_params(int argc, char *argv[])
 			}
 			params.tamper = true;
 			break;
-		case 34: /* hostlist */
+		case 35: /* hostlist */
 			if (!strlist_add(&params.hostlist_files, optarg))
 			{
 				fprintf(stderr, "strlist_add failed\n");
@@ -569,7 +594,7 @@ void parse_params(int argc, char *argv[])
 			}
 			params.tamper = true;
 			break;
-		case 35: /* hostlist-exclude */
+		case 36: /* hostlist-exclude */
 			if (!strlist_add(&params.hostlist_exclude_files, optarg))
 			{
 				fprintf(stderr, "strlist_add failed\n");
@@ -577,7 +602,7 @@ void parse_params(int argc, char *argv[])
 			}
 			params.tamper = true;
 			break;
-		case 36: /* hostlist-auto */
+		case 37: /* hostlist-auto */
 			if (*params.hostlist_auto_filename)
 			{
 				fprintf(stderr, "only one auto hostlist is supported\n");
@@ -609,7 +634,7 @@ void parse_params(int argc, char *argv[])
 			params.hostlist_auto_filename[sizeof(params.hostlist_auto_filename) - 1] = '\0';
 			params.tamper = true; // need to detect blocks and update autohostlist. cannot just slice.
 			break;
-		case 37: /* hostlist-auto-fail-threshold */
+		case 38: /* hostlist-auto-fail-threshold */
 			params.hostlist_auto_fail_threshold = (uint8_t)atoi(optarg);
 			if (params.hostlist_auto_fail_threshold<1 || params.hostlist_auto_fail_threshold>20)
 			{
@@ -617,7 +642,7 @@ void parse_params(int argc, char *argv[])
 				exit_clean(1);
 			}
 			break;
-		case 38: /* hostlist-auto-fail-time */
+		case 39: /* hostlist-auto-fail-time */
 			params.hostlist_auto_fail_time = (uint8_t)atoi(optarg);
 			if (params.hostlist_auto_fail_time<1)
 			{
@@ -625,7 +650,7 @@ void parse_params(int argc, char *argv[])
 				exit_clean(1);
 			}
 			break;
-		case 39: /* hostlist-auto-debug */
+		case 40: /* hostlist-auto-debug */
 			{
 				FILE *F = fopen(optarg,"a+t");
 				if (!F)
@@ -640,35 +665,35 @@ void parse_params(int argc, char *argv[])
 				params.hostlist_auto_debuglog[sizeof(params.hostlist_auto_debuglog) - 1] = '\0';
 			}
 			break;
-		case 40: /* pidfile */
+		case 41: /* pidfile */
 			strncpy(params.pidfile,optarg,sizeof(params.pidfile));
 			params.pidfile[sizeof(params.pidfile)-1]='\0';
 			break;
-		case 41:
+		case 42:
 			params.debug = optarg ? atoi(optarg) : 1;
 			break;
-		case 42: /* local-rcvbuf */
+		case 43: /* local-rcvbuf */
 			params.local_rcvbuf = atoi(optarg)/2;
 			break;
-		case 43: /* local-sndbuf */
+		case 44: /* local-sndbuf */
 			params.local_sndbuf = atoi(optarg)/2;
 			break;
-		case 44: /* remote-rcvbuf */
+		case 45: /* remote-rcvbuf */
 			params.remote_rcvbuf = atoi(optarg)/2;
 			break;
-		case 45: /* remote-sndbuf */
+		case 46: /* remote-sndbuf */
 			params.remote_sndbuf = atoi(optarg)/2;
 			break;
-		case 46: /* socks */
+		case 47: /* socks */
 			params.proxy_type = CONN_TYPE_SOCKS;
 			break;
-		case 47: /* no-resolve */
+		case 48: /* no-resolve */
 			params.no_resolve = true;
 			break;
-		case 48: /* skip-nodelay */
+		case 49: /* skip-nodelay */
 			params.skip_nodelay = true;
 			break;
-		case 49: /* tamper-start */
+		case 50: /* tamper-start */
 			{
 				const char *p=optarg;
 				if (*p=='n')
@@ -681,7 +706,7 @@ void parse_params(int argc, char *argv[])
 				params.tamper_start = atoi(p);
 			}
 			break;
-		case 50: /* tamper-cutoff */
+		case 51: /* tamper-cutoff */
 			{
 				const char *p=optarg;
 				if (*p=='n')
@@ -695,7 +720,7 @@ void parse_params(int argc, char *argv[])
 			}
 			break;
 #if defined(BSD) && !defined(__OpenBSD__) && !defined(__APPLE__)
-		case 51: /* enable-pf */
+		case 52: /* enable-pf */
 			params.pf_enable = true;
 			break;
 #endif

tpws/tpws_conn.c

@@ -101,6 +101,7 @@ ssize_t send_with_ttl(int fd, const void *buf, size_t len, int flags, int ttl)
 	{
 		DBGPRINT("send_with_ttl %d fd=%d",ttl,fd);
 		if (!set_ttl_hl(fd, ttl))
+			//fprintf(stderr,"could not set ttl %d to fd=%d\n",ttl,fd);
 			fprintf(stderr,"could not set ttl %d to fd=%d\n",ttl,fd);
 	}
 	wr = send(fd, buf, len, flags);
@@ -915,7 +916,7 @@ static bool in_tamper_out_range(tproxy_conn_t *conn)
 		(!params.tamper_cutoff || (params.tamper_cutoff_n ? (conn->tnrd+1) : conn->trd) < params.tamper_cutoff);
 }
 
-static void tamper(tproxy_conn_t *conn, uint8_t *segment, size_t segment_buffer_size, size_t *segment_size, size_t *split_pos)
+static void tamper(tproxy_conn_t *conn, uint8_t *segment, size_t segment_buffer_size, size_t *segment_size, size_t *split_pos, uint8_t *split_flags)
 {
 	*split_pos=0;
 	if (params.tamper)
@@ -935,7 +936,7 @@ static void tamper(tproxy_conn_t *conn, uint8_t *segment, size_t segment_buffer_
 				params.tamper_start_n ? "n" : "" , params.tamper_start,
 				params.tamper_cutoff_n ? "n" : "" , params.tamper_cutoff,
 				in_range ? "IN RANGE" : "OUT OF RANGE")
-			if (in_range) tamper_out(&conn->track,segment,segment_buffer_size,segment_size,split_pos);
+			if (in_range) tamper_out(&conn->track,segment,segment_buffer_size,segment_size,split_pos,split_flags);
 		}
 	}
 }
@@ -1029,28 +1030,30 @@ static bool handle_epoll(tproxy_conn_t *conn, struct tailhead *conn_list, uint32
 			if (rd>0)
 			{
 				size_t split_pos;
+				uint8_t split_flags;
 
 				bs = rd;
 
 				// tamper needs to know stream position of the block start
-				tamper(conn, buf, sizeof(buf), &bs, &split_pos);
+				tamper(conn, buf, sizeof(buf), &bs, &split_pos, &split_flags);
 				// increase after tamper
 				conn->tnrd++;
 				conn->trd+=rd;
 
 				if (split_pos)
 				{
-					VPRINT("Splitting at pos %zu", split_pos)
+					VPRINT("Splitting at pos %zu%s", split_pos, (split_flags & SPLIT_FLAG_DISORDER) ? " with disorder" : "")
-					if (params.oob)
+					if (split_flags && SPLIT_FLAG_OOB)
 					{
+						VPRINT("Sending OOB byte %02X", params.oob_byte)
 						uint8_t oob_save;
 						oob_save = buf[split_pos];
 						buf[split_pos] = params.oob_byte;
-						wr = send_or_buffer(conn->partner->wr_buf, conn->partner->fd, buf, split_pos+1, MSG_OOB, params.disorder ? 1 : 0);
+						wr = send_or_buffer(conn->partner->wr_buf, conn->partner->fd, buf, split_pos+1, MSG_OOB, !!(split_flags & SPLIT_FLAG_DISORDER));
 						buf[split_pos] = oob_save;
 					}
 					else
-						wr = send_or_buffer(conn->partner->wr_buf, conn->partner->fd, buf, split_pos, 0, params.disorder ? 1 : 0);
+						wr = send_or_buffer(conn->partner->wr_buf, conn->partner->fd, buf, split_pos, 0, !!(split_flags & SPLIT_FLAG_DISORDER));
 					DBGPRINT("send_or_buffer(1) fd=%d wr=%zd err=%d",conn->partner->fd,wr,errno)
 					if (wr >= 0)
 					{
@@ -1133,8 +1136,9 @@ static bool read_all_and_buffer(tproxy_conn_t *conn, int buffer_number)
 					conn->partner->bFlowOut = true;
 
 					size_t split_pos;
+					uint8_t split_flags;
 
-					tamper(conn, conn->partner->wr_buf[buffer_number].data, numbytes+5, &conn->partner->wr_buf[buffer_number].len, &split_pos);
+					tamper(conn, conn->partner->wr_buf[buffer_number].data, numbytes+5, &conn->partner->wr_buf[buffer_number].len, &split_pos, &split_flags);
 
 					if (epoll_update_flow(conn->partner))
 						return true;