diff --git a/binaries/aarch64/nfqws b/binaries/aarch64/nfqws index bc32395..259b2af 100755 Binary files a/binaries/aarch64/nfqws and b/binaries/aarch64/nfqws differ diff --git a/binaries/aarch64/tpws b/binaries/aarch64/tpws index c471d6e..c8151ad 100755 Binary files a/binaries/aarch64/tpws and b/binaries/aarch64/tpws differ diff --git a/binaries/arm/nfqws b/binaries/arm/nfqws index d09b961..b832c5e 100755 Binary files a/binaries/arm/nfqws and b/binaries/arm/nfqws differ diff --git a/binaries/arm/tpws b/binaries/arm/tpws index 1213012..0e1902d 100755 Binary files a/binaries/arm/tpws and b/binaries/arm/tpws differ diff --git a/binaries/freebsd-x64/dvtws b/binaries/freebsd-x64/dvtws index 3c10eab..ece170c 100755 Binary files a/binaries/freebsd-x64/dvtws and b/binaries/freebsd-x64/dvtws differ diff --git a/binaries/freebsd-x64/tpws b/binaries/freebsd-x64/tpws index d985e2f..8487acb 100755 Binary files a/binaries/freebsd-x64/tpws and b/binaries/freebsd-x64/tpws differ diff --git a/binaries/mac64/tpws b/binaries/mac64/tpws index 891bea5..f034c79 100755 Binary files a/binaries/mac64/tpws and b/binaries/mac64/tpws differ diff --git a/binaries/mips32r1-lsb/nfqws b/binaries/mips32r1-lsb/nfqws index 2433331..0f2f4d6 100755 Binary files a/binaries/mips32r1-lsb/nfqws and b/binaries/mips32r1-lsb/nfqws differ diff --git a/binaries/mips32r1-lsb/tpws b/binaries/mips32r1-lsb/tpws index f22a549..56cf476 100755 Binary files a/binaries/mips32r1-lsb/tpws and b/binaries/mips32r1-lsb/tpws differ diff --git a/binaries/mips32r1-msb/nfqws b/binaries/mips32r1-msb/nfqws index 02242e5..e217eb5 100755 Binary files a/binaries/mips32r1-msb/nfqws and b/binaries/mips32r1-msb/nfqws differ diff --git a/binaries/mips32r1-msb/tpws b/binaries/mips32r1-msb/tpws index 5c3f25a..58f4bfe 100755 Binary files a/binaries/mips32r1-msb/tpws and b/binaries/mips32r1-msb/tpws differ diff --git a/binaries/mips64r2-msb/nfqws b/binaries/mips64r2-msb/nfqws index 2715cff..91ff6e5 100755 Binary files a/binaries/mips64r2-msb/nfqws and b/binaries/mips64r2-msb/nfqws differ diff --git a/binaries/mips64r2-msb/tpws b/binaries/mips64r2-msb/tpws index aa0c9e3..b15615f 100755 Binary files a/binaries/mips64r2-msb/tpws and b/binaries/mips64r2-msb/tpws differ diff --git a/binaries/ppc/nfqws b/binaries/ppc/nfqws index 76245f7..304bf2e 100755 Binary files a/binaries/ppc/nfqws and b/binaries/ppc/nfqws differ diff --git a/binaries/ppc/tpws b/binaries/ppc/tpws index 26b1a42..8e92d86 100755 Binary files a/binaries/ppc/tpws and b/binaries/ppc/tpws differ diff --git a/binaries/x86/nfqws b/binaries/x86/nfqws index d4e7297..8f8a0f4 100755 Binary files a/binaries/x86/nfqws and b/binaries/x86/nfqws differ diff --git a/binaries/x86/tpws b/binaries/x86/tpws index 98b7fd8..186380d 100755 Binary files a/binaries/x86/tpws and b/binaries/x86/tpws differ diff --git a/binaries/x86_64/nfqws b/binaries/x86_64/nfqws index eb2694e..b6ce265 100755 Binary files a/binaries/x86_64/nfqws and b/binaries/x86_64/nfqws differ diff --git a/binaries/x86_64/tpws b/binaries/x86_64/tpws index 1dd0f94..8074ce1 100755 Binary files a/binaries/x86_64/tpws and b/binaries/x86_64/tpws differ diff --git a/binaries/x86_64/tpws_wsl.tgz b/binaries/x86_64/tpws_wsl.tgz index 5f8aa65..419741e 100644 Binary files a/binaries/x86_64/tpws_wsl.tgz and b/binaries/x86_64/tpws_wsl.tgz differ diff --git a/common/list.sh b/common/list.sh index d5628e4..cac5543 100644 --- a/common/list.sh +++ b/common/list.sh @@ -15,6 +15,7 @@ find_hostlists() [ -f "$HOSTLIST_EXCLUDE" ] || HOSTLIST_EXCLUDE= HOSTLIST_AUTO="$HOSTLIST_BASE/zapret-hosts-auto.txt" + HOSTLIST_AUTO_DEBUGLOG="$HOSTLIST_BASE/zapret-hosts-auto-debug.log" } filter_apply_autohostlist_target() @@ -23,9 +24,10 @@ filter_apply_autohostlist_target() local parm1="${AUTOHOSTLIST_FAIL_THRESHOLD:+--hostlist-auto-fail-threshold=$AUTOHOSTLIST_FAIL_THRESHOLD}" local parm2="${AUTOHOSTLIST_FAIL_TIME:+--hostlist-auto-fail-time=$AUTOHOSTLIST_FAIL_TIME}" - local parm3 + local parm3 parm4 [ "$MODE" = "tpws" -o "$MODE" = "tpws-socks" ] || parm3="${AUTOHOSTLIST_RETRANS_THRESHOLD:+--hostlist-auto-retrans-threshold=$AUTOHOSTLIST_RETRANS_THRESHOLD}" - eval $1="\"\$$1 --hostlist-auto=$HOSTLIST_AUTO $parm1 $parm2 $parm3\"" + [ "$AUTOHOSTLIST_DEBUGLOG" = 1 ] && parm4="--hostlist-auto-debug=$HOSTLIST_AUTO_DEBUGLOG" + eval $1="\"\$$1 --hostlist-auto=$HOSTLIST_AUTO $parm1 $parm2 $parm3 $parm4\"" } filter_apply_hostlist_target() diff --git a/config b/config index ff7e912..de22a03 100644 --- a/config +++ b/config @@ -23,6 +23,8 @@ IP2NET_OPT6="--prefix-length=56-64 --v6-threshold=5" AUTOHOSTLIST_RETRANS_THRESHOLD=3 AUTOHOSTLIST_FAIL_THRESHOLD=2 AUTOHOSTLIST_FAIL_TIME=60 +# 1 = debug autohostlist positives to ipset/zapret-hosts-auto-debug.log +AUTOHOSTLIST_DEBUGLOG=0 # number of parallel threads for domain list resolves MDIG_THREADS=30 diff --git a/docs/readme.eng.md b/docs/readme.eng.md index 763c51e..92110b8 100644 --- a/docs/readme.eng.md +++ b/docs/readme.eng.md @@ -184,6 +184,7 @@ nfqws takes the following parameters: --hostlist-auto-fail-threshold= ; how many failed attempts cause hostname to be added to auto hostlist (default : 2) --hostlist-auto-fail-time= ; all failed attemps must be within these seconds (default : 60) --hostlist-auto-retrans-threshold= ; how many request retransmissions cause attempt to fail (default : 3) + --hostlist-auto-debug= ; debug auto hostlist positives ``` The manipulation parameters can be combined in any way. @@ -539,6 +540,8 @@ tpws is transparent proxy. --hostlist-auto= ; detect DPI blocks and build hostlist automatically --hostlist-auto-fail-threshold= ; how many failed attempts cause hostname to be added to auto hostlist (default : 2) --hostlist-auto-fail-time= ; all failed attemps must be within these seconds (default : 60) + --hostlist-auto-debug= ; debug auto hostlist positives + --split-http-req=method|host ; split http request at specified logical position. --split-pos= ; split at specified pos. split-http-req takes precedence over split-pos for http reqs. --split-any-protocol ; split not only http and https @@ -911,6 +914,7 @@ autohostlist mode tuning. AUTOHOSTLIST_RETRANS_THRESHOLD=3 AUTOHOSTLIST_FAIL_THRESHOLD=2 AUTOHOSTLIST_FAIL_TIME=60 +AUTOHOSTLIST_DEBUG=0 ``` Enable gzip compression for large lists. Used by ipset/*.sh scripts. diff --git a/docs/readme.txt b/docs/readme.txt index cabe803..18f9d67 100644 --- a/docs/readme.txt +++ b/docs/readme.txt @@ -234,6 +234,7 @@ nfqws --hostlist-auto-fail-threshold= ; сколько раз нужно обнаружить ситуацию, похожую на блокировку, чтобы добавить хост в лист (по умолчанию: 2) --hostlist-auto-fail-time= ; все эти ситуации должны быть в пределах указанного количества секунд (по умолчанию: 60) --hostlist-auto-retrans-threshold= ; сколько ретрансмиссий запроса считать блокировкой (по умолчанию: 3) + --hostlist-auto-debug= ; лог положительных решений по autohostlist. позволяет разобраться почему там появляются хосты. Параметры манипуляции могут сочетаться в любых комбинациях. @@ -609,6 +610,7 @@ tpws - это transparent proxy. --hostlist-auto= ; обнаруживать автоматически блокировки и заполнять автоматический hostlist (требует перенаправления входящего трафика) --hostlist-auto-fail-threshold= ; сколько раз нужно обнаружить ситуацию, похожую на блокировку, чтобы добавить хост в лист (по умолчанию: 2) --hostlist-auto-fail-time= ; все эти ситуации должны быть в пределах указанного количества секунд (по умолчанию: 60) + --hostlist-auto-debug= ; лог положительных решений по autohostlist. позволяет разобраться почему там появляются хосты. Параметры манипуляции могут сочетаться в любых комбинациях. @@ -1123,6 +1125,7 @@ IP2NET_OPT6="--prefix-length=56-64 --v6-threshold=5" AUTOHOSTLIST_RETRANS_THRESHOLD=3 AUTOHOSTLIST_FAIL_THRESHOLD=2 AUTOHOSTLIST_FAIL_TIME=60 +AUTOHOSTLIST_DEBUG=0 Включить или выключить сжатие больших листов в скриптах ipset/*.sh. По умолчанию включено. GZIP_LISTS=1 diff --git a/nfq/desync.c b/nfq/desync.c index b050e33..568caed 100644 --- a/nfq/desync.c +++ b/nfq/desync.c @@ -175,8 +175,11 @@ static void wssize_cutoff(t_ctrack *ctrack) static void ctrack_stop_req_counter(t_ctrack *ctrack) { - ctrack->req_retrans_counter = RETRANS_COUNTER_STOP; - maybe_cutoff(ctrack, IPPROTO_TCP); + if (ctrack && *params.hostlist_auto_filename) + { + ctrack->req_retrans_counter = RETRANS_COUNTER_STOP; + maybe_cutoff(ctrack, IPPROTO_TCP); + } } // return true if retrans trigger fires @@ -221,6 +224,7 @@ static void auto_hostlist_failed(const char *hostname) } fail_counter->counter++; DLOG("auto hostlist : %s : fail counter %d/%d\n", hostname, fail_counter->counter, params.hostlist_auto_fail_threshold); + HOSTLIST_DEBUGLOG_APPEND("%s : fail counter %d/%d", hostname, fail_counter->counter, params.hostlist_auto_fail_threshold); if (fail_counter->counter >= params.hostlist_auto_fail_threshold) { DLOG("auto hostlist : fail threshold reached. adding %s to auto hostlist\n", hostname); @@ -231,6 +235,7 @@ static void auto_hostlist_failed(const char *hostname) if (!HostlistCheck(params.hostlist, params.hostlist_exclude, hostname, &bExcluded) && !bExcluded) { DLOG("auto hostlist : adding %s\n", hostname); + HOSTLIST_DEBUGLOG_APPEND("%s : adding", hostname); if (!StrPoolAddStr(¶ms.hostlist, hostname)) { fprintf(stderr, "StrPoolAddStr out of memory\n"); @@ -243,7 +248,10 @@ static void auto_hostlist_failed(const char *hostname) } } else + { DLOG("auto hostlist : NOT adding %s\n", hostname); + HOSTLIST_DEBUGLOG_APPEND("%s : NOT adding, duplicate detected", hostname); + } } } @@ -289,6 +297,7 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout, if (tcphdr->th_flags & TH_RST) { DLOG("incoming RST detected for hostname %s\n", ctrack->hostname); + HOSTLIST_DEBUGLOG_APPEND("%s : incoming RST", ctrack->hostname); bFail = bStop = true; } else if (len_payload && ctrack->l7proto==HTTP) @@ -298,7 +307,10 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout, DLOG("incoming HTTP reply detected for hostname %s\n", ctrack->hostname); bFail = HttpReplyLooksLikeDPIRedirect(data_payload, len_payload, ctrack->hostname); if (bFail) + { DLOG("redirect to another domain detected. possibly DPI redirect.\n") + HOSTLIST_DEBUGLOG_APPEND("%s : redirect to another domain", ctrack->hostname); + } else DLOG("local or in-domain redirect detected. it's not a DPI redirect.\n") } @@ -439,9 +451,8 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout, } else { - if (ctrack && *params.hostlist_auto_filename) - // received unknown payload. it means we are out of the request retransmission phase. stop counter - ctrack_stop_req_counter(ctrack); + // received unknown payload. it means we are out of the request retransmission phase. stop counter + ctrack_stop_req_counter(ctrack); if (!params.desync_any_proto) return res; DLOG("applying tampering to unknown protocol\n") @@ -456,12 +467,14 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout, if ((params.hostlist || params.hostlist_exclude) && !HostlistCheck(params.hostlist, params.hostlist_exclude, host, &bExcluded)) { DLOG("not applying tampering to this request\n") - if (!bExcluded) + if (!bExcluded && *params.hostlist_auto_filename && ctrack) { - if (*params.hostlist_auto_filename && ctrack && !ctrack->hostname) - ctrack->hostname=strdup(host); + if (!ctrack->hostname) ctrack->hostname=strdup(host); if (auto_hostlist_retrans(ctrack, IPPROTO_TCP, params.hostlist_auto_retrans_threshold)) + { + HOSTLIST_DEBUGLOG_APPEND("%s : tcp retrans threshold reached", ctrack->hostname); auto_hostlist_failed(host); + } } return res; } @@ -867,32 +880,34 @@ packet_process_result dpi_desync_udp_packet(uint32_t fwmark, const char *ifout, } bKnownProtocol = true; } - else if (IsWireguardHandshakeInitiation(data_payload,len_payload)) - { - DLOG("packet contains wireguard handshake initiation\n") - if (ctrack && !ctrack->l7proto) ctrack->l7proto = WIREGUARD; - fake = params.fake_wg; - fake_size = params.fake_wg_size; - bKnownProtocol = true; - } - else if (IsDhtD1(data_payload,len_payload)) - { - DLOG("packet contains DHT d1...e\n") - if (ctrack && !ctrack->l7proto) ctrack->l7proto = DHT; - fake = params.fake_dht; - fake_size = params.fake_dht_size; - bKnownProtocol = true; - } else { - if (ctrack && *params.hostlist_auto_filename) - // received unknown payload. it means we are out of the request retransmission phase. stop counter - ctrack_stop_req_counter(ctrack); - - if (!params.desync_any_proto) return res; - DLOG("applying tampering to unknown protocol\n") - fake = params.fake_unknown_udp; - fake_size = params.fake_unknown_udp_size; + // received payload without host. it means we are out of the request retransmission phase. stop counter + ctrack_stop_req_counter(ctrack); + + if (IsWireguardHandshakeInitiation(data_payload,len_payload)) + { + DLOG("packet contains wireguard handshake initiation\n") + if (ctrack && !ctrack->l7proto) ctrack->l7proto = WIREGUARD; + fake = params.fake_wg; + fake_size = params.fake_wg_size; + bKnownProtocol = true; + } + else if (IsDhtD1(data_payload,len_payload)) + { + DLOG("packet contains DHT d1...e\n") + if (ctrack && !ctrack->l7proto) ctrack->l7proto = DHT; + fake = params.fake_dht; + fake_size = params.fake_dht_size; + bKnownProtocol = true; + } + else + { + if (!params.desync_any_proto) return res; + DLOG("applying tampering to unknown protocol\n") + fake = params.fake_unknown_udp; + fake_size = params.fake_unknown_udp_size; + } } if (bHaveHost) @@ -902,12 +917,14 @@ packet_process_result dpi_desync_udp_packet(uint32_t fwmark, const char *ifout, if ((params.hostlist || params.hostlist_exclude) && !HostlistCheck(params.hostlist, params.hostlist_exclude, host, &bExcluded)) { DLOG("not applying tampering to this request\n") - if (!bExcluded) + if (!bExcluded && *params.hostlist_auto_filename && ctrack) { - if (*params.hostlist_auto_filename && ctrack && !ctrack->hostname) - ctrack->hostname=strdup(host); + if (!ctrack->hostname) ctrack->hostname=strdup(host); if (auto_hostlist_retrans(ctrack, IPPROTO_UDP, params.hostlist_auto_retrans_threshold)) + { + HOSTLIST_DEBUGLOG_APPEND("%s : udp retrans threshold reached", ctrack->hostname); auto_hostlist_failed(host); + } } return res; } diff --git a/nfq/helpers.c b/nfq/helpers.c index 95bcc88..a52bf0d 100644 --- a/nfq/helpers.c +++ b/nfq/helpers.c @@ -188,6 +188,12 @@ void phton64(uint8_t *p, uint64_t v) p[7] = (uint8_t)(v >> 0); } +bool ipv6_addr_is_zero(const struct in6_addr *a) +{ + return !memcmp(a,"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",16); +} + + #define INVALID_HEX_DIGIT ((uint8_t)-1) static inline uint8_t parse_hex_digit(char c) { @@ -234,3 +240,13 @@ void fill_pattern(uint8_t *buf,size_t bufsize,const void *pattern,size_t patsize bufsize -= size; } } + +int fprint_localtime(FILE *F) +{ + struct tm t; + time_t now; + + time(&now); + localtime_r(&now,&t); + return fprintf(F, "%02d.%02d.%04d %02d:%02d:%02d", t.tm_mday, t.tm_mon + 1, t.tm_year + 1900, t.tm_hour, t.tm_min, t.tm_sec); +} diff --git a/nfq/helpers.h b/nfq/helpers.h index 2e474d4..fa7d003 100644 --- a/nfq/helpers.h +++ b/nfq/helpers.h @@ -25,6 +25,8 @@ bool set_socket_buffers(int fd, int rcvbuf, int sndbuf); uint64_t pntoh64(const void *p); void phton64(uint8_t *p, uint64_t v); +bool ipv6_addr_is_zero(const struct in6_addr *a); + static inline uint16_t pntoh16(const uint8_t *p) { return ((uint16_t)p[0] << 8) | (uint16_t)p[1]; } @@ -38,3 +40,5 @@ static inline uint32_t pntoh32(const uint8_t *p) { bool parse_hex_str(const char *s, uint8_t *pbuf, size_t *size); void fill_pattern(uint8_t *buf,size_t bufsize,const void *pattern,size_t patsize); + +int fprint_localtime(FILE *F); diff --git a/nfq/nfqws.c b/nfq/nfqws.c index 5b0262b..dfa33ae 100644 --- a/nfq/nfqws.c +++ b/nfq/nfqws.c @@ -567,7 +567,8 @@ static void exithelp(void) " --hostlist-auto=\t\t\t; detect DPI blocks and build hostlist automatically\n" " --hostlist-auto-fail-threshold=\t\t; how many failed attempts cause hostname to be added to auto hostlist (default : %d)\n" " --hostlist-auto-fail-time=\t\t; all failed attemps must be within these seconds (default : %d)\n" - " --hostlist-auto-retrans-threshold=\t; how many request retransmissions cause attempt to fail (default : %d)\n", + " --hostlist-auto-retrans-threshold=\t; how many request retransmissions cause attempt to fail (default : %d)\n" + " --hostlist-auto-debug=\t\t; debug auto hostlist positives\n", CTRACK_T_SYN, CTRACK_T_EST, CTRACK_T_FIN, CTRACK_T_UDP, #if defined(__linux__) || defined(SO_USER_COOKIE) DPI_DESYNC_FWMARK_DEFAULT,DPI_DESYNC_FWMARK_DEFAULT, @@ -752,10 +753,11 @@ int main(int argc, char **argv) {"hostlist-auto-fail-threshold",required_argument,0,0}, // optidx=41 {"hostlist-auto-fail-time",required_argument,0,0}, // optidx=42 {"hostlist-auto-retrans-threshold",required_argument,0,0}, // optidx=43 + {"hostlist-auto-debug",required_argument,0,0}, // optidx=44 #ifdef __linux__ - {"bind-fix4",no_argument,0,0}, // optidx=44 - {"bind-fix6",no_argument,0,0}, // optidx=45 + {"bind-fix4",no_argument,0,0}, // optidx=45 + {"bind-fix6",no_argument,0,0}, // optidx=46 #endif {NULL,0,NULL,0} }; @@ -1140,11 +1142,26 @@ int main(int argc, char **argv) exit_clean(1); } break; + case 44: /* hostlist-auto-debug */ + { + FILE *F = fopen(optarg,"a+t"); + if (!F) + { + fprintf(stderr, "cannot create %s\n", optarg); + exit_clean(1); + } + fclose(F); + if (params.droproot && chown(optarg, params.uid, -1)) + fprintf(stderr, "could not chown %s. auto hostlist debug log may not be writable after privilege drop\n", optarg); + strncpy(params.hostlist_auto_debuglog, optarg, sizeof(params.hostlist_auto_debuglog)); + params.hostlist_auto_debuglog[sizeof(params.hostlist_auto_debuglog) - 1] = '\0'; + } + break; #ifdef __linux__ - case 44: /* bind-fix4 */ + case 45: /* bind-fix4 */ params.bind_fix4 = true; break; - case 45: /* bind-fix6 */ + case 46: /* bind-fix6 */ params.bind_fix6 = true; break; #endif diff --git a/nfq/params.h b/nfq/params.h index ac0a7d7..5b10332 100644 --- a/nfq/params.h +++ b/nfq/params.h @@ -64,7 +64,7 @@ struct params_s strpool *hostlist, *hostlist_exclude; struct str_list_head hostlist_files, hostlist_exclude_files; - char hostlist_auto_filename[PATH_MAX]; + char hostlist_auto_filename[PATH_MAX], hostlist_auto_debuglog[PATH_MAX]; int hostlist_auto_fail_threshold, hostlist_auto_fail_time, hostlist_auto_retrans_threshold; hostfail_pool *hostlist_auto_fail_counters; @@ -75,3 +75,15 @@ struct params_s extern struct params_s params; #define DLOG(format, ...) {if (params.debug) printf(format, ##__VA_ARGS__);} + +#define LOG_APPEND(filename, format, ...) \ +{ \ + FILE *F = fopen(filename,"at"); \ + if (F) \ + { \ + fprint_localtime(F); \ + fprintf(F, " : " format "\n", ##__VA_ARGS__); \ + fclose(F); \ + } \ +} +#define HOSTLIST_DEBUGLOG_APPEND(format, ...) if (*params.hostlist_auto_debuglog) LOG_APPEND(params.hostlist_auto_debuglog, format, ##__VA_ARGS__) diff --git a/tpws/helpers.c b/tpws/helpers.c index a4a7750..4ac46d0 100644 --- a/tpws/helpers.c +++ b/tpws/helpers.c @@ -8,6 +8,7 @@ #include #include #include +#include char *strncasestr(const char *s,const char *find, size_t slen) { @@ -211,3 +212,13 @@ int get_so_error(int fd) errn=errno; return errn; } + +int fprint_localtime(FILE *F) +{ + struct tm t; + time_t now; + + time(&now); + localtime_r(&now,&t); + return fprintf(F, "%02d.%02d.%04d %02d:%02d:%02d", t.tm_mday, t.tm_mon + 1, t.tm_year + 1900, t.tm_hour, t.tm_min, t.tm_sec); +} diff --git a/tpws/helpers.h b/tpws/helpers.h index a021baa..00e0522 100644 --- a/tpws/helpers.h +++ b/tpws/helpers.h @@ -5,6 +5,7 @@ #include #include #include +#include char *strncasestr(const char *s,const char *find, size_t slen); @@ -41,3 +42,5 @@ static inline void phton16(uint8_t *p, uint16_t v) { p[0] = (uint8_t)(v>>8); p[1] = (uint8_t)v; } + +int fprint_localtime(FILE *F); diff --git a/tpws/params.h b/tpws/params.h index c7e4c54..7184f1c 100644 --- a/tpws/params.h +++ b/tpws/params.h @@ -56,7 +56,7 @@ struct params_s strpool *hostlist, *hostlist_exclude; struct str_list_head hostlist_files, hostlist_exclude_files; - char hostlist_auto_filename[PATH_MAX]; + char hostlist_auto_filename[PATH_MAX], hostlist_auto_debuglog[PATH_MAX]; int hostlist_auto_fail_threshold, hostlist_auto_fail_time; hostfail_pool *hostlist_auto_fail_counters; @@ -72,3 +72,15 @@ extern struct params_s params; #define _DBGPRINT(format, level, ...) { if (params.debug>=level) printf(format "\n", ##__VA_ARGS__); } #define VPRINT(format, ...) _DBGPRINT(format,1,##__VA_ARGS__) #define DBGPRINT(format, ...) _DBGPRINT(format,2,##__VA_ARGS__) + +#define LOG_APPEND(filename, format, ...) \ +{ \ + FILE *F = fopen(filename,"at"); \ + if (F) \ + { \ + fprint_localtime(F); \ + fprintf(F, " : " format "\n", ##__VA_ARGS__); \ + fclose(F); \ + } \ +} +#define HOSTLIST_DEBUGLOG_APPEND(format, ...) if (*params.hostlist_auto_debuglog) LOG_APPEND(params.hostlist_auto_debuglog, format, ##__VA_ARGS__) diff --git a/tpws/tamper.c b/tpws/tamper.c index 98d50b5..782e218 100644 --- a/tpws/tamper.c +++ b/tpws/tamper.c @@ -299,6 +299,7 @@ static void auto_hostlist_failed(const char *hostname) } fail_counter->counter++; VPRINT("auto hostlist : %s : fail counter %d/%d", hostname, fail_counter->counter, params.hostlist_auto_fail_threshold); + HOSTLIST_DEBUGLOG_APPEND("%s : fail counter %d/%d", hostname, fail_counter->counter, params.hostlist_auto_fail_threshold); if (fail_counter->counter >= params.hostlist_auto_fail_threshold) { VPRINT("auto hostlist : fail threshold reached. adding %s to auto hostlist", hostname); @@ -309,6 +310,7 @@ static void auto_hostlist_failed(const char *hostname) if (!HostlistCheck(params.hostlist, params.hostlist_exclude, hostname, &bExcluded) && !bExcluded) { VPRINT("auto hostlist : adding %s", hostname); + HOSTLIST_DEBUGLOG_APPEND("%s : adding", hostname); if (!StrPoolAddStr(¶ms.hostlist, hostname)) { fprintf(stderr, "StrPoolAddStr out of memory\n"); @@ -321,7 +323,10 @@ static void auto_hostlist_failed(const char *hostname) } } else + { VPRINT("auto hostlist: NOT adding %s", hostname); + HOSTLIST_DEBUGLOG_APPEND("%s : NOT adding, duplicate detected", hostname); + } } } @@ -340,7 +345,10 @@ void tamper_in(t_ctrack *ctrack, uint8_t *segment,size_t segment_buffer_size,siz VPRINT("incoming HTTP reply detected for hostname %s", ctrack->hostname); bFail = HttpReplyLooksLikeDPIRedirect(segment, *size, ctrack->hostname); if (bFail) + { VPRINT("redirect to another domain detected. possibly DPI redirect.") + HOSTLIST_DEBUGLOG_APPEND("%s : redirect to another domain", ctrack->hostname); + } else VPRINT("local or in-domain redirect detected. it's not a DPI redirect.") } @@ -365,6 +373,7 @@ void rst_in(t_ctrack *ctrack) if (!ctrack->bTamperInCutoff && ctrack->hostname) { VPRINT("incoming RST detected for hostname %s", ctrack->hostname); + HOSTLIST_DEBUGLOG_APPEND("%s : incoming RST", ctrack->hostname); auto_hostlist_failed(ctrack->hostname); } } @@ -378,6 +387,7 @@ void hup_out(t_ctrack *ctrack) { // local leg dropped connection after first request. probably due to timeout. VPRINT("local leg closed connection after first request (timeout ?). hostname: %s", ctrack->hostname); + HOSTLIST_DEBUGLOG_APPEND("%s : client closed connection without server reply", ctrack->hostname); auto_hostlist_failed(ctrack->hostname); } } diff --git a/tpws/tpws.c b/tpws/tpws.c index f3abb69..b806ab1 100644 --- a/tpws/tpws.c +++ b/tpws/tpws.c @@ -166,6 +166,7 @@ static void exithelp(void) " --hostlist-auto=\t\t; detect DPI blocks and build hostlist automatically\n" " --hostlist-auto-fail-threshold=\t; how many failed attempts cause hostname to be added to auto hostlist (default : %d)\n" " --hostlist-auto-fail-time=\t; all failed attemps must be within these seconds (default : %d)\n" + " --hostlist-auto-debug=\t; debug auto hostlist positives\n" "\nTAMPER:\n" " --split-http-req=method|host\t\t; split at specified logical part of plain http request\n" " --split-pos=\t\t; split at specified pos. split-http-req takes precedence for http.\n" @@ -304,17 +305,18 @@ void parse_params(int argc, char *argv[]) { "hostlist-auto",required_argument,0,0}, // optidx=35 { "hostlist-auto-fail-threshold",required_argument,0,0}, // optidx=36 { "hostlist-auto-fail-time",required_argument,0,0}, // optidx=37 - { "pidfile",required_argument,0,0 },// optidx=38 - { "debug",optional_argument,0,0 },// optidx=39 - { "local-rcvbuf",required_argument,0,0 },// optidx=40 - { "local-sndbuf",required_argument,0,0 },// optidx=41 - { "remote-rcvbuf",required_argument,0,0 },// optidx=42 - { "remote-sndbuf",required_argument,0,0 },// optidx=43 - { "socks",no_argument,0,0 },// optidx=44 - { "no-resolve",no_argument,0,0 },// optidx=45 - { "skip-nodelay",no_argument,0,0 },// optidx=46 + { "hostlist-auto-debug",required_argument,0,0}, // optidx=38 + { "pidfile",required_argument,0,0 },// optidx=39 + { "debug",optional_argument,0,0 },// optidx=40 + { "local-rcvbuf",required_argument,0,0 },// optidx=41 + { "local-sndbuf",required_argument,0,0 },// optidx=42 + { "remote-rcvbuf",required_argument,0,0 },// optidx=43 + { "remote-sndbuf",required_argument,0,0 },// optidx=44 + { "socks",no_argument,0,0 },// optidx=45 + { "no-resolve",no_argument,0,0 },// optidx=46 + { "skip-nodelay",no_argument,0,0 },// optidx=47 #if defined(BSD) && !defined(__OpenBSD__) && !defined(__APPLE__) - { "enable-pf",no_argument,0,0 },// optidx=47 + { "enable-pf",no_argument,0,0 },// optidx=48 #endif { "hostlist-auto-retrans-threshold",optional_argument,0,0}, // ignored. for nfqws command line compatibility { NULL,0,NULL,0 } @@ -603,36 +605,51 @@ void parse_params(int argc, char *argv[]) exit_clean(1); } break; - case 38: /* pidfile */ + case 38: /* hostlist-auto-debug */ + { + FILE *F = fopen(optarg,"a+t"); + if (!F) + { + fprintf(stderr, "cannot create %s\n", optarg); + exit_clean(1); + } + fclose(F); + if (params.droproot && chown(optarg, params.uid, -1)) + fprintf(stderr, "could not chown %s. auto hostlist debug log may not be writable after privilege drop\n", optarg); + strncpy(params.hostlist_auto_debuglog, optarg, sizeof(params.hostlist_auto_debuglog)); + params.hostlist_auto_debuglog[sizeof(params.hostlist_auto_debuglog) - 1] = '\0'; + } + break; + case 39: /* pidfile */ strncpy(params.pidfile,optarg,sizeof(params.pidfile)); params.pidfile[sizeof(params.pidfile)-1]='\0'; break; - case 39: + case 40: params.debug = optarg ? atoi(optarg) : 1; break; - case 40: /* local-rcvbuf */ + case 41: /* local-rcvbuf */ params.local_rcvbuf = atoi(optarg)/2; break; - case 41: /* local-sndbuf */ + case 42: /* local-sndbuf */ params.local_sndbuf = atoi(optarg)/2; break; - case 42: /* remote-rcvbuf */ + case 43: /* remote-rcvbuf */ params.remote_rcvbuf = atoi(optarg)/2; break; - case 43: /* remote-sndbuf */ + case 44: /* remote-sndbuf */ params.remote_sndbuf = atoi(optarg)/2; break; - case 44: /* socks */ + case 45: /* socks */ params.proxy_type = CONN_TYPE_SOCKS; break; - case 45: /* no-resolve */ + case 46: /* no-resolve */ params.no_resolve = true; break; - case 46: /* skip-nodelay */ + case 47: /* skip-nodelay */ params.skip_nodelay = true; break; #if defined(BSD) && !defined(__OpenBSD__) && !defined(__APPLE__) - case 47: /* enable-pf */ + case 48: /* enable-pf */ params.pf_enable = true; break; #endif