diff --git a/binaries/aarch64/nfqws b/binaries/aarch64/nfqws index 259b2af..c449713 100755 Binary files a/binaries/aarch64/nfqws and b/binaries/aarch64/nfqws differ diff --git a/binaries/arm/nfqws b/binaries/arm/nfqws index b832c5e..fe22aa6 100755 Binary files a/binaries/arm/nfqws and b/binaries/arm/nfqws differ diff --git a/binaries/freebsd-x64/dvtws b/binaries/freebsd-x64/dvtws index ece170c..683c889 100755 Binary files a/binaries/freebsd-x64/dvtws and b/binaries/freebsd-x64/dvtws differ diff --git a/binaries/mips32r1-lsb/nfqws b/binaries/mips32r1-lsb/nfqws index 0f2f4d6..3d39484 100755 Binary files a/binaries/mips32r1-lsb/nfqws and b/binaries/mips32r1-lsb/nfqws differ diff --git a/binaries/mips32r1-msb/nfqws b/binaries/mips32r1-msb/nfqws index e217eb5..4051396 100755 Binary files a/binaries/mips32r1-msb/nfqws and b/binaries/mips32r1-msb/nfqws differ diff --git a/binaries/mips64r2-msb/nfqws b/binaries/mips64r2-msb/nfqws index 91ff6e5..5f18b6c 100755 Binary files a/binaries/mips64r2-msb/nfqws and b/binaries/mips64r2-msb/nfqws differ diff --git a/binaries/ppc/nfqws b/binaries/ppc/nfqws index 304bf2e..738d630 100755 Binary files a/binaries/ppc/nfqws and b/binaries/ppc/nfqws differ diff --git a/binaries/x86/nfqws b/binaries/x86/nfqws index 8f8a0f4..198a099 100755 Binary files a/binaries/x86/nfqws and b/binaries/x86/nfqws differ diff --git a/binaries/x86_64/nfqws b/binaries/x86_64/nfqws index b6ce265..96b3165 100755 Binary files a/binaries/x86_64/nfqws and b/binaries/x86_64/nfqws differ diff --git a/common/ipt.sh b/common/ipt.sh index 99a8861..0b42818 100644 --- a/common/ipt.sh +++ b/common/ipt.sh @@ -328,9 +328,9 @@ zapret_do_firewall_rules_ipt() # autohostlist mode requires incoming traffic sample # always use conntrack packet limiter or nfqws will deal with gigabytes if [ "$MODE_FILTER" = "autohostlist" ]; then - n=$((4+${AUTOHOSTLIST_RETRANS_THRESHOLD:-3})) + n=$((6+${AUTOHOSTLIST_RETRANS_THRESHOLD:-3})) else - n=4 + n=6 fi first_packet_only="${first_packet_only}$n" diff --git a/common/nft.sh b/common/nft.sh index 6344941..9f0b83e 100644 --- a/common/nft.sh +++ b/common/nft.sh @@ -576,9 +576,9 @@ zapret_apply_firewall_rules_nft() # autohostlist mode requires incoming traffic sample # always use conntrack packet limiter or nfqws will deal with gigabytes if [ "$MODE_FILTER" = "autohostlist" ]; then - first_packet_only=$((4+${AUTOHOSTLIST_RETRANS_THRESHOLD:-3})) + first_packet_only=$((6+${AUTOHOSTLIST_RETRANS_THRESHOLD:-3})) else - first_packet_only=4 + first_packet_only=6 fi first_packet_only="ct original packets 1-$first_packet_only" diff --git a/docs/changes.txt b/docs/changes.txt index 8383917..e492d66 100644 --- a/docs/changes.txt +++ b/docs/changes.txt @@ -251,3 +251,7 @@ tpws --tlsrec attack. v52 autohostlist mode + +v53 + +nfqws: tcp session reassemble for TLS ClientHello diff --git a/docs/iptables.txt b/docs/iptables.txt index e28bb6e..c3b8c6f 100644 --- a/docs/iptables.txt +++ b/docs/iptables.txt @@ -17,7 +17,7 @@ iptables -t mangle -I POSTROUTING -p udp --dport 443 -m mark ! --mark 0x40000000 # auto hostlist with avoiding wrong ACK numbers in RST,ACK packets sent by russian DPI sysctl net.netfilter.nf_conntrack_tcp_be_liberal=1 iptables -t mangle -I POSTROUTING -p tcp -m multiport --dports 80,443 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:12 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass -iptables -t mangle -I PREROUTING -p tcp -m multiport --sports 80,443 -m connbytes --connbytes-dir=reply --connbytes-mode=packets --connbytes 1:4 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass +iptables -t mangle -I PREROUTING -p tcp -m multiport --sports 80,443 -m connbytes --connbytes-dir=reply --connbytes-mode=packets --connbytes 1:6 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass For TPROXY : diff --git a/docs/readme.eng.md b/docs/readme.eng.md index 92110b8..c8a3b24 100644 --- a/docs/readme.eng.md +++ b/docs/readme.eng.md @@ -92,10 +92,10 @@ into IP addresses and put them to ipset 'zapret', then add a filter to the comma Some DPIs catch only the first http request, ignoring subsequent requests in a keep-alive session. Then we can reduce CPU load, refusing to process unnecessary packets. -`iptables -t mangle -I POSTROUTING -o -p tcp --dport 80 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:4 -m mark ! --mark 0x40000000/0x40000000 -m set --match-set zapret dst -j NFQUEUE --queue-num 200 --queue-bypass` +`iptables -t mangle -I POSTROUTING -o -p tcp --dport 80 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:6 -m mark ! --mark 0x40000000/0x40000000 -m set --match-set zapret dst -j NFQUEUE --queue-num 200 --queue-bypass` Mark filter does not allow nfqws-generated packets to enter the queue again. -Its necessary to use this filter when also using `connbytes 1:4`. Without it packet ordering can be changed breaking the whole idea. +Its necessary to use this filter when also using `connbytes 1:6`. Without it packet ordering can be changed breaking the whole idea. ## ip6tables @@ -295,13 +295,13 @@ Subdomains are applied automatically. gzip lists are supported. iptables for performing the attack on the first packet : -`iptables -t mangle -I POSTROUTING -o -p tcp -m multiport --dports 80,443 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:4 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass` +`iptables -t mangle -I POSTROUTING -o -p tcp -m multiport --dports 80,443 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:6 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass` This is good if DPI does not track all requests in http keep-alive session. If it does, then pass all outgoing packets for http and only first data packet for https : ``` -iptables -t mangle -I POSTROUTING -o -p tcp --dport 443 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:4 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass +iptables -t mangle -I POSTROUTING -o -p tcp --dport 443 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:6 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass iptables -t mangle -I POSTROUTING -o -p tcp --dport 80 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass ``` diff --git a/docs/readme.txt b/docs/readme.txt index 88b4d66..84fdb40 100644 --- a/docs/readme.txt +++ b/docs/readme.txt @@ -1,4 +1,4 @@ -zapret v.52 +zapret v.53 English ------- @@ -116,10 +116,10 @@ iptables -t mangle -I POSTROUTING -o <внешний_интерфейс> -p tcp DPI может ловить только первый http запрос, игнорируя последующие запросы в keep-alive сессии. Тогда можем уменьшить нагрузку на проц, отказавшись от процессинга ненужных пакетов. -iptables -t mangle -I POSTROUTING -o <внешний_интерфейс> -p tcp --dport 80 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:4 -m mark ! --mark 0x40000000/0x40000000 -m set --match-set zapret dst -j NFQUEUE --queue-num 200 --queue-bypass +iptables -t mangle -I POSTROUTING -o <внешний_интерфейс> -p tcp --dport 80 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:6 -m mark ! --mark 0x40000000/0x40000000 -m set --match-set zapret dst -j NFQUEUE --queue-num 200 --queue-bypass Фильтр по mark нужен для отсечения от очереди пакетов, сгенерированных внутри nfqws. -Если применяется фильтр по connbytes 1:4, то обязательно добавлять в iptables и фильтр по mark. Иначе возможно +Если применяется фильтр по connbytes 1:6, то обязательно добавлять в iptables и фильтр по mark. Иначе возможно перепутывание порядка следования пакетов, что приведет к неработоспособности метода. @@ -356,12 +356,12 @@ DPI может отстать от потока, если ClientHello его у iptables для задействования атаки на первый пакет данных : -iptables -t mangle -I POSTROUTING -o <внешний_интерфейс> -p tcp -m multiport --dports 80,443 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:4 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass +iptables -t mangle -I POSTROUTING -o <внешний_интерфейс> -p tcp -m multiport --dports 80,443 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:6 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass Этот вариант применяем, когда DPI не следит за всеми запросами http внутри keep-alive сессии. Если следит, направляем только первый пакет от https и все пакеты от http : -iptables -t mangle -I POSTROUTING -o <внешний_интерфейс> -p tcp --dport 443 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:4 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass +iptables -t mangle -I POSTROUTING -o <внешний_интерфейс> -p tcp --dport 443 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:6 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass iptables -t mangle -I POSTROUTING -o <внешний_интерфейс> -p tcp --dport 80 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass mark нужен, чтобы сгенерированный поддельный пакет не попал опять к нам на обработку. nfqws выставляет fwmark при его отсылке. @@ -372,11 +372,11 @@ mark нужен, чтобы сгенерированный поддельный При отсутствии ограничения на connbytes, атака будет работать и без фильтра по mark. Но лучше его все же оставить для увеличения скорости. -Почему --connbytes 1:4 : +Почему --connbytes 1:6 : 1 - для работы методов десинхронизации 0-й фазы и wssize 2 - иногда данные идут в 3-м пакете 3-way handshake -3 - стандартная ситуация -4 - для надежности. на случай, если выполнялась одна ретрансмиссия +3 - стандартная ситуация приема одного пакета запроса +4-6 - на случай ретрансмиссии или запроса длиной в несколько пакетов (TLSClientHello с kyber, например) КОМБИНИРОВАНИЕ МЕТОДОВ ДЕСИНХРОНИЗАЦИИ В параметре dpi-desync можно указать до 3 режимов через запятую. @@ -416,7 +416,6 @@ ip6tables -D zone_wan_output -m comment --comment '!fw3' -j zone_wan_dest_ACCEPT CONNTRACK nfqws оснащен ограниченной реализацией слежения за состоянием tcp соединений (conntrack). Он включается для реализации некоторых методов противодействия DPI. -На текущий момент это параметры --wssize и --dpi-desync-cutoff. conntrack способен следить за фазой соединения : SYN,ESTABLISHED,FIN , количеством пакетов в каждую сторону, sequence numbers. conntrack способен "кормиться" пакетами в обе или только в одну сторону. Соединение попадает в таблицу при обнаружении пакетов с выставленными флагами SYN или SYN,ACK. @@ -468,6 +467,17 @@ window size итоговый размер окна стал максимальн На склонных к бездействию соединениях следует изменить таймауты conntrack. Если соединение выпало из conntrack и задана опция --dpi-desync-cutoff, dpi desync применяться не будет. +РЕАССЕМБЛИНГ TCP +nfqws поддерживает реассемблинг некоторых видов tcp запросов. +На текущий момент это TLS ClientHello. Он бывает длинным, если в chrome включить пост-квантовую +криптографию tls-kyber, и занимает как правило 2 пакета. +chrome рандомизирует фингерпринт TLS. SNI может оказаться как в начале, так и в конце, то есть +попасть в 1 или 2 пакет. stateful DPI обычно реассемблирует запрос целиком, и только потом +принимает решение о блокировке. +nfqws реагирует десинхронизацией на каждый пакет из TLSClientHello, если задана опция +--dpi-desync-skip-nosni=0. В противном случае десинхронизация идет на сам пакет, +включающий SNI, и все последующие. + ПОДДЕРЖКА UDP Атаки на udp более ограничены в возможностях. udp нельзя фрагментировать иначе, чем на уровне ip. Для UDP действуют только режимы десинхронизации fake,hopbyhop,destopt,ipfrag1,ipfrag2,udplen,tamper. @@ -487,6 +497,11 @@ udplen увеличивает размер udp пакета на указанн Атака fake полезна только для stateful DPI, она бесполезна для анализа на уровне отдельных пакетов. По умолчанию fake наполнение - 64 нуля. Можно указать файл в --dpi-desync-fake-unknown-udp. +РЕАССЕМБЛИНГ QUIC +tls-kyber может так же размазываться по 2 пакетам QUIC Initial. +Пока их реассемблинг не реализован, поскольку русский DPI не регирует на такие пакеты. +Идет десинхронизация полных hello в одном пакете и частичных hello, где SNI попал в 1-й пакет. + IP ФРАГМЕНТАЦИЯ В современной сети с этом все очень плохо. Фрагментированные пакеты застревают по пути, часто отбрасываются. Иногда доходят. Иногда то доходят, то не доходят. Может зависеть от версии ipv4/ipv6. @@ -964,7 +979,7 @@ nfqws и tpws могут сечь варианты 1-3, 4 они не распо Заносите такие домены в ipset/zapret-hosts-user-exclude.txt, чтобы избежать повторения. Чтобы впоследствии разобраться почему домен был занесен в лист, можно включить autohostlist debug log. Он полезен тем, что работает без постоянного просмотра вывода nfqws в режиме debug. -В лог заносятся только основные события, ведушие к занесению хоста в лист. +В лог заносятся только основные события, ведущие к занесению хоста в лист. По логу можно понять как избежать ложных срабатываний и подходит ли вообще вам этот режим. Скрипты zapret ведут autohostlist в ipset/zapret-hosts-auto.txt. diff --git a/nfq/conntrack.c b/nfq/conntrack.c index dbd07cd..01368d0 100644 --- a/nfq/conntrack.c +++ b/nfq/conntrack.c @@ -25,9 +25,23 @@ static void connswap(const t_conn *c, t_conn *c2) c2->dport = c->sport; } +void ConntrackClearHostname(t_ctrack *track) +{ + if (track->hostname) + { + free(track->hostname); + track->hostname = NULL; + } +} +static void ConntrackClearTrack(t_ctrack *track) +{ + ConntrackClearHostname(track); + ReasmClear(&track->reasm_orig); +} + static void ConntrackFreeElem(t_conntrack_pool *elem) { - if (elem->track.hostname) free(elem->track.hostname); + ConntrackClearTrack(&elem->track); free(elem); } @@ -309,3 +323,36 @@ void ConntrackPoolDump(const t_conntrack *p) t->track.req_retrans_counter, t->track.b_cutoff, t->track.b_wssize_cutoff, t->track.b_desync_cutoff, t->track.hostname, ConntrackProtoName(t->track.l7proto)); }; } + + +void ReasmClear(t_reassemble *reasm) +{ + if (reasm->packet) + { + free(reasm->packet); + reasm->packet = NULL; + } + reasm->size = reasm->size_present = 0; +} +bool ReasmInit(t_reassemble *reasm, size_t size_requested, uint32_t seq_start) +{ + reasm->packet = malloc(size_requested); + if (!reasm->packet) return false; + reasm->size = size_requested; + reasm->size_present = 0; + reasm->seq = seq_start; + return true; +} +bool ReasmFeed(t_reassemble *reasm, uint32_t seq, const void *payload, size_t len) +{ + if (reasm->seq!=seq) return false; // fail session if out of sequence + + size_t szcopy; + szcopy = reasm->size - reasm->size_present; + if (lenpacket + reasm->size_present, payload, szcopy); + reasm->size_present += szcopy; + reasm->seq += (uint32_t)szcopy; + + return true; +} diff --git a/nfq/conntrack.h b/nfq/conntrack.h index d2fce2b..393365c 100644 --- a/nfq/conntrack.h +++ b/nfq/conntrack.h @@ -34,6 +34,14 @@ typedef struct uint8_t l4proto; // IPPROTO_TCP, IPPROTO_UDP } t_conn; +// this structure helps to reassemble continuous packets streams. it does not support out-of-orders +typedef struct { + uint8_t *packet; // allocated for size during reassemble request. requestor must know the message size. + uint32_t seq; // current seq number. if a packet comes with an unexpected seq - it fails reassemble session. + size_t size; // expected message size. success means that we have received exactly 'size' bytes and have them in 'packet' + size_t size_present; // how many bytes already stored in 'packet' +} t_reassemble; + // SYN - SYN or SYN/ACK received // ESTABLISHED - any except SYN or SYN/ACK received // FIN - FIN or RST received @@ -55,11 +63,14 @@ typedef struct uint8_t scale_orig, scale_reply; // last seen window scale factor. SCALE_NONE if none uint8_t req_retrans_counter; // number of request retransmissions - uint32_t req_seq; // sequence number of the request (to track retransmissions) + bool req_seq_start_present, req_seq_present; + uint32_t req_seq_start,req_seq_end; // sequence interval of the request (to track retransmissions) bool b_cutoff; // mark for deletion bool b_wssize_cutoff, b_desync_cutoff; + t_reassemble reasm_orig; + t_l7proto l7proto; char *hostname; } t_ctrack; @@ -85,3 +96,11 @@ bool ConntrackPoolDrop(t_conntrack *p, const struct ip *ip, const struct ip6_hdr void CaonntrackExtractConn(t_conn *c, bool bReverse, const struct ip *ip, const struct ip6_hdr *ip6, const struct tcphdr *tcphdr, const struct udphdr *udphdr); void ConntrackPoolDump(const t_conntrack *p); void ConntrackPoolPurge(t_conntrack *p); +void ConntrackClearHostname(t_ctrack *track); + +bool ReasmInit(t_reassemble *reasm, size_t size_requested, uint32_t seq_start); +void ReasmClear(t_reassemble *reasm); +// false means reassemble session has failed and we should ReasmClear() it +bool ReasmFeed(t_reassemble *reasm, uint32_t seq, const void *payload, size_t len); +inline static bool ReasmIsEmpty(t_reassemble *reasm) {return !reasm->size;} +inline static bool ReasmIsFull(t_reassemble *reasm) {return !ReasmIsEmpty(reasm) && (reasm->size==reasm->size_present);} diff --git a/nfq/desync.c b/nfq/desync.c index 568caed..ea2d791 100644 --- a/nfq/desync.c +++ b/nfq/desync.c @@ -155,13 +155,14 @@ static void maybe_cutoff(t_ctrack *ctrack, uint8_t proto) ctrack->b_wssize_cutoff |= cutoff_test(ctrack, params.wssize_cutoff, params.wssize_cutoff_mode); ctrack->b_desync_cutoff |= cutoff_test(ctrack, params.desync_cutoff, params.desync_cutoff_mode); + // we do not need conntrack entry anymore if all cutoff conditions are either not defined or reached + // do not drop udp entry because it will be recreated when next packet arrives if (proto==IPPROTO_TCP) - // we do not need conntrack entry anymore if all cutoff conditions are either not defined or reached - // do not drop udp entry because it will be recreated when next packet arrives ctrack->b_cutoff |= \ (!params.wssize || ctrack->b_wssize_cutoff) && (!params.desync_cutoff || ctrack->b_desync_cutoff) && - (!*params.hostlist_auto_filename || ctrack->req_retrans_counter==RETRANS_COUNTER_STOP); + (!*params.hostlist_auto_filename || ctrack->req_retrans_counter==RETRANS_COUNTER_STOP) && + ReasmIsEmpty(&ctrack->reasm_orig); } } static void wssize_cutoff(t_ctrack *ctrack) @@ -172,8 +173,16 @@ static void wssize_cutoff(t_ctrack *ctrack) maybe_cutoff(ctrack, IPPROTO_TCP); } } +static void forced_wssize_cutoff(t_ctrack *ctrack) +{ + if (ctrack && params.wssize && !ctrack->b_wssize_cutoff) + { + DLOG("forced wssize-cutoff\n"); + wssize_cutoff(ctrack); + } +} -static void ctrack_stop_req_counter(t_ctrack *ctrack) +static void ctrack_stop_retrans_counter(t_ctrack *ctrack) { if (ctrack && *params.hostlist_auto_filename) { @@ -187,24 +196,26 @@ static bool auto_hostlist_retrans(t_ctrack *ctrack, uint8_t l4proto, int thresho { if (*params.hostlist_auto_filename && ctrack && ctrack->req_retrans_counter!=RETRANS_COUNTER_STOP) { - ctrack->req_retrans_counter++; - DLOG("req retrans counter : %u/%u\n",ctrack->req_retrans_counter, threshold); - if (ctrack->req_retrans_counter >= threshold) - { - DLOG("req retrans threshold reached\n"); - ctrack_stop_req_counter(ctrack); - return true; - } if (l4proto==IPPROTO_TCP) { - if (!ctrack->req_seq) ctrack->req_seq = ctrack->seq_last; - if (ctrack->seq_last != ctrack->req_seq) - { - DLOG("another request, not retransmission. stop tracking.\n"); - ctrack_stop_req_counter(ctrack); - return false; - } + if (!ctrack->req_seq_present) + return false; + if (!seq_within(ctrack->seq_last, ctrack->req_seq_start, ctrack->req_seq_end)) + { + DLOG("req retrans : tcp seq %u not within the req range %u-%u. stop tracking.\n", ctrack->seq_last, ctrack->req_seq_start, ctrack->req_seq_end); + ctrack_stop_retrans_counter(ctrack); + ctrack->req_seq_present = false; + return false; + } } + ctrack->req_retrans_counter++; + if (ctrack->req_retrans_counter >= threshold) + { + DLOG("req retrans threshold reached : %u/%u\n",ctrack->req_retrans_counter, threshold); + ctrack_stop_retrans_counter(ctrack); + return true; + } + DLOG("req retrans counter : %u/%u\n",ctrack->req_retrans_counter, threshold); } return false; } @@ -227,7 +238,7 @@ static void auto_hostlist_failed(const char *hostname) HOSTLIST_DEBUGLOG_APPEND("%s : fail counter %d/%d", hostname, fail_counter->counter, params.hostlist_auto_fail_threshold); if (fail_counter->counter >= params.hostlist_auto_fail_threshold) { - DLOG("auto hostlist : fail threshold reached. adding %s to auto hostlist\n", hostname); + DLOG("auto hostlist : fail threshold reached. about to add %s to auto hostlist\n", hostname); HostFailPoolDel(¶ms.hostlist_auto_fail_counters, fail_counter); DLOG("auto hostlist : rechecking %s to avoid duplicates\n", hostname); @@ -255,7 +266,69 @@ static void auto_hostlist_failed(const char *hostname) } } -#define CONNTRACK_REQUIRED (params.wssize || params.desync_cutoff || *params.hostlist_auto_filename) +static void process_retrans_fail(t_ctrack *ctrack, uint8_t proto) +{ + if (ctrack && ctrack->hostname && auto_hostlist_retrans(ctrack, proto, params.hostlist_auto_retrans_threshold)) + { + HOSTLIST_DEBUGLOG_APPEND("%s : tcp retrans threshold reached", ctrack->hostname); + auto_hostlist_failed(ctrack->hostname); + } +} + +static bool reasm_start(t_ctrack *ctrack, t_reassemble *reasm, size_t sz, size_t szMax, const uint8_t *data_payload, size_t len_payload) +{ + ReasmClear(reasm); + if (sz<=szMax) + { + if (ReasmInit(reasm,sz,ctrack->seq_last)) + { + ReasmFeed(reasm,ctrack->seq_last,data_payload,len_payload); + DLOG("starting reassemble. now we have %zu/%zu\n",reasm->size_present,reasm->size); + return true; + } + else + DLOG("reassemble init failed. out of memory\n"); + } + else + DLOG("unexpected large payload for reassemble: size=%zu\n",sz); + return false; +} +static bool reasm_orig_start(t_ctrack *ctrack, size_t sz, size_t szMax, const uint8_t *data_payload, size_t len_payload) +{ + return reasm_start(ctrack,&ctrack->reasm_orig,sz,szMax,data_payload,len_payload); +} +static bool reasm_feed(t_ctrack *ctrack, t_reassemble *reasm, const uint8_t *data_payload, size_t len_payload) +{ + if (ctrack && !ReasmIsEmpty(reasm)) + { + if (ReasmFeed(reasm,ctrack->seq_last,data_payload,len_payload)) + { + DLOG("reassemble : feeding data payload size=%zu. now we have %zu/%zu\n",len_payload,reasm->size_present,reasm->size) + return true; + } + else + { + ReasmClear(reasm); + DLOG("reassemble session failed\n") + } + } + return false; +} +static bool reasm_orig_feed(t_ctrack *ctrack, const uint8_t *data_payload, size_t len_payload) +{ + return reasm_feed(ctrack, &ctrack->reasm_orig, data_payload, len_payload); +} +static void reasm_orig_fin(t_ctrack *ctrack) +{ + if (ctrack && ReasmIsFull(&ctrack->reasm_orig)) + { + DLOG("reassemble session finished\n"); + ReasmClear(&ctrack->reasm_orig); + } +} + + + // result : true - drop original packet, false = dont drop packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout, uint8_t *data_pkt, size_t len_pkt, struct ip *ip, struct ip6_hdr *ip6hdr, struct tcphdr *tcphdr, size_t len_tcp, uint8_t *data_payload, size_t len_payload) { @@ -271,13 +344,10 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout, if (!!ip == !!ip6hdr) return res; // one and only one must be present - if (CONNTRACK_REQUIRED) - { - ConntrackPoolPurge(¶ms.conntrack); - if (ConntrackPoolFeed(¶ms.conntrack, ip, ip6hdr, tcphdr, NULL, len_payload, &ctrack, &bReverse)) - maybe_cutoff(ctrack, IPPROTO_TCP); - HostFailPoolPurgeRateLimited(¶ms.hostlist_auto_fail_counters); - } + ConntrackPoolPurge(¶ms.conntrack); + if (ConntrackPoolFeed(¶ms.conntrack, ip, ip6hdr, tcphdr, NULL, len_payload, &ctrack, &bReverse)) + maybe_cutoff(ctrack, IPPROTO_TCP); + HostFailPoolPurgeRateLimited(¶ms.hostlist_auto_fail_counters); //ConntrackPoolDump(¶ms.conntrack); @@ -290,15 +360,16 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout, if (bReverse) { // process reply packets for auto hostlist mode - // by looking at RSTs or HTTP replies we decide whether original request looks to be blocked by DPI - if (*params.hostlist_auto_filename && ctrack && ctrack->hostname && ctrack->req_retrans_counter != RETRANS_COUNTER_STOP) + // by looking at RSTs or HTTP replies we decide whether original request looks like DPI blocked + // we only process first-sequence replies. do not react to subsequent redirects or RSTs + if (*params.hostlist_auto_filename && ctrack && ctrack->hostname && (ctrack->ack_last-ctrack->ack0)==1) { - bool bFail=false, bStop=false; + bool bFail=false; if (tcphdr->th_flags & TH_RST) { DLOG("incoming RST detected for hostname %s\n", ctrack->hostname); HOSTLIST_DEBUGLOG_APPEND("%s : incoming RST", ctrack->hostname); - bFail = bStop = true; + bFail = true; } else if (len_payload && ctrack->l7proto==HTTP) { @@ -319,12 +390,11 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout, // received not http reply. do not monitor this connection anymore DLOG("incoming unknown HTTP data detected for hostname %s\n", ctrack->hostname); } - bStop = true; } if (bFail) auto_hostlist_failed(ctrack->hostname); - if (bStop) - ctrack_stop_req_counter(ctrack); + if (tcphdr->th_flags & TH_RST) + ConntrackClearHostname(ctrack); // do not react to further dup RSTs } return res; // nothing to do. do not waste cpu @@ -408,78 +478,119 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout, bool bIsHttp; bool bKnownProtocol = false; uint8_t *p, *phost; - if ((bIsHttp = IsHttp(data_payload,len_payload))) + const uint8_t *rdata_payload = data_payload; + size_t rlen_payload = len_payload; + + if (reasm_orig_feed(ctrack,data_payload,len_payload)) + { + rdata_payload = ctrack->reasm_orig.packet; + rlen_payload = ctrack->reasm_orig.size_present; + } + + if ((bIsHttp = IsHttp(rdata_payload,rlen_payload))) { DLOG("packet contains HTTP request\n") if (ctrack && !ctrack->l7proto) ctrack->l7proto = HTTP; - if (params.wssize) - { - DLOG("forced wssize-cutoff\n"); - wssize_cutoff(ctrack); - } + forced_wssize_cutoff(ctrack); fake = params.fake_http; fake_size = params.fake_http_size; - if (params.hostlist || params.debug) bHaveHost=HttpExtractHost(data_payload,len_payload,host,sizeof(host)); - if (params.hostlist && !bHaveHost) + if (params.hostlist || params.hostlist_exclude) { - DLOG("not applying tampering to HTTP without Host:\n") - return res; - } - bKnownProtocol = true; - } - else if (IsTLSClientHello(data_payload,len_payload)) - { - DLOG("packet contains TLS ClientHello\n") - if (ctrack && !ctrack->l7proto) ctrack->l7proto = TLS; - if (params.wssize) - { - DLOG("forced wssize-cutoff\n"); - wssize_cutoff(ctrack); - } - fake = params.fake_tls; - fake_size = params.fake_tls_size; - if (params.hostlist || params.desync_skip_nosni || params.debug) - { - bHaveHost=TLSHelloExtractHost(data_payload,len_payload,host,sizeof(host)); - if (params.desync_skip_nosni && !bHaveHost) + bHaveHost=HttpExtractHost(rdata_payload,rlen_payload,host,sizeof(host)); + if (!bHaveHost) { - DLOG("not applying tampering to TLS ClientHello without hostname in the SNI\n") + DLOG("not applying tampering to HTTP without Host:\n") + process_retrans_fail(ctrack, IPPROTO_TCP); + reasm_orig_fin(ctrack); return res; } } + if (ctrack) + { + // we do not reassemble http + if (!ctrack->req_seq_present) + { + ctrack->req_seq_start=ctrack->seq_last; + ctrack->req_seq_end=ctrack->pos_orig-1; + ctrack->req_seq_start_present=ctrack->req_seq_present=true; + DLOG("req retrans : tcp seq interval %u-%u\n",ctrack->req_seq_start,ctrack->req_seq_end); + } + } bKnownProtocol = true; } - else + else if (IsTLSClientHello(rdata_payload,rlen_payload,TLS_PARTIALS_ENABLE)) + { + bool bReqFull = IsTLSRecordFull(rdata_payload,rlen_payload); + DLOG(bReqFull ? "packet contains full TLS ClientHello\n" : "packet contains partial TLS ClientHello\n") + fake = params.fake_tls; + fake_size = params.fake_tls_size; + bHaveHost=TLSHelloExtractHost(rdata_payload,rlen_payload,host,sizeof(host),TLS_PARTIALS_ENABLE); + if (ctrack) + { + if (!ctrack->l7proto) ctrack->l7proto = TLS; + if (!bReqFull && ReasmIsEmpty(&ctrack->reasm_orig)) + // do not reconstruct unexpected large payload (they are feeding garbage ?) + reasm_orig_start(ctrack,TLSRecordLen(data_payload),4096,data_payload,len_payload); + if (!ctrack->req_seq_start_present) + { + // lower bound of request seq interval + ctrack->req_seq_start=ctrack->seq_last; + ctrack->req_seq_start_present=true; + } + if (!ctrack->req_seq_present && bReqFull) + { + // upper bound of request seq interval + ctrack->req_seq_end=ctrack->pos_orig-1; + ctrack->req_seq_present=ctrack->req_seq_start_present; + DLOG("req retrans : seq interval %u-%u\n",ctrack->req_seq_start,ctrack->req_seq_end); + } + } + if (bReqFull || !ctrack || ReasmIsEmpty(&ctrack->reasm_orig)) forced_wssize_cutoff(ctrack); + + if (params.desync_skip_nosni && !bHaveHost) + { + DLOG("not applying tampering to TLS ClientHello without hostname in the SNI\n") + process_retrans_fail(ctrack, IPPROTO_TCP); + reasm_orig_fin(ctrack); + return res; + } + bKnownProtocol = true; + } + + reasm_orig_fin(ctrack); + rdata_payload=NULL; + + if (bHaveHost) + { + bool bExcluded; + DLOG("hostname: %s\n",host) + if ((params.hostlist || params.hostlist_exclude) && !HostlistCheck(params.hostlist, params.hostlist_exclude, host, &bExcluded)) + { + DLOG("not applying tampering to this request\n") + if (ctrack) + { + if (!bExcluded && *params.hostlist_auto_filename) + { + if (!ctrack->hostname) ctrack->hostname=strdup(host); + process_retrans_fail(ctrack, IPPROTO_TCP); + } + else + ctrack_stop_retrans_counter(ctrack); + } + return res; + } + ctrack_stop_retrans_counter(ctrack); + } + process_retrans_fail(ctrack, IPPROTO_TCP); + + if (!bKnownProtocol) { - // received unknown payload. it means we are out of the request retransmission phase. stop counter - ctrack_stop_req_counter(ctrack); - if (!params.desync_any_proto) return res; DLOG("applying tampering to unknown protocol\n") fake = params.fake_unknown; fake_size = params.fake_unknown_size; } - if (bHaveHost) - { - DLOG("hostname: %s\n",host) - bool bExcluded; - if ((params.hostlist || params.hostlist_exclude) && !HostlistCheck(params.hostlist, params.hostlist_exclude, host, &bExcluded)) - { - DLOG("not applying tampering to this request\n") - if (!bExcluded && *params.hostlist_auto_filename && ctrack) - { - if (!ctrack->hostname) ctrack->hostname=strdup(host); - if (auto_hostlist_retrans(ctrack, IPPROTO_TCP, params.hostlist_auto_retrans_threshold)) - { - HOSTLIST_DEBUGLOG_APPEND("%s : tcp retrans threshold reached", ctrack->hostname); - auto_hostlist_failed(host); - } - } - return res; - } - } - if (bIsHttp && (params.hostcase || params.hostnospace || params.domcase) && (phost = (uint8_t*)memmem(data_payload, len_payload, "\r\nHost: ", 8))) { if (params.hostcase) @@ -773,6 +884,7 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout, return frag; } } + } return res; @@ -793,13 +905,10 @@ packet_process_result dpi_desync_udp_packet(uint32_t fwmark, const char *ifout, if (!!ip == !!ip6hdr) return res; // one and only one must be present - if (CONNTRACK_REQUIRED) - { - ConntrackPoolPurge(¶ms.conntrack); - if (ConntrackPoolFeed(¶ms.conntrack, ip, ip6hdr, NULL, udphdr, len_payload, &ctrack, &bReverse)) - maybe_cutoff(ctrack, IPPROTO_UDP); - HostFailPoolPurgeRateLimited(¶ms.hostlist_auto_fail_counters); - } + ConntrackPoolPurge(¶ms.conntrack); + if (ConntrackPoolFeed(¶ms.conntrack, ip, ip6hdr, NULL, udphdr, len_payload, &ctrack, &bReverse)) + maybe_cutoff(ctrack, IPPROTO_UDP); + HostFailPoolPurgeRateLimited(¶ms.hostlist_auto_fail_counters); //ConntrackPoolDump(¶ms.conntrack); @@ -883,7 +992,7 @@ packet_process_result dpi_desync_udp_packet(uint32_t fwmark, const char *ifout, else { // received payload without host. it means we are out of the request retransmission phase. stop counter - ctrack_stop_req_counter(ctrack); + ctrack_stop_retrans_counter(ctrack); if (IsWireguardHandshakeInitiation(data_payload,len_payload)) { @@ -920,11 +1029,7 @@ packet_process_result dpi_desync_udp_packet(uint32_t fwmark, const char *ifout, if (!bExcluded && *params.hostlist_auto_filename && ctrack) { if (!ctrack->hostname) ctrack->hostname=strdup(host); - if (auto_hostlist_retrans(ctrack, IPPROTO_UDP, params.hostlist_auto_retrans_threshold)) - { - HOSTLIST_DEBUGLOG_APPEND("%s : udp retrans threshold reached", ctrack->hostname); - auto_hostlist_failed(host); - } + process_retrans_fail(ctrack, IPPROTO_UDP); } return res; } diff --git a/nfq/helpers.c b/nfq/helpers.c index a52bf0d..72b902d 100644 --- a/nfq/helpers.c +++ b/nfq/helpers.c @@ -5,6 +5,7 @@ #include #include #include +#include void hexdump_limited_dlog(const uint8_t *data, size_t size, size_t limit) { @@ -149,12 +150,12 @@ void dbgprint_socket_buffers(int fd) bool set_socket_buffers(int fd, int rcvbuf, int sndbuf) { DLOG("set_socket_buffers fd=%d rcvbuf=%d sndbuf=%d\n", fd, rcvbuf, sndbuf) - if (rcvbuf && setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(int)) < 0) - { - perror("setsockopt (SO_RCVBUF)"); - close(fd); - return false; - } + if (rcvbuf && setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(int)) < 0) + { + perror("setsockopt (SO_RCVBUF)"); + close(fd); + return false; + } if (sndbuf && setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof(int)) < 0) { perror("setsockopt (SO_SNDBUF)"); @@ -188,6 +189,11 @@ void phton64(uint8_t *p, uint64_t v) p[7] = (uint8_t)(v >> 0); } +bool seq_within(uint32_t s, uint32_t s1, uint32_t s2) +{ + return s2>=s1 && s>=s1 && s<=s2 || s2=s1); +} + bool ipv6_addr_is_zero(const struct in6_addr *a) { return !memcmp(a,"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",16); diff --git a/nfq/helpers.h b/nfq/helpers.h index fa7d003..c2d8762 100644 --- a/nfq/helpers.h +++ b/nfq/helpers.h @@ -5,6 +5,7 @@ #include #include #include +#include #include "params.h" @@ -19,6 +20,8 @@ void print_sockaddr(const struct sockaddr *sa); void ntop46(const struct sockaddr *sa, char *str, size_t len); void ntop46_port(const struct sockaddr *sa, char *str, size_t len); +bool seq_within(uint32_t s, uint32_t s1, uint32_t s2); + void dbgprint_socket_buffers(int fd); bool set_socket_buffers(int fd, int rcvbuf, int sndbuf); diff --git a/nfq/params.h b/nfq/params.h index 5b10332..8e08d9a 100644 --- a/nfq/params.h +++ b/nfq/params.h @@ -1,6 +1,5 @@ #pragma once -#include "params.h" #include "pools.h" #include "conntrack.h" #include "desync.h" @@ -12,6 +11,8 @@ #include #include +#define TLS_PARTIALS_ENABLE true + #define Q_RCVBUF (128*1024) // in bytes #define Q_SNDBUF (64*1024) // in bytes #define RAW_SNDBUF (64*1024) // in bytes diff --git a/nfq/protocol.c b/nfq/protocol.c index f9a6793..69eca78 100644 --- a/nfq/protocol.c +++ b/nfq/protocol.c @@ -108,12 +108,25 @@ bool HttpReplyLooksLikeDPIRedirect(const uint8_t *data, size_t len, const char * } - -bool IsTLSClientHello(const uint8_t *data, size_t len) +uint16_t TLSRecordDataLen(const uint8_t *data) { - return len >= 6 && data[0] == 0x16 && data[1] == 0x03 && data[2] >= 0x01 && data[2] <= 0x03 && data[5] == 0x01 && (pntoh16(data + 3) + 5) <= len; + return pntoh16(data + 3); } -bool TLSFindExtInHandshake(const uint8_t *data, size_t len, uint16_t type, const uint8_t **ext, size_t *len_ext) +size_t TLSRecordLen(const uint8_t *data) +{ + return TLSRecordDataLen(data) + 5; +} +bool IsTLSRecordFull(const uint8_t *data, size_t len) +{ + return TLSRecordLen(data)<=len; +} +bool IsTLSClientHello(const uint8_t *data, size_t len, bool bPartialIsOK) +{ + return len >= 6 && data[0] == 0x16 && data[1] == 0x03 && data[2] >= 0x01 && data[2] <= 0x03 && data[5] == 0x01 && (bPartialIsOK || TLSRecordLen(data) <= len); +} + +// bPartialIsOK=true - accept partial packets not containing the whole TLS message +bool TLSFindExtInHandshake(const uint8_t *data, size_t len, uint16_t type, const uint8_t **ext, size_t *len_ext, bool bPartialIsOK) { // +0 // u8 HandshakeType: ClientHello @@ -133,8 +146,11 @@ bool TLSFindExtInHandshake(const uint8_t *data, size_t len, uint16_t type, const l = 1 + 3 + 2 + 32; // SessionIDLength if (len < (l + 1)) return false; - ll = data[1] << 16 | data[2] << 8 | data[3]; // HandshakeProtocol length - if (len < (ll + 4)) return false; + if (!bPartialIsOK) + { + ll = data[1] << 16 | data[2] << 8 | data[3]; // HandshakeProtocol length + if (len < (ll + 4)) return false; + } l += data[l] + 1; // CipherSuitesLength if (len < (l + 2)) return false; @@ -148,7 +164,15 @@ bool TLSFindExtInHandshake(const uint8_t *data, size_t len, uint16_t type, const data += l; len -= l; l = pntoh16(data); data += 2; len -= 2; - if (len < l) return false; + + if (bPartialIsOK) + { + if (len < l) l = len; + } + else + { + if (len < l) return false; + } while (l >= 4) { @@ -170,14 +194,14 @@ bool TLSFindExtInHandshake(const uint8_t *data, size_t len, uint16_t type, const return false; } -bool TLSFindExt(const uint8_t *data, size_t len, uint16_t type, const uint8_t **ext, size_t *len_ext) +bool TLSFindExt(const uint8_t *data, size_t len, uint16_t type, const uint8_t **ext, size_t *len_ext, bool bPartialIsOK) { // +0 // u8 ContentType: Handshake // u16 Version: TLS1.0 // u16 Length - if (!IsTLSClientHello(data, len)) return false; - return TLSFindExtInHandshake(data + 5, len - 5, type, ext, len_ext); + if (!IsTLSClientHello(data, len, bPartialIsOK)) return false; + return TLSFindExtInHandshake(data + 5, len - 5, type, ext, len_ext, bPartialIsOK); } static bool TLSExtractHostFromExt(const uint8_t *ext, size_t elen, char *host, size_t len_host) { @@ -196,20 +220,20 @@ static bool TLSExtractHostFromExt(const uint8_t *ext, size_t elen, char *host, s } return true; } -bool TLSHelloExtractHost(const uint8_t *data, size_t len, char *host, size_t len_host) +bool TLSHelloExtractHost(const uint8_t *data, size_t len, char *host, size_t len_host, bool bPartialIsOK) { const uint8_t *ext; size_t elen; - if (!TLSFindExt(data, len, 0, &ext, &elen)) return false; + if (!TLSFindExt(data, len, 0, &ext, &elen, bPartialIsOK)) return false; return TLSExtractHostFromExt(ext, elen, host, len_host); } -bool TLSHelloExtractHostFromHandshake(const uint8_t *data, size_t len, char *host, size_t len_host) +bool TLSHelloExtractHostFromHandshake(const uint8_t *data, size_t len, char *host, size_t len_host, bool bPartialIsOK) { const uint8_t *ext; size_t elen; - if (!TLSFindExtInHandshake(data, len, 0, &ext, &elen)) return false; + if (!TLSFindExtInHandshake(data, len, 0, &ext, &elen, bPartialIsOK)) return false; return TLSExtractHostFromExt(ext, elen, host, len_host); } @@ -580,7 +604,7 @@ bool QUICExtractHostFromInitial(const uint8_t *data, size_t data_len, char *host if (!IsQUICCryptoHello(defrag, defrag_len, &hello_offset, &hello_len)) return false; if (bIsCryptoHello) *bIsCryptoHello=true; - return TLSHelloExtractHostFromHandshake(defrag + hello_offset, hello_len, host, len_host); + return TLSHelloExtractHostFromHandshake(defrag + hello_offset, hello_len, host, len_host, true); } bool IsQUICInitial(const uint8_t *data, size_t len) diff --git a/nfq/protocol.h b/nfq/protocol.h index 4c70def..d915eec 100644 --- a/nfq/protocol.h +++ b/nfq/protocol.h @@ -5,6 +5,7 @@ #include #include "crypto/sha.h" #include "crypto/aes-gcm.h" +#include "helpers.h" bool IsHttp(const uint8_t *data, size_t len); // header must be passed like this : "\nHost:" @@ -17,11 +18,14 @@ int HttpReplyCode(const uint8_t *data, size_t len); // must be pre-checked by IsHttpReply bool HttpReplyLooksLikeDPIRedirect(const uint8_t *data, size_t len, const char *host); -bool IsTLSClientHello(const uint8_t *data, size_t len); -bool TLSFindExt(const uint8_t *data, size_t len, uint16_t type, const uint8_t **ext, size_t *len_ext); -bool TLSFindExtInHandshake(const uint8_t *data, size_t len, uint16_t type, const uint8_t **ext, size_t *len_ext); -bool TLSHelloExtractHost(const uint8_t *data, size_t len, char *host, size_t len_host); -bool TLSHelloExtractHostFromHandshake(const uint8_t *data, size_t len, char *host, size_t len_host); +uint16_t TLSRecordDataLen(const uint8_t *data); +size_t TLSRecordLen(const uint8_t *data); +bool IsTLSRecordFull(const uint8_t *data, size_t len); +bool IsTLSClientHello(const uint8_t *data, size_t len, bool bPartialIsOK); +bool TLSFindExt(const uint8_t *data, size_t len, uint16_t type, const uint8_t **ext, size_t *len_ext, bool bPartialIsOK); +bool TLSFindExtInHandshake(const uint8_t *data, size_t len, uint16_t type, const uint8_t **ext, size_t *len_ext, bool bPartialIsOK); +bool TLSHelloExtractHost(const uint8_t *data, size_t len, char *host, size_t len_host, bool bPartialIsOK); +bool TLSHelloExtractHostFromHandshake(const uint8_t *data, size_t len, char *host, size_t len_host, bool bPartialIsOK); bool IsWireguardHandshakeInitiation(const uint8_t *data, size_t len); bool IsDhtD1(const uint8_t *data, size_t len);