Merge branch 'bol-van:master' into tpws-minimal

binaries/win64/zapret-winws/ipset-discord.txt

@@ -0,0 +1,721 @@
+5.200.14.249
+18.165.140.0/25
+23.227.38.74
+34.0.48.0/24
+34.0.49.64/26
+34.0.50.0/25
+34.0.51.0/24
+34.0.52.0/22
+34.0.56.0/23
+34.0.59.0/24
+34.0.60.0/24
+34.0.62.128/25
+34.0.63.228
+34.0.64.0/23
+34.0.66.130
+34.0.82.140
+34.0.129.128/25
+34.0.130.0/24
+34.0.131.130
+34.0.132.139
+34.0.133.75
+34.0.134.0/24
+34.0.135.251
+34.0.136.51
+34.0.137.0/24
+34.0.139.0/24
+34.0.140.0/23
+34.0.142.0/25
+34.0.144.0/23
+34.0.146.0/24
+34.0.148.25
+34.0.149.101
+34.0.151.0/25
+34.0.153.0/24
+34.0.155.0/24
+34.0.156.101
+34.0.157.0/25
+34.0.158.247
+34.0.159.188
+34.0.192.0/25
+34.0.193.0/24
+34.0.194.0/24
+34.0.195.172
+34.0.196.200/29
+34.0.197.81
+34.0.198.25
+34.0.199.0/24
+34.0.200.0/24
+34.0.201.81
+34.0.202.34
+34.0.203.0/24
+34.0.204.0/23
+34.0.206.0/25
+34.0.207.0/25
+34.0.208.195
+34.0.209.0/24
+34.0.210.20
+34.0.211.0/26
+34.0.212.0/24
+34.0.213.64/26
+34.0.215.128/25
+34.0.216.238
+34.0.217.0/24
+34.0.218.83
+34.0.220.103
+34.0.221.0/24
+34.0.222.193
+34.0.223.68
+34.0.227.0/24
+34.0.240.0/21
+34.0.248.0/23
+34.0.250.0/24
+34.0.251.0/25
+34.1.216.0/24
+34.1.221.166
+35.207.64.0/23
+35.207.67.116
+35.207.71.0/24
+35.207.72.32
+35.207.73.0/24
+35.207.74.0/24
+35.207.75.128/25
+35.207.76.128/26
+35.207.77.0/24
+35.207.78.129
+35.207.79.0/24
+35.207.80.76
+35.207.81.248/30
+35.207.82.0/23
+35.207.84.0/24
+35.207.85.160
+35.207.86.41
+35.207.87.184
+35.207.89.188
+35.207.91.146
+35.207.92.230
+35.207.95.0/24
+35.207.97.174
+35.207.99.134
+35.207.100.64/26
+35.207.101.130
+35.207.103.64/26
+35.207.104.0/24
+35.207.106.128/26
+35.207.107.19
+35.207.108.192/27
+35.207.109.185
+35.207.110.0/24
+35.207.111.174
+35.207.114.16
+35.207.115.163
+35.207.116.51
+35.207.117.0/24
+35.207.121.204
+35.207.122.0/25
+35.207.124.145
+35.207.125.116
+35.207.126.30
+35.207.129.0/24
+35.207.131.128/27
+35.207.132.247
+35.207.135.147
+35.207.136.69
+35.207.137.0/24
+35.207.139.0/24
+35.207.140.241
+35.207.141.119
+35.207.142.0/24
+35.207.143.96/27
+35.207.144.0/25
+35.207.145.0/24
+35.207.146.89
+35.207.147.0/24
+35.207.149.0/24
+35.207.150.0/24
+35.207.151.61
+35.207.153.117
+35.207.154.0/24
+35.207.155.128/25
+35.207.156.254
+35.207.157.7
+35.207.158.192
+35.207.160.160
+35.207.162.239
+35.207.163.0/24
+35.207.164.0/25
+35.207.165.147
+35.207.166.0/25
+35.207.167.0/24
+35.207.168.116
+35.207.170.0/23
+35.207.172.0/24
+35.207.174.55
+35.207.176.128/25
+35.207.178.0/24
+35.207.180.152
+35.207.181.76
+35.207.182.125
+35.207.184.101
+35.207.185.192
+35.207.186.128/25
+35.207.187.228
+35.207.188.0/24
+35.207.189.0/25
+35.207.190.194
+35.207.191.64/26
+35.207.193.165
+35.207.195.75
+35.207.196.0/24
+35.207.198.0/23
+35.207.201.186
+35.207.202.169
+35.207.205.211
+35.207.207.4
+35.207.209.0/25
+35.207.210.191
+35.207.211.253
+35.207.213.97
+35.207.214.0/24
+35.207.220.147
+35.207.221.58
+35.207.222.105
+35.207.224.151
+35.207.225.210
+35.207.227.0/24
+35.207.229.212
+35.207.232.26
+35.207.234.182
+35.207.238.0/24
+35.207.240.0/24
+35.207.245.0/24
+35.207.249.0/24
+35.207.250.212
+35.207.251.0/27
+35.212.4.134
+35.212.12.148
+35.212.88.11
+35.212.102.50
+35.212.111.0/26
+35.212.117.247
+35.212.120.122
+35.213.0.0/24
+35.213.2.8
+35.213.4.185
+35.213.6.118
+35.213.7.128/25
+35.213.8.168
+35.213.10.0/24
+35.213.11.21
+35.213.12.224/27
+35.213.13.19
+35.213.14.217
+35.213.16.67
+35.213.17.235
+35.213.23.166
+35.213.25.164
+35.213.26.62
+35.213.27.252
+35.213.32.0/24
+35.213.33.74
+35.213.34.204
+35.213.37.81
+35.213.38.186
+35.213.39.253
+35.213.42.0/24
+35.213.43.79
+35.213.45.0/24
+35.213.46.136
+35.213.49.17
+35.213.50.0/24
+35.213.51.213
+35.213.52.0/25
+35.213.53.0/24
+35.213.54.0/24
+35.213.56.0/25
+35.213.59.0/24
+35.213.61.58
+35.213.65.0/24
+35.213.67.0/24
+35.213.68.192/26
+35.213.70.151
+35.213.72.128/25
+35.213.73.245
+35.213.74.131
+35.213.78.0/24
+35.213.79.137
+35.213.80.0/25
+35.213.83.128/25
+35.213.84.245
+35.213.85.0/24
+35.213.88.145
+35.213.89.80/28
+35.213.90.0/24
+35.213.91.195
+35.213.92.0/24
+35.213.93.254
+35.213.94.78
+35.213.95.145
+35.213.96.87
+35.213.98.0/24
+35.213.99.126
+35.213.101.214
+35.213.102.0/24
+35.213.105.0/24
+35.213.106.128/25
+35.213.107.158
+35.213.109.0/24
+35.213.110.40
+35.213.111.0/25
+35.213.115.0/25
+35.213.120.0/24
+35.213.122.0/24
+35.213.124.89
+35.213.125.40
+35.213.126.185
+35.213.127.0/24
+35.213.128.0/22
+35.213.132.0/23
+35.213.134.140
+35.213.135.0/24
+35.213.136.0/23
+35.213.138.128/25
+35.213.139.0/24
+35.213.140.0/25
+35.213.141.164
+35.213.142.128/25
+35.213.143.0/24
+35.213.144.0/22
+35.213.148.0/23
+35.213.150.0/24
+35.213.152.0/23
+35.213.154.137
+35.213.155.134
+35.213.156.144
+35.213.157.0/24
+35.213.158.64/26
+35.213.160.90
+35.213.161.253
+35.213.162.0/25
+35.213.163.0/24
+35.213.164.0/23
+35.213.166.106
+35.213.167.160/27
+35.213.168.0/24
+35.213.169.179
+35.213.170.0/24
+35.213.171.201
+35.213.172.159
+35.213.173.0/24
+35.213.174.128/25
+35.213.175.128/26
+35.213.176.0/24
+35.213.177.0/25
+35.213.179.139
+35.213.180.0/24
+35.213.181.0/25
+35.213.182.0/23
+35.213.184.0/23
+35.213.186.70
+35.213.187.0/24
+35.213.188.128/25
+35.213.190.158
+35.213.191.0/24
+35.213.192.240/31
+35.213.193.74
+35.213.194.0/25
+35.213.195.178
+35.213.196.38
+35.213.197.68
+35.213.198.0/23
+35.213.200.0/23
+35.213.202.0/25
+35.213.203.195
+35.213.204.32/27
+35.213.205.170
+35.213.207.128/25
+35.213.208.85
+35.213.210.0/24
+35.213.211.176/29
+35.213.212.0/24
+35.213.213.225
+35.213.214.0/25
+35.213.215.255
+35.213.217.0/24
+35.213.218.248
+35.213.219.0/25
+35.213.220.211
+35.213.221.0/24
+35.213.222.215
+35.213.223.0/24
+35.213.225.0/24
+35.213.227.227
+35.213.229.17
+35.213.230.89
+35.213.231.0/24
+35.213.233.0/24
+35.213.234.134
+35.213.236.0/24
+35.213.237.212
+35.213.238.0/24
+35.213.240.212
+35.213.241.0/24
+35.213.242.10
+35.213.243.219
+35.213.244.146
+35.213.245.119
+35.213.246.0/23
+35.213.249.79
+35.213.250.0/24
+35.213.251.74
+35.213.252.0/24
+35.213.253.155
+35.213.254.89
+35.214.128.248
+35.214.129.220
+35.214.130.217
+35.214.131.144
+35.214.132.189
+35.214.133.0/24
+35.214.134.163
+35.214.137.0/24
+35.214.138.0/25
+35.214.140.0/24
+35.214.142.0/24
+35.214.143.41
+35.214.144.26
+35.214.145.200
+35.214.146.9
+35.214.147.135
+35.214.148.89
+35.214.149.110
+35.214.151.128/25
+35.214.152.0/24
+35.214.156.115
+35.214.158.181
+35.214.159.128/25
+35.214.160.128/25
+35.214.161.217
+35.214.162.0/24
+35.214.163.28
+35.214.165.102
+35.214.167.77
+35.214.169.0/24
+35.214.170.2
+35.214.171.0/25
+35.214.172.128/25
+35.214.173.0/24
+35.214.175.0/24
+35.214.177.183
+35.214.179.46
+35.214.180.0/23
+35.214.184.179
+35.214.185.28
+35.214.186.3
+35.214.187.0/24
+35.214.191.0/24
+35.214.192.128/25
+35.214.193.0/24
+35.214.194.128/25
+35.214.195.0/25
+35.214.196.64/26
+35.214.197.0/24
+35.214.198.7
+35.214.199.224
+35.214.201.0/25
+35.214.203.155
+35.214.204.0/23
+35.214.207.0/24
+35.214.208.128/25
+35.214.209.64
+35.214.210.0/24
+35.214.211.3
+35.214.212.64/26
+35.214.213.0/25
+35.214.214.0/24
+35.214.215.64/26
+35.214.216.0/23
+35.214.218.140
+35.214.219.0/24
+35.214.220.149
+35.214.221.0/24
+35.214.222.149
+35.214.223.0/24
+35.214.224.71
+35.214.225.0/24
+35.214.226.0/23
+35.214.228.0/23
+35.214.231.187
+35.214.233.8
+35.214.235.38
+35.214.237.0/24
+35.214.238.0/25
+35.214.239.0/24
+35.214.240.87
+35.214.241.0/24
+35.214.243.21
+35.214.244.0/24
+35.214.245.16/28
+35.214.246.106
+35.214.248.119
+35.214.249.154
+35.214.250.0/24
+35.214.251.128/25
+35.214.252.187
+35.214.253.0/24
+35.214.255.154
+35.215.72.85
+35.215.73.65
+35.215.83.0
+35.215.108.111
+35.215.115.120
+35.215.126.35
+35.215.127.34
+35.215.128.0/21
+35.215.136.0/26
+35.215.137.0/24
+35.215.138.0/23
+35.215.140.0/24
+35.215.141.64/27
+35.215.142.0/24
+35.215.143.83
+35.215.144.128/25
+35.215.145.0/24
+35.215.146.0/24
+35.215.147.86
+35.215.148.0/23
+35.215.150.0/26
+35.215.151.0/24
+35.215.152.0/24
+35.215.153.128/25
+35.215.154.240/28
+35.215.155.20
+35.215.156.0/24
+35.215.158.0/23
+35.215.160.192/26
+35.215.161.0/24
+35.215.163.0/24
+35.215.164.0/24
+35.215.165.236
+35.215.166.128/25
+35.215.167.128/25
+35.215.168.0/24
+35.215.169.12
+35.215.170.0/23
+35.215.172.0/22
+35.215.176.0/24
+35.215.177.72
+35.215.178.0/24
+35.215.179.161
+35.215.180.0/22
+35.215.184.253
+35.215.185.64/26
+35.215.186.0/25
+35.215.187.0/24
+35.215.188.0/23
+35.215.190.0/24
+35.215.191.61
+35.215.192.0/23
+35.215.194.192/28
+35.215.195.0/24
+35.215.196.0/25
+35.215.197.0/25
+35.215.198.230
+35.215.199.204
+35.215.200.0/23
+35.215.202.0/24
+35.215.203.0/25
+35.215.204.128/25
+35.215.205.0/25
+35.215.206.0/23
+35.215.208.0/24
+35.215.209.0/25
+35.215.210.0/23
+35.215.212.0/22
+35.215.216.0/22
+35.215.221.0/24
+35.215.222.128/25
+35.215.223.126
+35.215.224.0/23
+35.215.226.0/24
+35.215.227.0/25
+35.215.228.0/24
+35.215.229.64
+35.215.230.89
+35.215.231.0/24
+35.215.232.0/24
+35.215.233.0/25
+35.215.234.37
+35.215.235.0/24
+35.215.238.0/25
+35.215.239.119
+35.215.240.0/24
+35.215.241.128/25
+35.215.242.0/25
+35.215.243.0/24
+35.215.244.0/23
+35.215.246.222
+35.215.247.0/24
+35.215.248.0/22
+35.215.252.0/24
+35.215.253.118
+35.215.254.0/23
+35.217.0.0/24
+35.217.1.64/26
+35.217.2.5
+35.217.3.0/24
+35.217.4.72
+35.217.5.0/25
+35.217.6.0/24
+35.217.8.0/25
+35.217.9.0/24
+35.217.11.186
+35.217.12.0/24
+35.217.14.192/26
+35.217.15.65
+35.217.16.75
+35.217.17.128/25
+35.217.18.0/24
+35.217.19.183
+35.217.20.0/24
+35.217.21.128/25
+35.217.22.128/25
+35.217.23.128/25
+35.217.24.0/24
+35.217.25.81
+35.217.26.0/24
+35.217.27.128/25
+35.217.28.128/25
+35.217.29.0/24
+35.217.30.0/25
+35.217.31.0/25
+35.217.32.128/25
+35.217.33.0/24
+35.217.35.128/25
+35.217.36.0/23
+35.217.38.179
+35.217.39.186
+35.217.40.176
+35.217.41.204
+35.217.43.0/24
+35.217.45.248
+35.217.46.0/24
+35.217.47.128/25
+35.217.48.195
+35.217.49.160/27
+35.217.50.0/25
+35.217.51.0/24
+35.217.52.117
+35.217.53.128/25
+35.217.54.0/25
+35.217.55.96/27
+35.217.56.6
+35.217.57.184
+35.217.58.0/24
+35.217.59.64/26
+35.217.60.0/24
+35.217.61.128/25
+35.217.62.0/24
+35.217.63.128/25
+35.219.225.149
+35.219.226.57
+35.219.227.0/24
+35.219.228.37
+35.219.229.128/25
+35.219.230.0/23
+35.219.235.0/24
+35.219.236.198
+35.219.238.115
+35.219.239.0/24
+35.219.241.0/24
+35.219.242.221
+35.219.243.191
+35.219.244.1
+35.219.245.0/24
+35.219.246.159
+35.219.247.0/26
+35.219.248.0/24
+35.219.249.126
+35.219.251.186
+35.219.252.0/23
+35.219.254.0/24
+64.233.161.207
+64.233.162.207
+64.233.163.207
+64.233.164.207
+64.233.165.207
+66.22.196.0/26
+66.22.197.0/24
+66.22.198.0/26
+66.22.199.0/24
+66.22.200.0/26
+66.22.202.0/26
+66.22.204.0/24
+66.22.206.0/24
+66.22.208.0/25
+66.22.210.0/26
+66.22.212.0/24
+66.22.214.0/24
+66.22.216.0/23
+66.22.220.0/25
+66.22.221.0/24
+66.22.222.0/23
+66.22.224.0/25
+66.22.225.0/26
+66.22.226.0/25
+66.22.227.0/25
+66.22.228.0/22
+66.22.233.0/24
+66.22.234.0/24
+66.22.236.0/23
+66.22.238.0/24
+66.22.240.0/22
+66.22.244.0/23
+66.22.248.0/24
+74.125.131.207
+74.125.205.207
+104.17.51.93
+104.17.117.93
+104.18.4.161
+104.18.5.161
+104.18.8.105
+104.18.9.105
+104.18.30.128
+104.18.31.128
+104.21.2.204
+104.21.25.51
+104.21.40.151
+104.21.59.128
+104.21.72.221
+104.21.82.160
+108.177.14.207
+138.128.140.240/28
+142.250.150.207
+142.251.1.207
+162.159.128.232/30
+162.159.129.232/30
+162.159.130.232/30
+162.159.133.232/30
+162.159.134.232/30
+162.159.135.232/30
+162.159.136.232/30
+162.159.137.232/30
+162.159.138.232/30
+172.65.202.19
+172.66.41.34
+172.66.42.222
+172.67.152.224/28
+172.67.155.163
+172.67.159.89
+172.67.177.131
+172.67.222.182
+173.194.73.207
+173.194.220.207
+173.194.221.207
+173.194.222.207
+188.114.96.2
+188.114.97.2
+188.114.98.224
+188.114.99.224
+204.11.56.48
+209.85.233.207

binaries/win64/zapret-winws/preset_russia.cmd

@@ -1,5 +1,7 @@
 start "zapret: http,https,quic" /min "%~dp0winws.exe" ^
---wf-tcp=80,443 --wf-udp=443 ^
+--wf-tcp=80,443 --wf-udp=443,50000-50099 ^
+--filter-udp=50000-50099 --ipset="%~dp0ipset-discord.txt" --dpi-desync=fake --dpi-desync-repeats=6 --dpi-desync-any-protocol --dpi-desync-cutoff=n2 --new ^
+--filter-udp=50000-50099 --new ^
 --filter-udp=443 --hostlist="%~dp0list-youtube.txt" --dpi-desync=fake --dpi-desync-repeats=11 --dpi-desync-fake-quic="%~dp0quic_initial_www_google_com.bin" --new ^
 --filter-udp=443 --dpi-desync=fake --dpi-desync-repeats=11 --new ^
 --filter-tcp=80 --dpi-desync=fake,split2 --dpi-desync-autottl=2 --dpi-desync-fooling=md5sig --new ^

binaries/win64/zapret-winws/preset_russia_autohostlist.cmd

@@ -1,5 +1,7 @@
 start "zapret: http,https,quic" /min "%~dp0winws.exe" ^
---wf-tcp=80,443 --wf-udp=443 ^
+--wf-tcp=80,443 --wf-udp=443,50000-50099 ^
+--filter-udp=50000-50099 --ipset="%~dp0ipset-discord.txt" --dpi-desync=fake --dpi-desync-repeats=6 --dpi-desync-any-protocol --dpi-desync-cutoff=n2 --new ^
+--filter-udp=50000-50099 --new ^
 --filter-udp=443 --hostlist="%~dp0list-youtube.txt" --dpi-desync=fake --dpi-desync-repeats=11 --dpi-desync-fake-quic="%~dp0quic_initial_www_google_com.bin" --new ^
 --filter-udp=443 --dpi-desync=fake --dpi-desync-repeats=11 --new ^
 --filter-tcp=80 --dpi-desync=fake,split2 --dpi-desync-autottl=2 --dpi-desync-fooling=md5sig --hostlist-auto="%~dp0autohostlist.txt" --new ^

binaries/win64/zapret-winws/reload_lists.cmd

@@ -0,0 +1 @@
+%~dp0killall -HUP winws

docs/changes.txt

@@ -329,3 +329,10 @@ v65:
 init.d: dynamic number allocation for dnum,tpws_port,qnum
 init.d: FW_EXTRA_PRE, FW_EXTRA_POST
 init.d: zapret_custom_firewall_nft_flush
+nfqws,tpws: l7proto and client ip:port info in autohostlist debug log
+nfqws,tpws: user mode ipset filter support
+nfqws,tpws: l7proto filter support
+tpws: fixed MSS apply in transparent mode
+nfqws: fixed autottl apply if desync profile changed
+tpws,nfqws: fixed 100% cpu hang on gzipped list with comments
+ipset: get_refilter_ipsum.sh , get_refilter_domain.sh

docs/readme.eng.md

@@ -211,6 +211,9 @@ nfqws takes the following parameters:
  --filter-l3=ipv4|ipv6                          ; L3 protocol filter. multiple comma separated values allowed.
  --filter-tcp=[~]port1[-port2]                  ; TCP port filter. ~ means negation. setting tcp and not setting udp filter denies udp.
  --filter-udp=[~]port1[-port2]                  ; UDP port filter. ~ means negation. setting udp and not setting tcp filter denies tcp.
+ --filter-l7=[http|tls|quic|wireguard|dht|unknown] ; L6-L7 protocol filter. multiple comma separated values allowed.
+ --ipset=<filename>                             ; ipset include filter (one ip/CIDR per line, ipv4 and ipv6 accepted, gzip supported, multiple ipsets allowed)
+ --ipset-exclude=<filename>                     ; ipset exclude filter (one ip/CIDR per line, ipv4 and ipv6 accepted, gzip supported, multiple ipsets allowed)
 ```
 
 The manipulation parameters can be combined in any way.
@@ -575,18 +578,18 @@ You need to use nftables instead with hook priority 101 or higher.
 `nfqws` can apply different strategies to different requests. It's done with multiple desync profiles.
 Profiles are delimited by the `--new` parameter. First profile is created automatically and does not require `--new`.
 Each profile has a filter. By default it's empty and profile matches any packet.
-Filter can have hard parameters : ip version and tcp/udp port range.
+Filter can have hard parameters : ip version, ipset and tcp/udp port range.
-Hard parameters are always identified unambiguously even on zero-phase when hostname is unknown yet.
+Hard parameters are always identified unambiguously even on zero-phase when hostname and L7 are unknown yet.
-Hostlist can also act as a filter. They can be combined with hard parameters.
+Hostlists can also act as a filter. They can be combined with hard parameters.
 When a packet comes profiles are matched from the first to the last until first filter condition match.
 Hard filter is matched first. If it does not match verification goes to the next profile.
-If a profile matches hard filter and has autohostlist it's selected immediately.
+If a profile matches hard filter , L7 filter and has autohostlist it's selected immediately.
-If a profile matches hard filter and has normal hostlist(s) and hostname is unknown yet verification goes to the next profile.
+If a profile matches hard filter , L7 filter and has normal hostlist(s) and hostname is unknown yet verification goes to the next profile.
 Otherwise profile hostlist(s) are checked for the hostname. If it matches profile is selected.
 Otherwise verification goes to the next profile.
 
-It's possible that before getting hostname connection is served by one profile and after
+It's possible that before knowing L7 and hostname connection is served by one profile and after
-hostname is revealed it's switched to another profile.
+this information is revealed it's switched to another profile.
 If you use 0-phase desync methods think carefully what can happen during strategy switch.
 Use `--debug` logging to understand better what `nfqws` does.
 
@@ -597,6 +600,9 @@ IMPORTANT : multiple strategies exist only for the case when it's not possible t
 Copy-pasting blockcheck results of different websites to multiple strategies lead to the mess.
 This way you may never unblock all resources and only confuse yourself.
 
+IMPORTANT : user-mode ipset implementation was not designed as a kernel version replacement. Kernel version is much more effective.
+It's for the systems that lack ipset support : Windows and Linux without nftables and ipset kernel modules (Android, for example).
+
 ## tpws
 
 tpws is transparent proxy.
@@ -637,6 +643,9 @@ tpws is transparent proxy.
  --new                          ; begin new strategy
  --filter-l3=ipv4|ipv6          ; L3 protocol filter. multiple comma separated values allowed.
  --filter-tcp=[~]port1[-port2]  ; TCP port filter. ~ means negation
+ --filter-l7=[http|tls|unknown] ; L6-L7 protocol filter. multiple comma separated values allowed.
+ --ipset=<filename>             ; ipset include filter (one ip/CIDR per line, ipv4 and ipv6 accepted, gzip supported, multiple ipsets allowed)
+ --ipset-exclude=<filename>     ; ipset exclude filter (one ip/CIDR per line, ipv4 and ipv6 accepted, gzip supported, multiple ipsets allowed)
 
  --hostlist=<filename>          ; only act on hosts in the list (one host per line, subdomains auto apply, gzip supported, multiple hostlists allowed)
  --hostlist-exclude=<filename>  ; do not act on hosts in the list (one host per line, subdomains auto apply, gzip supported, multiple hostlists allowed)

docs/readme.txt

@@ -1,4 +1,4 @@
-zapret v.64
+zapret v.65
 
 English
 -------
@@ -280,6 +280,9 @@ nfqws
  --filter-l3=ipv4|ipv6                          ; фильтр версии ip для текущей стратегии
  --filter-tcp=[~]port1[-port2]                  ; фильтр портов tcp для текущей стратегии. ~ означает инверсию. установка фильтра tcp и неустановка фильтра udp запрещает udp.
  --filter-udp=[~]port1[-port2]                  ; фильтр портов udp для текущей стратегии. ~ означает инверсию. установка фильтра udp и неустановка фильтра tcp запрещает udp.
+ --filter-l7=[http|tls|quic|wireguard|dht|unknown] ; фильтр протокола L6-L7. поддерживается несколько значений через запятую.
+ --ipset=<filename>             	        ; включающий ip list. на каждой строчке ip или cidr ipv4 или ipv6. поддерживается множество листов и gzip.
+ --ipset-exclude=<filename>                     ; исключающий ip list. на каждой строчке ip или cidr ipv4 или ipv6. поддерживается множество листов и gzip.
 
 Параметры манипуляции могут сочетаться в любых комбинациях.
 
@@ -664,20 +667,22 @@ nfqws способен по-разному реагировать на разл
 Профили разделяются в командной строке параметром --new. Первый профиль создается автоматически.
 Для него не нужно --new. Каждый профиль имеет фильтр. По умолчанию он пуст, то есть профиль удовлетворяет
 любым условиям.
-Фильтр может содержать жесткие параметры : версия ip протокола или порты tcp/udp.
+Фильтр может содержать жесткие параметры : версия ip протокола, ipset и порты tcp/udp.
-Они всегда однозначно идентифицируются даже на нулевой фазе десинхронизации, когда еще хост неизвестен.
+Они всегда однозначно идентифицируются даже на нулевой фазе десинхронизации, когда еще хост и L7 неизвестны.
-В качестве фильтра могут выступать и хост-листы. Они могут сочетаться с жесткими параметрами.
+В качестве мягкого фильтра могут выступать хост-листы и протокол прикладного уровня (l7).
+L7 протокол становится известен обычно после первого пакета с данными.
 При поступлении запроса идет проверка профилей в порядке от первого до последнего до
 достижения первого совпадения с фильтром.
 Жесткие параметры фильтра сверяются первыми. При несовпадении идет сразу же переход к следующему профилю.
-Если какой-то профиль удовлетворяет жесткому фильтру и содержит авто-хостлист, он выбирается сразу.
+Если какой-то профиль удовлетворяет жесткому фильтру и L7 фильтру и содержит авто-хостлист, он выбирается сразу.
-Если профиль удовлетворяет жесткому фильтру, для него задан хостлист, и у нас еще нет имени хоста, 
+Если профиль удовлетворяет жесткому фильтру и L7 фильтру, для него задан хостлист, и у нас еще нет имени хоста, 
 идет переход к следующему профилю. В противном случае идет проверка по хостлистам этого профиля.
 Если имя хоста удовлетворяет листам, выбирается этот профиль. Иначе идет переход к следующему.
-Может так случиться, что до получения имени хоста соединение идет по одному профилю, а при получении
+Может так случиться, что до получения имени хоста или узнавания L7 протокола соединение идет по одному профилю,
-хоста профиль меняется на лету. Поэтому если у вас есть параметры дурения нулевой фазы, тщательно
+а при выяснении этих параметров профиль меняется на лету. Это может произойти даже дважды - при выяснении L7
-продумывайте что может произойти при переключении стратегии. Смотрите debug log, чтобы лучше
+и имени хоста. Чаще всего это выяснение совмещается в одно действие, поскольку по одному пакету как правило узнается и L7, и хост.
-понять что делает nfqws.
+Поэтому если у вас есть параметры дурения нулевой фазы, тщательно продумывайте что может произойти при переключении стратегии.
+Смотрите debug log, чтобы лучше понять что делает nfqws.
 Нумерация профилей идет с 1 до N. Последним в цепочке создается пустой профиль с номером 0.
 Он используется, когда никакие условия фильтров не совпали.
 
@@ -686,6 +691,10 @@ nfqws способен по-разному реагировать на разл
 во множество профилей без понимания как они работают приведет к нагромождению параметров, которые все равно 
 не покроют все возможные заблокированные ресурсы. Вы только увязните в этой каше.
 
+ВАЖНО : user-mode реализация ipset создавалась не как удобная замена *nix версии, реализованной в ядре.
+Вариант в ядре работает гораздо эффективнее. Это создавалось для систем без подержки ipset в ядре.
+Конкретно - Windows и ядра Linux, собранные без nftables и ipset модулей ядра. Например, в android нет ipset.
+
 
 tpws
 -----
@@ -777,6 +786,9 @@ tpws - это transparent proxy.
  --new                                  ; начало новой стратегии
  --filter-l3=ipv4|ipv6                  ; фильтр версии ip для текущей стратегии
  --filter-tcp=[~]port1[-port2]          ; фильтр портов tcp для текущей стратегии. ~ означает инверсию.
+ --filter-l7=[http|tls|quic|wireguard|dht|unknown] ; фильтр протокола L6-L7. поддерживается несколько значений через запятую.
+ --ipset=<filename>             	        ; включающий ip list. на каждой строчке ip или cidr ipv4 или ipv6. поддерживается множество листов и gzip.
+ --ipset-exclude=<filename>                     ; исключающий ip list. на каждой строчке ip или cidr ipv4 или ipv6. поддерживается множество листов и gzip.
 
 
 --debug позволяет выводить подробный лог действий на консоль, в syslog или в файл.

docs/windows.eng.md

@@ -80,11 +80,13 @@ curl -O https://www.cygwin.com/setup-x86_64.exe
 setup-x86_64.exe --allow-unsupported-windows --no-verify --site http://ctm.crouchingtigerhiddenfruitbat.org/pub/cygwin/circa/64bit/2024/01/30/231215
 ```
 You must choose to install `curl`. To compile from sources install `gcc-core`,`make`,`zlib-devel`.
+Make from directory `nfq` using `make cygwin`.
 
 `winws` requires `cygwin1.dll`, `windivert.dll`, `windivert64.sys`. You can take them from `binaries/win64/zapret-winws`.
 
-It's possible to build x86 32-bit version but this version is not shipped. You have to build it yourself.
+32-bit x86 version can be build from 32-bit cygwin using `make cygwin32`.
-32-bit `windivert` can be downloaded from it's developer github. Required version is 2.2.2.
+`windivert.dll` and `windivert32.sys` can be taken from [windivert 2.2.2 archive](https://reqrypt.org/download)
+
 There's no `arm64` signed `windivert` driver and no `cygwin`.
 But it's possible to use unsigned driver version in test mode and user mode components with x64 emulation.
 x64 emulation requires `windows 11` and not supported in `windows 10`.
@@ -159,3 +161,5 @@ unix2dos winws.log
 ```
 
 `winws.log` will be in `cygwin/home/<username>`. `unix2dos` helps with `windows 7` notepad. It's not necessary in `Windows 10` and later.
+
+Pre-compiled 32-bit binaries can be downloaded [here](https://github.com/bol-van/zapret-win32)

docs/windows.txt

@@ -120,8 +120,9 @@ setup-x86_64.exe --allow-unsupported-windows --no-verify --site http://ctm.crouc
 Для сборки из исходников требуется gcc-core,make,zlib-devel.
 Собирать из директории nfq командой "make cygwin".
 winws требует cygwin1.dll, windivert.dll, windivert64.sys. Их можно взять из binaries/win64/zapret-winws.
-Версию для 32-битных x86 windows собрать можно, но такие системы уже уходят в прошлое, поэтому если надо - собирайте сами.
+
-32-битный windivert можно взять с сайта разработчика. Требуется версия 2.2.2.
+32-битную x86 версию можно собрать из 32-битного cygwin командой "make cygwin32".
+windivert.dll и windivert32.sys можно взять из комплекта windivert 2.2.2 : https://reqrypt.org/download
 
 Для arm64 windows нет подписанного драйвера windivert и нет cygwin.
 Однако, эмуляция x64 windows 11 позволяет использовать все, кроме WinDivert64.sys без изменений.
@@ -220,3 +221,6 @@ winws --debug --wf-tcp=80,443 | tee winws.log
 winws.log будет в cygwin/home/<имя_пользователя>
 Если у вас windows 7, то блокнот не поймет переводы строк в стиле unix. Воспользуйтесь командой
 unix2dos winws.log
+
+Готовую 32-битную версию можно скачать здесь : https://github.com/bol-van/zapret-win32
+Поскольку 32-битные windows мало востребованы, выложены только бинарники и ничего больше.

nfq/Makefile

@@ -5,7 +5,11 @@ CFLAGS_MAC = -mmacosx-version-min=10.8
 CFLAGS_CYGWIN = -Wno-address-of-packed-member -static
 LIBS_LINUX = -lnetfilter_queue -lnfnetlink -lz
 LIBS_BSD = -lz
-LIBS_CYGWIN = -lz -Lwindivert -lwindivert -lwlanapi -lole32 -loleaut32 -luuid
+LIBS_CYGWIN = -lz -Lwindows/windivert -Iwindows -lwlanapi -lole32 -loleaut32 -luuid
+LIBS_CYGWIN32 = -lwindivert32
+LIBS_CYGWIN64 = -lwindivert64
+RES_CYGWIN32 = windows/res/32/winmanifest.o windows/res/32/winicon.o
+RES_CYGWIN64 = windows/res/64/winmanifest.o windows/res/64/winicon.o
 SRC_FILES = *.c crypto/*.c
 
 all: nfqws
@@ -23,8 +27,11 @@ mac: $(SRC_FILES)
 	lipo -create -output dvtws dvtwsx dvtwsa
 	rm -f dvtwsx dvtwsa
 
-cygwin:
+cygwin64:
-	$(CC) -s $(CFLAGS) $(CFLAGS_CYGWIN) -o winws $(SRC_FILES) $(LDFLAGS) $(LIBS_CYGWIN) winmanifest.o winicon.o
+	$(CC) -s $(CFLAGS) $(CFLAGS_CYGWIN) -o winws $(SRC_FILES) $(LDFLAGS) $(LIBS_CYGWIN) $(LIBS_CYGWIN64) $(RES_CYGWIN64)
+cygwin32:
+	$(CC) -s $(CFLAGS) $(CFLAGS_CYGWIN) -o winws $(SRC_FILES) $(LDFLAGS) $(LIBS_CYGWIN) $(LIBS_CYGWIN32) $(RES_CYGWIN32)
+cygwin: cygwin64
 
 clean:
 	rm -f nfqws dvtws winws.exe

nfq/conntrack.c

@@ -12,6 +12,29 @@ static void ut_oom_recover(void *elem)
 	oom = true;
 }
 
+const char *l7proto_str(t_l7proto l7)
+{
+	switch(l7)
+	{
+		case HTTP: return "http";
+		case TLS: return "tls";
+		case QUIC: return "quic";
+		case WIREGUARD: return "wireguard";
+		case DHT: return "dht";
+		default: return "unknown";
+	}
+}
+bool l7_proto_match(t_l7proto l7proto, uint32_t filter_l7)
+{
+	return  (l7proto==UNKNOWN && (filter_l7 & L7_PROTO_UNKNOWN)) ||
+		(l7proto==HTTP && (filter_l7 & L7_PROTO_HTTP)) ||
+		(l7proto==TLS && (filter_l7 & L7_PROTO_TLS)) ||
+		(l7proto==QUIC && (filter_l7 & L7_PROTO_QUIC)) ||
+		(l7proto==WIREGUARD && (filter_l7 & L7_PROTO_WIREGUARD)) ||
+		(l7proto==DHT && (filter_l7 & L7_PROTO_DHT));
+}
+
+
 static const char *connstate_s[]={"SYN","ESTABLISHED","FIN"};
 
 static void connswap(const t_conn *c, t_conn *c2)
@@ -316,18 +339,6 @@ static void taddr2str(uint8_t l3proto, const t_addr *a, char *buf, size_t bufsiz
 	if (!inet_ntop(family_from_proto(l3proto), a, buf, bufsize) && bufsize) *buf=0;
 }
 
-static const char *ConntrackProtoName(t_l7proto proto)
-{
-	switch(proto)
-	{
-		case HTTP: return "HTTP";
-		case TLS: return "TLS";
-		case QUIC: return "QUIC";
-		case WIREGUARD: return "WIREGUARD";
-		case DHT: return "DHT";
-		default: return "UNKNOWN";
-	}
-}
 void ConntrackPoolDump(const t_conntrack *p)
 {
 	t_conntrack_pool *t, *tmp;
@@ -354,7 +365,7 @@ void ConntrackPoolDump(const t_conntrack *p)
 				t->track.seq_last, t->track.pos_orig,
 				t->track.ack_last, t->track.pos_reply);
 		printf(" req_retrans=%u cutoff=%u wss_cutoff=%u d_cutoff=%u hostname=%s l7proto=%s\n",
-			t->track.req_retrans_counter, t->track.b_cutoff, t->track.b_wssize_cutoff, t->track.b_desync_cutoff, t->track.hostname, ConntrackProtoName(t->track.l7proto));
+			t->track.req_retrans_counter, t->track.b_cutoff, t->track.b_wssize_cutoff, t->track.b_desync_cutoff, t->track.hostname, l7proto_str(t->track.l7proto));
 	};
 }
 

nfq/conntrack.h

@@ -52,12 +52,23 @@ typedef struct {
 // ESTABLISHED - any except SYN or SYN/ACK received
 // FIN - FIN or RST received
 typedef enum {SYN=0, ESTABLISHED, FIN} t_connstate;
+
 typedef enum {UNKNOWN=0, HTTP, TLS, QUIC, WIREGUARD, DHT} t_l7proto;
+#define L7_PROTO_HTTP		0x00000001
+#define L7_PROTO_TLS		0x00000002
+#define L7_PROTO_QUIC		0x00000004
+#define L7_PROTO_WIREGUARD	0x00000008
+#define L7_PROTO_DHT		0x00000010
+#define L7_PROTO_UNKNOWN	0x80000000
+const char *l7proto_str(t_l7proto l7);
+bool l7_proto_match(t_l7proto l7proto, uint32_t filter_l7);
+
 typedef struct
 {
+	bool bCheckDone, bCheckResult, bCheckExcluded; // hostlist check result cache
+
 	struct desync_profile *dp;		// desync profile cache
 	bool dp_search_complete;
-	bool bCheckDone, bCheckResult, bCheckExcluded; // hostlist check result cache
 
 	// common state
 	time_t t_start, t_last;
@@ -76,12 +87,13 @@ typedef struct
 	bool req_seq_present,req_seq_finalized,req_seq_abandoned;
 	uint32_t req_seq_start,req_seq_end;	// sequence interval of the request (to track retransmissions)
 
-	uint8_t autottl;
+	uint8_t incoming_ttl, autottl;
 
 	bool b_cutoff;				// mark for deletion
 	bool b_wssize_cutoff, b_desync_cutoff;
 
 	t_l7proto l7proto;
+	bool l7proto_discovered;
 	char *hostname;
 	bool hostname_ah_check;			// should perform autohostlist checks
 	

nfq/darkmagic.c

@@ -22,6 +22,11 @@
 #ifdef __CYGWIN__
 #include <wlanapi.h>
 #include <netlistmgr.h>
+
+#ifndef ERROR_INVALID_IMAGE_HASH
+#define ERROR_INVALID_IMAGE_HASH __MSABI_LONG(577)
+#endif
+
 #endif
 
 uint32_t net32_add(uint32_t netorder_value, uint32_t cpuorder_increment)

nfq/desync.c

@@ -5,6 +5,7 @@
 #include "params.h"
 #include "helpers.h"
 #include "hostlist.h"
+#include "ipset.h"
 #include "conntrack.h"
 
 #include <string.h>
@@ -145,20 +146,32 @@ enum dpi_desync_mode desync_mode_from_string(const char *s)
 	return DESYNC_INVALID;
 }
 
-static bool dp_match_l3l4(struct desync_profile *dp, bool ipv6, uint16_t tcp_port, uint16_t udp_port)
+static bool dp_match_l3l4(struct desync_profile *dp, uint8_t l3proto, const struct sockaddr *dest)
 {
-	return \
+	return  ((dest->sa_family==AF_INET && dp->filter_ipv4) || (dest->sa_family==AF_INET6 && dp->filter_ipv6)) &&
-		((!ipv6 && dp->filter_ipv4) || (ipv6 && dp->filter_ipv6)) &&
+		(l3proto==IPPROTO_TCP && pf_in_range(saport(dest), &dp->pf_tcp) || l3proto==IPPROTO_UDP && pf_in_range(saport(dest), &dp->pf_udp)) &&
-		(!tcp_port || pf_in_range(tcp_port,&dp->pf_tcp)) &&
+		IpsetCheck(dp, dest->sa_family==AF_INET ? &((struct sockaddr_in*)dest)->sin_addr : NULL, dest->sa_family==AF_INET6 ? &((struct sockaddr_in6*)dest)->sin6_addr : NULL);
-		(!udp_port || pf_in_range(udp_port,&dp->pf_udp));
 }
+static bool dp_impossible(struct desync_profile *dp, const char *hostname, t_l7proto l7proto)
+{
+	return	!PROFILE_IPSETS_EMPTY(dp) &&
+		((dp->filter_l7 && !l7_proto_match(l7proto, dp->filter_l7)) || (!*dp->hostlist_auto_filename && !hostname && (dp->hostlist || dp->hostlist_exclude)));
+}
+
 static bool dp_match(
-	struct desync_profile *dp, bool ipv6, uint16_t tcp_port, uint16_t udp_port, const char *hostname,
+	struct desync_profile *dp,
+	uint8_t l3proto, const struct sockaddr *dest, const char *hostname, t_l7proto l7proto,
 	bool *bCheckDone, bool *bCheckResult, bool *bExcluded)
 {
 	if (bCheckDone) *bCheckDone = false;
-	if (dp_match_l3l4(dp,ipv6,tcp_port,udp_port))
+	// impossible case, hard filter
+	// impossible check avoids relatively slow ipset search
+	if (!dp_impossible(dp,hostname,l7proto) && dp_match_l3l4(dp,l3proto,dest))
 	{
+		// soft filter
+		if (dp->filter_l7 && !l7_proto_match(l7proto, dp->filter_l7))
+			return false;
+
 		// autohostlist profile matching l3/l4 filter always win
 		if (*dp->hostlist_auto_filename) return true;
 
@@ -181,15 +194,21 @@ static bool dp_match(
 	return false;
 }
 static struct desync_profile *dp_find(
-	struct desync_profile_list_head *head, bool ipv6, uint16_t tcp_port, uint16_t udp_port, const char *hostname,
+	struct desync_profile_list_head *head,
+	uint8_t l3proto, const struct sockaddr *dest, const char *hostname, t_l7proto l7proto,
 	bool *bCheckDone, bool *bCheckResult, bool *bExcluded)
 {
 	struct desync_profile_list *dpl;
-	DLOG("desync profile search for hostname='%s' ipv6=%u tcp_port=%u udp_port=%u\n", hostname ? hostname : "", ipv6, tcp_port, udp_port);
+	if (params.debug)
+	{
+		char ip_port[48];
+		ntop46_port(dest, ip_port,sizeof(ip_port));
+		DLOG("desync profile search for %s target=%s l7proto=%s hostname='%s'\n", proto_name(l3proto), ip_port, l7proto_str(l7proto), hostname ? hostname : "");
+	}
 	if (bCheckDone) *bCheckDone = false;
 	LIST_FOREACH(dpl, head, next)
 	{
-		if (dp_match(&dpl->dp,ipv6,tcp_port,udp_port,hostname,bCheckDone,bCheckResult,bExcluded))
+		if (dp_match(&dpl->dp,l3proto,dest,hostname,l7proto,bCheckDone,bCheckResult,bExcluded))
 		{
 			DLOG("desync profile %d matches\n",dpl->dp.n);
 			return &dpl->dp;
@@ -272,7 +291,7 @@ static void ctrack_stop_retrans_counter(t_ctrack *ctrack)
 	}
 }
 
-static void auto_hostlist_reset_fail_counter(struct desync_profile *dp, const char *hostname)
+static void auto_hostlist_reset_fail_counter(struct desync_profile *dp, const char *hostname, const char *client_ip_port, t_l7proto l7proto)
 {
 	if (hostname)
 	{
@@ -283,13 +302,13 @@ static void auto_hostlist_reset_fail_counter(struct desync_profile *dp, const ch
 		{
 			HostFailPoolDel(&dp->hostlist_auto_fail_counters, fail_counter);
 			DLOG("auto hostlist (profile %d) : %s : fail counter reset. website is working.\n", dp->n, hostname);
-			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : fail counter reset. website is working.", hostname, dp->n);
+			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : fail counter reset. website is working.", hostname, dp->n, client_ip_port, l7proto_str(l7proto));
 		}
 	}
 }
 
 // return true if retrans trigger fires
-static bool auto_hostlist_retrans(t_ctrack *ctrack, uint8_t l4proto, int threshold)
+static bool auto_hostlist_retrans(t_ctrack *ctrack, uint8_t l4proto, int threshold, const char *client_ip_port, t_l7proto l7proto)
 {
 	if (ctrack && ctrack->dp && ctrack->hostname_ah_check && ctrack->req_retrans_counter!=RETRANS_COUNTER_STOP)
 	{
@@ -301,7 +320,7 @@ static bool auto_hostlist_retrans(t_ctrack *ctrack, uint8_t l4proto, int thresho
 			{
 				DLOG("req retrans : tcp seq %u not within the req range %u-%u. stop tracking.\n", ctrack->seq_last, ctrack->req_seq_start, ctrack->req_seq_end);
 				ctrack_stop_retrans_counter(ctrack);
-				auto_hostlist_reset_fail_counter(ctrack->dp, ctrack->hostname);
+				auto_hostlist_reset_fail_counter(ctrack->dp, ctrack->hostname, client_ip_port, l7proto);
 				return false;
 			}
 		}
@@ -316,7 +335,7 @@ static bool auto_hostlist_retrans(t_ctrack *ctrack, uint8_t l4proto, int thresho
 	}
 	return false;
 }
-static void auto_hostlist_failed(struct desync_profile *dp, const char *hostname)
+static void auto_hostlist_failed(struct desync_profile *dp, const char *hostname, const char *client_ip_port, t_l7proto l7proto)
 {
 	hostfail_pool *fail_counter;
 	
@@ -326,13 +345,13 @@ static void auto_hostlist_failed(struct desync_profile *dp, const char *hostname
 		fail_counter = HostFailPoolAdd(&dp->hostlist_auto_fail_counters, hostname, dp->hostlist_auto_fail_time);
 		if (!fail_counter)
 		{
-			fprintf(stderr, "HostFailPoolAdd: out of memory\n");
+			DLOG_ERR("HostFailPoolAdd: out of memory\n");
 			return;
 		}
 	}
 	fail_counter->counter++;
 	DLOG("auto hostlist (profile %d) : %s : fail counter %d/%d\n", dp->n, hostname, fail_counter->counter, dp->hostlist_auto_fail_threshold);
-	HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : fail counter %d/%d", hostname, dp->n, fail_counter->counter, dp->hostlist_auto_fail_threshold);
+	HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : fail counter %d/%d", hostname, dp->n, client_ip_port, l7proto_str(l7proto), fail_counter->counter, dp->hostlist_auto_fail_threshold);
 	if (fail_counter->counter >= dp->hostlist_auto_fail_threshold)
 	{
 		DLOG("auto hostlist (profile %d) : fail threshold reached. about to add %s to auto hostlist\n", dp->n, hostname);
@@ -343,10 +362,10 @@ static void auto_hostlist_failed(struct desync_profile *dp, const char *hostname
 		if (!HostlistCheck(dp, hostname, &bExcluded) && !bExcluded)
 		{
 			DLOG("auto hostlist (profile %d) : adding %s to %s\n", dp->n, hostname, dp->hostlist_auto_filename);
-			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : adding to %s", hostname, dp->n, dp->hostlist_auto_filename);
+			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : adding to %s", hostname, dp->n, client_ip_port, l7proto_str(l7proto), dp->hostlist_auto_filename);
 			if (!StrPoolAddStr(&dp->hostlist, hostname))
 			{
-				fprintf(stderr, "StrPoolAddStr out of memory\n");
+				DLOG_ERR("StrPoolAddStr out of memory\n");
 				return;
 			}
 			if (!append_to_list_file(dp->hostlist_auto_filename, hostname))
@@ -359,17 +378,22 @@ static void auto_hostlist_failed(struct desync_profile *dp, const char *hostname
 		else
 		{
 			DLOG("auto hostlist (profile %d) : NOT adding %s\n", dp->n, hostname);
-			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : NOT adding, duplicate detected", hostname, dp->n);
+			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : NOT adding, duplicate detected", hostname, dp->n, client_ip_port, l7proto_str(l7proto));
 		}
 	}
 }
 
-static void process_retrans_fail(t_ctrack *ctrack, uint8_t proto)
+static void process_retrans_fail(t_ctrack *ctrack, uint8_t proto, const struct sockaddr *client)
 {
-	if (ctrack && ctrack->dp && ctrack->hostname && auto_hostlist_retrans(ctrack, proto, ctrack->dp->hostlist_auto_retrans_threshold))
+	char client_ip_port[48];
+	if (*params.hostlist_auto_debuglog)
+		ntop46_port((struct sockaddr*)client,client_ip_port,sizeof(client_ip_port));
+	else
+		*client_ip_port=0;
+	if (ctrack && ctrack->dp && ctrack->hostname && auto_hostlist_retrans(ctrack, proto, ctrack->dp->hostlist_auto_retrans_threshold, client_ip_port, ctrack->l7proto))
 	{
-		HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : tcp retrans threshold reached", ctrack->hostname, ctrack->dp->n);
+		HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : retrans threshold reached", ctrack->hostname, ctrack->dp->n, client_ip_port, l7proto_str(ctrack->l7proto));
-		auto_hostlist_failed(ctrack->dp, ctrack->hostname);
+		auto_hostlist_failed(ctrack->dp, ctrack->hostname, client_ip_port, ctrack->l7proto);
 	}
 }
 
@@ -575,6 +599,21 @@ static size_t pos_normalize(size_t split_pos, size_t reasm_offset, size_t len_pa
 	return split_pos;
 }
 
+static void autottl_discover(t_ctrack *ctrack, bool bIpv6)
+{
+	if (ctrack && ctrack->incoming_ttl)
+	{
+		autottl *attl = bIpv6 ? &ctrack->dp->desync_autottl6 : &ctrack->dp->desync_autottl;
+		if (AUTOTTL_ENABLED(*attl))
+		{
+			ctrack->autottl = autottl_guess(ctrack->incoming_ttl, attl);
+			if (ctrack->autottl)
+				DLOG("autottl: guessed %u\n",ctrack->autottl);
+			else
+				DLOG("autottl: could not guess\n");
+		}
+	}
+}
 
 static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint32_t fwmark, const char *ifout, uint8_t *data_pkt, size_t *len_pkt, struct ip *ip, struct ip6_hdr *ip6hdr, struct tcphdr *tcphdr, size_t transport_len, uint8_t *data_payload, size_t len_payload)
 {
@@ -593,10 +632,10 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 	size_t pkt1_len, pkt2_len;
 	uint8_t ttl_orig,ttl_fake,flags_orig,scale_factor;
 	uint32_t *timestamps;
-	t_l7proto l7proto = UNKNOWN;
 
 	ttl_orig = ip ? ip->ip_ttl : ip6hdr->ip6_ctlun.ip6_un1.ip6_un1_hlim;
 	uint32_t desync_fwmark = fwmark | params.desync_fwmark;
+	extract_endpoints(ip, ip6hdr, tcphdr, NULL, &src, &dst);
 	
 	if (replay)
 	{
@@ -611,10 +650,9 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 			DLOG("using cached desync profile %d\n",dp->n);
 		else if (!ctrack_replay->dp_search_complete)
 		{
-			dp = ctrack_replay->dp = dp_find(&params.desync_profiles, !!ip6hdr, ntohs(bReverse ? tcphdr->th_sport : tcphdr->th_dport), 0, ctrack_replay->hostname, NULL, NULL, NULL);
+			dp = ctrack_replay->dp = dp_find(&params.desync_profiles, IPPROTO_TCP, (struct sockaddr *)&dst, ctrack_replay->hostname, ctrack_replay->l7proto, NULL, NULL, NULL);
 			ctrack_replay->dp_search_complete = true;
 		}
-
 		if (!dp)
 		{
 			DLOG("matching desync profile not found\n");
@@ -635,7 +673,7 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 			DLOG("using cached desync profile %d\n",dp->n);
 		else if (!ctrack || !ctrack->dp_search_complete)
 		{
-			dp = dp_find(&params.desync_profiles, !!ip6hdr, ntohs(bReverse ? tcphdr->th_sport : tcphdr->th_dport), 0, ctrack ? ctrack->hostname : NULL, NULL, NULL, NULL);
+			dp = dp_find(&params.desync_profiles, IPPROTO_TCP, (struct sockaddr *)&dst, ctrack ? ctrack->hostname : NULL, ctrack ? ctrack->l7proto : UNKNOWN, NULL, NULL, NULL);
 			if (ctrack)
 			{
 				ctrack->dp = dp;
@@ -661,17 +699,14 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 
 		if (bReverse)
 		{
-			if (ctrack && !ctrack->autottl && ctrack->pcounter_reply==1)
+			if (ctrack)
 			{
-				autottl *attl = ip ? &dp->desync_autottl : &dp->desync_autottl6;
+				if (!ctrack->incoming_ttl)
-				if (AUTOTTL_ENABLED(*attl))
 				{
-					ctrack->autottl = autottl_guess(ttl_orig, attl);
+					DLOG("incoming TTL %u\n",ttl_orig);
-					if (ctrack->autottl)
+					ctrack->incoming_ttl = ttl_orig;
-						DLOG("autottl: guessed %u\n",ctrack->autottl);
-					else
-						DLOG("autottl: could not guess\n");
 				}
+				if (!ctrack->autottl) autottl_discover(ctrack,!!ip6hdr);
 			}
 
 			// process reply packets for auto hostlist mode
@@ -680,10 +715,17 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 			if (ctrack && ctrack->hostname && ctrack->hostname_ah_check && (ctrack->ack_last-ctrack->ack0)==1)
 			{
 				bool bFail=false;
+
+				char client_ip_port[48];
+				if (*params.hostlist_auto_debuglog)
+					ntop46_port((struct sockaddr*)&dst,client_ip_port,sizeof(client_ip_port));
+				else
+					*client_ip_port=0;
+
 				if (tcphdr->th_flags & TH_RST)
 				{
 					DLOG("incoming RST detected for hostname %s\n", ctrack->hostname);
-					HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : incoming RST", ctrack->hostname, ctrack->dp->n);
+					HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : incoming RST", ctrack->hostname, ctrack->dp->n, client_ip_port, l7proto_str(ctrack->l7proto));
 					bFail = true;
 				}
 				else if (len_payload && ctrack->l7proto==HTTP)
@@ -695,7 +737,7 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 						if (bFail)
 						{
 							DLOG("redirect to another domain detected. possibly DPI redirect.\n");
-							HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : redirect to another domain", ctrack->hostname, ctrack->dp->n);
+							HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : redirect to another domain", ctrack->hostname, ctrack->dp->n, client_ip_port, l7proto_str(ctrack->l7proto));
 						}
 						else
 							DLOG("local or in-domain redirect detected. it's not a DPI redirect.\n");
@@ -707,10 +749,10 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 					}
 				}
 				if (bFail)
-					auto_hostlist_failed(dp, ctrack->hostname);
+					auto_hostlist_failed(dp, ctrack->hostname, client_ip_port, ctrack->l7proto);
 				else
 					if (len_payload)
-						auto_hostlist_reset_fail_counter(dp, ctrack->hostname);
+						auto_hostlist_reset_fail_counter(dp, ctrack->hostname, client_ip_port, ctrack->l7proto);
 				if (tcphdr->th_flags & TH_RST)
 					ConntrackClearHostname(ctrack); // do not react to further dup RSTs
 			}
@@ -744,10 +786,12 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 	flags_orig = *((uint8_t*)tcphdr+13);
 	scale_factor = tcp_find_scale_factor(tcphdr);
 	timestamps = tcp_find_timestamps(tcphdr);
-	extract_endpoints(ip, ip6hdr, tcphdr, NULL, &src, &dst);
 
 	if (!replay)
 	{
+		// start and cutoff limiters
+		if (!process_desync_interval(dp, ctrack)) return verdict;
+
 		if (tcp_syn_segment(tcphdr))
 		{
 			switch (dp->desync_mode0)
@@ -797,8 +841,6 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 			return verdict;
 		}
 
-		// start and cutoff limiters
-		if (!process_desync_interval(dp, ctrack)) return verdict;
 	} // !replay
 	
 	if (!(tcphdr->th_flags & TH_SYN) && len_payload)
@@ -811,6 +853,7 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 		const uint8_t *rdata_payload = data_payload;
 		size_t rlen_payload = len_payload;
 		size_t split_pos;
+		t_l7proto l7proto = UNKNOWN;
 
 		if (replay)
 		{
@@ -823,13 +866,13 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 			rlen_payload = ctrack->reasm_orig.size_present;
 		}
 
-		process_retrans_fail(ctrack, IPPROTO_TCP);
+		process_retrans_fail(ctrack, IPPROTO_TCP, (struct sockaddr*)&src);
 
 		if (IsHttp(rdata_payload,rlen_payload))
 		{
 			DLOG("packet contains HTTP request\n");
 			l7proto = HTTP;
-			if (ctrack && !ctrack->l7proto) ctrack->l7proto = l7proto;
+			if (ctrack && ctrack->l7proto==UNKNOWN) ctrack->l7proto = l7proto;
 
 			// we do not reassemble http
 			reasm_orig_cancel(ctrack);
@@ -874,7 +917,6 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 						reasm_orig_cancel(ctrack);
 						return verdict;
 					}
-					
 				}
 				if (!ctrack->req_seq_finalized)
 				{
@@ -901,7 +943,7 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 					}
 					else
 					{
-						fprintf(stderr, "rawpacket_queue failed !'\n");
+						DLOG_ERR("rawpacket_queue failed !\n");
 						reasm_orig_cancel(ctrack);
 						return verdict;
 					}
@@ -929,13 +971,23 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 			if (dseq>=0x1000000 && !(dseq & 0x80000000)) ctrack->req_seq_abandoned=true;
 		}
 
-		if (bHaveHost)
+		if (bHaveHost) DLOG("hostname: %s\n",host);
-		{
+
-			bool bCheckDone=false, bCheckResult=false, bCheckExcluded=false;
+		bool bDiscoveredL7;
-			DLOG("hostname: %s\n",host);
 		if (ctrack_replay)
 		{
-				if (!ctrack_replay->hostname)
+			bDiscoveredL7 = !ctrack_replay->l7proto_discovered && ctrack_replay->l7proto!=UNKNOWN;
+			ctrack_replay->l7proto_discovered=true;
+		}
+		else
+			bDiscoveredL7 = !ctrack_replay && l7proto!=UNKNOWN;
+		if (bDiscoveredL7) DLOG("discovered l7 protocol\n");
+
+		bool bDiscoveredHostname = bHaveHost && !(ctrack_replay && ctrack_replay->hostname);
+		if (bDiscoveredHostname)
+		{
+			DLOG("discovered hostname\n");
+			if (ctrack_replay)
 			{
 				ctrack_replay->hostname=strdup(host);
 				if (!ctrack_replay->hostname)
@@ -944,10 +996,23 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 					reasm_orig_cancel(ctrack);
 					return verdict;
 				}
-					DLOG("we have hostname now. searching desync profile again.\n");
+			}
+		}
+
+		bool bCheckDone=false, bCheckResult=false, bCheckExcluded=false;
+		if (bDiscoveredL7 || bDiscoveredHostname)
+		{
 			struct desync_profile *dp_prev = dp;
-					dp = ctrack_replay->dp = dp_find(&params.desync_profiles, !!ip6hdr, ntohs(bReverse ? tcphdr->th_sport : tcphdr->th_dport), 0, ctrack_replay->hostname, &ctrack_replay->bCheckDone, &ctrack_replay->bCheckResult, &ctrack_replay->bCheckExcluded);
+
+			dp = dp_find(&params.desync_profiles, IPPROTO_TCP, (struct sockaddr *)&dst, ctrack_replay ? ctrack_replay->hostname : host, ctrack_replay ? ctrack_replay->l7proto : l7proto, &bCheckDone, &bCheckResult, &bCheckExcluded);
+			if (ctrack_replay)
+			{
+				ctrack_replay->dp = dp;
 				ctrack_replay->dp_search_complete = true;
+				ctrack_replay->bCheckDone = bCheckDone;
+				ctrack_replay->bCheckResult = bCheckResult;
+				ctrack_replay->bCheckExcluded = bCheckExcluded;
+			}
 			if (!dp)
 			{
 				reasm_orig_cancel(ctrack);
@@ -955,7 +1020,9 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 			}
 			if (dp!=dp_prev)
 			{
-						DLOG("desync profile changed by revealed hostname !\n");
+				DLOG("desync profile changed by revealed l7 protocol or hostname !\n");
+				// rediscover autottl
+				autottl_discover(ctrack_replay,!!ip6hdr);
 				// re-evaluate start/cutoff limiters
 				if (!replay)
 				{
@@ -968,11 +1035,14 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 				}
 			}
 		}
+		else if (ctrack_replay)
+		{
 			bCheckDone = ctrack_replay->bCheckDone;
 			bCheckResult = ctrack_replay->bCheckResult;
 			bCheckExcluded = ctrack_replay->bCheckExcluded;
 		}
-			if (dp->hostlist || dp->hostlist_exclude)
+
+		if (bHaveHost && (dp->hostlist || dp->hostlist_exclude))
 		{
 			if (!bCheckDone)
 				bCheckResult = HostlistCheck(dp, host, &bCheckExcluded);
@@ -991,7 +1061,6 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 				return verdict;
 			}
 		}
-		}
 
 		// desync profile may have changed after hostname was revealed
 		switch(l7proto)
@@ -1019,24 +1088,28 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 
 		if (l7proto==UNKNOWN)
 		{
-			if (!dp->desync_any_proto) return verdict;
+			if (!dp->desync_any_proto)
+			{
+				DLOG("not applying tampering to unknown protocol\n");
+				return verdict;
+			}
 			DLOG("applying tampering to unknown protocol\n");
 		}
 
 		ttl_fake = (ctrack_replay && ctrack_replay->autottl) ? ctrack_replay->autottl : (ip6hdr ? (dp->desync_ttl6 ? dp->desync_ttl6 : ttl_orig) : (dp->desync_ttl ? dp->desync_ttl : ttl_orig));
 
-		if ((l7proto == HTTP) && (dp->hostcase || dp->hostnospace || dp->domcase) && (phost = (uint8_t*)memmem(data_payload, len_payload, "\r\nHost: ", 8)))
+		if ((l7proto == HTTP) && (dp->hostcase || dp->hostnospace || dp->domcase) && HttpFindHost(&phost,data_payload,len_payload))
 		{
 			if (dp->hostcase)
 			{
 				DLOG("modifying Host: => %c%c%c%c:\n", dp->hostspell[0], dp->hostspell[1], dp->hostspell[2], dp->hostspell[3]);
-				memcpy(phost + 2, dp->hostspell, 4);
+				memcpy(phost, dp->hostspell, 4);
 				verdict=VERDICT_MODIFY;
 			}
 			if (dp->domcase)
 			{
 				DLOG("mixing domain case\n");
-				for (p = phost+7; p < (data_payload + len_payload) && *p != '\r' && *p != '\n'; p++)
+				for (p = phost+5; p < (data_payload + len_payload) && *p != '\r' && *p != '\n'; p++)
 					*p = (((size_t)p) & 1) ? tolower(*p) : toupper(*p);
 				verdict=VERDICT_MODIFY;
 			}
@@ -1048,12 +1121,12 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 				DLOG("removing space after Host: and adding it to User-Agent:\n");
 				if (pua > phost)
 				{
-					memmove(phost + 7, phost + 8, pua - phost - 8);
+					memmove(phost + 5, phost + 6, pua - phost - 6);
-					phost[pua - phost - 1] = ' ';
+					pua[-1]=' ';
 				}
 				else
 				{
-					memmove(pua + 1, pua, phost - pua + 7);
+					memmove(pua + 1, pua, phost - pua + 5);
 					*pua = ' ';
 				}
 				verdict=VERDICT_MODIFY;
@@ -1404,6 +1477,9 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint
 	uint8_t ttl_orig,ttl_fake;
 	t_l7proto l7proto = UNKNOWN;
 
+	ttl_orig = ip ? ip->ip_ttl : ip6hdr->ip6_ctlun.ip6_un1.ip6_un1_hlim;
+	extract_endpoints(ip, ip6hdr, NULL, udphdr, &src, &dst);
+
 	if (replay)
 	{
 		// in replay mode conntrack_replay is not NULL and ctrack is NULL
@@ -1417,7 +1493,7 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint
 			DLOG("using cached desync profile %d\n",dp->n);
 		else if (!ctrack_replay->dp_search_complete)
 		{
-			dp = ctrack_replay->dp = dp_find(&params.desync_profiles, !!ip6hdr, 0, ntohs(bReverse ? udphdr->uh_sport : udphdr->uh_dport), ctrack_replay->hostname, NULL, NULL, NULL);
+			dp = ctrack_replay->dp = dp_find(&params.desync_profiles, IPPROTO_UDP, (struct sockaddr *)&dst, ctrack_replay->hostname, ctrack_replay->l7proto, NULL, NULL, NULL);
 			ctrack_replay->dp_search_complete = true;
 		}
 		if (!dp)
@@ -1440,7 +1516,7 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint
 			DLOG("using cached desync profile %d\n",dp->n);
 		else if (!ctrack || !ctrack->dp_search_complete)
 		{
-			dp = dp_find(&params.desync_profiles, !!ip6hdr, 0, ntohs(bReverse ? udphdr->uh_sport : udphdr->uh_dport), ctrack ? ctrack->hostname : NULL, NULL, NULL, NULL);
+			dp = dp_find(&params.desync_profiles, IPPROTO_UDP, (struct sockaddr *)&dst, ctrack ? ctrack->hostname : NULL, ctrack ? ctrack->l7proto : UNKNOWN, NULL, NULL, NULL);
 			if (ctrack)
 			{
 				ctrack->dp = dp;
@@ -1458,15 +1534,21 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint
 		//ConntrackPoolDump(&params.conntrack);
 	}
 
-	if (bReverse) return verdict; // nothing to do. do not waste cpu
+	if (bReverse && ctrack)
+	{
+		if (!ctrack->incoming_ttl)
+		{
+			DLOG("incoming TTL %u\n",ttl_orig);
+			ctrack->incoming_ttl = ttl_orig;
+		}
+		if (!ctrack->autottl) autottl_discover(ctrack,!!ip6hdr);
+		return verdict; // nothing to do. do not waste cpu
+	}
 
 	// start and cutoff limiters
 	if (!replay && !process_desync_interval(dp, ctrack)) return verdict;
 
 	uint32_t desync_fwmark = fwmark | params.desync_fwmark;
-	ttl_orig = ip ? ip->ip_ttl : ip6hdr->ip6_ctlun.ip6_un1.ip6_un1_hlim;
-	ttl_fake = ip6hdr ? dp->desync_ttl6 ? dp->desync_ttl6 : ttl_orig : dp->desync_ttl ? dp->desync_ttl : ttl_orig;
-	extract_endpoints(ip, ip6hdr, NULL, udphdr, &src, &dst);
 
 	if (len_payload)
 	{
@@ -1480,7 +1562,7 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint
 		{
 			DLOG("packet contains QUIC initial\n");
 			l7proto = QUIC;
-			if (ctrack && !ctrack->l7proto) ctrack->l7proto = l7proto;
+			if (ctrack && ctrack->l7proto==UNKNOWN) ctrack->l7proto = l7proto;
 
 			uint8_t clean[16384], *pclean;
 			size_t clean_len;
@@ -1542,7 +1624,7 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint
 							}
 							else
 							{
-								fprintf(stderr, "rawpacket_queue failed !'\n");
+								DLOG_ERR("rawpacket_queue failed !\n");
 								reasm_orig_cancel(ctrack);
 								return verdict;
 							}
@@ -1593,28 +1675,42 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint
 			{
 				DLOG("packet contains wireguard handshake initiation\n");
 				l7proto = WIREGUARD;
-				if (ctrack && !ctrack->l7proto) ctrack->l7proto = l7proto;
+				if (ctrack && ctrack->l7proto==UNKNOWN) ctrack->l7proto = l7proto;
 			}
 			else if (IsDhtD1(data_payload,len_payload))
 			{
 				DLOG("packet contains DHT d1...e\n");
 				l7proto = DHT;
-				if (ctrack && !ctrack->l7proto) ctrack->l7proto = l7proto;
+				if (ctrack && ctrack->l7proto==UNKNOWN) ctrack->l7proto = l7proto;
 			}
 			else
 			{
-				if (!dp->desync_any_proto) return verdict;
+				if (!dp->desync_any_proto)
+				{
+					DLOG("not applying tampering to unknown protocol\n");
+					return verdict;
+				}
 				DLOG("applying tampering to unknown protocol\n");
 			}
 		}
 
-		if (bHaveHost)
+		if (bHaveHost) DLOG("hostname: %s\n",host);
-		{
+
-			bool bCheckDone=false, bCheckResult=false, bCheckExcluded=false;
+		bool bDiscoveredL7;
-			DLOG("hostname: %s\n",host);
 		if (ctrack_replay)
 		{
-				if (!ctrack_replay->hostname)
+			bDiscoveredL7 = !ctrack_replay->l7proto_discovered && ctrack_replay->l7proto!=UNKNOWN;
+			ctrack_replay->l7proto_discovered=true;
+		}
+		else
+			bDiscoveredL7 = !ctrack_replay && l7proto!=UNKNOWN;
+		if (bDiscoveredL7) DLOG("discovered l7 protocol\n");
+
+		bool bDiscoveredHostname = bHaveHost && !(ctrack_replay && ctrack_replay->hostname);
+		if (bDiscoveredHostname)
+		{
+			DLOG("discovered hostname\n");
+			if (ctrack_replay)
 			{
 				ctrack_replay->hostname=strdup(host);
 				if (!ctrack_replay->hostname)
@@ -1622,14 +1718,33 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint
 					DLOG_ERR("hostname dup : out of memory");
 					return verdict;
 				}
-					DLOG("we have hostname now. searching desync profile again.\n");
+			}
+		}
+
+		bool bCheckDone=false, bCheckResult=false, bCheckExcluded=false;
+		if (bDiscoveredL7 || bDiscoveredHostname)
+		{
 			struct desync_profile *dp_prev = dp;
-					dp = ctrack_replay->dp = dp_find(&params.desync_profiles, !!ip6hdr, 0, ntohs(bReverse ? udphdr->uh_sport : udphdr->uh_dport), ctrack_replay->hostname, &ctrack_replay->bCheckDone, &ctrack_replay->bCheckResult, &ctrack_replay->bCheckExcluded);
+
+			dp = dp_find(&params.desync_profiles, IPPROTO_UDP, (struct sockaddr *)&dst, ctrack_replay ? ctrack_replay->hostname : host, ctrack_replay ? ctrack_replay->l7proto : l7proto, &bCheckDone, &bCheckResult, &bCheckExcluded);
+			if (ctrack_replay)
+			{
+				ctrack_replay->dp = dp;
 				ctrack_replay->dp_search_complete = true;
-					if (!dp) return verdict;
+				ctrack_replay->bCheckDone = bCheckDone;
+				ctrack_replay->bCheckResult = bCheckResult;
+				ctrack_replay->bCheckExcluded = bCheckExcluded;
+			}
+			if (!dp)
+			{
+				reasm_orig_cancel(ctrack);
+				return verdict;
+			}
 			if (dp!=dp_prev)
 			{
-						DLOG("desync profile changed by reavealed hostname !\n");
+				DLOG("desync profile changed by revealed l7 protocol or hostname !\n");
+				// rediscover autottl
+				autottl_discover(ctrack_replay,!!ip6hdr);
 				// re-evaluate start/cutoff limiters
 				if (!replay)
 				{
@@ -1638,16 +1753,20 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint
 				}
 			}
 		}
+		else if (ctrack_replay)
+		{
 			bCheckDone = ctrack_replay->bCheckDone;
 			bCheckResult = ctrack_replay->bCheckResult;
 			bCheckExcluded = ctrack_replay->bCheckExcluded;
 		}
-			if (dp->hostlist || dp->hostlist_exclude)
+
+		if (bHaveHost && (dp->hostlist || dp->hostlist_exclude))
 		{
-				bool bCheckExcluded;
 			if (!bCheckDone)
 				bCheckResult = HostlistCheck(dp, host, &bCheckExcluded);
-				if (!bCheckResult)
+			if (bCheckResult)
+				ctrack_stop_retrans_counter(ctrack_replay);
+			else
 			{
 				if (ctrack_replay)
 				{
@@ -1655,17 +1774,14 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint
 					if (ctrack_replay->hostname_ah_check)
 					{
 						// first request is not retrans
-							if (ctrack_replay->hostname)
+						if (!bDiscoveredHostname)
-								process_retrans_fail(ctrack_replay, IPPROTO_UDP);
+							process_retrans_fail(ctrack_replay, IPPROTO_UDP, (struct sockaddr*)&src);
-							else
-								ctrack_replay->hostname=strdup(host);
 					}
 				}
 				DLOG("not applying tampering to this request\n");
 				return verdict;
 			}
 		}
-		}
 
 		// desync profile may have changed after hostname was revealed
 		switch(l7proto)
@@ -1687,7 +1803,8 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint
 				fake_size = dp->fake_unknown_udp_size;
 				break;
 		}
-		ttl_fake = ip6hdr ? dp->desync_ttl6 ? dp->desync_ttl6 : ttl_orig : dp->desync_ttl ? dp->desync_ttl : ttl_orig;
+
+		ttl_fake = (ctrack_replay && ctrack_replay->autottl) ? ctrack_replay->autottl : (ip6hdr ? (dp->desync_ttl6 ? dp->desync_ttl6 : ttl_orig) : (dp->desync_ttl ? dp->desync_ttl : ttl_orig));
 
 		enum dpi_desync_mode desync_mode = dp->desync_mode;
 		uint32_t fooling_orig = FOOL_NONE;

nfq/helpers.c

@@ -11,6 +11,12 @@
 
 #include "params.h"
 
+void rtrim(char *s)
+{
+	if (s)
+		for (char *p = s + strlen(s) - 1; p >= s && (*p == '\n' || *p == '\r'); p--) *p = '\0';
+}
+
 void hexdump_limited_dlog(const uint8_t *data, size_t size, size_t limit)
 {
 	size_t k;
@@ -181,6 +187,11 @@ bool pton6_port(const char *s, struct sockaddr_in6 *sa)
 	return true;
 }
 
+uint16_t saport(const struct sockaddr *sa)
+{
+	return htons(sa->sa_family==AF_INET ? ((struct sockaddr_in*)sa)->sin_port :
+		     sa->sa_family==AF_INET6 ? ((struct sockaddr_in6*)sa)->sin6_port : 0);
+}
 
 void dbgprint_socket_buffers(int fd)
 {
@@ -385,3 +396,99 @@ bool cd_to_exe_dir(const char *argv0)
 	}
 	return bOK;
 }
+
+
+static void mask_from_preflen6_make(uint8_t plen, struct in6_addr *a)
+{
+	if (plen >= 128)
+		memset(a->s6_addr,0xFF,16);
+	else
+	{
+		uint8_t n = plen >> 3;
+		memset(a->s6_addr,0xFF,n);
+		memset(a->s6_addr+n,0x00,16-n);
+		a->s6_addr[n] = (uint8_t)(0xFF00 >> (plen & 7));
+	}
+}
+struct in6_addr ip6_mask[129];
+void mask_from_preflen6_prepare(void)
+{
+	for (int plen=0;plen<=128;plen++) mask_from_preflen6_make(plen, ip6_mask+plen);
+}
+
+#if defined(__GNUC__) && !defined(__llvm__)
+__attribute__((optimize ("no-strict-aliasing")))
+#endif
+void ip6_and(const struct in6_addr * restrict a, const struct in6_addr * restrict b, struct in6_addr * restrict result)
+{
+	// int128 requires 16-bit alignment. in struct sockaddr_in6.sin6_addr is 8-byte aligned.
+	// it causes segfault on x64 arch with latest compiler. it can cause misalign slowdown on other archs
+	// use 64-bit AND
+	((uint64_t*)result->s6_addr)[0] = ((uint64_t*)a->s6_addr)[0] & ((uint64_t*)b->s6_addr)[0];
+	((uint64_t*)result->s6_addr)[1] = ((uint64_t*)a->s6_addr)[1] & ((uint64_t*)b->s6_addr)[1];
+}
+
+void str_cidr4(char *s, size_t s_len, const struct cidr4 *cidr)
+{
+	char s_ip[16];
+	*s_ip=0;
+	inet_ntop(AF_INET, &cidr->addr, s_ip, sizeof(s_ip));
+	snprintf(s,s_len,cidr->preflen<32 ? "%s/%u" : "%s", s_ip, cidr->preflen);
+}
+void print_cidr4(const struct cidr4 *cidr)
+{
+	char s[19];
+	str_cidr4(s,sizeof(s),cidr);
+	printf("%s",s);
+}
+void str_cidr6(char *s, size_t s_len, const struct cidr6 *cidr)
+{
+	char s_ip[40];
+	*s_ip=0;
+	inet_ntop(AF_INET6, &cidr->addr, s_ip, sizeof(s_ip));
+	snprintf(s,s_len,cidr->preflen<128 ? "%s/%u" : "%s", s_ip, cidr->preflen);
+}
+void print_cidr6(const struct cidr6 *cidr)
+{
+	char s[44];
+	str_cidr6(s,sizeof(s),cidr);
+	printf("%s",s);
+}
+bool parse_cidr4(char *s, struct cidr4 *cidr)
+{
+	char *p,d;
+	bool b;
+	unsigned int plen;
+
+	if ((p = strchr(s, '/')))
+	{
+		if (sscanf(p + 1, "%u", &plen)!=1 || plen>32)
+			return false;
+		cidr->preflen = (uint8_t)plen;
+		d=*p; *p=0; // backup char
+	}
+	else
+		cidr->preflen = 32;
+	b = (inet_pton(AF_INET, s, &cidr->addr)==1);
+	if (p) *p=d; // restore char
+	return b;
+}
+bool parse_cidr6(char *s, struct cidr6 *cidr)
+{
+	char *p,d;
+	bool b;
+	unsigned int plen;
+
+	if ((p = strchr(s, '/')))
+	{
+		if (sscanf(p + 1, "%u", &plen)!=1 || plen>128)
+			return false;
+		cidr->preflen = (uint8_t)plen;
+		d=*p; *p=0; // backup char
+	}
+	else
+		cidr->preflen = 128;
+	b = (inet_pton(AF_INET6, s, &cidr->addr)==1);
+	if (p) *p=d; // restore char
+	return b;
+}

nfq/helpers.h

@@ -9,6 +9,16 @@
 #include <stdio.h>
 #include <time.h>
 
+// this saves memory. sockaddr_storage is larger than required. it can be 128 bytes. sockaddr_in6 is 28 bytes.
+typedef union
+{
+	struct sockaddr_in sa4;		// size 16
+	struct sockaddr_in6 sa6;	// size 28
+	char _align[32];		// force 16-byte alignment for ip6_and int128 ops
+} sockaddr_in46;
+
+void rtrim(char *s);
+
 void hexdump_limited_dlog(const uint8_t *data, size_t size, size_t limit);
 char *strncasestr(const char *s,const char *find, size_t slen);
 bool load_file(const char *filename,void *buffer,size_t *buffer_size);
@@ -22,6 +32,8 @@ void ntop46_port(const struct sockaddr *sa, char *str, size_t len);
 bool pton4_port(const char *s, struct sockaddr_in *sa);
 bool pton6_port(const char *s, struct sockaddr_in6 *sa);
 
+uint16_t saport(const struct sockaddr *sa);
+
 bool seq_within(uint32_t s, uint32_t s1, uint32_t s2);
 
 void dbgprint_socket_buffers(int fd);
@@ -64,3 +76,33 @@ void fill_random_az(uint8_t *p,size_t sz);
 void fill_random_az09(uint8_t *p,size_t sz);
 
 bool cd_to_exe_dir(const char *argv0);
+
+
+struct cidr4
+{
+	struct in_addr addr;
+	uint8_t	preflen;
+};
+struct cidr6
+{
+	struct in6_addr addr;
+	uint8_t	preflen;
+};
+void str_cidr4(char *s, size_t s_len, const struct cidr4 *cidr);
+void print_cidr4(const struct cidr4 *cidr);
+void str_cidr6(char *s, size_t s_len, const struct cidr6 *cidr);
+void print_cidr6(const struct cidr6 *cidr);
+bool parse_cidr4(char *s, struct cidr4 *cidr);
+bool parse_cidr6(char *s, struct cidr6 *cidr);
+
+static inline uint32_t mask_from_preflen(uint32_t preflen)
+{
+	return preflen ? preflen<32 ? ~((1 << (32-preflen)) - 1) : 0xFFFFFFFF : 0;
+}
+void ip6_and(const struct in6_addr * restrict a, const struct in6_addr * restrict b, struct in6_addr * restrict result);
+extern struct in6_addr ip6_mask[129];
+void mask_from_preflen6_prepare(void);
+static inline const struct in6_addr *mask_from_preflen6(uint8_t preflen)
+{
+	return ip6_mask+preflen;
+}

nfq/hostlist.c

@@ -174,7 +174,7 @@ static bool LoadIncludeHostListsForProfile(struct desync_profile *dp)
 // return : true = apply fooling, false = do not apply
 bool HostlistCheck(struct desync_profile *dp, const char *host, bool *excluded)
 {
-	DLOG("* Hostlist check for profile %d\n",dp->n);
+	DLOG("* hostlist check for profile %d\n",dp->n);
 	if (*dp->hostlist_auto_filename)
 	{
 		time_t t = file_mod_time(dp->hostlist_auto_filename);

nfq/ipset.c

@@ -0,0 +1,195 @@
+#include <stdio.h>
+#include "ipset.h"
+#include "gzip.h"
+#include "helpers.h"
+
+// inplace tolower() and add to pool
+static bool addpool(ipset *ips, char **s, const char *end, int *ct)
+{
+	char *p, cidr[128];
+	size_t l;
+	struct cidr4 c4;
+	struct cidr6 c6;
+
+	// advance until eol
+	for (p=*s; p<end && *p && *p!='\r' && *p != '\n'; p++);
+
+	// comment line
+	if (!(**s == '#' || **s == ';' || **s == '/' || **s == '\r' || **s == '\n' ))
+	{
+		l = p-*s;
+		if (l>=sizeof(cidr)) l=sizeof(cidr)-1;
+		memcpy(cidr,*s,l);
+		cidr[l]=0;
+		rtrim(cidr);
+
+		if (parse_cidr4(cidr,&c4))
+		{
+			if (!ipset4AddCidr(&ips->ips4, &c4))
+			{
+				ipsetDestroy(ips);
+				return false;
+			}
+			(*ct)++;
+		}
+		else if (parse_cidr6(cidr,&c6))
+		{
+			if (!ipset6AddCidr(&ips->ips6, &c6))
+			{
+				ipsetDestroy(ips);
+				return false;
+			}
+			(*ct)++;
+		}
+		else
+			DLOG_ERR("bad ip or subnet : %s\n",cidr);
+	}
+
+	// advance to the next line
+	for (; p<end && (!*p || *p=='\r' || *p=='\n') ; p++);
+	*s = p;
+	return true;
+
+}
+
+static bool AppendIpset(ipset *ips, const char *filename)
+{
+	char *p, *e, s[256], *zbuf;
+	size_t zsize;
+	int ct = 0;
+	FILE *F;
+	int r;
+
+	DLOG_CONDUP("Loading ipset %s\n",filename);
+
+	if (!(F = fopen(filename, "rb")))
+	{
+		DLOG_ERR("Could not open %s\n", filename);
+		return false;
+	}
+
+	if (is_gzip(F))
+	{
+		r = z_readfile(F,&zbuf,&zsize);
+		fclose(F);
+		if (r==Z_OK)
+		{
+			DLOG_CONDUP("zlib compression detected. uncompressed size : %zu\n", zsize);
+
+			p = zbuf;
+			e = zbuf + zsize;
+			while(p<e)
+			{
+				if (!addpool(ips,&p,e,&ct))
+				{
+					DLOG_ERR("Not enough memory to store ipset : %s\n", filename);
+					free(zbuf);
+					return false;
+				}
+			}
+			free(zbuf);
+		}
+		else
+		{
+			DLOG_ERR("zlib decompression failed : result %d\n",r);
+			return false;
+		}
+	}
+	else
+	{
+		DLOG_CONDUP("loading plain text list\n");
+
+		while (fgets(s, sizeof(s)-1, F))
+		{
+			p = s;
+			if (!addpool(ips,&p,p+strlen(p),&ct))
+			{
+				DLOG_ERR("Not enough memory to store ipset : %s\n", filename);
+				fclose(F);
+				return false;
+			}
+		}
+		fclose(F);
+	}
+
+	DLOG_CONDUP("Loaded %d ip/subnets from %s\n", ct, filename);
+	return true;
+}
+
+static bool LoadIpsets(ipset *ips, struct str_list_head *file_list)
+{
+	struct str_list *file;
+
+	ipsetDestroy(ips);
+
+	LIST_FOREACH(file, file_list, next)
+	{
+		if (!AppendIpset(ips, file->str)) return false;
+	}
+	return true;
+}
+
+bool LoadIncludeIpsets()
+{
+	struct desync_profile_list *dpl;
+	LIST_FOREACH(dpl, &params.desync_profiles, next)
+		if (!LoadIpsets(&dpl->dp.ips, &dpl->dp.ipset_files))
+			return false;
+	return true;
+}
+bool LoadExcludeIpsets()
+{
+	struct desync_profile_list *dpl;
+	LIST_FOREACH(dpl, &params.desync_profiles, next)
+		if (!LoadIpsets(&dpl->dp.ips_exclude, &dpl->dp.ipset_exclude_files))
+			return false;
+	return true;
+}
+
+bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_addr *ipv6)
+{
+	char s_ip[40];
+	bool bInSet=false;
+
+	if (!!ipv4 != !!ipv6)
+	{
+		*s_ip=0;
+		if (ipv4)
+		{
+			if (params.debug) inet_ntop(AF_INET, ipv4, s_ip, sizeof(s_ip));
+			if (ips->ips4) bInSet = ipset4Check(ips->ips4, ipv4, 32);
+		}
+		if (ipv6)
+		{
+			if (params.debug) inet_ntop(AF_INET6, ipv6, s_ip, sizeof(s_ip));
+			if (ips->ips6) bInSet = ipset6Check(ips->ips6, ipv6, 128);
+		}
+		DLOG("ipset check for %s : %s\n", s_ip, bInSet ? "positive" : "negative");
+	}
+	else
+		// ipv4 and ipv6 are both empty or non-empty
+		DLOG("ipset check error !!!!!!!! ipv4=%p ipv6=%p\n",ipv4,ipv6);
+	return bInSet;
+}
+
+static bool IpsetCheck_(const ipset *ips, const ipset *ips_exclude, const struct in_addr *ipv4, const struct in6_addr *ipv6)
+{
+	if (!IPSET_EMPTY(ips_exclude))
+	{
+		DLOG("exclude ");
+		if (SearchIpset(ips_exclude, ipv4, ipv6))
+			return false;
+	}
+	if (!IPSET_EMPTY(ips))
+	{
+		DLOG("include ");
+		return SearchIpset(ips, ipv4, ipv6);
+	}
+	return true;
+}
+
+bool IpsetCheck(struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6)
+{
+	if (!PROFILE_IPSETS_EMPTY(dp)) DLOG("* ipset check for profile %d\n",dp->n);
+	return IpsetCheck_(&dp->ips,&dp->ips_exclude,ipv4,ipv6);
+}

nfq/ipset.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <stdbool.h>
+#include <arpa/inet.h>
+#include "params.h"
+#include "pools.h"
+
+bool LoadIncludeIpsets();
+bool LoadExcludeIpsets();
+bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_addr *ipv6);
+bool IpsetCheck(struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6);

nfq/nfqws.c

@@ -8,6 +8,7 @@
 #include "params.h"
 #include "protocol.h"
 #include "hostlist.h"
+#include "ipset.h"
 #include "gzip.h"
 #include "pools.h"
 
@@ -54,7 +55,7 @@ static bool bHup = false;
 static void onhup(int sig)
 {
 	printf("HUP received !\n");
-	printf("Will reload hostlist on next request (if any)\n");
+	printf("Will reload hostlists and ipsets on next request (if any)\n");
 	bHup = true;
 }
 // should be called in normal execution
@@ -62,7 +63,7 @@ static void dohup(void)
 {
 	if (bHup)
 	{
-		if (!LoadIncludeHostLists() || !LoadExcludeHostLists())
+		if (!LoadIncludeHostLists() || !LoadExcludeHostLists() || !LoadIncludeIpsets() || !LoadExcludeIpsets())
 		{
 			// what will we do without hostlist ?? sure, gonna die
 			exit(1);
@@ -490,6 +491,7 @@ static int win_main(const char *windivert_filter)
 				win_dark_deinit();
 				return w_win32_error;
 			}
+
 			*ifout=0;
 			if (wa.Outbound) snprintf(ifout,sizeof(ifout),"%u.%u", wa.Network.IfIdx, wa.Network.SubIfIdx);
 			DLOG("packet: id=%u len=%zu %s IPv6=%u IPChecksum=%u TCPChecksum=%u UDPChecksum=%u IfIdx=%u.%u\n", id, len, wa.Outbound ? "outbound" : "inbound", wa.IPv6, wa.IPChecksum, wa.TCPChecksum, wa.UDPChecksum, wa.Network.IfIdx, wa.Network.SubIfIdx);
@@ -505,6 +507,8 @@ static int win_main(const char *windivert_filter)
 			}
 			else
 			{
+				dohup();
+
 				mark=0;
 				// pseudo interface id IfIdx.SubIfIdx
 				verdict = processPacketData(&mark, ifout, packet, &len);
@@ -621,7 +625,7 @@ static void load_file_or_exit(const char *filename, void *buf, size_t *size)
 	}
 }
 
-bool parse_autottl(const char *s, autottl *t)
+static bool parse_autottl(const char *s, autottl *t)
 {
 	unsigned int delta,min,max;
 	AUTOTTL_SET_DEFAULT(*t);
@@ -647,6 +651,42 @@ bool parse_autottl(const char *s, autottl *t)
 	return true;
 }
 
+static bool parse_l7_list(char *opt, uint32_t *l7)
+{
+	char *e,*p,c;
+
+	for (p=opt,*l7=0 ; p ; )
+	{
+		if ((e = strchr(p,',')))
+		{
+			c=*e;
+			*e=0;
+		}
+
+		if (!strcmp(p,"http"))
+			*l7 |= L7_PROTO_HTTP;
+		else if (!strcmp(p,"tls"))
+			*l7 |= L7_PROTO_TLS;
+		else if (!strcmp(p,"quic"))
+			*l7 |= L7_PROTO_QUIC;
+		else if (!strcmp(p,"wireguard"))
+			*l7 |= L7_PROTO_WIREGUARD;
+		else if (!strcmp(p,"dht"))
+			*l7 |= L7_PROTO_DHT;
+		else if (!strcmp(p,"unknown"))
+			*l7 |= L7_PROTO_UNKNOWN;
+		else return false;
+
+		if (e)
+		{
+			*e++=c;
+		}
+		p = e;
+	}
+	return true;
+}
+
+
 static bool wf_make_l3(char *opt, bool *ipv4, bool *ipv6)
 {
 	char *e,*p,c;
@@ -673,6 +713,7 @@ static bool wf_make_l3(char *opt, bool *ipv4, bool *ipv6)
 	}
 	return true;
 }
+
 #ifdef __CYGWIN__
 static bool wf_make_pf(char *opt, const char *l4, const char *portname, char *buf, size_t len)
 {
@@ -831,6 +872,9 @@ static void exithelp(void)
 		" --filter-l3=ipv4|ipv6\t\t\t\t; L3 protocol filter. multiple comma separated values allowed.\n"
 		" --filter-tcp=[~]port1[-port2]\t\t\t; TCP port filter. ~ means negation. setting tcp and not setting udp filter denies udp.\n"
 		" --filter-udp=[~]port1[-port2]\t\t\t; UDP port filter. ~ means negation. setting udp and not setting tcp filter denies tcp.\n"
+		" --filter-l7=[http|tls|quic|wireguard|dht|unknown] ; L6-L7 protocol filter. multiple comma separated values allowed.\n"
+		" --ipset=<filename>\t\t\t\t; ipset include filter (one ip/CIDR per line, ipv4 and ipv6 accepted, gzip supported, multiple ipsets allowed)\n"
+		" --ipset-exclude=<filename>\t\t\t; ipset exclude filter (one ip/CIDR per line, ipv4 and ipv6 accepted, gzip supported, multiple ipsets allowed)\n"
 		"\nHOSTLIST FILTER:\n"
 		" --hostlist=<filename>\t\t\t\t; apply dpi desync only to the listed hosts (one host per line, subdomains auto apply, gzip supported, multiple hostlists allowed)\n"
 		" --hostlist-exclude=<filename>\t\t\t; do not apply dpi desync to the listed hosts (one host per line, subdomains auto apply, gzip supported, multiple hostlists allowed)\n"
@@ -945,6 +989,7 @@ int main(int argc, char **argv)
 #endif
 
 	srandom(time(NULL));
+	mask_from_preflen6_prepare();
 
 	memset(&params, 0, sizeof(params));
 	*pidfile = 0;
@@ -1056,19 +1101,22 @@ int main(int argc, char **argv)
 		{"filter-l3",required_argument,0,0},	// optidx=53
 		{"filter-tcp",required_argument,0,0},	// optidx=54
 		{"filter-udp",required_argument,0,0},	// optidx=55
+		{"filter-l7",required_argument,0,0},	// optidx=56
+		{"ipset",required_argument,0,0},	// optidx=57
+		{"ipset-exclude",required_argument,0,0},// optidx=58
 #ifdef __linux__
-		{"bind-fix4",no_argument,0,0},		// optidx=56
+		{"bind-fix4",no_argument,0,0},		// optidx=59
-		{"bind-fix6",no_argument,0,0},		// optidx=57
+		{"bind-fix6",no_argument,0,0},		// optidx=60
 #elif defined(__CYGWIN__)
-		{"wf-iface",required_argument,0,0},	// optidx=56
+		{"wf-iface",required_argument,0,0},	// optidx=59
-		{"wf-l3",required_argument,0,0},	// optidx=57
+		{"wf-l3",required_argument,0,0},	// optidx=60
-		{"wf-tcp",required_argument,0,0},	// optidx=58
+		{"wf-tcp",required_argument,0,0},	// optidx=61
-		{"wf-udp",required_argument,0,0},	// optidx=59
+		{"wf-udp",required_argument,0,0},	// optidx=62
-		{"wf-raw",required_argument,0,0},	// optidx=60
+		{"wf-raw",required_argument,0,0},	// optidx=63
-		{"wf-save",required_argument,0,0},	// optidx=61
+		{"wf-save",required_argument,0,0},	// optidx=64
-		{"ssid-filter",required_argument,0,0},	// optidx=62
+		{"ssid-filter",required_argument,0,0},	// optidx=65
-		{"nlm-filter",required_argument,0,0},	// optidx=63
+		{"nlm-filter",required_argument,0,0},	// optidx=66
-		{"nlm-list",optional_argument,0,0},	// optidx=64
+		{"nlm-list",optional_argument,0,0},	// optidx=67
 #endif
 		{NULL,0,NULL,0}
 	};
@@ -1596,30 +1644,53 @@ int main(int argc, char **argv)
 			// deny tcp if not set
 			if (pf_is_empty(&dp->pf_tcp)) dp->pf_tcp.neg=true;
 			break;
+		case 56: /* filter-l7 */
+			if (!parse_l7_list(optarg,&dp->filter_l7))
+			{
+				DLOG_ERR("Invalid l7 filter : %s\n",optarg);
+				exit_clean(1);
+			}
+			break;
+		case 57: /* ipset */
+			if (!strlist_add(&dp->ipset_files, optarg))
+			{
+				DLOG_ERR("strlist_add failed\n");
+				exit_clean(1);
+			}
+			break;
+		case 58: /* ipset-exclude */
+			if (!strlist_add(&dp->ipset_exclude_files, optarg))
+			{
+				DLOG_ERR("strlist_add failed\n");
+				exit_clean(1);
+			}
+			break;
+
+
 
 #ifdef __linux__
-		case 56: /* bind-fix4 */
+		case 59: /* bind-fix4 */
 			params.bind_fix4 = true;
 			break;
-		case 57: /* bind-fix6 */
+		case 60: /* bind-fix6 */
 			params.bind_fix6 = true;
 			break;
 #elif defined(__CYGWIN__)
-		case 56: /* wf-iface */
+		case 59: /* wf-iface */
 			if (!sscanf(optarg,"%u.%u",&IfIdx,&SubIfIdx))
 			{
 				DLOG_ERR("bad value for --wf-iface\n");
 				exit_clean(1);
 			}
 			break;
-		case 57: /* wf-l3 */
+		case 60: /* wf-l3 */
 			if (!wf_make_l3(optarg,&wf_ipv4,&wf_ipv6))
 			{
 				DLOG_ERR("bad value for --wf-l3\n");
 				exit_clean(1);
 			}
 			break;
-		case 58: /* wf-tcp */
+		case 61: /* wf-tcp */
 			hash_wf_tcp=hash_jen(optarg,strlen(optarg));
 			if (!wf_make_pf(optarg,"tcp","SrcPort",wf_pf_tcp_src,sizeof(wf_pf_tcp_src)) ||
 				!wf_make_pf(optarg,"tcp","DstPort",wf_pf_tcp_dst,sizeof(wf_pf_tcp_dst)))
@@ -1628,7 +1699,7 @@ int main(int argc, char **argv)
 				exit_clean(1);
 			}
 			break;
-		case 59: /* wf-udp */
+		case 62: /* wf-udp */
 			hash_wf_udp=hash_jen(optarg,strlen(optarg));
 			if (!wf_make_pf(optarg,"udp","SrcPort",wf_pf_udp_src,sizeof(wf_pf_udp_src)) ||
 				!wf_make_pf(optarg,"udp","DstPort",wf_pf_udp_dst,sizeof(wf_pf_udp_dst)))
@@ -1637,7 +1708,7 @@ int main(int argc, char **argv)
 				exit_clean(1);
 			}
 			break;
-		case 60: /* wf-raw */
+		case 63: /* wf-raw */
 			hash_wf_raw=hash_jen(optarg,strlen(optarg));
 			if (optarg[0]=='@')
 			{
@@ -1651,11 +1722,11 @@ int main(int argc, char **argv)
 				windivert_filter[sizeof(windivert_filter) - 1] = '\0';
 			}
 			break;
-		case 61: /* wf-save */
+		case 64: /* wf-save */
 			strncpy(wf_save_file, optarg, sizeof(wf_save_file));
 			wf_save_file[sizeof(wf_save_file) - 1] = '\0';
 			break;
-		case 62: /* ssid-filter */
+		case 65: /* ssid-filter */
 			hash_ssid_filter=hash_jen(optarg,strlen(optarg));
 			{
 				char *e,*p = optarg;
@@ -1673,7 +1744,7 @@ int main(int argc, char **argv)
 				}
 			}
 			break;
-		case 63: /* nlm-filter */
+		case 66: /* nlm-filter */
 			hash_nlm_filter=hash_jen(optarg,strlen(optarg));
 			{
 				char *e,*p = optarg;
@@ -1691,7 +1762,7 @@ int main(int argc, char **argv)
 				}
 			}
 			break;
-		case 64: /* nlm-list */
+		case 67: /* nlm-list */
 			if (!nlm_list(optarg && !strcmp(optarg,"all")))
 			{
 				DLOG_ERR("could not get list of NLM networks\n");
@@ -1776,9 +1847,9 @@ int main(int argc, char **argv)
 		if (dp->desync_ttl6 == 0xFF) dp->desync_ttl6=dp->desync_ttl;
 		if (!AUTOTTL_ENABLED(dp->desync_autottl6)) dp->desync_autottl6 = dp->desync_autottl;
 		if (AUTOTTL_ENABLED(dp->desync_autottl))
-			DLOG("[profile %d] autottl ipv4 %u:%u-%u\n",v,dp->desync_autottl.delta,dp->desync_autottl.min,dp->desync_autottl.max);
+			DLOG("[profile %d] autottl ipv4 %u:%u-%u\n",dp->n,dp->desync_autottl.delta,dp->desync_autottl.min,dp->desync_autottl.max);
 		if (AUTOTTL_ENABLED(dp->desync_autottl6))
-			DLOG("[profile %d] autottl ipv6 %u:%u-%u\n",v,dp->desync_autottl6.delta,dp->desync_autottl6.min,dp->desync_autottl6.max);
+			DLOG("[profile %d] autottl ipv6 %u:%u-%u\n",dp->n,dp->desync_autottl6.delta,dp->desync_autottl6.min,dp->desync_autottl6.max);
 		if (dp->desync_split_tls==tlspos_none && dp->desync_split_pos) dp->desync_split_tls=tlspos_pos;
 		if (dp->desync_split_http_req==httpreqpos_none && dp->desync_split_pos) dp->desync_split_http_req=httpreqpos_pos;
 	}
@@ -1793,6 +1864,16 @@ int main(int argc, char **argv)
 		DLOG_ERR("Exclude hostlists load failed\n");
 		exit_clean(1);
 	}
+	if (!LoadIncludeIpsets())
+	{
+		DLOG_ERR("Include ipset load failed\n");
+		exit_clean(1);
+	}
+	if (!LoadExcludeIpsets())
+	{
+		DLOG_ERR("Exclude ipset load failed\n");
+		exit_clean(1);
+	}
 	
 	if (daemon) daemonize();
 

nfq/params.c

@@ -207,8 +207,12 @@ static void dp_entry_destroy(struct desync_profile_list *entry)
 {
 	strlist_destroy(&entry->dp.hostlist_files);
 	strlist_destroy(&entry->dp.hostlist_exclude_files);
+	strlist_destroy(&entry->dp.ipset_files);
+	strlist_destroy(&entry->dp.ipset_exclude_files);
 	StrPoolDestroy(&entry->dp.hostlist_exclude);
 	StrPoolDestroy(&entry->dp.hostlist);
+	ipsetDestroy(&entry->dp.ips);
+	ipsetDestroy(&entry->dp.ips_exclude);
 	HostFailPoolDestroy(&entry->dp.hostlist_auto_fail_counters);
 	free(entry);
 }

nfq/params.h

@@ -66,6 +66,10 @@ struct desync_profile
 
 	bool filter_ipv4,filter_ipv6;
 	port_filter pf_tcp,pf_udp;
+	uint32_t filter_l7;	// L7_PROTO_* bits
+	ipset ips,ips_exclude;
+	struct str_list_head ipset_files, ipset_exclude_files;
+
 	strpool *hostlist, *hostlist_exclude;
 	struct str_list_head hostlist_files, hostlist_exclude_files;
 	char hostlist_auto_filename[PATH_MAX];
@@ -74,6 +78,8 @@ struct desync_profile
 	hostfail_pool *hostlist_auto_fail_counters;
 };
 
+#define PROFILE_IPSETS_EMPTY(dp) (IPSET_EMPTY(&dp->ips) && IPSET_EMPTY(&dp->ips_exclude))
+
 struct desync_profile_list {
 	struct desync_profile dp;
 	LIST_ENTRY(desync_profile_list) next;

nfq/pools.c

@@ -151,3 +151,127 @@ void strlist_destroy(struct str_list_head *head)
 		strlist_entry_destroy(entry);
 	}
 }
+
+
+
+void ipset4Destroy(ipset4 **ipset)
+{
+	ipset4 *elem, *tmp;
+	HASH_ITER(hh, *ipset, elem, tmp)
+	{
+		HASH_DEL(*ipset, elem);
+		free(elem);
+	}
+}
+bool ipset4Check(ipset4 *ipset, const struct in_addr *a, uint8_t preflen)
+{
+	uint32_t ip = ntohl(a->s_addr);
+	struct cidr4 cidr;
+	ipset4 *ips_found;
+
+	// zero alignment bytes
+	memset(&cidr,0,sizeof(cidr));
+	cidr.preflen = preflen+1;
+	do
+	{
+		cidr.preflen--;
+		cidr.addr.s_addr = htonl(ip & mask_from_preflen(cidr.preflen));
+		HASH_FIND(hh, ipset, &cidr, sizeof(cidr), ips_found);
+		if (ips_found) return true;
+	} while(cidr.preflen);
+
+	return false;
+}
+bool ipset4Add(ipset4 **ipset, const struct in_addr *a, uint8_t preflen)
+{
+	if (preflen>32) return false;
+
+	// avoid dups
+	if (ipset4Check(*ipset, a, preflen)) return true; // already included
+
+	struct ipset4 *entry = calloc(1,sizeof(ipset4));
+	if (!entry) return false;
+
+	entry->cidr.addr.s_addr = htonl(ntohl(a->s_addr) & mask_from_preflen(preflen));
+	entry->cidr.preflen = preflen;
+	oom = false;
+	HASH_ADD(hh, *ipset, cidr, sizeof(entry->cidr), entry);
+	if (oom) { free(entry); return false; }
+
+	return true;
+}
+void ipset4Print(ipset4 *ipset)
+{
+	ipset4 *ips, *tmp;
+	HASH_ITER(hh, ipset , ips, tmp)
+	{
+		print_cidr4(&ips->cidr);
+		printf("\n");
+	}
+}
+
+void ipset6Destroy(ipset6 **ipset)
+{
+	ipset6 *elem, *tmp;
+	HASH_ITER(hh, *ipset, elem, tmp)
+	{
+		HASH_DEL(*ipset, elem);
+		free(elem);
+	}
+}
+bool ipset6Check(ipset6 *ipset, const struct in6_addr *a, uint8_t preflen)
+{
+	struct cidr6 cidr;
+	ipset6 *ips_found;
+
+	// zero alignment bytes
+	memset(&cidr,0,sizeof(cidr));
+	cidr.preflen = preflen+1;
+	do
+	{
+		cidr.preflen--;
+		ip6_and(a, mask_from_preflen6(cidr.preflen), &cidr.addr);
+		HASH_FIND(hh, ipset, &cidr, sizeof(cidr), ips_found);
+		if (ips_found) return true;
+	} while(cidr.preflen);
+
+	return false;
+}
+bool ipset6Add(ipset6 **ipset, const struct in6_addr *a, uint8_t preflen)
+{
+	if (preflen>128) return false;
+
+	// avoid dups
+	if (ipset6Check(*ipset, a, preflen)) return true; // already included
+
+	struct ipset6 *entry = calloc(1,sizeof(ipset6));
+	if (!entry) return false;
+
+	ip6_and(a, mask_from_preflen6(preflen), &entry->cidr.addr);
+	entry->cidr.preflen = preflen;
+	oom = false;
+	HASH_ADD(hh, *ipset, cidr, sizeof(entry->cidr), entry);
+	if (oom) { free(entry); return false; }
+
+	return true;
+}
+void ipset6Print(ipset6 *ipset)
+{
+	ipset6 *ips, *tmp;
+	HASH_ITER(hh, ipset , ips, tmp)
+	{
+		print_cidr6(&ips->cidr);
+		printf("\n");
+	}
+}
+
+void ipsetDestroy(ipset *ipset)
+{
+	ipset4Destroy(&ipset->ips4);
+	ipset6Destroy(&ipset->ips6);
+}
+void ipsetPrint(ipset *ipset)
+{
+	ipset4Print(ipset->ips4);
+	ipset6Print(ipset->ips6);
+}

nfq/pools.h

@@ -5,6 +5,8 @@
 #include <sys/queue.h>
 #include <time.h>
 
+#include "helpers.h"
+
 //#define HASH_BLOOM 20
 #define HASH_NONFATAL_OOM 1
 #define HASH_FUNCTION HASH_BER
@@ -43,3 +45,41 @@ void HostFailPoolDump(hostfail_pool *p);
 
 bool strlist_add(struct str_list_head *head, const char *filename);
 void strlist_destroy(struct str_list_head *head);
+
+
+typedef struct ipset4 {
+	struct cidr4 cidr;	/* key */
+	UT_hash_handle hh;	/* makes this structure hashable */
+} ipset4;
+typedef struct ipset6 {
+	struct cidr6 cidr;	/* key */
+	UT_hash_handle hh;	/* makes this structure hashable */
+} ipset6;
+// combined ipset ipv4 and ipv6
+typedef struct ipset {
+	ipset4 *ips4;
+	ipset6 *ips6;
+} ipset;
+
+#define IPSET_EMPTY(ips) (!(ips)->ips4 && !(ips)->ips6)
+
+void ipset4Destroy(ipset4 **ipset);
+bool ipset4Add(ipset4 **ipset, const struct in_addr *a, uint8_t preflen);
+static inline bool ipset4AddCidr(ipset4 **ipset, const struct cidr4 *cidr)
+{
+	return ipset4Add(ipset,&cidr->addr,cidr->preflen);
+}
+bool ipset4Check(ipset4 *ipset, const struct in_addr *a, uint8_t preflen);
+void ipset4Print(ipset4 *ipset);
+
+void ipset6Destroy(ipset6 **ipset);
+bool ipset6Add(ipset6 **ipset, const struct in6_addr *a, uint8_t preflen);
+static inline bool ipset6AddCidr(ipset6 **ipset, const struct cidr6 *cidr)
+{
+	return ipset6Add(ipset,&cidr->addr,cidr->preflen);
+}
+bool ipset6Check(ipset6 *ipset, const struct in6_addr *a, uint8_t preflen);
+void ipset6Print(ipset6 *ipset);
+
+void ipsetDestroy(ipset *ipset);
+void ipsetPrint(ipset *ipset);

tpws/helpers.c

@@ -321,13 +321,9 @@ __attribute__((optimize ("no-strict-aliasing")))
 #endif
 void ip6_and(const struct in6_addr * restrict a, const struct in6_addr * restrict b, struct in6_addr * restrict result)
 {
-#ifdef __SIZEOF_INT128__
+	// int 128 can cause alignment segfaults because sin6_addr in struct sockaddr_in6 is 8-byte aligned, not 16-byte
-	// gcc and clang have 128 bit int types on some 64-bit archs. take some advantage
-	*((unsigned __int128*)result->s6_addr) = *((unsigned __int128*)a->s6_addr) & *((unsigned __int128*)b->s6_addr);
-#else
 	((uint64_t*)result->s6_addr)[0] = ((uint64_t*)a->s6_addr)[0] & ((uint64_t*)b->s6_addr)[0];
 	((uint64_t*)result->s6_addr)[1] = ((uint64_t*)a->s6_addr)[1] & ((uint64_t*)b->s6_addr)[1];
-#endif
 }
 
 void str_cidr4(char *s, size_t s_len, const struct cidr4 *cidr)

tpws/helpers.h

@@ -13,7 +13,6 @@ typedef union
 {
 	struct sockaddr_in sa4;		// size 16
 	struct sockaddr_in6 sa6;	// size 28
-	char _align[32];		// force 16-byte alignment for ip6_and int128 ops
 } sockaddr_in46;
 
 void rtrim(char *s);