Merge 92ba6b439e6aed8557a315ebfcd350016159be52 into c6e729b237be9a30817ce0b4e8a7ab49574a9e2f

blockcheck.sh

@@ -23,6 +23,7 @@ CURL=${CURL:-curl}
 . "$ZAPRET_BASE/common/fwtype.sh"
 . "$ZAPRET_BASE/common/virt.sh"
 
+DOMAINS_DEFAULT=${DOMAINS_DEFAULT:-rutracker.org}
 QNUM=${QNUM:-59780}
 SOCKS_PORT=${SOCKS_PORT:-1993}
 TPWS_UID=${TPWS_UID:-1}
@@ -35,9 +36,9 @@ MDIG=${MDIG:-${ZAPRET_BASE}/mdig/mdig}
 DESYNC_MARK=0x10000000
 IPFW_RULE_NUM=${IPFW_RULE_NUM:-1}
 IPFW_DIVERT_PORT=${IPFW_DIVERT_PORT:-59780}
-DOMAINS=${DOMAINS:-rutracker.org}
 CURL_MAX_TIME=${CURL_MAX_TIME:-2}
 CURL_MAX_TIME_QUIC=${CURL_MAX_TIME_QUIC:-$CURL_MAX_TIME}
+CURL_MAX_TIME_DOH=${CURL_MAX_TIME_DOH:-2}
 MIN_TTL=${MIN_TTL:-1}
 MAX_TTL=${MAX_TTL:-12}
 USER_AGENT=${USER_AGENT:-Mozilla}
@@ -45,8 +46,9 @@ HTTP_PORT=${HTTP_PORT:-80}
 HTTPS_PORT=${HTTPS_PORT:-443}
 QUIC_PORT=${QUIC_PORT:-443}
 UNBLOCKED_DOM=${UNBLOCKED_DOM:-iana.org}
+PARALLEL_OUT=/tmp/zapret_parallel
 
-HDRTEMP=/tmp/zapret-hdr.txt
+HDRTEMP=/tmp/zapret-hdr
 
 NFT_TABLE=blockcheck
 
@@ -77,9 +79,11 @@ exitp()
 {
 	local A
 
-	echo
+	[ "$BATCH" = 1 ] || {
-	echo press enter to continue
+		echo
-	read A
+		echo press enter to continue
+		read A
+	}
 	exit $1
 }
 
@@ -212,7 +216,7 @@ doh_resolve()
 	# $1 - ip version 4/6
 	# $2 - hostname
 	# $3 - doh server URL. use $DOH_SERVER if empty
-	$MDIG --family=$1 --dns-make-query=$2 | $CURL -s --data-binary @- -H "Content-Type: application/dns-message" "${3:-$DOH_SERVER}" | $MDIG --dns-parse-query
+	$MDIG --family=$1 --dns-make-query=$2 | $CURL --max-time $CURL_MAX_TIME_DOH -s --data-binary @- -H "Content-Type: application/dns-message" "${3:-$DOH_SERVER}" | $MDIG --dns-parse-query
 }
 doh_find_working()
 {
@@ -560,7 +564,7 @@ curl_supports_tls13()
 	[ $? = 2 ] && return 1
 	# curl can have tlsv1.3 key present but ssl library without TLS 1.3 support
 	# this is online test because there's no other way to trigger library incompatibility case
-	$CURL --tlsv1.3 --max-time $CURL_MAX_TIME -Is -o /dev/null https://iana.org 2>/dev/null
+	$CURL --tlsv1.3 --max-time 1 -Is -o /dev/null https://iana.org 2>/dev/null
 	r=$?
 	[ $r != 4 -a $r != 35 ]
 }
@@ -651,28 +655,28 @@ curl_test_http()
 	# $3 - subst ip
 	# $4 - "detail" - detail info
 
-	local code loc
+	local code loc hdrt="${HDRTEMP}_${!:-$$}.txt"
-	curl_probe $1 $2 $HTTP_PORT "$3" -SsD "$HDRTEMP" -A "$USER_AGENT" --max-time $CURL_MAX_TIME $CURL_OPT "http://$2" -o /dev/null 2>&1 || {
+	curl_probe $1 $2 $HTTP_PORT "$3" -SsD "$hdrt" -A "$USER_AGENT" --max-time $CURL_MAX_TIME $CURL_OPT "http://$2" -o /dev/null 2>&1 || {
 		code=$?
-		rm -f "$HDRTEMP"
+		rm -f "$hdrt"
 		return $code
 	}
 	if [ "$4" = "detail" ] ; then
-		head -n 1 "$HDRTEMP"
+		head -n 1 "$hdrt"
-		grep "^[lL]ocation:" "$HDRTEMP"
+		grep "^[lL]ocation:" "$hdrt"
 	else
-		code=$(hdrfile_http_code "$HDRTEMP")
+		code=$(hdrfile_http_code "$hdrt")
 		[ "$code" = 301 -o "$code" = 302 -o "$code" = 307 -o "$code" = 308 ] && {
-			loc=$(hdrfile_location "$HDRTEMP")
+			loc=$(hdrfile_location "$hdrt")
 			echo "$loc" | grep -qE "^https?://.*$2(/|$)" ||
 			echo "$loc" | grep -vqE '^https?://' || {
 				echo suspicious redirection $code to : $loc
-				rm -f "$HDRTEMP"
+				rm -f "$hdrt"
 				return 254
 			}
 		}
 	fi
-	rm -f "$HDRTEMP"
+	rm -f "$hdrt"
 	[ "$code" = 400 ] && {
 		# this can often happen if the server receives fake packets it should not receive
 		echo http code $code. likely the server receives fakes.
@@ -964,18 +968,38 @@ curl_test()
 	# $2 - domain
 	# $3 - subst ip
 	# $4 - param of test function
-	local code=0 n=0
+	local code=0 n=0 p pids
 
-	while [ $n -lt $REPEATS ]; do
+	if [ "$PARALLEL" = 1 ]; then
-		n=$(($n+1))
+		rm -f "${PARALLEL_OUT}"*
-		[ $REPEATS -gt 1 ] && printf "[attempt $n] "
+		for n in $(seq -s ' ' 1 $REPEATS); do
-		if $1 "$IPV" $2 $3 "$4" ; then
+			$1 "$IPV" $2 $3 "$4" >"${PARALLEL_OUT}_$n" &
-			[ $REPEATS -gt 1 ] && echo 'AVAILABLE'
+			pids=${pids:+$pids }$!
-		else
+		done
-			code=$?
+		n=1
-			[ "$SCANLEVEL" = quick ] && break
+		for p in $pids; do
-		fi
+			[ $REPEATS -gt 1 ] && printf "[attempt $n] "
-	done
+			if wait $p; then
+				[ $REPEATS -gt 1 ] && echo 'AVAILABLE'
+			else
+				code=$?
+				cat "${PARALLEL_OUT}_$n"
+			fi
+			n=$(($n+1))
+		done
+		rm -f "${PARALLEL_OUT}"*
+	else
+		while [ $n -lt $REPEATS ]; do
+			n=$(($n+1))
+			[ $REPEATS -gt 1 ] && printf "[attempt $n] "
+			if $1 "$IPV" $2 $3 "$4" ; then
+				[ $REPEATS -gt 1 ] && echo 'AVAILABLE'
+			else
+				code=$?
+				[ "$SCANLEVEL" = quick ] && break
+			fi
+		done
+	fi
 	[ "$4" = detail ] || {
 		if [ $code = 254 ]; then
 			echo "UNAVAILABLE"
@@ -1707,76 +1731,119 @@ ask_params()
 		exitp 1
 	}
 
-
-	echo "specify domain(s) to test. multiple domains are space separated."
-	printf "domain(s) (default: $DOMAINS) : "
 	local dom
-	read dom
+	[ -n "$DOMAINS" ] || {
-	[ -n "$dom" ] && DOMAINS="$dom"
+		DOMAINS="$DOMAINS_DEFAULT"
+		[ "$BATCH" = 1 ] || {
+			echo "specify domain(s) to test. multiple domains are space separated."
+			printf "domain(s) (default: $DOMAINS) : "
+			read dom
+			[ -n "$dom" ] && DOMAINS="$dom"
+		}
+	}
 
 	local IPVS_def=4
-	# yandex public dns
+	[ -n "$IPVS" ] || {
-	pingtest 6 2a02:6b8::feed:0ff && IPVS_def=46
+		# yandex public dns
-	printf "ip protocol version(s) - 4, 6 or 46 for both (default: $IPVS_def) : "
+		pingtest 6 2a02:6b8::feed:0ff && IPVS_def=46
-	read IPVS
+		[ "$BATCH" = 1 ] || {
-	[ -n "$IPVS" ] || IPVS=$IPVS_def
+			printf "ip protocol version(s) - 4, 6 or 46 for both (default: $IPVS_def) : "
-	[ "$IPVS" = 4 -o "$IPVS" = 6 -o "$IPVS" = 46 ] || {
+			read IPVS
-		echo 'invalid ip version(s). should be 4, 6 or 46.'
+		}
-		exitp 1
+		[ -n "$IPVS" ] || IPVS=$IPVS_def
+		[ "$IPVS" = 4 -o "$IPVS" = 6 -o "$IPVS" = 46 ] || {
+			echo 'invalid ip version(s). should be 4, 6 or 46.'
+			exitp 1
+		}
 	}
 	[ "$IPVS" = 46 ] && IPVS="4 6"
 
 	configure_curl_opt
 
-	ENABLE_HTTP=1
+	[ -n "$ENABLE_HTTP" ] || {
-	echo
+		ENABLE_HTTP=1
-	ask_yes_no_var ENABLE_HTTP "check http"
+		[ "$BATCH" = 1 ] || {
-
+			echo
-	ENABLE_HTTPS_TLS12=1
+			ask_yes_no_var ENABLE_HTTP "check http"
-	echo
+		}
-	ask_yes_no_var ENABLE_HTTPS_TLS12 "check https tls 1.2"
-
-	ENABLE_HTTPS_TLS13=0
-	echo
-	if [ -n "$TLS13" ]; then
-		echo "TLS 1.3 uses encrypted ServerHello. DPI cannot check domain name in server response."
-		echo "This can allow more bypass strategies to work."
-		echo "What works for TLS 1.2 will also work for TLS 1.3 but not vice versa."
-		echo "Most sites nowadays support TLS 1.3 but not all. If you can't find a strategy for TLS 1.2 use this test."
-		echo "TLS 1.3 only strategy is better than nothing."
-		ask_yes_no_var ENABLE_HTTPS_TLS13 "check https tls 1.3"
-	else
-		echo "installed curl version does not support TLS 1.3 . tests disabled."
-	fi
-
-	ENABLE_HTTP3=0
-	echo
-	if [ -n "$HTTP3" ]; then
-		echo "make sure target domain(s) support QUIC or result will be negative in any case"
-		ENABLE_HTTP3=1
-		ask_yes_no_var ENABLE_HTTP3 "check http3 QUIC"
-	else
-		echo "installed curl version does not support http3 QUIC. tests disabled."
-	fi
-
-	echo
-	echo "sometimes ISPs use multiple DPIs or load balancing. bypass strategies may work unstable."
-	printf "how many times to repeat each test (default: 1) : "
-	read REPEATS
-	REPEATS=$((0+${REPEATS:-1}))
-	[ "$REPEATS" = 0 ] && {
-		echo invalid repeat count
-		exitp 1
 	}
 
-	echo
+	[ -n "$ENABLE_HTTPS_TLS12" ] || {
-	echo quick    - scan as fast as possible to reveal any working strategy
+		ENABLE_HTTPS_TLS12=1
-	echo standard - do investigation what works on your DPI
+		[ "$BATCH" = 1 ] || {
-	echo force    - scan maximum despite of result
+			echo
-	SCANLEVEL=${SCANLEVEL:-standard}
+			ask_yes_no_var ENABLE_HTTPS_TLS12 "check https tls 1.2"
-	ask_list SCANLEVEL "quick standard force" "$SCANLEVEL"
+		}
-	# disable tpws checks by default in quick mode
+	}
-	[ "$SCANLEVEL" = quick -a -z "$SKIP_TPWS" -a "$UNAME" != Darwin ] && SKIP_TPWS=1
+
+	[ -n "$ENABLE_HTTPS_TLS13" ] || {
+		ENABLE_HTTPS_TLS13=0
+		if [ -n "$TLS13" ]; then
+			[ "$BATCH" = 1 ] || {
+				echo
+				echo "TLS 1.3 uses encrypted ServerHello. DPI cannot check domain name in server response."
+				echo "This can allow more bypass strategies to work."
+				echo "What works for TLS 1.2 will also work for TLS 1.3 but not vice versa."
+				echo "Most sites nowadays support TLS 1.3 but not all. If you can't find a strategy for TLS 1.2 use this test."
+				echo "TLS 1.3 only strategy is better than nothing."
+				ask_yes_no_var ENABLE_HTTPS_TLS13 "check https tls 1.3"
+			}
+		else
+			echo
+			echo "installed curl version does not support TLS 1.3 . tests disabled."
+		fi
+	}
+
+	[ -n "$ENABLE_HTTP3" ] || {
+		ENABLE_HTTP3=0
+		if [ -n "$HTTP3" ]; then
+			ENABLE_HTTP3=1
+			[ "$BATCH" = 1 ] || {
+				echo
+				echo "make sure target domain(s) support QUIC or result will be negative in any case"
+				ask_yes_no_var ENABLE_HTTP3 "check http3 QUIC"
+			}
+		else
+			echo
+			echo "installed curl version does not support http3 QUIC. tests disabled."
+		fi
+	}
+
+	[ -n "$REPEATS" ] || {
+		[ "$BATCH" = 1 ] || {
+			echo
+			echo "sometimes ISPs use multiple DPIs or load balancing. bypass strategies may work unstable."
+			printf "how many times to repeat each test (default: 1) : "
+			read REPEATS
+		}
+		REPEATS=$((0+${REPEATS:-1}))
+		[ "$REPEATS" = 0 ] && {
+			echo invalid repeat count
+			exitp 1
+		}
+	}
+	[ -z "$PARALLEL" -a $REPEATS -gt 1 ] && {
+		PARALLEL=0
+		[ "$BATCH" = 1 ] || {
+			echo
+			echo "parallel scan can greatly increase speed but may also trigger DDoS protection and cause false result"
+			ask_yes_no_var PARALLEL "enable parallel scan"
+		}
+	}
+	PARALLEL=${PARALLEL:-0}
+
+	[ -n "$SCANLEVEL" ] || {
+		SCANLEVEL=standard
+		[ "$BATCH" = 1 ] || {
+			echo
+			echo quick    - scan as fast as possible to reveal any working strategy
+			echo standard - do investigation what works on your DPI
+			echo force    - scan maximum despite of result
+			ask_list SCANLEVEL "quick standard force" "$SCANLEVEL"
+			# disable tpws checks by default in quick mode
+			[ "$SCANLEVEL" = quick -a -z "$SKIP_TPWS" -a "$UNAME" != Darwin ] && SKIP_TPWS=1
+		}
+	}
 
 	echo
 
@@ -1981,14 +2048,14 @@ check_dns()
 unprepare_all()
 {
 	# make sure we are not in a middle state that impacts connectivity
-	rm -f "$HDRTEMP"
+	ws_kill
 	[ -n "$IPV" ] && {
 		pktws_ipt_unprepare_tcp 80
 		pktws_ipt_unprepare_tcp 443
 		pktws_ipt_unprepare_udp 443
 	}
-	ws_kill
 	cleanup
+	rm -f "${HDRTEMP}"* "${PARALLEL_OUT}"*
 }
 sigint()
 {

docs/changes.txt

@@ -440,4 +440,9 @@ v69.9
 init.d: exclude ipban from tpws redirection
 macos: fix install_easy
 macos: fix national decimal separator in sleep
-nfqws: apply relative markers to partial TLS ClientHello
+ipset: scripts maintenance
+
+v70
+
+blockcheck: override all dialog questions and enable batch mode
+blockcheck: parallel attempts

docs/readme.en.md

@@ -1,4 +1,4 @@
-# zapret v69.9
+# zapret v70
 
 # SCAMMER WARNING
 

docs/readme.md

@@ -1,4 +1,4 @@
-# zapret v69.9
+# zapret v70
 
 # ВНИМАНИЕ, остерегайтесь мошенников
 
@@ -1427,9 +1427,19 @@ linux, но через раз приобретает статус INVALID в con
 CURL - замена программы curl
 CURL_MAX_TIME - время таймаута curl в секундах
 CURL_MAX_TIME_QUIC - время таймаута curl для quic. если не задано, используется значение CURL_MAX_TIME
+CURL_MAX_TIME_DOH - время таймаута curl для DoH серверов
 CURL_CMD=1 - показывать команды curl
 CURL_OPT - дополнительные параметры curl. `-k` - игнор сертификатов. `-v` - подробный вывод протокола
 DOMAINS - список тестируемых доменов через пробел
+IPVS=4|6|46 - тестируемые версии ip протокола
+ENABLE_HTTP=0|1 - включить тест plain http
+ENABLE_HTTPS_TLS12=0|1 - включить тест https TLS 1.2
+ENABLE_HTTPS_TLS13=0|1 - включить тест https TLS 1.3
+ENABLE_HTTP3=0|1 - включить тест QUIC
+REPEATS - количество попыток тестирования
+PARALLEL=0|1 - включить параллельные попытки. может обидеть сайт из-за долбежки и привести к неверному результату
+SCANLEVEL=quick|standard|force - уровень сканирования
+BATCH=1 - пакетный режим без вопросов и ожидания ввода в консоли
 HTTP_PORT, HTTPS_PORT, QUIC_PORT - номера портов для соответствующих протоколов
 SKIP_DNSCHECK=1 - отказ от проверки DNS
 SKIP_TPWS=1 - отказ от тестов tpws

ipset/get_antifilter_domains.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+IPSET_DIR="$(dirname "$0")"
+IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
+
+. "$IPSET_DIR/def.sh"
+
+TMPLIST="$TMPDIR/list.txt"
+
+URL="https://antifilter.download/list/domains.lst"
+
+dl()
+{
+  # $1 - url
+  # $2 - file
+  # $3 - minsize
+  # $4 - maxsize
+  curl -L -H "Accept-Encoding: gzip" -k --fail --max-time 60 --connect-timeout 10 --retry 4 --max-filesize 251658240 -o "$TMPLIST" "$1" ||
+  {
+   echo list download failed : $1
+   exit 2
+  }
+  dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
+  if test $dlsize -lt $3; then
+   echo list is too small : $dlsize bytes. can be bad.
+   exit 2
+  fi
+  zzcat "$TMPLIST" | tr -d '\015' | zz "$2"
+  rm -f "$TMPLIST"
+}
+
+# useful in case ipban set is used in custom scripts
+FAIL=
+getipban || FAIL=1
+"$IPSET_DIR/create_ipset.sh"
+[ -n "$FAIL" ] && exit
+
+dl "$URL" "$ZHOSTLIST" 32768 4194304
+
+exit 0