diff --git a/binaries/aarch64/mdig b/binaries/aarch64/mdig index 0fddd95..2b72833 100755 Binary files a/binaries/aarch64/mdig and b/binaries/aarch64/mdig differ diff --git a/binaries/armhf/mdig b/binaries/armhf/mdig index 02b4280..ed3566c 100755 Binary files a/binaries/armhf/mdig and b/binaries/armhf/mdig differ diff --git a/binaries/mips32r1-lsb/mdig b/binaries/mips32r1-lsb/mdig index 431d54c..5e2a1d7 100755 Binary files a/binaries/mips32r1-lsb/mdig and b/binaries/mips32r1-lsb/mdig differ diff --git a/binaries/mips32r1-msb/mdig b/binaries/mips32r1-msb/mdig index 51b91ef..23ab939 100755 Binary files a/binaries/mips32r1-msb/mdig and b/binaries/mips32r1-msb/mdig differ diff --git a/binaries/mips64r2-msb/mdig b/binaries/mips64r2-msb/mdig index 4e650a1..5f67859 100755 Binary files a/binaries/mips64r2-msb/mdig and b/binaries/mips64r2-msb/mdig differ diff --git a/binaries/ppc/mdig b/binaries/ppc/mdig index d783d5b..151c83c 100755 Binary files a/binaries/ppc/mdig and b/binaries/ppc/mdig differ diff --git a/binaries/x86/mdig b/binaries/x86/mdig index 1469587..1610684 100755 Binary files a/binaries/x86/mdig and b/binaries/x86/mdig differ diff --git a/binaries/x86_64/mdig b/binaries/x86_64/mdig index 1546137..3dce482 100755 Binary files a/binaries/x86_64/mdig and b/binaries/x86_64/mdig differ diff --git a/ipset/def.sh b/ipset/def.sh index c53e484..efaf4b4 100755 --- a/ipset/def.sh +++ b/ipset/def.sh @@ -34,7 +34,7 @@ zz() digger() { if [ -x "$MDIG" ]; then - zzcat "$1" | "$MDIG" --family=4 --threads=$MDIG_THREADS + zzcat "$1" | "$MDIG" --family=4 --threads=$MDIG_THREADS --stats=1000 else zzcat "$1" | dig A +short +time=8 +tries=2 -f - | grep -E '^[^;].*[^\.]$' fi diff --git a/ipset/get_antizapret.sh b/ipset/get_antizapret.sh index c7d9405..9ec0e1f 100755 --- a/ipset/get_antizapret.sh +++ b/ipset/get_antizapret.sh @@ -12,7 +12,7 @@ ZIPLISTTMP=$TMPDIR/zapret-ip.txt getuser -curl --fail --max-time 150 --max-filesize 20971520 -k -L "$ZURL" | cut_local >"$ZIPLISTTMP" && +curl --fail --max-time 150 --connect-timeout 5 --max-filesize 20971520 -k -L "$ZURL" | cut_local >"$ZIPLISTTMP" && { dlsize=$(wc -c "$ZIPLISTTMP" | cut -f 1 -d ' ') if test $dlsize -lt 204800; then diff --git a/ipset/get_combined.sh b/ipset/get_combined.sh index f6b36d4..84895c2 100755 --- a/ipset/get_combined.sh +++ b/ipset/get_combined.sh @@ -16,7 +16,7 @@ ZIPLIST_IPBAN_TMP=/tmp/zapret-ipban.txt getuser # assume all https banned by ip -curl -k --fail --max-time 300 --max-filesize 62914560 "$ZURL_REESTR" -o "$ZREESTR" || +curl -k --fail --max-time 150 --connect-timeout 5 --retry 3 --max-filesize 62914560 "$ZURL_REESTR" -o "$ZREESTR" || { echo reestr list download failed exit 2 @@ -35,7 +35,7 @@ fi rm -f "$ZREESTR" -curl --fail --max-time 150 --max-filesize 20971520 -k -L "$ZURL_AZ" | cut_local >"$ZAZ" || +curl --fail --max-time 150 --connect-timeout 5 --max-filesize 20971520 -k -L "$ZURL_AZ" | cut_local >"$ZAZ" || { rm -f "$ZIPLIST_IPBAN_TMP" echo antizapret list download failed diff --git a/ipset/get_hostlist.sh b/ipset/get_hostlist.sh index b912cc9..2583e68 100755 --- a/ipset/get_hostlist.sh +++ b/ipset/get_hostlist.sh @@ -10,7 +10,7 @@ ZREESTR=$TMPDIR/zapret.txt #ZURL=https://reestr.rublacklist.net/api/current ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv -curl -k --fail --max-time 300 --max-filesize 62914560 "$ZURL" >"$ZREESTR" || +curl -k --fail --max-time 150 --connect-timeout 5 --retry 3 --max-filesize 62914560 "$ZURL" >"$ZREESTR" || { echo reestr list download failed exit 2 diff --git a/ipset/get_reestr.sh b/ipset/get_reestr.sh index e60668e..f9dd13c 100755 --- a/ipset/get_reestr.sh +++ b/ipset/get_reestr.sh @@ -14,7 +14,7 @@ ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv getuser -curl -k --fail --max-time 300 --max-filesize 62914560 "$ZURL" >"$ZREESTR" || +curl -k --fail --max-time 150 --connect-timeout 5 --retry 3 --max-filesize 62914560 "$ZURL" >"$ZREESTR" || { echo reestr list download failed exit 2 @@ -29,7 +29,8 @@ echo preparing dig list .. #sed -nre 's/^[^;]*;([^;|\\]{4,250})\;.*$/\1/p' $ZREESTR | sort | uniq >$ZDIG cut -f2 -d ';' "$ZREESTR" | grep -avE '^$|\*|:' >"$ZDIG" rm -f "$ZREESTR" -echo digging started ... +echo digging started. this can take long ... +echo domains in the list : $(wc -l <"$ZDIG") digger "$ZDIG" | cut_local >"$ZIPLISTTMP" || { rm -f "$ZDIG" exit 1 diff --git a/mdig/mdig.c b/mdig/mdig.c index 96f3726..3822f9a 100644 --- a/mdig/mdig.c +++ b/mdig/mdig.c @@ -14,7 +14,7 @@ #include #include -#define RESOLVER_EAGAIN_ATTEMPTS 3 +#define RESOLVER_EAGAIN_ATTEMPTS 2 void trimstr(char *s) { @@ -61,6 +61,8 @@ static struct char family; int threads; pthread_mutex_t flock; + pthread_mutex_t slock; // stats lock + int stats_every,stats_ct,stats_ct_ok; // stats } glob; // get next domain. return 0 if failure @@ -107,11 +109,31 @@ static void print_addrinfo(struct addrinfo *ai) } } +void stat_print(int ct, int ct_ok) +{ + if (glob.stats_every > 0) + interlocked_fprintf(stderr, "mdig stats : domains=%d success=%d error=%d\n", ct, ct_ok, ct-ct_ok); +} + +void stat_plus(char is_ok) +{ + int ct,ct_ok; + if (glob.stats_every > 0) + { + pthread_mutex_lock(&glob.slock); + ct = ++glob.stats_ct; + ct_ok = glob.stats_ct_ok+=!!is_ok; + pthread_mutex_unlock(&glob.slock); + + if (!(ct % glob.stats_every)) stat_print(ct,ct_ok); + } +} + static void *t_resolver(void *arg) { int tid = (int)(size_t)arg; int i,r; - char dom[256]; + char dom[256],is_ok; struct addrinfo hints; struct addrinfo *result; @@ -126,20 +148,22 @@ static void *t_resolver(void *arg) if (*dom) { VLOG("resolving %s", dom); + is_ok=0; for (i = 0; i < RESOLVER_EAGAIN_ATTEMPTS; i++) { if (r = getaddrinfo(dom, NULL, &hints, &result)) { - ELOG("failed to resolve %s : result %d (%s)", dom, r, eai_str(r)); + VLOG("failed to resolve %s : result %d (%s)", dom, r, eai_str(r)); if (r == EAI_AGAIN) continue; // temporary failure. should retry } else { print_addrinfo(result); freeaddrinfo(result); + is_ok=1; } - break; } + stat_plus(is_ok); } } VLOG("ended"); @@ -151,15 +175,23 @@ static int run_threads() int i, thread; pthread_t *t; + glob.stats_ct=glob.stats_ct_ok=0; if (pthread_mutex_init(&glob.flock, NULL) != 0) { fprintf(stderr, "mutex init failed\n"); return 10; } + if (pthread_mutex_init(&glob.slock, NULL) != 0) + { + fprintf(stderr, "mutex init failed\n"); + pthread_mutex_destroy(&glob.flock); + return 10; + } t = (pthread_t*)malloc(sizeof(pthread_t)*glob.threads); if (!t) { fprintf(stderr, "out of memory\n"); + pthread_mutex_destroy(&glob.slock); pthread_mutex_destroy(&glob.flock); return 11; } @@ -176,6 +208,8 @@ static int run_threads() pthread_join(t[i], NULL); } free(t); + stat_print(glob.stats_ct,glob.stats_ct_ok); + pthread_mutex_destroy(&glob.slock); pthread_mutex_destroy(&glob.flock); return thread ? 0 : 12; } @@ -186,6 +220,7 @@ static void exithelp() " --threads=\n" " --family=<4|6|46>\t; ipv4, ipv6, ipv4+ipv6\n" " --verbose\t\t; print query progress to stderr\n" + " --stats=N\t\t; print resolve stats to stderr every N domains\n" ); exit(1); } @@ -196,12 +231,13 @@ int main(int argc, char **argv) static const struct option long_options[] = { {"threads",required_argument,0,0}, // optidx=0 {"family",required_argument,0,0}, // optidx=1 - {"verbose",no_argument,0,0}, // optidx=2 - {"help",no_argument,0,0}, // optidx=3 + {"verbose",no_argument,0,0}, // optidx=2 + {"stats",required_argument,0,0}, // optidx=3 + {"help",no_argument,0,0}, // optidx=4 {NULL,0,NULL,0} }; - glob.verbose = '\0'; + memset(&glob, 0, sizeof(glob)); glob.family = FAMILY4; glob.threads = 1; while ((v = getopt_long_only(argc, argv, "", long_options, &option_index)) != -1) @@ -233,7 +269,11 @@ int main(int argc, char **argv) case 2: /* verbose */ glob.verbose = '\1'; break; - case 3: /* help */ + glob.threads = optarg ? atoi(optarg) : 0; + case 3: /* stats */ + glob.stats_every = optarg ? atoi(optarg) : 0; + break; + case 4: /* help */ exithelp(); break; }