mdig : stats reporting. resolver errors only with -v. fail retry count 3=>2

ipset/def.sh

@@ -34,7 +34,7 @@ zz()
 digger()
 {
  if [ -x "$MDIG" ]; then
-  zzcat "$1" | "$MDIG" --family=4 --threads=$MDIG_THREADS
+  zzcat "$1" | "$MDIG" --family=4 --threads=$MDIG_THREADS --stats=1000
  else
   zzcat "$1" | dig A +short +time=8 +tries=2 -f - | grep -E '^[^;].*[^\.]$'
  fi

ipset/get_antizapret.sh

@@ -12,7 +12,7 @@ ZIPLISTTMP=$TMPDIR/zapret-ip.txt
 
 getuser
 
-curl --fail --max-time 150 --max-filesize 20971520 -k -L "$ZURL" | cut_local >"$ZIPLISTTMP" &&
+curl --fail --max-time 150 --connect-timeout 5 --max-filesize 20971520 -k -L "$ZURL" | cut_local >"$ZIPLISTTMP" &&
 {
  dlsize=$(wc -c "$ZIPLISTTMP" | cut -f 1 -d ' ')
  if test $dlsize -lt 204800; then

ipset/get_combined.sh

@@ -16,7 +16,7 @@ ZIPLIST_IPBAN_TMP=/tmp/zapret-ipban.txt
 getuser
 
 # assume all https banned by ip
-curl -k --fail --max-time 300 --max-filesize 62914560 "$ZURL_REESTR" -o "$ZREESTR" ||
+curl -k --fail --max-time 150 --connect-timeout 5 --retry 3 --max-filesize 62914560 "$ZURL_REESTR" -o "$ZREESTR" ||
 {
  echo reestr list download failed
  exit 2
@@ -35,7 +35,7 @@ fi
  
 rm -f "$ZREESTR"
 
-curl --fail --max-time 150 --max-filesize 20971520 -k -L "$ZURL_AZ" | cut_local >"$ZAZ" ||
+curl --fail --max-time 150 --connect-timeout 5 --max-filesize 20971520 -k -L "$ZURL_AZ" | cut_local >"$ZAZ" ||
 {
  rm -f "$ZIPLIST_IPBAN_TMP"
  echo antizapret list download failed

ipset/get_hostlist.sh

@@ -10,7 +10,7 @@ ZREESTR=$TMPDIR/zapret.txt
 #ZURL=https://reestr.rublacklist.net/api/current
 ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
 
-curl -k --fail --max-time 300 --max-filesize 62914560 "$ZURL" >"$ZREESTR" ||
+curl -k --fail --max-time 150 --connect-timeout 5 --retry 3 --max-filesize 62914560 "$ZURL" >"$ZREESTR" ||
 {
  echo reestr list download failed   
  exit 2

ipset/get_reestr.sh

@@ -14,7 +14,7 @@ ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
 
 getuser
 
-curl -k --fail --max-time 300 --max-filesize 62914560 "$ZURL" >"$ZREESTR" ||
+curl -k --fail --max-time 150 --connect-timeout 5 --retry 3 --max-filesize 62914560 "$ZURL" >"$ZREESTR" ||
 {
  echo reestr list download failed   
  exit 2
@@ -29,7 +29,8 @@ echo preparing dig list ..
 #sed -nre 's/^[^;]*;([^;|\\]{4,250})\;.*$/\1/p' $ZREESTR | sort | uniq >$ZDIG
 cut -f2 -d ';' "$ZREESTR"  | grep -avE '^$|\*|:' >"$ZDIG"
 rm -f "$ZREESTR"
-echo digging started ...
+echo digging started. this can take long ...
+echo domains in the list : $(wc -l <"$ZDIG")
 digger "$ZDIG" | cut_local >"$ZIPLISTTMP" || {
  rm -f "$ZDIG"
  exit 1

mdig/mdig.c

@@ -14,7 +14,7 @@
 #include <arpa/inet.h>
 #include <netdb.h>
 
-#define RESOLVER_EAGAIN_ATTEMPTS 3
+#define RESOLVER_EAGAIN_ATTEMPTS 2
 
 void trimstr(char *s)
 {
@@ -61,6 +61,8 @@ static struct
 	char family;
 	int threads;
 	pthread_mutex_t flock;
+	pthread_mutex_t slock; // stats lock
+	int stats_every,stats_ct,stats_ct_ok; // stats
 } glob;
 
 // get next domain. return 0 if failure
@@ -107,11 +109,31 @@ static void print_addrinfo(struct addrinfo *ai)
 	}
 }
 
+void stat_print(int ct, int ct_ok)
+{
+	if (glob.stats_every > 0)
+		interlocked_fprintf(stderr, "mdig stats : domains=%d success=%d error=%d\n", ct, ct_ok, ct-ct_ok);
+}
+
+void stat_plus(char is_ok)
+{
+	int ct,ct_ok;
+	if (glob.stats_every > 0)
+	{
+		pthread_mutex_lock(&glob.slock);
+		ct = ++glob.stats_ct;
+		ct_ok = glob.stats_ct_ok+=!!is_ok;
+		pthread_mutex_unlock(&glob.slock);
+
+		if (!(ct % glob.stats_every)) stat_print(ct,ct_ok);
+	}
+}
+
 static void *t_resolver(void *arg)
 {
 	int tid = (int)(size_t)arg;
 	int i,r;
-	char dom[256];
+	char dom[256],is_ok;
 	struct addrinfo hints;
 	struct addrinfo *result;
 
@@ -126,20 +148,22 @@ static void *t_resolver(void *arg)
 		if (*dom)
 		{
 			VLOG("resolving %s", dom);
+			is_ok=0;
 			for (i = 0; i < RESOLVER_EAGAIN_ATTEMPTS; i++)
 			{
 				if (r = getaddrinfo(dom, NULL, &hints, &result))
 				{
-					ELOG("failed to resolve %s : result %d (%s)", dom, r, eai_str(r));
+					VLOG("failed to resolve %s : result %d (%s)", dom, r, eai_str(r));
 					if (r == EAI_AGAIN) continue; // temporary failure. should retry
 				}
 				else
 				{
 					print_addrinfo(result);
 					freeaddrinfo(result);
+					is_ok=1;
 				}
-				break;
 			}
+			stat_plus(is_ok);
 		}
 	}
 	VLOG("ended");
@@ -151,15 +175,23 @@ static int run_threads()
 	int i, thread;
 	pthread_t *t;
 
+	glob.stats_ct=glob.stats_ct_ok=0;
 	if (pthread_mutex_init(&glob.flock, NULL) != 0)
 	{
 		fprintf(stderr, "mutex init failed\n");
 		return 10;
 	}
+	if (pthread_mutex_init(&glob.slock, NULL) != 0)
+	{
+		fprintf(stderr, "mutex init failed\n");
+		pthread_mutex_destroy(&glob.flock);
+		return 10;
+	}
 	t = (pthread_t*)malloc(sizeof(pthread_t)*glob.threads);
 	if (!t)
 	{
 		fprintf(stderr, "out of memory\n");
+		pthread_mutex_destroy(&glob.slock);
 		pthread_mutex_destroy(&glob.flock);
 		return 11;
 	}
@@ -176,6 +208,8 @@ static int run_threads()
 		pthread_join(t[i], NULL);
 	}
 	free(t);
+	stat_print(glob.stats_ct,glob.stats_ct_ok);
+	pthread_mutex_destroy(&glob.slock);
 	pthread_mutex_destroy(&glob.flock);
 	return thread ? 0 : 12;
 }
@@ -186,6 +220,7 @@ static void exithelp()
 		" --threads=<threads_number>\n"
 		" --family=<4|6|46>\t; ipv4, ipv6, ipv4+ipv6\n"
 		" --verbose\t\t; print query progress to stderr\n"
+		" --stats=N\t\t; print resolve stats to stderr every N domains\n"
 	);
 	exit(1);
 }
@@ -196,12 +231,13 @@ int main(int argc, char **argv)
 	static const struct option long_options[] = {
 			{"threads",required_argument,0,0},	// optidx=0
 			{"family",required_argument,0,0},	// optidx=1
-			{"verbose",no_argument,0,0},	// optidx=2
-			{"help",no_argument,0,0},	// optidx=3
+			{"verbose",no_argument,0,0},		// optidx=2
+			{"stats",required_argument,0,0},	// optidx=3
+			{"help",no_argument,0,0},		// optidx=4
 			{NULL,0,NULL,0}
 	};
 
-	glob.verbose = '\0';
+	memset(&glob, 0, sizeof(glob));
 	glob.family = FAMILY4;
 	glob.threads = 1;
 	while ((v = getopt_long_only(argc, argv, "", long_options, &option_index)) != -1)
@@ -233,7 +269,11 @@ int main(int argc, char **argv)
 		case 2: /* verbose */
 			glob.verbose = '\1';
 			break;
-		case 3: /* help */
+			glob.threads = optarg ? atoi(optarg) : 0;
+		case 3: /* stats */
+			glob.stats_every = optarg ? atoi(optarg) : 0;
+			break;
+		case 4: /* help */
 			exithelp();
 			break;
 		}