nfqws: hostlist/ipset dedup and auto reload

nfq/desync.c

@@ -146,50 +146,48 @@ enum dpi_desync_mode desync_mode_from_string(const char *s)
 	return DESYNC_INVALID;
 }
 
-static bool dp_match_l3l4(struct desync_profile *dp, uint8_t l3proto, const struct sockaddr *dest)
-{
-	return  ((dest->sa_family==AF_INET && dp->filter_ipv4) || (dest->sa_family==AF_INET6 && dp->filter_ipv6)) &&
-		(l3proto==IPPROTO_TCP && pf_in_range(saport(dest), &dp->pf_tcp) || l3proto==IPPROTO_UDP && pf_in_range(saport(dest), &dp->pf_udp)) &&
-		IpsetCheck(dp, dest->sa_family==AF_INET ? &((struct sockaddr_in*)dest)->sin_addr : NULL, dest->sa_family==AF_INET6 ? &((struct sockaddr_in6*)dest)->sin6_addr : NULL);
-}
-static bool dp_impossible(struct desync_profile *dp, const char *hostname, t_l7proto l7proto)
-{
-	return	!PROFILE_IPSETS_EMPTY(dp) &&
-		((dp->filter_l7 && !l7_proto_match(l7proto, dp->filter_l7)) || (!*dp->hostlist_auto_filename && !hostname && (dp->hostlist || dp->hostlist_exclude)));
-}
-
 static bool dp_match(
 	struct desync_profile *dp,
 	uint8_t l3proto, const struct sockaddr *dest, const char *hostname, t_l7proto l7proto,
 	bool *bCheckDone, bool *bCheckResult, bool *bExcluded)
 {
 	if (bCheckDone) *bCheckDone = false;
-	// impossible case, hard filter
+
-	// impossible check avoids relatively slow ipset search
+	if (!HostlistsReloadCheckForProfile(dp)) return false;
-	if (!dp_impossible(dp,hostname,l7proto) && dp_match_l3l4(dp,l3proto,dest))
+
+	if ((dest->sa_family==AF_INET && !dp->filter_ipv4) || (dest->sa_family==AF_INET6 && !dp->filter_ipv6))
+		// L3 filter does not match
+		return false;
+	if ((l3proto==IPPROTO_TCP && !pf_in_range(saport(dest), &dp->pf_tcp)) || (l3proto==IPPROTO_UDP && !pf_in_range(saport(dest), &dp->pf_udp)))
+		// L4 filter does not match
+		return false;
+	if (dp->filter_l7 && !l7_proto_match(l7proto, dp->filter_l7))
+		// L7 filter does not match
+		return false;
+	if (!dp->hostlist_auto && !hostname && !PROFILE_HOSTLISTS_EMPTY(dp))
+		// avoid cpu consuming ipset check. profile cannot win if regular hostlists are present without auto hostlist and hostname is unknown.
+		return false;
+	if (!IpsetCheck(dp, dest->sa_family==AF_INET ? &((struct sockaddr_in*)dest)->sin_addr : NULL, dest->sa_family==AF_INET6 ? &((struct sockaddr_in6*)dest)->sin6_addr : NULL))
+		// target ip does not match
+		return false;
+
+	// autohostlist profile matching l3/l4 filter always win
+	if (dp->hostlist_auto) return true;
+
+	if (PROFILE_HOSTLISTS_EMPTY(dp))
+		// profile without hostlist filter wins
+		return true;
+	else
 	{
-		// soft filter
+		// without known hostname first profile matching l3/l4 filter and without hostlist filter wins
-		if (dp->filter_l7 && !l7_proto_match(l7proto, dp->filter_l7))
+		if (hostname)
-			return false;
-
-		// autohostlist profile matching l3/l4 filter always win
-		if (*dp->hostlist_auto_filename) return true;
-
-		if (dp->hostlist || dp->hostlist_exclude)
 		{
-			// without known hostname first profile matching l3/l4 filter and without hostlist filter wins
+			if (bCheckDone) *bCheckDone = true;
-			if (hostname)
+			bool b;
-			{
+			b = HostlistCheck(dp, hostname, bExcluded, true);
-				if (bCheckDone) *bCheckDone = true;
+			if (bCheckResult) *bCheckResult = b;
-				bool b;
+			return b;
-				b = HostlistCheck(dp, hostname, bExcluded);
-				if (bCheckResult) *bCheckResult = b;
-				return b;
-			}
 		}
-		else
-			// profile without hostlist filter wins
-			return true;
 	}
 	return false;
 }
@@ -359,21 +357,21 @@ static void auto_hostlist_failed(struct desync_profile *dp, const char *hostname
 		
 		DLOG("auto hostlist (profile %d) : rechecking %s to avoid duplicates\n", dp->n, hostname);
 		bool bExcluded=false;
-		if (!HostlistCheck(dp, hostname, &bExcluded) && !bExcluded)
+		if (!HostlistCheck(dp, hostname, &bExcluded, false) && !bExcluded)
 		{
-			DLOG("auto hostlist (profile %d) : adding %s to %s\n", dp->n, hostname, dp->hostlist_auto_filename);
+			DLOG("auto hostlist (profile %d) : adding %s to %s\n", dp->n, hostname, dp->hostlist_auto->filename);
-			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : adding to %s", hostname, dp->n, client_ip_port, l7proto_str(l7proto), dp->hostlist_auto_filename);
+			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : adding to %s", hostname, dp->n, client_ip_port, l7proto_str(l7proto), dp->hostlist_auto->filename);
-			if (!StrPoolAddStr(&dp->hostlist, hostname))
+			if (!StrPoolAddStr(&dp->hostlist_auto->hostlist, hostname))
 			{
 				DLOG_ERR("StrPoolAddStr out of memory\n");
 				return;
 			}
-			if (!append_to_list_file(dp->hostlist_auto_filename, hostname))
+			if (!append_to_list_file(dp->hostlist_auto->filename, hostname))
 			{
 				DLOG_PERROR("write to auto hostlist:");
 				return;
 			}
-			dp->hostlist_auto_mod_time = file_mod_time(dp->hostlist_auto_filename);
+			dp->hostlist_auto->mod_time = file_mod_time(dp->hostlist_auto->filename);
 		}
 		else
 		{
@@ -1044,17 +1042,17 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 			bCheckExcluded = ctrack_replay->bCheckExcluded;
 		}
 
-		if (bHaveHost && (dp->hostlist || dp->hostlist_exclude))
+		if (bHaveHost && !PROFILE_HOSTLISTS_EMPTY(dp))
 		{
 			if (!bCheckDone)
-				bCheckResult = HostlistCheck(dp, host, &bCheckExcluded);
+				bCheckResult = HostlistCheck(dp, host, &bCheckExcluded, false);
 			if (bCheckResult)
 				ctrack_stop_retrans_counter(ctrack_replay);
 			else
 			{
 				if (ctrack_replay)
 				{
-					ctrack_replay->hostname_ah_check = *dp->hostlist_auto_filename && !bCheckExcluded;
+					ctrack_replay->hostname_ah_check = dp->hostlist_auto && !bCheckExcluded;
 					if (!ctrack_replay->hostname_ah_check)
 						ctrack_stop_retrans_counter(ctrack_replay);
 				}
@@ -1762,17 +1760,17 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint
 			bCheckExcluded = ctrack_replay->bCheckExcluded;
 		}
 
-		if (bHaveHost && (dp->hostlist || dp->hostlist_exclude))
+		if (bHaveHost && !PROFILE_HOSTLISTS_EMPTY(dp))
 		{
 			if (!bCheckDone)
-				bCheckResult = HostlistCheck(dp, host, &bCheckExcluded);
+				bCheckResult = HostlistCheck(dp, host, &bCheckExcluded, false);
 			if (bCheckResult)
 				ctrack_stop_retrans_counter(ctrack_replay);
 			else
 			{
 				if (ctrack_replay)
 				{
-					ctrack_replay->hostname_ah_check = *dp->hostlist_auto_filename && !bCheckExcluded;
+					ctrack_replay->hostname_ah_check = dp->hostlist_auto && !bCheckExcluded;
 					if (ctrack_replay->hostname_ah_check)
 					{
 						// first request is not retrans

nfq/hostlist.c

@@ -96,22 +96,38 @@ bool AppendHostList(strpool **hostlist, const char *filename)
 	return true;
 }
 
-bool LoadHostLists(strpool **hostlist, struct str_list_head *file_list)
+static bool LoadHostList(struct hostlist_file *hfile)
 {
-	struct str_list *file;
+	time_t t = file_mod_time(hfile->filename);
-
+	if (!t)
-	if (*hostlist)
 	{
-		StrPoolDestroy(hostlist);
+		// stat() error
-		*hostlist = NULL;
+		DLOG_ERR("cannot access hostlist file '%s'. in-memory content remains unchanged.\n",hfile->filename);
+		return true;
 	}
-
+	if (t==hfile->mod_time) return true; // up to date
-	LIST_FOREACH(file, file_list, next)
+	StrPoolDestroy(&hfile->hostlist);
+	if (!AppendHostList(&hfile->hostlist, hfile->filename))
 	{
-		if (!AppendHostList(hostlist, file->str)) return false;
+		StrPoolDestroy(&hfile->hostlist);
+		return false;
 	}
+	hfile->mod_time=t;
 	return true;
 }
+static bool LoadHostLists(struct hostlist_files_head *list)
+{
+	bool bres=true;
+	struct hostlist_file *hfile;
+
+	LIST_FOREACH(hfile, list, next)
+	{
+		if (!LoadHostList(hfile))
+			// at least one failed
+			bres=false;
+	}
+	return bres;
+}
 
 bool NonEmptyHostlist(strpool **hostlist)
 {
@@ -119,8 +135,27 @@ bool NonEmptyHostlist(strpool **hostlist)
 	return *hostlist ? true : StrPoolAddStrLen(hostlist, "@&()", 4);
 }
 
+static void MakeAutolistsNonEmpty()
+{
+	struct desync_profile_list *dpl;
+	LIST_FOREACH(dpl, &params.desync_profiles, next)
+	{
+		if (dpl->dp.hostlist_auto)
+			NonEmptyHostlist(&dpl->dp.hostlist_auto->hostlist);
+	}
+}
 
-bool SearchHostList(strpool *hostlist, const char *host)
+bool LoadAllHostLists()
+{
+	if (!LoadHostLists(&params.hostlists))
+		return false;
+	MakeAutolistsNonEmpty();
+	return true;
+}
+
+
+
+static bool SearchHostList(strpool *hostlist, const char *host)
 {
 	if (hostlist)
 	{
@@ -129,7 +164,7 @@ bool SearchHostList(strpool *hostlist, const char *host)
 		while (p)
 		{
 			bInHostList = StrPoolCheckStr(hostlist, p);
-			DLOG("Hostlist check for %s : %s\n", p, bInHostList ? "positive" : "negative");
+			DLOG("hostlist check for %s : %s\n", p, bInHostList ? "positive" : "negative");
 			if (bInHostList) return true;
 			p = strchr(p, '.');
 			if (p) p++;
@@ -138,74 +173,103 @@ bool SearchHostList(strpool *hostlist, const char *host)
 	return false;
 }
 
-// return : true = apply fooling, false = do not apply
+
-static bool HostlistCheck_(strpool *hostlist, strpool *hostlist_exclude, const char *host, bool *excluded)
+static bool HostlistsReloadCheck(const struct hostlist_collection_head *hostlists)
 {
-	if (excluded) *excluded = false;
+	struct hostlist_item *item;
-	if (hostlist_exclude)
+	LIST_FOREACH(item, hostlists, next)
 	{
-		DLOG("Checking exclude hostlist\n");
+		if (!LoadHostList(item->hfile))
-		if (SearchHostList(hostlist_exclude, host))
+			return false;
+	}
+	MakeAutolistsNonEmpty();
+	return true;
+}
+bool HostlistsReloadCheckForProfile(const struct desync_profile *dp)
+{
+	return HostlistsReloadCheck(&dp->hl_collection) && HostlistsReloadCheck(&dp->hl_collection_exclude);
+}
+// return : true = apply fooling, false = do not apply
+static bool HostlistCheck_(const struct hostlist_collection_head *hostlists, const struct hostlist_collection_head *hostlists_exclude, const char *host, bool *excluded, bool bSkipReloadCheck)
+{
+	struct hostlist_item *item;
+
+	if (excluded) *excluded = false;
+
+	if (!bSkipReloadCheck)
+		if (!HostlistsReloadCheck(hostlists) || !HostlistsReloadCheck(hostlists_exclude))
+			return false;
+
+	LIST_FOREACH(item, hostlists_exclude, next)
+	{
+		DLOG("[%s] exclude ", item->hfile->filename);
+		if (SearchHostList(item->hfile->hostlist, host))
 		{
 			if (excluded) *excluded = true;
 			return false;
 		}
 	}
-	if (hostlist)
+	// old behavior compat: all include lists are empty means check passes
+	if (!hostlist_collection_is_empty(hostlists))
 	{
-		DLOG("Checking include hostlist\n");
+		LIST_FOREACH(item, hostlists, next)
-		return SearchHostList(hostlist, host);
+		{
+			DLOG("[%s] include ", item->hfile->filename);
+			if (SearchHostList(item->hfile->hostlist, host))
+				return true;
+		}
+		return false;
 	}
 	return true;
 }
 
-static bool LoadIncludeHostListsForProfile(struct desync_profile *dp)
-{
-	if (!LoadHostLists(&dp->hostlist, &dp->hostlist_files))
-		return false;
-	if (*dp->hostlist_auto_filename)
-	{
-		dp->hostlist_auto_mod_time = file_mod_time(dp->hostlist_auto_filename);
-		NonEmptyHostlist(&dp->hostlist);
-	}
-	return true;
-}
 
 // return : true = apply fooling, false = do not apply
-bool HostlistCheck(struct desync_profile *dp, const char *host, bool *excluded)
+bool HostlistCheck(const struct desync_profile *dp, const char *host, bool *excluded, bool bSkipReloadCheck)
 {
 	DLOG("* hostlist check for profile %d\n",dp->n);
-	if (*dp->hostlist_auto_filename)
+	return HostlistCheck_(&dp->hl_collection, &dp->hl_collection_exclude, host, excluded, bSkipReloadCheck);
-	{
-		time_t t = file_mod_time(dp->hostlist_auto_filename);
-		if (t!=dp->hostlist_auto_mod_time)
-		{
-			DLOG_CONDUP("Autohostlist '%s' from profile %d was modified. Reloading include hostlists for this profile.\n",dp->hostlist_auto_filename, dp->n);
-			if (!LoadIncludeHostListsForProfile(dp))
-			{
-				// what will we do without hostlist ?? sure, gonna die
-				exit(1);
-			}
-			dp->hostlist_auto_mod_time = t;
-			NonEmptyHostlist(&dp->hostlist);
-		}
-	}
-	return HostlistCheck_(dp->hostlist, dp->hostlist_exclude, host, excluded);
 }
 
-bool LoadIncludeHostLists()
+
+static struct hostlist_file *RegisterHostlist_(struct hostlist_files_head *hostlists, struct hostlist_collection_head *hl_collection, const char *filename)
 {
-	struct desync_profile_list *dpl;
+	struct hostlist_file *hfile;
-	LIST_FOREACH(dpl, &params.desync_profiles, next)
+	if (!(hfile=hostlist_files_search(hostlists, filename)))
-		if (!LoadIncludeHostListsForProfile(&dpl->dp))
+		if (!(hfile=hostlist_files_add(hostlists, filename)))
-			return false;
+			return NULL;
-	return true;
+	if (!hostlist_collection_search(hl_collection, filename))
+		if (!hostlist_collection_add(hl_collection, hfile))
+			return NULL;
+	return hfile;
 }
-bool LoadExcludeHostLists()
+struct hostlist_file *RegisterHostlist(struct desync_profile *dp, bool bExclude, const char *filename)
 {
-	struct desync_profile_list *dpl;
+	return RegisterHostlist_(
-	LIST_FOREACH(dpl, &params.desync_profiles, next)
+		&params.hostlists,
-		if (!LoadHostLists(&dpl->dp.hostlist_exclude, &dpl->dp.hostlist_exclude_files))
+		bExclude ? &dp->hl_collection_exclude : &dp->hl_collection,
-			return false;
+		filename);
-	return true;
+}
+
+void HostlistsDebug()
+{
+	if (!params.debug) return;
+
+	struct hostlist_file *hfile;
+	struct desync_profile_list *dpl;
+	struct hostlist_item *hl_item;
+
+	LIST_FOREACH(hfile, &params.hostlists, next)
+		DLOG("hostlist file %s%s\n",hfile->filename,hfile->hostlist ? "" : " (empty)");
+
+	LIST_FOREACH(dpl, &params.desync_profiles, next)
+	{
+		LIST_FOREACH(hl_item, &dpl->dp.hl_collection, next)
+			if (hl_item->hfile!=dpl->dp.hostlist_auto)
+				DLOG("profile %d include hostlist %s%s\n",dpl->dp.n, hl_item->hfile->filename,hl_item->hfile->hostlist ? "" : " (empty)");
+		LIST_FOREACH(hl_item, &dpl->dp.hl_collection_exclude, next)
+			DLOG("profile %d exclude hostlist %s%s\n",dpl->dp.n,hl_item->hfile->filename,hl_item->hfile->hostlist ? "" : " (empty)");
+		if (dpl->dp.hostlist_auto)
+			DLOG("profile %d auto hostlist %s%s\n",dpl->dp.n,dpl->dp.hostlist_auto->filename,dpl->dp.hostlist_auto->hostlist ? "" : " (empty)");
+	}
 }

nfq/hostlist.h

@@ -5,10 +5,10 @@
 #include "params.h"
 
 bool AppendHostList(strpool **hostlist, const char *filename);
-bool LoadHostLists(strpool **hostlist, struct str_list_head *file_list);
+bool LoadAllHostLists();
-bool LoadIncludeHostLists();
-bool LoadExcludeHostLists();
 bool NonEmptyHostlist(strpool **hostlist);
-bool SearchHostList(strpool *hostlist, const char *host);
 // return : true = apply fooling, false = do not apply
-bool HostlistCheck(struct desync_profile *dp,const char *host, bool *excluded);
+bool HostlistCheck(const struct desync_profile *dp,const char *host, bool *excluded, bool bSkipReloadCheck);
+struct hostlist_file *RegisterHostlist(struct desync_profile *dp, bool bExclude, const char *filename);
+bool HostlistsReloadCheckForProfile(const struct desync_profile *dp);
+void HostlistsDebug();

nfq/ipset.c

@@ -3,6 +3,7 @@
 #include "gzip.h"
 #include "helpers.h"
 
+
 // inplace tolower() and add to pool
 static bool addpool(ipset *ips, char **s, const char *end, int *ct)
 {
@@ -116,37 +117,45 @@ static bool AppendIpset(ipset *ips, const char *filename)
 	return true;
 }
 
-static bool LoadIpsets(ipset *ips, struct str_list_head *file_list)
+static bool LoadIpset(struct ipset_file *hfile)
 {
-	struct str_list *file;
+	time_t t = file_mod_time(hfile->filename);
-
+	if (!t)
-	ipsetDestroy(ips);
-
-	LIST_FOREACH(file, file_list, next)
 	{
-		if (!AppendIpset(ips, file->str)) return false;
+		// stat() error
+		DLOG_ERR("cannot access ipset file '%s'. in-memory content remains unchanged.\n",hfile->filename);
+		return true;
 	}
+	if (t==hfile->mod_time) return true; // up to date
+	ipsetDestroy(&hfile->ipset);
+	if (!AppendIpset(&hfile->ipset, hfile->filename))
+	{
+		ipsetDestroy(&hfile->ipset);
+		return false;
+	}
+	hfile->mod_time=t;
 	return true;
 }
+static bool LoadIpsets(struct ipset_files_head *list)
+{
+	bool bres=true;
+	struct ipset_file *hfile;
+
+	LIST_FOREACH(hfile, list, next)
+	{
+		if (!LoadIpset(hfile))
+			// at least one failed
+			bres=false;
+	}
+	return bres;
+}
 
-bool LoadIncludeIpsets()
+bool LoadAllIpsets()
 {
-	struct desync_profile_list *dpl;
+	return LoadIpsets(&params.ipsets);
-	LIST_FOREACH(dpl, &params.desync_profiles, next)
-		if (!LoadIpsets(&dpl->dp.ips, &dpl->dp.ipset_files))
-			return false;
-	return true;
-}
-bool LoadExcludeIpsets()
-{
-	struct desync_profile_list *dpl;
-	LIST_FOREACH(dpl, &params.desync_profiles, next)
-		if (!LoadIpsets(&dpl->dp.ips_exclude, &dpl->dp.ipset_exclude_files))
-			return false;
-	return true;
 }
 
-bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_addr *ipv6)
+static bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_addr *ipv6)
 {
 	char s_ip[40];
 	bool bInSet=false;
@@ -172,24 +181,103 @@ bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_
 	return bInSet;
 }
 
-static bool IpsetCheck_(const ipset *ips, const ipset *ips_exclude, const struct in_addr *ipv4, const struct in6_addr *ipv6)
+static bool IpsetsReloadCheck(const struct ipset_collection_head *ipsets)
 {
-	if (!IPSET_EMPTY(ips_exclude))
+	struct ipset_item *item;
+	LIST_FOREACH(item, ipsets, next)
 	{
-		DLOG("exclude ");
+		if (!LoadIpset(item->hfile))
-		if (SearchIpset(ips_exclude, ipv4, ipv6))
 			return false;
 	}
-	if (!IPSET_EMPTY(ips))
+	return true;
+}
+bool IpsetsReloadCheckForProfile(const struct desync_profile *dp)
+{
+	return IpsetsReloadCheck(&dp->ips_collection) && IpsetsReloadCheck(&dp->ips_collection_exclude);
+}
+
+static bool IpsetCheck_(const struct ipset_collection_head *ips, const struct ipset_collection_head *ips_exclude, const struct in_addr *ipv4, const struct in6_addr *ipv6)
+{
+	struct ipset_item *item;
+
+	if (!IpsetsReloadCheck(ips) || !IpsetsReloadCheck(ips_exclude))
+		return false;
+
+	LIST_FOREACH(item, ips_exclude, next)
 	{
-		DLOG("include ");
+		DLOG("[%s] exclude ",item->hfile->filename);
-		return SearchIpset(ips, ipv4, ipv6);
+		if (SearchIpset(&item->hfile->ipset, ipv4, ipv6))
+			return false;
+	}
+	// old behavior compat: all include lists are empty means check passes
+	if (!ipset_collection_is_empty(ips))
+	{
+		LIST_FOREACH(item, ips, next)
+		{
+			DLOG("[%s] include ",item->hfile->filename);
+			if (SearchIpset(&item->hfile->ipset, ipv4, ipv6))
+				return true;
+		}
+		return false;
 	}
 	return true;
 }
 
-bool IpsetCheck(struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6)
+bool IpsetCheck(const struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6)
 {
 	if (!PROFILE_IPSETS_EMPTY(dp)) DLOG("* ipset check for profile %d\n",dp->n);
-	return IpsetCheck_(&dp->ips,&dp->ips_exclude,ipv4,ipv6);
+	return IpsetCheck_(&dp->ips_collection,&dp->ips_collection_exclude,ipv4,ipv6);
+}
+
+
+static struct ipset_file *RegisterIpset_(struct ipset_files_head *ipsets, struct ipset_collection_head *ips_collection, const char *filename)
+{
+	struct ipset_file *hfile;
+	if (!(hfile=ipset_files_search(ipsets, filename)))
+		if (!(hfile=ipset_files_add(ipsets, filename)))
+			return NULL;
+	if (!ipset_collection_search(ips_collection, filename))
+		if (!ipset_collection_add(ips_collection, hfile))
+			return NULL;
+	return hfile;
+}
+struct ipset_file *RegisterIpset(struct desync_profile *dp, bool bExclude, const char *filename)
+{
+	return RegisterIpset_(
+		&params.ipsets,
+		bExclude ? &dp->ips_collection_exclude : &dp->ips_collection,
+		filename);
+}
+
+static const char *dbg_ipset_fill(const ipset *ips)
+{
+	if (ips->ips4)
+		if (ips->ips6)
+			return "ipv4+ipv6";
+		else
+			return "ipv4";
+	else
+		if (ips->ips6)
+			return "ipv6";
+		else
+			return "empty";
+}
+void IpsetsDebug()
+{
+	if (!params.debug) return;
+
+	struct ipset_file *hfile;
+	struct desync_profile_list *dpl;
+	struct ipset_item *ips_item;
+
+	LIST_FOREACH(hfile, &params.ipsets, next)
+		DLOG("ipset file %s (%s)\n",hfile->filename,dbg_ipset_fill(&hfile->ipset));
+
+	LIST_FOREACH(dpl, &params.desync_profiles, next)
+	{
+		LIST_FOREACH(ips_item, &dpl->dp.ips_collection, next)
+			DLOG("profile %d include ipset %s (%s)\n",dpl->dp.n,ips_item->hfile->filename,dbg_ipset_fill(&ips_item->hfile->ipset));
+		LIST_FOREACH(ips_item, &dpl->dp.ips_collection_exclude, next)
+			DLOG("profile %d exclude ipset %s (%s)\n",dpl->dp.n,ips_item->hfile->filename,dbg_ipset_fill(&ips_item->hfile->ipset));
+	}
 }

nfq/ipset.h

@@ -5,7 +5,7 @@
 #include "params.h"
 #include "pools.h"
 
-bool LoadIncludeIpsets();
+bool LoadAllIpsets();
-bool LoadExcludeIpsets();
+bool IpsetCheck(const struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6);
-bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_addr *ipv6);
+struct ipset_file *RegisterIpset(struct desync_profile *dp, bool bExclude, const char *filename);
-bool IpsetCheck(struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6);
+void IpsetsDebug();

nfq/nfqws.c

@@ -51,25 +51,10 @@ struct params_s params;
 bool bQuit=false;
 #endif
 
-static bool bHup = false;
 static void onhup(int sig)
 {
 	printf("HUP received !\n");
-	printf("Will reload hostlists and ipsets on next request (if any)\n");
+	// do not do anything. lists auto reload themselves based on file time.
-	bHup = true;
-}
-// should be called in normal execution
-static void dohup(void)
-{
-	if (bHup)
-	{
-		if (!LoadIncludeHostLists() || !LoadExcludeHostLists() || !LoadIncludeIpsets() || !LoadExcludeIpsets())
-		{
-			// what will we do without hostlist ?? sure, gonna die
-			exit(1);
-		}
-		bHup = false;
-	}
 }
 
 static void onusr1(int sig)
@@ -239,7 +224,6 @@ static int nfq_main(void)
 	{
 		while ((rv = recv(fd, buf, sizeof(buf), 0)) > 0)
 		{
-			dohup();
 			int r = nfq_handle_packet(h, (char *)buf, rv);
 			if (r) DLOG_ERR("nfq_handle_packet error %d\n", r);
 		}
@@ -351,7 +335,6 @@ static int dvt_main(void)
 			if (errno==EINTR)
 			{
 				// a signal received
-				dohup();
 				continue;
 			}
 			DLOG_PERROR("select");
@@ -507,8 +490,6 @@ static int win_main(const char *windivert_filter)
 			}
 			else
 			{
-				dohup();
-
 				mark=0;
 				// pseudo interface id IfIdx.SubIfIdx
 				verdict = processPacketData(&mark, ifout, packet, &len);
@@ -575,6 +556,8 @@ static void cleanup_params(void)
 
 	dp_list_destroy(&params.desync_profiles);
 
+	hostlist_files_destroy(&params.hostlists);
+	ipset_files_destroy(&params.ipsets);
 #ifdef __CYGWIN__
 	strlist_destroy(&params.ssid_filter);
 	strlist_destroy(&params.nlm_filter);
@@ -997,6 +980,7 @@ int main(int argc, char **argv)
 	struct desync_profile_list *dpl;
 	struct desync_profile *dp;
 	int desync_profile_count=0;
+
 	if (!(dpl = dp_list_add(&params.desync_profiles)))
 	{
 		DLOG_ERR("desync_profile_add: out of memory\n");
@@ -1015,6 +999,9 @@ int main(int argc, char **argv)
 	params.ctrack_t_est = CTRACK_T_EST;
 	params.ctrack_t_fin = CTRACK_T_FIN;
 	params.ctrack_t_udp = CTRACK_T_UDP;
+	
+	LIST_INIT(&params.hostlists);
+	LIST_INIT(&params.ipsets);
 
 #ifdef __CYGWIN__
 	LIST_INIT(&params.ssid_filter);
@@ -1522,27 +1509,27 @@ int main(int argc, char **argv)
 			}
 			break;
 		case 45: /* hostlist */
-			if (!strlist_add(&dp->hostlist_files, optarg))
+			if (!RegisterHostlist(dp, false, optarg))
 			{
-				DLOG_ERR("strlist_add failed\n");
+				DLOG_ERR("failed to register hostlist %s\n", optarg);
 				exit_clean(1);
 			}
 			break;
 		case 46: /* hostlist-exclude */
-			if (!strlist_add(&dp->hostlist_exclude_files, optarg))
+			if (!RegisterHostlist(dp, true, optarg))
 			{
-				DLOG_ERR("strlist_add failed\n");
+				DLOG_ERR("failed to register hostlist %s\n", optarg);
 				exit_clean(1);
 			}
 			break;
 		case 47: /* hostlist-auto */
-			if (*dp->hostlist_auto_filename)
+			if (dp->hostlist_auto)
 			{
 				DLOG_ERR("only one auto hostlist per profile is supported\n");
 				exit_clean(1);
 			}
 			{
-				FILE *F = fopen(optarg,"at");
+				FILE *F = fopen(optarg,"a+b");
 				if (!F)
 				{
 					DLOG_ERR("cannot create %s\n", optarg);
@@ -1560,13 +1547,11 @@ int main(int argc, char **argv)
 					DLOG_ERR("could not chown %s. auto hostlist file may not be writable after privilege drop\n", optarg);
 #endif
 			}
-			if (!strlist_add(&dp->hostlist_files, optarg))
+			if (!(dp->hostlist_auto=RegisterHostlist(dp, false, optarg)))
 			{
-				DLOG_ERR("strlist_add failed\n");
+				DLOG_ERR("failed to register hostlist %s\n", optarg);
 				exit_clean(1);
 			}
-			strncpy(dp->hostlist_auto_filename, optarg, sizeof(dp->hostlist_auto_filename));
-			dp->hostlist_auto_filename[sizeof(dp->hostlist_auto_filename) - 1] = '\0';
 			break;
 		case 48: /* hostlist-auto-fail-threshold */
 			dp->hostlist_auto_fail_threshold = (uint8_t)atoi(optarg);
@@ -1652,16 +1637,16 @@ int main(int argc, char **argv)
 			}
 			break;
 		case 57: /* ipset */
-			if (!strlist_add(&dp->ipset_files, optarg))
+			if (!RegisterIpset(dp, false, optarg))
 			{
-				DLOG_ERR("strlist_add failed\n");
+				DLOG_ERR("failed to register ipset %s\n", optarg);
 				exit_clean(1);
 			}
 			break;
 		case 58: /* ipset-exclude */
-			if (!strlist_add(&dp->ipset_exclude_files, optarg))
+			if (!RegisterIpset(dp, true, optarg))
 			{
-				DLOG_ERR("strlist_add failed\n");
+				DLOG_ERR("failed to register ipset %s\n", optarg);
 				exit_clean(1);
 			}
 			break;
@@ -1839,7 +1824,7 @@ int main(int argc, char **argv)
 	}
 
 	DLOG_CONDUP("we have %d user defined desync profile(s) and default low priority profile 0\n",desync_profile_count);
-
+	
 	LIST_FOREACH(dpl, &params.desync_profiles, next)
 	{
 		dp = &dpl->dp;
@@ -1854,27 +1839,22 @@ int main(int argc, char **argv)
 		if (dp->desync_split_http_req==httpreqpos_none && dp->desync_split_pos) dp->desync_split_http_req=httpreqpos_pos;
 	}
 
-	if (!LoadIncludeHostLists())
+	if (!LoadAllHostLists())
 	{
-		DLOG_ERR("Include hostlists load failed\n");
+		DLOG_ERR("hostlists load failed\n");
 		exit_clean(1);
 	}
-	if (!LoadExcludeHostLists())
+	if (!LoadAllIpsets())
 	{
-		DLOG_ERR("Exclude hostlists load failed\n");
+		DLOG_ERR("ipset load failed\n");
-		exit_clean(1);
-	}
-	if (!LoadIncludeIpsets())
-	{
-		DLOG_ERR("Include ipset load failed\n");
-		exit_clean(1);
-	}
-	if (!LoadExcludeIpsets())
-	{
-		DLOG_ERR("Exclude ipset load failed\n");
 		exit_clean(1);
 	}
 	
+	DLOG("\nlists summary:\n");
+	HostlistsDebug();
+	IpsetsDebug();
+	DLOG("\n");
+
 	if (daemon) daemonize();
 
 	if (*pidfile && !writepid(pidfile))

nfq/params.c

@@ -160,8 +160,10 @@ struct desync_profile_list *dp_list_add(struct desync_profile_list_head *head)
 	struct desync_profile_list *entry = calloc(1,sizeof(struct desync_profile_list));
 	if (!entry) return NULL;
 	
-	LIST_INIT(&entry->dp.hostlist_files);
+	LIST_INIT(&entry->dp.hl_collection);
-	LIST_INIT(&entry->dp.hostlist_exclude_files);
+	LIST_INIT(&entry->dp.hl_collection_exclude);
+	LIST_INIT(&entry->dp.ips_collection);
+	LIST_INIT(&entry->dp.ips_collection_exclude);
 	memcpy(entry->dp.hostspell, "host", 4); // default hostspell
 	entry->dp.desync_skip_nosni = true;
 	entry->dp.desync_split_pos = 2;
@@ -205,14 +207,10 @@ struct desync_profile_list *dp_list_add(struct desync_profile_list_head *head)
 }
 static void dp_entry_destroy(struct desync_profile_list *entry)
 {
-	strlist_destroy(&entry->dp.hostlist_files);
+	hostlist_collection_destroy(&entry->dp.hl_collection);
-	strlist_destroy(&entry->dp.hostlist_exclude_files);
+	hostlist_collection_destroy(&entry->dp.hl_collection_exclude);
-	strlist_destroy(&entry->dp.ipset_files);
+	ipset_collection_destroy(&entry->dp.ips_collection);
-	strlist_destroy(&entry->dp.ipset_exclude_files);
+	ipset_collection_destroy(&entry->dp.ips_collection_exclude);
-	StrPoolDestroy(&entry->dp.hostlist_exclude);
-	StrPoolDestroy(&entry->dp.hostlist);
-	ipsetDestroy(&entry->dp.ips);
-	ipsetDestroy(&entry->dp.ips_exclude);
 	HostFailPoolDestroy(&entry->dp.hostlist_auto_fail_counters);
 	free(entry);
 }
@@ -229,8 +227,7 @@ bool dp_list_have_autohostlist(struct desync_profile_list_head *head)
 {
 	struct desync_profile_list *dpl;
 	LIST_FOREACH(dpl, head, next)
-		if (*dpl->dp.hostlist_auto_filename)
+		if (dpl->dp.hostlist_auto)
 			return true;
 	return false;
 }
-

nfq/params.h

@@ -67,18 +67,21 @@ struct desync_profile
 	bool filter_ipv4,filter_ipv6;
 	port_filter pf_tcp,pf_udp;
 	uint32_t filter_l7;	// L7_PROTO_* bits
-	ipset ips,ips_exclude;
-	struct str_list_head ipset_files, ipset_exclude_files;
 
-	strpool *hostlist, *hostlist_exclude;
+	// list of pointers to ipsets
-	struct str_list_head hostlist_files, hostlist_exclude_files;
+	struct ipset_collection_head ips_collection, ips_collection_exclude;
-	char hostlist_auto_filename[PATH_MAX];
+
+	// list of pointers to hostlist files
+	struct hostlist_collection_head hl_collection, hl_collection_exclude;
+	// pointer to autohostlist. NULL if no autohostlist for the profile.
+	struct hostlist_file *hostlist_auto;
 	int hostlist_auto_fail_threshold, hostlist_auto_fail_time, hostlist_auto_retrans_threshold;
-	time_t hostlist_auto_mod_time;
+
 	hostfail_pool *hostlist_auto_fail_counters;
 };
 
-#define PROFILE_IPSETS_EMPTY(dp) (IPSET_EMPTY(&dp->ips) && IPSET_EMPTY(&dp->ips_exclude))
+#define PROFILE_IPSETS_EMPTY(dp) (ipset_collection_is_empty(&dp->ips_collection) && ipset_collection_is_empty(&dp->ips_collection_exclude))
+#define PROFILE_HOSTLISTS_EMPTY(dp) (hostlist_collection_is_empty(&dp->hl_collection) && hostlist_collection_is_empty(&dp->hl_collection_exclude))
 
 struct desync_profile_list {
 	struct desync_profile dp;
@@ -114,7 +117,12 @@ struct params_s
 #endif
 
 	char hostlist_auto_debuglog[PATH_MAX];
-	
+
+	// hostlist files with data for all profiles
+	struct hostlist_files_head hostlists;
+	// ipset files with data for all profiles
+	struct ipset_files_head ipsets;
+
 	unsigned int ctrack_t_syn, ctrack_t_est, ctrack_t_fin, ctrack_t_udp;
 	t_conntrack conntrack;
 };

nfq/pools.c

@@ -154,6 +154,93 @@ void strlist_destroy(struct str_list_head *head)
 
 
 
+
+struct hostlist_file *hostlist_files_add(struct hostlist_files_head *head, const char *filename)
+{
+	struct hostlist_file *entry = malloc(sizeof(struct hostlist_file));
+	if (entry)
+	{
+		if (!(entry->filename = strdup(filename)))
+		{
+			free(entry);
+			return false;
+		}
+		entry->mod_time=0;
+		entry->hostlist = NULL;
+		LIST_INSERT_HEAD(head, entry, next);
+	}
+	return entry;
+}
+static void hostlist_files_entry_destroy(struct hostlist_file *entry)
+{
+	if (entry->filename) free(entry->filename);
+	StrPoolDestroy(&entry->hostlist);
+	free(entry);
+}
+void hostlist_files_destroy(struct hostlist_files_head *head)
+{
+	struct hostlist_file *entry;
+	while ((entry = LIST_FIRST(head)))
+	{
+		LIST_REMOVE(entry, next);
+		hostlist_files_entry_destroy(entry);
+	}
+}
+struct hostlist_file *hostlist_files_search(struct hostlist_files_head *head, const char *filename)
+{
+	struct hostlist_file *hfile;
+
+	LIST_FOREACH(hfile, head, next)
+	{
+		if (!strcmp(hfile->filename,filename))
+			return hfile;
+	}
+	return NULL;
+}
+
+struct hostlist_item *hostlist_collection_add(struct hostlist_collection_head *head, struct hostlist_file *hfile)
+{
+	struct hostlist_item *entry = malloc(sizeof(struct hostlist_item));
+	if (entry)
+	{
+		entry->hfile = hfile;
+		LIST_INSERT_HEAD(head, entry, next);
+	}
+	return entry;
+}
+void hostlist_collection_destroy(struct hostlist_collection_head *head)
+{
+	struct hostlist_item *entry;
+	while ((entry = LIST_FIRST(head)))
+	{
+		LIST_REMOVE(entry, next);
+		free(entry);
+	}
+}
+struct hostlist_item *hostlist_collection_search(struct hostlist_collection_head *head, const char *filename)
+{
+	struct hostlist_item *item;
+
+	LIST_FOREACH(item, head, next)
+	{
+		if (!strcmp(item->hfile->filename,filename))
+			return item;
+	}
+	return NULL;
+}
+bool hostlist_collection_is_empty(const struct hostlist_collection_head *head)
+{
+	const struct hostlist_item *item;
+
+	LIST_FOREACH(item, head, next)
+	{
+		if (item->hfile->hostlist)
+			return false;
+	}
+	return true;
+}
+
+
 void ipset4Destroy(ipset4 **ipset)
 {
 	ipset4 *elem, *tmp;
@@ -275,3 +362,89 @@ void ipsetPrint(ipset *ipset)
 	ipset4Print(ipset->ips4);
 	ipset6Print(ipset->ips6);
 }
+
+
+struct ipset_file *ipset_files_add(struct ipset_files_head *head, const char *filename)
+{
+	struct ipset_file *entry = malloc(sizeof(struct ipset_file));
+	if (entry)
+	{
+		if (!(entry->filename = strdup(filename)))
+		{
+			free(entry);
+			return false;
+		}
+		entry->mod_time=0;
+		memset(&entry->ipset,0,sizeof(entry->ipset));
+		LIST_INSERT_HEAD(head, entry, next);
+	}
+	return entry;
+}
+static void ipset_files_entry_destroy(struct ipset_file *entry)
+{
+	if (entry->filename) free(entry->filename);
+	ipsetDestroy(&entry->ipset);
+	free(entry);
+}
+void ipset_files_destroy(struct ipset_files_head *head)
+{
+	struct ipset_file *entry;
+	while ((entry = LIST_FIRST(head)))
+	{
+		LIST_REMOVE(entry, next);
+		ipset_files_entry_destroy(entry);
+	}
+}
+struct ipset_file *ipset_files_search(struct ipset_files_head *head, const char *filename)
+{
+	struct ipset_file *hfile;
+
+	LIST_FOREACH(hfile, head, next)
+	{
+		if (!strcmp(hfile->filename,filename))
+			return hfile;
+	}
+	return NULL;
+}
+
+struct ipset_item *ipset_collection_add(struct ipset_collection_head *head, struct ipset_file *hfile)
+{
+	struct ipset_item *entry = malloc(sizeof(struct ipset_item));
+	if (entry)
+	{
+		entry->hfile = hfile;
+		LIST_INSERT_HEAD(head, entry, next);
+	}
+	return entry;
+}
+void ipset_collection_destroy(struct ipset_collection_head *head)
+{
+	struct ipset_item *entry;
+	while ((entry = LIST_FIRST(head)))
+	{
+		LIST_REMOVE(entry, next);
+		free(entry);
+	}
+}
+struct ipset_item *ipset_collection_search(struct ipset_collection_head *head, const char *filename)
+{
+	struct ipset_item *item;
+
+	LIST_FOREACH(item, head, next)
+	{
+		if (!strcmp(item->hfile->filename,filename))
+			return item;
+	}
+	return NULL;
+}
+bool ipset_collection_is_empty(const struct ipset_collection_head *head)
+{
+	const struct ipset_item *item;
+
+	LIST_FOREACH(item, head, next)
+	{
+		if (!IPSET_EMPTY(&item->hfile->ipset))
+			return false;
+	}
+	return true;
+}

nfq/pools.h

@@ -47,6 +47,30 @@ bool strlist_add(struct str_list_head *head, const char *filename);
 void strlist_destroy(struct str_list_head *head);
 
 
+
+struct hostlist_file {
+	char *filename;
+	time_t mod_time;
+	strpool *hostlist;
+	LIST_ENTRY(hostlist_file) next;
+};
+LIST_HEAD(hostlist_files_head, hostlist_file);
+
+struct hostlist_file *hostlist_files_add(struct hostlist_files_head *head, const char *filename);
+void hostlist_files_destroy(struct hostlist_files_head *head);
+struct hostlist_file *hostlist_files_search(struct hostlist_files_head *head, const char *filename);
+
+struct hostlist_item {
+	struct hostlist_file *hfile;
+	LIST_ENTRY(hostlist_item) next;
+};
+LIST_HEAD(hostlist_collection_head, hostlist_item);
+struct hostlist_item *hostlist_collection_add(struct hostlist_collection_head *head, struct hostlist_file *hfile);
+void hostlist_collection_destroy(struct hostlist_collection_head *head);
+struct hostlist_item *hostlist_collection_search(struct hostlist_collection_head *head, const char *filename);
+bool hostlist_collection_is_empty(const struct hostlist_collection_head *head);
+
+
 typedef struct ipset4 {
 	struct cidr4 cidr;	/* key */
 	UT_hash_handle hh;	/* makes this structure hashable */
@@ -83,3 +107,26 @@ void ipset6Print(ipset6 *ipset);
 
 void ipsetDestroy(ipset *ipset);
 void ipsetPrint(ipset *ipset);
+
+
+struct ipset_file {
+	char *filename;
+	time_t mod_time;
+	ipset ipset;
+	LIST_ENTRY(ipset_file) next;
+};
+LIST_HEAD(ipset_files_head, ipset_file);
+
+struct ipset_file *ipset_files_add(struct ipset_files_head *head, const char *filename);
+void ipset_files_destroy(struct ipset_files_head *head);
+struct ipset_file *ipset_files_search(struct ipset_files_head *head, const char *filename);
+
+struct ipset_item {
+	struct ipset_file *hfile;
+	LIST_ENTRY(ipset_item) next;
+};
+LIST_HEAD(ipset_collection_head, ipset_item);
+struct ipset_item * ipset_collection_add(struct ipset_collection_head *head, struct ipset_file *hfile);
+void ipset_collection_destroy(struct ipset_collection_head *head);
+struct ipset_item *ipset_collection_search(struct ipset_collection_head *head, const char *filename);
+bool ipset_collection_is_empty(const struct ipset_collection_head *head);