nfq/desync.c

@@ -146,49 +146,51 @@ enum dpi_desync_mode desync_mode_from_string(const char *s)
 	return DESYNC_INVALID;
 }
 
+static bool dp_match_l3l4(struct desync_profile *dp, uint8_t l3proto, const struct sockaddr *dest)
+{
+	return  ((dest->sa_family==AF_INET && dp->filter_ipv4) || (dest->sa_family==AF_INET6 && dp->filter_ipv6)) &&
+		(l3proto==IPPROTO_TCP && pf_in_range(saport(dest), &dp->pf_tcp) || l3proto==IPPROTO_UDP && pf_in_range(saport(dest), &dp->pf_udp)) &&
+		IpsetCheck(dp, dest->sa_family==AF_INET ? &((struct sockaddr_in*)dest)->sin_addr : NULL, dest->sa_family==AF_INET6 ? &((struct sockaddr_in6*)dest)->sin6_addr : NULL);
+}
+static bool dp_impossible(struct desync_profile *dp, const char *hostname, t_l7proto l7proto)
+{
+	return	!PROFILE_IPSETS_EMPTY(dp) &&
+		((dp->filter_l7 && !l7_proto_match(l7proto, dp->filter_l7)) || (!*dp->hostlist_auto_filename && !hostname && (dp->hostlist || dp->hostlist_exclude)));
+}
+
 static bool dp_match(
 	struct desync_profile *dp,
 	uint8_t l3proto, const struct sockaddr *dest, const char *hostname, t_l7proto l7proto,
 	bool *bCheckDone, bool *bCheckResult, bool *bExcluded)
 {
 	if (bCheckDone) *bCheckDone = false;
-
+	// impossible case, hard filter
-	if (!HostlistsReloadCheckForProfile(dp)) return false;
+	// impossible check avoids relatively slow ipset search
-
+	if (!dp_impossible(dp,hostname,l7proto) && dp_match_l3l4(dp,l3proto,dest))
-	if ((dest->sa_family==AF_INET && !dp->filter_ipv4) || (dest->sa_family==AF_INET6 && !dp->filter_ipv6))
+	{
-		// L3 filter does not match
+		// soft filter
-		return false;
-	if ((l3proto==IPPROTO_TCP && !pf_in_range(saport(dest), &dp->pf_tcp)) || (l3proto==IPPROTO_UDP && !pf_in_range(saport(dest), &dp->pf_udp)))
-		// L4 filter does not match
-		return false;
 		if (dp->filter_l7 && !l7_proto_match(l7proto, dp->filter_l7))
-		// L7 filter does not match
-		return false;
-	if (!dp->hostlist_auto && !hostname && !PROFILE_HOSTLISTS_EMPTY(dp))
-		// avoid cpu consuming ipset check. profile cannot win if regular hostlists are present without auto hostlist and hostname is unknown.
-		return false;
-	if (!IpsetCheck(dp, dest->sa_family==AF_INET ? &((struct sockaddr_in*)dest)->sin_addr : NULL, dest->sa_family==AF_INET6 ? &((struct sockaddr_in6*)dest)->sin6_addr : NULL))
-		// target ip does not match
 			return false;
 
 		// autohostlist profile matching l3/l4 filter always win
-	if (dp->hostlist_auto) return true;
+		if (*dp->hostlist_auto_filename) return true;
 
-	if (PROFILE_HOSTLISTS_EMPTY(dp))
+		if (dp->hostlist || dp->hostlist_exclude)
-		// profile without hostlist filter wins
-		return true;
-	else
 		{
 			// without known hostname first profile matching l3/l4 filter and without hostlist filter wins
 			if (hostname)
 			{
 				if (bCheckDone) *bCheckDone = true;
 				bool b;
-			b = HostlistCheck(dp, hostname, bExcluded, true);
+				b = HostlistCheck(dp, hostname, bExcluded);
 				if (bCheckResult) *bCheckResult = b;
 				return b;
 			}
 		}
+		else
+			// profile without hostlist filter wins
+			return true;
+	}
 	return false;
 }
 static struct desync_profile *dp_find(
@@ -357,21 +359,21 @@ static void auto_hostlist_failed(struct desync_profile *dp, const char *hostname
 		
 		DLOG("auto hostlist (profile %d) : rechecking %s to avoid duplicates\n", dp->n, hostname);
 		bool bExcluded=false;
-		if (!HostlistCheck(dp, hostname, &bExcluded, false) && !bExcluded)
+		if (!HostlistCheck(dp, hostname, &bExcluded) && !bExcluded)
 		{
-			DLOG("auto hostlist (profile %d) : adding %s to %s\n", dp->n, hostname, dp->hostlist_auto->filename);
+			DLOG("auto hostlist (profile %d) : adding %s to %s\n", dp->n, hostname, dp->hostlist_auto_filename);
-			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : adding to %s", hostname, dp->n, client_ip_port, l7proto_str(l7proto), dp->hostlist_auto->filename);
+			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : adding to %s", hostname, dp->n, client_ip_port, l7proto_str(l7proto), dp->hostlist_auto_filename);
-			if (!StrPoolAddStr(&dp->hostlist_auto->hostlist, hostname))
+			if (!StrPoolAddStr(&dp->hostlist, hostname))
 			{
 				DLOG_ERR("StrPoolAddStr out of memory\n");
 				return;
 			}
-			if (!append_to_list_file(dp->hostlist_auto->filename, hostname))
+			if (!append_to_list_file(dp->hostlist_auto_filename, hostname))
 			{
 				DLOG_PERROR("write to auto hostlist:");
 				return;
 			}
-			dp->hostlist_auto->mod_time = file_mod_time(dp->hostlist_auto->filename);
+			dp->hostlist_auto_mod_time = file_mod_time(dp->hostlist_auto_filename);
 		}
 		else
 		{
@@ -1042,17 +1044,17 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 			bCheckExcluded = ctrack_replay->bCheckExcluded;
 		}
 
-		if (bHaveHost && !PROFILE_HOSTLISTS_EMPTY(dp))
+		if (bHaveHost && (dp->hostlist || dp->hostlist_exclude))
 		{
 			if (!bCheckDone)
-				bCheckResult = HostlistCheck(dp, host, &bCheckExcluded, false);
+				bCheckResult = HostlistCheck(dp, host, &bCheckExcluded);
 			if (bCheckResult)
 				ctrack_stop_retrans_counter(ctrack_replay);
 			else
 			{
 				if (ctrack_replay)
 				{
-					ctrack_replay->hostname_ah_check = dp->hostlist_auto && !bCheckExcluded;
+					ctrack_replay->hostname_ah_check = *dp->hostlist_auto_filename && !bCheckExcluded;
 					if (!ctrack_replay->hostname_ah_check)
 						ctrack_stop_retrans_counter(ctrack_replay);
 				}
@@ -1760,17 +1762,17 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint
 			bCheckExcluded = ctrack_replay->bCheckExcluded;
 		}
 
-		if (bHaveHost && !PROFILE_HOSTLISTS_EMPTY(dp))
+		if (bHaveHost && (dp->hostlist || dp->hostlist_exclude))
 		{
 			if (!bCheckDone)
-				bCheckResult = HostlistCheck(dp, host, &bCheckExcluded, false);
+				bCheckResult = HostlistCheck(dp, host, &bCheckExcluded);
 			if (bCheckResult)
 				ctrack_stop_retrans_counter(ctrack_replay);
 			else
 			{
 				if (ctrack_replay)
 				{
-					ctrack_replay->hostname_ah_check = dp->hostlist_auto && !bCheckExcluded;
+					ctrack_replay->hostname_ah_check = *dp->hostlist_auto_filename && !bCheckExcluded;
 					if (ctrack_replay->hostname_ah_check)
 					{
 						// first request is not retrans

nfq/hostlist.c

@@ -96,37 +96,21 @@ bool AppendHostList(strpool **hostlist, const char *filename)
 	return true;
 }
 
-static bool LoadHostList(struct hostlist_file *hfile)
+bool LoadHostLists(strpool **hostlist, struct str_list_head *file_list)
 {
-	time_t t = file_mod_time(hfile->filename);
+	struct str_list *file;
-	if (!t)
-	{
-		// stat() error
-		DLOG_ERR("cannot access hostlist file '%s'. in-memory content remains unchanged.\n",hfile->filename);
-		return true;
-	}
-	if (t==hfile->mod_time) return true; // up to date
-	StrPoolDestroy(&hfile->hostlist);
-	if (!AppendHostList(&hfile->hostlist, hfile->filename))
-	{
-		StrPoolDestroy(&hfile->hostlist);
-		return false;
-	}
-	hfile->mod_time=t;
-	return true;
-}
-static bool LoadHostLists(struct hostlist_files_head *list)
-{
-	bool bres=true;
-	struct hostlist_file *hfile;
 
-	LIST_FOREACH(hfile, list, next)
+	if (*hostlist)
 	{
-		if (!LoadHostList(hfile))
+		StrPoolDestroy(hostlist);
-			// at least one failed
+		*hostlist = NULL;
-			bres=false;
 	}
-	return bres;
+
+	LIST_FOREACH(file, file_list, next)
+	{
+		if (!AppendHostList(hostlist, file->str)) return false;
+	}
+	return true;
 }
 
 bool NonEmptyHostlist(strpool **hostlist)
@@ -135,27 +119,8 @@ bool NonEmptyHostlist(strpool **hostlist)
 	return *hostlist ? true : StrPoolAddStrLen(hostlist, "@&()", 4);
 }
 
-static void MakeAutolistsNonEmpty()
-{
-	struct desync_profile_list *dpl;
-	LIST_FOREACH(dpl, &params.desync_profiles, next)
-	{
-		if (dpl->dp.hostlist_auto)
-			NonEmptyHostlist(&dpl->dp.hostlist_auto->hostlist);
-	}
-}
 
-bool LoadAllHostLists()
+bool SearchHostList(strpool *hostlist, const char *host)
-{
-	if (!LoadHostLists(&params.hostlists))
-		return false;
-	MakeAutolistsNonEmpty();
-	return true;
-}
-
-
-
-static bool SearchHostList(strpool *hostlist, const char *host)
 {
 	if (hostlist)
 	{
@@ -164,7 +129,7 @@ static bool SearchHostList(strpool *hostlist, const char *host)
 		while (p)
 		{
 			bInHostList = StrPoolCheckStr(hostlist, p);
-			DLOG("hostlist check for %s : %s\n", p, bInHostList ? "positive" : "negative");
+			DLOG("Hostlist check for %s : %s\n", p, bInHostList ? "positive" : "negative");
 			if (bInHostList) return true;
 			p = strchr(p, '.');
 			if (p) p++;
@@ -173,108 +138,74 @@ static bool SearchHostList(strpool *hostlist, const char *host)
 	return false;
 }
 
-
-static bool HostlistsReloadCheck(const struct hostlist_collection_head *hostlists)
-{
-	struct hostlist_item *item;
-	LIST_FOREACH(item, hostlists, next)
-	{
-		if (!LoadHostList(item->hfile))
-			return false;
-	}
-	MakeAutolistsNonEmpty();
-	return true;
-}
-bool HostlistsReloadCheckForProfile(const struct desync_profile *dp)
-{
-	return HostlistsReloadCheck(&dp->hl_collection) && HostlistsReloadCheck(&dp->hl_collection_exclude);
-}
 // return : true = apply fooling, false = do not apply
-static bool HostlistCheck_(const struct hostlist_collection_head *hostlists, const struct hostlist_collection_head *hostlists_exclude, const char *host, bool *excluded, bool bSkipReloadCheck)
+static bool HostlistCheck_(strpool *hostlist, strpool *hostlist_exclude, const char *host, bool *excluded)
 {
-	struct hostlist_item *item;
-
 	if (excluded) *excluded = false;
-
+	if (hostlist_exclude)
-	if (!bSkipReloadCheck)
-		if (!HostlistsReloadCheck(hostlists) || !HostlistsReloadCheck(hostlists_exclude))
-			return false;
-
-	LIST_FOREACH(item, hostlists_exclude, next)
 	{
-		DLOG("[%s] exclude ", item->hfile->filename);
+		DLOG("Checking exclude hostlist\n");
-		if (SearchHostList(item->hfile->hostlist, host))
+		if (SearchHostList(hostlist_exclude, host))
 		{
 			if (excluded) *excluded = true;
 			return false;
 		}
 	}
-	// old behavior compat: all include lists are empty means check passes
+	if (hostlist)
-	if (!hostlist_collection_is_empty(hostlists))
 	{
-		LIST_FOREACH(item, hostlists, next)
+		DLOG("Checking include hostlist\n");
-		{
+		return SearchHostList(hostlist, host);
-			DLOG("[%s] include ", item->hfile->filename);
-			if (SearchHostList(item->hfile->hostlist, host))
-				return true;
-		}
-		return false;
 	}
 	return true;
 }
 
+static bool LoadIncludeHostListsForProfile(struct desync_profile *dp)
+{
+	if (!LoadHostLists(&dp->hostlist, &dp->hostlist_files))
+		return false;
+	if (*dp->hostlist_auto_filename)
+	{
+		dp->hostlist_auto_mod_time = file_mod_time(dp->hostlist_auto_filename);
+		NonEmptyHostlist(&dp->hostlist);
+	}
+	return true;
+}
 
 // return : true = apply fooling, false = do not apply
-bool HostlistCheck(const struct desync_profile *dp, const char *host, bool *excluded, bool bSkipReloadCheck)
+bool HostlistCheck(struct desync_profile *dp, const char *host, bool *excluded)
 {
 	DLOG("* hostlist check for profile %d\n",dp->n);
-	return HostlistCheck_(&dp->hl_collection, &dp->hl_collection_exclude, host, excluded, bSkipReloadCheck);
+	if (*dp->hostlist_auto_filename)
-}
-
-
-static struct hostlist_file *RegisterHostlist_(struct hostlist_files_head *hostlists, struct hostlist_collection_head *hl_collection, const char *filename)
-{
-	struct hostlist_file *hfile;
-	if (!(hfile=hostlist_files_search(hostlists, filename)))
-		if (!(hfile=hostlist_files_add(hostlists, filename)))
-			return NULL;
-	if (!hostlist_collection_search(hl_collection, filename))
-		if (!hostlist_collection_add(hl_collection, hfile))
-			return NULL;
-	return hfile;
-}
-struct hostlist_file *RegisterHostlist(struct desync_profile *dp, bool bExclude, const char *filename)
-{
-	if (!file_mod_time(filename))
 	{
-		DLOG_ERR("cannot access hostlist file '%s'\n",filename);
+		time_t t = file_mod_time(dp->hostlist_auto_filename);
-		return NULL;
+		if (t!=dp->hostlist_auto_mod_time)
+		{
+			DLOG_CONDUP("Autohostlist '%s' from profile %d was modified. Reloading include hostlists for this profile.\n",dp->hostlist_auto_filename, dp->n);
+			if (!LoadIncludeHostListsForProfile(dp))
+			{
+				// what will we do without hostlist ?? sure, gonna die
+				exit(1);
 			}
-	return RegisterHostlist_(
+			dp->hostlist_auto_mod_time = t;
-		&params.hostlists,
+			NonEmptyHostlist(&dp->hostlist);
-		bExclude ? &dp->hl_collection_exclude : &dp->hl_collection,
+		}
-		filename);
+	}
+	return HostlistCheck_(dp->hostlist, dp->hostlist_exclude, host, excluded);
 }
 
-void HostlistsDebug()
+bool LoadIncludeHostLists()
 {
-	if (!params.debug) return;
-
-	struct hostlist_file *hfile;
 	struct desync_profile_list *dpl;
-	struct hostlist_item *hl_item;
-
-	LIST_FOREACH(hfile, &params.hostlists, next)
-		DLOG("hostlist file %s%s\n",hfile->filename,hfile->hostlist ? "" : " (empty)");
-
 	LIST_FOREACH(dpl, &params.desync_profiles, next)
-	{
+		if (!LoadIncludeHostListsForProfile(&dpl->dp))
-		LIST_FOREACH(hl_item, &dpl->dp.hl_collection, next)
+			return false;
-			if (hl_item->hfile!=dpl->dp.hostlist_auto)
+	return true;
-				DLOG("profile %d include hostlist %s%s\n",dpl->dp.n, hl_item->hfile->filename,hl_item->hfile->hostlist ? "" : " (empty)");
+}
-		LIST_FOREACH(hl_item, &dpl->dp.hl_collection_exclude, next)
+bool LoadExcludeHostLists()
-			DLOG("profile %d exclude hostlist %s%s\n",dpl->dp.n,hl_item->hfile->filename,hl_item->hfile->hostlist ? "" : " (empty)");
+{
-		if (dpl->dp.hostlist_auto)
+	struct desync_profile_list *dpl;
-			DLOG("profile %d auto hostlist %s%s\n",dpl->dp.n,dpl->dp.hostlist_auto->filename,dpl->dp.hostlist_auto->hostlist ? "" : " (empty)");
+	LIST_FOREACH(dpl, &params.desync_profiles, next)
-	}
+		if (!LoadHostLists(&dpl->dp.hostlist_exclude, &dpl->dp.hostlist_exclude_files))
+			return false;
+	return true;
 }

nfq/hostlist.h

@@ -5,10 +5,10 @@
 #include "params.h"
 
 bool AppendHostList(strpool **hostlist, const char *filename);
-bool LoadAllHostLists();
+bool LoadHostLists(strpool **hostlist, struct str_list_head *file_list);
+bool LoadIncludeHostLists();
+bool LoadExcludeHostLists();
 bool NonEmptyHostlist(strpool **hostlist);
+bool SearchHostList(strpool *hostlist, const char *host);
 // return : true = apply fooling, false = do not apply
-bool HostlistCheck(const struct desync_profile *dp,const char *host, bool *excluded, bool bSkipReloadCheck);
+bool HostlistCheck(struct desync_profile *dp,const char *host, bool *excluded);
-struct hostlist_file *RegisterHostlist(struct desync_profile *dp, bool bExclude, const char *filename);
-bool HostlistsReloadCheckForProfile(const struct desync_profile *dp);
-void HostlistsDebug();

nfq/ipset.c

@@ -3,7 +3,6 @@
 #include "gzip.h"
 #include "helpers.h"
 
-
 // inplace tolower() and add to pool
 static bool addpool(ipset *ips, char **s, const char *end, int *ct)
 {
@@ -117,45 +116,37 @@ static bool AppendIpset(ipset *ips, const char *filename)
 	return true;
 }
 
-static bool LoadIpset(struct ipset_file *hfile)
+static bool LoadIpsets(ipset *ips, struct str_list_head *file_list)
 {
-	time_t t = file_mod_time(hfile->filename);
+	struct str_list *file;
-	if (!t)
+
+	ipsetDestroy(ips);
+
+	LIST_FOREACH(file, file_list, next)
 	{
-		// stat() error
+		if (!AppendIpset(ips, file->str)) return false;
-		DLOG_ERR("cannot access ipset file '%s'. in-memory content remains unchanged.\n",hfile->filename);
-		return true;
 	}
-	if (t==hfile->mod_time) return true; // up to date
+	return true;
-	ipsetDestroy(&hfile->ipset);
+}
-	if (!AppendIpset(&hfile->ipset, hfile->filename))
+
-	{
+bool LoadIncludeIpsets()
-		ipsetDestroy(&hfile->ipset);
+{
+	struct desync_profile_list *dpl;
+	LIST_FOREACH(dpl, &params.desync_profiles, next)
+		if (!LoadIpsets(&dpl->dp.ips, &dpl->dp.ipset_files))
 			return false;
-	}
-	hfile->mod_time=t;
 	return true;
 }
-static bool LoadIpsets(struct ipset_files_head *list)
+bool LoadExcludeIpsets()
 {
-	bool bres=true;
+	struct desync_profile_list *dpl;
-	struct ipset_file *hfile;
+	LIST_FOREACH(dpl, &params.desync_profiles, next)
-
+		if (!LoadIpsets(&dpl->dp.ips_exclude, &dpl->dp.ipset_exclude_files))
-	LIST_FOREACH(hfile, list, next)
+			return false;
-	{
+	return true;
-		if (!LoadIpset(hfile))
-			// at least one failed
-			bres=false;
-	}
-	return bres;
 }
 
-bool LoadAllIpsets()
+bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_addr *ipv6)
-{
-	return LoadIpsets(&params.ipsets);
-}
-
-static bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_addr *ipv6)
 {
 	char s_ip[40];
 	bool bInSet=false;
@@ -181,109 +172,24 @@ static bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const stru
 	return bInSet;
 }
 
-static bool IpsetsReloadCheck(const struct ipset_collection_head *ipsets)
+static bool IpsetCheck_(const ipset *ips, const ipset *ips_exclude, const struct in_addr *ipv4, const struct in6_addr *ipv6)
 {
-	struct ipset_item *item;
+	if (!IPSET_EMPTY(ips_exclude))
-	LIST_FOREACH(item, ipsets, next)
 	{
-		if (!LoadIpset(item->hfile))
+		DLOG("exclude ");
+		if (SearchIpset(ips_exclude, ipv4, ipv6))
 			return false;
 	}
-	return true;
+	if (!IPSET_EMPTY(ips))
-}
-bool IpsetsReloadCheckForProfile(const struct desync_profile *dp)
-{
-	return IpsetsReloadCheck(&dp->ips_collection) && IpsetsReloadCheck(&dp->ips_collection_exclude);
-}
-
-static bool IpsetCheck_(const struct ipset_collection_head *ips, const struct ipset_collection_head *ips_exclude, const struct in_addr *ipv4, const struct in6_addr *ipv6)
-{
-	struct ipset_item *item;
-
-	if (!IpsetsReloadCheck(ips) || !IpsetsReloadCheck(ips_exclude))
-		return false;
-
-	LIST_FOREACH(item, ips_exclude, next)
 	{
-		DLOG("[%s] exclude ",item->hfile->filename);
+		DLOG("include ");
-		if (SearchIpset(&item->hfile->ipset, ipv4, ipv6))
+		return SearchIpset(ips, ipv4, ipv6);
-			return false;
-	}
-	// old behavior compat: all include lists are empty means check passes
-	if (!ipset_collection_is_empty(ips))
-	{
-		LIST_FOREACH(item, ips, next)
-		{
-			DLOG("[%s] include ",item->hfile->filename);
-			if (SearchIpset(&item->hfile->ipset, ipv4, ipv6))
-				return true;
-		}
-		return false;
 	}
 	return true;
 }
 
-bool IpsetCheck(const struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6)
+bool IpsetCheck(struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6)
 {
-	if (PROFILE_IPSETS_EMPTY(dp)) return true;
+	if (!PROFILE_IPSETS_EMPTY(dp)) DLOG("* ipset check for profile %d\n",dp->n);
-	DLOG("* ipset check for profile %d\n",dp->n);
+	return IpsetCheck_(&dp->ips,&dp->ips_exclude,ipv4,ipv6);
-	return IpsetCheck_(&dp->ips_collection,&dp->ips_collection_exclude,ipv4,ipv6);
-}
-
-
-static struct ipset_file *RegisterIpset_(struct ipset_files_head *ipsets, struct ipset_collection_head *ips_collection, const char *filename)
-{
-	struct ipset_file *hfile;
-	if (!(hfile=ipset_files_search(ipsets, filename)))
-		if (!(hfile=ipset_files_add(ipsets, filename)))
-			return NULL;
-	if (!ipset_collection_search(ips_collection, filename))
-		if (!ipset_collection_add(ips_collection, hfile))
-			return NULL;
-	return hfile;
-}
-struct ipset_file *RegisterIpset(struct desync_profile *dp, bool bExclude, const char *filename)
-{
-	if (!file_mod_time(filename))
-	{
-		DLOG_ERR("cannot access ipset file '%s'\n",filename);
-		return NULL;
-	}
-	return RegisterIpset_(
-		&params.ipsets,
-		bExclude ? &dp->ips_collection_exclude : &dp->ips_collection,
-		filename);
-}
-
-static const char *dbg_ipset_fill(const ipset *ips)
-{
-	if (ips->ips4)
-		if (ips->ips6)
-			return "ipv4+ipv6";
-		else
-			return "ipv4";
-	else
-		if (ips->ips6)
-			return "ipv6";
-		else
-			return "empty";
-}
-void IpsetsDebug()
-{
-	if (!params.debug) return;
-
-	struct ipset_file *hfile;
-	struct desync_profile_list *dpl;
-	struct ipset_item *ips_item;
-
-	LIST_FOREACH(hfile, &params.ipsets, next)
-		DLOG("ipset file %s (%s)\n",hfile->filename,dbg_ipset_fill(&hfile->ipset));
-
-	LIST_FOREACH(dpl, &params.desync_profiles, next)
-	{
-		LIST_FOREACH(ips_item, &dpl->dp.ips_collection, next)
-			DLOG("profile %d include ipset %s (%s)\n",dpl->dp.n,ips_item->hfile->filename,dbg_ipset_fill(&ips_item->hfile->ipset));
-		LIST_FOREACH(ips_item, &dpl->dp.ips_collection_exclude, next)
-			DLOG("profile %d exclude ipset %s (%s)\n",dpl->dp.n,ips_item->hfile->filename,dbg_ipset_fill(&ips_item->hfile->ipset));
-	}
 }

nfq/ipset.h

@@ -5,7 +5,7 @@
 #include "params.h"
 #include "pools.h"
 
-bool LoadAllIpsets();
+bool LoadIncludeIpsets();
-bool IpsetCheck(const struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6);
+bool LoadExcludeIpsets();
-struct ipset_file *RegisterIpset(struct desync_profile *dp, bool bExclude, const char *filename);
+bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_addr *ipv6);
-void IpsetsDebug();
+bool IpsetCheck(struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6);

nfq/nfqws.c

@@ -51,10 +51,25 @@ struct params_s params;
 bool bQuit=false;
 #endif
 
+static bool bHup = false;
 static void onhup(int sig)
 {
 	printf("HUP received !\n");
-	// do not do anything. lists auto reload themselves based on file time.
+	printf("Will reload hostlists and ipsets on next request (if any)\n");
+	bHup = true;
+}
+// should be called in normal execution
+static void dohup(void)
+{
+	if (bHup)
+	{
+		if (!LoadIncludeHostLists() || !LoadExcludeHostLists() || !LoadIncludeIpsets() || !LoadExcludeIpsets())
+		{
+			// what will we do without hostlist ?? sure, gonna die
+			exit(1);
+		}
+		bHup = false;
+	}
 }
 
 static void onusr1(int sig)
@@ -224,6 +239,7 @@ static int nfq_main(void)
 	{
 		while ((rv = recv(fd, buf, sizeof(buf), 0)) > 0)
 		{
+			dohup();
 			int r = nfq_handle_packet(h, (char *)buf, rv);
 			if (r) DLOG_ERR("nfq_handle_packet error %d\n", r);
 		}
@@ -335,6 +351,7 @@ static int dvt_main(void)
 			if (errno==EINTR)
 			{
 				// a signal received
+				dohup();
 				continue;
 			}
 			DLOG_PERROR("select");
@@ -490,6 +507,8 @@ static int win_main(const char *windivert_filter)
 			}
 			else
 			{
+				dohup();
+
 				mark=0;
 				// pseudo interface id IfIdx.SubIfIdx
 				verdict = processPacketData(&mark, ifout, packet, &len);
@@ -556,8 +575,6 @@ static void cleanup_params(void)
 
 	dp_list_destroy(&params.desync_profiles);
 
-	hostlist_files_destroy(&params.hostlists);
-	ipset_files_destroy(&params.ipsets);
 #ifdef __CYGWIN__
 	strlist_destroy(&params.ssid_filter);
 	strlist_destroy(&params.nlm_filter);
@@ -980,7 +997,6 @@ int main(int argc, char **argv)
 	struct desync_profile_list *dpl;
 	struct desync_profile *dp;
 	int desync_profile_count=0;
-
 	if (!(dpl = dp_list_add(&params.desync_profiles)))
 	{
 		DLOG_ERR("desync_profile_add: out of memory\n");
@@ -1000,9 +1016,6 @@ int main(int argc, char **argv)
 	params.ctrack_t_fin = CTRACK_T_FIN;
 	params.ctrack_t_udp = CTRACK_T_UDP;
 
-	LIST_INIT(&params.hostlists);
-	LIST_INIT(&params.ipsets);
-
 #ifdef __CYGWIN__
 	LIST_INIT(&params.ssid_filter);
 	LIST_INIT(&params.nlm_filter);
@@ -1509,27 +1522,27 @@ int main(int argc, char **argv)
 			}
 			break;
 		case 45: /* hostlist */
-			if (!RegisterHostlist(dp, false, optarg))
+			if (!strlist_add(&dp->hostlist_files, optarg))
 			{
-				DLOG_ERR("failed to register hostlist '%s'\n", optarg);
+				DLOG_ERR("strlist_add failed\n");
 				exit_clean(1);
 			}
 			break;
 		case 46: /* hostlist-exclude */
-			if (!RegisterHostlist(dp, true, optarg))
+			if (!strlist_add(&dp->hostlist_exclude_files, optarg))
 			{
-				DLOG_ERR("failed to register hostlist '%s'\n", optarg);
+				DLOG_ERR("strlist_add failed\n");
 				exit_clean(1);
 			}
 			break;
 		case 47: /* hostlist-auto */
-			if (dp->hostlist_auto)
+			if (*dp->hostlist_auto_filename)
 			{
 				DLOG_ERR("only one auto hostlist per profile is supported\n");
 				exit_clean(1);
 			}
 			{
-				FILE *F = fopen(optarg,"a+b");
+				FILE *F = fopen(optarg,"at");
 				if (!F)
 				{
 					DLOG_ERR("cannot create %s\n", optarg);
@@ -1547,11 +1560,13 @@ int main(int argc, char **argv)
 					DLOG_ERR("could not chown %s. auto hostlist file may not be writable after privilege drop\n", optarg);
 #endif
 			}
-			if (!(dp->hostlist_auto=RegisterHostlist(dp, false, optarg)))
+			if (!strlist_add(&dp->hostlist_files, optarg))
 			{
-				DLOG_ERR("failed to register hostlist '%s'\n", optarg);
+				DLOG_ERR("strlist_add failed\n");
 				exit_clean(1);
 			}
+			strncpy(dp->hostlist_auto_filename, optarg, sizeof(dp->hostlist_auto_filename));
+			dp->hostlist_auto_filename[sizeof(dp->hostlist_auto_filename) - 1] = '\0';
 			break;
 		case 48: /* hostlist-auto-fail-threshold */
 			dp->hostlist_auto_fail_threshold = (uint8_t)atoi(optarg);
@@ -1637,16 +1652,16 @@ int main(int argc, char **argv)
 			}
 			break;
 		case 57: /* ipset */
-			if (!RegisterIpset(dp, false, optarg))
+			if (!strlist_add(&dp->ipset_files, optarg))
 			{
-				DLOG_ERR("failed to register ipset '%s'\n", optarg);
+				DLOG_ERR("strlist_add failed\n");
 				exit_clean(1);
 			}
 			break;
 		case 58: /* ipset-exclude */
-			if (!RegisterIpset(dp, true, optarg))
+			if (!strlist_add(&dp->ipset_exclude_files, optarg))
 			{
-				DLOG_ERR("failed to register ipset '%s'\n", optarg);
+				DLOG_ERR("strlist_add failed\n");
 				exit_clean(1);
 			}
 			break;
@@ -1839,21 +1854,26 @@ int main(int argc, char **argv)
 		if (dp->desync_split_http_req==httpreqpos_none && dp->desync_split_pos) dp->desync_split_http_req=httpreqpos_pos;
 	}
 
-	if (!LoadAllHostLists())
+	if (!LoadIncludeHostLists())
 	{
-		DLOG_ERR("hostlists load failed\n");
+		DLOG_ERR("Include hostlists load failed\n");
 		exit_clean(1);
 	}
-	if (!LoadAllIpsets())
+	if (!LoadExcludeHostLists())
 	{
-		DLOG_ERR("ipset load failed\n");
+		DLOG_ERR("Exclude hostlists load failed\n");
+		exit_clean(1);
+	}
+	if (!LoadIncludeIpsets())
+	{
+		DLOG_ERR("Include ipset load failed\n");
+		exit_clean(1);
+	}
+	if (!LoadExcludeIpsets())
+	{
+		DLOG_ERR("Exclude ipset load failed\n");
 		exit_clean(1);
 	}
-	
-	DLOG("\nlists summary:\n");
-	HostlistsDebug();
-	IpsetsDebug();
-	DLOG("\n");
 	
 	if (daemon) daemonize();
 

nfq/params.c

@@ -160,10 +160,8 @@ struct desync_profile_list *dp_list_add(struct desync_profile_list_head *head)
 	struct desync_profile_list *entry = calloc(1,sizeof(struct desync_profile_list));
 	if (!entry) return NULL;
 	
-	LIST_INIT(&entry->dp.hl_collection);
+	LIST_INIT(&entry->dp.hostlist_files);
-	LIST_INIT(&entry->dp.hl_collection_exclude);
+	LIST_INIT(&entry->dp.hostlist_exclude_files);
-	LIST_INIT(&entry->dp.ips_collection);
-	LIST_INIT(&entry->dp.ips_collection_exclude);
 	memcpy(entry->dp.hostspell, "host", 4); // default hostspell
 	entry->dp.desync_skip_nosni = true;
 	entry->dp.desync_split_pos = 2;
@@ -207,10 +205,14 @@ struct desync_profile_list *dp_list_add(struct desync_profile_list_head *head)
 }
 static void dp_entry_destroy(struct desync_profile_list *entry)
 {
-	hostlist_collection_destroy(&entry->dp.hl_collection);
+	strlist_destroy(&entry->dp.hostlist_files);
-	hostlist_collection_destroy(&entry->dp.hl_collection_exclude);
+	strlist_destroy(&entry->dp.hostlist_exclude_files);
-	ipset_collection_destroy(&entry->dp.ips_collection);
+	strlist_destroy(&entry->dp.ipset_files);
-	ipset_collection_destroy(&entry->dp.ips_collection_exclude);
+	strlist_destroy(&entry->dp.ipset_exclude_files);
+	StrPoolDestroy(&entry->dp.hostlist_exclude);
+	StrPoolDestroy(&entry->dp.hostlist);
+	ipsetDestroy(&entry->dp.ips);
+	ipsetDestroy(&entry->dp.ips_exclude);
 	HostFailPoolDestroy(&entry->dp.hostlist_auto_fail_counters);
 	free(entry);
 }
@@ -223,3 +225,12 @@ void dp_list_destroy(struct desync_profile_list_head *head)
 		dp_entry_destroy(entry);
 	}
 }
+bool dp_list_have_autohostlist(struct desync_profile_list_head *head)
+{
+	struct desync_profile_list *dpl;
+	LIST_FOREACH(dpl, head, next)
+		if (*dpl->dp.hostlist_auto_filename)
+			return true;
+	return false;
+}
+

nfq/params.h

@@ -67,21 +67,18 @@ struct desync_profile
 	bool filter_ipv4,filter_ipv6;
 	port_filter pf_tcp,pf_udp;
 	uint32_t filter_l7;	// L7_PROTO_* bits
+	ipset ips,ips_exclude;
+	struct str_list_head ipset_files, ipset_exclude_files;
 
-	// list of pointers to ipsets
+	strpool *hostlist, *hostlist_exclude;
-	struct ipset_collection_head ips_collection, ips_collection_exclude;
+	struct str_list_head hostlist_files, hostlist_exclude_files;
-
+	char hostlist_auto_filename[PATH_MAX];
-	// list of pointers to hostlist files
-	struct hostlist_collection_head hl_collection, hl_collection_exclude;
-	// pointer to autohostlist. NULL if no autohostlist for the profile.
-	struct hostlist_file *hostlist_auto;
 	int hostlist_auto_fail_threshold, hostlist_auto_fail_time, hostlist_auto_retrans_threshold;
-
+	time_t hostlist_auto_mod_time;
 	hostfail_pool *hostlist_auto_fail_counters;
 };
 
-#define PROFILE_IPSETS_EMPTY(dp) (ipset_collection_is_empty(&dp->ips_collection) && ipset_collection_is_empty(&dp->ips_collection_exclude))
+#define PROFILE_IPSETS_EMPTY(dp) (IPSET_EMPTY(&dp->ips) && IPSET_EMPTY(&dp->ips_exclude))
-#define PROFILE_HOSTLISTS_EMPTY(dp) (hostlist_collection_is_empty(&dp->hl_collection) && hostlist_collection_is_empty(&dp->hl_collection_exclude))
 
 struct desync_profile_list {
 	struct desync_profile dp;
@@ -90,6 +87,7 @@ struct desync_profile_list {
 LIST_HEAD(desync_profile_list_head, desync_profile_list);
 struct desync_profile_list *dp_list_add(struct desync_profile_list_head *head);
 void dp_list_destroy(struct desync_profile_list_head *head);
+bool dp_list_have_autohostlist(struct desync_profile_list_head *head);
 
 struct params_s
 {
@@ -117,11 +115,6 @@ struct params_s
 
 	char hostlist_auto_debuglog[PATH_MAX];
 	
-	// hostlist files with data for all profiles
-	struct hostlist_files_head hostlists;
-	// ipset files with data for all profiles
-	struct ipset_files_head ipsets;
-
 	unsigned int ctrack_t_syn, ctrack_t_est, ctrack_t_fin, ctrack_t_udp;
 	t_conntrack conntrack;
 };

nfq/pools.c

@@ -154,93 +154,6 @@ void strlist_destroy(struct str_list_head *head)
 
 
 
-
-struct hostlist_file *hostlist_files_add(struct hostlist_files_head *head, const char *filename)
-{
-	struct hostlist_file *entry = malloc(sizeof(struct hostlist_file));
-	if (entry)
-	{
-		if (!(entry->filename = strdup(filename)))
-		{
-			free(entry);
-			return false;
-		}
-		entry->mod_time=0;
-		entry->hostlist = NULL;
-		LIST_INSERT_HEAD(head, entry, next);
-	}
-	return entry;
-}
-static void hostlist_files_entry_destroy(struct hostlist_file *entry)
-{
-	if (entry->filename) free(entry->filename);
-	StrPoolDestroy(&entry->hostlist);
-	free(entry);
-}
-void hostlist_files_destroy(struct hostlist_files_head *head)
-{
-	struct hostlist_file *entry;
-	while ((entry = LIST_FIRST(head)))
-	{
-		LIST_REMOVE(entry, next);
-		hostlist_files_entry_destroy(entry);
-	}
-}
-struct hostlist_file *hostlist_files_search(struct hostlist_files_head *head, const char *filename)
-{
-	struct hostlist_file *hfile;
-
-	LIST_FOREACH(hfile, head, next)
-	{
-		if (!strcmp(hfile->filename,filename))
-			return hfile;
-	}
-	return NULL;
-}
-
-struct hostlist_item *hostlist_collection_add(struct hostlist_collection_head *head, struct hostlist_file *hfile)
-{
-	struct hostlist_item *entry = malloc(sizeof(struct hostlist_item));
-	if (entry)
-	{
-		entry->hfile = hfile;
-		LIST_INSERT_HEAD(head, entry, next);
-	}
-	return entry;
-}
-void hostlist_collection_destroy(struct hostlist_collection_head *head)
-{
-	struct hostlist_item *entry;
-	while ((entry = LIST_FIRST(head)))
-	{
-		LIST_REMOVE(entry, next);
-		free(entry);
-	}
-}
-struct hostlist_item *hostlist_collection_search(struct hostlist_collection_head *head, const char *filename)
-{
-	struct hostlist_item *item;
-
-	LIST_FOREACH(item, head, next)
-	{
-		if (!strcmp(item->hfile->filename,filename))
-			return item;
-	}
-	return NULL;
-}
-bool hostlist_collection_is_empty(const struct hostlist_collection_head *head)
-{
-	const struct hostlist_item *item;
-
-	LIST_FOREACH(item, head, next)
-	{
-		if (item->hfile->hostlist)
-			return false;
-	}
-	return true;
-}
-
-
 void ipset4Destroy(ipset4 **ipset)
 {
 	ipset4 *elem, *tmp;
@@ -362,89 +275,3 @@ void ipsetPrint(ipset *ipset)
 	ipset4Print(ipset->ips4);
 	ipset6Print(ipset->ips6);
 }
-
-
-struct ipset_file *ipset_files_add(struct ipset_files_head *head, const char *filename)
-{
-	struct ipset_file *entry = malloc(sizeof(struct ipset_file));
-	if (entry)
-	{
-		if (!(entry->filename = strdup(filename)))
-		{
-			free(entry);
-			return false;
-		}
-		entry->mod_time=0;
-		memset(&entry->ipset,0,sizeof(entry->ipset));
-		LIST_INSERT_HEAD(head, entry, next);
-	}
-	return entry;
-}
-static void ipset_files_entry_destroy(struct ipset_file *entry)
-{
-	if (entry->filename) free(entry->filename);
-	ipsetDestroy(&entry->ipset);
-	free(entry);
-}
-void ipset_files_destroy(struct ipset_files_head *head)
-{
-	struct ipset_file *entry;
-	while ((entry = LIST_FIRST(head)))
-	{
-		LIST_REMOVE(entry, next);
-		ipset_files_entry_destroy(entry);
-	}
-}
-struct ipset_file *ipset_files_search(struct ipset_files_head *head, const char *filename)
-{
-	struct ipset_file *hfile;
-
-	LIST_FOREACH(hfile, head, next)
-	{
-		if (!strcmp(hfile->filename,filename))
-			return hfile;
-	}
-	return NULL;
-}
-
-struct ipset_item *ipset_collection_add(struct ipset_collection_head *head, struct ipset_file *hfile)
-{
-	struct ipset_item *entry = malloc(sizeof(struct ipset_item));
-	if (entry)
-	{
-		entry->hfile = hfile;
-		LIST_INSERT_HEAD(head, entry, next);
-	}
-	return entry;
-}
-void ipset_collection_destroy(struct ipset_collection_head *head)
-{
-	struct ipset_item *entry;
-	while ((entry = LIST_FIRST(head)))
-	{
-		LIST_REMOVE(entry, next);
-		free(entry);
-	}
-}
-struct ipset_item *ipset_collection_search(struct ipset_collection_head *head, const char *filename)
-{
-	struct ipset_item *item;
-
-	LIST_FOREACH(item, head, next)
-	{
-		if (!strcmp(item->hfile->filename,filename))
-			return item;
-	}
-	return NULL;
-}
-bool ipset_collection_is_empty(const struct ipset_collection_head *head)
-{
-	const struct ipset_item *item;
-
-	LIST_FOREACH(item, head, next)
-	{
-		if (!IPSET_EMPTY(&item->hfile->ipset))
-			return false;
-	}
-	return true;
-}

nfq/pools.h

@@ -47,30 +47,6 @@ bool strlist_add(struct str_list_head *head, const char *filename);
 void strlist_destroy(struct str_list_head *head);
 
 
-
-struct hostlist_file {
-	char *filename;
-	time_t mod_time;
-	strpool *hostlist;
-	LIST_ENTRY(hostlist_file) next;
-};
-LIST_HEAD(hostlist_files_head, hostlist_file);
-
-struct hostlist_file *hostlist_files_add(struct hostlist_files_head *head, const char *filename);
-void hostlist_files_destroy(struct hostlist_files_head *head);
-struct hostlist_file *hostlist_files_search(struct hostlist_files_head *head, const char *filename);
-
-struct hostlist_item {
-	struct hostlist_file *hfile;
-	LIST_ENTRY(hostlist_item) next;
-};
-LIST_HEAD(hostlist_collection_head, hostlist_item);
-struct hostlist_item *hostlist_collection_add(struct hostlist_collection_head *head, struct hostlist_file *hfile);
-void hostlist_collection_destroy(struct hostlist_collection_head *head);
-struct hostlist_item *hostlist_collection_search(struct hostlist_collection_head *head, const char *filename);
-bool hostlist_collection_is_empty(const struct hostlist_collection_head *head);
-
-
 typedef struct ipset4 {
 	struct cidr4 cidr;	/* key */
 	UT_hash_handle hh;	/* makes this structure hashable */
@@ -107,26 +83,3 @@ void ipset6Print(ipset6 *ipset);
 
 void ipsetDestroy(ipset *ipset);
 void ipsetPrint(ipset *ipset);
-
-
-struct ipset_file {
-	char *filename;
-	time_t mod_time;
-	ipset ipset;
-	LIST_ENTRY(ipset_file) next;
-};
-LIST_HEAD(ipset_files_head, ipset_file);
-
-struct ipset_file *ipset_files_add(struct ipset_files_head *head, const char *filename);
-void ipset_files_destroy(struct ipset_files_head *head);
-struct ipset_file *ipset_files_search(struct ipset_files_head *head, const char *filename);
-
-struct ipset_item {
-	struct ipset_file *hfile;
-	LIST_ENTRY(ipset_item) next;
-};
-LIST_HEAD(ipset_collection_head, ipset_item);
-struct ipset_item * ipset_collection_add(struct ipset_collection_head *head, struct ipset_file *hfile);
-void ipset_collection_destroy(struct ipset_collection_head *head);
-struct ipset_item *ipset_collection_search(struct ipset_collection_head *head, const char *filename);
-bool ipset_collection_is_empty(const struct ipset_collection_head *head);