From 2464d27550b766413019057d066fbbb07bbe8a37 Mon Sep 17 00:00:00 2001 From: bol-van Date: Tue, 29 Oct 2024 15:08:05 +0300 Subject: [PATCH] tpws: hostlist/ipset dedup and auto reload --- tpws/hostlist.c | 195 ++++++++++++++++++++++++++++++++--------------- tpws/hostlist.h | 10 +-- tpws/ipset.c | 162 ++++++++++++++++++++++++++++++--------- tpws/ipset.h | 8 +- tpws/params.c | 28 +++---- tpws/params.h | 27 ++++--- tpws/pools.c | 179 +++++++++++++++++++++++++++++++++++++++++-- tpws/pools.h | 49 +++++++++++- tpws/tamper.c | 72 +++++++++-------- tpws/tpws.c | 73 +++++++----------- tpws/tpws_conn.c | 2 - 11 files changed, 577 insertions(+), 228 deletions(-) diff --git a/tpws/hostlist.c b/tpws/hostlist.c index b429dd8..48b3a79 100644 --- a/tpws/hostlist.c +++ b/tpws/hostlist.c @@ -1,7 +1,6 @@ #include #include "hostlist.h" #include "gzip.h" -#include "params.h" #include "helpers.h" // inplace tolower() and add to pool @@ -56,7 +55,7 @@ bool AppendHostList(strpool **hostlist, const char *filename) if (r==Z_OK) { DLOG_CONDUP("zlib compression detected. uncompressed size : %zu\n", zsize); - + p = zbuf; e = zbuf + zsize; while(pfilename); + if (!t) { - StrPoolDestroy(hostlist); - *hostlist = NULL; + // stat() error + DLOG_ERR("cannot access hostlist file '%s'. in-memory content remains unchanged.\n",hfile->filename); + return true; } - - LIST_FOREACH(file, file_list, next) + if (t==hfile->mod_time) return true; // up to date + StrPoolDestroy(&hfile->hostlist); + if (!AppendHostList(&hfile->hostlist, hfile->filename)) { - if (!AppendHostList(hostlist, file->str)) return false; + StrPoolDestroy(&hfile->hostlist); + return false; } + hfile->mod_time=t; return true; } +static bool LoadHostLists(struct hostlist_files_head *list) +{ + bool bres=true; + struct hostlist_file *hfile; + + LIST_FOREACH(hfile, list, next) + { + if (!LoadHostList(hfile)) + // at least one failed + bres=false; + } + return bres; +} bool NonEmptyHostlist(strpool **hostlist) { @@ -120,8 +135,27 @@ bool NonEmptyHostlist(strpool **hostlist) return *hostlist ? true : StrPoolAddStrLen(hostlist, "@&()", 4); } +static void MakeAutolistsNonEmpty() +{ + struct desync_profile_list *dpl; + LIST_FOREACH(dpl, ¶ms.desync_profiles, next) + { + if (dpl->dp.hostlist_auto) + NonEmptyHostlist(&dpl->dp.hostlist_auto->hostlist); + } +} -bool SearchHostList(strpool *hostlist, const char *host) +bool LoadAllHostLists() +{ + if (!LoadHostLists(¶ms.hostlists)) + return false; + MakeAutolistsNonEmpty(); + return true; +} + + + +static bool SearchHostList(strpool *hostlist, const char *host) { if (hostlist) { @@ -130,7 +164,7 @@ bool SearchHostList(strpool *hostlist, const char *host) while (p) { bInHostList = StrPoolCheckStr(hostlist, p); - VPRINT("Hostlist check for %s : %s\n", p, bInHostList ? "positive" : "negative"); + VPRINT("hostlist check for %s : %s\n", p, bInHostList ? "positive" : "negative"); if (bInHostList) return true; p = strchr(p, '.'); if (p) p++; @@ -139,73 +173,108 @@ bool SearchHostList(strpool *hostlist, const char *host) return false; } -// return : true = apply fooling, false = do not apply -static bool HostlistCheck_(strpool *hostlist, strpool *hostlist_exclude, const char *host, bool *excluded) + +static bool HostlistsReloadCheck(const struct hostlist_collection_head *hostlists) { - if (excluded) *excluded = false; - if (hostlist_exclude) + struct hostlist_item *item; + LIST_FOREACH(item, hostlists, next) { - VPRINT("Checking exclude hostlist\n"); - if (SearchHostList(hostlist_exclude, host)) + if (!LoadHostList(item->hfile)) + return false; + } + MakeAutolistsNonEmpty(); + return true; +} +bool HostlistsReloadCheckForProfile(const struct desync_profile *dp) +{ + return HostlistsReloadCheck(&dp->hl_collection) && HostlistsReloadCheck(&dp->hl_collection_exclude); +} +// return : true = apply fooling, false = do not apply +static bool HostlistCheck_(const struct hostlist_collection_head *hostlists, const struct hostlist_collection_head *hostlists_exclude, const char *host, bool *excluded, bool bSkipReloadCheck) +{ + struct hostlist_item *item; + + if (excluded) *excluded = false; + + if (!bSkipReloadCheck) + if (!HostlistsReloadCheck(hostlists) || !HostlistsReloadCheck(hostlists_exclude)) + return false; + + LIST_FOREACH(item, hostlists_exclude, next) + { + VPRINT("[%s] exclude ", item->hfile->filename); + if (SearchHostList(item->hfile->hostlist, host)) { if (excluded) *excluded = true; return false; } } - if (hostlist) + // old behavior compat: all include lists are empty means check passes + if (!hostlist_collection_is_empty(hostlists)) { - VPRINT("Checking include hostlist\n"); - return SearchHostList(hostlist, host); - } - return true; -} - -static bool LoadIncludeHostListsForProfile(struct desync_profile *dp) -{ - if (!LoadHostLists(&dp->hostlist, &dp->hostlist_files)) + LIST_FOREACH(item, hostlists, next) + { + VPRINT("[%s] include ", item->hfile->filename); + if (SearchHostList(item->hfile->hostlist, host)) + return true; + } return false; - if (*dp->hostlist_auto_filename) - { - dp->hostlist_auto_mod_time = file_mod_time(dp->hostlist_auto_filename); - NonEmptyHostlist(&dp->hostlist); } return true; } -bool HostlistCheck(struct desync_profile *dp, const char *host, bool *excluded) + +// return : true = apply fooling, false = do not apply +bool HostlistCheck(const struct desync_profile *dp, const char *host, bool *excluded, bool bSkipReloadCheck) { VPRINT("* hostlist check for profile %d\n",dp->n); - if (*dp->hostlist_auto_filename) - { - time_t t = file_mod_time(dp->hostlist_auto_filename); - if (t!=dp->hostlist_auto_mod_time) - { - DLOG_CONDUP("Autohostlist '%s' from profile %d was modified. Reloading include hostlists for this profile.\n",dp->hostlist_auto_filename, dp->n); - if (!LoadIncludeHostListsForProfile(dp)) - { - // what will we do without hostlist ?? sure, gonna die - exit(1); - } - dp->hostlist_auto_mod_time = t; - NonEmptyHostlist(&dp->hostlist); - } - } - return HostlistCheck_(dp->hostlist, dp->hostlist_exclude, host, excluded); + return HostlistCheck_(&dp->hl_collection, &dp->hl_collection_exclude, host, excluded, bSkipReloadCheck); } -bool LoadIncludeHostLists() + +static struct hostlist_file *RegisterHostlist_(struct hostlist_files_head *hostlists, struct hostlist_collection_head *hl_collection, const char *filename) { - struct desync_profile_list *dpl; - LIST_FOREACH(dpl, ¶ms.desync_profiles, next) - if (!LoadIncludeHostListsForProfile(&dpl->dp)) - return false; - return true; + struct hostlist_file *hfile; + if (!(hfile=hostlist_files_search(hostlists, filename))) + if (!(hfile=hostlist_files_add(hostlists, filename))) + return NULL; + if (!hostlist_collection_search(hl_collection, filename)) + if (!hostlist_collection_add(hl_collection, hfile)) + return NULL; + return hfile; } -bool LoadExcludeHostLists() +struct hostlist_file *RegisterHostlist(struct desync_profile *dp, bool bExclude, const char *filename) { - struct desync_profile_list *dpl; - LIST_FOREACH(dpl, ¶ms.desync_profiles, next) - if (!LoadHostLists(&dpl->dp.hostlist_exclude, &dpl->dp.hostlist_exclude_files)) - return false; - return true; + if (!file_mod_time(filename)) + { + DLOG_ERR("cannot access hostlist file '%s'\n",filename); + return NULL; + } + return RegisterHostlist_( + ¶ms.hostlists, + bExclude ? &dp->hl_collection_exclude : &dp->hl_collection, + filename); +} + +void HostlistsDebug() +{ + if (!params.debug) return; + + struct hostlist_file *hfile; + struct desync_profile_list *dpl; + struct hostlist_item *hl_item; + + LIST_FOREACH(hfile, ¶ms.hostlists, next) + VPRINT("hostlist file %s%s\n",hfile->filename,hfile->hostlist ? "" : " (empty)"); + + LIST_FOREACH(dpl, ¶ms.desync_profiles, next) + { + LIST_FOREACH(hl_item, &dpl->dp.hl_collection, next) + if (hl_item->hfile!=dpl->dp.hostlist_auto) + VPRINT("profile %d include hostlist %s%s\n",dpl->dp.n, hl_item->hfile->filename,hl_item->hfile->hostlist ? "" : " (empty)"); + LIST_FOREACH(hl_item, &dpl->dp.hl_collection_exclude, next) + VPRINT("profile %d exclude hostlist %s%s\n",dpl->dp.n,hl_item->hfile->filename,hl_item->hfile->hostlist ? "" : " (empty)"); + if (dpl->dp.hostlist_auto) + VPRINT("profile %d auto hostlist %s%s\n",dpl->dp.n,dpl->dp.hostlist_auto->filename,dpl->dp.hostlist_auto->hostlist ? "" : " (empty)"); + } } diff --git a/tpws/hostlist.h b/tpws/hostlist.h index d909737..f49db2d 100644 --- a/tpws/hostlist.h +++ b/tpws/hostlist.h @@ -5,10 +5,10 @@ #include "params.h" bool AppendHostList(strpool **hostlist, const char *filename); -bool LoadHostLists(strpool **hostlist, struct str_list_head *file_list); -bool LoadIncludeHostLists(); -bool LoadExcludeHostLists(); +bool LoadAllHostLists(); bool NonEmptyHostlist(strpool **hostlist); -bool SearchHostList(strpool *hostlist, const char *host); // return : true = apply fooling, false = do not apply -bool HostlistCheck(struct desync_profile *dp,const char *host, bool *excluded); \ No newline at end of file +bool HostlistCheck(const struct desync_profile *dp,const char *host, bool *excluded, bool bSkipReloadCheck); +struct hostlist_file *RegisterHostlist(struct desync_profile *dp, bool bExclude, const char *filename); +bool HostlistsReloadCheckForProfile(const struct desync_profile *dp); +void HostlistsDebug(); diff --git a/tpws/ipset.c b/tpws/ipset.c index 93b60fd..6a8e4c9 100644 --- a/tpws/ipset.c +++ b/tpws/ipset.c @@ -3,6 +3,7 @@ #include "gzip.h" #include "helpers.h" + // inplace tolower() and add to pool static bool addpool(ipset *ips, char **s, const char *end, int *ct) { @@ -75,7 +76,7 @@ static bool AppendIpset(ipset *ips, const char *filename) if (r==Z_OK) { DLOG_CONDUP("zlib compression detected. uncompressed size : %zu\n", zsize); - + p = zbuf; e = zbuf + zsize; while(pfilename); + if (!t) { - if (!AppendIpset(ips, file->str)) return false; + // stat() error + DLOG_ERR("cannot access ipset file '%s'. in-memory content remains unchanged.\n",hfile->filename); + return true; } + if (t==hfile->mod_time) return true; // up to date + ipsetDestroy(&hfile->ipset); + if (!AppendIpset(&hfile->ipset, hfile->filename)) + { + ipsetDestroy(&hfile->ipset); + return false; + } + hfile->mod_time=t; return true; } +static bool LoadIpsets(struct ipset_files_head *list) +{ + bool bres=true; + struct ipset_file *hfile; + + LIST_FOREACH(hfile, list, next) + { + if (!LoadIpset(hfile)) + // at least one failed + bres=false; + } + return bres; +} -bool LoadIncludeIpsets() +bool LoadAllIpsets() { - struct desync_profile_list *dpl; - LIST_FOREACH(dpl, ¶ms.desync_profiles, next) - if (!LoadIpsets(&dpl->dp.ips, &dpl->dp.ipset_files)) - return false; - return true; -} -bool LoadExcludeIpsets() -{ - struct desync_profile_list *dpl; - LIST_FOREACH(dpl, ¶ms.desync_profiles, next) - if (!LoadIpsets(&dpl->dp.ips_exclude, &dpl->dp.ipset_exclude_files)) - return false; - return true; + return LoadIpsets(¶ms.ipsets); } -bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_addr *ipv6) +static bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_addr *ipv6) { char s_ip[40]; bool bInSet=false; @@ -172,24 +181,109 @@ bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_ return bInSet; } -static bool IpsetCheck_(const ipset *ips, const ipset *ips_exclude, const struct in_addr *ipv4, const struct in6_addr *ipv6) +static bool IpsetsReloadCheck(const struct ipset_collection_head *ipsets) { - if (!IPSET_EMPTY(ips_exclude)) + struct ipset_item *item; + LIST_FOREACH(item, ipsets, next) { - VPRINT("exclude "); - if (SearchIpset(ips_exclude, ipv4, ipv6)) + if (!LoadIpset(item->hfile)) return false; } - if (!IPSET_EMPTY(ips)) + return true; +} +bool IpsetsReloadCheckForProfile(const struct desync_profile *dp) +{ + return IpsetsReloadCheck(&dp->ips_collection) && IpsetsReloadCheck(&dp->ips_collection_exclude); +} + +static bool IpsetCheck_(const struct ipset_collection_head *ips, const struct ipset_collection_head *ips_exclude, const struct in_addr *ipv4, const struct in6_addr *ipv6) +{ + struct ipset_item *item; + + if (!IpsetsReloadCheck(ips) || !IpsetsReloadCheck(ips_exclude)) + return false; + + LIST_FOREACH(item, ips_exclude, next) { - VPRINT("include "); - return SearchIpset(ips, ipv4, ipv6); + VPRINT("[%s] exclude ",item->hfile->filename); + if (SearchIpset(&item->hfile->ipset, ipv4, ipv6)) + return false; + } + // old behavior compat: all include lists are empty means check passes + if (!ipset_collection_is_empty(ips)) + { + LIST_FOREACH(item, ips, next) + { + VPRINT("[%s] include ",item->hfile->filename); + if (SearchIpset(&item->hfile->ipset, ipv4, ipv6)) + return true; + } + return false; } return true; } -bool IpsetCheck(struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6) +bool IpsetCheck(const struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6) { - if (!PROFILE_IPSETS_EMPTY(dp)) VPRINT("* ipset check for profile %d\n",dp->n); - return IpsetCheck_(&dp->ips,&dp->ips_exclude,ipv4,ipv6); + if (PROFILE_IPSETS_EMPTY(dp)) return true; + VPRINT("* ipset check for profile %d\n",dp->n); + return IpsetCheck_(&dp->ips_collection,&dp->ips_collection_exclude,ipv4,ipv6); +} + + +static struct ipset_file *RegisterIpset_(struct ipset_files_head *ipsets, struct ipset_collection_head *ips_collection, const char *filename) +{ + struct ipset_file *hfile; + if (!(hfile=ipset_files_search(ipsets, filename))) + if (!(hfile=ipset_files_add(ipsets, filename))) + return NULL; + if (!ipset_collection_search(ips_collection, filename)) + if (!ipset_collection_add(ips_collection, hfile)) + return NULL; + return hfile; +} +struct ipset_file *RegisterIpset(struct desync_profile *dp, bool bExclude, const char *filename) +{ + if (!file_mod_time(filename)) + { + DLOG_ERR("cannot access ipset file '%s'\n",filename); + return NULL; + } + return RegisterIpset_( + ¶ms.ipsets, + bExclude ? &dp->ips_collection_exclude : &dp->ips_collection, + filename); +} + +static const char *dbg_ipset_fill(const ipset *ips) +{ + if (ips->ips4) + if (ips->ips6) + return "ipv4+ipv6"; + else + return "ipv4"; + else + if (ips->ips6) + return "ipv6"; + else + return "empty"; +} +void IpsetsDebug() +{ + if (!params.debug) return; + + struct ipset_file *hfile; + struct desync_profile_list *dpl; + struct ipset_item *ips_item; + + LIST_FOREACH(hfile, ¶ms.ipsets, next) + VPRINT("ipset file %s (%s)\n",hfile->filename,dbg_ipset_fill(&hfile->ipset)); + + LIST_FOREACH(dpl, ¶ms.desync_profiles, next) + { + LIST_FOREACH(ips_item, &dpl->dp.ips_collection, next) + VPRINT("profile %d include ipset %s (%s)\n",dpl->dp.n,ips_item->hfile->filename,dbg_ipset_fill(&ips_item->hfile->ipset)); + LIST_FOREACH(ips_item, &dpl->dp.ips_collection_exclude, next) + VPRINT("profile %d exclude ipset %s (%s)\n",dpl->dp.n,ips_item->hfile->filename,dbg_ipset_fill(&ips_item->hfile->ipset)); + } } diff --git a/tpws/ipset.h b/tpws/ipset.h index ea9d1b8..9852ca6 100644 --- a/tpws/ipset.h +++ b/tpws/ipset.h @@ -5,7 +5,7 @@ #include "params.h" #include "pools.h" -bool LoadIncludeIpsets(); -bool LoadExcludeIpsets(); -bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_addr *ipv6); -bool IpsetCheck(struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6); +bool LoadAllIpsets(); +bool IpsetCheck(const struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6); +struct ipset_file *RegisterIpset(struct desync_profile *dp, bool bExclude, const char *filename); +void IpsetsDebug(); diff --git a/tpws/params.c b/tpws/params.c index 14d4a15..e56236a 100644 --- a/tpws/params.c +++ b/tpws/params.c @@ -145,10 +145,12 @@ struct desync_profile_list *dp_list_add(struct desync_profile_list_head *head) struct desync_profile_list *entry = calloc(1,sizeof(struct desync_profile_list)); if (!entry) return NULL; - LIST_INIT(&entry->dp.hostlist_files); - LIST_INIT(&entry->dp.hostlist_exclude_files); - entry->dp.filter_ipv4 = entry->dp.filter_ipv6 = true; + LIST_INIT(&entry->dp.hl_collection); + LIST_INIT(&entry->dp.hl_collection_exclude); + LIST_INIT(&entry->dp.ips_collection); + LIST_INIT(&entry->dp.ips_collection_exclude); + entry->dp.filter_ipv4 = entry->dp.filter_ipv6 = true; memcpy(entry->dp.hostspell, "host", 4); // default hostspell entry->dp.hostlist_auto_fail_threshold = HOSTLIST_AUTO_FAIL_THRESHOLD_DEFAULT; entry->dp.hostlist_auto_fail_time = HOSTLIST_AUTO_FAIL_TIME_DEFAULT; @@ -167,14 +169,10 @@ struct desync_profile_list *dp_list_add(struct desync_profile_list_head *head) } static void dp_entry_destroy(struct desync_profile_list *entry) { - strlist_destroy(&entry->dp.hostlist_files); - strlist_destroy(&entry->dp.hostlist_exclude_files); - strlist_destroy(&entry->dp.ipset_files); - strlist_destroy(&entry->dp.ipset_exclude_files); - StrPoolDestroy(&entry->dp.hostlist_exclude); - StrPoolDestroy(&entry->dp.hostlist); - ipsetDestroy(&entry->dp.ips); - ipsetDestroy(&entry->dp.ips_exclude); + hostlist_collection_destroy(&entry->dp.hl_collection); + hostlist_collection_destroy(&entry->dp.hl_collection_exclude); + ipset_collection_destroy(&entry->dp.ips_collection); + ipset_collection_destroy(&entry->dp.ips_collection_exclude); HostFailPoolDestroy(&entry->dp.hostlist_auto_fail_counters); free(entry); } @@ -187,11 +185,3 @@ void dp_list_destroy(struct desync_profile_list_head *head) dp_entry_destroy(entry); } } -bool dp_list_have_autohostlist(struct desync_profile_list_head *head) -{ - struct desync_profile_list *dpl; - LIST_FOREACH(dpl, head, next) - if (*dpl->dp.hostlist_auto_filename) - return true; - return false; -} diff --git a/tpws/params.h b/tpws/params.h index 2e9cb93..d3f2cc5 100644 --- a/tpws/params.h +++ b/tpws/params.h @@ -48,22 +48,25 @@ struct desync_profile bool tamper_start_n,tamper_cutoff_n; unsigned int tamper_start,tamper_cutoff; - + bool filter_ipv4,filter_ipv6; port_filter pf_tcp; uint32_t filter_l7; // L7_PROTO_* bits - ipset ips,ips_exclude; - struct str_list_head ipset_files, ipset_exclude_files; - strpool *hostlist, *hostlist_exclude; - struct str_list_head hostlist_files, hostlist_exclude_files; - char hostlist_auto_filename[PATH_MAX]; - int hostlist_auto_fail_threshold, hostlist_auto_fail_time; - time_t hostlist_auto_mod_time; + // list of pointers to ipsets + struct ipset_collection_head ips_collection, ips_collection_exclude; + + // list of pointers to hostlist files + struct hostlist_collection_head hl_collection, hl_collection_exclude; + // pointer to autohostlist. NULL if no autohostlist for the profile. + struct hostlist_file *hostlist_auto; + int hostlist_auto_fail_threshold, hostlist_auto_fail_time, hostlist_auto_retrans_threshold; + hostfail_pool *hostlist_auto_fail_counters; }; -#define PROFILE_IPSETS_EMPTY(dp) (IPSET_EMPTY(&dp->ips) && IPSET_EMPTY(&dp->ips_exclude)) +#define PROFILE_IPSETS_EMPTY(dp) (ipset_collection_is_empty(&dp->ips_collection) && ipset_collection_is_empty(&dp->ips_collection_exclude)) +#define PROFILE_HOSTLISTS_EMPTY(dp) (hostlist_collection_is_empty(&dp->hl_collection) && hostlist_collection_is_empty(&dp->hl_collection_exclude)) struct desync_profile_list { struct desync_profile dp; @@ -72,7 +75,6 @@ struct desync_profile_list { LIST_HEAD(desync_profile_list_head, desync_profile_list); struct desync_profile_list *dp_list_add(struct desync_profile_list_head *head); void dp_list_destroy(struct desync_profile_list_head *head); -bool dp_list_have_autohostlist(struct desync_profile_list_head *head); struct params_s { @@ -112,6 +114,11 @@ struct params_s int ttl_default; char hostlist_auto_debuglog[PATH_MAX]; + // hostlist files with data for all profiles + struct hostlist_files_head hostlists; + // ipset files with data for all profiles + struct ipset_files_head ipsets; + bool tamper; // any tamper option is set bool tamper_lim; // tamper-start or tamper-cutoff set in any profile struct desync_profile_list_head desync_profiles; diff --git a/tpws/pools.c b/tpws/pools.c index 8d10f73..9df09f4 100644 --- a/tpws/pools.c +++ b/tpws/pools.c @@ -52,12 +52,6 @@ bool StrPoolAddStr(strpool **pp, const char *s) { return StrPoolAddStrLen(pp, s, strlen(s)); } -bool StrPoolAddUniqueStr(strpool **pp,const char *s) -{ - if (StrPoolCheckStr(*pp,s)) - return true; - return StrPoolAddStr(pp,s); -} bool StrPoolCheckStr(strpool *p, const char *s) { @@ -160,6 +154,93 @@ void strlist_destroy(struct str_list_head *head) + +struct hostlist_file *hostlist_files_add(struct hostlist_files_head *head, const char *filename) +{ + struct hostlist_file *entry = malloc(sizeof(struct hostlist_file)); + if (entry) + { + if (!(entry->filename = strdup(filename))) + { + free(entry); + return false; + } + entry->mod_time=0; + entry->hostlist = NULL; + LIST_INSERT_HEAD(head, entry, next); + } + return entry; +} +static void hostlist_files_entry_destroy(struct hostlist_file *entry) +{ + if (entry->filename) free(entry->filename); + StrPoolDestroy(&entry->hostlist); + free(entry); +} +void hostlist_files_destroy(struct hostlist_files_head *head) +{ + struct hostlist_file *entry; + while ((entry = LIST_FIRST(head))) + { + LIST_REMOVE(entry, next); + hostlist_files_entry_destroy(entry); + } +} +struct hostlist_file *hostlist_files_search(struct hostlist_files_head *head, const char *filename) +{ + struct hostlist_file *hfile; + + LIST_FOREACH(hfile, head, next) + { + if (!strcmp(hfile->filename,filename)) + return hfile; + } + return NULL; +} + +struct hostlist_item *hostlist_collection_add(struct hostlist_collection_head *head, struct hostlist_file *hfile) +{ + struct hostlist_item *entry = malloc(sizeof(struct hostlist_item)); + if (entry) + { + entry->hfile = hfile; + LIST_INSERT_HEAD(head, entry, next); + } + return entry; +} +void hostlist_collection_destroy(struct hostlist_collection_head *head) +{ + struct hostlist_item *entry; + while ((entry = LIST_FIRST(head))) + { + LIST_REMOVE(entry, next); + free(entry); + } +} +struct hostlist_item *hostlist_collection_search(struct hostlist_collection_head *head, const char *filename) +{ + struct hostlist_item *item; + + LIST_FOREACH(item, head, next) + { + if (!strcmp(item->hfile->filename,filename)) + return item; + } + return NULL; +} +bool hostlist_collection_is_empty(const struct hostlist_collection_head *head) +{ + const struct hostlist_item *item; + + LIST_FOREACH(item, head, next) + { + if (item->hfile->hostlist) + return false; + } + return true; +} + + void ipset4Destroy(ipset4 **ipset) { ipset4 *elem, *tmp; @@ -281,3 +362,89 @@ void ipsetPrint(ipset *ipset) ipset4Print(ipset->ips4); ipset6Print(ipset->ips6); } + + +struct ipset_file *ipset_files_add(struct ipset_files_head *head, const char *filename) +{ + struct ipset_file *entry = malloc(sizeof(struct ipset_file)); + if (entry) + { + if (!(entry->filename = strdup(filename))) + { + free(entry); + return false; + } + entry->mod_time=0; + memset(&entry->ipset,0,sizeof(entry->ipset)); + LIST_INSERT_HEAD(head, entry, next); + } + return entry; +} +static void ipset_files_entry_destroy(struct ipset_file *entry) +{ + if (entry->filename) free(entry->filename); + ipsetDestroy(&entry->ipset); + free(entry); +} +void ipset_files_destroy(struct ipset_files_head *head) +{ + struct ipset_file *entry; + while ((entry = LIST_FIRST(head))) + { + LIST_REMOVE(entry, next); + ipset_files_entry_destroy(entry); + } +} +struct ipset_file *ipset_files_search(struct ipset_files_head *head, const char *filename) +{ + struct ipset_file *hfile; + + LIST_FOREACH(hfile, head, next) + { + if (!strcmp(hfile->filename,filename)) + return hfile; + } + return NULL; +} + +struct ipset_item *ipset_collection_add(struct ipset_collection_head *head, struct ipset_file *hfile) +{ + struct ipset_item *entry = malloc(sizeof(struct ipset_item)); + if (entry) + { + entry->hfile = hfile; + LIST_INSERT_HEAD(head, entry, next); + } + return entry; +} +void ipset_collection_destroy(struct ipset_collection_head *head) +{ + struct ipset_item *entry; + while ((entry = LIST_FIRST(head))) + { + LIST_REMOVE(entry, next); + free(entry); + } +} +struct ipset_item *ipset_collection_search(struct ipset_collection_head *head, const char *filename) +{ + struct ipset_item *item; + + LIST_FOREACH(item, head, next) + { + if (!strcmp(item->hfile->filename,filename)) + return item; + } + return NULL; +} +bool ipset_collection_is_empty(const struct ipset_collection_head *head) +{ + const struct ipset_item *item; + + LIST_FOREACH(item, head, next) + { + if (!IPSET_EMPTY(&item->hfile->ipset)) + return false; + } + return true; +} diff --git a/tpws/pools.h b/tpws/pools.h index d1f1b9f..1fd361c 100644 --- a/tpws/pools.h +++ b/tpws/pools.h @@ -19,7 +19,6 @@ typedef struct strpool { void StrPoolDestroy(strpool **pp); bool StrPoolAddStr(strpool **pp,const char *s); -bool StrPoolAddUniqueStr(strpool **pp,const char *s); bool StrPoolAddStrLen(strpool **pp,const char *s,size_t slen); bool StrPoolCheckStr(strpool *p,const char *s); @@ -29,7 +28,6 @@ struct str_list { }; LIST_HEAD(str_list_head, str_list); - typedef struct hostfail_pool { char *str; /* key */ int counter; /* value */ @@ -49,6 +47,30 @@ bool strlist_add(struct str_list_head *head, const char *filename); void strlist_destroy(struct str_list_head *head); + +struct hostlist_file { + char *filename; + time_t mod_time; + strpool *hostlist; + LIST_ENTRY(hostlist_file) next; +}; +LIST_HEAD(hostlist_files_head, hostlist_file); + +struct hostlist_file *hostlist_files_add(struct hostlist_files_head *head, const char *filename); +void hostlist_files_destroy(struct hostlist_files_head *head); +struct hostlist_file *hostlist_files_search(struct hostlist_files_head *head, const char *filename); + +struct hostlist_item { + struct hostlist_file *hfile; + LIST_ENTRY(hostlist_item) next; +}; +LIST_HEAD(hostlist_collection_head, hostlist_item); +struct hostlist_item *hostlist_collection_add(struct hostlist_collection_head *head, struct hostlist_file *hfile); +void hostlist_collection_destroy(struct hostlist_collection_head *head); +struct hostlist_item *hostlist_collection_search(struct hostlist_collection_head *head, const char *filename); +bool hostlist_collection_is_empty(const struct hostlist_collection_head *head); + + typedef struct ipset4 { struct cidr4 cidr; /* key */ UT_hash_handle hh; /* makes this structure hashable */ @@ -85,3 +107,26 @@ void ipset6Print(ipset6 *ipset); void ipsetDestroy(ipset *ipset); void ipsetPrint(ipset *ipset); + + +struct ipset_file { + char *filename; + time_t mod_time; + ipset ipset; + LIST_ENTRY(ipset_file) next; +}; +LIST_HEAD(ipset_files_head, ipset_file); + +struct ipset_file *ipset_files_add(struct ipset_files_head *head, const char *filename); +void ipset_files_destroy(struct ipset_files_head *head); +struct ipset_file *ipset_files_search(struct ipset_files_head *head, const char *filename); + +struct ipset_item { + struct ipset_file *hfile; + LIST_ENTRY(ipset_item) next; +}; +LIST_HEAD(ipset_collection_head, ipset_item); +struct ipset_item * ipset_collection_add(struct ipset_collection_head *head, struct ipset_file *hfile); +void ipset_collection_destroy(struct ipset_collection_head *head); +struct ipset_item *ipset_collection_search(struct ipset_collection_head *head, const char *filename); +bool ipset_collection_is_empty(const struct ipset_collection_head *head); diff --git a/tpws/tamper.c b/tpws/tamper.c index 41f9626..4c8b3b7 100644 --- a/tpws/tamper.c +++ b/tpws/tamper.c @@ -24,40 +24,36 @@ static bool l7_proto_match(t_l7proto l7proto, uint32_t filter_l7) (l7proto==TLS && (filter_l7 & L7_PROTO_TLS)); } -static bool dp_match_l3l4(struct desync_profile *dp, const struct sockaddr *dest) -{ - return ((dest->sa_family==AF_INET && dp->filter_ipv4) || (dest->sa_family==AF_INET6 && dp->filter_ipv6)) && - pf_in_range(saport(dest), &dp->pf_tcp) && - IpsetCheck(dp, dest->sa_family==AF_INET ? &((struct sockaddr_in*)dest)->sin_addr : NULL, dest->sa_family==AF_INET6 ? &((struct sockaddr_in6*)dest)->sin6_addr : NULL); -} -static bool dp_impossible(struct desync_profile *dp, const char *hostname, t_l7proto l7proto) -{ - return !PROFILE_IPSETS_EMPTY(dp) && - ((dp->filter_l7 && !l7_proto_match(l7proto, dp->filter_l7)) || (!*dp->hostlist_auto_filename && !hostname && (dp->hostlist || dp->hostlist_exclude))); -} static bool dp_match(struct desync_profile *dp, const struct sockaddr *dest, const char *hostname, t_l7proto l7proto) { - // impossible case, hard filter - // impossible check avoids relatively slow ipset search - if (!dp_impossible(dp,hostname,l7proto) && dp_match_l3l4(dp,dest)) - { - // soft filter - if (dp->filter_l7 && !l7_proto_match(l7proto, dp->filter_l7)) - return false; + if (!HostlistsReloadCheckForProfile(dp)) return false; - // autohostlist profile matching l3/l4 filter always win - if (*dp->hostlist_auto_filename) return true; + if ((dest->sa_family==AF_INET && !dp->filter_ipv4) || (dest->sa_family==AF_INET6 && !dp->filter_ipv6)) + // L3 filter does not match + return false; + if (!pf_in_range(saport(dest), &dp->pf_tcp)) + // L4 filter does not match + return false; + if (dp->filter_l7 && !l7_proto_match(l7proto, dp->filter_l7)) + // L7 filter does not match + return false; + if (!dp->hostlist_auto && !hostname && !PROFILE_HOSTLISTS_EMPTY(dp)) + // avoid cpu consuming ipset check. profile cannot win if regular hostlists are present without auto hostlist and hostname is unknown. + return false; + if (!IpsetCheck(dp, dest->sa_family==AF_INET ? &((struct sockaddr_in*)dest)->sin_addr : NULL, dest->sa_family==AF_INET6 ? &((struct sockaddr_in6*)dest)->sin6_addr : NULL)) + // target ip does not match + return false; + + // autohostlist profile matching l3/l4 filter always win + if (dp->hostlist_auto) return true; + + if (PROFILE_HOSTLISTS_EMPTY(dp)) + // profile without hostlist filter wins + return true; + else if (hostname) + // without known hostname first profile matching l3/l4 filter and without hostlist filter wins + return HostlistCheck(dp, hostname, NULL, true); - if (dp->hostlist || dp->hostlist_exclude) - { - // without known hostname first profile matching l3/l4 filter and without hostlist filter wins - if (hostname) - return HostlistCheck(dp, hostname, NULL); - } - else - // profile without hostlist filter wins - return true; - } return false; } static struct desync_profile *dp_find(struct desync_profile_list_head *head, const struct sockaddr *dest, const char *hostname, t_l7proto l7proto) @@ -181,10 +177,10 @@ void tamper_out(t_ctrack *ctrack, const struct sockaddr *dest, uint8_t *segment, VPRINT("desync profile changed by revealed l7 protocol or hostname !\n"); } - if (bDiscoveredHostname && *ctrack->dp->hostlist_auto_filename) + if (bDiscoveredHostname && ctrack->dp->hostlist_auto) { bool bHostExcluded; - if (!HostlistCheck(ctrack->dp, Host, &bHostExcluded)) + if (!HostlistCheck(ctrack->dp, Host, &bHostExcluded, false)) { ctrack->b_ah_check = !bHostExcluded; VPRINT("Not acting on this request\n"); @@ -415,21 +411,21 @@ static void auto_hostlist_failed(struct desync_profile *dp, const char *hostname VPRINT("auto hostlist (profile %d) : rechecking %s to avoid duplicates\n", dp->n, hostname); bool bExcluded=false; - if (!HostlistCheck(dp, hostname, &bExcluded) && !bExcluded) + if (!HostlistCheck(dp, hostname, &bExcluded, false) && !bExcluded) { - VPRINT("auto hostlist (profile %d) : adding %s to %s\n", dp->n, hostname, dp->hostlist_auto_filename); - HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : adding to %s", hostname, dp->n, client_ip_port, l7proto_str(l7proto), dp->hostlist_auto_filename); - if (!StrPoolAddStr(&dp->hostlist, hostname)) + VPRINT("auto hostlist (profile %d) : adding %s to %s\n", dp->n, hostname, dp->hostlist_auto->filename); + HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : adding to %s", hostname, dp->n, client_ip_port, l7proto_str(l7proto), dp->hostlist_auto->filename); + if (!StrPoolAddStr(&dp->hostlist_auto->hostlist, hostname)) { DLOG_ERR("StrPoolAddStr out of memory\n"); return; } - if (!append_to_list_file(dp->hostlist_auto_filename, hostname)) + if (!append_to_list_file(dp->hostlist_auto->filename, hostname)) { DLOG_PERROR("write to auto hostlist:"); return; } - dp->hostlist_auto_mod_time = file_mod_time(dp->hostlist_auto_filename); + dp->hostlist_auto->mod_time = file_mod_time(dp->hostlist_auto->filename); } else { diff --git a/tpws/tpws.c b/tpws/tpws.c index a9a1cc3..bebf320 100644 --- a/tpws/tpws.c +++ b/tpws/tpws.c @@ -43,25 +43,9 @@ struct params_s params; -bool bHup = false; static void onhup(int sig) { printf("HUP received !\n"); - printf("Will reload hostlists and ipsets on next request (if any)\n"); - bHup = true; -} -// should be called in normal execution -void dohup(void) -{ - if (bHup) - { - if (!LoadIncludeHostLists() || !LoadExcludeHostLists() || !LoadIncludeIpsets() || !LoadExcludeIpsets()) - { - // what will we do without hostlist or ipset ?? sure, gonna die - exit(1); - } - bHup = false; - } } static void onusr2(int sig) @@ -230,6 +214,9 @@ static void exithelp(void) static void cleanup_params(void) { dp_list_destroy(¶ms.desync_profiles); + + hostlist_files_destroy(¶ms.hostlists); + ipset_files_destroy(¶ms.ipsets); } static void exithelp_clean(void) { @@ -367,6 +354,10 @@ void parse_params(int argc, char *argv[]) #if defined(__OpenBSD__) || defined(__APPLE__) params.pf_enable = true; // OpenBSD and MacOS have no other choice #endif + + LIST_INIT(¶ms.hostlists); + LIST_INIT(¶ms.ipsets); + if (can_drop_root()) { params.uid = params.gid = 0x7FFFFFFF; // default uid:gid @@ -726,29 +717,29 @@ void parse_params(int argc, char *argv[]) params.tamper = true; break; case 36: /* hostlist */ - if (!strlist_add(&dp->hostlist_files, optarg)) + if (!RegisterHostlist(dp, false, optarg)) { - DLOG_ERR("strlist_add failed\n"); + DLOG_ERR("failed to register hostlist '%s'\n", optarg); exit_clean(1); } params.tamper = true; break; case 37: /* hostlist-exclude */ - if (!strlist_add(&dp->hostlist_exclude_files, optarg)) + if (!RegisterHostlist(dp, true, optarg)) { - DLOG_ERR("strlist_add failed\n"); + DLOG_ERR("failed to register hostlist '%s'\n", optarg); exit_clean(1); } params.tamper = true; break; case 38: /* hostlist-auto */ - if (*dp->hostlist_auto_filename) + if (dp->hostlist_auto) { DLOG_ERR("only one auto hostlist per profile is supported\n"); exit_clean(1); } { - FILE *F = fopen(optarg,"a+t"); + FILE *F = fopen(optarg,"a+b"); if (!F) { DLOG_ERR("cannot create %s\n", optarg); @@ -764,13 +755,11 @@ void parse_params(int argc, char *argv[]) if (params.droproot && chown(optarg, params.uid, -1)) DLOG_ERR("could not chown %s. auto hostlist file may not be writable after privilege drop\n", optarg); } - if (!strlist_add(&dp->hostlist_files, optarg)) + if (!(dp->hostlist_auto=RegisterHostlist(dp, false, optarg))) { - DLOG_ERR("strlist_add failed\n"); + DLOG_ERR("failed to register hostlist '%s'\n", optarg); exit_clean(1); } - strncpy(dp->hostlist_auto_filename, optarg, sizeof(dp->hostlist_auto_filename)); - dp->hostlist_auto_filename[sizeof(dp->hostlist_auto_filename) - 1] = '\0'; params.tamper = true; // need to detect blocks and update autohostlist. cannot just slice. break; case 39: /* hostlist-auto-fail-threshold */ @@ -979,17 +968,17 @@ void parse_params(int argc, char *argv[]) } break; case 60: /* ipset */ - if (!strlist_add(&dp->ipset_files, optarg)) + if (!RegisterIpset(dp, false, optarg)) { - DLOG_ERR("strlist_add failed\n"); + DLOG_ERR("failed to register ipset '%s'\n", optarg); exit_clean(1); } params.tamper = true; break; case 61: /* ipset-exclude */ - if (!strlist_add(&dp->ipset_exclude_files, optarg)) + if (!RegisterIpset(dp, true, optarg)) { - DLOG_ERR("strlist_add failed\n"); + DLOG_ERR("failed to register ipset '%s'\n", optarg); exit_clean(1); } params.tamper = true; @@ -1062,7 +1051,6 @@ void parse_params(int argc, char *argv[]) dp = &dpl->dp; if (dp->split_tls==tlspos_none && dp->split_pos) dp->split_tls=tlspos_pos; if (dp->split_http_req==httpreqpos_none && dp->split_pos) dp->split_http_req=httpreqpos_pos; - if (*dp->hostlist_auto_filename) dp->hostlist_auto_mod_time = file_mod_time(dp->hostlist_auto_filename); if (params.skip_nodelay && (dp->split_tls || dp->split_http_req || dp->split_pos)) { DLOG_ERR("Cannot split with --skip-nodelay\n"); @@ -1070,26 +1058,21 @@ void parse_params(int argc, char *argv[]) } } - if (!LoadIncludeHostLists()) + if (!LoadAllHostLists()) { - DLOG_ERR("Include hostlist load failed\n"); + DLOG_ERR("hostlists load failed\n"); exit_clean(1); } - if (!LoadExcludeHostLists()) + if (!LoadAllIpsets()) { - DLOG_ERR("Exclude hostlist load failed\n"); - exit_clean(1); - } - if (!LoadIncludeIpsets()) - { - DLOG_ERR("Include ipset load failed\n"); - exit_clean(1); - } - if (!LoadExcludeIpsets()) - { - DLOG_ERR("Exclude ipset load failed\n"); + DLOG_ERR("ipset load failed\n"); exit_clean(1); } + + VPRINT("\nlists summary:\n"); + HostlistsDebug(); + IpsetsDebug(); + VPRINT("\n"); } diff --git a/tpws/tpws_conn.c b/tpws/tpws_conn.c index 66cbd21..c1d335e 100644 --- a/tpws/tpws_conn.c +++ b/tpws/tpws_conn.c @@ -1469,8 +1469,6 @@ int event_loop(const int *listen_fd, size_t listen_fd_ct) break; } - dohup(); - for (i = 0; i < num_events; i++) { conn = (tproxy_conn_t*)events[i].data.ptr;