From 0e6016f429530b4fed0ad17d97cad1a5bb128ef9 Mon Sep 17 00:00:00 2001 From: Landon Curt Noll Date: Fri, 6 Oct 2023 21:59:06 -0700 Subject: [PATCH] improve Address Sanitizer (ASAN) support and chk_tree Updated BUGS about MSYS2 on Windows compiling of calc. Added more git related checks and sanity checks to chk_tree. Added ${FSANITIZE} make variable to Makefile.config to hold common Address Sanitizer (ASAN) optins to modern Linux and macOS. The Address Sanitizer is NOT enabled not compiled in by default. Improved comments in Makefile.local for RHEL9.2 (Linux) and for macOS 14.0 that, when uncommented and calc is recompiled (i.e., make clobber all) will enable the Address Sanitizer (ASAN) for calc. --- CHANGES | 14 ++++++ Makefile.config | 31 +++++++++++++ Makefile.local | 116 +++++++++++++----------------------------------- chk_tree | 52 +++++++++++++++++----- trailblank | 34 ++++++++++++-- 5 files changed, 149 insertions(+), 98 deletions(-) diff --git a/CHANGES b/CHANGES index d6ae51a..3aec5c3 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,17 @@ +The following are the changes from calc version 2.15.0.2 to date: + + Updated BUGS about MSYS2 on Windows compiling of calc. + + Added more git related checks and sanity checks to chk_tree. + + Added ${FSANITIZE} make variable to Makefile.config to hold + common Address Sanitizer (ASAN) optins to modern Linux and macOS. + The Address Sanitizer is NOT enabled not compiled in by default. + Improved comments in Makefile.local for RHEL9.2 (Linux) and for + macOS 14.0 that, when uncommented and calc is recompiled (i.e., + make clobber all) will enable the Address Sanitizer (ASAN) for calc. + + The following are the changes from calc version 2.14.3.5 to 2.15.0.1: The tarball for calc version 2.15.0.0 was missing version.h. diff --git a/Makefile.config b/Makefile.config index 15439e2..d8fb070 100644 --- a/Makefile.config +++ b/Makefile.config @@ -1382,3 +1382,34 @@ endif # ($(ALLOW_CUSTOM),-DCUSTOM) # intermediate and final calc and calc related programs # COMMON_LDFLAGS= ${EXTRA_LDFLAGS} + +# Common Address Sanitizer (ASAN) +# +# For more info see: https://github.com/google/sanitizers/wiki/AddressSanitizer +# See also: https://developer.apple.com/documentation/xcode/diagnosing-memory-thread-and-crash-issues-early +# +# The following Address Sanitizer (ASAN) are common to both REHL9.2 (Linux) and macOS 14.0. +# +# By default, the Address Sanitizer is NOT enabled, not compiled into calc. +# To enable the Address Sanitizer, uncomment the appropriate lines in Makefile.local !!! +# +FSANITIZE:= -Wno-invalid-command-line-argument +FSANITIZE+= -fsanitize=address +FSANITIZE+= -fsanitize=alignment +FSANITIZE+= -fsanitize=bool +FSANITIZE+= -fsanitize=enum +FSANITIZE+= -fsanitize=vptr +FSANITIZE+= -fsanitize=integer-divide-by-zero +FSANITIZE+= -fsanitize=float-divide-by-zero +FSANITIZE+= -fsanitize=float-cast-overflow +FSANITIZE+= -fsanitize=nonnull-attribute +FSANITIZE+= -fsanitize=returns-nonnull-attribute +FSANITIZE+= -fsanitize=null +FSANITIZE+= -fsanitize=object-size +FSANITIZE+= -fsanitize=shift +FSANITIZE+= -fsanitize=signed-integer-overflow +FSANITIZE+= -fsanitize=undefined +FSANITIZE+= -fsanitize=unreachable +FSANITIZE+= -fsanitize=vla-bound +FSANITIZE+= -fno-omit-frame-pointer +FSANITIZE+= -fno-common diff --git a/Makefile.local b/Makefile.local index 188bae0..8242c04 100644 --- a/Makefile.local +++ b/Makefile.local @@ -50,103 +50,51 @@ ################################################################### #### -# # Force calc to install under /usr/local # -# BINDIR:= /usr/local/bin -# LIBDIR:= /usr/local/lib -# CALC_SHAREDIR:= /usr/local/share/calc -# CALC_INCDIR:= /usr/local/include/calc -# +# PREFIX:= /usr/local +# BINDIR:= ${PREFIX}/bin +# LIBDIR:= ${PREFIX}/lib +# CALC_SHAREDIR:= ${PREFIX}/share/calc +# CALC_INCDIR:= ${PREFIX}/include/calc #### #### -# RHEL Diagnosing memory, thread, and crash issues: -# -# For more info see: https://github.com/google/sanitizers/wiki/AddressSanitizer +# macOS Address Sanitizer (ASAN) # # This comment block was tested under: # -# RHEL9 gcc version 11.3.1 20221121 (Red Hat 11.3.1-4) (GCC) +# macOS 14.0 with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64 # -# with: +# See the FSANITIZE comment block in Makefile.config for common FSANITIZE values and more info. # -# libasan-11.3.1-4.3.el9.x86_64 libubsan-11.3.1-4.3.el9.x86_64 +# To use the Address Sanitizer, uncomment this set set of lines and recompile (make clobber all): # -# NOTE: With the above version, these are NOT supported: -# -# UNSUPPORTED_FSANITIZE:= -fsanitize=nullability-arg -fsanitize=nullability-assign -# -# Uncomment these lines: -# -# FSANITIZE:= -fsanitize=address -# FSANITIZE:= -fsanitize=bool -# FSANITIZE:= -fsanitize=bounds -# FSANITIZE+= -fsanitize=enum -# FSANITIZE+= -fsanitize=float-cast-overflow -# FSANITIZE+= -fsanitize=float-divide-by-zero -# FSANITIZE+= -fsanitize=integer-divide-by-zero -# FSANITIZE+= -fsanitize=nonnull-attribute -# FSANITIZE+= -fsanitize=null -# FSANITIZE+= -fsanitize=returns-nonnull-attribute -# FSANITIZE+= -fsanitize=shift -# FSANITIZE+= -fsanitize=signed-integer-overflow -# FSANITIZE:= -fsanitize=undefined -# FSANITIZE+= -fsanitize=unreachable -# FSANITIZE+= -fsanitize=vla-bound -# FSANITIZE+= -fsanitize=vptr -# CFLAGS+= -Wno-invalid-command-line-argument ${FSANITIZE} -fno-omit-frame-pointer -# LDFLAGS+= -Wno-invalid-command-line-argument ${FSANITIZE} -fno-omit-frame-pointer -# CALC_ENV+= ASAN_OPTIONS=detect_stack_use_after_return=1 -# DEBUG:= -O0 -g -#### - -#### -# macOS Diagnosing memory, thread, and crash issues: -# -# For more info see: https://github.com/google/sanitizers/wiki/AddressSanitizer -# -# This comment block was tested under: -# -# macOS 14.0 with clang version 15.0.7 (Red Hat 15.0.7-2.el9) -# -# For more info for clang to Diagnosing memory, thread, and crash issues early, see: -# -# https://developer.apple.com/documentation/xcode/diagnosing-memory-thread-and-crash-issues-early -# -# Uncomment these lines: -# -# These values are presented in the order listed for the above mentioned URL: -# -# FSANITIZE:= -fsanitize=alignment -# FSANITIZE:= -fsanitize=bool -# FSANITIZE:= -fsanitize=bounds -# FSANITIZE+= -fsanitize=enum -# FSANITIZE+= -fsanitize=vptr -# FSANITIZE+= -fsanitize=integer-divide-by-zero -# FSANITIZE+= -fsanitize=float-divide-by-zero -# FSANITIZE+= -fsanitize=float-cast-overflow -# FSANITIZE+= -fsanitize=nonnull-attribute # FSANITIZE+= -fsanitize=nullability-arg # FSANITIZE+= -fsanitize=nullability-assign -# FSANITIZE+= -fsanitize=returns-nonnull-attribute # FSANITIZE+= -fsanitize=nullability-return -# FSANITIZE+= -fsanitize=null -# FSANITIZE+= -fsanitize=object-size -# FSANITIZE+= -fsanitize=shift -# FSANITIZE+= -fsanitize=signed-integer-overflow -# FSANITIZE:= -fsanitize=undefined -# FSANITIZE+= -fsanitize=unreachable -# FSANITIZE+= -fsanitize=vla-bound -# -# We also observe these options, while NOT order listed the above mentioned URL, work: -# -# FSANITIZE:= -fsanitize=address -# -# We combine the above FSANITIZE with these compile and link options: -# -# CFLAGS+= -Wno-invalid-command-line-argument ${FSANITIZE} -fno-omit-frame-pointer -# LDFLAGS+= -Wno-invalid-command-line-argument ${FSANITIZE} -fno-omit-frame-pointer -# CALC_ENV+= ASAN_OPTIONS=detect_stack_use_after_return=1 +# CFLAGS+= ${FSANITIZE} +# LDFLAGS+= ${FSANITIZE} # DEBUG:= -O0 -g3 #### + +#### +# RHEL (Linux) Address Sanitizer (ASAN) +# +# This comment block was tested under: +# +# RHEL9.2 with clang version 15.0.7 (Red Hat 15.0.7-2.el9) for x86_64 +# +# with these RPMs installed: +# +# libasan-11.3.1-4.3.el9.x86_64 libubsan-11.3.1-4.3.el9.x86_64 +# +# See the FSANITIZE comment block in Makefile.config for common FSANITIZE values and more info. +# +# To use the Address Sanitizer, uncomment this set set of lines and recompile (make clobber all): +# +# FSANITIZE+= -fsanitize=bounds +# CFLAGS+= ${FSANITIZE} +# LDFLAGS+= ${FSANITIZE} +# DEBUG:= -O0 -g3 +### diff --git a/chk_tree b/chk_tree index bc7cd04..1888678 100755 --- a/chk_tree +++ b/chk_tree @@ -46,6 +46,11 @@ fi # declare -a DISTDIR mapfile -t DISTDIR < <(make -s distdir | grep -v '^make\[[0-9]*\]: ' | sort -u) +if [[ ${#DISTDIR[@]} -le 0 ]]; then + echo "$0: ERROR: distdir is empty" 1>&2 + EXIT_CODE=11 + echo "$0: Warning: set EXIT_CODE: $EXIT_CODE" 1>&2 +fi # collect directories # @@ -58,6 +63,11 @@ mapfile -t FINDDIR < <(find . -type d \ ! -path './.git/*' ! -name .git \ ! -path './.github/*' ! -name .github | \ sed -e 's/^\.\///' | sort -u) +if [[ ${#FINDDIR[@]} -le 0 ]]; then + echo "$0: ERROR: find dir is empty" 1>&2 + EXIT_CODE=12 + echo "$0: Warning: set EXIT_CODE: $EXIT_CODE" 1>&2 +fi # compare DISTDIR and FINDDIR # @@ -81,7 +91,7 @@ if [[ ${#DIFF_DISTDIR_FINDDIR[@]} -ne 0 ]]; then if [[ ${#ONLY_DISTDIR[@]} -ne 0 ]]; then echo "$0: ERROR: found only in distdir: ${ONLY_DISTDIR[*]}" 1>&2 fi - EXIT_CODE=11 + EXIT_CODE=13 echo "$0: Warning: set EXIT_CODE: $EXIT_CODE" 1>&2 fi @@ -91,7 +101,7 @@ make distlist >/dev/null 2>&1 status="$?" if [[ $status -ne 0 ]]; then echo "$0: ERROR: make distlist exit code: $status" 1>&2 - EXIT_CODE=12 + EXIT_CODE=14 echo "$0: Warning: set EXIT_CODE: $EXIT_CODE" 1>&2 fi @@ -99,6 +109,11 @@ fi # declare -a DISTLIST mapfile -t DISTLIST < <(make -s distlist | grep -v '^make\[[0-9]*\]: ' | sort -u) +if [[ ${#DISTLIST[@]} -le 0 ]]; then + echo "$0: ERROR: distlist is empty" 1>&2 + EXIT_CODE=15 + echo "$0: Warning: set EXIT_CODE: $EXIT_CODE" 1>&2 +fi declare -A DISTLIST_A for i in "${DISTLIST[@]}"; do DISTLIST_A["$i"]="$i" @@ -116,6 +131,11 @@ if [[ -d .git ]]; then mapfile -t GITLS < <(git ls | grep -v -E '^\.github/|^\.gitignore$|^CODE_OF_CONDUCT\.md$|^CONTRIBUTING\.md$|^SECURITY\.md$' | sort -u) + if [[ ${#GITLS[@]} -le 0 ]]; then + echo "$0: ERROR: git ls is empty" 1>&2 + EXIT_CODE=16 + echo "$0: Warning: set EXIT_CODE: $EXIT_CODE" 1>&2 + fi # compare DISTLIST and GITLS # @@ -139,7 +159,7 @@ if [[ -d .git ]]; then if [[ ${#ONLY_DISTLIST[@]} -ne 0 ]]; then echo "$0: ERROR: found only in distlist: ${ONLY_DISTLIST[*]}" 1>&2 fi - EXIT_CODE=13 + EXIT_CODE=17 echo "$0: Warning: set EXIT_CODE: $EXIT_CODE" 1>&2 fi @@ -149,7 +169,7 @@ if [[ -d .git ]]; then if [[ -n $GIT_STATUS ]] || ! git diff --cached --quiet --exit-code; then echo "$0: ERROR: there are staged uncommitted changes" 1>&2 git status --short - EXIT_CODE=14 + EXIT_CODE=18 echo "$0: Warning: set EXIT_CODE: $EXIT_CODE" 1>&2 fi @@ -158,7 +178,7 @@ if [[ -d .git ]]; then if ! git diff --quiet --exit-code; then echo "$0: ERROR: there are unstaged changes" 1>&2 git status --untracked-files=no --porcelain --short - EXIT_CODE=15 + EXIT_CODE=19 echo "$0: Warning: set EXIT_CODE: $EXIT_CODE" 1>&2 fi @@ -169,12 +189,12 @@ if [[ -d .git ]]; then if [[ $GIT_MASTER -gt $GIT_ORIGIN_MASTER ]]; then echo "$0: ERROR: master branch is behind of origin/master" 1>&2 git status master - EXIT_CODE=16 + EXIT_CODE=20 echo "$0: Warning: set EXIT_CODE: $EXIT_CODE" 1>&2 elif [[ $GIT_MASTER -lt $GIT_ORIGIN_MASTER ]]; then echo "$0: ERROR: master branch is ahead of origin/master" 1>&2 git status master - EXIT_CODE=17 + EXIT_CODE=21 echo "$0: Warning: set EXIT_CODE: $EXIT_CODE" 1>&2 fi @@ -185,12 +205,12 @@ if [[ -d .git ]]; then if [[ $GIT_HEAD -gt $GIT_ORIGIN_HEAD ]]; then echo "$0: ERROR: HEAD is behind of origin/HEAD" 1>&2 git status HEAD - EXIT_CODE=18 + EXIT_CODE=22 echo "$0: Warning: set EXIT_CODE: $EXIT_CODE" 1>&2 elif [[ $GIT_HEAD -lt $GIT_ORIGIN_HEAD ]]; then echo "$0: ERROR: HEAD is ahead of origin/HEAD" 1>&2 git status HEAD - EXIT_CODE=19 + EXIT_CODE=23 echo "$0: Warning: set EXIT_CODE: $EXIT_CODE" 1>&2 fi fi @@ -199,6 +219,11 @@ fi # declare -a BUILDLIST mapfile -t BUILDLIST < <(make -s buildlist | grep -v '^make\[[0-9]*\]: ' | sort -u) +if [[ ${#BUILDLIST[@]} -le 0 ]]; then + echo "$0: ERROR: buildlist is empty" 1>&2 + EXIT_CODE=24 + echo "$0: Warning: set EXIT_CODE: $EXIT_CODE" 1>&2 +fi declare -A BUILDLIST_A for i in "${BUILDLIST[@]}"; do BUILDLIST_A["$i"]="$i" @@ -224,7 +249,7 @@ if [[ ${#DISTLIST_ALSO_IN_BUILDLIST[@]} -ne 0 ]]; then # echo "$0: ERROR: distlist files found in buildlist" 1>&2 echo "$0: ERROR: distlist files found in buildlist: ${DISTLIST_ALSO_IN_BUILDLIST[*]}" 1>&2 - EXIT_CODE=20 + EXIT_CODE=25 echo "$0: Warning: set EXIT_CODE: $EXIT_CODE" 1>&2 fi @@ -244,6 +269,11 @@ mapfile -t FINDFILE < <(find . -type f \ ! -name 'CODE_OF_CONDUCT.md' ! -name 'CONTRIBUTING.md' ! -name '.gitignore' ! -name 'SECURITY.md' \ ! -name '*.swp' ! -name '*.out' ! -name '?' | sed -e 's/^\.\///' | sort -u) +if [[ ${#FINDFILE[@]} -le 0 ]]; then + echo "$0: ERROR: find file is empty" 1>&2 + EXIT_CODE=26 + echo "$0: Warning: set EXIT_CODE: $EXIT_CODE" 1>&2 +fi # look for something in FINDFILE that in neither DISTLIST nor BUILDLIST # @@ -282,7 +312,7 @@ if [[ ${#UNKNOWN_FILE[@]} -ne 0 ]]; then # echo "$0: ERROR: files that are neither built nor distlist are found" 1>&2 echo "$0: ERROR: distlist files found in buildlist: ${UNKNOWN_FILE[*]}" 1>&2 - EXIT_CODE=21 + EXIT_CODE=27 echo "$0: Warning: set EXIT_CODE: $EXIT_CODE" 1>&2 fi diff --git a/trailblank b/trailblank index fc1b539..61fe68d 100755 --- a/trailblank +++ b/trailblank @@ -38,6 +38,14 @@ export EXIT_CODE=0 # We exclude binary files, RCS source code history, intermediate # compiled files, patch droppings and compiled binary libraries. # +# We also ignore filenames with a single letter, and files +# that end in .out, and files containing the any of the following +# in any letter case: +# +# foo bar baz curds whey rmme +# +# Such files are temporary/test files and are not part of calc. +# # Last, we have two files that have long lines that, for now, # we cannot be as picky about without significant work. # @@ -53,7 +61,9 @@ LEADING_SPACES_BEFORE_TAB=$( -path './longbits' -o -name '.*.swp' -o -name 'conf.h' -o \ -name '.git' -o -path './custom/libcustcalc*' -o -path './libcustcalc*' -o \ -name 'sample_many-static' -o -name 'sample_rand-static' -o \ - -name 'codeql-analysis.yml' -o -name tags -o -name '*.out' \ + -name 'codeql-analysis.yml' -o -name tags -o -name '*.out' -o \ + -name '?' -o -iname '*foo*' -o -iname '*bar*' -o -iname '*baz*' -o \ + -iname '*curds*' -o -iname '*whey*' -o -iname '*rmme*' \ \) -prune -o -type f -print0 | \ xargs -0 egrep -l '^ * * ' ) @@ -69,6 +79,14 @@ fi # We exclude binary files, RCS source code history, intermediate # compiled files, patch droppings and compiled binary libraries. # +# We also ignore filenames with a single letter, and files +# that end in .out, and files containing the any of the following +# in any letter case: +# +# foo bar baz curds whey rmme +# +# Such files are temporary/test files and are not part of calc. +# # Last, we have two files that have long lines that, for now, # we cannot be as picky about without significant work. # @@ -84,7 +102,8 @@ TRAILING_WHITESPACE=$( -path './longbits' -o -name '.*.swp' -o -name 'conf.h' -o \ -name '.git' -o -path './custom/libcustcalc*' -o -path './libcustcalc*' -o \ -name 'sample_many-static' -o -name 'sample_rand-static' -o \ - -name 'codeql-analysis.yml' -o -name tags -o -name '*.out' \ + -name '?' -o -iname '*foo*' -o -iname '*bar*' -o -iname '*baz*' -o \ + -iname '*curds*' -o -iname '*whey*' -o -iname '*rmme*' \ \) -prune -o -type f -print0 | \ xargs -0 egrep -l '[ ]$' ) @@ -139,6 +158,14 @@ fi # We exclude binary files, source code history, intermediate # compiled files, patch droppings and compiled binary libraries. # +# We also ignore filenames with a single letter, and files +# that end in .out, and files containing the any of the following +# in any letter case: +# +# foo bar baz curds whey rmme +# +# Such files are temporary/test files and are not part of calc. +# # Last, we have files that have long lines that, for now, # we cannot be as picky about without significant work. # @@ -155,7 +182,8 @@ PICKY_PHASE_1=$( -name '.git' -o -path './custom/libcustcalc*' -o -path './libcustcalc*' -o \ -name 'sample_many-static' -o -name 'sample_rand-static' -o \ -name 'codeql-analysis.yml' -o -name tags -o -name '*.out' -o \ - -name '.gitignore' -o -name 'README.md' -o -name '.lldbinit' \ + -name '?' -o -iname '*foo*' -o -iname '*bar*' -o -iname '*baz*' -o \ + -iname '*curds*' -o -iname '*whey*' -o -iname '*rmme*' \ \) -prune -o -type f -print0 | \ if [[ -x /usr/local/bin/picky ]]; then xargs -0 /usr/local/bin/picky -s -v -w132