Build system tweaks; other small tweaks & cleanups

Main changes:

libast: Reduce the dependencies on ast_release.h (which defines
_AST_release for release builds, disabling debugging code). No
longer auto-include this from ast.h. Files that need it should
include it explicitly and declare this dependency in the Mamfile.

bin/package: Rework the way of dealing with LD_LIBRARY_PATH and
friends. On init, ensure $INSTALLROOT-based directories are *not*
in there (potentially inherited from a 'bin/package use'
environmetn) as this may break the build. When doing 'test' and/or
'use', re-add these as appropriate.

bin/package

@@ -131,7 +131,7 @@ command=${0##*/}
 case $(getopts '[-][123:xyz]' opt --xyz 2>/dev/null; echo 0$opt) in
 0123)	USAGE=$'
 [-?
-@(#)$Id: '$command$' (ksh 93u+m) 2024-12-07 $
+@(#)$Id: '$command$' (ksh 93u+m) 2024-12-16 $
 ]
 [-author?Glenn Fowler <[email protected]>]
 [-author?Contributors to https://github.com/ksh93/ksh]
@@ -564,7 +564,7 @@ SEE ALSO
   pkgadd(1), pkgmk(1), rpm(1), sh(1), tar(1), optget(3)
 
 IMPLEMENTATION
-  version         package (ksh 93u+m) 2024-12-07
+  version         package (ksh 93u+m) 2024-12-16
   author          Glenn Fowler <[email protected]>
   author          Contributors to https://github.com/ksh93/ksh
   copyright       (c) 1994-2012 AT&T Intellectual Property
@@ -1794,6 +1794,43 @@ export DEFPATH  # for iffe, etc.
 
 PATH=$(sanitize_PATH "/opt/ast/bin:$DEFPATH:$PATH")
 
+# remove $INSTALLROOT/... from these; may be harmful while building
+# (these comes pre-set when running package from a 'package use' environment)
+case $LD_LIBRARY_PATH in
+"$INSTALLROOT"/dyn/lib)
+	unset LD_LIBRARY_PATH ;;
+"$INSTALLROOT"/dyn/lib:*)
+	LD_LIBRARY_PATH=${LD_LIBRARY_PATH#*:} ;;
+esac
+case $LIBPATH in
+"$INSTALLROOT"/dyn/bin:"$INSTALLROOT"/dyn/lib:*)
+	LIBPATH=${LIBPATH#*:*:} ;;
+esac
+case $SHLIB_PATH in
+"$INSTALLROOT"/dyn/lib)
+	unset SHLIB_PATH ;;
+"$INSTALLROOT"/dyn/lib:*)
+	SHLIB_PATH=${SHLIB_PATH#*:} ;;
+esac
+case $DYLD_LIBRARY_PATH in
+"$INSTALLROOT"/dyn/lib)
+	unset DYLD_LIBRARY_PATH ;;
+"$INSTALLROOT"/dyn/lib:*)
+	DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH#*:} ;;
+esac
+case $LIBRARY_PATH in
+"$INSTALLROOT"/dyn/lib)
+	unset LIBRARY_PATH ;;
+"$INSTALLROOT"/dyn/lib:*)
+	LIBRARY_PATH=${LIBRARY_PATH#*:} ;;
+esac
+case $C_INCLUDE_PATH in
+"$INSTALLROOT"/include/ast)
+	unset C_INCLUDE_PATH ;;
+"$INSTALLROOT"/include/ast:*)
+	C_INCLUDE_PATH=${C_INCLUDE_PATH#*:} ;;
+esac
+
 # some actions have their own PACKAGEROOT or kick out early
 
 case $action in
@@ -2032,74 +2069,32 @@ case $x in
 			# Allow loading dynamic libraries from $INSTALLROOT/dyn/lib by setting library
 			# path variables for various systems; instead of bothering to detect the system,
 			# just set them all. (Note: not needed and potentially harmful while building.)
-			case $LD_LIBRARY_PATH: in
-			$INSTALLROOT/dyn/lib:*)
-				;;
-			*)	LD_LIBRARY_PATH=$INSTALLROOT/dyn/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
-				$show LD_LIBRARY_PATH=$LD_LIBRARY_PATH
-				$show export LD_LIBRARY_PATH
-				export LD_LIBRARY_PATH
-				;;
-			esac
-			case $LIBPATH: in
-			$INSTALLROOT/dyn/bin:$INSTALLROOT/dyn/lib:*)
-				;;
-			*)	case $LIBPATH in
-				'')	LIBPATH=/usr/lib:/lib ;;
-				esac
-				LIBPATH=$INSTALLROOT/dyn/bin:$INSTALLROOT/dyn/lib:$LIBPATH
-				$show LIBPATH=$LIBPATH
-				$show export LIBPATH
-				export LIBPATH
-				;;
-			esac
-			case $SHLIB_PATH: in
-			$INSTALLROOT/dyn/lib:*)
-				;;
-			*)	SHLIB_PATH=$INSTALLROOT/dyn/lib${SHLIB_PATH:+:$SHLIB_PATH}
-				$show SHLIB_PATH=$SHLIB_PATH
-				$show export SHLIB_PATH
-				export SHLIB_PATH
-				;;
-			esac
-			case $DYLD_LIBRARY_PATH: in
-			$INSTALLROOT/dyn/lib:*)
-				;;
-			*)	DYLD_LIBRARY_PATH=$INSTALLROOT/dyn/lib${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}
-				$show DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH
-				$show export DYLD_LIBRARY_PATH
-				export DYLD_LIBRARY_PATH
-				;;
-			esac
-			case $_RLD_ROOT in
-			$INSTALLROOT/arch*)	;;
-			':')	_RLD_ROOT=$INSTALLROOT/arch:/ ;;
-			/|*:/)	_RLD_ROOT=$INSTALLROOT/arch:$_RLD_ROOT ;;
-			*)	_RLD_ROOT=$INSTALLROOT/arch:$_RLD_ROOT:/ ;;
+			export LD_LIBRARY_PATH=$INSTALLROOT/dyn/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
+			$show export LD_LIBRARY_PATH=$LD_LIBRARY_PATH
+
+			case $LIBPATH in
+			'')	LIBPATH=/usr/lib:/lib ;;
 			esac
-			$show _RLD_ROOT=$_RLD_ROOT
-			$show export _RLD_ROOT
-			export _RLD_ROOT
+			export LIBPATH=$INSTALLROOT/dyn/bin:$INSTALLROOT/dyn/lib:$LIBPATH
+			$show export LIBPATH=$LIBPATH
+
+			export SHLIB_PATH=$INSTALLROOT/dyn/lib${SHLIB_PATH:+:$SHLIB_PATH}
+			$show export SHLIB_PATH=$SHLIB_PATH
+
+			export DYLD_LIBRARY_PATH=$INSTALLROOT/dyn/lib${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}
+			$show export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH
+
 			# dynamic library search path for gcc and clang; also used for runtime linking on Haiku
-			case $LIBRARY_PATH: in
-			$INSTALLROOT/dyn/lib:*)
-				;;
-			*)	LIBRARY_PATH=$INSTALLROOT/dyn/lib${LIBRARY_PATH:+:$LIBRARY_PATH}
-				$show LIBRARY_PATH=$LIBRARY_PATH
-				$show export LIBRARY_PATH
-				export LIBRARY_PATH
-				;;
-			esac
-			# include header search path for gcc and clang
-			case $C_INCLUDE_PATH: in
-			$INSTALLROOT/include/ast:*)
-				;;
-			*)	C_INCLUDE_PATH=$INSTALLROOT/include/ast${C_INCLUDE_PATH:+:$C_INCLUDE_PATH}
-				$show C_INCLUDE_PATH=$C_INCLUDE_PATH
-				$show export C_INCLUDE_PATH
-				export C_INCLUDE_PATH
-				;;
-			esac
+			export LIBRARY_PATH=$INSTALLROOT/dyn/lib${LIBRARY_PATH:+:$LIBRARY_PATH}
+			$show export LIBRARY_PATH=$LIBRARY_PATH
+			;;
+		esac
+		case $action in
+		use)
+			# Include header search path for gcc and clang. Allows skipping '-I ~ast/include/ast'
+			# when compiling a test program from the command line in a 'package use' environment.
+			export C_INCLUDE_PATH=$INSTALLROOT/include/ast${C_INCLUDE_PATH:+:$C_INCLUDE_PATH}
+			$show export C_INCLUDE_PATH=$C_INCLUDE_PATH
 			;;
 		esac
 

src/cmd/INIT/Mamfile

@@ -11,7 +11,7 @@ note * initialization for the build system
 note *
 
 make install virtual
-	loop DIR bin include/ast lib/lib lib/probe/C/make lib/probe/C/pp lib/probe/C/mam
+	loop DIR bin include/ast lib/lib lib/probe/C/make lib/probe/C/mam
 		make %{INSTALLROOT}/%{DIR}
 			exec - mkdir -p %{@}
 		done
@@ -184,7 +184,6 @@ make test dontcare virtual
 	done
 	make test.iffe virtual
 		makp iffe.tst
-		exec - unset C_INCLUDE_PATH  # set by bin/package, but invalidates some iffe tests
 		exec - regress --verbose %{<} iffe
 	done
 done test

src/cmd/INIT/iffe.sh

@@ -33,13 +33,16 @@ esac
 set -o noglob
 
 command=iffe
-version=2024-08-02
+version=2024-12-16
 
 # DEFPATH should be inherited from package(1)
 case $DEFPATH in
 /*)	;;
-*)	echo "$command: DEFPATH not set" >&2
-	exit 1 ;;
+*)	DEFPATH=$(getconf PATH) ||
+	{
+		echo "$command: DEFPATH not set" >&2
+		exit 1
+	} ;;
 esac
 
 compile() # $cc ...
@@ -1173,8 +1176,10 @@ esac
 		{	test -f "$3" && test "$o" -nt "$3"
 		}
 	} || exit 1
-	echo "$command: test results in $o are up to date" >&$stderr
-	# update timestamp for correct dependency resolution in mamake
+	# still bump the timestamp for correct dependency resolution in mamake
+	case $verbose in
+	1)	echo "$command: test results in $o are up to date; bumping timestamp" >&$stderr ;;
+	esac
 	touch "$o" || kill "$$"
 ) && exit 0
 
@@ -1211,9 +1216,11 @@ std='/* AST backward compatibility macros */
 #define _END_EXTERNS_'
 # To ensure the environment tested is the same as that used, add standards
 # compliance macros as probed by libast as soon as they are available.
-if	test -f "${INSTALLROOT}/src/lib/libast/${dir}/standards"
-then	std=${std}${nl}$(cat "${INSTALLROOT}/src/lib/libast/${dir}/standards")
-fi
+case $INSTALLROOT in
+/*)	if	test -f "${INSTALLROOT}/src/lib/libast/${dir}/standards"
+	then	std="${std}${nl}#include \"${INSTALLROOT}/src/lib/libast/${dir}/standards\""
+	fi ;;
+esac
 tst=
 ext="#include <stdio.h>"
 
@@ -2855,18 +2862,13 @@ int x;
 									sed 's,/\*[^/]*\*/, ,g' $x > $tmp.t
 									;;
 								esac
-								if	cmp -s $tmp.c $tmp.t
-								then	rm -f $tmp.h
-									case $verbose in
-									1)	echo "$command: $x: unchanged;" \
-											"updating timestamp" >&$stderr ;;
-									esac
-									touch "$x"  # needed for mamake dependency tree integrity
-								else	case $x in
-									${dir}[\\/]$cur)	test -d $dir || mkdir $dir || exit 1 ;;
-									esac
-									mv $tmp.h $x
-								fi
+								# to ensure correct dependency resolution, always move the new file
+								# over, to bump the timestamp even if the results haven't changed
+								case $x in
+								${dir}[\\/]$cur)
+									test -d $dir || mkdir $dir || exit ;;
+								esac
+								mv -f $tmp.h $x || exit
 								;;
 							esac
 							;;

src/cmd/INIT/utils/Mamfile_rm_unused_vars

@@ -20,7 +20,7 @@ IFS=$CCn; for varname in $vars; do IFS=
 	MAMAKE_STRICT)	continue ;;
 	esac
 	case $mamfile in
-	*"\${$varname"[}[:=?+-]* | *"%{$varname"[}\|?+-]* )
+	*"\${$varname"[}\[:=?+-]* | *"%{$varname"[}\|?+-]* )
 		;;
 	* )	# add with '|' separator for Extended Regular Expression
 		rm_unused_ere="${rm_unused_ere:+$rm_unused_ere|}setv[[:blank:]]+$varname([[:blank:]]|$)" ;;

src/cmd/ksh93/Mamfile

@@ -576,6 +576,7 @@ make install virtual
 		done
 
 		make sh/arith.c
+			prev %{INCLUDE_AST}/ast_release.h
 			prev include/builtins.h
 			prev include/variables.h
 			prev include/streval.h
@@ -599,6 +600,7 @@ make install virtual
 		done
 
 		make sh/array.c
+			prev %{INCLUDE_AST}/ast_release.h
 			prev include/name.h
 			prev include/defs.h
 			prev shopt.h
@@ -743,6 +745,7 @@ make install virtual
 		done
 
 		make sh/lex.c
+			prev %{INCLUDE_AST}/ast_release.h
 			prev include/shlex.h
 			prev include/io.h
 			prev include/lexstates.h
@@ -989,6 +992,7 @@ make install virtual
 		done
 
 		make edit/emacs.c
+			prev %{INCLUDE_AST}/ast_release.h
 			prev include/terminal.h
 			prev include/edit.h
 			prev include/history.h

src/cmd/ksh93/edit/emacs.c

@@ -72,6 +72,7 @@ One line screen editor for any program
 #if SHOPT_MULTIBYTE
 #include	<wctype.h>
 #endif /* SHOPT_MULTIBYTE */
+#include	<ast_release.h>
 
 #define ESH_NFIRST
 #define ESH_KAPPEND

src/cmd/ksh93/include/edit.h

@@ -35,9 +35,7 @@
 #define LOOKAHEAD	80
 
 #if SHOPT_MULTIBYTE
-#   ifndef ESS_MAXCHAR
-#	include	"national.h"
-#   endif /* ESS_MAXCHAR */
+#   include	"national.h"
     typedef wchar_t genchar;
 #   define CHARSIZE	(sizeof(wchar_t)<=2?3:sizeof(wchar_t))
 #else

src/cmd/ksh93/sh/arith.c

@@ -29,6 +29,7 @@
 #include	"streval.h"
 #include	"variables.h"
 #include	"builtins.h"
+#include	<ast_release.h>
 
 #ifndef LLONG_MAX
 #define LLONG_MAX	LONG_MAX

src/cmd/ksh93/sh/array.c

@@ -28,6 +28,7 @@
 #include	"shopt.h"
 #include	"defs.h"
 #include	"name.h"
+#include	<ast_release.h>
 
 #if _AST_release
 #define NDEBUG

src/cmd/ksh93/sh/lex.c

@@ -35,6 +35,7 @@
 #include	"lexstates.h"
 #include	"io.h"
 #include	"shlex.h"
+#include	<ast_release.h>
 
 #define TEST_RE		3
 #define SYNBAD		3	/* exit value for syntax errors */

src/cmd/ksh93/tests/builtins.sh

@@ -594,7 +594,7 @@ fi
 	while (( i <2))
 	do	(( i++))
 	done) == $'0\n0\n1\n1\n2' ]]  || err_exit  "DEBUG trap not working"
-if builtin getconf 2> /dev/null; then
+if ((!SHOPT_ECHOPRINT)) && builtin getconf 2> /dev/null; then
 	getconf UNIVERSE - ucb
 	[[ $($SHELL -c 'echo -3') == -3 ]] || err_exit "echo -3 not working in ucb universe"
 fi

src/cmd/ksh93/tests/case.sh

@@ -2,7 +2,7 @@
 #                                                                      #
 #               This software is part of the ast package               #
 #          Copyright (c) 1982-2011 AT&T Intellectual Property          #
-#          Copyright (c) 2020-2023 Contributors to ksh 93u+m           #
+#          Copyright (c) 2020-2024 Contributors to ksh 93u+m           #
 #                      and is licensed under the                       #
 #                 Eclipse Public License, Version 2.0                  #
 #                                                                      #
@@ -55,7 +55,7 @@ fi
 a)      print -n a > /dev/null ;&
 b)      print b;;
 esac') != b ]] && err_exit 'bug in ;& at end of script'
-[[ $(VMDEBUG=1 $SHELL -c '
+[[ $("$SHELL" -c '
 	tmp=foo
 	for i in a b
 	do	case $i in

src/cmd/ksh93/tests/path.sh

@@ -310,7 +310,7 @@ status=$($SHELL -c $'trap \'print $?\' ERR;/dev/null 2> /dev/null')
 
 # universe via PATH
 
-if builtin getconf 2> /dev/null; then
+if ((!SHOPT_ECHOPRINT)) && builtin getconf 2> /dev/null; then
 	getconf UNIVERSE - att # override sticky default 'UNIVERSE = foo'
 
 	[[ $(PATH=/usr/ucb/bin:/usr/bin echo -n ucb) == 'ucb' ]] || err_exit "ucb universe echo ignores -n option"

src/cmd/ksh93/tests/shtests

@@ -278,8 +278,9 @@ then	s=${SHELL:##*sh}
 fi
 
 # when testing a shell outside of INSTALLROOT, make sure libraries in INSTALLROOT are not loaded
+# The variables unset below should be the same ones that are set for 'test' or 'use' in bin/package
 if	! [[ ${SHELL%/*} -ef $INSTALLROOT/bin || ${SHELL%/*} -ef $INSTALLROOT/dyn/bin ]]
-then	unset -v LD_LIBRARY_PATH LIBPATH SHLIB_PATH DYLD_LIBRARY_PATH _RLD_ROOT
+then	unset -v LD_LIBRARY_PATH LIBPATH SHLIB_PATH DYLD_LIBRARY_PATH LIBRARY_PATH C_INCLUDE_PATH
 fi
 
 tmp=$(