Use kernel CFLAGS for 'kernels' subdirs in addons. (#658)

Details:
- Updated Makefile and common.mk so that the targeted configuration's
  kernel CFLAGS are applied to source files that are found in a
  'kernels' subdirectory within an enabled addon. For now, this
  behavior only applies when the 'kernels' directory is at the top
  level of the addon directory structure. For example, if there is an
  addon named 'foobar', the source code must be located in
  addon/foobar/kernels/ in order for it to be compiled with the target
  configurations's kernel CFLAGS. Any other source code within
  addon/foobar/ will be compiled with general-purpose CFLAGS (the same
  ones that were used on all addon code prior to this commit). Thanks
  to AMD (esp. Mithun Mohan) for suggesting this change and catching an
  intermediate bug in the PR.
- Comment/whitespace updates.

Makefile

@@ -213,8 +213,19 @@ MK_REFKERN_OBJS     := $(foreach arch, $(CONFIG_LIST), \
 MK_FRAME_OBJS       := $(call gen-obj-paths-from-src,$(FRAME_SRC_SUFS),$(MK_FRAME_SRC),$(FRAME_PATH),$(BASE_OBJ_FRAME_PATH))
 
 # Generate object file paths for the addon source code. If one or more addons
-# were not enabled a configure-time, this variable will we empty.
-MK_ADDON_OBJS       := $(call gen-obj-paths-from-src,$(ADDON_SRC_SUFS),$(MK_ADDON_SRC),$(ADDON_PATH),$(BASE_OBJ_ADDON_PATH))
+# were not enabled a configure-time, these variable will we empty.
+# NOTE: We separate the source and objects into kernel and non-kernel lists.
+MK_ADDON_KERS_SRC   := $(foreach addon, $(ADDON_LIST), \
+                           $(filter $(ADDON_PATH)/$(addon)/$(KERNELS_DIR)/%, \
+                                    $(MK_ADDON_SRC)) \
+                        )
+MK_ADDON_OTHER_SRC  := $(foreach addon, $(ADDON_LIST), \
+                           $(filter-out $(ADDON_PATH)/$(addon)/$(KERNELS_DIR)/%, \
+                                        $(MK_ADDON_SRC)) \
+                        )
+MK_ADDON_KERS_OBJS  := $(call gen-obj-paths-from-src,$(ADDON_SRC_SUFS),$(MK_ADDON_KERS_SRC),$(ADDON_PATH),$(BASE_OBJ_ADDON_PATH))
+MK_ADDON_OTHER_OBJS := $(call gen-obj-paths-from-src,$(ADDON_SRC_SUFS),$(MK_ADDON_OTHER_SRC),$(ADDON_PATH),$(BASE_OBJ_ADDON_PATH))
+MK_ADDON_OBJS       := $(MK_ADDON_KERS_OBJS) $(MK_ADDON_OTHER_OBJS)
 
 # Generate object file paths for the sandbox source code. If a sandbox was not
 # enabled a configure-time, this variable will we empty.
@@ -580,6 +591,7 @@ endef
 
 # first argument: a configuration name from the union of config_list and
 # config_name, used to look up the CFLAGS to use during compilation.
+# second argument: the C99 addon file suffix being considered.
 define make-c99-addon-rule
 $(BASE_OBJ_ADDON_PATH)/%.o: $(ADDON_PATH)/%.$(2) $(BLIS_H_FLAT) $(ADDON_H99_FILES) $(MAKE_DEFS_MK_PATHS)
 ifeq ($(ENABLE_VERBOSE),yes)
@@ -590,6 +602,23 @@ else
 endif
 endef
 
+# first argument: a configuration name from the union of config_list and
+# config_name, used to look up the CFLAGS to use during compilation.
+# second argument: the C99 addon file suffix being considered.
+# third argument: the name of the addon being considered.
+define make-c99-addon-kers-rule
+$(BASE_OBJ_ADDON_PATH)/$(3)/$(KERNELS_DIR)/%.o: $(ADDON_PATH)/$(3)/$(KERNELS_DIR)/%.$(2) $(BLIS_H_FLAT) $(ADDON_H99_FILES) $(MAKE_DEFS_MK_PATHS)
+ifeq ($(ENABLE_VERBOSE),yes)
+	$(CC) $(call get-addon-kernel-c99flags-for,$(1)) -c $$< -o $$@
+else
+	@echo "Compiling $$@" $(call get-addon-kernel-text-for,$(1))
+	@$(CC) $(call get-addon-kernel-c99flags-for,$(1)) -c $$< -o $$@
+endif
+endef
+
+# first argument: a configuration name from the union of config_list and
+# config_name, used to look up the CFLAGS to use during compilation.
+# second argument: the C++ addon file suffix being considered.
 define make-cxx-addon-rule
 $(BASE_OBJ_ADDON_PATH)/%.o: $(ADDON_PATH)/%.$(2) $(BLIS_H_FLAT) $(ADDON_HXX_FILES) $(MAKE_DEFS_MK_PATHS)
 ifeq ($(ENABLE_VERBOSE),yes)
@@ -602,6 +631,7 @@ endef
 
 # first argument: a configuration name from the union of config_list and
 # config_name, used to look up the CFLAGS to use during compilation.
+# second argument: the C99 sandbox file suffix being considered.
 define make-c99-sandbox-rule
 $(BASE_OBJ_SANDBOX_PATH)/%.o: $(SANDBOX_PATH)/%.$(2) $(BLIS_H_FLAT) $(SANDBOX_H99_FILES) $(MAKE_DEFS_MK_PATHS)
 ifeq ($(ENABLE_VERBOSE),yes)
@@ -612,6 +642,9 @@ else
 endif
 endef
 
+# first argument: a configuration name from the union of config_list and
+# config_name, used to look up the CFLAGS to use during compilation.
+# second argument: the C++ sandbox file suffix being considered.
 define make-cxx-sandbox-rule
 $(BASE_OBJ_SANDBOX_PATH)/%.o: $(SANDBOX_PATH)/%.$(2) $(BLIS_H_FLAT) $(SANDBOX_HXX_FILES) $(MAKE_DEFS_MK_PATHS)
 ifeq ($(ENABLE_VERBOSE),yes)
@@ -657,6 +690,12 @@ $(foreach kset, $(KERNEL_LIST), $(eval $(call make-kernels-rule,$(kset),$(call g
 $(foreach suf, $(ADDON_C99_SUFS), \
 $(foreach conf, $(CONFIG_NAME), $(eval $(call make-c99-addon-rule,$(conf),$(suf)))))
 
+# Instantiate the build rule for C addon/kernels files. Use the CFLAGS for the
+# configuration family.
+$(foreach addon, $(ADDON_LIST), \
+$(foreach suf, $(ADDON_C99_SUFS), \
+$(foreach conf, $(CONFIG_NAME), $(eval $(call make-c99-addon-kers-rule,$(conf),$(suf),$(addon))))))
+
 # Instantiate the build rule for C++ addon files. Use the CFLAGS for the
 # configuration family.
 $(foreach suf, $(ADDON_CXX_SUFS), \

common.mk

@@ -154,7 +154,7 @@ get-kernel-cflags-for    = $(strip $(call load-var-for,CKOPTFLAGS,$(1)) \
                                    $(BUILD_SYMFLAGS) \
                             )
 
-# When compiling sandboxes, we use flags similar to those of general framework
+# When compiling addons, we use flags similar to those of general framework
 # source. This ensures that the same code can be linked and run across various
 # sub-configurations.
 get-addon-c99flags-for   = $(strip $(call load-var-for,COPTFLAGS,$(1)) \
@@ -169,6 +169,15 @@ get-addon-cxxflags-for   = $(strip $(call load-var-for,COPTFLAGS,$(1)) \
                                    $(BUILD_CPPFLAGS) \
                                    $(BUILD_SYMFLAGS) \
                             )
+# When compiling addon kernels, we use flags similar to those of kernels
+# flags, except we also include the addon header paths.
+get-addon-kernel-c99flags-for = $(strip $(call load-var-for,CKOPTFLAGS,$(1)) \
+                                        $(call load-var-for,CKVECFLAGS,$(1)) \
+                                        $(call get-noopt-cflags-for,$(1)) \
+                                        $(CADDONINCFLAGS) \
+                                        $(BUILD_CPPFLAGS) \
+                                        $(BUILD_SYMFLAGS) \
+                                 )
 
 # When compiling sandboxes, we use flags similar to those of general framework
 # source. This ensures that the same code can be linked and run across various
@@ -203,16 +212,17 @@ get-user-cflags-for      = $(strip $(call load-var-for,COPTFLAGS,$(1)) \
 
 # Define functions that return messages appropriate for each non-verbose line
 # of compilation output.
-get-noopt-text          = "(CFLAGS for no optimization)"
-get-refinit-text-for    = "('$(1)' CFLAGS for ref. kernel init)"
-get-refkern-text-for    = "('$(1)' CFLAGS for ref. kernels)"
-get-config-text-for     = "('$(1)' CFLAGS for config code)"
-get-frame-text-for      = "('$(1)' CFLAGS for framework code)"
-get-kernel-text-for     = "('$(1)' CFLAGS for kernels)"
-get-addon-c99text-for   = "('$(1)' CFLAGS for addons)"
-get-addon-cxxtext-for   = "('$(1)' CXXFLAGS for addons)"
-get-sandbox-c99text-for = "('$(1)' CFLAGS for sandboxes)"
-get-sandbox-cxxtext-for = "('$(1)' CXXFLAGS for sandboxes)"
+get-noopt-text            = "(CFLAGS for no optimization)"
+get-refinit-text-for      = "('$(1)' CFLAGS for ref. kernel init)"
+get-refkern-text-for      = "('$(1)' CFLAGS for ref. kernels)"
+get-config-text-for       = "('$(1)' CFLAGS for config code)"
+get-frame-text-for        = "('$(1)' CFLAGS for framework code)"
+get-kernel-text-for       = "('$(1)' CFLAGS for kernels)"
+get-addon-c99text-for     = "('$(1)' CFLAGS for addons)"
+get-addon-cxxtext-for     = "('$(1)' CXXFLAGS for addons)"
+get-addon-kernel-text-for = "('$(1)' CFLAGS for addon kernels)"
+get-sandbox-c99text-for   = "('$(1)' CFLAGS for sandboxes)"
+get-sandbox-cxxtext-for   = "('$(1)' CXXFLAGS for sandboxes)"