diff --git a/aspect/BUILD b/aspect/BUILD index 7b857a764ae..0a0679e91b8 100644 --- a/aspect/BUILD +++ b/aspect/BUILD @@ -28,17 +28,35 @@ filegroup( # the aspect files that will be bundled with the final plugin zip filegroup( name = "aspect_files", + srcs = [ + ":aspect_files_only", + ":aspect_tools", + ], + visibility = ["//visibility:public"], +) + +# Separate different targets as they are put into different directories +filegroup( + name = "aspect_files_only", srcs = [ "WORKSPACE", "artifacts.bzl", + "build_compose_dependencies.bzl", "build_dependencies.bzl", - "fast_build_info_bundled.bzl", + "build_dependencies_deps.bzl", "intellij_info.bzl", "intellij_info_bundled.bzl", "intellij_info_impl_bundled.bzl", "java_classpath.bzl", "make_variables.bzl", ":BUILD.bazel", + ], + visibility = ["//visibility:public"], +) + +filegroup( + name = "aspect_tools", + srcs = [ "//aspect/tools:CreateAar", "//aspect/tools:JarFilter_deploy.jar", "//aspect/tools:PackageParser_deploy.jar", @@ -81,17 +99,7 @@ genrule( srcs = ["intellij_info_impl.bzl"], outs = ["intellij_info_impl_bundled.bzl"], cmd = "cat $(SRCS) >$@ && " + - "sed -i -e '/BUNDLED-IGNORE-BEGIN/,/BUNDLED-IGNORE-END/d' $@ && " + - "sed -i -e 's,load(\".*/rules_java/,load(\"@rules_java//', $@", -) - -# Makes bundled fast_build_info.bzl use bundled intellij_info_impl.bzl -genrule( - name = "create_fast_build_info_bundle", - srcs = ["fast_build_info.bzl"], - outs = ["fast_build_info_bundled.bzl"], - cmd = "cat $(SRCS) >$@ && " + - "sed -i -e 's,:intellij_info_impl.bzl,:intellij_info_impl_bundled.bzl,g' $@", + "sed -i -e '/BUNDLED-IGNORE-BEGIN/,/BUNDLED-IGNORE-END/d' $@", ) define_flag_hack() diff --git a/aspect/build_compose_dependencies.bzl b/aspect/build_compose_dependencies.bzl new file mode 100644 index 00000000000..090a9e330ce --- /dev/null +++ b/aspect/build_compose_dependencies.bzl @@ -0,0 +1,37 @@ +"""Aspects to build and collect project's compose dependencies.""" + +load("@rules_java//java:defs.bzl", "JavaInfo") + +ComposeDependenciesInfo = provider( + "The compose dependencies", + fields = { + "render_jars": "a list of render jars generated for project files and external dependencies", + }, +) + +def _package_compose_dependencies_impl(target, ctx): # @unused + return [OutputGroupInfo( + render_jars = target[ComposeDependenciesInfo].render_jars.to_list(), + )] + +package_compose_dependencies = aspect( + implementation = _package_compose_dependencies_impl, + required_aspect_providers = [[ComposeDependenciesInfo]], +) + +def _collect_compose_dependencies_impl(target, ctx): # @unused + if JavaInfo not in target: + return [ComposeDependenciesInfo( + render_jars = depset(), + )] + return [ + ComposeDependenciesInfo( + render_jars = depset([], transitive = [target[JavaInfo].transitive_runtime_jars]), + ), + ] + +collect_compose_dependencies = aspect( + implementation = _collect_compose_dependencies_impl, + provides = [ComposeDependenciesInfo], + attr_aspects = ["deps", "exports", "_android_sdk"], +) diff --git a/aspect/build_dependencies.bzl b/aspect/build_dependencies.bzl index 778d91c775f..44e47bacb80 100644 --- a/aspect/build_dependencies.bzl +++ b/aspect/build_dependencies.bzl @@ -1,13 +1,17 @@ """Aspects to build and collect project dependencies.""" +# Load external dependencies of this aspect. These are loaded in a separate file and re-exported as necessary +# to make supporting other versions of bazel easier, by replacing build_dependencies_deps.bzl. load( - "@bazel_tools//tools/build_defs/cc:action_names.bzl", - "CPP_COMPILE_ACTION_NAME", - "C_COMPILE_ACTION_NAME", + ":build_dependencies_deps.bzl", + "ANDROID_IDE_INFO", + "ZIP_TOOL_LABEL", + _ide_cc_not_validated = "IDE_CC", + _ide_java_not_validated = "IDE_JAVA", + _ide_kotlin_not_validated = "IDE_KOTLIN", ) -load("@rules_java//java:defs.bzl", "JavaInfo") -ALWAYS_BUILD_RULES = "java_proto_library,java_lite_proto_library,java_mutable_proto_library,kt_proto_library_helper,_java_grpc_library,_java_lite_grpc_library,kt_grpc_library_helper,java_stubby_library,kt_stubby_library_helper,aar_import,java_import" +ALWAYS_BUILD_RULES = "java_proto_library,java_lite_proto_library,java_mutable_proto_library,kt_proto_library_helper,_java_grpc_library,_java_lite_grpc_library,kt_grpc_library_helper,java_stubby_library,kt_stubby_library_helper,aar_import,java_import, j2kt_native_import" PROTO_RULE_KINDS = [ "java_proto_library", @@ -16,50 +20,114 @@ PROTO_RULE_KINDS = [ "kt_proto_library_helper", ] +def _rule_function( + rule): # @unused + return [] + +def _target_rule_function( + target, # @unused + rule): # @unused + return [] + +def _unique(values): + return {k: None for k in values}.keys() + +def _validate_ide(unvalidated, template): + "Basic validation that a provided implementation conforms to a given template" + for a in dir(template): + if not hasattr(unvalidated, a): + fail("attribute missing: ", a, unvalidated) + elif type(getattr(unvalidated, a)) != type(getattr(template, a)): + fail("attribute type mismatch: ", a, type(getattr(unvalidated, a)), type(getattr(template, a))) + return struct(**{a: getattr(unvalidated, a) for a in dir(template) if a not in dir(struct())}) + +IDE_JAVA = _validate_ide( + _ide_java_not_validated, + template = struct( + srcs_attributes = [], # Additional srcs like attributes. + get_java_info = _target_rule_function, # A function that takes a rule and returns a JavaInfo like structure (or the provider itself). + ), +) + +IDE_KOTLIN = _validate_ide( + _ide_kotlin_not_validated, + template = struct( + srcs_attributes = [], # Additional srcs like attributes. + follow_attributes = [], # Additional attributes for the aspect to follow and request DependenciesInfo provider. + follow_additional_attributes = [], # Additional attributes for the aspect to follow without requesting DependenciesInfo provider. + followed_dependencies = _rule_function, # A function that takes a rule and returns a list of dependencies (targets or toolchain containers). + toolchains_aspects = [], # Toolchain types for the aspect to follow. + ), +) + +IDE_CC = _validate_ide( + _ide_cc_not_validated, + template = struct( + c_compile_action_name = "", # An action named to be used with cc_common.get_memory_inefficient_command_line or similar. + cpp_compile_action_name = "", # An action named to be used with cc_common.get_memory_inefficient_command_line or similar. + follow_attributes = ["_cc_toolchain"], # Additional attributes for the aspect to follow and request DependenciesInfo provider. + toolchains_aspects = [], # Toolchain types for the aspect to follow. + toolchain_target = _rule_function, # A function that takes a rule and returns a toolchain target (or a toolchain container). + ), +) + +JVM_SRC_ATTRS = _unique(["srcs"] + IDE_JAVA.srcs_attributes + IDE_KOTLIN.srcs_attributes) + def _package_dependencies_impl(target, ctx): java_info_file = _write_java_target_info(target, ctx) cc_info_file = _write_cc_target_info(target, ctx) + dep_info = target[DependenciesInfo] return [OutputGroupInfo( - qsync_jars = target[DependenciesInfo].compile_time_jars.to_list(), + qsync_jars = dep_info.compile_time_jars.to_list() if dep_info.compile_time_jars else [], artifact_info_file = java_info_file, - qsync_aars = target[DependenciesInfo].aars.to_list(), - qsync_gensrcs = target[DependenciesInfo].gensrcs.to_list(), - cc_headers = target[DependenciesInfo].cc_headers.to_list(), - cc_info_file = cc_info_file + [target[DependenciesInfo].cc_toolchain_info.file] if target[DependenciesInfo].cc_toolchain_info else [], + qsync_aars = dep_info.aars.to_list() if dep_info.aars else [], + qsync_gensrcs = dep_info.gensrcs.to_list() if dep_info.gensrcs else [], + cc_headers = dep_info.cc_headers.to_list() if dep_info.cc_headers else [], + cc_info_file = cc_info_file + [dep_info.cc_toolchain_info.file] if dep_info.cc_toolchain_info else [], )] -def _write_java_target_info(target, ctx): - if not target[DependenciesInfo].target_to_artifacts: +def _write_java_target_info(target, ctx, custom_prefix = ""): + """Write java target info to a file in proto format. + + The proto format used is defined by proto bazel.intellij.JavaArtifacts. + """ + target_to_artifacts = target[DependenciesInfo].target_to_artifacts + if not target_to_artifacts: return [] - file_name = target.label.name + ".java-info.txt" + file_name = custom_prefix + target.label.name + ".java-info.txt" artifact_info_file = ctx.actions.declare_file(file_name) ctx.actions.write( artifact_info_file, - _encode_target_info_proto(target[DependenciesInfo].target_to_artifacts), + _encode_target_info_proto(target_to_artifacts), ) return [artifact_info_file] def _write_cc_target_info(target, ctx): - if not target[DependenciesInfo].cc_info: + """Write CC target info to a file in proto format. + + The proto format used defined by proto bazel.intellij.CcCompilationInfo. + """ + if not target[DependenciesInfo].cc_compilation_info: return [] cc_info_file_name = target.label.name + ".cc-info.txt" cc_info_file = ctx.actions.declare_file(cc_info_file_name) ctx.actions.write( cc_info_file, - _encode_cc_info_proto(target.label, target[DependenciesInfo].cc_info), + _encode_cc_compilation_info_proto(target.label, target[DependenciesInfo].cc_compilation_info), ) return [cc_info_file] DependenciesInfo = provider( "The out-of-project dependencies", fields = { + "label": "the label of a target it describes", "compile_time_jars": "a list of jars generated by targets", "target_to_artifacts": "a map between a target and all its artifacts", "aars": "a list of aars with resource files", "gensrcs": "a list of sources generated by project targets", "expand_sources": "boolean, true if the sources for this target should be expanded when it appears inside another rules srcs list", - "cc_info": "a structure containing info required to compile cc sources", + "cc_compilation_info": "a structure containing info required to compile cc sources", "cc_headers": "a depset of generated headers required to compile cc sources", "cc_toolchain_info": "struct containing cc toolchain info, with keys file (the output file) and id (unique ID for the toolchain info, referred to from elsewhere)", "test_mode_own_files": "a structure describing Java artifacts required when the target is requested within the project scope", @@ -68,39 +136,145 @@ DependenciesInfo = provider( ) def create_dependencies_info( + label, compile_time_jars = depset(), target_to_artifacts = {}, aars = depset(), gensrcs = depset(), expand_sources = False, - cc_info = None, + cc_compilation_info = None, cc_headers = depset(), cc_toolchain_info = None, test_mode_own_files = None, test_mode_cc_src_deps = depset()): """A helper function to create a DependenciesInfo provider instance.""" return DependenciesInfo( + label = label, compile_time_jars = compile_time_jars, target_to_artifacts = target_to_artifacts, aars = aars, gensrcs = gensrcs, expand_sources = expand_sources, - cc_info = cc_info, + cc_compilation_info = cc_compilation_info, cc_headers = cc_headers, cc_toolchain_info = cc_toolchain_info, test_mode_own_files = test_mode_own_files, test_mode_cc_src_deps = test_mode_cc_src_deps, ) +def create_java_dependencies_info( + compile_time_jars, + target_to_artifacts, + aars, + gensrcs, + expand_sources, + test_mode_own_files): + """A helper function to create a DependenciesInfo provider instance.""" + return struct( + compile_time_jars = compile_time_jars, + target_to_artifacts = target_to_artifacts, + aars = aars, + gensrcs = gensrcs, + expand_sources = expand_sources, + test_mode_own_files = test_mode_own_files, + ) + +def create_cc_dependencies_info( + cc_compilation_info = None, + cc_headers = depset(), + cc_toolchain_info = None, + test_mode_cc_src_deps = depset()): + """A helper function to create a DependenciesInfo provider instance.""" + return struct( + cc_compilation_info = cc_compilation_info, + cc_headers = cc_headers, + cc_toolchain_info = cc_toolchain_info, + test_mode_cc_src_deps = test_mode_cc_src_deps, + ) + +def create_cc_toolchain_info( + cc_toolchain_info = None, + test_mode_cc_src_deps = depset()): + """A helper function to create a DependenciesInfo provider instance.""" + return struct( + cc_toolchain_info = cc_toolchain_info, + test_mode_cc_src_deps = test_mode_cc_src_deps, + ) + +def merge_dependencies_info( + target, + ctx, # @unused + java_dep_info, + cc_dep_info, + cc_toolchain_dep_info): + """Merge multiple DependenciesInfo providers into one. + + Depsets and dicts are merged. For members such as `cc_compilation_info`, we require that at most one of the + DependenciesInfo's defines this which should always be the case. + + Args: + target: the target label whose dependencies will be merged + ctx: unused context + java_dep_info: java dep info + cc_dep_info: cc dep info + cc_toolchain_dep_info: cc toolchain info + + Returns: + Merged DependenciesInfo + """ + + if not java_dep_info and not cc_dep_info and not cc_toolchain_dep_info: + return [] + + if cc_dep_info and cc_toolchain_dep_info: + test_mode_cc_src_deps = depset(transitive = [cc_dep_info.test_mode_cc_src_deps, cc_toolchain_dep_info.test_mode_cc_src_deps]) + elif cc_dep_info: + test_mode_cc_src_deps = cc_dep_info.test_mode_cc_src_deps + elif cc_toolchain_dep_info: + test_mode_cc_src_deps = cc_toolchain_dep_info.test_mode_cc_src_deps + else: + test_mode_cc_src_deps = None + + merged = create_dependencies_info( + label = target.label, + compile_time_jars = java_dep_info.compile_time_jars if java_dep_info else None, + target_to_artifacts = java_dep_info.target_to_artifacts if java_dep_info else None, + aars = java_dep_info.aars if java_dep_info else None, + gensrcs = java_dep_info.gensrcs if java_dep_info else None, + expand_sources = java_dep_info.expand_sources if java_dep_info else None, + cc_compilation_info = cc_dep_info.cc_compilation_info if cc_dep_info else None, + cc_headers = cc_dep_info.cc_headers if cc_dep_info else None, + cc_toolchain_info = cc_toolchain_dep_info.cc_toolchain_info if cc_toolchain_dep_info else None, + test_mode_own_files = java_dep_info.test_mode_own_files if java_dep_info else None, + test_mode_cc_src_deps = test_mode_cc_src_deps, + ) + return merged + +def one_of(a, b): + """Returns whichever of a or b is not None, None if both are, or fails if neither are. + + Args: + a: possible None value + b: possible None value + + Returns: + Whichever of a or b is not None, None if both are, or fails if neither are + """ + if a == None: + return b + if b == None: + return a + fail("Expected at most one, but got both", a, b) + def _encode_target_info_proto(target_to_artifacts): contents = [] for label, target_info in target_to_artifacts.items(): contents.append( struct( target = label, - jars = target_info["jars"], - ide_aars = target_info["ide_aars"], - gen_srcs = target_info["gen_srcs"], + jars = _encode_file_list(target_info["jars"]) if "jars" in target_info else [], + ide_aars = _encode_file_list(target_info["ide_aars"]), + gen_srcs = _encode_file_list(target_info["gen_srcs"]), srcs = target_info["srcs"], srcjars = target_info["srcjars"], android_resources_package = target_info["android_resources_package"], @@ -108,18 +282,32 @@ def _encode_target_info_proto(target_to_artifacts): ) return proto.encode_text(struct(artifacts = contents)) -def _encode_cc_info_proto(label, cc_info): +def _encode_file_list(files): + """Encodes a list of files as a struct. + + Returns: + A list fo structs matching the bazel.intellij.OutputArtifact proto message. + """ + r = [] + for f in files: + if f.is_directory: + r.append(struct(directory = f.path)) + else: + r.append(struct(file = f.path)) + return r + +def _encode_cc_compilation_info_proto(label, cc_compilation_info): return proto.encode_text( struct(targets = [ struct( label = str(label), - defines = cc_info.transitive_defines, - include_directories = cc_info.transitive_include_directory, - quote_include_directories = cc_info.transitive_quote_include_directory, - system_include_directories = cc_info.transitive_system_include_directory, - framework_include_directories = cc_info.framework_include_directory, - gen_hdrs = cc_info.gen_headers, - toolchain_id = cc_info.toolchain_id, + defines = cc_compilation_info.transitive_defines, + include_directories = cc_compilation_info.transitive_include_directory, + quote_include_directories = cc_compilation_info.transitive_quote_include_directory, + system_include_directories = cc_compilation_info.transitive_system_include_directory, + framework_include_directories = cc_compilation_info.framework_include_directory, + gen_hdrs = _encode_file_list(cc_compilation_info.gen_headers), + toolchain_id = cc_compilation_info.toolchain_id, ), ]), ) @@ -148,7 +336,9 @@ def declares_android_resources(target, ctx): return hasattr(ctx.rule.attr, "resource_files") and len(ctx.rule.attr.resource_files) > 0 def _get_android_provider(target): - if hasattr(android_common, "AndroidIdeInfo"): + if ANDROID_IDE_INFO and ANDROID_IDE_INFO in target: + return target[ANDROID_IDE_INFO] + elif hasattr(android_common, "AndroidIdeInfo"): if android_common.AndroidIdeInfo in target: return target[android_common.AndroidIdeInfo] else: @@ -204,43 +394,80 @@ def _collect_all_dependencies_for_tests_impl(target, ctx): exclude = None, always_build_rules = ALWAYS_BUILD_RULES, generate_aidl_classes = None, - use_generated_srcjars = False, + use_generated_srcjars = True, test_mode = True, ) +def collect_dependencies_for_test(target, ctx, include = []): + return _collect_dependencies_core_impl( + target, + ctx, + include = ",".join(include), + exclude = "", + always_build_rules = ALWAYS_BUILD_RULES, + generate_aidl_classes = None, + use_generated_srcjars = True, + test_mode = False, + ) + +def _package_prefix_match(package, prefix): + if (package == prefix): + return True + if package.startswith(prefix) and package[len(prefix)] == "/": + return True + return False + +def _get_repo_name(label): + # The API to get the repo name changed between bazel versions. Use whichever exists: + return label.repo_name if "repo_name" in dir(label) else label.workspace_name + def _target_within_project_scope(label, include, exclude): + repo = _get_repo_name(label) + package = label.package result = False if include: - for inc in include.split(","): - if label.startswith(inc): - if label[len(inc)] in [":", "/"]: + if include == "//": + # when workspace root is included + result = True + else: + for inc in [Label(i) for i in include.split(",")]: + if _get_repo_name(inc) == repo and _package_prefix_match(package, inc.package): result = True break if result and len(exclude) > 0: - for exc in exclude.split(","): - if label.startswith(exc): - if label[len(exc)] in [":", "/"]: - result = False - break + for exc in [Label(i) for i in exclude.split(",")]: + if _get_repo_name(exc) == repo and _package_prefix_match(package, exc.package): + result = False + break return result -def _get_followed_kotlin_toolchain_dependency_info(rule): +def _get_dependency_attribute(rule, attr): + if hasattr(rule.attr, attr): + to_add = getattr(rule.attr, attr) + if type(to_add) == "list": + return [t for t in to_add if type(t) == "Target"] + elif type(to_add) == "Target": + return [to_add] return [] +def _get_followed_java_proto_dependencies(rule): + deps = [] + if rule.kind in ["proto_lang_toolchain", "java_rpc_toolchain"]: + deps.extend(_get_dependency_attribute(rule, "runtime")) + if rule.kind in ["_java_grpc_library", "_java_lite_grpc_library"]: + deps.extend(_get_dependency_attribute(rule, "_toolchain")) + return deps + def _get_followed_java_dependency_infos(rule): deps = [] - for (attr, kinds) in FOLLOW_JAVA_ATTRIBUTES_BY_RULE_KIND: - if hasattr(rule.attr, attr) and (not kinds or rule.kind in kinds): - to_add = getattr(rule.attr, attr) - if type(to_add) == "list": - deps += [t for t in to_add if type(t) == "Target"] - elif type(to_add) == "Target": - deps.append(to_add) + for attr in FOLLOW_JAVA_ATTRIBUTES: + deps.extend(_get_dependency_attribute(rule, attr)) - deps.extend(_get_followed_kotlin_toolchain_dependency_info(rule)) + deps.extend(_get_followed_java_proto_dependencies(rule)) + deps.extend(IDE_KOTLIN.followed_dependencies(rule)) return { - str(dep.label): dep[DependenciesInfo] + str(dep[DependenciesInfo].label): dep[DependenciesInfo] # NOTE: This handles duplicates. for dep in deps if DependenciesInfo in dep and dep[DependenciesInfo].target_to_artifacts } @@ -248,17 +475,13 @@ def _get_followed_java_dependency_infos(rule): def _collect_own_java_artifacts( target, ctx, - dependency_infos, + can_follow_dependencies, always_build_rules, generate_aidl_classes, use_generated_srcjars, target_is_within_project_scope): rule = ctx.rule - # Toolchains are collected for proto targets via aspect traversal, but jars - # produced for proto deps of the underlying proto_library are not - can_follow_dependencies = bool(dependency_infos) and not ctx.rule.kind in PROTO_RULE_KINDS - must_build_main_artifacts = ( not target_is_within_project_scope or rule.kind in always_build_rules.split(",") ) @@ -271,6 +494,8 @@ def _collect_own_java_artifacts( own_srcjar_files = [] resource_package = "" + java_info = IDE_JAVA.get_java_info(target, ctx.rule) + if must_build_main_artifacts: # For rules that we do not follow dependencies of (either because they don't # have further dependencies with JavaInfo or do so in attributes we don't care) @@ -279,18 +504,24 @@ def _collect_own_java_artifacts( # This is done primarily for rules like proto, whose toolchain classes # are collected via attribute traversal, but still requires jars for any # proto deps of the underlying proto_library. - if JavaInfo in target: + if java_info: if can_follow_dependencies: - own_jar_depsets.append(target[JavaInfo].compile_jars) + own_jar_depsets.append(java_info.compile_jars) else: - own_jar_depsets.append(target[JavaInfo].transitive_compile_time_jars) + own_jar_depsets.append(java_info.transitive_compile_time_jars) if declares_android_resources(target, ctx): ide_aar = _get_ide_aar_file(target, ctx) if ide_aar: - own_ide_aar_files.append(ide_aar) + # TODO(mathewi) - handle source aars + if not ide_aar.is_source: + own_ide_aar_files.append(ide_aar) elif declares_aar_import(ctx): - own_ide_aar_files.append(rule.attr.aar.files.to_list()[0]) + ide_aar = rule.attr.aar.files.to_list()[0] + + # TODO(mathewi) - handle source aars + if not ide_aar.is_source: + own_ide_aar_files.append(ide_aar) else: android = _get_android_provider(target) @@ -319,9 +550,9 @@ def _collect_own_java_artifacts( # Add generated java_outputs (e.g. from annotation processing) generated_class_jars = [] - if JavaInfo in target: - for java_output in target[JavaInfo].java_outputs: - # Prefer source jars if they exist: + if java_info: + for java_output in java_info.java_outputs: + # Prefer source jars if they exist and are requested: if use_generated_srcjars and java_output.generated_source_jar: own_gensrc_files.append(java_output.generated_source_jar) elif java_output.generated_class_jar: @@ -331,30 +562,27 @@ def _collect_own_java_artifacts( own_jar_files += generated_class_jars # Add generated sources for included targets - if hasattr(rule.attr, "srcs"): - for src in rule.attr.srcs: - for file in src.files.to_list(): - if not file.is_source: - expand_sources = False - if str(file.owner) in dependency_infos: - src_depinfo = dependency_infos[str(file.owner)] - expand_sources = src_depinfo.expand_sources - - # If the target that generates this source specifies that - # the sources should be expanded, we ignore the generated - # sources - the IDE will substitute the target sources - # themselves instead. - if not expand_sources: - own_gensrc_files.append(file) + for src_attr in JVM_SRC_ATTRS: + if hasattr(rule.attr, src_attr): + for src in getattr(rule.attr, src_attr): + # If the target that generates this source specifies that + # the sources should be expanded, we ignore the generated + # sources - the IDE will substitute the target sources + # themselves instead. + if not (DependenciesInfo in src and src[DependenciesInfo].expand_sources): + for file in src.files.to_list(): + if not file.is_source: + own_gensrc_files.append(file) if not target_is_within_project_scope: - if hasattr(rule.attr, "srcs"): - for src in rule.attr.srcs: - for file in src.files.to_list(): - if file.is_source: - own_src_files.append(file.path) - else: - own_gensrc_files.append(file) + for src_attr in JVM_SRC_ATTRS: + if hasattr(rule.attr, src_attr): + for src in getattr(rule.attr, src_attr): + for file in src.files.to_list(): + if file.is_source: + own_src_files.append(file.path) + else: + own_gensrc_files.append(file) if hasattr(rule.attr, "srcjar"): if rule.attr.srcjar and type(rule.attr.srcjar) == "Target": for file in rule.attr.srcjar.files.to_list(): @@ -373,6 +601,27 @@ def _collect_own_java_artifacts( android_resources_package = resource_package, ) +def _target_to_artifact_entry( + jars = [], + ide_aars = [], + gen_srcs = [], + srcs = [], + srcjars = [], + android_resources_package = ""): + return { + "jars": jars, + "ide_aars": ide_aars, + "gen_srcs": gen_srcs, + "srcs": srcs, + "srcjars": srcjars, + "android_resources_package": android_resources_package, + } + +def _can_follow_dependencies(ctx): + # Toolchains are collected for proto targets via aspect traversal, but jars + # produced for proto deps of the underlying proto_library are not + return not ctx.rule.kind in PROTO_RULE_KINDS + def _collect_own_and_dependency_java_artifacts( target, ctx, @@ -381,10 +630,12 @@ def _collect_own_and_dependency_java_artifacts( generate_aidl_classes, use_generated_srcjars, target_is_within_project_scope): + can_follow_dependencies = _can_follow_dependencies(ctx) + own_files = _collect_own_java_artifacts( target, ctx, - dependency_infos, + can_follow_dependencies, always_build_rules, generate_aidl_classes, use_generated_srcjars, @@ -403,19 +654,20 @@ def _collect_own_and_dependency_java_artifacts( target_to_artifacts = {} if has_own_artifacts: + # Pass the following lists through depset() to to remove any duplicates. jars = depset(own_files.jars, transitive = own_files.jar_depsets).to_list() - - # Pass the following lists through depset() too to remove any duplicates. ide_aars = depset(own_files.ide_aars).to_list() gen_srcs = depset(own_files.gensrcs).to_list() - target_to_artifacts[str(target.label)] = { - "jars": [_output_relative_path(file.path) for file in jars], - "ide_aars": [_output_relative_path(file.path) for file in ide_aars], - "gen_srcs": [_output_relative_path(file.path) for file in gen_srcs], - "srcs": own_files.srcs, - "srcjars": own_files.srcjars, - "android_resources_package": own_files.android_resources_package, - } + target_to_artifacts[str(target.label)] = _target_to_artifact_entry( + jars = jars, + ide_aars = ide_aars, + gen_srcs = gen_srcs, + srcs = own_files.srcs, + srcjars = own_files.srcjars, + android_resources_package = own_files.android_resources_package, + ) + elif target_is_within_project_scope: + target_to_artifacts[str(target.label)] = _target_to_artifact_entry() own_and_transitive_jar_depsets = list(own_files.jar_depsets) # Copy to prevent changes to own_jar_depsets. own_and_transitive_ide_aar_depsets = [] @@ -434,17 +686,14 @@ def _collect_own_and_dependency_java_artifacts( depset(own_files.gensrcs, transitive = own_and_transitive_gensrc_depsets), ) -def _get_followed_cc_toolchain_dependency_info(rule): +def _get_cc_toolchain_dependency_info(rule): + cc_toolchain_target = IDE_CC.toolchain_target(rule) + if cc_toolchain_target and DependenciesInfo in cc_toolchain_target: + return cc_toolchain_target[DependenciesInfo] return None -def _get_followed_cc_dependency_info(rule): - if hasattr(rule.attr, "_cc_toolchain"): - cc_toolchain_target = getattr(rule.attr, "_cc_toolchain") - if DependenciesInfo in cc_toolchain_target: - return cc_toolchain_target[DependenciesInfo] - return _get_followed_cc_toolchain_dependency_info(rule) - -def _collect_own_and_dependency_cc_info(target, dependency_info, test_mode): +def _collect_own_and_dependency_cc_info(target, rule, test_mode): + dependency_info = _get_cc_toolchain_dependency_info(rule) compilation_context = target[CcInfo].compilation_context cc_toolchain_info = None test_mode_cc_src_deps = depset() @@ -468,9 +717,14 @@ def _collect_own_and_dependency_cc_info(target, dependency_info, test_mode): transitive_defines = compilation_context.defines.to_list(), transitive_include_directory = compilation_context.includes.to_list(), transitive_quote_include_directory = compilation_context.quote_includes.to_list(), - transitive_system_include_directory = compilation_context.system_includes.to_list() + compilation_context.external_includes.to_list(), + transitive_system_include_directory = ( + compilation_context.system_includes.to_list() + ( + # external_includes was added in newer versions of bazel + compilation_context.external_includes.to_list() if hasattr(compilation_context, "external_includes") else [] + ) + ), framework_include_directory = compilation_context.framework_includes.to_list(), - gen_headers = [f.path for f in gen_headers.to_list()], + gen_headers = gen_headers.to_list(), toolchain_id = cc_toolchain_info.id if cc_toolchain_info else None, ) return struct( @@ -489,7 +743,7 @@ def _collect_dependencies_core_impl( generate_aidl_classes, use_generated_srcjars, test_mode): - dep_infos = _collect_java_dependencies_core_impl( + java_dep_info = _collect_java_dependencies_core_impl( target, ctx, include, @@ -499,11 +753,13 @@ def _collect_dependencies_core_impl( use_generated_srcjars, test_mode, ) + cc_dep_info = None if CcInfo in target: - dep_infos.append(_collect_cc_dependencies_core_impl(target, ctx, test_mode)) + cc_dep_info = _collect_cc_dependencies_core_impl(target, ctx, test_mode) + cc_toolchain_dep_info = None if cc_common.CcToolchainInfo in target: - dep_infos.append(_collect_cc_toolchain_info(target, ctx)) - return dep_infos + cc_toolchain_dep_info = _collect_cc_toolchain_info(target, ctx) + return merge_dependencies_info(target, ctx, java_dep_info, cc_dep_info, cc_toolchain_dep_info) def _collect_java_dependencies_core_impl( target, @@ -514,7 +770,7 @@ def _collect_java_dependencies_core_impl( generate_aidl_classes, use_generated_srcjars, test_mode): - target_is_within_project_scope = _target_within_project_scope(str(target.label), include, exclude) and not test_mode + target_is_within_project_scope = _target_within_project_scope(target.label, include, exclude) and not test_mode dependency_infos = _get_followed_java_dependency_infos(ctx.rule) target_to_artifacts, compile_jars, aars, gensrcs = _collect_own_and_dependency_java_artifacts( @@ -529,10 +785,11 @@ def _collect_java_dependencies_core_impl( test_mode_own_files = None if test_mode: + can_follow_dependencies = _can_follow_dependencies(ctx) within_scope_own_files = _collect_own_java_artifacts( target, ctx, - dependency_infos, + can_follow_dependencies, always_build_rules, generate_aidl_classes, use_generated_srcjars, @@ -549,24 +806,20 @@ def _collect_java_dependencies_core_impl( if "ij-ignore-source-transform" in ctx.rule.attr.tags: expand_sources = True - return [ - create_dependencies_info( - target_to_artifacts = target_to_artifacts, - compile_time_jars = compile_jars, - aars = aars, - gensrcs = gensrcs, - expand_sources = expand_sources, - test_mode_own_files = test_mode_own_files, - ), - ] + return create_java_dependencies_info( + target_to_artifacts = target_to_artifacts, + compile_time_jars = compile_jars, + aars = aars, + gensrcs = gensrcs, + expand_sources = expand_sources, + test_mode_own_files = test_mode_own_files, + ) def _collect_cc_dependencies_core_impl(target, ctx, test_mode): - dependency_info = _get_followed_cc_dependency_info(ctx.rule) - - cc_info = _collect_own_and_dependency_cc_info(target, dependency_info, test_mode) + cc_info = _collect_own_and_dependency_cc_info(target, ctx.rule, test_mode) - return create_dependencies_info( - cc_info = cc_info.compilation_info, + return create_cc_dependencies_info( + cc_compilation_info = cc_info.compilation_info, cc_headers = cc_info.gen_headers, cc_toolchain_info = cc_info.cc_toolchain_info, test_mode_cc_src_deps = cc_info.test_mode_cc_src_deps, @@ -607,12 +860,12 @@ def _collect_cc_toolchain_info(target, ctx): ) c_options = cc_common.get_memory_inefficient_command_line( feature_configuration = feature_config, - action_name = C_COMPILE_ACTION_NAME, + action_name = IDE_CC.c_compile_action_name, variables = c_variables, ) cpp_options = cc_common.get_memory_inefficient_command_line( feature_configuration = feature_config, - action_name = CPP_COMPILE_ACTION_NAME, + action_name = IDE_CC.cpp_compile_action_name, variables = cpp_variables, ) toolchain_id = str(target.label) + "%" + toolchain_info.target_gnu_system_name @@ -637,7 +890,7 @@ def _collect_cc_toolchain_info(target, ctx): ), ) - return create_dependencies_info( + return create_cc_toolchain_info( cc_toolchain_info = struct(file = cc_toolchain_file, id = toolchain_id), test_mode_cc_src_deps = depset([f for f in toolchain_info.all_files.to_list() if f.is_source]), ) @@ -716,40 +969,27 @@ def _package_ide_aar(ctx, aar, file_map): arguments = ["c", aar.path] + files_map_args, ) -def _output_relative_path(path): - """Get file path relative to the output path. - - Args: - path: path of artifact path = (../repo_name)? + (root_fragment)? + relative_path - - Returns: - path relative to the output path - """ - if (path.startswith("blaze-out/")) or (path.startswith("bazel-out/")): - # len("blaze-out/") or len("bazel-out/") - path = path[10:] - return path - # List of tuples containing: # 1. An attribute for the aspect to traverse # 2. A list of rule kinds to specify which rules for which the attribute labels # need to be added as dependencies. If empty, the attribute is followed for # all rules. -FOLLOW_JAVA_ATTRIBUTES_BY_RULE_KIND = [ - ("deps", []), - ("exports", []), - ("srcs", []), - ("_junit", []), - ("_aspect_proto_toolchain_for_javalite", []), - ("_aspect_java_proto_toolchain", []), - ("runtime", ["proto_lang_toolchain", "java_rpc_toolchain"]), - ("_toolchain", ["_java_grpc_library", "_java_lite_grpc_library", "kt_jvm_library_helper", "android_library", "kt_android_library"]), - ("kotlin_libs", ["kt_jvm_toolchain"]), -] +FOLLOW_JAVA_ATTRIBUTES = [ + "deps", + "exports", + "srcs", + "_junit", + "_aspect_proto_toolchain_for_javalite", + "_aspect_java_proto_toolchain", +] + IDE_KOTLIN.follow_attributes + +FOLLOW_CC_ATTRIBUTES = IDE_CC.follow_attributes -FOLLOW_CC_ATTRIBUTES = ["_cc_toolchain"] +FOLLOW_ADDITIONAL_ATTRIBUTES = ["runtime", "_toolchain"] + IDE_KOTLIN.follow_additional_attributes -FOLLOW_ATTRIBUTES = [attr for (attr, _) in FOLLOW_JAVA_ATTRIBUTES_BY_RULE_KIND] + FOLLOW_CC_ATTRIBUTES +FOLLOW_ATTRIBUTES = _unique(FOLLOW_JAVA_ATTRIBUTES + FOLLOW_CC_ATTRIBUTES + FOLLOW_ADDITIONAL_ATTRIBUTES) + +TOOLCHAINS_ASPECTS = IDE_KOTLIN.toolchains_aspects + IDE_CC.toolchains_aspects collect_dependencies = aspect( implementation = _collect_dependencies_impl, @@ -780,10 +1020,13 @@ collect_dependencies = aspect( allow_files = True, cfg = "exec", executable = True, - default = "@@bazel_tools//tools/zip:zipper", + default = ZIP_TOOL_LABEL, ), }, fragments = ["cpp"], + **{ + "toolchains_aspects": TOOLCHAINS_ASPECTS, + } if TOOLCHAINS_ASPECTS else {} ) collect_all_dependencies_for_tests = aspect( @@ -803,8 +1046,72 @@ collect_all_dependencies_for_tests = aspect( allow_files = True, cfg = "exec", executable = True, - default = "@@bazel_tools//tools/zip:zipper", + default = ZIP_TOOL_LABEL, ), }, fragments = ["cpp"], ) + +def _write_java_info_txt_impl(ctx): + info_txt_files = [] + for dep in ctx.attr.deps: + info_txt_files.extend(_write_java_target_info(dep, ctx, custom_prefix = ctx.label.name + ".")) + return DefaultInfo(files = depset(info_txt_files)) + +def collect_dependencies_aspect_for_tests(custom_aspect_impl): + """Creates a custom aspect for use in test code. + + This is used to run the `collect_dependencies` aspect with a custom set of project includes. + It's necessary to create a custom aspect to do this, as when invoking as aspect from a build + rule it's not possible to give arbitrary values to their parameters. + + This is necessary as bazel requires that all aspects are assigned to top level build vars. + + Args: + custom_aspect_impl: A method that invokes `collect_dependencies_for_test` passing the + required set of project includes. This should be defined thus: + + def custom_aspect_impl(target, ctx): + return collect_dependencies_for_test(target, ctx, include=[ + "//package/path/to/include", + ]) + + The `includes` are bazel packages corresponding to the `directories` in a `.bazelproject`. + + Returns: + An aspect for use with `write_java_info_txt_rule_for_tests`. + """ + return aspect( + implementation = custom_aspect_impl, + attr_aspects = FOLLOW_ATTRIBUTES, + provides = [DependenciesInfo], + ) + +def write_java_info_txt_rule_for_tests(aspect_name): + """Create a rule to run the aspect for use in test code. + + This is used in conjunction with `collect_dependencies_aspect_for_tests` to run the collect + dependencies aspect and write the resulting `.java-info.txt` files to an artifact. + + Args: + aspect_name: The name that the return value from `write_java_info_txt_aspect_for_tests` + as written to. + + Returns: + A custom run implementation that should be assigned to a variable inside a `.bzl` file + and them subsequently used thus: + + custom_aspect_rule( + name = "java_info", + deps = [":my_target"], + ) + + This will run the aspect as if an "enable analysis" action was run from the IDE on the + targets given in `deps`. + """ + return rule( + implementation = _write_java_info_txt_impl, + attrs = { + "deps": attr.label_list(aspects = [aspect_name]), + }, + ) diff --git a/aspect/build_dependencies_deps.bzl b/aspect/build_dependencies_deps.bzl new file mode 100644 index 00000000000..05d8c5b1cdc --- /dev/null +++ b/aspect/build_dependencies_deps.bzl @@ -0,0 +1,79 @@ +"""Loads and re-exports dependencies of build_dependencies.bzl to support different versions of bazel""" + +load( + "@bazel_tools//tools/build_defs/cc:action_names.bzl", + _CPP_COMPILE_ACTION_NAME = "CPP_COMPILE_ACTION_NAME", + _C_COMPILE_ACTION_NAME = "C_COMPILE_ACTION_NAME", +) + +ZIP_TOOL_LABEL = "@bazel_tools@bazel_tools//tools/zip:zipper" + +ANDROID_IDE_INFO = None + +# JAVA + +def _get_java_info( + target, + rule): # @unused + if not JavaInfo in target: + return None + p = target[JavaInfo] + return struct( + compile_jars = p.compile_jars, + transitive_compile_time_jars = p.transitive_compile_time_jars, + java_outputs = p.java_outputs, + ) + +IDE_JAVA = struct( + srcs_attributes = ["java_srcs", "java_test_srcs"], + get_java_info = _get_java_info, +) + +# KOTLIN + +def _get_dependency_attribute(rule, attr): + if hasattr(rule.attr, attr): + to_add = getattr(rule.attr, attr) + if type(to_add) == "list": + return [t for t in to_add if type(t) == "Target"] + elif type(to_add) == "Target": + return [to_add] + return [] + +def _get_followed_kotlin_dependencies(rule): + deps = [] + if rule.kind in ["kt_jvm_library_helper", "kt_android_library", "android_library"]: + deps.extend(_get_dependency_attribute(rule, "_toolchain")) + if rule.kind in ["kt_jvm_toolchain"]: + deps.extend(_get_dependency_attribute(rule, "kotlin_libs")) + return deps + +IDE_KOTLIN = struct( + srcs_attributes = [ + "kotlin_srcs", + "kotlin_test_srcs", + "common_srcs", + ], + follow_attributes = [], + follow_additional_attributes = [ + "_toolchain", + "kotlin_libs", + ], + followed_dependencies = _get_followed_kotlin_dependencies, + toolchains_aspects = [], +) + +# CC + +def _get_cc_toolchain_target(rule): + if hasattr(rule.attr, "_cc_toolchain"): + return getattr(rule.attr, "_cc_toolchain") + return None + +IDE_CC = struct( + c_compile_action_name = _C_COMPILE_ACTION_NAME, + cpp_compile_action_name = _CPP_COMPILE_ACTION_NAME, + follow_attributes = ["_cc_toolchain"], + toolchains_aspects = [], + toolchain_target = _get_cc_toolchain_target, +) diff --git a/aspect/fast_build_info.bzl b/aspect/fast_build_info.bzl deleted file mode 100644 index c764d67e5e1..00000000000 --- a/aspect/fast_build_info.bzl +++ /dev/null @@ -1,148 +0,0 @@ -"""An aspect to gather info needed by the FastBuildService.""" - -load("@rules_java//java:defs.bzl", "JavaInfo", "java_common") -load( - ":artifacts.bzl", - "artifact_location", - "sources_from_target", - "struct_omit_none", -) -load( - ":intellij_info_impl.bzl", - "stringify_label", -) - -_DEP_ATTRS = ["deps", "exports", "runtime_deps", "_java_toolchain"] - -def _get_android_ide_info(target): - if hasattr(android_common, "AndroidIdeInfo") and android_common.AndroidIdeInfo in target: - return target[android_common.AndroidIdeInfo] - if hasattr(target, "android"): - return target.android - return None - -def _fast_build_info_impl(target, ctx): - dep_targets = _get_all_dep_targets(target, ctx) - dep_outputs = _get_all_dep_outputs(dep_targets) - - output_files = [] - - info = { - "workspace_name": ctx.workspace_name, - "label": stringify_label(target.label), - "dependencies": [stringify_label(t.label) for t in dep_targets], - } - - write_output = False - if hasattr(ctx.rule.attr, "data") and ctx.rule.attr.data: - # The data attribute can reference artifacts directly (like deploy jars) that the aspect - # will skip. So we need to gather them up here, in the referencing target. - write_output = True - info["data"] = [ - struct( - label = stringify_label(datadep.label), - artifacts = [artifact_location(file) for file in datadep.files.to_list()], - ) - for datadep in ctx.rule.attr.data - ] - - if hasattr(target, "java_toolchain"): - toolchain = target.java_toolchain - elif java_common.JavaToolchainInfo != platform_common.ToolchainInfo and \ - java_common.JavaToolchainInfo in target: - toolchain = target[java_common.JavaToolchainInfo] - else: - toolchain = None - if toolchain: - write_output = True - javac_jars = [] - if hasattr(toolchain, "tools"): - javac_jars = [artifact_location(f) for f in toolchain.tools.to_list()] - bootclasspath_jars = [] - if hasattr(toolchain, "bootclasspath"): - bootclasspath_jars = [artifact_location(f) for f in toolchain.bootclasspath.to_list()] - info["java_toolchain_info"] = struct_omit_none( - javac_jars = javac_jars, - bootclasspath_jars = bootclasspath_jars, - source_version = toolchain.source_version, - target_version = toolchain.target_version, - ) - if JavaInfo in target: - write_output = True - launcher = None - if hasattr(ctx.rule.attr, "use_launcher") and not ctx.rule.attr.use_launcher: - launcher = None - elif hasattr(ctx.rule.attr, "launcher") and ctx.rule.attr.launcher: - launcher = stringify_label(ctx.rule.attr.launcher.label) - elif hasattr(ctx.rule.attr, "_java_launcher") and ctx.rule.attr._java_launcher: - # TODO: b/295221112 - remove _java_launcher when it's removed from Java rules - launcher = stringify_label(ctx.rule.attr._java_launcher.label) - elif hasattr(ctx.rule.attr, "_javabase") and ctx.rule.attr._javabase: - launcher = stringify_label(ctx.rule.attr._javabase.label) - java_info = { - "sources": sources_from_target(ctx), - "test_class": getattr(ctx.rule.attr, "test_class", None), - "test_size": getattr(ctx.rule.attr, "size", None), - "launcher": launcher, - "swigdeps": getattr(ctx.rule.attr, "swigdeps", True), - "jvm_flags": getattr(ctx.rule.attr, "jvm_flags", []), - } - annotation_processing = target[JavaInfo].annotation_processing - if annotation_processing: - java_info["annotation_processor_class_names"] = annotation_processing.processor_classnames - java_info["annotation_processor_classpath"] = [ - artifact_location(t) - for t in annotation_processing.processor_classpath.to_list() - ] - info["java_info"] = struct_omit_none(**java_info) - - android_ide_info = _get_android_ide_info(target) - if android_ide_info: - write_output = True - android_info = struct_omit_none( - aar = artifact_location(android_ide_info.aar), - merged_manifest = artifact_location( - getattr(android_ide_info, "generated_manifest", None) or - getattr(android_ide_info, "merged_manifest", None), - ), - ) - info["android_info"] = android_info - - if write_output: - output_file = ctx.actions.declare_file(target.label.name + ".ide-fast-build-info.txt") - ctx.actions.write(output_file, proto.encode_text(struct_omit_none(**info))) - output_files += [output_file] - - output_groups = depset(output_files, transitive = dep_outputs) - - return struct(output_groups = {"ide-fast-build": output_groups}) - -def _get_all_dep_outputs(dep_targets): - """Get the ide-fast-build output files for all dependencies""" - return [ - dep_target[OutputGroupInfo]["ide-fast-build"] - for dep_target in dep_targets - if _has_ide_fast_build(dep_target) - ] - -def _get_all_dep_targets(target, ctx): - """Get all the targets mentioned in one of the _DEP_ATTRS attributes of the target""" - targets = [] - for attr_name in _DEP_ATTRS: - attr_val = getattr(ctx.rule.attr, attr_name, None) - if not attr_val: - continue - attr_type = type(attr_val) - if attr_type == type(target): - targets.append(attr_val) - elif attr_type == type([]): - targets += [list_val for list_val in attr_val if type(list_val) == type(target)] - return targets - -def _has_ide_fast_build(target): - return OutputGroupInfo in target and "ide-fast-build" in target[OutputGroupInfo] - -fast_build_info_aspect = aspect( - attr_aspects = _DEP_ATTRS, - implementation = _fast_build_info_impl, -) diff --git a/aspect/java_classpath.bzl b/aspect/java_classpath.bzl index 1a47bd9448a..be3dfb648c6 100644 --- a/aspect/java_classpath.bzl +++ b/aspect/java_classpath.bzl @@ -1,7 +1,5 @@ """An aspect which extracts the runtime classpath from a java target.""" -load("@rules_java//java:defs.bzl", "JavaInfo") - def _runtime_classpath_impl(target, ctx): """The top level aspect implementation function. diff --git a/aspect/tools/BUILD b/aspect/tools/BUILD index cfefd3a5579..0810f91ccbe 100644 --- a/aspect/tools/BUILD +++ b/aspect/tools/BUILD @@ -3,7 +3,9 @@ # Tools needed by the bazel plugin's aspect. # -load("@rules_java//java:defs.bzl", "java_binary", "java_library", "java_test") +load("@rules_java//java:java_binary.bzl", "java_binary") +load("@rules_java//java:java_library.bzl", "java_library") +load("@rules_java//java:java_test.bzl", "java_test") package(default_visibility = ["//aspect:__pkg__"]) @@ -24,6 +26,7 @@ java_library( deps = [ ":guava", "//proto:proto_deps", + "@com_google_protobuf//:protobuf_java", "@jsr305_annotations//jar", ], ) @@ -84,6 +87,7 @@ java_test( test_class = "com.google.idea.blaze.aspect.PackageParserTest", deps = [ ":test_lib", + "@com_google_protobuf//:protobuf_java", "@error_prone_annotations//jar", ], )