Skip to content

Commit

Permalink
feat: add opt-in exports_directories_only mode to yarn_install and np…
Browse files Browse the repository at this point in the history
…m_install (defaults to False)

When `True`, `yarn_install` and `npm_install` export only top-level package directory artifacts from node_modules.

Turning this on will decrease the time it takes for Bazel to setup runfiles and sandboxing when
there are a large number of npm dependencies as inputs to an action.

This breaks compatabilty for label that reference files within npm packages such as `@npm//:node_modules/prettier/bin-prettier.js`.
To reference files within npm packages, you can use the `directory_file_path` rule and/or `DirectoryFilePathInfo` provider.
Note, some rules still need upgrading to support consuming `DirectoryFilePathInfo` where needed.

NB: This feature requires runfiles be enabled due to an issue in Bazel which we are still investigating.
    On Windows runfiles are off by default and must be enabled with the `--enable_runfiles` flag when
    using this feature.

NB: `ts_library` does not support directory artifact npm deps due to internal dependency on having all input sources files explicitly specified

NB: `karma_web_test` and `karma_web_test_suite` do not yet support directory artifact npm deps as they require `node_modules/requirejs/require.js` & `node_modules/karma-requirejs/lib/adapter.js` explicit source files in deps

For the `nodejs_binary` & `nodejs_test` `entry_point` attribute (which often needs to reference a file within
an npm package) you can set the entry_point to a dict with a single entry, where the key corresponds to the directory artifact
label and the value corresponds to the path within that directory to the entry point.

For example,

```
nodejs_binary(
    name = "prettier",
    data = ["@npm//prettier"],
    entry_point = "@npm//:node_modules/prettier/bin-prettier.js",
)
```

becomes,

```
nodejs_binary(
    name = "prettier",
    data = ["@npm//prettier"],
    entry_point = { "@npm//:node_modules/prettier": "bin-prettier.js" },
)
```

For other labels that are passed to `$(rootpath)`, `$(execpath)`, or `$(location)` you can simply break these apart into
the directory artifact label that gets passed to the expander & path part to follows it.

For example,

```
$(rootpath @npm//:node_modules/prettier/bin-prettier.js")
```

becomes,

```
$(rootpath @npm//:node_modules/prettier)/bin-prettier.js
```
  • Loading branch information
Greg Magolan authored and alexeagle committed Jun 9, 2021
1 parent 51676ef commit 3d182cf
Show file tree
Hide file tree
Showing 93 changed files with 3,386 additions and 135 deletions.
27 changes: 18 additions & 9 deletions .bazelci/presubmit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -382,65 +382,74 @@ tasks:
# Disabled due to https://github.com/bazelbuild/rules_nodejs/issues/1486
#- "@fine_grained_deps_yarn//typescript/bin:tsc"
build_flags:
- "--build_tag_filters=-e2e,-examples,-fix-windows,-manual"
- "--build_tag_filters=-e2e,-examples,-manual,-fix-windows,-no-bazelci-windows,-requires-runfiles"
build_targets:
- "//..."
test_flags:
# Firefox not supported on Windows with rules_webtesting (if run it exit with success)
# See https://github.com/bazelbuild/rules_webtesting/blob/0.3.3/browsers/BUILD.bazel#L66.
- "--test_tag_filters=-e2e,-examples,-fix-windows,-no-bazelci-windows,-manual,-browser:firefox-local,-cypress"
- "--test_tag_filters=-e2e,-examples,-fix-windows,-no-bazelci-windows,-requires-runfiles,-manual,-browser:firefox-local,-cypress"
test_targets:
- "//..."
# //internal/node/test:nodejs_toolchain_windows_amd64_test is a "manual" test that must be run
# explicitly; it should pass when running on Windows with no --platform set.
- "//internal/node/test:nodejs_toolchain_windows_amd64_test"
windows_runfiles_enabled:
name: windows_runfiles_enabled
platform: windows
test_flags:
- "--test_tag_filters=requires-runfiles"
- "--enable_runfiles"
test_targets:
- "//..."
windows_e2e:
name: windows_e2e
platform: windows
build_flags:
- "--build_tag_filters=e2e,-fix-windows"
- "--build_tag_filters=e2e,-fix-windows,-no-bazelci-windows,-requires-runfiles"
build_targets:
- "//..."
# We control Bazel version in integration tests, so we don't need USE_BAZEL_VERSION for tests.
skip_use_bazel_version_for_test: true
test_flags:
- "--test_tag_filters=e2e,-fix-windows,-no-bazelci-windows"
- "--test_tag_filters=e2e,-fix-windows,-no-bazelci-windows,-requires-runfiles"
- "--local_ram_resources=792"
# test_args will be passed to the nested bazel process
- "--test_arg=--local_ram_resources=13288"
# Firefox not supported on Windows with rules_webtesting (if run it exit with success)
# See https://github.com/bazelbuild/rules_webtesting/blob/0.3.3/browsers/BUILD.bazel#L66.
# Chrome fails to launch on Windows inside bazel-in-bazel with: [17:12:04] E/launcher -
# spawn D:\...\external\io_bazel_rules_webtesting\third_party\chromedriver\chromedriver.out\chromedriver_win32\chromedriver.exe ENOENT
- "--test_arg=--test_tag_filters=-fix-windows,-manual,-browser:chromium-local,-browser:firefox-local"
- "--test_arg=--test_tag_filters=-fix-windows,-no-bazelci-windows,-requires-runfiles,-manual,-browser:chromium-local,-browser:firefox-local"
test_targets:
- "//..."
windows_examples:
name: windows_examples
platform: windows
build_flags:
- "--build_tag_filters=examples,-fix-windows"
- "--build_tag_filters=examples,-fix-windows,-no-bazelci-windows,-requires-runfiles"
build_targets:
- "//..."
# We control Bazel version in integration tests, so we don't need USE_BAZEL_VERSION for tests.
skip_use_bazel_version_for_test: true
test_flags:
- "--test_tag_filters=examples,-manual,-fix-windows,-no-bazelci-windows,-cypress"
- "--test_tag_filters=examples,-manual,-fix-windows,-no-bazelci-windows,-requires-runfiles,-cypress"
- "--local_ram_resources=792"
# test_args will be passed to the nested bazel process
- "--test_arg=--local_ram_resources=13288"
# Firefox not supported on Windows with rules_webtesting (if run it exit with success)
# See https://github.com/bazelbuild/rules_webtesting/blob/0.3.3/browsers/BUILD.bazel#L66.
# Chrome fails to launch on Windows inside bazel-in-bazel with: [17:12:04] E/launcher -
# spawn D:\...\external\io_bazel_rules_webtesting\third_party\chromedriver\chromedriver.out\chromedriver_win32\chromedriver.exe ENOENT
- "--test_arg=--test_tag_filters=-no-bazelci-windows,-fix-windows,-manual,-browser:chromium-local,-browser:firefox-local,-cypress"
- "--test_arg=--test_tag_filters=-fix-windows,-no-bazelci-windows,-requires-runfiles,-manual,-browser:chromium-local,-browser:firefox-local,-cypress"
test_targets:
- "//..."
windows_cross_compile_linux:
name: windows_cross_compile_linux
platform: windows
# Build on windows with the node --platform set to linux
build_flags:
- "--build_tag_filters=-fix-windows,-no-bazelci-windows,-requires-runfiles"
# Build on windows with the node --platform set to linux
- "--platforms=@build_bazel_rules_nodejs//toolchains/node:linux_amd64"
build_targets:
- "//internal/..."
Expand Down
4 changes: 2 additions & 2 deletions .bazelrc
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ import %workspace%/common.bazelrc
# This lets us glob() up all the files inside the examples to make them inputs to tests
# To update these lines, just run `yarn bazel:update-deleted-packages`
# (Note, we cannot use common --deleted_packages because the bazel version command doesn't support it)
build --deleted_packages=examples/angular,examples/angular/e2e,examples/angular/src/app/home,examples/angular/src/app/hello-world,examples/angular/src/app,examples/angular/src/app/todos,examples/angular/src/app/todos/reducers,examples/angular/src/shared/material,examples/angular/src/lib/shorten,examples/angular/src,examples/angular/src/assets,examples/angular/tools,examples/angular_bazel_architect,examples/angular_bazel_architect/projects/frontend-lib,examples/angular_view_engine,examples/angular_view_engine/e2e,examples/angular_view_engine/src/app/home,examples/angular_view_engine/src/app/hello-world,examples/angular_view_engine/src/app,examples/angular_view_engine/src/app/todos,examples/angular_view_engine/src/app/todos/reducers,examples/angular_view_engine/src/shared/material,examples/angular_view_engine/src/lib/typography,examples/angular_view_engine/src/lib/shorten,examples/angular_view_engine/src,examples/angular_view_engine/src/assets,examples/app,examples/app/styles,examples/app/test,examples/closure,examples/create-react-app,examples/cypress,examples/esbuild,examples/esbuild/src,examples/from_source,examples/jest,examples/jest/ts/test,examples/jest/ts,examples/jest/ts/src,examples/kotlin,examples/kotlin/node_modules/@bazel/rollup,examples/kotlin/node_modules/@bazel/terser,examples/nestjs,examples/nestjs/src,examples/parcel,examples/protobufjs,examples/protocol_buffers,examples/react_webpack,examples/user_managed_deps,examples/vendored_node,examples/vendored_node_and_yarn,examples/vue,examples/vue/src,examples/web_testing,examples/webapp,examples/worker,e2e/bazel_managed_deps,e2e/concatjs_devserver,e2e/concatjs_devserver/genrule,e2e/concatjs_devserver/subpackage,e2e/coverage,e2e/fine_grained_symlinks,e2e/jasmine,e2e/node_loader_no_preserve_symlinks,e2e/node_loader_preserve_symlinks,e2e/nodejs_image,e2e/nodejs_image/foolib,e2e/nodejs_repository,e2e/packages,e2e/symlinked_node_modules_npm,e2e/symlinked_node_modules_yarn,e2e/typescript,e2e/webapp
query --deleted_packages=examples/angular,examples/angular/e2e,examples/angular/src/app/home,examples/angular/src/app/hello-world,examples/angular/src/app,examples/angular/src/app/todos,examples/angular/src/app/todos/reducers,examples/angular/src/shared/material,examples/angular/src/lib/shorten,examples/angular/src,examples/angular/src/assets,examples/angular/tools,examples/angular_bazel_architect,examples/angular_bazel_architect/projects/frontend-lib,examples/angular_view_engine,examples/angular_view_engine/e2e,examples/angular_view_engine/src/app/home,examples/angular_view_engine/src/app/hello-world,examples/angular_view_engine/src/app,examples/angular_view_engine/src/app/todos,examples/angular_view_engine/src/app/todos/reducers,examples/angular_view_engine/src/shared/material,examples/angular_view_engine/src/lib/typography,examples/angular_view_engine/src/lib/shorten,examples/angular_view_engine/src,examples/angular_view_engine/src/assets,examples/app,examples/app/styles,examples/app/test,examples/closure,examples/create-react-app,examples/cypress,examples/esbuild,examples/esbuild/src,examples/from_source,examples/jest,examples/jest/ts/test,examples/jest/ts,examples/jest/ts/src,examples/kotlin,examples/kotlin/node_modules/@bazel/rollup,examples/kotlin/node_modules/@bazel/terser,examples/nestjs,examples/nestjs/src,examples/parcel,examples/protobufjs,examples/protocol_buffers,examples/react_webpack,examples/user_managed_deps,examples/vendored_node,examples/vendored_node_and_yarn,examples/vue,examples/vue/src,examples/web_testing,examples/webapp,examples/worker,e2e/bazel_managed_deps,e2e/concatjs_devserver,e2e/concatjs_devserver/genrule,e2e/concatjs_devserver/subpackage,e2e/coverage,e2e/fine_grained_symlinks,e2e/jasmine,e2e/node_loader_no_preserve_symlinks,e2e/node_loader_preserve_symlinks,e2e/nodejs_image,e2e/nodejs_image/foolib,e2e/nodejs_repository,e2e/packages,e2e/symlinked_node_modules_npm,e2e/symlinked_node_modules_yarn,e2e/typescript,e2e/webapp
build --deleted_packages=examples/angular,examples/angular/e2e,examples/angular/src/app/home,examples/angular/src/app/hello-world,examples/angular/src/app,examples/angular/src/app/todos,examples/angular/src/app/todos/reducers,examples/angular/src/shared/material,examples/angular/src/lib/shorten,examples/angular/src,examples/angular/src/assets,examples/angular/tools,examples/angular_bazel_architect,examples/angular_bazel_architect/projects/frontend-lib,examples/angular_view_engine,examples/angular_view_engine/e2e,examples/angular_view_engine/src/app/home,examples/angular_view_engine/src/app/hello-world,examples/angular_view_engine/src/app,examples/angular_view_engine/src/app/todos,examples/angular_view_engine/src/app/todos/reducers,examples/angular_view_engine/src/shared/material,examples/angular_view_engine/src/lib/typography,examples/angular_view_engine/src/lib/shorten,examples/angular_view_engine/src,examples/angular_view_engine/src/assets,examples/app,examples/app/styles,examples/app/test,examples/closure,examples/create-react-app,examples/cypress,examples/esbuild,examples/esbuild/src,examples/from_source,examples/jest,examples/jest/ts/test,examples/jest/ts,examples/jest/ts/src,examples/kotlin,examples/nestjs,examples/nestjs/src,examples/parcel,examples/protobufjs,examples/protocol_buffers,examples/react_webpack,examples/user_managed_deps,examples/vendored_node,examples/vendored_node_and_yarn,examples/vue,examples/vue/src,examples/web_testing,examples/webapp,examples/worker,e2e/bazel_managed_deps,e2e/concatjs_devserver,e2e/concatjs_devserver/genrule,e2e/concatjs_devserver/subpackage,e2e/concatjs_devserver_directory_artifacts,e2e/concatjs_devserver_directory_artifacts/genrule,e2e/concatjs_devserver_directory_artifacts/subpackage,e2e/coverage,e2e/fine_grained_symlinks,e2e/jasmine,e2e/node_loader_no_preserve_symlinks,e2e/node_loader_preserve_symlinks,e2e/nodejs_image,e2e/nodejs_image/foolib,e2e/nodejs_repository,e2e/packages,e2e/symlinked_node_modules_npm,e2e/symlinked_node_modules_yarn,e2e/typescript,e2e/webapp
query --deleted_packages=examples/angular,examples/angular/e2e,examples/angular/src/app/home,examples/angular/src/app/hello-world,examples/angular/src/app,examples/angular/src/app/todos,examples/angular/src/app/todos/reducers,examples/angular/src/shared/material,examples/angular/src/lib/shorten,examples/angular/src,examples/angular/src/assets,examples/angular/tools,examples/angular_bazel_architect,examples/angular_bazel_architect/projects/frontend-lib,examples/angular_view_engine,examples/angular_view_engine/e2e,examples/angular_view_engine/src/app/home,examples/angular_view_engine/src/app/hello-world,examples/angular_view_engine/src/app,examples/angular_view_engine/src/app/todos,examples/angular_view_engine/src/app/todos/reducers,examples/angular_view_engine/src/shared/material,examples/angular_view_engine/src/lib/typography,examples/angular_view_engine/src/lib/shorten,examples/angular_view_engine/src,examples/angular_view_engine/src/assets,examples/app,examples/app/styles,examples/app/test,examples/closure,examples/create-react-app,examples/cypress,examples/esbuild,examples/esbuild/src,examples/from_source,examples/jest,examples/jest/ts/test,examples/jest/ts,examples/jest/ts/src,examples/kotlin,examples/nestjs,examples/nestjs/src,examples/parcel,examples/protobufjs,examples/protocol_buffers,examples/react_webpack,examples/user_managed_deps,examples/vendored_node,examples/vendored_node_and_yarn,examples/vue,examples/vue/src,examples/web_testing,examples/webapp,examples/worker,e2e/bazel_managed_deps,e2e/concatjs_devserver,e2e/concatjs_devserver/genrule,e2e/concatjs_devserver/subpackage,e2e/concatjs_devserver_directory_artifacts,e2e/concatjs_devserver_directory_artifacts/genrule,e2e/concatjs_devserver_directory_artifacts/subpackage,e2e/coverage,e2e/fine_grained_symlinks,e2e/jasmine,e2e/node_loader_no_preserve_symlinks,e2e/node_loader_preserve_symlinks,e2e/nodejs_image,e2e/nodejs_image/foolib,e2e/nodejs_repository,e2e/packages,e2e/symlinked_node_modules_npm,e2e/symlinked_node_modules_yarn,e2e/typescript,e2e/webapp

# Mock versioning command to test the --stamp behavior
build --workspace_status_command="echo BUILD_SCM_VERSION 1.2.3"
Expand Down
2 changes: 1 addition & 1 deletion docs/Built-ins.html

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion docs/Built-ins.md
Original file line number Diff line number Diff line change
Expand Up @@ -1777,7 +1777,7 @@ Defaults to `[]`

The name it will be imported by. Should match the "name" field in the package.json file.

If package_name == "$node_modules$" this indictates that this js_library target is one or more external npm
If package_name is “$node_modules$” or “$node_modules_dir$” this indictates that this js_library target is one or more external npm
packages in node_modules. This is a special case that used be covered by the internal only
`external_npm_package` attribute. NB: '$' is an illegal character
for npm packages names so this reserved name will not conflict with any valid package_name values
Expand Down
15 changes: 15 additions & 0 deletions e2e/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,21 @@ e2e_integration_test(
},
)

e2e_integration_test(
name = "e2e_concatjs_devserver_directory_artifacts",
bazel_commands = [
"info",
# enable runfiles explicitly so it is on for Windows since directory
# artifacts requires runfiles
"test ... --enable_runfiles",
],
npm_packages = {
"//packages/concatjs:npm_package": "@bazel/concatjs",
"//packages/protractor:npm_package": "@bazel/protractor",
"//packages/typescript:npm_package": "@bazel/typescript",
},
)

[e2e_integration_test(
name = "e2e_typescript_%s" % tsc_version.replace(".", "_"),
npm_packages = {
Expand Down
1 change: 1 addition & 0 deletions e2e/concatjs_devserver_directory_artifacts/.bazelignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
node_modules
1 change: 1 addition & 0 deletions e2e/concatjs_devserver_directory_artifacts/.bazelrc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
import %workspace%/../../common.bazelrc
85 changes: 85 additions & 0 deletions e2e/concatjs_devserver_directory_artifacts/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
# Copyright 2017 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
load("@build_bazel_rules_nodejs//internal/npm_install:npm_umd_bundle.bzl", "npm_umd_bundle")
load("@npm//@bazel/concatjs:index.bzl", "concatjs_devserver")
load("@npm//@bazel/protractor:index.bzl", "protractor_web_test_suite")
load("@npm//@bazel/typescript:index.bzl", "ts_project")

exports_files([
"red-body-style.css",
"tsconfig.json",
"protractor.on-prepare.js",
])

ts_project(
name = "compile",
srcs = [
"app.ts",
"app_e2e-spec.ts",
],
declaration = True,
tsconfig = ":tsconfig.json",
validate = False,
deps = [
"@npm//@types",
"@npm//date-fns",
"@npm//protractor",
"@npm//rxjs",
"@npm//typeorm",
],
)

js_library(
name = "lib",
named_module_srcs = [":compile"],
deps = [":compile"],
)

# typeorm is a special case for npm_umd_bundle as we need to excluded
# the `react-native-sqlite-storage` dynamic require from the umd bundle
npm_umd_bundle(
name = "typeorm_umd",
package_name = "typeorm",
entry_point = {"@npm//:node_modules/typeorm": "browser/index.js"},
excluded = ["react-native-sqlite-storage"],
package = "@npm//typeorm",
)

concatjs_devserver(
name = "devserver",
entry_module = "concatjs_devserver_directory_artifacts/app",
scripts = [
"@npm//date-fns:date-fns.umd.js",
"@npm//rxjs:rxjs.umd.js",
":typeorm.umd.js",
],
# We'll collect all the devmode JS sources from these TypeScript libraries
deps = [":lib"],
)

protractor_web_test_suite(
name = "devserver_test",
on_prepare = ":protractor.on-prepare.js",
protractor_entry_point = {"@npm//:node_modules/protractor": "bin/protractor"},
server = ":devserver",
deps = [":lib"],
)

# Just a dummy test so that we have a test target for //... on certain bazelci platforms with bazel_integration_test
sh_test(
name = "dummy_test",
srcs = ["dummy_test.sh"],
)
51 changes: 51 additions & 0 deletions e2e/concatjs_devserver_directory_artifacts/WORKSPACE
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
# Copyright 2017 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

workspace(
name = "concatjs_devserver_directory_artifacts",
managed_directories = {"@npm": ["node_modules"]},
)

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

http_archive(
name = "build_bazel_rules_nodejs",
sha256 = "4a5d654a4ccd4a4c24eca5d319d85a88a650edf119601550c95bf400c8cc897e",
urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/3.5.1/rules_nodejs-3.5.1.tar.gz"],
)

load("@build_bazel_rules_nodejs//:index.bzl", "yarn_install")

yarn_install(
name = "npm",
exports_directories_only = True,
package_json = "//:package.json",
yarn_lock = "//:yarn.lock",
)

load("@npm//@bazel/protractor:package.bzl", "npm_bazel_protractor_dependencies")

npm_bazel_protractor_dependencies()

# Setup the rules_webtesting toolchain
load("@io_bazel_rules_webtesting//web:repositories.bzl", "web_test_repositories")

web_test_repositories()

load("@io_bazel_rules_webtesting//web/versioned:browsers-0.3.2.bzl", "browser_repositories")

browser_repositories(
chromium = True,
firefox = True,
)
Loading

0 comments on commit 3d182cf

Please sign in to comment.