Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: subprocess build #3814

Merged
merged 6 commits into from
May 3, 2023
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/bentoml/_internal/bento/bento.py
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ def create(
model.save(bento_model_store)

# create ignore specs
specs = BentoPathSpec(build_config.include, build_config.exclude) # type: ignore (unfinished attrs converter type)
specs = BentoPathSpec(build_config.include, build_config.exclude)

# Copy all files base on include and exclude, into `src` directory
relpaths = [s for s in build_config.include if s.startswith("../")]
Expand Down
9 changes: 6 additions & 3 deletions src/bentoml/_internal/bento/build_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -835,9 +835,12 @@ def from_yaml(cls, stream: t.TextIO) -> BentoBuildConfig:
raise InvalidArgument(str(e)) from e

def to_yaml(self, stream: t.TextIO) -> None:
# TODO: Save BentoBuildOptions to a yaml file
# This is reserved for building interactive build file creation CLI
raise NotImplementedError
try:
yaml.dump(bentoml_cattr.unstructure(self), stream)
except yaml.YAMLError as e:
logger.error("Error while deserializing BentoBuildConfig to yaml:")
logger.error(e)
aarnphm marked this conversation as resolved.
Show resolved Hide resolved
raise


@attr.define(frozen=True)
Expand Down
62 changes: 45 additions & 17 deletions src/bentoml/bentos.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,11 @@

from __future__ import annotations

import os
import typing as t
import logging
import tempfile
import subprocess

from simple_di import inject
from simple_di import Provide
Expand Down Expand Up @@ -273,7 +276,7 @@ def build(
version: str | None = None,
build_ctx: str | None = None,
_bento_store: BentoStore = Provide[BentoMLContainer.bento_store],
) -> "Bento":
) -> Bento:
"""
User-facing API for building a Bento. The available build options are identical to the keys of a
valid 'bentofile.yaml' file.
Expand Down Expand Up @@ -349,21 +352,39 @@ def build(
conda=conda,
)

return Bento.create(
build_config=build_config,
version=version,
build_ctx=build_ctx,
).save(_bento_store)
build_args = ["bentoml", "build"]

if build_ctx is None:
build_ctx = "."
build_args.append(build_ctx)

if version is not None:
build_args.extend(["--version", version])
build_args.extend(["--output", "tag"])

with tempfile.NamedTemporaryFile(
"w", encoding="utf-8", prefix="bentoml-build-", suffix=".yaml"
) as f:
build_config.to_yaml(f)
bentofile_path = os.path.join(os.path.dirname(f.name), f.name)
build_args.extend(["--bentofile", bentofile_path])
try:
output = subprocess.check_output(build_args)
except subprocess.CalledProcessError as e:
logger.error("Failed to build BentoService bundle: %s", e)
raise

return get(output.decode("utf-8").strip().split("\n")[-1])


@inject
def build_bentofile(
bentofile: str = "bentofile.yaml",
*,
version: t.Optional[str] = None,
build_ctx: t.Optional[str] = None,
_bento_store: "BentoStore" = Provide[BentoMLContainer.bento_store],
) -> "Bento":
version: str | None = None,
build_ctx: str | None = None,
_bento_store: BentoStore = Provide[BentoMLContainer.bento_store],
) -> Bento:
"""
Build a Bento base on options specified in a bentofile.yaml file.

Expand All @@ -381,14 +402,21 @@ def build_bentofile(
except FileNotFoundError:
raise InvalidArgument(f'bentofile "{bentofile}" not found')

with open(bentofile, "r", encoding="utf-8") as f:
build_config = BentoBuildConfig.from_yaml(f)
build_args = ["bentoml", "build"]
if build_ctx is None:
build_ctx = "."
build_args.append(build_ctx)
if version is not None:
build_args.extend(["--version", version])
build_args.extend(["--bentofile", bentofile, "--output", "tag"])

return Bento.create(
build_config=build_config,
version=version,
build_ctx=build_ctx,
).save(_bento_store)
try:
output = subprocess.check_output(build_args)
except subprocess.CalledProcessError as e:
logger.error("Failed to build BentoService bundle: %s", e)
raise

return get(output.decode("utf-8").strip().split("\n")[-1])
aarnphm marked this conversation as resolved.
Show resolved Hide resolved


def containerize(bento_tag: Tag | str, **kwargs: t.Any) -> bool:
Expand Down
87 changes: 72 additions & 15 deletions src/bentoml_cli/bentos.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@

import yaml
import click
from simple_di import inject
from simple_di import Provide
from rich.table import Table
from rich.syntax import Syntax

Expand All @@ -17,6 +19,8 @@
from click import Context
from click import Parameter

from bentoml._internal.bento import BentoStore

BENTOML_FIGLET = """
██████╗░███████╗███╗░░██╗████████╗░█████╗░███╗░░░███╗██╗░░░░░
██╔══██╗██╔════╝████╗░██║╚══██╔══╝██╔══██╗████╗░████║██║░░░░░
Expand Down Expand Up @@ -45,14 +49,18 @@ def parse_delete_targets_argument_callback(


def add_bento_management_commands(cli: Group):
import bentoml
from bentoml import Tag
from bentoml.bentos import import_bento
from bentoml.bentos import build_bentofile
from bentoml._internal.utils import rich_console as console
from bentoml._internal.utils import calc_dir_size
from bentoml._internal.utils import human_readable_size
from bentoml._internal.utils import resolve_user_filepath
from bentoml._internal.utils import display_path_under_home
from bentoml._internal.bento.bento import Bento
from bentoml._internal.bento.bento import DEFAULT_BENTO_BUILD_FILE
from bentoml._internal.configuration import get_quiet_mode
from bentoml._internal.bento.build_config import BentoBuildConfig
from bentoml._internal.configuration.containers import BentoMLContainer

bento_store = BentoMLContainer.bento_store.get()
Expand Down Expand Up @@ -278,22 +286,71 @@ def push(bento_tag: str, force: bool, threads: int, context: str) -> None: # ty
@cli.command()
@click.argument("build_ctx", type=click.Path(), default=".")
@click.option(
"-f", "--bentofile", type=click.STRING, default=DEFAULT_BENTO_BUILD_FILE
"-f",
"--bentofile",
type=click.STRING,
default=DEFAULT_BENTO_BUILD_FILE,
help="Path to bentofile. Default to 'bentofile.yaml'",
)
@click.option(
"--version",
type=click.STRING,
default=None,
help="Bento version. By default the version will be generated.",
)
@click.option("--version", type=click.STRING, default=None)
def build(build_ctx: str, bentofile: str, version: str) -> None: # type: ignore (not accessed)
@click.option(
"-o",
"--output",
type=click.Choice(["tag", "default"]),
default="default",
show_default=True,
help="Output log format. '-o tag' to display only bento tag.",
)
@inject
def build( # type: ignore (not accessed)
build_ctx: str,
bentofile: str,
version: str,
output: t.Literal["tag", "default"],
_bento_store: BentoStore = Provide[BentoMLContainer.bento_store],
) -> None:
"""Build a new Bento from current directory."""
if sys.path[0] != build_ctx:
sys.path.insert(0, build_ctx)

bento = build_bentofile(bentofile, build_ctx=build_ctx, version=version)
click.echo(BENTOML_FIGLET)
click.secho(f"Successfully built {bento}.", fg="green")
click.secho(
f"\nPossible next steps:\n\n * Containerize your Bento with `bentoml containerize`:\n $ bentoml containerize {bento.tag}",
fg="yellow",
)
click.secho(
f"\n * Push to BentoCloud with `bentoml push`:\n $ bentoml push {bento.tag}",
fg="yellow",
)
try:
bentofile = resolve_user_filepath(bentofile, build_ctx)
except FileNotFoundError:
raise bentoml.exceptions.InvalidArgument(
f'bentofile "{bentofile}" not found'
)

with open(bentofile, "r", encoding="utf-8") as f:
build_config = BentoBuildConfig.from_yaml(f)

bento = Bento.create(
build_config=build_config,
version=version,
build_ctx=build_ctx,
).save(_bento_store)

if output == "tag":
click.echo(bento.tag)
return

if not get_quiet_mode():
click.echo(BENTOML_FIGLET)
click.secho(f"Successfully built {bento}.", fg="green")

click.secho(
f"\nPossible next steps:\n\n * Containerize your Bento with `bentoml containerize`:\n $ bentoml containerize {bento.tag}",
fg="blue",
)
click.secho(
f"\n * Push to BentoCloud with `bentoml push`:\n $ bentoml push {bento.tag}",
fg="blue",
)

# We need to pop the build_ctx from sys.path to avoid conflicts.
if sys.path[0] == build_ctx:
aarnphm marked this conversation as resolved.
Show resolved Hide resolved
sys.path.pop(0)