Skip to content

Commit

Permalink
remove old batch size example
Browse files Browse the repository at this point in the history
Signed-off-by: Yee Hing Tong <[email protected]>
  • Loading branch information
wild-endeavor committed Dec 9, 2024
1 parent aba1579 commit 123271e
Showing 1 changed file with 0 additions and 33 deletions.
33 changes: 0 additions & 33 deletions examples/data_types_and_io/data_types_and_io/pickle_type.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,39 +26,6 @@ def superhero_wf(name: str = "Thor", power: str = "Flight") -> str:
return greet_superhero(superhero=superhero)


# Batch size
# By default, if the list subtype is unrecognized, a single pickle file is generated.
# To optimize serialization and deserialization performance for scenarios involving a large number of items
# or significant list elements, you can specify a batch size.
# This feature allows for the processing of each batch as a separate pickle file.
# The following example demonstrates how to set the batch size.
from typing import Iterator

from flytekit.types.pickle.pickle import BatchSize
from typing_extensions import Annotated


@task
def welcome_superheroes(names: list[str], powers: list[str]) -> Annotated[list[Superhero], BatchSize(3)]:
return [Superhero(name, power) for name, power in zip(names, powers)]


@task
def greet_superheroes(superheroes: list[Superhero]) -> Iterator[str]:
for superhero in superheroes:
yield f"👋 Hello {superhero.name}! Your superpower is {superhero.power}."


@workflow
def superheroes_wf(
names: list[str] = ["Thor", "Spiderman", "Hulk"],
powers: list[str] = ["Flight", "Surface clinger", "Shapeshifting"],
) -> Iterator[str]:
superheroes = welcome_superheroes(names=names, powers=powers)
return greet_superheroes(superheroes=superheroes)


# Run the workflow locally
if __name__ == "__main__":
print(f"Superhero wf: {superhero_wf()}")
print(f"Superhero(es) wf: {superheroes_wf()}")

0 comments on commit 123271e

Please sign in to comment.