Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add add_bos=False, add_eos=False to SentencePieceTokenizer.__init__() #1811

Merged
merged 4 commits into from
Sep 9, 2024
Merged

Add add_bos=False, add_eos=False to SentencePieceTokenizer.__init__() #1811

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
dad52a9
Add add_bos, add_eos parameters to SentencePieceTokenizer.__init__()
briango28 Sep 5, 2024
cd2994f
Add test_scalar_bos_eos() & test_string_bos_eos() tests to SentencePi…
briango28 Sep 5, 2024
32ee45f
Add docstrings for add_bos, add_eos in SentencePieceTokenizer.
briango28 Sep 6, 2024
b57396c
Add add_bos, add_eos to SentencePieceTokenizer.get_config()
briango28 Sep 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions keras_nlp/src/tokenizers/sentence_piece_tokenizer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,8 @@ def __init__(
proto=None,
sequence_length=None,
dtype="int32",
add_bos=False,
mattdangerw marked this conversation as resolved.
Show resolved Hide resolved
add_eos=False,
**kwargs,
) -> None:
if not is_int_dtype(dtype) and not is_string_dtype(dtype):
Expand All @@ -128,6 +130,8 @@ def __init__(

self.proto = None
self.sequence_length = sequence_length
self.add_bos = add_bos
self.add_eos = add_eos
self.set_proto(proto)
self.file_assets = [VOCAB_FILENAME]

Expand Down Expand Up @@ -172,6 +176,8 @@ def set_proto(self, proto):
self._sentence_piece = tf_text.SentencepieceTokenizer(
model=proto_bytes,
out_type=self.compute_dtype,
add_bos=self.add_bos,
mattdangerw marked this conversation as resolved.
Show resolved Hide resolved
add_eos=self.add_eos,
)
# Keras cannot serialize a bytestring, so we base64 encode the model
# byte array as a string for saving.
Expand Down
23 changes: 23 additions & 0 deletions keras_nlp/src/tokenizers/sentence_piece_tokenizer_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,29 @@ def test_string_tokenize(self):
[["▁the", "▁quick", "▁brown", "▁fox."]],
)

def test_scalar_bos_eos(self):
input_data = "the quick brown fox."
tokenizer = SentencePieceTokenizer(
proto=self.proto,
add_bos=True,
add_eos=True,
)
output_data = tokenizer(input_data)
self.assertAllEqual(output_data, [1, 6, 5, 3, 4, 2])

def test_string_bos_eos(self):
input_data = ["the quick brown fox."]
tokenizer = SentencePieceTokenizer(
proto=self.proto,
dtype="string",
add_bos=True,
add_eos=True,
)
output_data = tokenizer(input_data)
self.assertAllEqual(
output_data, [["<s>", "▁the", "▁quick", "▁brown", "▁fox.", "</s>"]]
)

def test_detokenize(self):
tokenizer = SentencePieceTokenizer(proto=self.proto)
outputs = tokenizer.detokenize([6, 5, 3, 4])
Expand Down
Loading