update pip pkg

rwkv_pip_package/README.md

@@ -53,6 +53,7 @@ from rwkv.utils import PIPELINE, PIPELINE_ARGS
 # download models: https://huggingface.co/BlinkDL
 model = RWKV(model='/fsx/BlinkDL/HF-MODEL/rwkv-4-pile-169m/RWKV-4-Pile-169M-20220807-8023', strategy='cpu fp32')
 pipeline = PIPELINE(model, "20B_tokenizer.json") # 20B_tokenizer.json is in https://github.com/BlinkDL/ChatRWKV
+# use pipeline = PIPELINE(model, "rwkv_vocab_v20230424") for rwkv "world" models
 
 ctx = "\nIn a shocking finding, scientist discovered a herd of dragons living in a remote, previously unexplored valley, in Tibet. Even more surprising to the researchers was the fact that the dragons spoke perfect Chinese."
 print(ctx, end='')

rwkv_pip_package/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "rwkv"
-version = "0.8.22"
+version = "0.8.23"
 authors = [
   { name="Bo PENG" },
 ]

rwkv_pip_package/src/rwkv/utils.py

@@ -109,10 +109,11 @@ def generate(self, ctx, token_count=100, args=PIPELINE_ARGS(), callback=None, st
             all_tokens += [token]
             for xxx in occurrence:
                 occurrence[xxx] *= args.alpha_decay
-            if token not in occurrence:
-                occurrence[token] = 1
-            else:
-                occurrence[token] += 1
+            if self.decode([token]) not in ' \r\n\t,.;?!"\':0123456789+-*/=#@$%^&_`~|<>\\()[]{}，。；“”：？！（）【】':
+                if token not in occurrence:
+                    occurrence[token] = 1
+                else:
+                    occurrence[token] += 1
             # print(occurrence) # debug
 
             # output