CRAYON-tokenizer / decode_examples.py
Phase-Technologies's picture
Upload folder using huggingface_hub
708f4a3 verified
from crayon import CrayonVocab
vocab = CrayonVocab(device="auto")
vocab.load_profile("lite")
text = "Hello, world!"
tokens = vocab.tokenize(text)
print(tokens)
decode=vocab.decode(tokens)
print(decode)