| library_name: transformers | |
| tags: [] | |
| # Tokenizer Card | |
| <!-- Provide a quick summary of what the model is/does. --> | |
| ```bash | |
| from transformers import AutoTokenizer | |
| tokenizer = AutoTokenizer.from_pretrained("azherali/python-code-search-tokenizer") | |
| example = """class LinearLayer(): | |
| def __init__(self, input_size, output_size): | |
| self.weight = torch.randn(input_size, output_size) | |
| self.bias = torch.zeros(output_size) | |
| def __call__(self, x): | |
| return x @ self.weights + self.bias | |
| """ | |
| tokenizer.tokenize(example) | |