| { | |
| "additional_special_tokens": [ | |
| "<unk>", | |
| "<s>", | |
| "</s>", | |
| "<|fim_prefix|>", | |
| "<|fim_middle|>", | |
| "<|fim_suffix|>", | |
| "<|fim_pad|>", | |
| "<|filename|>", | |
| "<|gh_stars|>", | |
| "<|issue_start|>", | |
| "<|issue_comment|>", | |
| "<|issue_closed|>", | |
| "<|jupyter_start|>", | |
| "<|jupyter_text|>", | |
| "<|jupyter_code|>", | |
| "<|jupyter_output|>", | |
| "<|empty_output|>", | |
| "<|commit_before|>", | |
| "<|commit_msg|>", | |
| "<|commit_after|>", | |
| "<|reponame|>", | |
| "<|im_start|>", | |
| "<|im_end|>", | |
| "<|sys_start|>", | |
| "<|sys_end|>" | |
| ], | |
| "bos_token": "<s>", | |
| "eos_token": "</s>", | |
| "pad_token": "<unk>", | |
| "unk_token": "<unk>" | |
| } | |