115 lines
1.7 KiB
JSON
115 lines
1.7 KiB
JSON
{
|
|
"version": "1.0",
|
|
"truncation": null,
|
|
"padding": null,
|
|
"added_tokens": [
|
|
{
|
|
"id": 58,
|
|
"content": "<unk>",
|
|
"single_word": false,
|
|
"lstrip": false,
|
|
"rstrip": false,
|
|
"normalized": false,
|
|
"special": true
|
|
}
|
|
],
|
|
"normalizer": {
|
|
"type": "Sequence",
|
|
"normalizers": [
|
|
{
|
|
"type": "Lowercase"
|
|
},
|
|
{
|
|
"type": "Replace",
|
|
"pattern": {
|
|
"Regex": "[^012345679 '_aஅஆஇஈஉஊஎஏஐஒஓகஙசஜஞடணதநனபமயரறலளழவஷஸஹாிீுூெேைொோௌ்]"
|
|
},
|
|
"content": ""
|
|
},
|
|
{
|
|
"type": "Strip",
|
|
"strip_left": true,
|
|
"strip_right": true
|
|
},
|
|
{
|
|
"type": "Replace",
|
|
"pattern": {
|
|
"Regex": "(?=.)|(?<!^)$"
|
|
},
|
|
"content": "3"
|
|
}
|
|
]
|
|
},
|
|
"pre_tokenizer": {
|
|
"type": "Split",
|
|
"pattern": {
|
|
"Regex": ""
|
|
},
|
|
"behavior": "Isolated",
|
|
"invert": false
|
|
},
|
|
"post_processor": null,
|
|
"decoder": null,
|
|
"model": {
|
|
"vocab": {
|
|
"0": 47,
|
|
"1": 44,
|
|
"2": 23,
|
|
"3": 0,
|
|
"4": 54,
|
|
"5": 57,
|
|
"6": 36,
|
|
"7": 14,
|
|
"9": 31,
|
|
" ": 7,
|
|
"'": 13,
|
|
"_": 4,
|
|
"a": 15,
|
|
"அ": 1,
|
|
"ஆ": 45,
|
|
"இ": 38,
|
|
"ஈ": 2,
|
|
"உ": 3,
|
|
"ஊ": 11,
|
|
"எ": 37,
|
|
"ஏ": 16,
|
|
"ஐ": 52,
|
|
"ஒ": 27,
|
|
"ஓ": 49,
|
|
"க": 6,
|
|
"ங": 50,
|
|
"ச": 30,
|
|
"ஜ": 53,
|
|
"ஞ": 29,
|
|
"ட": 22,
|
|
"ண": 48,
|
|
"த": 41,
|
|
"ந": 5,
|
|
"ன": 35,
|
|
"ப": 46,
|
|
"ம": 26,
|
|
"ய": 39,
|
|
"ர": 25,
|
|
"ற": 28,
|
|
"ல": 21,
|
|
"ள": 43,
|
|
"ழ": 24,
|
|
"வ": 17,
|
|
"ஷ": 55,
|
|
"ஸ": 33,
|
|
"ஹ": 19,
|
|
"ா": 9,
|
|
"ி": 32,
|
|
"ீ": 12,
|
|
"ு": 51,
|
|
"ூ": 20,
|
|
"ெ": 10,
|
|
"ே": 8,
|
|
"ை": 34,
|
|
"ொ": 56,
|
|
"ோ": 42,
|
|
"ௌ": 40,
|
|
"்": 18
|
|
}
|
|
}
|
|
} |