""" UCE-8 hybrid - a 3 tier encoding sized for Asian text. tier 1 [0xxxxxxx] 128 plain ASCII tier 2 [1xxxxxxx][0xxxxxxx] 8,704 68 pages tier 3 [1xxxxxxx][1xxxxxxx][0xxxxxxx] 1,114,112 everything else The high bit is the length signal: set means another byte follows, clear means the character ends here. 60 of the 128 possible terminating values are excluded: the 33 control codes and space, plus 26 characters that carry structural meaning where text flows - " & ' / < = > \\ ` HTML, shell quoting, JSON strings, path separators % , - . : ? URL encoding, CSV, file extensions, key:value # $ * + ; @ [ ] { | } shell, glob, email, JSON arrays and objects Only 6 punctuation marks survive as trail bytes - ! ( ) ^ _ ~ - and not one of them is a metacharacter in JSON, CSV, URLs, file paths or the shell. Every other printable ASCII punctuation mark is kept out of the encoded stream entirely. Tier 2 pages come in two kinds: 38 BLOCK pages 28 world scripts + 10 Chinese. A page is a 128 code point aligned block, so the slot is arithmetic: slot = code_point & 0x7F 30 INDEXED pages 2 world (Cyrillic ext/Georgian/Mongolian, then Tamil + currency + extra Latin) + 24 Chinese + 6 Korean. A page is 128 entries of a curated list, so the slot is a lookup, same multi-page pattern for all three: slot = WORLD_SLOT[code_point] trail = WORLD_TRAILS[slot >> 7] Why Chinese and Korean are indexed: Unicode orders Han characters by radical and Hangul syllables by jamo combination, neither of which follows frequency. The first 1,920 Hangul syllables that 15 block pages would cover reach only 28% of real Korean text; 6 indexed pages reach 89%. All 3,500 characters of the national standard primary set (通用规范汉字表) reach 2 bytes, with 24 slots spare. Why Tamil is folded into the world pages: Tamil shares one fixed layout template with the other Brahmi-derived Indic blocks (Devanagari, Bengali, Gurmukhi, ...), where every block reserves the same code point offset for the same consonant row - voiced, aspirated, voiced-aspirated members included - regardless of whether a given script distinguishes them. Tamil doesn't, so of the 128 code points in its block only 61 are real characters; the other 67 are permanently template gaps that can never be arithmetic-mapped to anything. Indexing packs the 61 real characters down and leaves room, in the same page, for 20 currency signs of independent countries with no other 2-byte reach, 38 Latin letters everyday for Romanian, Azerbaijani, Yoruba, Hausa, Ewe, Bambara and Pinyin, and a handful of letters moved in from the other world page (Vietnamese Ơơ Ưư, Igbo Ṅṅ, Uzbek/Hawaiian/Tongan ʻ) - 238 of 256 slots used across both world pages, 18 spare. CHINESE_CHARS, KOREAN_CHARS and WORLD_CHARS are FROZEN. Changing or reordering any of them changes the meaning of every byte sequence already written. """ LEAD_BASE = 0x80 MAX_CODE_POINT = 0x10FFFF # markers used in TRAIL_TO_PAGE for the three indexed kinds CHINESE = "chinese" KOREAN = "korean" WORLD = "world" # ---------------------------------------------------------------- block pages PAGE_TO_TRAIL = { # 28 world language pages 0x008: 0x41, # U+0080-00FF Latin-1 Supplement 0x010: 0x42, # U+0100-017F Latin Extended-A 0x038: 0x43, # U+0380-03FF Greek and Coptic 0x040: 0x44, # U+0400-047F Cyrillic 0x050: 0x45, # U+0500-057F Armenian + Cyrillic Supplement 0x058: 0x46, # U+0580-05FF Hebrew + Armenian tail 0x060: 0x47, # U+0600-067F Arabic 0x068: 0x48, # U+0680-06FF Arabic 2 - Persian, Urdu, Pashto, Kurdish 0x090: 0x49, # U+0900-097F Devanagari, 600M 0x098: 0x4A, # U+0980-09FF Bengali, 284M 0x0A0: 0x4B, # U+0A00-0A7F Gurmukhi, 30M 0x0A8: 0x4C, # U+0A80-0AFF Gujarati, 57M 0x0B0: 0x4D, # U+0B00-0B7F Oriya, 41M 0x0C0: 0x4E, # U+0C00-0C7F Telugu, 83M 0x0C8: 0x4F, # U+0C80-0CFF Kannada, 43M 0x0D0: 0x50, # U+0D00-0D7F Malayalam, 45M 0x0D8: 0x51, # U+0D80-0DFF Sinhala, 17M 0x0E0: 0x52, # U+0E00-0E7F Thai, 71M 0x0E8: 0x53, # U+0E80-0EFF Lao, 7.7M 0x0F0: 0x54, # U+0F00-0F7F Tibetan, 6M 0x100: 0x55, # U+1000-107F Myanmar, 33M 0x120: 0x56, # U+1200-127F Ethiopic, 135M 0x128: 0x57, # U+1280-12FF Ethiopic 2 - Amharic, Tigrinya (89 everyday) 0x130: 0x58, # U+1300-137F Ethiopic 3 - Amharic + punctuation and numerals (92) 0x178: 0x59, # U+1780-17FF Khmer, 18M 0x1E8: 0x5A, # U+1E80-1EFF Vietnamese, Yoruba, Igbo, Welsh - 164M 0x300: 0x38, # U+3000-307F CJK punctuation + Hiragana, 122M 0x308: 0x39, # U+3080-30FF Katakana + Hiragana # 8 Chinese block pages - the densest pages in the Han range 0x4E0: 0x61, # U+4E00-4E7F Chinese 0x4E8: 0x62, # U+4E80-4EFF Chinese 0x538: 0x63, # U+5380-53FF Chinese 0x540: 0x64, # U+5400-547F Chinese 0x5B8: 0x65, # U+5B80-5BFF Chinese 0x628: 0x66, # U+6280-62FF Chinese 0x7E8: 0x67, # U+7E80-7EFF Chinese 0x8B8: 0x68, # U+8B80-8BFF Chinese 0x8D0: 0x69, # U+8D00-8D7F Chinese 0x8F8: 0x6A, # U+8F80-8FFF Chinese } # ------------------------------------------------------- indexed world pages # 2 pages, same multi-page pattern as CHINESE_TRAILS/KOREAN_TRAILS below: # slot >> 7 picks the trail byte, slot & 0x7F picks the position in it. WORLD_TRAILS = (0x5E, 0x5F) # Page 0 (trail 0x50): Cyrillic Extension letters (Kazakh, Bashkir, # Mongolian Cyrillic), Georgian, and 32 curated Mongolian traditional-script # characters - comma, full stop, free variation selector one, free # variation selector three, vowel separator, 25 core letters (A E I O U OE # EE NA BA PA QA GA MA LA SA SHA TA DA CHA JA YA RA WA FA KA - UE and ANG # excluded), plus 2 Ali Gali transliteration letters. # # Page 1 (trail 0x51): 61 real Tamil characters - every code point in # U+0B80-0BFF that isn't a permanently ISCII-template gap # (independent vowels, consonants, vowel signs, virama, digits). Followed # by 20 currency signs - only the ones that are a live currency of a # sovereign country today (colon, naira, rupee, won, new sheqel, dong, # euro, kip, tugrik, peso, guarani, hryvnia, cedi, tenge, Indian rupee, # Turkish lira, manat, ruble, lari, som; the other 13 code points in the # Unicode Currency Symbols block are excluded - 10 historic/superseded, # 3 not a country's currency at all). Followed by 38 Latin letters # everyday for languages with no other 2-byte reach: Romanian ș ț Ș Ț, # Azerbaijani ə Ə, Yoruba ṣ Ṣ, Hausa ɓ ɗ ƙ ƴ Ɓ Ɗ Ƙ Ƴ, Ewe ɖ ɛ ƒ ɣ ɔ ʋ Ɖ Ɛ # Ƒ Ɣ Ɔ Ʋ, Bambara ɲ Ɲ, and 8 Pinyin tone letters ǎ ǐ ǒ ǔ ǖ ǘ ǚ ǜ. Followed # by ƠơƯư (Vietnamese O/U-horn) and Ṅṅʻ (Igbo, Uzbek/Hawaiian/Tongan). WORLD_CHARS = ( "ஂஃஅஆஇஈஉஊஎஏஐஒஓஔகஙசஜஞடணதநனபமயரறலளழவஶஷஸஹாிீுூெேைொோௌ்ௐௗ௦௧௨௩௪௫௬௭௮௯" "აბგდევზთიკლმნოპჟრსტუფქღყშჩცძწჭხჯჰ" "᠂᠃᠋᠍᠊᠎ᠠᠡᠢᠣᠤᠥᠦᠧᠨᠪᠫᠬᠭᠮᠯᠰᠱᠲᠳᠴᠵᠶᠷᠸᠹᠺᢋᢣ" "ӘәҒғҚқҢңӨөҰұҮүҺһҖҗҘҙҠҡҪҫҜҝҸҹҔҕҤҥӢӣӮӯҲҳҶҷӐӑӖӗӲӳӀҐґӏӕӧʼ" "șțȘȚəƏṣṢɓɗƙƴƁƊƘƳɖɛƒɣɔʋƉƐƑƔƆƲɲƝǎǐǒǔǖǘǚǜƠơƯưṄṅʻ" "₡₦₨₩₪₫€₭₮₱₲₴₵₸₹₺₼₽₾⃀" "，？！：№≠≤≥℃→" ) WORLD_SLOT = {ord(c): i for i, c in enumerate(WORLD_CHARS)} WORLD_CODE_POINTS = tuple(ord(c) for c in WORLD_CHARS) WORLD_PAGE = tuple( WORLD_TRAILS.index(b) if b in WORLD_TRAILS else None for b in range(0x80) ) # ----------------------------------------------------- indexed Chinese pages CHINESE_TRAILS = ( 0x21, 0x28, 0x29, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, ) # 2,816 Chinese characters, most frequent first, excluding everything the # 10 block pages already reach. CHINESE_CHARS = ( "十卜八入儿匕几刁刀力干工土士才大小山巾千川夕勺凡广门尸己已巳弓子卫女刃飞马王开天夫元无扎艺木支犬太区歹尤匹车巨牙屯戈比切瓦止少曰日冈内水见午牛手气毛壬升夭长片化币斤爪父凶分公月氏勿欠风匀勾凤六文方火斗忆户冗心尺引巴孔队办允邓劝幻玉刊未末示击" "打巧正扑卉扒功扔甘艾节本术左石布夯戊龙平灭轧卡北占凸卢旧帅归旦目甲申电田由央兄冉皿凹囚四生矢失禾白斥瓜用甩印尔匆册卯犯外处冬鸟务包饥市立冯玄闪兰半汁汇头汉穴写礼必永尼民弗弘出奶奴加皮孕圣矛母幼邦式刑戎动扛扣考托老巩圾执扩扫地场扬耳芋共芒芝朽" "朴机权臣再协西戌在百有存而页匠夸夺灰列死成夹夷轨邪尧划毕至此师尘尖劣光当早虫曲团因屿屹岁帆回岂则刚网肉年朱先廷舌竹伟传休伍伏优臼伐延伤伦华伙伪自伊血似行舟全会杀兆企众爷伞创肌肋朵杂危旬旨旭匈多色壮冲妆冰庄庆刘齐衣次决充妄闭问羊并关米灯州汗污" "江汛池汝汤忙兴字军农那尽异弛孙阵阳收阶阴防奸如妇妃好她妈戏羽观欢驮驯驰巡弄麦玖玛形戒韧扶坛坏扰扼找批址扯汞坝攻扳扮孝坎均坟坑坊壳志块扭声却劫芙芜芽花芹芥芬苍芳芦芯劳克芭苏杆杠杜材村杖杏杉巫极李杨求甫匣更束豆医励尬歼来步卤坚肖旱时助里园旷围足" "邮男困邑囤别岖岗帐针钉牡我利秀私每兵估体何佐佑但伸佃作伯伶佣低你住位伴身皂伺佛囱彻役余希坐谷妥邻岔肝肛肚肠龟甸免狂犹狈角删条彤卵灸岛刨饭饮系言冻状况床库庇疗应冷庐序弃冶忘闰闲间闷判兑灶灿灼弟汪沐沛汰沥沙汽沃沦汹泛沧没沟沪沈沉沁怀忧忱快牢究穷" "灾良补初社祀罕灵即层屁尿尾局改张忌际陆阿陈阻附坠妓妙妖姊妨妒努忍劲矣鸡驱驳驴奉玩环武青现玫表规卦坷坯坪坦坤者顶势垃幸坡其苦昔苛若茂苹苗英苑范直茄茎苔茅枉林枝杯枢柜枚析板松枪枫构杭杰枕或画卧刺枣雨卖郁矿码奈奔奇奋态欧殴垄妻轰顷转斩轮软到非歧肯" "齿卓虎虏肾尚旺具果昆国哎咕昌畅明易咙昂典固忠咒咋咐鸣咏咄咖岸岩帖罗帜帕岭凯图钓制知氛垂牧物刮秆和季委秉佳侍岳供使例侠侥版侄侦侣侧凭侨佩侈依卑的欣征往爬彼径所舍金刹肴斧爸采觅念忿肤肺肢肿胀朋股肪肥服胁昏鱼兔狐忽狗狞备饰饱饲庞店夜庙府底疟疙疚剂" "卒郊庚废净盲放刻育氓闸闹郑券卷单炬炒炊炕炎炉沫浅法泄沽河沾泪沮油泊沿泡注泣泞泻泌泳泥沸沼波泼泽治怔怯怖性怕怜怪怡学空帘郎肩房衬衫视祈建肃录隶帚屉居届刷屈弧弥弦承孟陋陌孤陕降函限妹姑姐姓妮始姆驾艰驶驹驻驼契奏春帮玷珍玲珊玻毒型挂封持项垮挎城挟" "挠政挡哉挺垢挑垛指垫挣挤挖按挥挪某甚荆茸革荐巷带草茵茶荒茫荡荣荤荧故胡荫荔南药标栈柑枯柄栋相查柏栅柳柱柿栏柠树勃要柬咸威歪研砖砌砂泵砚砍面耐耍牵残殃轴轻鸦皆韭背战点虐览竖省削尝昧盹是盼哇哄哑显冒映星昨咧昭畏趴胃界虹虾蚁思蚂虽品咽骂勋哗咱响哈" "哆咬咳咪哪哟炭峡罚骨幽钙钝钞钟钢钠钦钧钩钮卸看矩毡氢怎牲选适秒香种秋科重复竿段便俩顺修俏保促俄俐侮俭俗俘信皇泉鬼侵禹侯俊盾待徊衍律很须剑逃食盆胚胆胜胞胖脉胎勉狭狮独狰狡狱狠怨急饵饶蚀饺饼峦弯将奖哀度庭疮疯疫疤咨姿音帝施闺闻闽阀阁差养美姜送类" "籽娄前首逆兹总炼炸烁炮炫烂剃洼洁洪洒柒浇浊洞测洗活派洽染洛浏济洋洲浑浓津恃恒恢恍恬恤恰恼恨觉突穿窃冠扁袄祖神祝祠垦退既屋昼屏屎逊眉孩除险院娃姥姨姻娇姚娜怒架盈勇怠癸蚤柔垒骄骆骇耕耗艳泰秦珠班素匿蚕顽匪捞栽捕埂捂振载盐捎捍捏埋捉捆捐损袁捌都哲" "逝捡挫换挽挚热恐捣壶捅埃挨耻耿耽恭莽莱莲莫莉荷获晋恶莹莺真框梆桂桔栖档桐株桥桦栓桃格桩校核样根索哥速逗栗配翅唇夏砸砾础破套逐烈殊殉顾轿顿毙致柴桌虑监紧党晒眠晓哮唠鸭晃哺晌剔晕蚌畔蚊蚪蚓哨哩圃哭哦恩唤唁哼唧啊唉唆罢峭峨峰圆峻钱钳钻钾铁铃铅缺氧" "氨特牺造敌租积秧秩称秘透笔笑笋债借值倚俺倾倒倘俱倡候俯倍倦健臭射躬息倔徒徐殷舰舱般航途爹爱豺颁颂翁胰脆脂胸脏脐胶脑逛狸狼卿逢留皱饿凌凄恋桨浆衰衷高郭席准座症病疾斋疹疼疲脊效离紊唐瓷凉站剖竞部旁旅畜阅羞瓶粉料益兼烤烘烦烧烛烟烙递涛浙涝浦酒涉消" "涡浩海涂浴浮涣涤流润涧涕浪浸涨烫涩涌悖悟悄悍悔悯悦窍窄案朗扇袜袖袍被祥冥谁调冤谅谈谊剥恳展剧屑弱陵陶陷陪娱娟恕娥娘通能难预桑验骏球琐理琉琅捧堵措描域捺掩捷排焉掉捶堆推埠掀授捻教掏掐掠掂培接掷控探据掘掺职基聆勘聊娶著菱勒黄菲萌萝菌萎菜萄菊菩萍" "菠萤营萧萨菇械彬梦婪梗梧梢梅检梳梯桶梭救曹副票酗戚硅硕奢盔爽聋袭盛匾雪虚彪雀堂常匙晨睁眼悬野啪啦曼晦晚啄啡距趾啃跃略蚯蛇唬累鄂唱患啰唾唯啤啥啸崖崎崭逻崔帷崩崇崛婴圈铐铛铝铜铭铲银矫甜梨犁秽移笼笛符第敏做袋悠偿偶偎偷您售停偏躯兜假衅徘徙得衔盘" "舶船斜盒鸽敛悉欲彩领脚豚脸脱象够逸猜猪猎猫凰猖猛祭馅馆凑减毫烹庶麻庵痊痒痕廊康庸鹿盗章竟商族旋望率阎阐着盖眷粘粗粒断剪兽焊焕清添鸿淋涯淹渠渐淑淌混淮淆渊淫渔淘淳液淤淡淀深涮涵婆梁渗情惜惭悼惧惕惟惊惦悴惋惨惯窑谋谍谎谐袱祸谓谚谜逮敢尉屠弹隋堕" "随蛋隅隆隐婚婶婉颇颈骑缀巢琴琳琢琼斑替揍款堪塔搭堰揩越趁趋超揽堤提博揭喜彭揣插揪搜煮援搀裁搁搓搂搅壹握搔揉斯期欺联葫散惹葬募葛董葡敬葱蒋蒂落韩朝葵棒棱棋椰植森焚椅椒棵棍椎棉棚棕棺榔椭惠惑逼粟棘酣酥硬硝确硫雁殖裂雄雳暂雅翘悲紫凿敞棠掌晴睐暑最" "晰量鼎喷喳晶喇遇喊遏晾景畴践跋跌跑遗蛙蛛蜓蛤喝喂喘喉喻啼喧嵌幅帽黑铸铺链销锁锅锈锋锌锐甥掰短智氮毯氯鹅剩稍程稀税等筑策筛筒答筋筝傲傅牌堡集焦傍储皖粤奥街惩御循艇舒逾番释禽腊脾腋腔腕鲁猩猬猾猴惫然馈装蛮就敦斌痘痪痛童竣阔善翔羡普粪尊奠道遂曾焰" "港滞湖湘渣渤渺湿温渴溃溅滑湃渝湾渡游滋渲溉愤慌惰愕愣惶愧愉慨割窜窝窖窗窘遍雇裕裤裙禅禄谢谣谦犀属屡强粥疏隔隙媒絮嫂媚婿登缅缆缉缓缔骗编骚缘瑟瑞瑰瑙魂肆摄摸填搏塌鼓摆携搬摇搞塘摊聘斟蒜勤靴蓝墓幕蓬蓄蒲蓉蒙蒸献椿禁楚楷榄想槐榆楼概酪酬感碍碘碑碎" "碰碗尴雷零雾雹督频龄鉴睛睹睦瞄睫睡嗜鄙嗦愚暖盟歇暗暇照畸跨跷跳路跤跟遣蜗蛾蜂蜕嗅嗡嗓署置罪罩蜀幌错锚锡锣锥锦键锯锰矮稚稠愁筹签简筷毁舅鼠催傻像躲魁衙微愈遥腻腰腮腹腺腾腿鲍猿颖触解煞雏馍馏酱禀痹廓痴廉靖新韵意粮数煎塑慈煤煌满漠滇源滤滥滔溪溜漓" "滚溢溯滨溶溺粱滩慎誉塞窟谨褂裸福谬群殿障媳嫉嫌嫁缚缝缠剿静碧璃熬墙墟嘉摧截誓境摘摔撇聚慕暮摹蔓蔑蔡蔗蔽熙蔚兢模槛榴榜榨榕歌遭酵酷酿酸碟碱碳磁愿需雌裳颗墅嗽蜻蜡蝇蜘嘛嘀锻镀舞舔稳熏箕算管箫舆僚僧鼻魄魅貌膜膀鲜疑孵馒裹敲豪膏遮腐瘟瘦彰竭端旗精粹" "歉弊熄熔煽潇漆漱漂漫滴漾演漏慢慷蜜谭肇褐褪谱隧嫩翠熊凳骡缩慧撵撕撒撩趣撑撮撬播擒墩撞撤增撰聪鞋鞍蕉蔬蕴横槽樱橡樟橄敷豌飘醋醇醉磅碾震霄霉瞒题暴瞎嘻嘶嘲嘹影踢踏踩踪蝶蝴蝠蝌蝗蝙嘿嘱幢墨镇镐镑靠稽稻黎稿稼箱箭篇僵躺僻德艘膝鲤鲫熟摩褒瘤瘫凛颜毅糊" "遵憋潜澎潮潭鲨澳潘澈澜澄懂憔懊憎额翩褥谴鹤憨慰劈履豫缭撼擂操擅燕蕾薯薛薇擎薪薄颠翰噩橱橙橘整融瓢醒霍冀餐嘴蹄蟆螃器噪默黔镜穆篮篡篷篱儒邀衡膨雕鲸磨瘾凝糖糕燃濒澡激懒憾懈壁避缴戴擦藉鞠藏檬檐檀礁磷霜霞瞧瞬瞳瞩曙蹈螺嚎穗魏簧簇繁徽爵朦臊鳄癌糟燥" "懦豁臀臂翼骤藕鞭藤覆瞻蹦嚣翻鳍鹰瀑襟璧戳孽警蘑藻攀曝巅簸簿蟹颤靡瓣羹爆疆壤耀蠕嚼嚷巍籍鳞魔灌譬蠢霸露霹髓囊镶瓤罐" ) CHINESE_SLOT = {ord(c): i for i, c in enumerate(CHINESE_CHARS)} CHINESE_CODE_POINTS = tuple(ord(c) for c in CHINESE_CHARS) # ------------------------------------------------------ indexed Korean pages KOREAN_TRAILS = ( 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x7E, ) # 768 Hangul syllables, ranked by the frequency of their component jamo. KOREAN_CHARS = ( "아이가으나기사하다오니그시히디안어우느자마스고흐드알인바여라에지미노간거구은소일애요압비호즈도리난므너누앙악갈긴겨산서수을게한브" "입허후단더두르온날닌차잉암녀와근조익네길개교갑모살신읍셔강할힌파달딘세언운혀는각뎌올응닐내뇨헤치납데보임잔저주윽글만로머무낭깁슨" "의실새쇼삽낙곤흔깅든힐해효감합피딜대됴얼울과답상늘반긱츠연버부음옵닙란항잘삭엔진러루당말민져논급학옹닝며닥슬남제건군놔메십옥골긍" "닉흘외유프들김힙손앤욘극딥싱업웁발삼빈열늡혼즌솨긔랄돈벼린힝엘식질재죠함믄넌눈잡딩담엉웅밀베려화매묘놀능까맙타돠초힉옴레딕님억욱" "습워걸굴장늑견망금곱선순흡작늬겐듭솔승앨욜막븐헌훈빌심배뵤던둔공밥홀른흥슥즐엽포았돌등릴래료믈힘랍찬널눌엡집처추곡딤흑싀괴규년엄" "움방끼밉티득늠완영놉존넨갠굔희랑엥징카듸몬박잠겁굽역결밍좌맘농설술예락엑직겔뫄션블슴믹솝겅궁앱욥헐훌녹판센뇌뉴퍼푸덜둘현를빕곰흠" "뎐홉있즙걱국궈듬낸뇬찰돕헨립친송앵용끄믑트덴본넙눕녈전준쳐빙왈밤염론홍졸속증먼문넬액욕갤굘쇠슈동봐체링람키몰믕엠읜넝눙빅짐앟혹롸" "즉샌숀겹회휴밈독릭놈갔되듀섭숩믁셜넉눅눠겝즤핸횬팔븝핀검굼셀댄됸헙훕경관믜읐혈덥둡성숭펴뎔릅츤번분낼뇰겡헬칠채쵸격페델볼솜찹븡앰" "욤헝훙런룬석숙절줄숴계녑덩둥겍빔롤릉났젼왑멀물크홈븍즘좁넵헉훅돔갭굡면훠림창읠젠싸짜덕둑믐넘눔꼬둬몹토잏륵샐숄놘녕멘븨왕깄야왼위" "윤착종않앗넹픈갱굥릐셥핼횰녁샀필몽패표댈됼왁팝괄셉녜족겸넥죄쥬객굑협츨벌불섬숨핬뎝목셩솬겜긘뫼뮤닸냅뇹변헵칩갛잰죤럴룰븜팡뎁봅셍" "험훔접줍벤옸련환맨묜형셕졀덤둠닜깐탄름뎡돤롭촌꺼꾸멉뭅터투냉뇽팍읗멸셰렌헹섹젤칭참씨찌뎅봉혁원읩긌놜멜정중뎍촤념샙숍혜낵뇩욀율헥" "왐롱칙멍뭉뎨코플읺잇덱복좀넴늰갬굠뵈뷰적죽싰낳줘햅횹읭몸핍록댑둅생숑먹묵괍뢰류뭐밴뵨힜츱읙솰법붑긜딨폰별셤랜룐행횽깋잴죨색쇽었웄" ) KOREAN_SLOT = {ord(c): i for i, c in enumerate(KOREAN_CHARS)} KOREAN_CODE_POINTS = tuple(ord(c) for c in KOREAN_CHARS) # --------------------------------------------------------------- decode tables TRAIL_TO_PAGE = ( None, # 0x00 ctrl unused None, # 0x01 ctrl unused None, # 0x02 ctrl unused None, # 0x03 ctrl unused None, # 0x04 ctrl unused None, # 0x05 ctrl unused None, # 0x06 ctrl unused None, # 0x07 ctrl unused None, # 0x08 ctrl unused None, # 0x09 ctrl unused None, # 0x0A ctrl unused None, # 0x0B ctrl unused None, # 0x0C ctrl unused None, # 0x0D ctrl unused None, # 0x0E ctrl unused None, # 0x0F ctrl unused None, # 0x10 ctrl unused None, # 0x11 ctrl unused None, # 0x12 ctrl unused None, # 0x13 ctrl unused None, # 0x14 ctrl unused None, # 0x15 ctrl unused None, # 0x16 ctrl unused None, # 0x17 ctrl unused None, # 0x18 ctrl unused None, # 0x19 ctrl unused None, # 0x1A ctrl unused None, # 0x1B ctrl unused None, # 0x1C ctrl unused None, # 0x1D ctrl unused None, # 0x1E ctrl unused None, # 0x1F ctrl unused None, # 0x20 ctrl unused CHINESE, # 0x21 '!' indexed Chinese page 0 None, # 0x22 ''' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x23 '#' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x24 '$' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x25 '%' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x26 '&' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x27 ''' excluded - structural in HTML/shell/CSV/URL/JSON CHINESE, # 0x28 '(' indexed Chinese page 1 CHINESE, # 0x29 ')' indexed Chinese page 2 None, # 0x2A '*' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x2B '+' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x2C ',' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x2D '-' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x2E '.' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x2F '/' excluded - structural in HTML/shell/CSV/URL/JSON CHINESE, # 0x30 '0' indexed Chinese page 3 CHINESE, # 0x31 '1' indexed Chinese page 4 CHINESE, # 0x32 '2' indexed Chinese page 5 CHINESE, # 0x33 '3' indexed Chinese page 6 CHINESE, # 0x34 '4' indexed Chinese page 7 CHINESE, # 0x35 '5' indexed Chinese page 8 CHINESE, # 0x36 '6' indexed Chinese page 9 CHINESE, # 0x37 '7' indexed Chinese page 10 0x300, # 0x38 '8' U+3000-307F CJK punctuation + Hiragana, 122M 0x308, # 0x39 '9' U+3080-30FF Katakana + Hiragana None, # 0x3A ':' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x3B ';' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x3C '<' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x3D '=' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x3E '>' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x3F '?' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x40 '@' excluded - structural in HTML/shell/CSV/URL/JSON 0x008, # 0x41 'A' U+0080-00FF Latin-1 Supplement 0x010, # 0x42 'B' U+0100-017F Latin Extended-A 0x038, # 0x43 'C' U+0380-03FF Greek and Coptic 0x040, # 0x44 'D' U+0400-047F Cyrillic 0x050, # 0x45 'E' U+0500-057F Armenian + Cyrillic Supplement 0x058, # 0x46 'F' U+0580-05FF Hebrew + Armenian tail 0x060, # 0x47 'G' U+0600-067F Arabic 0x068, # 0x48 'H' U+0680-06FF Arabic 2 - Persian, Urdu, Pashto, Kurdish 0x090, # 0x49 'I' U+0900-097F Devanagari, 600M 0x098, # 0x4A 'J' U+0980-09FF Bengali, 284M 0x0A0, # 0x4B 'K' U+0A00-0A7F Gurmukhi, 30M 0x0A8, # 0x4C 'L' U+0A80-0AFF Gujarati, 57M 0x0B0, # 0x4D 'M' U+0B00-0B7F Oriya, 41M 0x0C0, # 0x4E 'N' U+0C00-0C7F Telugu, 83M 0x0C8, # 0x4F 'O' U+0C80-0CFF Kannada, 43M 0x0D0, # 0x50 'P' U+0D00-0D7F Malayalam, 45M 0x0D8, # 0x51 'Q' U+0D80-0DFF Sinhala, 17M 0x0E0, # 0x52 'R' U+0E00-0E7F Thai, 71M 0x0E8, # 0x53 'S' U+0E80-0EFF Lao, 7.7M 0x0F0, # 0x54 'T' U+0F00-0F7F Tibetan, 6M 0x100, # 0x55 'U' U+1000-107F Myanmar, 33M 0x120, # 0x56 'V' U+1200-127F Ethiopic, 135M 0x128, # 0x57 'W' U+1280-12FF Ethiopic 2 - Amharic, Tigrinya (89 everyday) 0x130, # 0x58 'X' U+1300-137F Ethiopic 3 - Amharic + punctuation and numerals (92) 0x178, # 0x59 'Y' U+1780-17FF Khmer, 18M 0x1E8, # 0x5A 'Z' U+1E80-1EFF Vietnamese, Yoruba, Igbo, Welsh - 164M None, # 0x5B '[' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x5C '\\' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x5D ']' excluded - structural in HTML/shell/CSV/URL/JSON WORLD, # 0x5E '^' indexed world page 0 WORLD, # 0x5F '_' indexed world page 1 None, # 0x60 '`' excluded - structural in HTML/shell/CSV/URL/JSON 0x4E0, # 0x61 'a' U+4E00-4E7F Chinese 0x4E8, # 0x62 'b' U+4E80-4EFF Chinese 0x538, # 0x63 'c' U+5380-53FF Chinese 0x540, # 0x64 'd' U+5400-547F Chinese 0x5B8, # 0x65 'e' U+5B80-5BFF Chinese 0x628, # 0x66 'f' U+6280-62FF Chinese 0x7E8, # 0x67 'g' U+7E80-7EFF Chinese 0x8B8, # 0x68 'h' U+8B80-8BFF Chinese 0x8D0, # 0x69 'i' U+8D00-8D7F Chinese 0x8F8, # 0x6A 'j' U+8F80-8FFF Chinese KOREAN, # 0x6B 'k' indexed Korean page 0 KOREAN, # 0x6C 'l' indexed Korean page 1 KOREAN, # 0x6D 'm' indexed Korean page 2 KOREAN, # 0x6E 'n' indexed Korean page 3 KOREAN, # 0x6F 'o' indexed Korean page 4 CHINESE, # 0x70 'p' indexed Chinese page 11 CHINESE, # 0x71 'q' indexed Chinese page 12 CHINESE, # 0x72 'r' indexed Chinese page 13 CHINESE, # 0x73 's' indexed Chinese page 14 CHINESE, # 0x74 't' indexed Chinese page 15 CHINESE, # 0x75 'u' indexed Chinese page 16 CHINESE, # 0x76 'v' indexed Chinese page 17 CHINESE, # 0x77 'w' indexed Chinese page 18 CHINESE, # 0x78 'x' indexed Chinese page 19 CHINESE, # 0x79 'y' indexed Chinese page 20 CHINESE, # 0x7A 'z' indexed Chinese page 21 None, # 0x7B '{' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x7C '|' excluded - structural in HTML/shell/CSV/URL/JSON None, # 0x7D '}' excluded - structural in HTML/shell/CSV/URL/JSON KOREAN, # 0x7E '~' indexed Korean page 5 None, # 0x7F ctrl unused ) TAIL_BYTES = ( 0x21, 0x28, 0x29, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5E, 0x5F, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x7E, ) TAIL_COUNT = len(TAIL_BYTES) TAIL_INDEX = tuple( TAIL_BYTES.index(b) if b in TAIL_BYTES else None for b in range(0x80) ) CHINESE_PAGE = tuple( CHINESE_TRAILS.index(b) if b in CHINESE_TRAILS else None for b in range(0x80) ) KOREAN_PAGE = tuple( KOREAN_TRAILS.index(b) if b in KOREAN_TRAILS else None for b in range(0x80) ) def handle_error(msg="Invalid code point"): raise ValueError(msg) def encode(code_point): """ Encode a Unicode code point into 1, 2 or 3 bytes """ if code_point < 0: handle_error("Negative code point") if code_point > MAX_CODE_POINT: handle_error("Beyond the Unicode range") # ------------------------- # TIER 1 - plain ascii passes straight through # ------------------------- if code_point < 0x80: return bytes([code_point]) # ------------------------- # TIER 2a - block pages: world scripts and the 8 Chinese blocks # ------------------------- trail = PAGE_TO_TRAIL.get((code_point >> 7) << 3) if trail is not None: return bytes([LEAD_BASE | (code_point & 0x7F), trail]) # ------------------------- # TIER 2b - indexed Chinese # ------------------------- slot = CHINESE_SLOT.get(code_point) if slot is not None: return bytes([LEAD_BASE | (slot & 0x7F), CHINESE_TRAILS[slot >> 7]]) # ------------------------- # TIER 2c - indexed Korean # ------------------------- slot = KOREAN_SLOT.get(code_point) if slot is not None: return bytes([LEAD_BASE | (slot & 0x7F), KOREAN_TRAILS[slot >> 7]]) # ------------------------- # TIER 2d - indexed world pages # ------------------------- slot = WORLD_SLOT.get(code_point) if slot is not None: return bytes([LEAD_BASE | (slot & 0x7F), WORLD_TRAILS[slot >> 7]]) # ------------------------- # TIER 3 - everything else # ------------------------- index = code_point - 0x0 tail = index % TAIL_COUNT rest = index // TAIL_COUNT return bytes([ LEAD_BASE | (rest >> 7), LEAD_BASE | (rest & 0x7F), TAIL_BYTES[tail], ]) def decode(data): """ Decode a 1, 2 or 3 byte sequence back into a Unicode code point A character is a run of lead bytes closed by one trail byte, so a sequence that runs out before the terminator arrives is not decodable. Every such case raises ValueError naming what is missing, rather than letting an IndexError escape from the table lookups. """ if not data: handle_error("Empty sequence: nothing to decode") first = data[0] if first < 0x80: return first if len(data) < 2: handle_error("Truncated sequence: 1 lead byte with no terminating byte") second = data[1] # ------------------------- # TIER 2 - second byte clears the high bit, so it terminates # ------------------------- if second < 0x80: page = TRAIL_TO_PAGE[second] if page is None: handle_error("Trail byte is not a page index") if page is WORLD: return WORLD_CODE_POINTS[(WORLD_PAGE[second] << 7) | (first & 0x7F)] if page is CHINESE: return CHINESE_CODE_POINTS[(CHINESE_PAGE[second] << 7) | (first & 0x7F)] if page is KOREAN: return KOREAN_CODE_POINTS[(KOREAN_PAGE[second] << 7) | (first & 0x7F)] return ((page >> 3) << 7) | (first & 0x7F) # ------------------------- # TIER 3 - second byte keeps the high bit, a third follows # ------------------------- if len(data) < 3: handle_error("Truncated sequence: 2 lead bytes with no terminating byte") third = data[2] if third >= 0x80: handle_error( "No terminating byte within 3 bytes: UCE-8 characters are at most 3" ) tail = TAIL_INDEX[third] if tail is None: handle_error("Third byte is not a legal terminating value") rest = ((first & 0x7F) << 7) | (second & 0x7F) code_point = rest * TAIL_COUNT + tail # ------------------------- # CANONICAL FORM - a character may only be spelled the shortest way # ------------------------- # Tier 3's arithmetic runs over every code point from zero, so it also # covers the 128 ASCII characters and the 8,704 that tier 2 reaches. The # encoder never emits those long forms, but a decoder that accepts them # gives every character two spellings - which is UTF-8's overlong bug, # including the dangerous case: 80 80 67 would otherwise decode to '/' # and walk straight past a validator scanning raw bytes for 0x2F. if len(encode(code_point)) < 3: handle_error( "Overlong sequence: U+%04X has a shorter form" % code_point ) return code_point if __name__ == "__main__": tests = [ "A", "\u00e9", "\u0416", "\u04e9", "\u0645", "\u0928", "\u0b95", "\u0e01", "\u0f40", "\u1200", "\u1290", "\u1308", "\u10d0", "\u1820", "\u3042", "\u4e2d", "\u7684", "\uc740", "\ud55c", "\u9fff", "\U0001f600", "\u20ac", "\u20b9", "\u20a9", "\u20bd", "\u20ba", "\u20bf", "\u0219", "\u0259", "\u1e63", "\u0253", "\u025b", "\u01ce", ] for ch in tests: code_point = ord(ch) data = encode(code_point) decoded = decode(data) status = "OK" if decoded == code_point else "MISMATCH" utf8 = len(ch.encode("utf-8")) shown = " ".join(f"{b:02X}" for b in data) kind = ("ascii" if code_point < 0x80 else "block" if PAGE_TO_TRAIL.get((code_point >> 7) << 3) is not None else "cn-idx" if code_point in CHINESE_SLOT else "kr-idx" if code_point in KOREAN_SLOT else "world" if code_point in WORLD_SLOT else "tier3") print( f"{ch!r:12} U+{code_point:04X} {len(data)}-byte [{shown:8}] {kind:6} " f"utf-8 {utf8} -> U+{decoded:04X} [{status}]" )