rpms/glibc
7
0
Fork 1
Code Issues Pull Requests Releases Activity
e61b8f41f2
glibc/glibc-rh1238412-update-the-translit-files-to-unicode-7.0.0.patch

5406 lines
200 KiB
Diff
Raw Normal View History

Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
From ee7a6e89183bdc8453dd9a0dadf133e65deb9e0b Mon Sep 17 00:00:00 2001
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
From: Mike FABIAN <mfabian@redhat.com>
Date: Fri, 17 Apr 2015 09:12:05 +0200
Subject: [PATCH 3/5] Update the translit files to Unicode 7.0.0
for localedata/ChangeLog
[BZ #16061]
* unicode_utils.py: New.
* existing scripts changed to used unicode_utils.py
* gen_translit_circle.py: New
* gen_translit_cjk_compat.py: New
* gen_translit_combining.py: New
* gen_translit_compat.py: New
* gen_translit_font.py: New
* gen_translit_fraction.py: New
* locales/translit_circle: Update.
* locales/translit_cjk_compat: Update.
* locales/translit_combining: Update.
* locales/translit_compat: Update.
* locales/translit_font: Update.
* locales/translit_fraction: Update.
---
localedata/locales/translit_circle | 30 +-
localedata/locales/translit_cjk_compat | 422 +++++++++++++-
localedata/locales/translit_combining | 636 +++++++++++++++++++++-
localedata/locales/translit_compat | 578 +++++++++++++++++++-
localedata/locales/translit_font | 151 ++++-
localedata/locales/translit_fraction | 15 +-
localedata/unicode-gen/Makefile | 42 +-
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
localedata/unicode-gen/gen_translit_circle.py | 150 +++++
localedata/unicode-gen/gen_translit_cjk_compat.py | 220 ++++++++
localedata/unicode-gen/gen_translit_combining.py | 442 +++++++++++++++
localedata/unicode-gen/gen_translit_compat.py | 326 +++++++++++
localedata/unicode-gen/gen_translit_font.py | 156 ++++++
localedata/unicode-gen/gen_translit_fraction.py | 197 +++++++
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
localedata/unicode-gen/gen_unicode_ctype.py | 497 +----------------
localedata/unicode-gen/unicode_utils.py | 502 +++++++++++++++++
localedata/unicode-gen/utf8_compatibility.py | 217 ++------
localedata/unicode-gen/utf8_gen.py | 28 +-
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
17 files changed, 3896 insertions(+), 713 deletions(-)
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
create mode 100755 localedata/unicode-gen/gen_translit_circle.py
create mode 100755 localedata/unicode-gen/gen_translit_cjk_compat.py
create mode 100755 localedata/unicode-gen/gen_translit_combining.py
create mode 100755 localedata/unicode-gen/gen_translit_compat.py
create mode 100755 localedata/unicode-gen/gen_translit_font.py
create mode 100755 localedata/unicode-gen/gen_translit_fraction.py
create mode 100644 localedata/unicode-gen/unicode_utils.py
diff --git a/localedata/locales/translit_circle b/localedata/locales/translit_circle
index f701bc9..5d5f58c 100644
--- a/localedata/locales/translit_circle
+++ b/localedata/locales/translit_circle
@@ -2,9 +2,7 @@ escape_char /
comment_char %
% Transliterations of encircled characters.
-% Generated through
-% $ grep '^[^;]*;[^;]*;[^;]*;[^;]*;[^;]*;<circle>[^;]*;' UnicodeData.txt | \
-% sed -e 's/^\([^;]*\);\([^;]*\);[^;]*;[^;]*;[^;]*;<circle> \([^;]*\);.*$/<U\1> "<U0028 \3 0029>"% \2/' -e 'h' -e 's/^\([^%]*\)% .*$/\1/' -e 's/\([0-9A-F]\) \([0-9A-F]\)/\1><U\2/g' -e 'x' -e 's/^[^%]*\(% .*\)$/\1/' -e 'G'
+% Generated automatically from UnicodeData.txt by gen_translit_circle.py on 2015-06-10 for Unicode 7.0.0.
LC_CTYPE
@@ -156,6 +154,14 @@ translit_start
<U24E9> "<U0028><U007A><U0029>"
% CIRCLED DIGIT ZERO
<U24EA> "<U0028><U0030><U0029>"
+% CIRCLED IDEOGRAPH QUESTION
+<U3244> "<U0028><U554F><U0029>"
+% CIRCLED IDEOGRAPH KINDERGARTEN
+<U3245> "<U0028><U5E7C><U0029>"
+% CIRCLED IDEOGRAPH SCHOOL
+<U3246> "<U0028><U6587><U0029>"
+% CIRCLED IDEOGRAPH KOTO
+<U3247> "<U0028><U7B8F><U0029>"
% CIRCLED NUMBER TWENTY ONE
<U3251> "<U0028><U0032><U0031><U0029>"
% CIRCLED NUMBER TWENTY TWO
@@ -242,6 +248,12 @@ translit_start
<U327A> "<U0028><U1111><U1161><U0029>"
% CIRCLED HANGUL HIEUH A
<U327B> "<U0028><U1112><U1161><U0029>"
+% CIRCLED KOREAN CHARACTER CHAMKO
+<U327C> "<U0028><U110E><U1161><U11B7><U1100><U1169><U0029>"
+% CIRCLED KOREAN CHARACTER JUEUI
+<U327D> "<U0028><U110C><U116E><U110B><U1174><U0029>"
+% CIRCLED HANGUL IEUNG U
+<U327E> "<U0028><U110B><U116E><U0029>"
% CIRCLED IDEOGRAPH ONE
<U3280> "<U0028><U4E00><U0029>"
% CIRCLED IDEOGRAPH TWO
@@ -464,6 +476,18 @@ translit_start
<U32FD> "<U0028><U30F1><U0029>"
% CIRCLED KATAKANA WO
<U32FE> "<U0028><U30F2><U0029>"
+% CIRCLED ITALIC LATIN CAPITAL LETTER C
+<U0001F12B> "<U0028><U0043><U0029>"
+% CIRCLED ITALIC LATIN CAPITAL LETTER R
+<U0001F12C> "<U0028><U0052><U0029>"
+% CIRCLED CD
+<U0001F12D> "<U0028><U0043><U0044><U0029>"
+% CIRCLED WZ
+<U0001F12E> "<U0028><U0057><U005A><U0029>"
+% CIRCLED IDEOGRAPH ADVANTAGE
+<U0001F250> "<U0028><U5F97><U0029>"
+% CIRCLED IDEOGRAPH ACCEPT
+<U0001F251> "<U0028><U53EF><U0029>"
translit_end
diff --git a/localedata/locales/translit_cjk_compat b/localedata/locales/translit_cjk_compat
index c73e5e3..a20c6ca 100644
--- a/localedata/locales/translit_cjk_compat
+++ b/localedata/locales/translit_cjk_compat
@@ -2,18 +2,22 @@ escape_char /
comment_char %
% Transliterations of CJK compatibility characters.
-% Generated through
-% $ grep '^[^;]*;[^;]*;[^;]*;[^;]*;[^;]*;<square>[^;]*;' UnicodeData.txt | \
-% sed -e 's/^\([^;]*\);\([^;]*\);[^;]*;[^;]*;[^;]*;<square> \([^;]*\);.*$/<U\1> "<U\3>"% \2/' | sed -e 'h' -e 's/^\([^%]*\)% .*$/\1/' -e 's/\([0-9A-F]\) \([0-9A-F]\)/\1><U\2/g' -e 'x' -e 's/^[^%]*\(% .*\)$/\1/' -e 'G'
-% and
-% $ grep '[^;]*;CJK COMPATIBILITY IDEOGRAPH[^;]*;[^;]*;[^;]*;[^;]*;[^;]' UnicodeData.txt | \
-% sed -e 's/^\([^;]*\);\([^;]*\);[^;]*;[^;]*;[^;]*;\([^;]*\);.*$/<U\1> <U\3>% \2/' | sed -e 'h' -e 's/^\([^%]*\)% .*$/\1/' -e 's/\([0-9A-F]\) \([0-9A-F]\)/\1><U\2/g' -e 'x' -e 's/^[^%]*\(% .*\)$/\1/' -e 'G' | \
-% sed -e 's/<U\(.....\)>/<U000\1>/g'
+% Generated automatically from UnicodeData.txt by gen_translit_cjk_compat.py on 2015-06-10 for Unicode 7.0.0.
LC_CTYPE
translit_start
+% PARTNERSHIP SIGN
+<U3250> "<U0050><U0054><U0045>"
+% SQUARE HG
+<U32CC> "<U0048><U0067>"
+% SQUARE ERG
+<U32CD> "<U0065><U0072><U0067>"
+% SQUARE EV
+<U32CE> "<U0065><U0056>"
+% LIMITED LIABILITY SIGN
+<U32CF> "<U004C><U0054><U0044>"
% SQUARE APAATO
<U3300> "<U30A2><U30D1><U30FC><U30C8>"
% SQUARE ARUHUA
@@ -202,6 +206,14 @@ translit_start
<U3375> "<U006F><U0056>"
% SQUARE PC
<U3376> "<U0070><U0063>"
+% SQUARE DM
+<U3377> "<U0064><U006D>"
+% SQUARE DM SQUARED
+<U3378> "<U0064><U006D><U00B2>";"<U0064><U006D><U005E><U0032>"
+% SQUARE DM CUBED
+<U3379> "<U0064><U006D><U00B3>";"<U0064><U006D><U005E><U0033>"
+% SQUARE IU
+<U337A> "<U0049><U0055>"
% SQUARE ERA NAME HEISEI
<U337B> "<U5E73><U6210>"
% SQUARE ERA NAME SYOUWA
@@ -400,6 +412,170 @@ translit_start
<U33DC> "<U0053><U0076>"
% SQUARE WB
<U33DD> "<U0057><U0062>"
+% SQUARE V OVER M
+<U33DE> "<U0056><U2215><U006D>";"<U0056><U002F><U006D>"
+% SQUARE A OVER M
+<U33DF> "<U0041><U2215><U006D>";"<U0041><U002F><U006D>"
+% SQUARE GAL
+<U33FF> "<U0067><U0061><U006C>"
+% SQUARED LATIN CAPITAL LETTER A
+<U0001F130> <U0041>
+% SQUARED LATIN CAPITAL LETTER B
+<U0001F131> <U0042>
+% SQUARED LATIN CAPITAL LETTER C
+<U0001F132> <U0043>
+% SQUARED LATIN CAPITAL LETTER D
+<U0001F133> <U0044>
+% SQUARED LATIN CAPITAL LETTER E
+<U0001F134> <U0045>
+% SQUARED LATIN CAPITAL LETTER F
+<U0001F135> <U0046>
+% SQUARED LATIN CAPITAL LETTER G
+<U0001F136> <U0047>
+% SQUARED LATIN CAPITAL LETTER H
+<U0001F137> <U0048>
+% SQUARED LATIN CAPITAL LETTER I
+<U0001F138> <U0049>
+% SQUARED LATIN CAPITAL LETTER J
+<U0001F139> <U004A>
+% SQUARED LATIN CAPITAL LETTER K
+<U0001F13A> <U004B>
+% SQUARED LATIN CAPITAL LETTER L
+<U0001F13B> <U004C>
+% SQUARED LATIN CAPITAL LETTER M
+<U0001F13C> <U004D>
+% SQUARED LATIN CAPITAL LETTER N
+<U0001F13D> <U004E>
+% SQUARED LATIN CAPITAL LETTER O
+<U0001F13E> <U004F>
+% SQUARED LATIN CAPITAL LETTER P
+<U0001F13F> <U0050>
+% SQUARED LATIN CAPITAL LETTER Q
+<U0001F140> <U0051>
+% SQUARED LATIN CAPITAL LETTER R
+<U0001F141> <U0052>
+% SQUARED LATIN CAPITAL LETTER S
+<U0001F142> <U0053>
+% SQUARED LATIN CAPITAL LETTER T
+<U0001F143> <U0054>
+% SQUARED LATIN CAPITAL LETTER U
+<U0001F144> <U0055>
+% SQUARED LATIN CAPITAL LETTER V
+<U0001F145> <U0056>
+% SQUARED LATIN CAPITAL LETTER W
+<U0001F146> <U0057>
+% SQUARED LATIN CAPITAL LETTER X
+<U0001F147> <U0058>
+% SQUARED LATIN CAPITAL LETTER Y
+<U0001F148> <U0059>
+% SQUARED LATIN CAPITAL LETTER Z
+<U0001F149> <U005A>
+% SQUARED HV
+<U0001F14A> "<U0048><U0056>"
+% SQUARED MV
+<U0001F14B> "<U004D><U0056>"
+% SQUARED SD
+<U0001F14C> "<U0053><U0044>"
+% SQUARED SS
+<U0001F14D> "<U0053><U0053>"
+% SQUARED PPV
+<U0001F14E> "<U0050><U0050><U0056>"
+% SQUARED WC
+<U0001F14F> "<U0057><U0043>"
+% SQUARE DJ
+<U0001F190> "<U0044><U004A>"
+% SQUARE HIRAGANA HOKA
+<U0001F200> "<U307B><U304B>"
+% SQUARED KATAKANA KOKO
+<U0001F201> "<U30B3><U30B3>"
+% SQUARED KATAKANA SA
+<U0001F202> <U30B5>
+% SQUARED CJK UNIFIED IDEOGRAPH-624B
+<U0001F210> <U624B>
+% SQUARED CJK UNIFIED IDEOGRAPH-5B57
+<U0001F211> <U5B57>
+% SQUARED CJK UNIFIED IDEOGRAPH-53CC
+<U0001F212> <U53CC>
+% SQUARED KATAKANA DE
+<U0001F213> <U30C7>
+% SQUARED CJK UNIFIED IDEOGRAPH-4E8C
+<U0001F214> <U4E8C>
+% SQUARED CJK UNIFIED IDEOGRAPH-591A
+<U0001F215> <U591A>
+% SQUARED CJK UNIFIED IDEOGRAPH-89E3
+<U0001F216> <U89E3>
+% SQUARED CJK UNIFIED IDEOGRAPH-5929
+<U0001F217> <U5929>
+% SQUARED CJK UNIFIED IDEOGRAPH-4EA4
+<U0001F218> <U4EA4>
+% SQUARED CJK UNIFIED IDEOGRAPH-6620
+<U0001F219> <U6620>
+% SQUARED CJK UNIFIED IDEOGRAPH-7121
+<U0001F21A> <U7121>
+% SQUARED CJK UNIFIED IDEOGRAPH-6599
+<U0001F21B> <U6599>
+% SQUARED CJK UNIFIED IDEOGRAPH-524D
+<U0001F21C> <U524D>
+% SQUARED CJK UNIFIED IDEOGRAPH-5F8C
+<U0001F21D> <U5F8C>
+% SQUARED CJK UNIFIED IDEOGRAPH-518D
+<U0001F21E> <U518D>
+% SQUARED CJK UNIFIED IDEOGRAPH-65B0
+<U0001F21F> <U65B0>
+% SQUARED CJK UNIFIED IDEOGRAPH-521D
+<U0001F220> <U521D>
+% SQUARED CJK UNIFIED IDEOGRAPH-7D42
+<U0001F221> <U7D42>
+% SQUARED CJK UNIFIED IDEOGRAPH-751F
+<U0001F222> <U751F>
+% SQUARED CJK UNIFIED IDEOGRAPH-8CA9
+<U0001F223> <U8CA9>
+% SQUARED CJK UNIFIED IDEOGRAPH-58F0
+<U0001F224> <U58F0>
+% SQUARED CJK UNIFIED IDEOGRAPH-5439
+<U0001F225> <U5439>
+% SQUARED CJK UNIFIED IDEOGRAPH-6F14
+<U0001F226> <U6F14>
+% SQUARED CJK UNIFIED IDEOGRAPH-6295
+<U0001F227> <U6295>
+% SQUARED CJK UNIFIED IDEOGRAPH-6355
+<U0001F228> <U6355>
+% SQUARED CJK UNIFIED IDEOGRAPH-4E00
+<U0001F229> <U4E00>
+% SQUARED CJK UNIFIED IDEOGRAPH-4E09
+<U0001F22A> <U4E09>
+% SQUARED CJK UNIFIED IDEOGRAPH-904A
+<U0001F22B> <U904A>
+% SQUARED CJK UNIFIED IDEOGRAPH-5DE6
+<U0001F22C> <U5DE6>
+% SQUARED CJK UNIFIED IDEOGRAPH-4E2D
+<U0001F22D> <U4E2D>
+% SQUARED CJK UNIFIED IDEOGRAPH-53F3
+<U0001F22E> <U53F3>
+% SQUARED CJK UNIFIED IDEOGRAPH-6307
+<U0001F22F> <U6307>
+% SQUARED CJK UNIFIED IDEOGRAPH-8D70
+<U0001F230> <U8D70>
+% SQUARED CJK UNIFIED IDEOGRAPH-6253
+<U0001F231> <U6253>
+% SQUARED CJK UNIFIED IDEOGRAPH-7981
+<U0001F232> <U7981>
+% SQUARED CJK UNIFIED IDEOGRAPH-7A7A
+<U0001F233> <U7A7A>
+% SQUARED CJK UNIFIED IDEOGRAPH-5408
+<U0001F234> <U5408>
+% SQUARED CJK UNIFIED IDEOGRAPH-6E80
+<U0001F235> <U6E80>
+% SQUARED CJK UNIFIED IDEOGRAPH-6709
+<U0001F236> <U6709>
+% SQUARED CJK UNIFIED IDEOGRAPH-6708
+<U0001F237> <U6708>
+% SQUARED CJK UNIFIED IDEOGRAPH-7533
+<U0001F238> <U7533>
+% SQUARED CJK UNIFIED IDEOGRAPH-5272
+<U0001F239> <U5272>
+% SQUARED CJK UNIFIED IDEOGRAPH-55B6
+<U0001F23A> <U55B6>
% CJK COMPATIBILITY IDEOGRAPH-F900
<UF900> <U8C48>
% CJK COMPATIBILITY IDEOGRAPH-F901
@@ -980,6 +1156,10 @@ translit_start
<UFA2C> <U9928>
% CJK COMPATIBILITY IDEOGRAPH-FA2D
<UFA2D> <U9DB4>
+% CJK COMPATIBILITY IDEOGRAPH-FA2E
+<UFA2E> <U90DE>
+% CJK COMPATIBILITY IDEOGRAPH-FA2F
+<UFA2F> <U96B7>
% CJK COMPATIBILITY IDEOGRAPH-FA30
<UFA30> <U4FAE>
% CJK COMPATIBILITY IDEOGRAPH-FA31
@@ -1098,6 +1278,224 @@ translit_start
<UFA69> <U97FF>
% CJK COMPATIBILITY IDEOGRAPH-FA6A
<UFA6A> <U983B>
+% CJK COMPATIBILITY IDEOGRAPH-FA6B
+<UFA6B> <U6075>
+% CJK COMPATIBILITY IDEOGRAPH-FA6C
+<UFA6C> <U000242EE>
+% CJK COMPATIBILITY IDEOGRAPH-FA6D
+<UFA6D> <U8218>
+% CJK COMPATIBILITY IDEOGRAPH-FA70
+<UFA70> <U4E26>
+% CJK COMPATIBILITY IDEOGRAPH-FA71
+<UFA71> <U51B5>
+% CJK COMPATIBILITY IDEOGRAPH-FA72
+<UFA72> <U5168>
+% CJK COMPATIBILITY IDEOGRAPH-FA73
+<UFA73> <U4F80>
+% CJK COMPATIBILITY IDEOGRAPH-FA74
+<UFA74> <U5145>
+% CJK COMPATIBILITY IDEOGRAPH-FA75
+<UFA75> <U5180>
+% CJK COMPATIBILITY IDEOGRAPH-FA76
+<UFA76> <U52C7>
+% CJK COMPATIBILITY IDEOGRAPH-FA77
+<UFA77> <U52FA>
+% CJK COMPATIBILITY IDEOGRAPH-FA78
+<UFA78> <U559D>
+% CJK COMPATIBILITY IDEOGRAPH-FA79
+<UFA79> <U5555>
+% CJK COMPATIBILITY IDEOGRAPH-FA7A
+<UFA7A> <U5599>
+% CJK COMPATIBILITY IDEOGRAPH-FA7B
+<UFA7B> <U55E2>
+% CJK COMPATIBILITY IDEOGRAPH-FA7C
+<UFA7C> <U585A>
+% CJK COMPATIBILITY IDEOGRAPH-FA7D
+<UFA7D> <U58B3>
+% CJK COMPATIBILITY IDEOGRAPH-FA7E
+<UFA7E> <U5944>
+% CJK COMPATIBILITY IDEOGRAPH-FA7F
+<UFA7F> <U5954>
+% CJK COMPATIBILITY IDEOGRAPH-FA80
+<UFA80> <U5A62>
+% CJK COMPATIBILITY IDEOGRAPH-FA81
+<UFA81> <U5B28>
+% CJK COMPATIBILITY IDEOGRAPH-FA82
+<UFA82> <U5ED2>
+% CJK COMPATIBILITY IDEOGRAPH-FA83
+<UFA83> <U5ED9>
+% CJK COMPATIBILITY IDEOGRAPH-FA84
+<UFA84> <U5F69>
+% CJK COMPATIBILITY IDEOGRAPH-FA85
+<UFA85> <U5FAD>
+% CJK COMPATIBILITY IDEOGRAPH-FA86
+<UFA86> <U60D8>
+% CJK COMPATIBILITY IDEOGRAPH-FA87
+<UFA87> <U614E>
+% CJK COMPATIBILITY IDEOGRAPH-FA88
+<UFA88> <U6108>
+% CJK COMPATIBILITY IDEOGRAPH-FA89
+<UFA89> <U618E>
+% CJK COMPATIBILITY IDEOGRAPH-FA8A
+<UFA8A> <U6160>
+% CJK COMPATIBILITY IDEOGRAPH-FA8B
+<UFA8B> <U61F2>
+% CJK COMPATIBILITY IDEOGRAPH-FA8C
+<UFA8C> <U6234>
+% CJK COMPATIBILITY IDEOGRAPH-FA8D
+<UFA8D> <U63C4>
+% CJK COMPATIBILITY IDEOGRAPH-FA8E
+<UFA8E> <U641C>
+% CJK COMPATIBILITY IDEOGRAPH-FA8F
+<UFA8F> <U6452>
+% CJK COMPATIBILITY IDEOGRAPH-FA90
+<UFA90> <U6556>
+% CJK COMPATIBILITY IDEOGRAPH-FA91
+<UFA91> <U6674>
+% CJK COMPATIBILITY IDEOGRAPH-FA92
+<UFA92> <U6717>
+% CJK COMPATIBILITY IDEOGRAPH-FA93
+<UFA93> <U671B>
+% CJK COMPATIBILITY IDEOGRAPH-FA94
+<UFA94> <U6756>
+% CJK COMPATIBILITY IDEOGRAPH-FA95
+<UFA95> <U6B79>
+% CJK COMPATIBILITY IDEOGRAPH-FA96
+<UFA96> <U6BBA>
+% CJK COMPATIBILITY IDEOGRAPH-FA97
+<UFA97> <U6D41>
+% CJK COMPATIBILITY IDEOGRAPH-FA98
+<UFA98> <U6EDB>
+% CJK COMPATIBILITY IDEOGRAPH-FA99
+<UFA99> <U6ECB>
+% CJK COMPATIBILITY IDEOGRAPH-FA9A
+<UFA9A> <U6F22>
+% CJK COMPATIBILITY IDEOGRAPH-FA9B
+<UFA9B> <U701E>
+% CJK COMPATIBILITY IDEOGRAPH-FA9C
+<UFA9C> <U716E>
+% CJK COMPATIBILITY IDEOGRAPH-FA9D
+<UFA9D> <U77A7>
+% CJK COMPATIBILITY IDEOGRAPH-FA9E
+<UFA9E> <U7235>
+% CJK COMPATIBILITY IDEOGRAPH-FA9F
+<UFA9F> <U72AF>
+% CJK COMPATIBILITY IDEOGRAPH-FAA0
+<UFAA0> <U732A>
+% CJK COMPATIBILITY IDEOGRAPH-FAA1
+<UFAA1> <U7471>
+% CJK COMPATIBILITY IDEOGRAPH-FAA2
+<UFAA2> <U7506>
+% CJK COMPATIBILITY IDEOGRAPH-FAA3
+<UFAA3> <U753B>
+% CJK COMPATIBILITY IDEOGRAPH-FAA4
+<UFAA4> <U761D>
+% CJK COMPATIBILITY IDEOGRAPH-FAA5
+<UFAA5> <U761F>
+% CJK COMPATIBILITY IDEOGRAPH-FAA6
+<UFAA6> <U76CA>
+% CJK COMPATIBILITY IDEOGRAPH-FAA7
+<UFAA7> <U76DB>
+% CJK COMPATIBILITY IDEOGRAPH-FAA8
+<UFAA8> <U76F4>
+% CJK COMPATIBILITY IDEOGRAPH-FAA9
+<UFAA9> <U774A>
+% CJK COMPATIBILITY IDEOGRAPH-FAAA
+<UFAAA> <U7740>
+% CJK COMPATIBILITY IDEOGRAPH-FAAB
+<UFAAB> <U78CC>
+% CJK COMPATIBILITY IDEOGRAPH-FAAC
+<UFAAC> <U7AB1>
+% CJK COMPATIBILITY IDEOGRAPH-FAAD
+<UFAAD> <U7BC0>
+% CJK COMPATIBILITY IDEOGRAPH-FAAE
+<UFAAE> <U7C7B>
+% CJK COMPATIBILITY IDEOGRAPH-FAAF
+<UFAAF> <U7D5B>
+% CJK COMPATIBILITY IDEOGRAPH-FAB0
+<UFAB0> <U7DF4>
+% CJK COMPATIBILITY IDEOGRAPH-FAB1
+<UFAB1> <U7F3E>
+% CJK COMPATIBILITY IDEOGRAPH-FAB2
+<UFAB2> <U8005>
+% CJK COMPATIBILITY IDEOGRAPH-FAB3
+<UFAB3> <U8352>
+% CJK COMPATIBILITY IDEOGRAPH-FAB4
+<UFAB4> <U83EF>
+% CJK COMPATIBILITY IDEOGRAPH-FAB5
+<UFAB5> <U8779>
+% CJK COMPATIBILITY IDEOGRAPH-FAB6
+<UFAB6> <U8941>
+% CJK COMPATIBILITY IDEOGRAPH-FAB7
+<UFAB7> <U8986>
+% CJK COMPATIBILITY IDEOGRAPH-FAB8
+<UFAB8> <U8996>
+% CJK COMPATIBILITY IDEOGRAPH-FAB9
+<UFAB9> <U8ABF>
+% CJK COMPATIBILITY IDEOGRAPH-FABA
+<UFABA> <U8AF8>
+% CJK COMPATIBILITY IDEOGRAPH-FABB
+<UFABB> <U8ACB>
+% CJK COMPATIBILITY IDEOGRAPH-FABC
+<UFABC> <U8B01>
+% CJK COMPATIBILITY IDEOGRAPH-FABD
+<UFABD> <U8AFE>
+% CJK COMPATIBILITY IDEOGRAPH-FABE
+<UFABE> <U8AED>
+% CJK COMPATIBILITY IDEOGRAPH-FABF
+<UFABF> <U8B39>
+% CJK COMPATIBILITY IDEOGRAPH-FAC0
+<UFAC0> <U8B8A>
+% CJK COMPATIBILITY IDEOGRAPH-FAC1
+<UFAC1> <U8D08>
+% CJK COMPATIBILITY IDEOGRAPH-FAC2
+<UFAC2> <U8F38>
+% CJK COMPATIBILITY IDEOGRAPH-FAC3
+<UFAC3> <U9072>
+% CJK COMPATIBILITY IDEOGRAPH-FAC4
+<UFAC4> <U9199>
+% CJK COMPATIBILITY IDEOGRAPH-FAC5
+<UFAC5> <U9276>
+% CJK COMPATIBILITY IDEOGRAPH-FAC6
+<UFAC6> <U967C>
+% CJK COMPATIBILITY IDEOGRAPH-FAC7
+<UFAC7> <U96E3>
+% CJK COMPATIBILITY IDEOGRAPH-FAC8
+<UFAC8> <U9756>
+% CJK COMPATIBILITY IDEOGRAPH-FAC9
+<UFAC9> <U97DB>
+% CJK COMPATIBILITY IDEOGRAPH-FACA
+<UFACA> <U97FF>
+% CJK COMPATIBILITY IDEOGRAPH-FACB
+<UFACB> <U980B>
+% CJK COMPATIBILITY IDEOGRAPH-FACC
+<UFACC> <U983B>
+% CJK COMPATIBILITY IDEOGRAPH-FACD
+<UFACD> <U9B12>
+% CJK COMPATIBILITY IDEOGRAPH-FACE
+<UFACE> <U9F9C>
+% CJK COMPATIBILITY IDEOGRAPH-FACF
+<UFACF> <U0002284A>
+% CJK COMPATIBILITY IDEOGRAPH-FAD0
+<UFAD0> <U00022844>
+% CJK COMPATIBILITY IDEOGRAPH-FAD1
+<UFAD1> <U000233D5>
+% CJK COMPATIBILITY IDEOGRAPH-FAD2
+<UFAD2> <U3B9D>
+% CJK COMPATIBILITY IDEOGRAPH-FAD3
+<UFAD3> <U4018>
+% CJK COMPATIBILITY IDEOGRAPH-FAD4
+<UFAD4> <U4039>
+% CJK COMPATIBILITY IDEOGRAPH-FAD5
+<UFAD5> <U00025249>
+% CJK COMPATIBILITY IDEOGRAPH-FAD6
+<UFAD6> <U00025CD0>
+% CJK COMPATIBILITY IDEOGRAPH-FAD7
+<UFAD7> <U00027ED3>
+% CJK COMPATIBILITY IDEOGRAPH-FAD8
+<UFAD8> <U9F43>
+% CJK COMPATIBILITY IDEOGRAPH-FAD9
+<UFAD9> <U9F8E>
% CJK COMPATIBILITY IDEOGRAPH-2F800
<U0002F800> <U4E3D>
% CJK COMPATIBILITY IDEOGRAPH-2F801
@@ -1307,7 +1705,7 @@ translit_start
% CJK COMPATIBILITY IDEOGRAPH-2F867
<U0002F867> <U36EE>
% CJK COMPATIBILITY IDEOGRAPH-2F868
-<U0002F868> <U0002136A>
+<U0002F868> <U36FC>
% CJK COMPATIBILITY IDEOGRAPH-2F869
<U0002F869> <U5B08>
% CJK COMPATIBILITY IDEOGRAPH-2F86A
@@ -1331,7 +1729,7 @@ translit_start
% CJK COMPATIBILITY IDEOGRAPH-2F873
<U0002F873> <U5C06>
% CJK COMPATIBILITY IDEOGRAPH-2F874
-<U0002F874> <U5F33>
+<U0002F874> <U5F53>
% CJK COMPATIBILITY IDEOGRAPH-2F875
<U0002F875> <U5C22>
% CJK COMPATIBILITY IDEOGRAPH-2F876
@@ -1673,7 +2071,7 @@ translit_start
% CJK COMPATIBILITY IDEOGRAPH-2F91E
<U0002F91E> <U719C>
% CJK COMPATIBILITY IDEOGRAPH-2F91F
-<U0002F91F> <U43AB>
+<U0002F91F> <U000243AB>
% CJK COMPATIBILITY IDEOGRAPH-2F920
<U0002F920> <U7228>
% CJK COMPATIBILITY IDEOGRAPH-2F921
@@ -1801,7 +2199,7 @@ translit_start
% CJK COMPATIBILITY IDEOGRAPH-2F95E
<U0002F95E> <U00025AA7>
% CJK COMPATIBILITY IDEOGRAPH-2F95F
-<U0002F95F> <U7AAE>
+<U0002F95F> <U7AEE>
% CJK COMPATIBILITY IDEOGRAPH-2F960
<U0002F960> <U4202>
% CJK COMPATIBILITY IDEOGRAPH-2F961
@@ -1993,7 +2391,7 @@ translit_start
% CJK COMPATIBILITY IDEOGRAPH-2F9BE
<U0002F9BE> <U8786>
% CJK COMPATIBILITY IDEOGRAPH-2F9BF
-<U0002F9BF> <U4D57>
+<U0002F9BF> <U45D7>
% CJK COMPATIBILITY IDEOGRAPH-2F9C0
<U0002F9C0> <U87E1>
% CJK COMPATIBILITY IDEOGRAPH-2F9C1
diff --git a/localedata/locales/translit_combining b/localedata/locales/translit_combining
index 44c62f9..b1b5345 100644
--- a/localedata/locales/translit_combining
+++ b/localedata/locales/translit_combining
@@ -3,7 +3,7 @@ comment_char %
% Transliterations that remove all combining characters (accents,
% pronounciation marks, etc.).
-% Generated from UnicodeData.txt.
+% Generated automatically from UnicodeData.txt by gen_translit_combining.py on 2015-06-10 for Unicode 7.0.0.
LC_CTYPE
@@ -167,6 +167,40 @@ translit_start
<U034D> ""
% COMBINING UPWARDS ARROW BELOW
<U034E> ""
+% COMBINING GRAPHEME JOINER
+<U034F> ""
+% COMBINING RIGHT ARROWHEAD ABOVE
+<U0350> ""
+% COMBINING LEFT HALF RING ABOVE
+<U0351> ""
+% COMBINING FERMATA
+<U0352> ""
+% COMBINING X BELOW
+<U0353> ""
+% COMBINING LEFT ARROWHEAD BELOW
+<U0354> ""
+% COMBINING RIGHT ARROWHEAD BELOW
+<U0355> ""
+% COMBINING RIGHT ARROWHEAD AND UP ARROWHEAD BELOW
+<U0356> ""
+% COMBINING RIGHT HALF RING ABOVE
+<U0357> ""
+% COMBINING DOT ABOVE RIGHT
+<U0358> ""
+% COMBINING ASTERISK BELOW
+<U0359> ""
+% COMBINING DOUBLE RING BELOW
+<U035A> ""
+% COMBINING ZIGZAG ABOVE
+<U035B> ""
+% COMBINING DOUBLE BREVE BELOW
+<U035C> ""
+% COMBINING DOUBLE BREVE
+<U035D> ""
+% COMBINING DOUBLE MACRON
+<U035E> ""
+% COMBINING DOUBLE MACRON BELOW
+<U035F> ""
% COMBINING DOUBLE TILDE
<U0360> ""
% COMBINING DOUBLE INVERTED BREVE
@@ -199,6 +233,68 @@ translit_start
<U036E> ""
% COMBINING LATIN SMALL LETTER X
<U036F> ""
+% HEBREW ACCENT ETNAHTA
+<U0591> ""
+% HEBREW ACCENT SEGOL
+<U0592> ""
+% HEBREW ACCENT SHALSHELET
+<U0593> ""
+% HEBREW ACCENT ZAQEF QATAN
+<U0594> ""
+% HEBREW ACCENT ZAQEF GADOL
+<U0595> ""
+% HEBREW ACCENT TIPEHA
+<U0596> ""
+% HEBREW ACCENT REVIA
+<U0597> ""
+% HEBREW ACCENT ZARQA
+<U0598> ""
+% HEBREW ACCENT PASHTA
+<U0599> ""
+% HEBREW ACCENT YETIV
+<U059A> ""
+% HEBREW ACCENT TEVIR
+<U059B> ""
+% HEBREW ACCENT GERESH
+<U059C> ""
+% HEBREW ACCENT GERESH MUQDAM
+<U059D> ""
+% HEBREW ACCENT GERSHAYIM
+<U059E> ""
+% HEBREW ACCENT QARNEY PARA
+<U059F> ""
+% HEBREW ACCENT TELISHA GEDOLA
+<U05A0> ""
+% HEBREW ACCENT PAZER
+<U05A1> ""
+% HEBREW ACCENT ATNAH HAFUKH
+<U05A2> ""
+% HEBREW ACCENT MUNAH
+<U05A3> ""
+% HEBREW ACCENT MAHAPAKH
+<U05A4> ""
+% HEBREW ACCENT MERKHA
+<U05A5> ""
+% HEBREW ACCENT MERKHA KEFULA
+<U05A6> ""
+% HEBREW ACCENT DARGA
+<U05A7> ""
+% HEBREW ACCENT QADMA
+<U05A8> ""
+% HEBREW ACCENT TELISHA QETANA
+<U05A9> ""
+% HEBREW ACCENT YERAH BEN YOMO
+<U05AA> ""
+% HEBREW ACCENT OLE
+<U05AB> ""
+% HEBREW ACCENT ILUY
+<U05AC> ""
+% HEBREW ACCENT DEHI
+<U05AD> ""
+% HEBREW ACCENT ZINOR
+<U05AE> ""
+% HEBREW MARK MASORA CIRCLE
+<U05AF> ""
% HEBREW POINT SHEVA
<U05B0> ""
% HEBREW POINT HATAF SEGOL
@@ -219,6 +315,8 @@ translit_start
<U05B8> ""
% HEBREW POINT HOLAM
<U05B9> ""
+% HEBREW POINT HOLAM HASER FOR VAV
+<U05BA> ""
% HEBREW POINT QUBUTS
<U05BB> ""
% HEBREW POINT DAGESH OR MAPIQ
@@ -231,12 +329,358 @@ translit_start
<U05C1> ""
% HEBREW POINT SIN DOT
<U05C2> ""
+% HEBREW MARK UPPER DOT
+<U05C4> ""
+% HEBREW MARK LOWER DOT
+<U05C5> ""
+% HEBREW POINT QAMATS QATAN
+<U05C7> ""
+% ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM
+<U0610> ""
+% ARABIC SIGN ALAYHE ASSALLAM
+<U0611> ""
+% ARABIC SIGN RAHMATULLAH ALAYHE
+<U0612> ""
+% ARABIC SIGN RADI ALLAHOU ANHU
+<U0613> ""
+% ARABIC SIGN TAKHALLUS
+<U0614> ""
+% ARABIC SMALL HIGH TAH
+<U0615> ""
+% ARABIC SMALL HIGH LIGATURE ALEF WITH LAM WITH YEH
+<U0616> ""
+% ARABIC SMALL HIGH ZAIN
+<U0617> ""
+% ARABIC SMALL FATHA
+<U0618> ""
+% ARABIC SMALL DAMMA
+<U0619> ""
+% ARABIC SMALL KASRA
+<U061A> ""
+% ARABIC FATHATAN
+<U064B> ""
+% ARABIC DAMMATAN
+<U064C> ""
+% ARABIC KASRATAN
+<U064D> ""
+% ARABIC FATHA
+<U064E> ""
+% ARABIC DAMMA
+<U064F> ""
+% ARABIC KASRA
+<U0650> ""
+% ARABIC SHADDA
+<U0651> ""
+% ARABIC SUKUN
+<U0652> ""
% ARABIC MADDAH ABOVE
<U0653> ""
% ARABIC HAMZA ABOVE
<U0654> ""
% ARABIC HAMZA BELOW
<U0655> ""
+% ARABIC SUBSCRIPT ALEF
+<U0656> ""
+% ARABIC INVERTED DAMMA
+<U0657> ""
+% ARABIC MARK NOON GHUNNA
+<U0658> ""
+% ARABIC ZWARAKAY
+<U0659> ""
+% ARABIC VOWEL SIGN SMALL V ABOVE
+<U065A> ""
+% ARABIC VOWEL SIGN INVERTED SMALL V ABOVE
+<U065B> ""
+% ARABIC VOWEL SIGN DOT BELOW
+<U065C> ""
+% ARABIC REVERSED DAMMA
+<U065D> ""
+% ARABIC FATHA WITH TWO DOTS
+<U065E> ""
+% ARABIC WAVY HAMZA BELOW
+<U065F> ""
+% ARABIC LETTER SUPERSCRIPT ALEF
+<U0670> ""
+% ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA
+<U06D6> ""
+% ARABIC SMALL HIGH LIGATURE QAF WITH LAM WITH ALEF MAKSURA
+<U06D7> ""
+% ARABIC SMALL HIGH MEEM INITIAL FORM
+<U06D8> ""
+% ARABIC SMALL HIGH LAM ALEF
+<U06D9> ""
+% ARABIC SMALL HIGH JEEM
+<U06DA> ""
+% ARABIC SMALL HIGH THREE DOTS
+<U06DB> ""
+% ARABIC SMALL HIGH SEEN
+<U06DC> ""
+% ARABIC SMALL HIGH ROUNDED ZERO
+<U06DF> ""
+% ARABIC SMALL HIGH UPRIGHT RECTANGULAR ZERO
+<U06E0> ""
+% ARABIC SMALL HIGH DOTLESS HEAD OF KHAH
+<U06E1> ""
+% ARABIC SMALL HIGH MEEM ISOLATED FORM
+<U06E2> ""
+% ARABIC SMALL LOW SEEN
+<U06E3> ""
+% ARABIC SMALL HIGH MADDA
+<U06E4> ""
+% ARABIC SMALL HIGH YEH
+<U06E7> ""
+% ARABIC SMALL HIGH NOON
+<U06E8> ""
+% ARABIC EMPTY CENTRE LOW STOP
+<U06EA> ""
+% ARABIC EMPTY CENTRE HIGH STOP
+<U06EB> ""
+% ARABIC ROUNDED HIGH STOP WITH FILLED CENTRE
+<U06EC> ""
+% ARABIC SMALL LOW MEEM
+<U06ED> ""
+% ARABIC CURLY FATHA
+<U08E4> ""
+% ARABIC CURLY DAMMA
+<U08E5> ""
+% ARABIC CURLY KASRA
+<U08E6> ""
+% ARABIC CURLY FATHATAN
+<U08E7> ""
+% ARABIC CURLY DAMMATAN
+<U08E8> ""
+% ARABIC CURLY KASRATAN
+<U08E9> ""
+% ARABIC TONE ONE DOT ABOVE
+<U08EA> ""
+% ARABIC TONE TWO DOTS ABOVE
+<U08EB> ""
+% ARABIC TONE LOOP ABOVE
+<U08EC> ""
+% ARABIC TONE ONE DOT BELOW
+<U08ED> ""
+% ARABIC TONE TWO DOTS BELOW
+<U08EE> ""
+% ARABIC TONE LOOP BELOW
+<U08EF> ""
+% ARABIC OPEN FATHATAN
+<U08F0> ""
+% ARABIC OPEN DAMMATAN
+<U08F1> ""
+% ARABIC OPEN KASRATAN
+<U08F2> ""
+% ARABIC SMALL HIGH WAW
+<U08F3> ""
+% ARABIC FATHA WITH RING
+<U08F4> ""
+% ARABIC FATHA WITH DOT ABOVE
+<U08F5> ""
+% ARABIC KASRA WITH DOT BELOW
+<U08F6> ""
+% ARABIC LEFT ARROWHEAD ABOVE
+<U08F7> ""
+% ARABIC RIGHT ARROWHEAD ABOVE
+<U08F8> ""
+% ARABIC LEFT ARROWHEAD BELOW
+<U08F9> ""
+% ARABIC RIGHT ARROWHEAD BELOW
+<U08FA> ""
+% ARABIC DOUBLE RIGHT ARROWHEAD ABOVE
+<U08FB> ""
+% ARABIC DOUBLE RIGHT ARROWHEAD ABOVE WITH DOT
+<U08FC> ""
+% ARABIC RIGHT ARROWHEAD ABOVE WITH DOT
+<U08FD> ""
+% ARABIC DAMMA WITH DOT
+<U08FE> ""
+% ARABIC MARK SIDEWAYS NOON GHUNNA
+<U08FF> ""
+% COMBINING DOUBLED CIRCUMFLEX ACCENT
+<U1AB0> ""
+% COMBINING DIAERESIS-RING
+<U1AB1> ""
+% COMBINING INFINITY
+<U1AB2> ""
+% COMBINING DOWNWARDS ARROW
+<U1AB3> ""
+% COMBINING TRIPLE DOT
+<U1AB4> ""
+% COMBINING X-X BELOW
+<U1AB5> ""
+% COMBINING WIGGLY LINE BELOW
+<U1AB6> ""
+% COMBINING OPEN MARK BELOW
+<U1AB7> ""
+% COMBINING DOUBLE OPEN MARK BELOW
+<U1AB8> ""
+% COMBINING LIGHT CENTRALIZATION STROKE BELOW
+<U1AB9> ""
+% COMBINING STRONG CENTRALIZATION STROKE BELOW
+<U1ABA> ""
+% COMBINING PARENTHESES ABOVE
+<U1ABB> ""
+% COMBINING DOUBLE PARENTHESES ABOVE
+<U1ABC> ""
+% COMBINING PARENTHESES BELOW
+<U1ABD> ""
+% COMBINING PARENTHESES OVERLAY
+<U1ABE> ""
+% COMBINING DOTTED GRAVE ACCENT
+<U1DC0> ""
+% COMBINING DOTTED ACUTE ACCENT
+<U1DC1> ""
+% COMBINING SNAKE BELOW
+<U1DC2> ""
+% COMBINING SUSPENSION MARK
+<U1DC3> ""
+% COMBINING MACRON-ACUTE
+<U1DC4> ""
+% COMBINING GRAVE-MACRON
+<U1DC5> ""
+% COMBINING MACRON-GRAVE
+<U1DC6> ""
+% COMBINING ACUTE-MACRON
+<U1DC7> ""
+% COMBINING GRAVE-ACUTE-GRAVE
+<U1DC8> ""
+% COMBINING ACUTE-GRAVE-ACUTE
+<U1DC9> ""
+% COMBINING LATIN SMALL LETTER R BELOW
+<U1DCA> ""
+% COMBINING BREVE-MACRON
+<U1DCB> ""
+% COMBINING MACRON-BREVE
+<U1DCC> ""
+% COMBINING DOUBLE CIRCUMFLEX ABOVE
+<U1DCD> ""
+% COMBINING OGONEK ABOVE
+<U1DCE> ""
+% COMBINING ZIGZAG BELOW
+<U1DCF> ""
+% COMBINING IS BELOW
+<U1DD0> ""
+% COMBINING UR ABOVE
+<U1DD1> ""
+% COMBINING US ABOVE
+<U1DD2> ""
+% COMBINING LATIN SMALL LETTER FLATTENED OPEN A ABOVE
+<U1DD3> ""
+% COMBINING LATIN SMALL LETTER AE
+<U1DD4> ""
+% COMBINING LATIN SMALL LETTER AO
+<U1DD5> ""
+% COMBINING LATIN SMALL LETTER AV
+<U1DD6> ""
+% COMBINING LATIN SMALL LETTER C CEDILLA
+<U1DD7> ""
+% COMBINING LATIN SMALL LETTER INSULAR D
+<U1DD8> ""
+% COMBINING LATIN SMALL LETTER ETH
+<U1DD9> ""
+% COMBINING LATIN SMALL LETTER G
+<U1DDA> ""
+% COMBINING LATIN LETTER SMALL CAPITAL G
+<U1DDB> ""
+% COMBINING LATIN SMALL LETTER K
+<U1DDC> ""
+% COMBINING LATIN SMALL LETTER L
+<U1DDD> ""
+% COMBINING LATIN LETTER SMALL CAPITAL L
+<U1DDE> ""
+% COMBINING LATIN LETTER SMALL CAPITAL M
+<U1DDF> ""
+% COMBINING LATIN SMALL LETTER N
+<U1DE0> ""
+% COMBINING LATIN LETTER SMALL CAPITAL N
+<U1DE1> ""
+% COMBINING LATIN LETTER SMALL CAPITAL R
+<U1DE2> ""
+% COMBINING LATIN SMALL LETTER R ROTUNDA
+<U1DE3> ""
+% COMBINING LATIN SMALL LETTER S
+<U1DE4> ""
+% COMBINING LATIN SMALL LETTER LONG S
+<U1DE5> ""
+% COMBINING LATIN SMALL LETTER Z
+<U1DE6> ""
+% COMBINING LATIN SMALL LETTER ALPHA
+<U1DE7> ""
+% COMBINING LATIN SMALL LETTER B
+<U1DE8> ""
+% COMBINING LATIN SMALL LETTER BETA
+<U1DE9> ""
+% COMBINING LATIN SMALL LETTER SCHWA
+<U1DEA> ""
+% COMBINING LATIN SMALL LETTER F
+<U1DEB> ""
+% COMBINING LATIN SMALL LETTER L WITH DOUBLE MIDDLE TILDE
+<U1DEC> ""
+% COMBINING LATIN SMALL LETTER O WITH LIGHT CENTRALIZATION STROKE
+<U1DED> ""
+% COMBINING LATIN SMALL LETTER P
+<U1DEE> ""
+% COMBINING LATIN SMALL LETTER ESH
+<U1DEF> ""
+% COMBINING LATIN SMALL LETTER U WITH LIGHT CENTRALIZATION STROKE
+<U1DF0> ""
+% COMBINING LATIN SMALL LETTER W
+<U1DF1> ""
+% COMBINING LATIN SMALL LETTER A WITH DIAERESIS
+<U1DF2> ""
+% COMBINING LATIN SMALL LETTER O WITH DIAERESIS
+<U1DF3> ""
+% COMBINING LATIN SMALL LETTER U WITH DIAERESIS
+<U1DF4> ""
+% COMBINING UP TACK ABOVE
+<U1DF5> ""
+% COMBINING DOUBLE INVERTED BREVE BELOW
+<U1DFC> ""
+% COMBINING ALMOST EQUAL TO BELOW
+<U1DFD> ""
+% COMBINING LEFT ARROWHEAD ABOVE
+<U1DFE> ""
+% COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW
+<U1DFF> ""
+% COMBINING LEFT HARPOON ABOVE
+<U20D0> ""
+% COMBINING RIGHT HARPOON ABOVE
+<U20D1> ""
+% COMBINING LONG VERTICAL LINE OVERLAY
+<U20D2> ""
+% COMBINING SHORT VERTICAL LINE OVERLAY
+<U20D3> ""
+% COMBINING ANTICLOCKWISE ARROW ABOVE
+<U20D4> ""
+% COMBINING CLOCKWISE ARROW ABOVE
+<U20D5> ""
+% COMBINING LEFT ARROW ABOVE
+<U20D6> ""
+% COMBINING RIGHT ARROW ABOVE
+<U20D7> ""
+% COMBINING RING OVERLAY
+<U20D8> ""
+% COMBINING CLOCKWISE RING OVERLAY
+<U20D9> ""
+% COMBINING ANTICLOCKWISE RING OVERLAY
+<U20DA> ""
+% COMBINING THREE DOTS ABOVE
+<U20DB> ""
+% COMBINING FOUR DOTS ABOVE
+<U20DC> ""
+% COMBINING ENCLOSING CIRCLE
+<U20DD> ""
+% COMBINING ENCLOSING SQUARE
+<U20DE> ""
+% COMBINING ENCLOSING DIAMOND
+<U20DF> ""
+% COMBINING ENCLOSING CIRCLE BACKSLASH
+<U20E0> ""
+% COMBINING LEFT RIGHT ARROW ABOVE
+<U20E1> ""
+% COMBINING ENCLOSING SCREEN
+<U20E2> ""
+% COMBINING ENCLOSING KEYCAP
+<U20E3> ""
% COMBINING ENCLOSING UPWARD POINTING TRIANGLE
<U20E4> ""
% COMBINING REVERSE SOLIDUS OVERLAY
@@ -251,10 +695,70 @@ translit_start
<U20E9> ""
% COMBINING LEFTWARDS ARROW OVERLAY
<U20EA> ""
+% COMBINING LONG DOUBLE SOLIDUS OVERLAY
+<U20EB> ""
+% COMBINING RIGHTWARDS HARPOON WITH BARB DOWNWARDS
+<U20EC> ""
+% COMBINING LEFTWARDS HARPOON WITH BARB DOWNWARDS
+<U20ED> ""
+% COMBINING LEFT ARROW BELOW
+<U20EE> ""
+% COMBINING RIGHT ARROW BELOW
+<U20EF> ""
+% COMBINING ASTERISK ABOVE
+<U20F0> ""
% COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK
<U3099> ""
% COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK
<U309A> ""
+% HEBREW POINT JUDEO-SPANISH VARIKA
+<UFB1E> ""
+% COMBINING LIGATURE LEFT HALF
+<UFE20> ""
+% COMBINING LIGATURE RIGHT HALF
+<UFE21> ""
+% COMBINING DOUBLE TILDE LEFT HALF
+<UFE22> ""
+% COMBINING DOUBLE TILDE RIGHT HALF
+<UFE23> ""
+% COMBINING MACRON LEFT HALF
+<UFE24> ""
+% COMBINING MACRON RIGHT HALF
+<UFE25> ""
+% COMBINING CONJOINING MACRON
+<UFE26> ""
+% COMBINING LIGATURE LEFT HALF BELOW
+<UFE27> ""
+% COMBINING LIGATURE RIGHT HALF BELOW
+<UFE28> ""
+% COMBINING TILDE LEFT HALF BELOW
+<UFE29> ""
+% COMBINING TILDE RIGHT HALF BELOW
+<UFE2A> ""
+% COMBINING MACRON LEFT HALF BELOW
+<UFE2B> ""
+% COMBINING MACRON RIGHT HALF BELOW
+<UFE2C> ""
+% COMBINING CONJOINING MACRON BELOW
+<UFE2D> ""
+% PHAISTOS DISC SIGN COMBINING OBLIQUE STROKE
+<U000101FD> ""
+% COMBINING OLD PERMIC LETTER AN
+<U00010376> ""
+% COMBINING OLD PERMIC LETTER DOI
+<U00010377> ""
+% COMBINING OLD PERMIC LETTER ZATA
+<U00010378> ""
+% COMBINING OLD PERMIC LETTER NENOE
+<U00010379> ""
+% COMBINING OLD PERMIC LETTER SII
+<U0001037A> ""
+% COMBINING GREEK MUSICAL TRISEME
+<U0001D242> ""
+% COMBINING GREEK MUSICAL TETRASEME
+<U0001D243> ""
+% COMBINING GREEK MUSICAL PENTASEME
+<U0001D244> ""
% LATIN CAPITAL LETTER A WITH GRAVE
<U00C0> <U0041>
@@ -268,6 +772,8 @@ translit_start
<U00C4> <U0041>
% LATIN CAPITAL LETTER A WITH RING ABOVE
<U00C5> <U0041>
+% LATIN CAPITAL LETTER AE
+<U00C6> "<U0041><U0045>"
% LATIN CAPITAL LETTER C WITH CEDILLA
<U00C7> <U0043>
% LATIN CAPITAL LETTER E WITH GRAVE
@@ -298,6 +804,8 @@ translit_start
<U00D5> <U004F>
% LATIN CAPITAL LETTER O WITH DIAERESIS
<U00D6> <U004F>
+% LATIN CAPITAL LETTER O WITH STROKE
+<U00D8> <U004F>
% LATIN CAPITAL LETTER U WITH GRAVE
<U00D9> <U0055>
% LATIN CAPITAL LETTER U WITH ACUTE
@@ -320,6 +828,8 @@ translit_start
<U00E4> <U0061>
% LATIN SMALL LETTER A WITH RING ABOVE
<U00E5> <U0061>
+% LATIN SMALL LETTER AE
+<U00E6> "<U0061><U0065>"
% LATIN SMALL LETTER C WITH CEDILLA
<U00E7> <U0063>
% LATIN SMALL LETTER E WITH GRAVE
@@ -350,6 +860,8 @@ translit_start
<U00F5> <U006F>
% LATIN SMALL LETTER O WITH DIAERESIS
<U00F6> <U006F>
+% LATIN SMALL LETTER O WITH STROKE
+<U00F8> <U006F>
% LATIN SMALL LETTER U WITH GRAVE
<U00F9> <U0075>
% LATIN SMALL LETTER U WITH ACUTE
@@ -472,10 +984,6 @@ translit_start
<U013D> <U004C>
% LATIN SMALL LETTER L WITH CARON
<U013E> <U006C>
-% LATIN CAPITAL LETTER L WITH STROKE
-<U0141> <U004C>
-% LATIN SMALL LETTER L WITH STROKE
-<U0142> <U006C>
% LATIN CAPITAL LETTER N WITH ACUTE
<U0143> <U004E>
% LATIN SMALL LETTER N WITH ACUTE
@@ -673,9 +1181,9 @@ translit_start
% LATIN SMALL LETTER AE WITH ACUTE
<U01FD> <U00E6>;"<U0061><U0065>"
% LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
-<U01FE> <U004F>
+<U01FE> <U00D8>;<U004F>
% LATIN SMALL LETTER O WITH STROKE AND ACUTE
-<U01FF> <U006F>
+<U01FF> <U00F8>;<U006F>
% LATIN CAPITAL LETTER A WITH DOUBLE GRAVE
<U0200> <U0041>
% LATIN SMALL LETTER A WITH DOUBLE GRAVE
@@ -764,14 +1272,6 @@ translit_start
<U0232> <U0059>
% LATIN SMALL LETTER Y WITH MACRON
<U0233> <U0079>
-% COMBINING GRAVE TONE MARK
-<U0340> <U0300>
-% COMBINING ACUTE TONE MARK
-<U0341> <U0301>
-% COMBINING GREEK KORONIS
-<U0343> <U0313>
-% COMBINING GREEK DIALYTIKA TONOS
-<U0344> <U0308>
% GREEK NUMERAL SIGN
<U0374> <U02B9>
% GREEK QUESTION MARK
@@ -928,6 +1428,8 @@ translit_start
<U04F8> <U042B>
% CYRILLIC SMALL LETTER YERU WITH DIAERESIS
<U04F9> <U044B>
+% HEBREW LIGATURE YIDDISH DOUBLE YOD
+<U05F2> "<U05D9><U05D9>"
% ARABIC LETTER ALEF WITH MADDA ABOVE
<U0622> <U0627>
% ARABIC LETTER ALEF WITH HAMZA ABOVE
@@ -1017,7 +1519,7 @@ translit_start
% KANNADA VOWEL SIGN O
<U0CCA> "<U0CC6><U0CC2>"
% KANNADA VOWEL SIGN OO
-<U0CCB> "<U0CCA><U0CD5>"
+<U0CCB> "<U0CC6><U0CC2><U0CD5>"
% MALAYALAM VOWEL SIGN O
<U0D4A> "<U0D46><U0D3E>"
% MALAYALAM VOWEL SIGN OO
@@ -1029,7 +1531,7 @@ translit_start
% SINHALA VOWEL SIGN KOMBUVA HAA AELA-PILLA
<U0DDC> "<U0DD9><U0DCF>"
% SINHALA VOWEL SIGN KOMBUVA HAA DIGA AELA-PILLA
-<U0DDD> "<U0DDC><U0DCA>"
+<U0DDD> "<U0DD9><U0DCF><U0DCA>"
% SINHALA VOWEL SIGN KOMBUVA HAA GAYANUKITTA
<U0DDE> "<U0DD9><U0DDF>"
% TIBETAN LETTER GHA
@@ -2020,16 +2522,114 @@ translit_start
<U2000> <U2002>;<U0020>
% EM QUAD
<U2001> <U2003>;<U0020>
+% EN SPACE
+<U2002> <U0020>
+% EM SPACE
+<U2003> <U0020>
% OHM SIGN
<U2126> <U03A9>
% KELVIN SIGN
<U212A> <U004B>
% ANGSTROM SIGN
-<U212B> <U00C5>
+<U212B> <U0041>
+% LEFTWARDS ARROW WITH STROKE
+<U219A> <U2190>
+% RIGHTWARDS ARROW WITH STROKE
+<U219B> <U2192>
+% LEFT RIGHT ARROW WITH STROKE
+<U21AE> "<U0021><U003C><U002D><U003E>"
+% LEFTWARDS DOUBLE ARROW WITH STROKE
+<U21CD> "<U0021><U003C><U003D>"
+% LEFT RIGHT DOUBLE ARROW WITH STROKE
+<U21CE> "<U0021><U003C><U003D><U003E>"
+% RIGHTWARDS DOUBLE ARROW WITH STROKE
+<U21CF> "<U0021><U003D><U003E>"
+% THERE DOES NOT EXIST
+<U2204> "<U0021><U2203>"
+% NOT AN ELEMENT OF
+<U2209> "<U0021><U2208>"
+% DOES NOT CONTAIN AS MEMBER
+<U220C> "<U0021><U220B>"
+% DOES NOT DIVIDE
+<U2224> "<U0021><U2223>"
+% NOT PARALLEL TO
+<U2226> "<U0021><U2225>"
+% NOT TILDE
+<U2241> "<U0021><U007E>"
+% NOT ASYMPTOTICALLY EQUAL TO
+<U2244> "<U0021><U007E><U002D>"
+% NEITHER APPROXIMATELY NOR ACTUALLY EQUAL TO
+<U2247> "<U0021><U007E><U003D>"
+% NOT ALMOST EQUAL TO
+<U2249> "<U0021><U007E><U007E>"
+% NOT EQUAL TO
+<U2260> "<U0021><U003D>"
+% NOT IDENTICAL TO
+<U2262> "<U0021><U003D><U003D>"
+% NOT EQUIVALENT TO
+<U226D> "<U0021><U224D>"
+% NOT LESS-THAN
+<U226E> "<U0021><U003C>"
+% NOT GREATER-THAN
+<U226F> "<U0021><U003E>"
+% NEITHER LESS-THAN NOR EQUAL TO
+<U2270> "<U0021><U003C><U003D>"
+% NEITHER GREATER-THAN NOR EQUAL TO
+<U2271> "<U0021><U003E><U003D>"
+% NEITHER LESS-THAN NOR EQUIVALENT TO
+<U2274> "<U0021><U003C><U007E>"
+% NEITHER GREATER-THAN NOR EQUIVALENT TO
+<U2275> "<U0021><U003E><U007E>"
+% NEITHER LESS-THAN NOR GREATER-THAN
+<U2278> "<U0021><U003C><U003E>"
+% NEITHER GREATER-THAN NOR LESS-THAN
+<U2279> "<U0021><U003E><U003C>"
+% DOES NOT PRECEDE
+<U2280> "<U0021><U227A>"
+% DOES NOT SUCCEED
+<U2281> "<U0021><U227B>"
+% NOT A SUBSET OF
+<U2284> "<U0021><U2282>"
+% NOT A SUPERSET OF
+<U2285> "<U0021><U2283>"
+% NEITHER A SUBSET OF NOR EQUAL TO
+<U2288> "<U0021><U2282><U003D>"
+% NEITHER A SUPERSET OF NOR EQUAL TO
+<U2289> "<U0021><U2283><U003D>"
+% DOES NOT PROVE
+<U22AC> "<U0021><U22A2>"
+% NOT TRUE
+<U22AD> "<U0021><U22A8>"
+% DOES NOT FORCE
+<U22AE> "<U0021><U22A9>"
+% NEGATED DOUBLE VERTICAL BAR DOUBLE RIGHT TURNSTILE
+<U22AF> "<U0021><U22AB>"
+% DOES NOT PRECEDE OR EQUAL
+<U22E0> "<U0021><U227C>"
+% DOES NOT SUCCEED OR EQUAL
+<U22E1> "<U0021><U227D>"
+% NOT SQUARE IMAGE OF OR EQUAL TO
+<U22E2> "<U0021><U2291>"
+% NOT SQUARE ORIGINAL OF OR EQUAL TO
+<U22E3> "<U0021><U2292>"
+% NOT NORMAL SUBGROUP OF
+<U22EA> "<U0021><U22B2>"
+% DOES NOT CONTAIN AS NORMAL SUBGROUP
+<U22EB> "<U0021><U22B3>"
+% NOT NORMAL SUBGROUP OF OR EQUAL TO
+<U22EC> "<U0021><U22B4>"
+% DOES NOT CONTAIN AS NORMAL SUBGROUP OR EQUAL
+<U22ED> "<U0021><U22B5>"
% LEFT-POINTING ANGLE BRACKET
<U2329> <U3008>;<U003C>
% RIGHT-POINTING ANGLE BRACKET
<U232A> <U3009>;<U003E>
+% FORKING
+<U2ADC> "<U0021><U2ADD>"
+% LEFT ANGLE BRACKET
+<U3008> <U003C>
+% RIGHT ANGLE BRACKET
+<U3009> <U003E>
% HIRAGANA LETTER GA
<U304C> <U304B>
% HIRAGANA LETTER GI
diff --git a/localedata/locales/translit_compat b/localedata/locales/translit_compat
index bb9d660..6e45220 100644
--- a/localedata/locales/translit_compat
+++ b/localedata/locales/translit_compat
@@ -2,18 +2,24 @@ escape_char /
comment_char %
% Transliterations of compatibility characters and ligatures.
-% Generated through
-% $ grep '^[^;]*;[^;]*;[^;]*;[^;]*;[^;]*;<compat>[^;]*;' UnicodeData.txt | \
-% sed -e 's/^\([^;]*\);\([^;]*\);[^;]*;[^;]*;[^;]*;<compat> \([^;]*\);.*$/<U\1> "<U\3>"% \2/' | grep -v '0020 03[0-6][0-9A-F]' | sed -e 'h' -e 's/^\([^%]*\)% .*$/\1/' -e 's/\([0-9A-F]\) \([0-9A-F]\)/\1><U\2/g' -e 'x' -e 's/^[^%]*\(% .*\)$/\1/' -e 'G'
-% and
-% $ grep '[^;]*;[^;]*LIGATURE[^;]*;' UnicodeData.txt
+% Generated automatically from UnicodeData.txt by gen_translit_compat.py on 2015-06-10 for Unicode 7.0.0.
LC_CTYPE
translit_start
+% FEMININE ORDINAL INDICATOR
+<U00AA> "<U0061>"
+% SUPERSCRIPT TWO
+<U00B2> "<U0032>"
+% SUPERSCRIPT THREE
+<U00B3> "<U0033>"
% MICRO SIGN
-<U00B5> "<U03BC>";<U0075>
+<U00B5> "<U03BC>";"<U0075>"
+% SUPERSCRIPT ONE
+<U00B9> "<U0031>"
+% MASCULINE ORDINAL INDICATOR
+<U00BA> "<U006F>"
% LATIN CAPITAL LIGATURE IJ
<U0132> "<U0049><U004A>"
% LATIN SMALL LIGATURE IJ
@@ -54,6 +60,38 @@ translit_start
<U01F2> "<U0044><U007A>"
% LATIN SMALL LETTER DZ
<U01F3> "<U0064><U007A>"
+% MODIFIER LETTER SMALL H
+<U02B0> "<U0068>"
+% MODIFIER LETTER SMALL H WITH HOOK
+<U02B1> "<U0266>"
+% MODIFIER LETTER SMALL J
+<U02B2> "<U006A>"
+% MODIFIER LETTER SMALL R
+<U02B3> "<U0072>"
+% MODIFIER LETTER SMALL TURNED R
+<U02B4> "<U0279>"
+% MODIFIER LETTER SMALL TURNED R WITH HOOK
+<U02B5> "<U027B>"
+% MODIFIER LETTER SMALL CAPITAL INVERTED R
+<U02B6> "<U0281>"
+% MODIFIER LETTER SMALL W
+<U02B7> "<U0077>"
+% MODIFIER LETTER SMALL Y
+<U02B8> "<U0079>"
+% MODIFIER LETTER APOSTROPHE
+<U02BC> "<U0027>"
+% MODIFIER LETTER SMALL GAMMA
+<U02E0> "<U0263>"
+% MODIFIER LETTER SMALL L
+<U02E1> "<U006C>"
+% MODIFIER LETTER SMALL S
+<U02E2> "<U0073>"
+% MODIFIER LETTER SMALL X
+<U02E3> "<U0078>"
+% MODIFIER LETTER SMALL REVERSED GLOTTAL STOP
+<U02E4> "<U0295>"
+% GREEK SMALL LETTER MU
+<U03BC> "<U0075>"
% GREEK BETA SYMBOL
<U03D0> "<U03B2>"
% GREEK THETA SYMBOL
@@ -74,6 +112,20 @@ translit_start
<U03F4> "<U0398>"
% GREEK LUNATE EPSILON SYMBOL
<U03F5> "<U03B5>"
+% GREEK CAPITAL LUNATE SIGMA SYMBOL
+<U03F9> "<U03A3>"
+% CYRILLIC CAPITAL LIGATURE EN GHE
+<U04A4> "<U041D><U0413>"
+% CYRILLIC SMALL LIGATURE EN GHE
+<U04A5> "<U043D><U0433>"
+% CYRILLIC CAPITAL LIGATURE TE TSE
+<U04B4> "<U0422><U0426>"
+% CYRILLIC SMALL LIGATURE TE TSE
+<U04B5> "<U0442><U0446>"
+% CYRILLIC CAPITAL LIGATURE A IE
+<U04D4> "<U0410><U0415>"
+% CYRILLIC SMALL LIGATURE A IE
+<U04D5> "<U0430><U0435>"
% ARMENIAN SMALL LIGATURE ECH YIWN
<U0587> "<U0565><U0582>"
% HEBREW LIGATURE YIDDISH DOUBLE VAV
@@ -102,6 +154,204 @@ translit_start
<U0F77> "<U0FB2><U0F81>"
% TIBETAN VOWEL SIGN VOCALIC LL
<U0F79> "<U0FB3><U0F81>"
+% MODIFIER LETTER GEORGIAN NAR
+<U10FC> "<U10DC>"
+% MODIFIER LETTER CAPITAL A
+<U1D2C> "<U0041>"
+% MODIFIER LETTER CAPITAL AE
+<U1D2D> "<U00C6>"
+% MODIFIER LETTER CAPITAL B
+<U1D2E> "<U0042>"
+% MODIFIER LETTER CAPITAL D
+<U1D30> "<U0044>"
+% MODIFIER LETTER CAPITAL E
+<U1D31> "<U0045>"
+% MODIFIER LETTER CAPITAL REVERSED E
+<U1D32> "<U018E>"
+% MODIFIER LETTER CAPITAL G
+<U1D33> "<U0047>"
+% MODIFIER LETTER CAPITAL H
+<U1D34> "<U0048>"
+% MODIFIER LETTER CAPITAL I
+<U1D35> "<U0049>"
+% MODIFIER LETTER CAPITAL J
+<U1D36> "<U004A>"
+% MODIFIER LETTER CAPITAL K
+<U1D37> "<U004B>"
+% MODIFIER LETTER CAPITAL L
+<U1D38> "<U004C>"
+% MODIFIER LETTER CAPITAL M
+<U1D39> "<U004D>"
+% MODIFIER LETTER CAPITAL N
+<U1D3A> "<U004E>"
+% MODIFIER LETTER CAPITAL O
+<U1D3C> "<U004F>"
+% MODIFIER LETTER CAPITAL OU
+<U1D3D> "<U0222>"
+% MODIFIER LETTER CAPITAL P
+<U1D3E> "<U0050>"
+% MODIFIER LETTER CAPITAL R
+<U1D3F> "<U0052>"
+% MODIFIER LETTER CAPITAL T
+<U1D40> "<U0054>"
+% MODIFIER LETTER CAPITAL U
+<U1D41> "<U0055>"
+% MODIFIER LETTER CAPITAL W
+<U1D42> "<U0057>"
+% MODIFIER LETTER SMALL A
+<U1D43> "<U0061>"
+% MODIFIER LETTER SMALL TURNED A
+<U1D44> "<U0250>"
+% MODIFIER LETTER SMALL ALPHA
+<U1D45> "<U0251>"
+% MODIFIER LETTER SMALL TURNED AE
+<U1D46> "<U1D02>"
+% MODIFIER LETTER SMALL B
+<U1D47> "<U0062>"
+% MODIFIER LETTER SMALL D
+<U1D48> "<U0064>"
+% MODIFIER LETTER SMALL E
+<U1D49> "<U0065>"
+% MODIFIER LETTER SMALL SCHWA
+<U1D4A> "<U0259>"
+% MODIFIER LETTER SMALL OPEN E
+<U1D4B> "<U025B>"
+% MODIFIER LETTER SMALL TURNED OPEN E
+<U1D4C> "<U025C>"
+% MODIFIER LETTER SMALL G
+<U1D4D> "<U0067>"
+% MODIFIER LETTER SMALL K
+<U1D4F> "<U006B>"
+% MODIFIER LETTER SMALL M
+<U1D50> "<U006D>"
+% MODIFIER LETTER SMALL ENG
+<U1D51> "<U014B>"
+% MODIFIER LETTER SMALL O
+<U1D52> "<U006F>"
+% MODIFIER LETTER SMALL OPEN O
+<U1D53> "<U0254>"
+% MODIFIER LETTER SMALL TOP HALF O
+<U1D54> "<U1D16>"
+% MODIFIER LETTER SMALL BOTTOM HALF O
+<U1D55> "<U1D17>"
+% MODIFIER LETTER SMALL P
+<U1D56> "<U0070>"
+% MODIFIER LETTER SMALL T
+<U1D57> "<U0074>"
+% MODIFIER LETTER SMALL U
+<U1D58> "<U0075>"
+% MODIFIER LETTER SMALL SIDEWAYS U
+<U1D59> "<U1D1D>"
+% MODIFIER LETTER SMALL TURNED M
+<U1D5A> "<U026F>"
+% MODIFIER LETTER SMALL V
+<U1D5B> "<U0076>"
+% MODIFIER LETTER SMALL AIN
+<U1D5C> "<U1D25>"
+% MODIFIER LETTER SMALL BETA
+<U1D5D> "<U03B2>"
+% MODIFIER LETTER SMALL GREEK GAMMA
+<U1D5E> "<U03B3>"
+% MODIFIER LETTER SMALL DELTA
+<U1D5F> "<U03B4>"
+% MODIFIER LETTER SMALL GREEK PHI
+<U1D60> "<U03C6>"
+% MODIFIER LETTER SMALL CHI
+<U1D61> "<U03C7>"
+% LATIN SUBSCRIPT SMALL LETTER I
+<U1D62> "<U0069>"
+% LATIN SUBSCRIPT SMALL LETTER R
+<U1D63> "<U0072>"
+% LATIN SUBSCRIPT SMALL LETTER U
+<U1D64> "<U0075>"
+% LATIN SUBSCRIPT SMALL LETTER V
+<U1D65> "<U0076>"
+% GREEK SUBSCRIPT SMALL LETTER BETA
+<U1D66> "<U03B2>"
+% GREEK SUBSCRIPT SMALL LETTER GAMMA
+<U1D67> "<U03B3>"
+% GREEK SUBSCRIPT SMALL LETTER RHO
+<U1D68> "<U03C1>"
+% GREEK SUBSCRIPT SMALL LETTER PHI
+<U1D69> "<U03C6>"
+% GREEK SUBSCRIPT SMALL LETTER CHI
+<U1D6A> "<U03C7>"
+% MODIFIER LETTER CYRILLIC EN
+<U1D78> "<U043D>"
+% MODIFIER LETTER SMALL TURNED ALPHA
+<U1D9B> "<U0252>"
+% MODIFIER LETTER SMALL C
+<U1D9C> "<U0063>"
+% MODIFIER LETTER SMALL C WITH CURL
+<U1D9D> "<U0255>"
+% MODIFIER LETTER SMALL ETH
+<U1D9E> "<U00F0>"
+% MODIFIER LETTER SMALL REVERSED OPEN E
+<U1D9F> "<U025C>"
+% MODIFIER LETTER SMALL F
+<U1DA0> "<U0066>"
+% MODIFIER LETTER SMALL DOTLESS J WITH STROKE
+<U1DA1> "<U025F>"
+% MODIFIER LETTER SMALL SCRIPT G
+<U1DA2> "<U0261>"
+% MODIFIER LETTER SMALL TURNED H
+<U1DA3> "<U0265>"
+% MODIFIER LETTER SMALL I WITH STROKE
+<U1DA4> "<U0268>"
+% MODIFIER LETTER SMALL IOTA
+<U1DA5> "<U0269>"
+% MODIFIER LETTER SMALL CAPITAL I
+<U1DA6> "<U026A>"
+% MODIFIER LETTER SMALL CAPITAL I WITH STROKE
+<U1DA7> "<U1D7B>"
+% MODIFIER LETTER SMALL J WITH CROSSED-TAIL
+<U1DA8> "<U029D>"
+% MODIFIER LETTER SMALL L WITH RETROFLEX HOOK
+<U1DA9> "<U026D>"
+% MODIFIER LETTER SMALL L WITH PALATAL HOOK
+<U1DAA> "<U1D85>"
+% MODIFIER LETTER SMALL CAPITAL L
+<U1DAB> "<U029F>"
+% MODIFIER LETTER SMALL M WITH HOOK
+<U1DAC> "<U0271>"
+% MODIFIER LETTER SMALL TURNED M WITH LONG LEG
+<U1DAD> "<U0270>"
+% MODIFIER LETTER SMALL N WITH LEFT HOOK
+<U1DAE> "<U0272>"
+% MODIFIER LETTER SMALL N WITH RETROFLEX HOOK
+<U1DAF> "<U0273>"
+% MODIFIER LETTER SMALL CAPITAL N
+<U1DB0> "<U0274>"
+% MODIFIER LETTER SMALL BARRED O
+<U1DB1> "<U0275>"
+% MODIFIER LETTER SMALL PHI
+<U1DB2> "<U0278>"
+% MODIFIER LETTER SMALL S WITH HOOK
+<U1DB3> "<U0282>"
+% MODIFIER LETTER SMALL ESH
+<U1DB4> "<U0283>"
+% MODIFIER LETTER SMALL T WITH PALATAL HOOK
+<U1DB5> "<U01AB>"
+% MODIFIER LETTER SMALL U BAR
+<U1DB6> "<U0289>"
+% MODIFIER LETTER SMALL UPSILON
+<U1DB7> "<U028A>"
+% MODIFIER LETTER SMALL CAPITAL U
+<U1DB8> "<U1D1C>"
+% MODIFIER LETTER SMALL V WITH HOOK
+<U1DB9> "<U028B>"
+% MODIFIER LETTER SMALL TURNED V
+<U1DBA> "<U028C>"
+% MODIFIER LETTER SMALL Z
+<U1DBB> "<U007A>"
+% MODIFIER LETTER SMALL Z WITH RETROFLEX HOOK
+<U1DBC> "<U0290>"
+% MODIFIER LETTER SMALL Z WITH CURL
+<U1DBD> "<U0291>"
+% MODIFIER LETTER SMALL EZH
+<U1DBE> "<U0292>"
+% MODIFIER LETTER SMALL THETA
+<U1DBF> "<U03B8>"
% LATIN SMALL LETTER A WITH RIGHT HALF RING
<U1E9A> "<U0061><U02BE>"
% EN SPACE
@@ -146,6 +396,90 @@ translit_start
<U2057> "<U2032><U2032><U2032><U2032>"
% MEDIUM MATHEMATICAL SPACE
<U205F> "<U0020>"
+% SUPERSCRIPT ZERO
+<U2070> "<U0030>"
+% SUPERSCRIPT LATIN SMALL LETTER I
+<U2071> "<U0069>"
+% SUPERSCRIPT FOUR
+<U2074> "<U0034>"
+% SUPERSCRIPT FIVE
+<U2075> "<U0035>"
+% SUPERSCRIPT SIX
+<U2076> "<U0036>"
+% SUPERSCRIPT SEVEN
+<U2077> "<U0037>"
+% SUPERSCRIPT EIGHT
+<U2078> "<U0038>"
+% SUPERSCRIPT NINE
+<U2079> "<U0039>"
+% SUPERSCRIPT PLUS SIGN
+<U207A> "<U002B>"
+% SUPERSCRIPT MINUS
+<U207B> "<U2212>"
+% SUPERSCRIPT EQUALS SIGN
+<U207C> "<U003D>"
+% SUPERSCRIPT LEFT PARENTHESIS
+<U207D> "<U0028>"
+% SUPERSCRIPT RIGHT PARENTHESIS
+<U207E> "<U0029>"
+% SUPERSCRIPT LATIN SMALL LETTER N
+<U207F> "<U006E>"
+% SUBSCRIPT ZERO
+<U2080> "<U0030>"
+% SUBSCRIPT ONE
+<U2081> "<U0031>"
+% SUBSCRIPT TWO
+<U2082> "<U0032>"
+% SUBSCRIPT THREE
+<U2083> "<U0033>"
+% SUBSCRIPT FOUR
+<U2084> "<U0034>"
+% SUBSCRIPT FIVE
+<U2085> "<U0035>"
+% SUBSCRIPT SIX
+<U2086> "<U0036>"
+% SUBSCRIPT SEVEN
+<U2087> "<U0037>"
+% SUBSCRIPT EIGHT
+<U2088> "<U0038>"
+% SUBSCRIPT NINE
+<U2089> "<U0039>"
+% SUBSCRIPT PLUS SIGN
+<U208A> "<U002B>"
+% SUBSCRIPT MINUS
+<U208B> "<U2212>"
+% SUBSCRIPT EQUALS SIGN
+<U208C> "<U003D>"
+% SUBSCRIPT LEFT PARENTHESIS
+<U208D> "<U0028>"
+% SUBSCRIPT RIGHT PARENTHESIS
+<U208E> "<U0029>"
+% LATIN SUBSCRIPT SMALL LETTER A
+<U2090> "<U0061>"
+% LATIN SUBSCRIPT SMALL LETTER E
+<U2091> "<U0065>"
+% LATIN SUBSCRIPT SMALL LETTER O
+<U2092> "<U006F>"
+% LATIN SUBSCRIPT SMALL LETTER X
+<U2093> "<U0078>"
+% LATIN SUBSCRIPT SMALL LETTER SCHWA
+<U2094> "<U0259>"
+% LATIN SUBSCRIPT SMALL LETTER H
+<U2095> "<U0068>"
+% LATIN SUBSCRIPT SMALL LETTER K
+<U2096> "<U006B>"
+% LATIN SUBSCRIPT SMALL LETTER L
+<U2097> "<U006C>"
+% LATIN SUBSCRIPT SMALL LETTER M
+<U2098> "<U006D>"
+% LATIN SUBSCRIPT SMALL LETTER N
+<U2099> "<U006E>"
+% LATIN SUBSCRIPT SMALL LETTER P
+<U209A> "<U0070>"
+% LATIN SUBSCRIPT SMALL LETTER S
+<U209B> "<U0073>"
+% LATIN SUBSCRIPT SMALL LETTER T
+<U209C> "<U0074>"
% RUPEE SIGN
<U20A8> "<U0052><U0073>"
% ACCOUNT OF
@@ -164,8 +498,12 @@ translit_start
<U2109> "<U00B0><U0046>"
% NUMERO SIGN
<U2116> "<U004E><U006F>"
+% SERVICE MARK
+<U2120> "<U0053><U004D>"
% TELEPHONE SIGN
<U2121> "<U0054><U0045><U004C>"
+% TRADE MARK SIGN
+<U2122> "<U0054><U004D>"
% ALEF SYMBOL
<U2135> "<U05D0>"
% BET SYMBOL
@@ -174,6 +512,8 @@ translit_start
<U2137> "<U05D2>"
% DALET SYMBOL
<U2138> "<U05D3>"
+% FACSIMILE SIGN
+<U213B> "<U0046><U0041><U0058>"
% ROMAN NUMERAL ONE
<U2160> "<U0049>"
% ROMAN NUMERAL TWO
@@ -386,6 +726,12 @@ translit_start
<U2A75> "<U003D><U003D>"
% THREE CONSECUTIVE EQUALS SIGNS
<U2A76> "<U003D><U003D><U003D>"
+% LATIN SUBSCRIPT SMALL LETTER J
+<U2C7C> "<U006A>"
+% MODIFIER LETTER CAPITAL V
+<U2C7D> "<U0056>"
+% TIFINAGH MODIFIER LETTER LABIALIZATION MARK
+<U2D6F> "<U2D61>"
% CJK RADICAL MOTHER
<U2E9F> "<U6BCD>"
% CJK RADICAL C-SIMPLIFIED TURTLE
@@ -830,6 +1176,10 @@ translit_start
<U309B> "<U0020><U3099>"
% KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK
<U309C> "<U0020><U309A>"
+% HIRAGANA DIGRAPH YORI
+<U309F> "<U3088><U308A>"
+% KATAKANA DIGRAPH KOTO
+<U30FF> "<U30B3><U30C8>"
% HANGUL LETTER KIYEOK
<U3131> "<U1100>"
% HANGUL LETTER SSANGKIYEOK
@@ -1018,6 +1368,34 @@ translit_start
<U318D> "<U119E>"
% HANGUL LETTER ARAEAE
<U318E> "<U11A1>"
+% IDEOGRAPHIC ANNOTATION ONE MARK
+<U3192> "<U4E00>"
+% IDEOGRAPHIC ANNOTATION TWO MARK
+<U3193> "<U4E8C>"
+% IDEOGRAPHIC ANNOTATION THREE MARK
+<U3194> "<U4E09>"
+% IDEOGRAPHIC ANNOTATION FOUR MARK
+<U3195> "<U56DB>"
+% IDEOGRAPHIC ANNOTATION TOP MARK
+<U3196> "<U4E0A>"
+% IDEOGRAPHIC ANNOTATION MIDDLE MARK
+<U3197> "<U4E2D>"
+% IDEOGRAPHIC ANNOTATION BOTTOM MARK
+<U3198> "<U4E0B>"
+% IDEOGRAPHIC ANNOTATION FIRST MARK
+<U3199> "<U7532>"
+% IDEOGRAPHIC ANNOTATION SECOND MARK
+<U319A> "<U4E59>"
+% IDEOGRAPHIC ANNOTATION THIRD MARK
+<U319B> "<U4E19>"
+% IDEOGRAPHIC ANNOTATION FOURTH MARK
+<U319C> "<U4E01>"
+% IDEOGRAPHIC ANNOTATION HEAVEN MARK
+<U319D> "<U5929>"
+% IDEOGRAPHIC ANNOTATION EARTH MARK
+<U319E> "<U5730>"
+% IDEOGRAPHIC ANNOTATION MAN MARK
+<U319F> "<U4EBA>"
% PARENTHESIZED HANGUL KIYEOK
<U3200> "<U0028><U1100><U0029>"
% PARENTHESIZED HANGUL NIEUN
@@ -1076,6 +1454,10 @@ translit_start
<U321B> "<U0028><U1112><U1161><U0029>"
% PARENTHESIZED HANGUL CIEUC U
<U321C> "<U0028><U110C><U116E><U0029>"
+% PARENTHESIZED KOREAN CHARACTER OJEON
+<U321D> "<U0028><U110B><U1169><U110C><U1165><U11AB><U0029>"
+% PARENTHESIZED KOREAN CHARACTER O HU
+<U321E> "<U0028><U110B><U1169><U1112><U116E><U0029>"
% PARENTHESIZED IDEOGRAPH ONE
<U3220> "<U0028><U4E00><U0029>"
% PARENTHESIZED IDEOGRAPH TWO
@@ -1284,6 +1666,24 @@ translit_start
<U33FD> "<U0033><U0030><U65E5>"
% IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY THIRTY-ONE
<U33FE> "<U0033><U0031><U65E5>"
+% MODIFIER LETTER CYRILLIC HARD SIGN
+<UA69C> "<U044A>"
+% MODIFIER LETTER CYRILLIC SOFT SIGN
+<UA69D> "<U044C>"
+% MODIFIER LETTER US
+<UA770> "<UA76F>"
+% MODIFIER LETTER CAPITAL H WITH STROKE
+<UA7F8> "<U0126>"
+% MODIFIER LETTER SMALL LIGATURE OE
+<UA7F9> "<U0153>"
+% MODIFIER LETTER SMALL HENG
+<UAB5C> "<UA727>"
+% MODIFIER LETTER SMALL L WITH INVERTED LAZY S
+<UAB5D> "<UAB37>"
+% MODIFIER LETTER SMALL L WITH MIDDLE TILDE
+<UAB5E> "<U026B>"
+% MODIFIER LETTER SMALL U WITH LEFT HOOK
+<UAB5F> "<UAB52>"
% LATIN SMALL LIGATURE FF
<UFB00> "<U0066><U0066>"
% LATIN SMALL LIGATURE FI
@@ -1295,7 +1695,7 @@ translit_start
% LATIN SMALL LIGATURE FFL
<UFB04> "<U0066><U0066><U006C>"
% LATIN SMALL LIGATURE LONG S T
-<UFB05> "<U017F><U0074>"
+<UFB05> "<U0073><U0074>"
% LATIN SMALL LIGATURE ST
<UFB06> "<U0073><U0074>"
% ARMENIAN SMALL LIGATURE MEN NOW
@@ -1310,6 +1710,72 @@ translit_start
<UFB17> "<U0574><U056D>"
% HEBREW LIGATURE ALEF LAMED
<UFB4F> "<U05D0><U05DC>"
+% PRESENTATION FORM FOR VERTICAL COMMA
+<UFE10> "<U002C>"
+% PRESENTATION FORM FOR VERTICAL IDEOGRAPHIC COMMA
+<UFE11> "<U3001>"
+% PRESENTATION FORM FOR VERTICAL IDEOGRAPHIC FULL STOP
+<UFE12> "<U3002>"
+% PRESENTATION FORM FOR VERTICAL COLON
+<UFE13> "<U003A>"
+% PRESENTATION FORM FOR VERTICAL SEMICOLON
+<UFE14> "<U003B>"
+% PRESENTATION FORM FOR VERTICAL EXCLAMATION MARK
+<UFE15> "<U0021>"
+% PRESENTATION FORM FOR VERTICAL QUESTION MARK
+<UFE16> "<U003F>"
+% PRESENTATION FORM FOR VERTICAL LEFT WHITE LENTICULAR BRACKET
+<UFE17> "<U3016>"
+% PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRAKCET
+<UFE18> "<U3017>"
+% PRESENTATION FORM FOR VERTICAL HORIZONTAL ELLIPSIS
+<UFE19> "<U002E><U002E><U002E>"
+% PRESENTATION FORM FOR VERTICAL TWO DOT LEADER
+<UFE30> "<U002E><U002E>"
+% PRESENTATION FORM FOR VERTICAL EM DASH
+<UFE31> "<U2014>"
+% PRESENTATION FORM FOR VERTICAL EN DASH
+<UFE32> "<U2013>"
+% PRESENTATION FORM FOR VERTICAL LOW LINE
+<UFE33> "<U005F>"
+% PRESENTATION FORM FOR VERTICAL WAVY LOW LINE
+<UFE34> "<U005F>"
+% PRESENTATION FORM FOR VERTICAL LEFT PARENTHESIS
+<UFE35> "<U0028>"
+% PRESENTATION FORM FOR VERTICAL RIGHT PARENTHESIS
+<UFE36> "<U0029>"
+% PRESENTATION FORM FOR VERTICAL LEFT CURLY BRACKET
+<UFE37> "<U007B>"
+% PRESENTATION FORM FOR VERTICAL RIGHT CURLY BRACKET
+<UFE38> "<U007D>"
+% PRESENTATION FORM FOR VERTICAL LEFT TORTOISE SHELL BRACKET
+<UFE39> "<U3014>"
+% PRESENTATION FORM FOR VERTICAL RIGHT TORTOISE SHELL BRACKET
+<UFE3A> "<U3015>"
+% PRESENTATION FORM FOR VERTICAL LEFT BLACK LENTICULAR BRACKET
+<UFE3B> "<U3010>"
+% PRESENTATION FORM FOR VERTICAL RIGHT BLACK LENTICULAR BRACKET
+<UFE3C> "<U3011>"
+% PRESENTATION FORM FOR VERTICAL LEFT DOUBLE ANGLE BRACKET
+<UFE3D> "<U300A>"
+% PRESENTATION FORM FOR VERTICAL RIGHT DOUBLE ANGLE BRACKET
+<UFE3E> "<U300B>"
+% PRESENTATION FORM FOR VERTICAL LEFT ANGLE BRACKET
+<UFE3F> "<U3008>"
+% PRESENTATION FORM FOR VERTICAL RIGHT ANGLE BRACKET
+<UFE40> "<U3009>"
+% PRESENTATION FORM FOR VERTICAL LEFT CORNER BRACKET
+<UFE41> "<U300C>"
+% PRESENTATION FORM FOR VERTICAL RIGHT CORNER BRACKET
+<UFE42> "<U300D>"
+% PRESENTATION FORM FOR VERTICAL LEFT WHITE CORNER BRACKET
+<UFE43> "<U300E>"
+% PRESENTATION FORM FOR VERTICAL RIGHT WHITE CORNER BRACKET
+<UFE44> "<U300F>"
+% PRESENTATION FORM FOR VERTICAL LEFT SQUARE BRACKET
+<UFE47> "<U005B>"
+% PRESENTATION FORM FOR VERTICAL RIGHT SQUARE BRACKET
+<UFE48> "<U005D>"
% DASHED OVERLINE
<UFE49> "<U203E>"
% CENTRELINE OVERLINE
@@ -1324,6 +1790,104 @@ translit_start
<UFE4E> "<U005F>"
% WAVY LOW LINE
<UFE4F> "<U005F>"
+% DIGIT ZERO FULL STOP
+<U0001F100> "<U0030><U002E>"
+% DIGIT ZERO COMMA
+<U0001F101> "<U0030><U002C>"
+% DIGIT ONE COMMA
+<U0001F102> "<U0031><U002C>"
+% DIGIT TWO COMMA
+<U0001F103> "<U0032><U002C>"
+% DIGIT THREE COMMA
+<U0001F104> "<U0033><U002C>"
+% DIGIT FOUR COMMA
+<U0001F105> "<U0034><U002C>"
+% DIGIT FIVE COMMA
+<U0001F106> "<U0035><U002C>"
+% DIGIT SIX COMMA
+<U0001F107> "<U0036><U002C>"
+% DIGIT SEVEN COMMA
+<U0001F108> "<U0037><U002C>"
+% DIGIT EIGHT COMMA
+<U0001F109> "<U0038><U002C>"
+% DIGIT NINE COMMA
+<U0001F10A> "<U0039><U002C>"
+% PARENTHESIZED LATIN CAPITAL LETTER A
+<U0001F110> "<U0028><U0041><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER B
+<U0001F111> "<U0028><U0042><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER C
+<U0001F112> "<U0028><U0043><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER D
+<U0001F113> "<U0028><U0044><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER E
+<U0001F114> "<U0028><U0045><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER F
+<U0001F115> "<U0028><U0046><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER G
+<U0001F116> "<U0028><U0047><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER H
+<U0001F117> "<U0028><U0048><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER I
+<U0001F118> "<U0028><U0049><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER J
+<U0001F119> "<U0028><U004A><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER K
+<U0001F11A> "<U0028><U004B><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER L
+<U0001F11B> "<U0028><U004C><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER M
+<U0001F11C> "<U0028><U004D><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER N
+<U0001F11D> "<U0028><U004E><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER O
+<U0001F11E> "<U0028><U004F><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER P
+<U0001F11F> "<U0028><U0050><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER Q
+<U0001F120> "<U0028><U0051><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER R
+<U0001F121> "<U0028><U0052><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER S
+<U0001F122> "<U0028><U0053><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER T
+<U0001F123> "<U0028><U0054><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER U
+<U0001F124> "<U0028><U0055><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER V
+<U0001F125> "<U0028><U0056><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER W
+<U0001F126> "<U0028><U0057><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER X
+<U0001F127> "<U0028><U0058><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER Y
+<U0001F128> "<U0028><U0059><U0029>"
+% PARENTHESIZED LATIN CAPITAL LETTER Z
+<U0001F129> "<U0028><U005A><U0029>"
+% TORTOISE SHELL BRACKETED LATIN CAPITAL LETTER S
+<U0001F12A> "<U3014><U0053><U3015>"
+% RAISED MC SIGN
+<U0001F16A> "<U004D><U0043>"
+% RAISED MD SIGN
+<U0001F16B> "<U004D><U0044>"
+% TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-672C
+<U0001F240> "<U3014><U672C><U3015>"
+% TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-4E09
+<U0001F241> "<U3014><U4E09><U3015>"
+% TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-4E8C
+<U0001F242> "<U3014><U4E8C><U3015>"
+% TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-5B89
+<U0001F243> "<U3014><U5B89><U3015>"
+% TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-70B9
+<U0001F244> "<U3014><U70B9><U3015>"
+% TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-6253
+<U0001F245> "<U3014><U6253><U3015>"
+% TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-76D7
+<U0001F246> "<U3014><U76D7><U3015>"
+% TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-52DD
+<U0001F247> "<U3014><U52DD><U3015>"
+% TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-6557
+<U0001F248> "<U3014><U6557><U3015>"
translit_end
diff --git a/localedata/locales/translit_font b/localedata/locales/translit_font
index 9347bd4..65e0d90 100644
--- a/localedata/locales/translit_font
+++ b/localedata/locales/translit_font
@@ -2,9 +2,7 @@ escape_char /
comment_char %
% Transliterations of font equivalents.
-% Generated through
-% $ grep '^[^;]*;[^;]*;[^;]*;[^;]*;[^;]*;<font>[^;]*;' UnicodeData.txt | \
-% sed -e 's/^\([^;]*\);\([^;]*\);[^;]*;[^;]*;[^;]*;<font> \([^;]*\);.*$/<U\1> <U\3> % \2/'
+% Generated automatically from UnicodeData.txt by gen_translit_font.py on 2015-06-10 for Unicode 7.0.0.
LC_CTYPE
@@ -37,6 +35,7 @@ translit_start
<U2133> <U004D> % SCRIPT CAPITAL M
<U2134> <U006F> % SCRIPT SMALL O
<U2139> <U0069> % INFORMATION SOURCE
+<U213C> <U03C0> % DOUBLE-STRUCK SMALL PI
<U213D> <U03B3> % DOUBLE-STRUCK SMALL GAMMA
<U213E> <U0393> % DOUBLE-STRUCK CAPITAL GAMMA
<U213F> <U03A0> % DOUBLE-STRUCK CAPITAL PI
@@ -238,6 +237,7 @@ translit_start
<U0001D4BE> <U0069> % MATHEMATICAL SCRIPT SMALL I
<U0001D4BF> <U006A> % MATHEMATICAL SCRIPT SMALL J
<U0001D4C0> <U006B> % MATHEMATICAL SCRIPT SMALL K
+<U0001D4C1> <U006C> % MATHEMATICAL SCRIPT SMALL L
<U0001D4C2> <U006D> % MATHEMATICAL SCRIPT SMALL M
<U0001D4C3> <U006E> % MATHEMATICAL SCRIPT SMALL N
<U0001D4C5> <U0070> % MATHEMATICAL SCRIPT SMALL P
@@ -707,6 +707,8 @@ translit_start
<U0001D6A1> <U0078> % MATHEMATICAL MONOSPACE SMALL X
<U0001D6A2> <U0079> % MATHEMATICAL MONOSPACE SMALL Y
<U0001D6A3> <U007A> % MATHEMATICAL MONOSPACE SMALL Z
+<U0001D6A4> <U0131> % MATHEMATICAL ITALIC SMALL DOTLESS I
+<U0001D6A5> <U0237> % MATHEMATICAL ITALIC SMALL DOTLESS J
<U0001D6A8> <U0391> % MATHEMATICAL BOLD CAPITAL ALPHA
<U0001D6A9> <U0392> % MATHEMATICAL BOLD CAPITAL BETA
<U0001D6AA> <U0393> % MATHEMATICAL BOLD CAPITAL GAMMA
@@ -997,6 +999,8 @@ translit_start
<U0001D7C7> <U03D5> % MATHEMATICAL SANS-SERIF BOLD ITALIC PHI SYMBOL
<U0001D7C8> <U03F1> % MATHEMATICAL SANS-SERIF BOLD ITALIC RHO SYMBOL
<U0001D7C9> <U03D6> % MATHEMATICAL SANS-SERIF BOLD ITALIC PI SYMBOL
+<U0001D7CA> <U03DC> % MATHEMATICAL BOLD CAPITAL DIGAMMA
+<U0001D7CB> <U03DD> % MATHEMATICAL BOLD SMALL DIGAMMA
<U0001D7CE> <U0030> % MATHEMATICAL BOLD DIGIT ZERO
<U0001D7CF> <U0031> % MATHEMATICAL BOLD DIGIT ONE
<U0001D7D0> <U0032> % MATHEMATICAL BOLD DIGIT TWO
@@ -1047,6 +1051,147 @@ translit_start
<U0001D7FD> <U0037> % MATHEMATICAL MONOSPACE DIGIT SEVEN
<U0001D7FE> <U0038> % MATHEMATICAL MONOSPACE DIGIT EIGHT
<U0001D7FF> <U0039> % MATHEMATICAL MONOSPACE DIGIT NINE
+<U0001EE00> <U0627> % ARABIC MATHEMATICAL ALEF
+<U0001EE01> <U0628> % ARABIC MATHEMATICAL BEH
+<U0001EE02> <U062C> % ARABIC MATHEMATICAL JEEM
+<U0001EE03> <U062F> % ARABIC MATHEMATICAL DAL
+<U0001EE05> <U0648> % ARABIC MATHEMATICAL WAW
+<U0001EE06> <U0632> % ARABIC MATHEMATICAL ZAIN
+<U0001EE07> <U062D> % ARABIC MATHEMATICAL HAH
+<U0001EE08> <U0637> % ARABIC MATHEMATICAL TAH
+<U0001EE09> <U064A> % ARABIC MATHEMATICAL YEH
+<U0001EE0A> <U0643> % ARABIC MATHEMATICAL KAF
+<U0001EE0B> <U0644> % ARABIC MATHEMATICAL LAM
+<U0001EE0C> <U0645> % ARABIC MATHEMATICAL MEEM
+<U0001EE0D> <U0646> % ARABIC MATHEMATICAL NOON
+<U0001EE0E> <U0633> % ARABIC MATHEMATICAL SEEN
+<U0001EE0F> <U0639> % ARABIC MATHEMATICAL AIN
+<U0001EE10> <U0641> % ARABIC MATHEMATICAL FEH
+<U0001EE11> <U0635> % ARABIC MATHEMATICAL SAD
+<U0001EE12> <U0642> % ARABIC MATHEMATICAL QAF
+<U0001EE13> <U0631> % ARABIC MATHEMATICAL REH
+<U0001EE14> <U0634> % ARABIC MATHEMATICAL SHEEN
+<U0001EE15> <U062A> % ARABIC MATHEMATICAL TEH
+<U0001EE16> <U062B> % ARABIC MATHEMATICAL THEH
+<U0001EE17> <U062E> % ARABIC MATHEMATICAL KHAH
+<U0001EE18> <U0630> % ARABIC MATHEMATICAL THAL
+<U0001EE19> <U0636> % ARABIC MATHEMATICAL DAD
+<U0001EE1A> <U0638> % ARABIC MATHEMATICAL ZAH
+<U0001EE1B> <U063A> % ARABIC MATHEMATICAL GHAIN
+<U0001EE1C> <U066E> % ARABIC MATHEMATICAL DOTLESS BEH
+<U0001EE1D> <U06BA> % ARABIC MATHEMATICAL DOTLESS NOON
+<U0001EE1E> <U06A1> % ARABIC MATHEMATICAL DOTLESS FEH
+<U0001EE1F> <U066F> % ARABIC MATHEMATICAL DOTLESS QAF
+<U0001EE21> <U0628> % ARABIC MATHEMATICAL INITIAL BEH
+<U0001EE22> <U062C> % ARABIC MATHEMATICAL INITIAL JEEM
+<U0001EE24> <U0647> % ARABIC MATHEMATICAL INITIAL HEH
+<U0001EE27> <U062D> % ARABIC MATHEMATICAL INITIAL HAH
+<U0001EE29> <U064A> % ARABIC MATHEMATICAL INITIAL YEH
+<U0001EE2A> <U0643> % ARABIC MATHEMATICAL INITIAL KAF
+<U0001EE2B> <U0644> % ARABIC MATHEMATICAL INITIAL LAM
+<U0001EE2C> <U0645> % ARABIC MATHEMATICAL INITIAL MEEM
+<U0001EE2D> <U0646> % ARABIC MATHEMATICAL INITIAL NOON
+<U0001EE2E> <U0633> % ARABIC MATHEMATICAL INITIAL SEEN
+<U0001EE2F> <U0639> % ARABIC MATHEMATICAL INITIAL AIN
+<U0001EE30> <U0641> % ARABIC MATHEMATICAL INITIAL FEH
+<U0001EE31> <U0635> % ARABIC MATHEMATICAL INITIAL SAD
+<U0001EE32> <U0642> % ARABIC MATHEMATICAL INITIAL QAF
+<U0001EE34> <U0634> % ARABIC MATHEMATICAL INITIAL SHEEN
+<U0001EE35> <U062A> % ARABIC MATHEMATICAL INITIAL TEH
+<U0001EE36> <U062B> % ARABIC MATHEMATICAL INITIAL THEH
+<U0001EE37> <U062E> % ARABIC MATHEMATICAL INITIAL KHAH
+<U0001EE39> <U0636> % ARABIC MATHEMATICAL INITIAL DAD
+<U0001EE3B> <U063A> % ARABIC MATHEMATICAL INITIAL GHAIN
+<U0001EE42> <U062C> % ARABIC MATHEMATICAL TAILED JEEM
+<U0001EE47> <U062D> % ARABIC MATHEMATICAL TAILED HAH
+<U0001EE49> <U064A> % ARABIC MATHEMATICAL TAILED YEH
+<U0001EE4B> <U0644> % ARABIC MATHEMATICAL TAILED LAM
+<U0001EE4D> <U0646> % ARABIC MATHEMATICAL TAILED NOON
+<U0001EE4E> <U0633> % ARABIC MATHEMATICAL TAILED SEEN
+<U0001EE4F> <U0639> % ARABIC MATHEMATICAL TAILED AIN
+<U0001EE51> <U0635> % ARABIC MATHEMATICAL TAILED SAD
+<U0001EE52> <U0642> % ARABIC MATHEMATICAL TAILED QAF
+<U0001EE54> <U0634> % ARABIC MATHEMATICAL TAILED SHEEN
+<U0001EE57> <U062E> % ARABIC MATHEMATICAL TAILED KHAH
+<U0001EE59> <U0636> % ARABIC MATHEMATICAL TAILED DAD
+<U0001EE5B> <U063A> % ARABIC MATHEMATICAL TAILED GHAIN
+<U0001EE5D> <U06BA> % ARABIC MATHEMATICAL TAILED DOTLESS NOON
+<U0001EE5F> <U066F> % ARABIC MATHEMATICAL TAILED DOTLESS QAF
+<U0001EE61> <U0628> % ARABIC MATHEMATICAL STRETCHED BEH
+<U0001EE62> <U062C> % ARABIC MATHEMATICAL STRETCHED JEEM
+<U0001EE64> <U0647> % ARABIC MATHEMATICAL STRETCHED HEH
+<U0001EE67> <U062D> % ARABIC MATHEMATICAL STRETCHED HAH
+<U0001EE68> <U0637> % ARABIC MATHEMATICAL STRETCHED TAH
+<U0001EE69> <U064A> % ARABIC MATHEMATICAL STRETCHED YEH
+<U0001EE6A> <U0643> % ARABIC MATHEMATICAL STRETCHED KAF
+<U0001EE6C> <U0645> % ARABIC MATHEMATICAL STRETCHED MEEM
+<U0001EE6D> <U0646> % ARABIC MATHEMATICAL STRETCHED NOON
+<U0001EE6E> <U0633> % ARABIC MATHEMATICAL STRETCHED SEEN
+<U0001EE6F> <U0639> % ARABIC MATHEMATICAL STRETCHED AIN
+<U0001EE70> <U0641> % ARABIC MATHEMATICAL STRETCHED FEH
+<U0001EE71> <U0635> % ARABIC MATHEMATICAL STRETCHED SAD
+<U0001EE72> <U0642> % ARABIC MATHEMATICAL STRETCHED QAF
+<U0001EE74> <U0634> % ARABIC MATHEMATICAL STRETCHED SHEEN
+<U0001EE75> <U062A> % ARABIC MATHEMATICAL STRETCHED TEH
+<U0001EE76> <U062B> % ARABIC MATHEMATICAL STRETCHED THEH
+<U0001EE77> <U062E> % ARABIC MATHEMATICAL STRETCHED KHAH
+<U0001EE79> <U0636> % ARABIC MATHEMATICAL STRETCHED DAD
+<U0001EE7A> <U0638> % ARABIC MATHEMATICAL STRETCHED ZAH
+<U0001EE7B> <U063A> % ARABIC MATHEMATICAL STRETCHED GHAIN
+<U0001EE7C> <U066E> % ARABIC MATHEMATICAL STRETCHED DOTLESS BEH
+<U0001EE7E> <U06A1> % ARABIC MATHEMATICAL STRETCHED DOTLESS FEH
+<U0001EE80> <U0627> % ARABIC MATHEMATICAL LOOPED ALEF
+<U0001EE81> <U0628> % ARABIC MATHEMATICAL LOOPED BEH
+<U0001EE82> <U062C> % ARABIC MATHEMATICAL LOOPED JEEM
+<U0001EE83> <U062F> % ARABIC MATHEMATICAL LOOPED DAL
+<U0001EE84> <U0647> % ARABIC MATHEMATICAL LOOPED HEH
+<U0001EE85> <U0648> % ARABIC MATHEMATICAL LOOPED WAW
+<U0001EE86> <U0632> % ARABIC MATHEMATICAL LOOPED ZAIN
+<U0001EE87> <U062D> % ARABIC MATHEMATICAL LOOPED HAH
+<U0001EE88> <U0637> % ARABIC MATHEMATICAL LOOPED TAH
+<U0001EE89> <U064A> % ARABIC MATHEMATICAL LOOPED YEH
+<U0001EE8B> <U0644> % ARABIC MATHEMATICAL LOOPED LAM
+<U0001EE8C> <U0645> % ARABIC MATHEMATICAL LOOPED MEEM
+<U0001EE8D> <U0646> % ARABIC MATHEMATICAL LOOPED NOON
+<U0001EE8E> <U0633> % ARABIC MATHEMATICAL LOOPED SEEN
+<U0001EE8F> <U0639> % ARABIC MATHEMATICAL LOOPED AIN
+<U0001EE90> <U0641> % ARABIC MATHEMATICAL LOOPED FEH
+<U0001EE91> <U0635> % ARABIC MATHEMATICAL LOOPED SAD
+<U0001EE92> <U0642> % ARABIC MATHEMATICAL LOOPED QAF
+<U0001EE93> <U0631> % ARABIC MATHEMATICAL LOOPED REH
+<U0001EE94> <U0634> % ARABIC MATHEMATICAL LOOPED SHEEN
+<U0001EE95> <U062A> % ARABIC MATHEMATICAL LOOPED TEH
+<U0001EE96> <U062B> % ARABIC MATHEMATICAL LOOPED THEH
+<U0001EE97> <U062E> % ARABIC MATHEMATICAL LOOPED KHAH
+<U0001EE98> <U0630> % ARABIC MATHEMATICAL LOOPED THAL
+<U0001EE99> <U0636> % ARABIC MATHEMATICAL LOOPED DAD
+<U0001EE9A> <U0638> % ARABIC MATHEMATICAL LOOPED ZAH
+<U0001EE9B> <U063A> % ARABIC MATHEMATICAL LOOPED GHAIN
+<U0001EEA1> <U0628> % ARABIC MATHEMATICAL DOUBLE-STRUCK BEH
+<U0001EEA2> <U062C> % ARABIC MATHEMATICAL DOUBLE-STRUCK JEEM
+<U0001EEA3> <U062F> % ARABIC MATHEMATICAL DOUBLE-STRUCK DAL
+<U0001EEA5> <U0648> % ARABIC MATHEMATICAL DOUBLE-STRUCK WAW
+<U0001EEA6> <U0632> % ARABIC MATHEMATICAL DOUBLE-STRUCK ZAIN
+<U0001EEA7> <U062D> % ARABIC MATHEMATICAL DOUBLE-STRUCK HAH
+<U0001EEA8> <U0637> % ARABIC MATHEMATICAL DOUBLE-STRUCK TAH
+<U0001EEA9> <U064A> % ARABIC MATHEMATICAL DOUBLE-STRUCK YEH
+<U0001EEAB> <U0644> % ARABIC MATHEMATICAL DOUBLE-STRUCK LAM
+<U0001EEAC> <U0645> % ARABIC MATHEMATICAL DOUBLE-STRUCK MEEM
+<U0001EEAD> <U0646> % ARABIC MATHEMATICAL DOUBLE-STRUCK NOON
+<U0001EEAE> <U0633> % ARABIC MATHEMATICAL DOUBLE-STRUCK SEEN
+<U0001EEAF> <U0639> % ARABIC MATHEMATICAL DOUBLE-STRUCK AIN
+<U0001EEB0> <U0641> % ARABIC MATHEMATICAL DOUBLE-STRUCK FEH
+<U0001EEB1> <U0635> % ARABIC MATHEMATICAL DOUBLE-STRUCK SAD
+<U0001EEB2> <U0642> % ARABIC MATHEMATICAL DOUBLE-STRUCK QAF
+<U0001EEB3> <U0631> % ARABIC MATHEMATICAL DOUBLE-STRUCK REH
+<U0001EEB4> <U0634> % ARABIC MATHEMATICAL DOUBLE-STRUCK SHEEN
+<U0001EEB5> <U062A> % ARABIC MATHEMATICAL DOUBLE-STRUCK TEH
+<U0001EEB6> <U062B> % ARABIC MATHEMATICAL DOUBLE-STRUCK THEH
+<U0001EEB7> <U062E> % ARABIC MATHEMATICAL DOUBLE-STRUCK KHAH
+<U0001EEB8> <U0630> % ARABIC MATHEMATICAL DOUBLE-STRUCK THAL
+<U0001EEB9> <U0636> % ARABIC MATHEMATICAL DOUBLE-STRUCK DAD
+<U0001EEBA> <U0638> % ARABIC MATHEMATICAL DOUBLE-STRUCK ZAH
+<U0001EEBB> <U063A> % ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN
translit_end
diff --git a/localedata/locales/translit_fraction b/localedata/locales/translit_fraction
index 50dbd78..30f2843 100644
--- a/localedata/locales/translit_fraction
+++ b/localedata/locales/translit_fraction
@@ -2,10 +2,7 @@ escape_char /
comment_char %
% Transliterations of fractions.
-% Generated through
-% $ grep '^[^;]*;[^;]*;[^;]*;[^;]*;[^;]*;<fraction>[^;]*;' UnicodeData.txt | \
-% sed -e 's/^\([^;]*\);\([^;]*\);[^;]*;[^;]*;[^;]*;<fraction> \([^;]*\);.*$/<U\1> "<U\3>"% \2/' -e 'h' -e 's/^\([^%]*\)% .*$/\1/' -e 's/\([0-9A-F]\) \([0-9A-F]\)/\1><U\2/g' -e 'x' -e 's/^[^%]*\(% .*\)$/\1/' -e 'G'
-%
+% Generated automatically from UnicodeData.txt by gen_translit_fraction.py on 2015-06-10 for Unicode 7.0.0.
% The replacements have been surrounded with spaces, because fractions are
% often preceded by a decimal number and followed by a unit or a math symbol.
@@ -19,6 +16,12 @@ translit_start
<U00BD> "<U0020><U0031><U2044><U0032><U0020>";"<U0020><U0031><U002F><U0032><U0020>"
% VULGAR FRACTION THREE QUARTERS
<U00BE> "<U0020><U0033><U2044><U0034><U0020>";"<U0020><U0033><U002F><U0034><U0020>"
+% VULGAR FRACTION ONE SEVENTH
+<U2150> "<U0020><U0031><U2044><U0037><U0020>";"<U0020><U0031><U002F><U0037><U0020>"
+% VULGAR FRACTION ONE NINTH
+<U2151> "<U0020><U0031><U2044><U0039><U0020>";"<U0020><U0031><U002F><U0039><U0020>"
+% VULGAR FRACTION ONE TENTH
+<U2152> "<U0020><U0031><U2044><U0031><U0030><U0020>";"<U0020><U0031><U002F><U0031><U0030><U0020>"
% VULGAR FRACTION ONE THIRD
<U2153> "<U0020><U0031><U2044><U0033><U0020>";"<U0020><U0031><U002F><U0033><U0020>"
% VULGAR FRACTION TWO THIRDS
@@ -44,7 +47,9 @@ translit_start
% VULGAR FRACTION SEVEN EIGHTHS
<U215E> "<U0020><U0037><U2044><U0038><U0020>";"<U0020><U0037><U002F><U0038><U0020>"
% FRACTION NUMERATOR ONE
-<U215F> "<U0020><U0031><U2044>";"<U0020><U0031><U002F>"
+<U215F> "<U0020><U0031><U2044><U0020>";"<U0020><U0031><U002F><U0020>"
+% VULGAR FRACTION ZERO THIRDS
+<U2189> "<U0020><U0030><U2044><U0033><U0020>";"<U0020><U0030><U002F><U0033><U0020>"
translit_end
diff --git a/localedata/unicode-gen/Makefile b/localedata/unicode-gen/Makefile
index 166ee31..920bf0e 100644
--- a/localedata/unicode-gen/Makefile
+++ b/localedata/unicode-gen/Makefile
@@ -41,7 +41,7 @@ PYTHON3 = python3
WGET = wget
DOWNLOADS = UnicodeData.txt DerivedCoreProperties.txt EastAsianWidth.txt
-GENERATED = i18n UTF-8
+GENERATED = i18n UTF-8 translit_combining translit_compat translit_circle translit_cjk_compat translit_font translit_fraction
REPORTS = i18n-report UTF-8-report
all: $(GENERATED)
@@ -51,6 +51,12 @@ check: check-i18n check-UTF-8
install:
cp -p i18n ../locales/i18n
cp -p UTF-8 ../charmaps/UTF-8
+ cp -p translit_combining ../locales/translit_combining
+ cp -p translit_compat ../locales/translit_compat
+ cp -p translit_circle ../locales/translit_circle
+ cp -p translit_cjk_compat ../locales/translit_cjk_compat
+ cp -p translit_font ../locales/translit_font
+ cp -p translit_fraction ../locales/translit_fraction
clean: mostlyclean
-rm -rf __pycache__
@@ -82,13 +88,43 @@ UTF-8: utf8_gen.py
UTF-8-report: UTF-8 ../charmaps/UTF-8
UTF-8-report: utf8_compatibility.py
- $(PYTHON3) ./utf8_compatibility.py -o ../charmaps/UTF-8 \
- -n UTF-8 -a -m > $@
+ $(PYTHON3) ./utf8_compatibility.py -u UnicodeData.txt \
+ -e EastAsianWidth.txt -o ../charmaps/UTF-8 \
+ -n UTF-8 -a -m -c > $@
check-UTF-8: UTF-8-report
@if grep '^Total.*: [^0]' UTF-8-report; \
then echo manual verification required; false; else true; fi
+translit_combining: UnicodeData.txt
+translit_combining: gen_translit_combining.py
+ $(PYTHON3) ./gen_translit_combining.py -u UnicodeData.txt \
+ -o $@ --unicode_version $(UNICODE_VERSION)
+
+translit_compat: UnicodeData.txt
+translit_compat: gen_translit_compat.py
+ $(PYTHON3) ./gen_translit_compat.py -u UnicodeData.txt \
+ -o $@ --unicode_version $(UNICODE_VERSION)
+
+translit_circle: UnicodeData.txt
+translit_circle: gen_translit_circle.py
+ $(PYTHON3) ./gen_translit_circle.py -u UnicodeData.txt \
+ -o $@ --unicode_version $(UNICODE_VERSION)
+
+translit_cjk_compat: UnicodeData.txt
+translit_cjk_compat: gen_translit_cjk_compat.py
+ $(PYTHON3) ./gen_translit_cjk_compat.py -u UnicodeData.txt \
+ -o $@ --unicode_version $(UNICODE_VERSION)
+
+translit_font: UnicodeData.txt
+translit_font: gen_translit_font.py
+ $(PYTHON3) ./gen_translit_font.py -u UnicodeData.txt \
+ -o $@ --unicode_version $(UNICODE_VERSION)
+
+translit_fraction: UnicodeData.txt
+translit_fraction: gen_translit_fraction.py
+ $(PYTHON3) ./gen_translit_fraction.py -u UnicodeData.txt \
+ -o $@ --unicode_version $(UNICODE_VERSION)
.PHONY: downloads clean-downloads
downloads: $(DOWNLOADS)
diff --git a/localedata/unicode-gen/gen_translit_circle.py b/localedata/unicode-gen/gen_translit_circle.py
new file mode 100755
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
index 0000000..6142859
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
--- /dev/null
+++ b/localedata/unicode-gen/gen_translit_circle.py
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
@@ -0,0 +1,150 @@
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+#!/usr/bin/python3
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+# -*- coding: utf-8 -*-
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+#
+# Generate a translit_circle file from a UnicodeData file.
+# Copyright (C) 2015 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+#
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <http://www.gnu.org/licenses/>.
+
+'''
+Generate a translit_circle file from UnicodeData.txt
+
+To see how this script is used, call it with the “-h” option:
+
+ $ ./gen_translit_circle -h
+ … prints usage message …
+'''
+
+import argparse
+import time
+import unicode_utils
+
+def read_input_file(filename):
+ '''Reads the original glibc translit_circle file to get the
+ original head and tail.
+
+ We want to replace only the part of the file between
+ “translit_start” and “translit_end”
+ '''
+ head = tail = ''
+ with open(filename, mode='r') as translit_file:
+ for line in translit_file:
+ head = head + line
+ if line.startswith('translit_start'):
+ break
+ for line in translit_file:
+ if line.startswith('translit_end'):
+ tail = line
+ break
+ for line in translit_file:
+ tail = tail + line
+ return (head, tail)
+
+def output_head(translit_file, unicode_version, head=''):
+ '''Write the header of the output file, i.e. the part of the file
+ before the “translit_start” line.
+ '''
+ if ARGS.input_file and head:
+ translit_file.write(head)
+ else:
+ translit_file.write('escape_char /\n')
+ translit_file.write('comment_char %\n')
+ translit_file.write('\n')
+ translit_file.write('% Transliterations of encircled characters.\n')
+ translit_file.write('% Generated automatically from UnicodeData.txt '
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+ + 'by gen_translit_circle.py '
+ + 'on {:s} '.format(time.strftime('%Y-%m-%d'))
+ + 'for Unicode {:s}.\n'.format(unicode_version))
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+ translit_file.write('\n')
+ translit_file.write('LC_CTYPE\n')
+ translit_file.write('\n')
+ translit_file.write('translit_start\n')
+
+def output_tail(translit_file, tail=''):
+ '''Write the tail of the output file'''
+ if ARGS.input_file and tail:
+ translit_file.write(tail)
+ else:
+ translit_file.write('translit_end\n')
+ translit_file.write('\n')
+ translit_file.write('END LC_CTYPE\n')
+
+def output_transliteration(translit_file):
+ '''Write the new transliteration to the output file'''
+ translit_file.write('\n')
+ for code_point in sorted(unicode_utils.UNICODE_ATTRIBUTES):
+ name = unicode_utils.UNICODE_ATTRIBUTES[code_point]['name']
+ decomposition = unicode_utils.UNICODE_ATTRIBUTES[
+ code_point]['decomposition']
+ if decomposition.startswith('<circle>'):
+ decomposition = decomposition[9:]
+ decomposed_code_points = [int(x, 16)
+ for x in decomposition.split(' ')]
+ translit_file.write('% {:s}\n'.format(name))
+ translit_file.write('{:s} "<U0028>'.format(
+ unicode_utils.ucs_symbol(code_point)))
+ for decomposed_code_point in decomposed_code_points:
+ translit_file.write('{:s}'.format(
+ unicode_utils.ucs_symbol(decomposed_code_point)))
+ translit_file.write('<U0029>"\n')
+ translit_file.write('\n')
+
+
+if __name__ == "__main__":
+ PARSER = argparse.ArgumentParser(
+ description='''
+ Generate a translit_circle file from UnicodeData.txt.
+ ''')
+ PARSER.add_argument(
+ '-u', '--unicode_data_file',
+ nargs='?',
+ type=str,
+ default='UnicodeData.txt',
+ help=('The UnicodeData.txt file to read, '
+ + 'default: %(default)s'))
+ PARSER.add_argument(
+ '-i', '--input_file',
+ nargs='?',
+ type=str,
+ help=''' The original glibc/localedata/locales/translit_combining
+ file.''')
+ PARSER.add_argument(
+ '-o', '--output_file',
+ nargs='?',
+ type=str,
+ default='translit_circle.new',
+ help='''The new translit_circle file, default: %(default)s. If the
+ original glibc/localedata/locales/translit_circle file has
+ been given as an option, the header up to the
+ “translit_start” line and the tail from the “translit_end”
+ line to the end of the file will be copied unchanged into the
+ output file. ''')
+ PARSER.add_argument(
+ '--unicode_version',
+ nargs='?',
+ required=True,
+ type=str,
+ help='The Unicode version of the input files used.')
+ ARGS = PARSER.parse_args()
+
+ unicode_utils.fill_attributes(ARGS.unicode_data_file)
+ HEAD = TAIL = ''
+ if ARGS.input_file:
+ (HEAD, TAIL) = read_input_file(ARGS.input_file)
+ with open(ARGS.output_file, mode='w') as TRANSLIT_FILE:
+ output_head(TRANSLIT_FILE, ARGS.unicode_version, head=HEAD)
+ output_transliteration(TRANSLIT_FILE)
+ output_tail(TRANSLIT_FILE, tail=TAIL)
diff --git a/localedata/unicode-gen/gen_translit_cjk_compat.py b/localedata/unicode-gen/gen_translit_cjk_compat.py
new file mode 100755
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
index 0000000..627ff6b
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
--- /dev/null
+++ b/localedata/unicode-gen/gen_translit_cjk_compat.py
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
@@ -0,0 +1,220 @@
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+#!/usr/bin/python3
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+# -*- coding: utf-8 -*-
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+#
+# Generate a translit_cjk_compat file from a UnicodeData file.
+# Copyright (C) 2015 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+#
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <http://www.gnu.org/licenses/>.
+
+'''
+Generate a translit_cjk_compat file from UnicodeData.txt
+
+To see how this script is used, call it with the “-h” option:
+
+ $ ./gen_translit_cjk_compat -h
+ … prints usage message …
+'''
+
+import argparse
+import time
+import sys
+import unicode_utils
+
+def read_input_file(filename):
+ '''Reads the original glibc translit_cjk_compat file to get the
+ original head and tail.
+
+ We want to replace only the part of the file between
+ “translit_start” and “translit_end”
+ '''
+ head = tail = ''
+ with open(filename, mode='r') as translit_file:
+ for line in translit_file:
+ head = head + line
+ if line.startswith('translit_start'):
+ break
+ for line in translit_file:
+ if line.startswith('translit_end'):
+ tail = line
+ break
+ for line in translit_file:
+ tail = tail + line
+ return (head, tail)
+
+def output_head(translit_file, unicode_version, head=''):
+ '''Write the header of the output file, i.e. the part of the file
+ before the “translit_start” line.
+ '''
+ if ARGS.input_file and head:
+ translit_file.write(head)
+ else:
+ translit_file.write('escape_char /\n')
+ translit_file.write('comment_char %\n')
+ translit_file.write('\n')
+ translit_file.write('% Transliterations of CJK compatibility ')
+ translit_file.write('characters.\n')
+ translit_file.write('% Generated automatically from UnicodeData.txt '
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+ + 'by gen_translit_cjk_compat.py '
+ + 'on {:s} '.format(time.strftime('%Y-%m-%d'))
+ + 'for Unicode {:s}.\n'.format(unicode_version))
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+ translit_file.write('\n')
+ translit_file.write('LC_CTYPE\n')
+ translit_file.write('\n')
+ translit_file.write('translit_start\n')
+
+def output_tail(translit_file, tail=''):
+ '''Write the tail of the output file'''
+ if ARGS.input_file and tail:
+ translit_file.write(tail)
+ else:
+ translit_file.write('translit_end\n')
+ translit_file.write('\n')
+ translit_file.write('END LC_CTYPE\n')
+
+def special_decompose(code_point_list):
+ '''
+ Decompositions which are not in UnicodeData.txt at all but which
+ were used in the original translit_cjk_compat file in glibc and
+ which seem to make sense. I want to keep the update of
+ translit_cjk_compat close to the spirit of the original file,
+ therefore I added this special decomposition rules here.
+ '''
+ special_decompose_dict = {
+ (0x2215,): [0x002F], # ∕ → /
+ (0x00B2,): [0x005E, 0x0032], # ² → ^2
+ (0x03BC,): [0x00B5], # μ → µ (GREEK SMALL LETTER MU → MICRO SIGN)
+ (0x2113,): [0x006C], # ℓ → l
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+ (0x00B3,): [0x005E, 0x0033], # ³ → ^3
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+ (0x00B5,): [0x0075], # µ → u
+ (0x03BC, 0x2113): [0x03BC, 0x006C], # μℓ → μl
+ (0x0072, 0x0061, 0x0064, 0x2215, 0x0073, 0x00B2): [
+ 0x0072, 0x0061, 0x0064, 0x002F, 0x0073, 0x00B2],
+ (0x006D, 0x2215, 0x0073, 0x00B2): [0x006D, 0x002F, 0x0073, 0x00B2],
+ }
+ if tuple(code_point_list) in special_decompose_dict:
+ return special_decompose_dict[tuple(code_point_list)]
+ else:
+ return code_point_list
+
+def output_transliteration(translit_file):
+ '''Write the new transliteration to the output file'''
+ translit_file.write('\n')
+ for code_point in sorted(unicode_utils.UNICODE_ATTRIBUTES):
+ name = unicode_utils.UNICODE_ATTRIBUTES[code_point]['name']
+ decomposition = unicode_utils.UNICODE_ATTRIBUTES[
+ code_point]['decomposition']
+ if decomposition.startswith('<square>'):
+ decomposition = decomposition[9:]
+ decomposed_code_points = [[int(x, 16)
+ for x in decomposition.split(' ')]]
+ if decomposed_code_points[0]:
+ while True:
+ special_decomposed_code_points = special_decompose(
+ decomposed_code_points[-1])
+ if (special_decomposed_code_points
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+ != decomposed_code_points[-1]):
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+ decomposed_code_points.append(
+ special_decomposed_code_points)
+ continue
+ special_decomposed_code_points = []
+ for decomposed_code_point in decomposed_code_points[-1]:
+ special_decomposed_code_points += special_decompose(
+ [decomposed_code_point])
+ if (special_decomposed_code_points
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+ == decomposed_code_points[-1]):
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+ break
+ decomposed_code_points.append(
+ special_decomposed_code_points)
+ translit_file.write('% {:s}\n'.format(name))
+ translit_file.write('{:s} '.format(
+ unicode_utils.ucs_symbol(code_point)))
+ for index in range(0, len(decomposed_code_points)):
+ if index > 0:
+ translit_file.write(';')
+ if len(decomposed_code_points[index]) > 1:
+ translit_file.write('"')
+ for decomposed_code_point in decomposed_code_points[index]:
+ translit_file.write('{:s}'.format(
+ unicode_utils.ucs_symbol(decomposed_code_point)))
+ if len(decomposed_code_points[index]) > 1:
+ translit_file.write('"')
+ translit_file.write('\n')
+ for code_point in sorted(unicode_utils.UNICODE_ATTRIBUTES):
+ name = unicode_utils.UNICODE_ATTRIBUTES[code_point]['name']
+ decomposition = unicode_utils.UNICODE_ATTRIBUTES[
+ code_point]['decomposition']
+ if decomposition and name.startswith('CJK COMPATIBILITY IDEOGRAPH'):
+ decomposed_code_points = [int(x, 16)
+ for x in decomposition.split(' ')]
+ if len(decomposed_code_points) != 1:
+ sys.stderr.write(
+ 'Unexpected decomposition length {:x} {:s} {:s}\n'.format(
+ code_point, name, decomposition))
+ exit(1)
+ translit_file.write('% {:s}\n'.format(name))
+ translit_file.write('{:s} '.format(
+ unicode_utils.ucs_symbol(code_point)))
+ for decomposed_code_point in decomposed_code_points:
+ translit_file.write('{:s}'.format(
+ unicode_utils.ucs_symbol(decomposed_code_point)))
+ translit_file.write('\n')
+ translit_file.write('\n')
+
+if __name__ == "__main__":
+ PARSER = argparse.ArgumentParser(
+ description='''
+ Generate a translit_cjk_compat file from UnicodeData.txt.
+ ''')
+ PARSER.add_argument(
+ '-u', '--unicode_data_file',
+ nargs='?',
+ type=str,
+ default='UnicodeData.txt',
+ help=('The UnicodeData.txt file to read, '
+ + 'default: %(default)s'))
+ PARSER.add_argument(
+ '-i', '--input_file',
+ nargs='?',
+ type=str,
+ help=''' The original glibc/localedata/locales/translit_cjk_compat
+ file.''')
+ PARSER.add_argument(
+ '-o', '--output_file',
+ nargs='?',
+ type=str,
+ default='translit_cjk_compat.new',
+ help='''The new translit_cjk_compat file, default: %(default)s. If the
+ original glibc/localedata/locales/translit_cjk_compat file has
+ been given as an option, the header up to the
+ “translit_start” line and the tail from the “translit_end”
+ line to the end of the file will be copied unchanged into the
+ output file. ''')
+ PARSER.add_argument(
+ '--unicode_version',
+ nargs='?',
+ required=True,
+ type=str,
+ help='The Unicode version of the input files used.')
+ ARGS = PARSER.parse_args()
+
+ unicode_utils.fill_attributes(ARGS.unicode_data_file)
+ HEAD = TAIL = ''
+ if ARGS.input_file:
+ (HEAD, TAIL) = read_input_file(ARGS.input_file)
+ with open(ARGS.output_file, mode='w') as TRANSLIT_FILE:
+ output_head(TRANSLIT_FILE, ARGS.unicode_version, head=HEAD)
+ output_transliteration(TRANSLIT_FILE)
+ output_tail(TRANSLIT_FILE, tail=TAIL)
diff --git a/localedata/unicode-gen/gen_translit_combining.py b/localedata/unicode-gen/gen_translit_combining.py
new file mode 100755
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
index 0000000..2551ce1
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
--- /dev/null
+++ b/localedata/unicode-gen/gen_translit_combining.py
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
@@ -0,0 +1,442 @@
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+#!/usr/bin/python3
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+# -*- coding: utf-8 -*-
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+#
+# Generate a translit_combining file from a UnicodeData file.
+# Copyright (C) 2015 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+#
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <http://www.gnu.org/licenses/>.
+
+'''
+Generate a translit_combining file from UnicodeData.txt
+
+To see how this script is used, call it with the “-h” option:
+
+ $ ./gen_translit_combining -h
+ … prints usage message …
+'''
+
+import argparse
+import time
+import unicode_utils
+
+def read_input_file(filename):
+ '''Reads the original glibc translit_combining file to get the
+ original head and tail.
+
+ We want to replace only the part of the file between
+ “translit_start” and “translit_end”
+ '''
+ head = tail = ''
+ with open(filename, mode='r') as translit_file:
+ for line in translit_file:
+ head = head + line
+ if line.startswith('translit_start'):
+ break
+ for line in translit_file:
+ if line.startswith('translit_end'):
+ tail = line
+ break
+ for line in translit_file:
+ tail = tail + line
+ return (head, tail)
+
+def output_head(translit_file, unicode_version, head=''):
+ '''Write the header of the output file, i.e. the part of the file
+ before the “translit_start” line.
+ '''
+ if ARGS.input_file and head:
+ translit_file.write(head)
+ else:
+ translit_file.write('escape_char /\n')
+ translit_file.write('comment_char %\n')
+ translit_file.write('\n')
+ translit_file.write('% Transliterations that remove all ')
+ translit_file.write('combining characters (accents,\n')
+ translit_file.write('% pronounciation marks, etc.).\n')
+ translit_file.write('% Generated automatically from UnicodeData.txt '
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+ + 'by gen_translit_combining.py '
+ + 'on {:s} '.format(time.strftime('%Y-%m-%d'))
+ + 'for Unicode {:s}.\n'.format(unicode_version))
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+ translit_file.write('\n')
+ translit_file.write('LC_CTYPE\n')
+ translit_file.write('\n')
+ translit_file.write('translit_start\n')
+
+def output_tail(translit_file, tail=''):
+ '''Write the tail of the output file'''
+ if ARGS.input_file and tail:
+ translit_file.write(tail)
+ else:
+ translit_file.write('translit_end\n')
+ translit_file.write('\n')
+ translit_file.write('END LC_CTYPE\n')
+
+def is_combining_remove(code_point):
+ '''Check whether this is a combining character which should be listed
+ in the section of the translit_combining file where combining
+ characters are replaced by empty strings.
+
+ We ignore combining characters from many scripts here because
+ the original translit_combining file didn’t do this for the
+ combining characters from these scripts either and I am not
+ sure yet whether this would be useful to do for all combining
+ characters or not. For the moment I think it is better to keep
+ close to the spirit of the original file.
+ '''
+ if not unicode_utils.is_combining(code_point):
+ return False
+ name = unicode_utils.UNICODE_ATTRIBUTES[code_point]['name']
+ for substring in ('DEVANAGARI',
+ 'BENGALI',
+ 'CYRILLIC',
+ 'SYRIAC',
+ 'THAANA',
+ 'NKO',
+ 'GURMUKHI',
+ 'TAMIL',
+ 'GUJARATI',
+ 'ORIYA',
+ 'TELUGU',
+ 'KANNADA',
+ 'MALAYALAM',
+ 'SINHALA',
+ 'THAI',
+ 'LAO',
+ 'TIBETAN',
+ 'MYANMAR',
+ 'ETHIOPIC',
+ 'TAGALOG',
+ 'HANUNOO',
+ 'BUHID',
+ 'TAGBANWA',
+ 'KHMER',
+ 'MONGOLIAN',
+ 'LIMBU',
+ 'NEW TAI LUE',
+ 'BUGINESE',
+ 'BALINESE',
+ 'SUNDANESE',
+ 'LEPCHA',
+ 'IDEOGRAPHIC',
+ 'HANGUL',
+ 'SYLOTI',
+ 'SAURASHTRA',
+ 'KAYAH',
+ 'REJANG',
+ 'CHAM',
+ 'VARIATION SELECTOR',
+ 'KHAROSHTHI',
+ 'MUSICAL SYMBOL',
+ 'SAMARITAN',
+ 'MANDAIC',
+ 'TAI THAM',
+ 'BATAK',
+ 'VEDIC',
+ 'COPTIC',
+ 'TIFINAGH',
+ 'BAMUM',
+ 'JAVANESE',
+ 'TAI VIET',
+ 'MEETEI',
+ 'MANICHAEAN',
+ 'BRAHMI',
+ 'KAITHI',
+ 'CHAKMA',
+ 'MAHAJANI',
+ 'SHARADA',
+ 'KHOJKI',
+ 'KHUDAWADI',
+ 'GRANTHA',
+ 'TIRHUTA',
+ 'SIDDHAM',
+ 'MODI VOWEL',
+ 'MODI SIGN',
+ 'TAKRI',
+ 'BASSA VAH',
+ 'PAHAWH HMONG',
+ 'MIAO',
+ 'DUPLOYAN',
+ 'MENDE KIKAKUI'
+ ):
+ if substring in name:
+ return False
+ return True
+
+def canonical_decompose(code_point):
+ '''http://www.unicode.org/reports/tr44/#Character_Decomposition_Mappings
+
+ In some instances a canonical mapping or a compatibility mapping
+ may consist of a single character. For a canonical mapping, this
+ indicates that the character is a canonical equivalent of another
+ single character. For a compatibility mapping, this indicates that
+ the character is a compatibility equivalent of another single
+ character.
+
+ A canonical mapping may also consist of a pair of characters, but
+ is never longer than two characters. When a canonical mapping
+ consists of a pair of characters, the first character may itself
+ be a character with a decomposition mapping, but the second
+ character never has a decomposition mapping.
+
+ We ignore the canonical decomposition for code points
+ matching certain substrings because the original translit_combining
+ file didn’t include these types of characters either. I am unsure
+ about the usefulness of including them and want to keep close
+ to the spirit of the original file for the moment.
+ '''
+ name = unicode_utils.UNICODE_ATTRIBUTES[code_point]['name']
+ for substring in ('MUSICAL SYMBOL',
+ 'CJK COMPATIBILITY IDEOGRAPH',
+ 'BALINESE',
+ 'KAITHI LETTER',
+ 'CHAKMA VOWEL',
+ 'GRANTHA VOWEL',
+ 'TIRHUTA VOWEL',
+ 'SIDDHAM VOWEL'):
+ if substring in name:
+ return []
+ decomposition = unicode_utils.UNICODE_ATTRIBUTES[
+ code_point]['decomposition']
+ if decomposition and not decomposition.startswith('<'):
+ decomposed_code_points = [int(x, 16) for x in decomposition.split(' ')]
+ if decomposed_code_points:
+ cd0 = canonical_decompose(decomposed_code_points[0])
+ if cd0:
+ decomposed_code_points = cd0 + decomposed_code_points[1:]
+ return decomposed_code_points
+ else:
+ return []
+
+def special_decompose(code_point_list):
+ '''
+ Decompositions which are not canonical or which are not in
+ UnicodeData.txt at all but some of these were used in the original
+ translit_combining file in glibc and they seemed to make sense.
+ I want to keep the update of translit_combining close to the
+ spirit of the original file, therefore I added these special
+ decomposition rules here.
+ '''
+ special_decompose_dict = {
+ # Ø U+00D8 is already handled in translit_neutral. But
+ # translit_combining is usually included after translit_neutral
+ # and Ǿ U+01FE LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
+ # has a canonical decomposition to Ø U+00D8 and we want to
+ # further decompose this to U+004F.
+ (0x00D8,): [0x004F], # Ø → O
+ # ø U+00F8 is already handled in translit_neutral. But
+ # translit_combining is usually included after translit_neutral
+ # and ǿ U+01FF LATIN SMALL LETTER O WITH STROKE AND ACUTE
+ # has a canonical decomposition to ø U+00F8 and we want to
+ # further decompose this to U+006F.
+ (0x00F8,): [0x006F], # ø → o
+ # æ U+00E6 is already in translit_compat because ligatures
+ # are handled in translit_compat. But ǣ U+01E3 has a
+ # canonical decomposition to U+00E6, U+0304 and we want to
+ # further decompose this to “ae”.
+ (0x00E6,): [0x0061, 0x0065], # æ → ae
+ # Æ U+00C6 is already in translit_compat because ligatures
+ # are handled in translit_compat. But Ǣ U+01E2 has a
+ # canonical decomposition to U+00C6, U+0304 and we want to
+ # further decompose this to “AE”
+ (0x00C6,): [0x0041, 0x0045], # Æ → AE
+ # U+05F2 HEBREW LIGATURE YIDDISH DOUBLE YOD is already in
+ # translit_compat because ligatures are handled in translit_compat.
+ # But U+FB1F has a canonical decomposition to U+05F2 and
+ # we want to further decompose this to U+05D9, U+05D9.
+ (0x05F2,): [0x05D9, 0x05D9], # ײ → יי
+ # 0x2002 has a <compat> decomposition to 0x0020 in UnicodeData.txt
+ # But U+2000 EN QUAD has a canonical decomposition U+2002
+ # and we want to further decompose this to U+0020.
+ (0x2002,): [0x0020], # EN SPACE → SPACE
+ # 0x2003 has a <compat> decomposition to 0x0020 in UnicodeData.txt
+ # But U+2001 EM QUAD has a canonical decomposition to U+2003
+ # and we want to further decompose this to U+0020.
+ (0x2003,): [0x0020], # EM SPACE → SPACE
+ # U+2260 ≠ has the canonical decomposition U+003D U+0338
+ # (= followed by ̸). After stripping the combining characters,
+ # the result is only = which reverses the meaning.
+ # Therefore, we add a special rules here for such mathematical
+ # negations:
+ (0x21AE,): [0x0021, 0x003C, 0x002D, 0x003E], # ↮ → !<->
+ (0x21CD,): [0x0021, 0x003C, 0x003D], # ⇍ → !<=
+ (0x21CE,): [0x0021, 0x003C, 0x003D, 0x003E], # ⇎ → !<=>
+ (0x21CF,): [0x0021, 0x003D, 0x003E], # ⇏ → !=>
+ (0x2204,): [0x0021, 0x2203], # ∄ → !∃
+ (0x2209,): [0x0021, 0x2208], # ∉ → !∈
+ (0x220C,): [0x0021, 0x220B], # ∌ → !∋
+ (0x2224,): [0x0021, 0x2223], # ∤ → !∣
+ (0x2226,): [0x0021, 0x2225], # ∦ → !∥
+ (0x2241,): [0x0021, 0x007E], # ≁ → !~
+ (0x2244,): [0x0021, 0x007E, 0x002D], # ≄ → !~-
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+ (0x2247,): [0x0021, 0x007E, 0x003D], # ≇ → !~=
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+ (0x2249,): [0x0021, 0x007E, 0x007E], # ≉ → !~~
+ (0x2260,): [0x0021, 0x003D], # ≠ → !=
+ (0x2262,): [0x0021, 0x003D, 0x003D], # ≢ → !==
+ (0x226D,): [0x0021, 0x224D], # ≭ → !≍
+ (0x226E,): [0x0021, 0x003C], # ≮ → !<
+ (0x226F,): [0x0021, 0x003E], # ≯ → !>
+ (0x2270,): [0x0021, 0x003C, 0x003D], # ≰ → !<=
+ (0x2271,): [0x0021, 0x003E, 0x003D], # ≱ → !>=
+ (0x2274,): [0x0021, 0x003C, 0x007E], # ≴ → !<~
+ (0x2275,): [0x0021, 0x003E, 0x007E], # ≵ → !>~
+ (0x2278,): [0x0021, 0x003C, 0x003E], # ≸ → !<>
+ (0x2279,): [0x0021, 0x003E, 0x003C], # ≹ → !><
+ (0x2280,): [0x0021, 0x227A], # ⊀ → !≺
+ (0x2281,): [0x0021, 0x227B], # ⊁ → !≻
+ (0x2284,): [0x0021, 0x2282], # ⊄ → !⊂
+ (0x2285,): [0x0021, 0x2283], # ⊅ → !⊃
+ (0x2288,): [0x0021, 0x2282, 0x003D], # ⊈ → !⊂=
+ (0x2289,): [0x0021, 0x2283, 0x003D], # ⊉ → !⊃=
+ (0x22AC,): [0x0021, 0x22A2], # ⊬ → !⊢
+ (0x22AD,): [0x0021, 0x22A8], # ⊭ → !⊨
+ (0x22AE,): [0x0021, 0x22A9], # ⊮ → !⊩
+ (0x22AF,): [0x0021, 0x22AB], # ⊯ → !⊫
+ (0x22E0,): [0x0021, 0x227C], # ⋠ → !≼
+ (0x22E1,): [0x0021, 0x227D], # ⋡ → !≽
+ (0x22E2,): [0x0021, 0x2291], # ⋢ → !⊑
+ (0x22E3,): [0x0021, 0x2292], # ⋣ → !⊒
+ (0x22EA,): [0x0021, 0x22B2], # ⋪ → !⊲
+ (0x22EB,): [0x0021, 0x22B3], # ⋫ → !⊳
+ (0x22EC,): [0x0021, 0x22B4], # ⋬ → !⊴
+ (0x22ED,): [0x0021, 0x22B5], # ⋭ → !⊵
+ (0x2ADC,): [0x0021, 0x2ADD], # ⫝̸ → !⫝
+ # Special rule for 〈 U+3008 is added
+ # because 〉 U+2329 has the canonical decomposition U+3008
+ # and we want to further decompose this to > U+003C.
+ (0x3008,): [0x003C], # 〈 → <
+ # Special rule for 〉 U+3009 is added
+ # because 〉 U+232A has the canonical decomposition U+3009
+ # and we want to further decompose this to < U+003E.
+ (0x3009,): [0x003E], # 〉→ >
+ }
+ if tuple(code_point_list) in special_decompose_dict:
+ return special_decompose_dict[tuple(code_point_list)]
+ else:
+ return code_point_list
+
+def output_combining_remove(translit_file):
+ '''Write the section of the translit_combining file where combining
+ characters are replaced by empty strings.
+ '''
+ translit_file.write('\n')
+ for code_point in sorted(unicode_utils.UNICODE_ATTRIBUTES):
+ name = unicode_utils.UNICODE_ATTRIBUTES[code_point]['name']
+ if is_combining_remove(code_point):
+ translit_file.write('% {:s}\n'.format(name))
+ translit_file.write('{:s} ""\n'.format(
+ unicode_utils.ucs_symbol(code_point)))
+ translit_file.write('\n')
+
+def output_decompositions(translit_file):
+ '''Write the section of the translit_combining file where characters
+ characters are decomposed and combining characters stripped from
+ the decompositions.
+ '''
+ for code_point in sorted(unicode_utils.UNICODE_ATTRIBUTES):
+ if special_decompose([code_point]) != [code_point]:
+ decomposed_code_points = [special_decompose([code_point])]
+ else:
+ decomposed_code_points = [canonical_decompose(code_point)]
+ if decomposed_code_points[0]:
+ while True:
+ special_decomposed_code_points = special_decompose(
+ decomposed_code_points[-1])
+ if (special_decomposed_code_points
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+ != decomposed_code_points[-1]):
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+ decomposed_code_points.append(
+ special_decomposed_code_points)
+ continue
+ special_decomposed_code_points = []
+ for decomposed_code_point in decomposed_code_points[-1]:
+ special_decomposed_code_points += special_decompose(
+ [decomposed_code_point])
+ if (special_decomposed_code_points
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+ == decomposed_code_points[-1]):
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+ break
+ decomposed_code_points.append(
+ special_decomposed_code_points)
+ for index in range(0, len(decomposed_code_points)):
+ decomposed_code_points[index] = [
+ x for x in decomposed_code_points[index]
+ if not is_combining_remove(x)]
+ if decomposed_code_points[0]:
+ translit_file.write('% {:s}\n'.format(
+ unicode_utils.UNICODE_ATTRIBUTES[code_point]['name']))
+ translit_file.write('{:s} '.format(
+ unicode_utils.ucs_symbol(code_point)))
+ for index in range(0, len(decomposed_code_points)):
+ if index > 0:
+ translit_file.write(';')
+ if len(decomposed_code_points[index]) > 1:
+ translit_file.write('"')
+ for decomposed_code_point in decomposed_code_points[index]:
+ translit_file.write('{:s}'.format(
+ unicode_utils.ucs_symbol(decomposed_code_point)))
+ if len(decomposed_code_points[index]) > 1:
+ translit_file.write('"')
+ translit_file.write('\n')
+ translit_file.write('\n')
+
+def output_transliteration(translit_file):
+ '''Write the new transliteration to the output file'''
+ output_combining_remove(translit_file)
+ output_decompositions(translit_file)
+
+if __name__ == "__main__":
+ PARSER = argparse.ArgumentParser(
+ description='''
+ Generate a translit_combining file from UnicodeData.txt.
+ ''')
+ PARSER.add_argument(
+ '-u', '--unicode_data_file',
+ nargs='?',
+ type=str,
+ default='UnicodeData.txt',
+ help=('The UnicodeData.txt file to read, '
+ + 'default: %(default)s'))
+ PARSER.add_argument(
+ '-i', '--input_file',
+ nargs='?',
+ type=str,
+ help=''' The original glibc/localedata/locales/translit_combining
+ file.''')
+ PARSER.add_argument(
+ '-o', '--output_file',
+ nargs='?',
+ type=str,
+ default='translit_combining.new',
+ help='''The new translit_combining file, default: %(default)s. If the
+ original glibc/localedata/locales/translit_combining file has
+ been given as an option, the header up to the
+ “translit_start” line and the tail from the “translit_end”
+ line to the end of the file will be copied unchanged into the
+ output file. ''')
+ PARSER.add_argument(
+ '--unicode_version',
+ nargs='?',
+ required=True,
+ type=str,
+ help='The Unicode version of the input files used.')
+ ARGS = PARSER.parse_args()
+
+ unicode_utils.fill_attributes(ARGS.unicode_data_file)
+ HEAD = TAIL = ''
+ if ARGS.input_file:
+ (HEAD, TAIL) = read_input_file(ARGS.input_file)
+ with open(ARGS.output_file, mode='w') as TRANSLIT_FILE:
+ output_head(TRANSLIT_FILE, ARGS.unicode_version, head=HEAD)
+ output_transliteration(TRANSLIT_FILE)
+ output_tail(TRANSLIT_FILE, tail=TAIL)
diff --git a/localedata/unicode-gen/gen_translit_compat.py b/localedata/unicode-gen/gen_translit_compat.py
new file mode 100755
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
index 0000000..0e824a8
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
--- /dev/null
+++ b/localedata/unicode-gen/gen_translit_compat.py
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
@@ -0,0 +1,326 @@
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+#!/usr/bin/python3
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+# -*- coding: utf-8 -*-
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+#
+# Generate a translit_compat file from a UnicodeData file.
+# Copyright (C) 2015 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+#
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <http://www.gnu.org/licenses/>.
+
+'''
+Generate a translit_compat file from UnicodeData.txt
+
+To see how this script is used, call it with the “-h” option:
+
+ $ ./gen_translit_compat -h
+ … prints usage message …
+'''
+
+import argparse
+import time
+import unicode_utils
+
+def read_input_file(filename):
+ '''Reads the original glibc translit_compat file to get the
+ original head and tail.
+
+ We want to replace only the part of the file between
+ “translit_start” and “translit_end”
+ '''
+ head = tail = ''
+ with open(filename, mode='r') as translit_file:
+ for line in translit_file:
+ head = head + line
+ if line.startswith('translit_start'):
+ break
+ for line in translit_file:
+ if line.startswith('translit_end'):
+ tail = line
+ break
+ for line in translit_file:
+ tail = tail + line
+ return (head, tail)
+
+def output_head(translit_file, unicode_version, head=''):
+ '''Write the header of the output file, i.e. the part of the file
+ before the “translit_start” line.
+ '''
+ if ARGS.input_file and head:
+ translit_file.write(head)
+ else:
+ translit_file.write('escape_char /\n')
+ translit_file.write('comment_char %\n')
+ translit_file.write('\n')
+ translit_file.write('% Transliterations of compatibility characters ')
+ translit_file.write('and ligatures.\n')
+ translit_file.write('% Generated automatically from UnicodeData.txt '
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+ + 'by gen_translit_compat.py '
+ + 'on {:s} '.format(time.strftime('%Y-%m-%d'))
+ + 'for Unicode {:s}.\n'.format(unicode_version))
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+ translit_file.write('\n')
+ translit_file.write('LC_CTYPE\n')
+ translit_file.write('\n')
+ translit_file.write('translit_start\n')
+
+def output_tail(translit_file, tail=''):
+ '''Write the tail of the output file'''
+ if ARGS.input_file and tail:
+ translit_file.write(tail)
+ else:
+ translit_file.write('translit_end\n')
+ translit_file.write('\n')
+ translit_file.write('END LC_CTYPE\n')
+
+def compatibility_decompose(code_point):
+ '''http://www.unicode.org/reports/tr44/#Character_Decomposition_Mappings
+
+ “The compatibility decomposition is formed by recursively applying
+ the canonical and compatibility mappings, then applying the
+ Canonical Ordering Algorithm.”
+
+ We don’t do the canonical decomposition here because this is
+ done in gen_translit_combining.py to generate translit_combining.
+
+ And we ignore some of the possible compatibility formatting tags
+ here. Some of them are used in other translit_* files, not
+ translit_compat:
+
+ <font>: translit_font
+ <circle>: translit_circle
+ <wide>: translit_wide
+ <narrow>: translit_narrow
+ <square>: translit_cjk_compat
+ <fraction>: translit_fraction
+
+ And we ignore
+
+ <noBreak>, <initial>, <medial>, <final>, <isolated>
+
+ because they seem to be not useful for transliteration.
+ '''
+ decomposition = unicode_utils.UNICODE_ATTRIBUTES[
+ code_point]['decomposition']
+ compatibility_tags = (
+ '<compat>', '<super>', '<sub>', '<vertical>')
+ for compatibility_tag in compatibility_tags:
+ if decomposition.startswith(compatibility_tag):
+ decomposition = decomposition[len(compatibility_tag)+1:]
+ decomposed_code_points = [int(x, 16)
+ for x in decomposition.split(' ')]
+ if (len(decomposed_code_points) > 1
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+ and decomposed_code_points[0] == 0x0020
+ and decomposed_code_points[1] >= 0x0300
+ and decomposed_code_points[1] <= 0x03FF):
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+ # Decomposes into a space followed by a combining character.
+ # This is not useful fo transliteration.
+ return []
+ else:
+ return_value = []
+ for index in range(0, len(decomposed_code_points)):
+ cd_code_points = compatibility_decompose(
+ decomposed_code_points[index])
+ if cd_code_points:
+ return_value += cd_code_points
+ else:
+ return_value += [decomposed_code_points[index]]
+ return return_value
+ return []
+
+def special_decompose(code_point_list):
+ '''
+ Decompositions which are not in UnicodeData.txt at all but which
+ were used in the original translit_compat file in glibc and
+ which seem to make sense. I want to keep the update of
+ translit_compat close to the spirit of the original file,
+ therefore I added this special decomposition rules here.
+ '''
+ special_decompose_dict = {
+ (0x03BC,): [0x0075], # μ → u
+ (0x02BC,): [0x0027], # ʼ → '
+ }
+ if tuple(code_point_list) in special_decompose_dict:
+ return special_decompose_dict[tuple(code_point_list)]
+ else:
+ return code_point_list
+
+def special_ligature_decompose(code_point):
+ '''
+ Decompositions for ligatures which are not in UnicodeData.txt at
+ all but which were used in the original translit_compat file in
+ glibc and which seem to make sense. I want to keep the update of
+ translit_compat close to the spirit of the original file,
+ therefore I added these special ligature decomposition rules here.
+
+ '''
+ special_ligature_decompose_dict = {
+ 0x00E6: [0x0061, 0x0065], # æ → ae
+ 0x00C6: [0x0041, 0x0045], # Æ → AE
+ # These following 5 special ligature decompositions were
+ # in the original glibc/localedata/locales/translit_compat file
+ 0x0152: [0x004F, 0x0045], # Œ → OE
+ 0x0153: [0x006F, 0x0065], # œ → oe
+ 0x05F0: [0x05D5, 0x05D5], # װ → וו
+ 0x05F1: [0x05D5, 0x05D9], # ױ → וי
+ 0x05F2: [0x05D9, 0x05D9], # ײ → יי
+ # The following special ligature decompositions were
+ # not in the original glibc/localedata/locales/translit_compat file
+ # U+04A4 CYRILLIC CAPITAL LIGATURE EN GHE
+ # → U+041D CYRILLIC CAPITAL LETTER EN,
+ # U+0413 CYRILLIC CAPITAL LETTER GHE
+ 0x04A4: [0x041D, 0x0413], # Ҥ → НГ
+ # U+04A5 CYRILLIC SMALL LIGATURE EN GHE
+ # → U+043D CYRILLIC SMALL LETTER EN,
+ # U+0433 CYRILLIC SMALL LETTER GHE
+ 0x04A5: [0x043D, 0x0433], # ҥ → нг
+ # U+04B4 CYRILLIC CAPITAL LIGATURE TE TSE
+ # → U+0422 CYRILLIC CAPITAL LETTER TE,
+ # U+0426 CYRILLIC CAPITAL LETTER TSE
+ 0x04B4: [0x0422, 0x0426], # Ҵ → ТЦ
+ # U+04B5 CYRILLIC SMALL LIGATURE TE TSE
+ # → U+0442 CYRILLIC SMALL LETTER TE,
+ # U+0446 CYRILLIC SMALL LETTER TSE
+ 0x04B5: [0x0442, 0x0446], # ҵ → тц
+ # U+04d4 CYRILLIC CAPITAL LIGATURE A IE
+ # → U+0410 CYRILLIC CAPITAL LETTER A
+ # U+0415;CYRILLIC CAPITAL LETTER IE
+ 0x04D4: [0x0410, 0x0415], # Ӕ → АЕ
+ # U+04D5 CYRILLIC SMALL LIGATURE A IE
+ # → U+0430 CYRILLIC SMALL LETTER A,
+ # U+0435 CYRILLIC SMALL LETTER IE
+ 0x04D5: [0x0430, 0x0435], # ӕ → ае
+ # I am not sure what to do with the following ligatures
+ # maybe it makes no sense to decompose them:
+ # U+0616 ARABIC SMALL HIGH LIGATURE ALEF WITH LAM WITH YEH
+ # U+06d6 ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA
+ # U+06d7 ARABIC SMALL HIGH LIGATURE QAF WITH LAM WITH ALEF MAKSURA
+ # U+fdfd ARABIC LIGATURE BISMILLAH AR-RAHMAN AR-RAHEEM
+ # U+fe20 COMBINING LIGATURE LEFT HALF
+ # U+fe21 COMBINING LIGATURE RIGHT HALF
+ # U+fe27 COMBINING LIGATURE LEFT HALF BELOW
+ # U+fe28 COMBINING LIGATURE RIGHT HALF BELOW
+ # U+11176 MAHAJANI LIGATURE SHRI
+ # U+1f670 SCRIPT LIGATURE ET ORNAMENT
+ # U+1f671 HEAVY SCRIPT LIGATURE ET ORNAMENT
+ # U+1f672 LIGATURE OPEN ET ORNAMENT
+ # U+1f673 HEAVY LIGATURE OPEN ET ORNAMENT
+ }
+ if code_point in special_ligature_decompose_dict:
+ return special_ligature_decompose_dict[code_point]
+ else:
+ return [code_point]
+
+def output_transliteration(translit_file):
+ '''Write the new transliteration to the output file'''
+ translit_file.write('\n')
+ for code_point in sorted(unicode_utils.UNICODE_ATTRIBUTES):
+ name = unicode_utils.UNICODE_ATTRIBUTES[code_point]['name']
+ decomposed_code_points = [compatibility_decompose(code_point)]
+ if not decomposed_code_points[0]:
+ if special_decompose([code_point]) != [code_point]:
+ decomposed_code_points[0] = special_decompose([code_point])
+ else:
+ special_decomposed_code_points = []
+ while True:
+ special_decomposed_code_points = special_decompose(
+ decomposed_code_points[-1])
+ if (special_decomposed_code_points
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+ != decomposed_code_points[-1]):
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+ decomposed_code_points.append(
+ special_decomposed_code_points)
+ continue
+ special_decomposed_code_points = []
+ for decomposed_code_point in decomposed_code_points[-1]:
+ special_decomposed_code_points += special_decompose(
+ [decomposed_code_point])
+ if (special_decomposed_code_points
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+ == decomposed_code_points[-1]):
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+ break
+ decomposed_code_points.append(
+ special_decomposed_code_points)
+ if decomposed_code_points[0]:
+ translit_file.write('% {:s}\n'.format(name))
+ translit_file.write('{:s} '.format(
+ unicode_utils.ucs_symbol(code_point)))
+ for index in range(0, len(decomposed_code_points)):
+ if index > 0:
+ translit_file.write(';')
+ translit_file.write('"')
+ for decomposed_code_point in decomposed_code_points[index]:
+ translit_file.write('{:s}'.format(
+ unicode_utils.ucs_symbol(decomposed_code_point)))
+ translit_file.write('"')
+ translit_file.write('\n')
+ elif 'LIGATURE' in name and 'ARABIC' not in name:
+ decomposed_code_points = special_ligature_decompose(code_point)
+ if decomposed_code_points[0] != code_point:
+ translit_file.write('% {:s}\n'.format(name))
+ translit_file.write('{:s} '.format(
+ unicode_utils.ucs_symbol(code_point)))
+ translit_file.write('"')
+ for decomposed_code_point in decomposed_code_points:
+ translit_file.write('{:s}'.format(
+ unicode_utils.ucs_symbol(decomposed_code_point)))
+ translit_file.write('"')
+ translit_file.write('\n')
+ else:
+ print('Warning: unhandled ligature: {:x} {:s}'.format(
+ code_point, name))
+ translit_file.write('\n')
+
+if __name__ == "__main__":
+ PARSER = argparse.ArgumentParser(
+ description='''
+ Generate a translit_compat file from UnicodeData.txt.
+ ''')
+ PARSER.add_argument(
+ '-u', '--unicode_data_file',
+ nargs='?',
+ type=str,
+ default='UnicodeData.txt',
+ help=('The UnicodeData.txt file to read, '
+ + 'default: %(default)s'))
+ PARSER.add_argument(
+ '-i', '--input_file',
+ nargs='?',
+ type=str,
+ help=''' The original glibc/localedata/locales/translit_compat
+ file.''')
+ PARSER.add_argument(
+ '-o', '--output_file',
+ nargs='?',
+ type=str,
+ default='translit_compat.new',
+ help='''The new translit_compat file, default: %(default)s. If the
+ original glibc/localedata/locales/translit_compat file has
+ been given as an option, the header up to the
+ “translit_start” line and the tail from the “translit_end”
+ line to the end of the file will be copied unchanged into the
+ output file. ''')
+ PARSER.add_argument(
+ '--unicode_version',
+ nargs='?',
+ required=True,
+ type=str,
+ help='The Unicode version of the input files used.')
+ ARGS = PARSER.parse_args()
+
+ unicode_utils.fill_attributes(ARGS.unicode_data_file)
+ HEAD = TAIL = ''
+ if ARGS.input_file:
+ (HEAD, TAIL) = read_input_file(ARGS.input_file)
+ with open(ARGS.output_file, mode='w') as TRANSLIT_FILE:
+ output_head(TRANSLIT_FILE, ARGS.unicode_version, head=HEAD)
+ output_transliteration(TRANSLIT_FILE)
+ output_tail(TRANSLIT_FILE, tail=TAIL)
diff --git a/localedata/unicode-gen/gen_translit_font.py b/localedata/unicode-gen/gen_translit_font.py
new file mode 100755
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
index 0000000..0723622
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
--- /dev/null
+++ b/localedata/unicode-gen/gen_translit_font.py
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
@@ -0,0 +1,156 @@
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+#!/usr/bin/python3
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+# -*- coding: utf-8 -*-
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+#
+# Generate a translit_font file from a UnicodeData file.
+# Copyright (C) 2015 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+#
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <http://www.gnu.org/licenses/>.
+
+'''
+Generate a translit_font file from UnicodeData.txt
+
+To see how this script is used, call it with the “-h” option:
+
+ $ ./gen_translit_font -h
+ … prints usage message …
+'''
+
+import argparse
+import time
+import unicode_utils
+
+def read_input_file(filename):
+ '''Reads the original glibc translit_font file to get the
+ original head and tail.
+
+ We want to replace only the part of the file between
+ “translit_start” and “translit_end”
+ '''
+ head = tail = ''
+ with open(filename, mode='r') as translit_file:
+ for line in translit_file:
+ head = head + line
+ if line.startswith('translit_start'):
+ break
+ for line in translit_file:
+ if line.startswith('translit_end'):
+ tail = line
+ break
+ for line in translit_file:
+ tail = tail + line
+ return (head, tail)
+
+def output_head(translit_file, unicode_version, head=''):
+ '''Write the header of the output file, i.e. the part of the file
+ before the “translit_start” line.
+ '''
+ if ARGS.input_file and head:
+ translit_file.write(head)
+ else:
+ translit_file.write('escape_char /\n')
+ translit_file.write('comment_char %\n')
+ translit_file.write('\n')
+ translit_file.write('% Transliterations of font equivalents.\n')
+ translit_file.write('% Generated automatically from UnicodeData.txt '
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+ + 'by gen_translit_font.py '
+ + 'on {:s} '.format(time.strftime('%Y-%m-%d'))
+ + 'for Unicode {:s}.\n'.format(unicode_version))
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+ translit_file.write('\n')
+ translit_file.write('LC_CTYPE\n')
+ translit_file.write('\n')
+ translit_file.write('translit_start\n')
+
+def output_tail(translit_file, tail=''):
+ '''Write the tail of the output file'''
+ if ARGS.input_file and tail:
+ translit_file.write(tail)
+ else:
+ translit_file.write('translit_end\n')
+ translit_file.write('\n')
+ translit_file.write('END LC_CTYPE\n')
+
+def output_transliteration(translit_file):
+ '''Write the new transliteration to the output file'''
+ translit_file.write('\n')
+ for code_point in sorted(unicode_utils.UNICODE_ATTRIBUTES):
+ name = unicode_utils.UNICODE_ATTRIBUTES[code_point]['name']
+ decomposition = unicode_utils.UNICODE_ATTRIBUTES[
+ code_point]['decomposition']
+ if decomposition.startswith('<font>'):
+ decomposition = decomposition[7:]
+ decomposed_code_points = [[int(x, 16)
+ for x in decomposition.split(' ')]]
+ if decomposed_code_points[0]:
+ translit_file.write('{:s} '.format(
+ unicode_utils.ucs_symbol(code_point)))
+ for index in range(0, len(decomposed_code_points)):
+ if index > 0:
+ translit_file.write(';')
+ if len(decomposed_code_points[index]) > 1:
+ translit_file.write('"')
+ for decomposed_code_point in decomposed_code_points[index]:
+ translit_file.write('{:s}'.format(
+ unicode_utils.ucs_symbol(decomposed_code_point)))
+ if len(decomposed_code_points[index]) > 1:
+ translit_file.write('"')
+ translit_file.write(' % {:s}\n'.format(name))
+ translit_file.write('\n')
+
+if __name__ == "__main__":
+ PARSER = argparse.ArgumentParser(
+ description='''
+ Generate a translit_font file from UnicodeData.txt.
+ ''')
+ PARSER.add_argument(
+ '-u', '--unicode_data_file',
+ nargs='?',
+ type=str,
+ default='UnicodeData.txt',
+ help=('The UnicodeData.txt file to read, '
+ + 'default: %(default)s'))
+ PARSER.add_argument(
+ '-i', '--input_file',
+ nargs='?',
+ type=str,
+ help=''' The original glibc/localedata/locales/translit_font
+ file.''')
+ PARSER.add_argument(
+ '-o', '--output_file',
+ nargs='?',
+ type=str,
+ default='translit_font.new',
+ help='''The new translit_font file, default: %(default)s. If the
+ original glibc/localedata/locales/translit_font file has
+ been given as an option, the header up to the
+ “translit_start” line and the tail from the “translit_end”
+ line to the end of the file will be copied unchanged into the
+ output file. ''')
+ PARSER.add_argument(
+ '--unicode_version',
+ nargs='?',
+ required=True,
+ type=str,
+ help='The Unicode version of the input files used.')
+ ARGS = PARSER.parse_args()
+
+ unicode_utils.fill_attributes(ARGS.unicode_data_file)
+ HEAD = TAIL = ''
+ if ARGS.input_file:
+ (HEAD, TAIL) = read_input_file(ARGS.input_file)
+ with open(ARGS.output_file, mode='w') as TRANSLIT_FILE:
+ output_head(TRANSLIT_FILE, ARGS.unicode_version, head=HEAD)
+ output_transliteration(TRANSLIT_FILE)
+ output_tail(TRANSLIT_FILE, tail=TAIL)
diff --git a/localedata/unicode-gen/gen_translit_fraction.py b/localedata/unicode-gen/gen_translit_fraction.py
new file mode 100755
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
index 0000000..5bf63ea
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
--- /dev/null
+++ b/localedata/unicode-gen/gen_translit_fraction.py
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
@@ -0,0 +1,197 @@
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+#!/usr/bin/python3
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+# -*- coding: utf-8 -*-
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+#
+# Generate a translit_fraction file from a UnicodeData file.
+# Copyright (C) 2015 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+#
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <http://www.gnu.org/licenses/>.
+
+'''
+Generate a translit_fraction file from UnicodeData.txt
+
+To see how this script is used, call it with the “-h” option:
+
+ $ ./gen_translit_fraction -h
+ … prints usage message …
+'''
+
+import argparse
+import time
+import unicode_utils
+
+def read_input_file(filename):
+ '''Reads the original glibc translit_fraction file to get the
+ original head and tail.
+
+ We want to replace only the part of the file between
+ “translit_start” and “translit_end”
+ '''
+ head = tail = ''
+ with open(filename, mode='r') as translit_file:
+ for line in translit_file:
+ head = head + line
+ if line.startswith('translit_start'):
+ break
+ for line in translit_file:
+ if line.startswith('translit_end'):
+ tail = line
+ break
+ for line in translit_file:
+ tail = tail + line
+ return (head, tail)
+
+def output_head(translit_file, unicode_version, head=''):
+ '''Write the header of the output file, i.e. the part of the file
+ before the “translit_start” line.
+ '''
+ if ARGS.input_file and head:
+ translit_file.write(head)
+ else:
+ translit_file.write('escape_char /\n')
+ translit_file.write('comment_char %\n')
+ translit_file.write('\n')
+ translit_file.write('% Transliterations of fractions.\n')
+ translit_file.write('% Generated automatically from UnicodeData.txt '
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+ + 'by gen_translit_fraction.py '
+ + 'on {:s} '.format(time.strftime('%Y-%m-%d'))
+ + 'for Unicode {:s}.\n'.format(unicode_version))
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+ translit_file.write('% The replacements have been surrounded ')
+ translit_file.write('with spaces, because fractions are\n')
+ translit_file.write('% often preceded by a decimal number and ')
+ translit_file.write('followed by a unit or a math symbol.\n')
+ translit_file.write('\n')
+ translit_file.write('LC_CTYPE\n')
+ translit_file.write('\n')
+ translit_file.write('translit_start\n')
+
+def output_tail(translit_file, tail=''):
+ '''Write the tail of the output file'''
+ if ARGS.input_file and tail:
+ translit_file.write(tail)
+ else:
+ translit_file.write('translit_end\n')
+ translit_file.write('\n')
+ translit_file.write('END LC_CTYPE\n')
+
+def special_decompose(code_point_list):
+ '''
+ Decompositions which are not in UnicodeData.txt at all but which
+ were used in the original translit_fraction file in glibc and
+ which seem to make sense. I want to keep the update of
+ translit_fraction close to the spirit of the original file,
+ therefore I added this special decomposition rules here.
+ '''
+ special_decompose_dict = {
+ (0x2044,): [0x002F], # ⁄ → /
+ }
+ if tuple(code_point_list) in special_decompose_dict:
+ return special_decompose_dict[tuple(code_point_list)]
+ else:
+ return code_point_list
+
+def output_transliteration(translit_file):
+ '''Write the new transliteration to the output file'''
+ translit_file.write('\n')
+ for code_point in sorted(unicode_utils.UNICODE_ATTRIBUTES):
+ name = unicode_utils.UNICODE_ATTRIBUTES[code_point]['name']
+ decomposition = unicode_utils.UNICODE_ATTRIBUTES[
+ code_point]['decomposition']
+ if decomposition.startswith('<fraction>'):
+ decomposition = decomposition[11:]
+ decomposed_code_points = [[int(x, 16)
+ for x in decomposition.split(' ')]]
+ if decomposed_code_points[0]:
+ decomposed_code_points[0] = [0x0020] \
+ + decomposed_code_points[0] \
+ + [0x0020]
+ while True:
+ special_decomposed_code_points = special_decompose(
+ decomposed_code_points[-1])
+ if (special_decomposed_code_points
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+ != decomposed_code_points[-1]):
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+ decomposed_code_points.append(
+ special_decomposed_code_points)
+ continue
+ special_decomposed_code_points = []
+ for decomposed_code_point in decomposed_code_points[-1]:
+ special_decomposed_code_points += special_decompose(
+ [decomposed_code_point])
+ if (special_decomposed_code_points
Some more additions to the translit_neutral file by Marko Myllynen (and pylint warning fixes to the gen_translit_* scripts by Pravin Satpute)
2015-07-23 13:44:24 +00:00
+ == decomposed_code_points[-1]):
Unicode 8.0.0 updates (including the transliteration files) - Resolves: rhbz#1238412
2015-06-10 13:11:53 +00:00
+ break
+ decomposed_code_points.append(
+ special_decomposed_code_points)
+ translit_file.write('% {:s}\n'.format(name))
+ translit_file.write('{:s} '.format(
+ unicode_utils.ucs_symbol(code_point)))
+ for index in range(0, len(decomposed_code_points)):
+ if index > 0:
+ translit_file.write(';')
+ if len(decomposed_code_points[index]) > 1:
+ translit_file.write('"')
+ for decomposed_code_point in decomposed_code_points[index]:
+ translit_file.write('{:s}'.format(
+ unicode_utils.ucs_symbol(decomposed_code_point)))
+ if len(decomposed_code_points[index]) > 1:
+ translit_file.write('"')
+ translit_file.write('\n')
+ translit_file.write('\n')
+
+if __name__ == "__main__":
+ PARSER = argparse.ArgumentParser(
+ description='''
+ Generate a translit_cjk_compat file from UnicodeData.txt.
+ ''')
+ PARSER.add_argument(
+ '-u', '--unicode_data_file',
+ nargs='?',
+ type=str,
+ default='UnicodeData.txt',
+ help=('The UnicodeData.txt file to read, '
+ + 'default: %(default)s'))
+ PARSER.add_argument(
+ '-i', '--input_file',
+ nargs='?',
+ type=str,
+ help=''' The original glibc/localedata/locales/translit_fraction
+ file.''')
+ PARSER.add_argument(
+ '-o', '--output_file',
+ nargs='?',
+ type=str,
+ default='translit_fraction.new',
+ help='''The new translit_fraction file, default: %(default)s. If the
+ original glibc/localedata/locales/translit_fraction file has
+ been given as an option, the header up to the
+ “translit_start” line and the tail from the “translit_end”
+ line to the end of the file will be copied unchanged into the
+ output file. ''')
+ PARSER.add_argument(
+ '--unicode_version',
+ nargs='?',
+ required=True,
+ type=str,
+ help='The Unicode version of the input files used.')
+ ARGS = PARSER.parse_args()
+
+ unicode_utils.fill_attributes(ARGS.unicode_data_file)
+ HEAD = TAIL = ''
+ if ARGS.input_file:
+ (HEAD, TAIL) = read_input_file(ARGS.input_file)
+ with open(ARGS.output_file, mode='w') as TRANSLIT_FILE:
+ output_head(TRANSLIT_FILE, ARGS.unicode_version, head=HEAD)
+ output_transliteration(TRANSLIT_FILE)
+ output_tail(TRANSLIT_FILE, tail=TAIL)
diff --git a/localedata/unicode-gen/gen_unicode_ctype.py b/localedata/unicode-gen/gen_unicode_ctype.py
index 0c74f2a..0f064f5 100755
--- a/localedata/unicode-gen/gen_unicode_ctype.py
+++ b/localedata/unicode-gen/gen_unicode_ctype.py
@@ -30,345 +30,9 @@ To see how this script is used, call it with the “-h” option:
'''
import argparse
-import sys
import time
import re
-
-# Dictionary holding the entire contents of the UnicodeData.txt file
-#
-# Contents of this dictionary look like this:
-#
-# {0: {'category': 'Cc',
-# 'title': None,
-# 'digit': '',
-# 'name': '<control>',
-# 'bidi': 'BN',
-# 'combining': '0',
-# 'comment': '',
-# 'oldname': 'NULL',
-# 'decomposition': '',
-# 'upper': None,
-# 'mirrored': 'N',
-# 'lower': None,
-# 'decdigit': '',
-# 'numeric': ''},
-# …
-# }
-UNICODE_ATTRIBUTES = {}
-
-# Dictionary holding the entire contents of the DerivedCoreProperties.txt file
-#
-# Contents of this dictionary look like this:
-#
-# {917504: ['Default_Ignorable_Code_Point'],
-# 917505: ['Case_Ignorable', 'Default_Ignorable_Code_Point'],
-# …
-# }
-DERIVED_CORE_PROPERTIES = {}
-
-def fill_attribute(code_point, fields):
- '''Stores in UNICODE_ATTRIBUTES[code_point] the values from the fields.
-
- One entry in the UNICODE_ATTRIBUTES dictionary represents one line
- in the UnicodeData.txt file.
-
- '''
- UNICODE_ATTRIBUTES[code_point] = {
- 'name': fields[1], # Character name
- 'category': fields[2], # General category
- 'combining': fields[3], # Canonical combining classes
- 'bidi': fields[4], # Bidirectional category
- 'decomposition': fields[5], # Character decomposition mapping
- 'decdigit': fields[6], # Decimal digit value
- 'digit': fields[7], # Digit value
- 'numeric': fields[8], # Numeric value
- 'mirrored': fields[9], # mirrored
- 'oldname': fields[10], # Old Unicode 1.0 name
- 'comment': fields[11], # comment
- # Uppercase mapping
- 'upper': int(fields[12], 16) if fields[12] else None,
- # Lowercase mapping
- 'lower': int(fields[13], 16) if fields[13] else None,
- # Titlecase mapping
- 'title': int(fields[14], 16) if fields[14] else None,
- }
-
-def fill_attributes(filename):
- '''Stores the entire contents of the UnicodeData.txt file
- in the UNICODE_ATTRIBUTES dictionary.
-
- A typical line for a single code point in UnicodeData.txt looks
- like this:
-
- 0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061;
-
- Code point ranges are indicated by pairs of lines like this:
-
- 4E00;<CJK Ideograph, First>;Lo;0;L;;;;;N;;;;;
- 9FCC;<CJK Ideograph, Last>;Lo;0;L;;;;;N;;;;;
- '''
- with open(filename, mode='r') as unicode_data_file:
- fields_start = []
- for line in unicode_data_file:
- fields = line.strip().split(';')
- if len(fields) != 15:
- sys.stderr.write(
- 'short line in file "%(f)s": %(l)s\n' %{
- 'f': filename, 'l': line})
- exit(1)
- if fields[2] == 'Cs':
- # Surrogates are UTF-16 artefacts,
- # not real characters. Ignore them.
- fields_start = []
- continue
- if fields[1].endswith(', First>'):
- fields_start = fields
- fields_start[1] = fields_start[1].split(',')[0][1:]
- continue
- if fields[1].endswith(', Last>'):
- fields[1] = fields[1].split(',')[0][1:]
- if fields[1:] != fields_start[1:]:
- sys.stderr.write(
- 'broken code point range in file "%(f)s": %(l)s\n' %{
- 'f': filename, 'l': line})
- exit(1)
- for code_point in range(
- int(fields_start[0], 16),
- int(fields[0], 16)+1):
- fill_attribute(code_point, fields)
- fields_start = []
- continue
- fill_attribute(int(fields[0], 16), fields)
- fields_start = []
-
-def fill_derived_core_properties(filename):
- '''Stores the entire contents of the DerivedCoreProperties.txt file
- in the DERIVED_CORE_PROPERTIES dictionary.
-
- Lines in DerivedCoreProperties.txt are either a code point range like
- this:
-
- 0061..007A ; Lowercase # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z
-
- or a single code point like this:
-
- 00AA ; Lowercase # Lo FEMININE ORDINAL INDICATOR
-
- '''
- with open(filename, mode='r') as derived_core_properties_file:
- for line in derived_core_properties_file:
- match = re.match(
- r'^(?P<codepoint1>[0-9A-F]{4,6})'
- + r'(?:\.\.(?P<codepoint2>[0-9A-F]{4,6}))?'
- + r'\s*;\s*(?P<property>[a-zA-Z_]+)',
- line)
- if not match:
- continue
- start = match.group('codepoint1')
- end = match.group('codepoint2')
- if not end:
- end = start
- for code_point in range(int(start, 16), int(end, 16)+1):
- prop = match.group('property')
- if code_point in DERIVED_CORE_PROPERTIES:
- DERIVED_CORE_PROPERTIES[code_point].append(prop)
- else:
- DERIVED_CORE_PROPERTIES[code_point] = [prop]
-
-def to_upper(code_point):
- '''Returns the code point of the uppercase version
- of the given code point'''
- if (UNICODE_ATTRIBUTES[code_point]['name']
- and UNICODE_ATTRIBUTES[code_point]['upper']):
- return UNICODE_ATTRIBUTES[code_point]['upper']
- else:
- return code_point
-
-def to_lower(code_point):
- '''Returns the code point of the lowercase version
- of the given code point'''
- if (UNICODE_ATTRIBUTES[code_point]['name']
- and UNICODE_ATTRIBUTES[code_point]['lower']):
- return UNICODE_ATTRIBUTES[code_point]['lower']
- else:
- return code_point
-
-def to_title(code_point):
- '''Returns the code point of the titlecase version
- of the given code point'''
- if (UNICODE_ATTRIBUTES[code_point]['name']
- and UNICODE_ATTRIBUTES[code_point]['title']):
- return UNICODE_ATTRIBUTES[code_point]['title']
- else:
- return code_point
-
-def is_upper(code_point):
- '''Checks whether the character with this code point is uppercase'''
- return (to_lower(code_point) != code_point
- or (code_point in DERIVED_CORE_PROPERTIES
- and 'Uppercase' in DERIVED_CORE_PROPERTIES[code_point]))
-
-def is_lower(code_point):
- '''Checks whether the character with this code point is lowercase'''
- # Some characters are defined as “Lowercase” in
- # DerivedCoreProperties.txt but do not have a mapping to upper
- # case. For example, ꜰ U+A72F “LATIN LETTER SMALL CAPITAL F” is
- # one of these.
- return (to_upper(code_point) != code_point
- # <U00DF> is lowercase, but without simple to_upper mapping.
- or code_point == 0x00DF
- or (code_point in DERIVED_CORE_PROPERTIES
- and 'Lowercase' in DERIVED_CORE_PROPERTIES[code_point]))
-
-def is_alpha(code_point):
- '''Checks whether the character with this code point is alphabetic'''
- return ((code_point in DERIVED_CORE_PROPERTIES
- and
- 'Alphabetic' in DERIVED_CORE_PROPERTIES[code_point])
- or
- # Consider all the non-ASCII digits as alphabetic.
- # ISO C 99 forbids us to have them in category “digit”,
- # but we want iswalnum to return true on them.
- (UNICODE_ATTRIBUTES[code_point]['category'] == 'Nd'
- and not (code_point >= 0x0030 and code_point <= 0x0039)))
-
-def is_digit(code_point):
- '''Checks whether the character with this code point is a digit'''
- if False:
- return (UNICODE_ATTRIBUTES[code_point]['name']
- and UNICODE_ATTRIBUTES[code_point]['category'] == 'Nd')
- # Note: U+0BE7..U+0BEF and U+1369..U+1371 are digit systems without
- # a zero. Must add <0> in front of them by hand.
- else:
- # SUSV2 gives us some freedom for the "digit" category, but ISO C 99
- # takes it away:
- # 7.25.2.1.5:
- # The iswdigit function tests for any wide character that
- # corresponds to a decimal-digit character (as defined in 5.2.1).
- # 5.2.1:
- # the 10 decimal digits 0 1 2 3 4 5 6 7 8 9
- return (code_point >= 0x0030 and code_point <= 0x0039)
-
-def is_outdigit(code_point):
- '''Checks whether the character with this code point is outdigit'''
- return (code_point >= 0x0030 and code_point <= 0x0039)
-
-def is_blank(code_point):
- '''Checks whether the character with this code point is blank'''
- return (code_point == 0x0009 # '\t'
- # Category Zs without mention of '<noBreak>'
- or (UNICODE_ATTRIBUTES[code_point]['name']
- and UNICODE_ATTRIBUTES[code_point]['category'] == 'Zs'
- and '<noBreak>' not in
- UNICODE_ATTRIBUTES[code_point]['decomposition']))
-
-def is_space(code_point):
- '''Checks whether the character with this code point is a space'''
- # Don’t make U+00A0 a space. Non-breaking space means that all programs
- # should treat it like a punctuation character, not like a space.
- return (code_point == 0x0020 # ' '
- or code_point == 0x000C # '\f'
- or code_point == 0x000A # '\n'
- or code_point == 0x000D # '\r'
- or code_point == 0x0009 # '\t'
- or code_point == 0x000B # '\v'
- # Categories Zl, Zp, and Zs without mention of "<noBreak>"
- or (UNICODE_ATTRIBUTES[code_point]['name']
- and
- (UNICODE_ATTRIBUTES[code_point]['category'] in ['Zl', 'Zp']
- or
- (UNICODE_ATTRIBUTES[code_point]['category'] in ['Zs']
- and
- '<noBreak>' not in
- UNICODE_ATTRIBUTES[code_point]['decomposition']))))
-
-def is_cntrl(code_point):
- '''Checks whether the character with this code point is
- a control character'''
- return (UNICODE_ATTRIBUTES[code_point]['name']
- and (UNICODE_ATTRIBUTES[code_point]['name'] == '<control>'
- or
- UNICODE_ATTRIBUTES[code_point]['category'] in ['Zl', 'Zp']))
-
-def is_xdigit(code_point):
- '''Checks whether the character with this code point is
- a hexadecimal digit'''
- if False:
- return (is_digit(code_point)
- or (code_point >= 0x0041 and code_point <= 0x0046)
- or (code_point >= 0x0061 and code_point <= 0x0066))
- else:
- # SUSV2 gives us some freedom for the "xdigit" category, but ISO C 99
- # takes it away:
- # 7.25.2.1.12:
- # The iswxdigit function tests for any wide character that
- # corresponds to a hexadecimal-digit character (as defined
- # in 6.4.4.1).
- # 6.4.4.1:
- # hexadecimal-digit: one of
- # 0 1 2 3 4 5 6 7 8 9 a b c d e f A B C D E F
- return ((code_point >= 0x0030 and code_point <= 0x0039)
- or (code_point >= 0x0041 and code_point <= 0x0046)
- or (code_point >= 0x0061 and code_point <= 0x0066))
-
-def is_graph(code_point):
- '''Checks whether the character with this code point is
- a graphical character'''
- return (UNICODE_ATTRIBUTES[code_point]['name']
- and UNICODE_ATTRIBUTES[code_point]['name'] != '<control>'
- and not is_space(code_point))
-
-def is_print(code_point):
- '''Checks whether the character with this code point is printable'''
- return (UNICODE_ATTRIBUTES[code_point]['name']
- and UNICODE_ATTRIBUTES[code_point]['name'] != '<control>'
- and UNICODE_ATTRIBUTES[code_point]['category'] not in ['Zl', 'Zp'])
-
-def is_punct(code_point):
- '''Checks whether the character with this code point is punctuation'''
- if False:
- return (UNICODE_ATTRIBUTES[code_point]['name']
- and UNICODE_ATTRIBUTES[code_point]['category'].startswith('P'))
- else:
- # The traditional POSIX definition of punctuation is every graphic,
- # non-alphanumeric character.
- return (is_graph(code_point)
- and not is_alpha(code_point)
- and not is_digit(code_point))
-
-def is_combining(code_point):
- '''Checks whether the character with this code point is
- a combining character'''
- # Up to Unicode 3.0.1 we took the Combining property from the PropList.txt
- # file. In 3.0.1 it was identical to the union of the general categories
- # "Mn", "Mc", "Me". In Unicode 3.1 this property has been dropped from the
- # PropList.txt file, so we take the latter definition.
- return (UNICODE_ATTRIBUTES[code_point]['name']
- and
- UNICODE_ATTRIBUTES[code_point]['category'] in ['Mn', 'Mc', 'Me'])
-
-def is_combining_level3(code_point):
- '''Checks whether the character with this code point is
- a combining level3 character'''
- return (is_combining(code_point)
- and
- int(UNICODE_ATTRIBUTES[code_point]['combining']) in range(0, 200))
-
-def ucs_symbol(code_point):
- '''Return the UCS symbol string for a Unicode character.'''
- if code_point < 0x10000:
- return '<U{:04X}>'.format(code_point)
- else:
- return '<U{:08X}>'.format(code_point)
-
-def ucs_symbol_range(code_point_low, code_point_high):
- '''Returns a string UCS symbol string for a code point range.
-
- Example:
-
- <U0041>..<U005A>
- '''
- return ucs_symbol(code_point_low) + '..' + ucs_symbol(code_point_high)
+import unicode_utils
def code_point_ranges(is_class_function):
'''Returns a list of ranges of code points for which is_class_function
@@ -379,7 +43,7 @@ def code_point_ranges(is_class_function):
[[65, 90], [192, 214], [216, 222], [256], … ]
'''
cp_ranges = []
- for code_point in sorted(UNICODE_ATTRIBUTES):
+ for code_point in sorted(unicode_utils.UNICODE_ATTRIBUTES):
if is_class_function(code_point):
if (cp_ranges
and cp_ranges[-1][-1] == code_point - 1):
@@ -413,9 +77,9 @@ def output_charclass(i18n_file, class_name, is_class_function):
if line.strip():
line += ';'
if len(code_point_range) == 1:
- range_string = ucs_symbol(code_point_range[0])
+ range_string = unicode_utils.ucs_symbol(code_point_range[0])
else:
- range_string = ucs_symbol_range(
+ range_string = unicode_utils.ucs_symbol_range(
code_point_range[0], code_point_range[-1])
if len(line+range_string) > max_column:
i18n_file.write(line+'/\n')
@@ -441,15 +105,15 @@ def output_charmap(i18n_file, map_name, map_function):
line = prefix
map_string = ''
i18n_file.write('%s /\n' %map_name)
- for code_point in sorted(UNICODE_ATTRIBUTES):
+ for code_point in sorted(unicode_utils.UNICODE_ATTRIBUTES):
mapped = map_function(code_point)
if code_point != mapped:
if line.strip():
line += ';'
map_string = '(' \
- + ucs_symbol(code_point) \
+ + unicode_utils.ucs_symbol(code_point) \
+ ',' \
- + ucs_symbol(mapped) \
+ + unicode_utils.ucs_symbol(mapped) \
+ ')'
if len(line+map_string) > max_column:
i18n_file.write(line+'/\n')
@@ -459,110 +123,6 @@ def output_charmap(i18n_file, map_name, map_function):
i18n_file.write(line+'\n')
i18n_file.write('\n')
-def verifications():
- '''Tests whether the is_* functions observe the known restrictions'''
- for code_point in sorted(UNICODE_ATTRIBUTES):
- # toupper restriction: "Only characters specified for the keywords
- # lower and upper shall be specified.
- if (to_upper(code_point) != code_point
- and not (is_lower(code_point) or is_upper(code_point))):
- sys.stderr.write(
- ('%(sym)s is not upper|lower '
- + 'but toupper(0x%(c)04X) = 0x%(uc)04X\n') %{
- 'sym': ucs_symbol(code_point),
- 'c': code_point,
- 'uc': to_upper(code_point)})
- # tolower restriction: "Only characters specified for the keywords
- # lower and upper shall be specified.
- if (to_lower(code_point) != code_point
- and not (is_lower(code_point) or is_upper(code_point))):
- sys.stderr.write(
- ('%(sym)s is not upper|lower '
- + 'but tolower(0x%(c)04X) = 0x%(uc)04X\n') %{
- 'sym': ucs_symbol(code_point),
- 'c': code_point,
- 'uc': to_lower(code_point)})
- # alpha restriction: "Characters classified as either upper or lower
- # shall automatically belong to this class.
- if ((is_lower(code_point) or is_upper(code_point))
- and not is_alpha(code_point)):
- sys.stderr.write('%(sym)s is upper|lower but not alpha\n' %{
- 'sym': ucs_symbol(code_point)})
- # alpha restriction: “No character specified for the keywords cntrl,
- # digit, punct or space shall be specified.”
- if (is_alpha(code_point) and is_cntrl(code_point)):
- sys.stderr.write('%(sym)s is alpha and cntrl\n' %{
- 'sym': ucs_symbol(code_point)})
- if (is_alpha(code_point) and is_digit(code_point)):
- sys.stderr.write('%(sym)s is alpha and digit\n' %{
- 'sym': ucs_symbol(code_point)})
- if (is_alpha(code_point) and is_punct(code_point)):
- sys.stderr.write('%(sym)s is alpha and punct\n' %{
- 'sym': ucs_symbol(code_point)})
- if (is_alpha(code_point) and is_space(code_point)):
- sys.stderr.write('%(sym)s is alpha and space\n' %{
- 'sym': ucs_symbol(code_point)})
- # space restriction: “No character specified for the keywords upper,
- # lower, alpha, digit, graph or xdigit shall be specified.”
- # upper, lower, alpha already checked above.
- if (is_space(code_point) and is_digit(code_point)):
- sys.stderr.write('%(sym)s is space and digit\n' %{
- 'sym': ucs_symbol(code_point)})
- if (is_space(code_point) and is_graph(code_point)):
- sys.stderr.write('%(sym)s is space and graph\n' %{
- 'sym': ucs_symbol(code_point)})
- if (is_space(code_point) and is_xdigit(code_point)):
- sys.stderr.write('%(sym)s is space and xdigit\n' %{
- 'sym': ucs_symbol(code_point)})
- # cntrl restriction: “No character specified for the keywords upper,
- # lower, alpha, digit, punct, graph, print or xdigit shall be
- # specified.” upper, lower, alpha already checked above.
- if (is_cntrl(code_point) and is_digit(code_point)):
- sys.stderr.write('%(sym)s is cntrl and digit\n' %{
- 'sym': ucs_symbol(code_point)})
- if (is_cntrl(code_point) and is_punct(code_point)):
- sys.stderr.write('%(sym)s is cntrl and punct\n' %{
- 'sym': ucs_symbol(code_point)})
- if (is_cntrl(code_point) and is_graph(code_point)):
- sys.stderr.write('%(sym)s is cntrl and graph\n' %{
- 'sym': ucs_symbol(code_point)})
- if (is_cntrl(code_point) and is_print(code_point)):
- sys.stderr.write('%(sym)s is cntrl and print\n' %{
- 'sym': ucs_symbol(code_point)})
- if (is_cntrl(code_point) and is_xdigit(code_point)):
- sys.stderr.write('%(sym)s is cntrl and xdigit\n' %{
- 'sym': ucs_symbol(code_point)})
- # punct restriction: “No character specified for the keywords upper,
- # lower, alpha, digit, cntrl, xdigit or as the <space> character shall
- # be specified.” upper, lower, alpha, cntrl already checked above.
- if (is_punct(code_point) and is_digit(code_point)):
- sys.stderr.write('%(sym)s is punct and digit\n' %{
- 'sym': ucs_symbol(code_point)})
- if (is_punct(code_point) and is_xdigit(code_point)):
- sys.stderr.write('%(sym)s is punct and xdigit\n' %{
- 'sym': ucs_symbol(code_point)})
- if (is_punct(code_point) and code_point == 0x0020):
- sys.stderr.write('%(sym)s is punct\n' %{
- 'sym': ucs_symbol(code_point)})
- # graph restriction: “No character specified for the keyword cntrl
- # shall be specified.” Already checked above.
-
- # print restriction: “No character specified for the keyword cntrl
- # shall be specified.” Already checked above.
-
- # graph - print relation: differ only in the <space> character.
- # How is this possible if there are more than one space character?!
- # I think susv2/xbd/locale.html should speak of “space characters”,
- # not “space character”.
- if (is_print(code_point)
- and not (is_graph(code_point) or is_space(code_point))):
- sys.stderr.write('%(sym)s is print but not graph|<space>\n' %{
- 'sym': ucs_symbol(code_point)})
- if (not is_print(code_point)
- and (is_graph(code_point) or code_point == 0x0020)):
- sys.stderr.write('%(sym)s is graph|<space> but not print\n' %{
- 'sym': ucs_symbol(code_point)})
-
def read_input_file(filename):
'''Reads the original glibc i18n file to get the original head
and tail.
@@ -648,18 +208,18 @@ def output_tables(i18n_file, unicode_version):
+ 'program.\n\n')
i18n_file.write('% The "upper" class reflects the uppercase '
+ 'characters of class "alpha"\n')
- output_charclass(i18n_file, 'upper', is_upper)
+ output_charclass(i18n_file, 'upper', unicode_utils.is_upper)
i18n_file.write('% The "lower" class reflects the lowercase '
+ 'characters of class "alpha"\n')
- output_charclass(i18n_file, 'lower', is_lower)
+ output_charclass(i18n_file, 'lower', unicode_utils.is_lower)
i18n_file.write('% The "alpha" class of the "i18n" FDCC-set is '
+ 'reflecting\n')
i18n_file.write('% the recommendations in TR 10176 annex A\n')
- output_charclass(i18n_file, 'alpha', is_alpha)
+ output_charclass(i18n_file, 'alpha', unicode_utils.is_alpha)
i18n_file.write('% The "digit" class must only contain the '
+ 'BASIC LATIN digits, says ISO C 99\n')
i18n_file.write('% (sections 7.25.2.1.5 and 5.2.1).\n')
- output_charclass(i18n_file, 'digit', is_digit)
+ output_charclass(i18n_file, 'digit', unicode_utils.is_digit)
i18n_file.write('% The "outdigit" information is by default '
+ '"0" to "9". We don\'t have to\n')
i18n_file.write('% provide it here since localedef will fill '
@@ -669,29 +229,30 @@ def output_tables(i18n_file, unicode_version):
i18n_file.write('% outdigit /\n')
i18n_file.write('% <U0030>..<U0039>\n\n')
# output_charclass(i18n_file, 'outdigit', is_outdigit)
- output_charclass(i18n_file, 'space', is_space)
- output_charclass(i18n_file, 'cntrl', is_cntrl)
- output_charclass(i18n_file, 'punct', is_punct)
- output_charclass(i18n_file, 'graph', is_graph)
- output_charclass(i18n_file, 'print', is_print)
+ output_charclass(i18n_file, 'space', unicode_utils.is_space)
+ output_charclass(i18n_file, 'cntrl', unicode_utils.is_cntrl)
+ output_charclass(i18n_file, 'punct', unicode_utils.is_punct)
+ output_charclass(i18n_file, 'graph', unicode_utils.is_graph)
+ output_charclass(i18n_file, 'print', unicode_utils.is_print)
i18n_file.write('% The "xdigit" class must only contain the '
+ 'BASIC LATIN digits and A-F, a-f,\n')
i18n_file.write('% says ISO C 99 '
+ '(sections 7.25.2.1.12 and 6.4.4.1).\n')
- output_charclass(i18n_file, 'xdigit', is_xdigit)
- output_charclass(i18n_file, 'blank', is_blank)
- output_charmap(i18n_file, 'toupper', to_upper)
- output_charmap(i18n_file, 'tolower', to_lower)
- output_charmap(i18n_file, 'map "totitle";', to_title)
+ output_charclass(i18n_file, 'xdigit', unicode_utils.is_xdigit)
+ output_charclass(i18n_file, 'blank', unicode_utils.is_blank)
+ output_charmap(i18n_file, 'toupper', unicode_utils.to_upper)
+ output_charmap(i18n_file, 'tolower', unicode_utils.to_lower)
+ output_charmap(i18n_file, 'map "totitle";', unicode_utils.to_title)
i18n_file.write('% The "combining" class reflects ISO/IEC 10646-1 '
+ 'annex B.1\n')
i18n_file.write('% That is, all combining characters (level 2+3).\n')
- output_charclass(i18n_file, 'class "combining";', is_combining)
+ output_charclass(i18n_file, 'class "combining";',
+ unicode_utils.is_combining)
i18n_file.write('% The "combining_level3" class reflects '
+ 'ISO/IEC 10646-1 annex B.2\n')
i18n_file.write('% That is, combining characters of level 3.\n')
- output_charclass(i18n_file,
- 'class "combining_level3";', is_combining_level3)
+ output_charclass(i18n_file, 'class "combining_level3";',
+ unicode_utils.is_combining_level3)
if __name__ == "__main__":
PARSER = argparse.ArgumentParser(
@@ -739,9 +300,11 @@ if __name__ == "__main__":
help='The Unicode version of the input files used.')
ARGS = PARSER.parse_args()
- fill_attributes(ARGS.unicode_data_file)
- fill_derived_core_properties(ARGS.derived_core_properties_file)
- verifications()
+ unicode_utils.fill_attributes(
+ ARGS.unicode_data_file)
+ unicode_utils.fill_derived_core_properties(
+ ARGS.derived_core_properties_file)
+ unicode_utils.verifications()
HEAD = TAIL = ''
if ARGS.input_file:
(HEAD, TAIL) = read_input_file(ARGS.input_file)
diff --git a/localedata/unicode-gen/unicode_utils.py b/localedata/unicode-gen/unicode_utils.py
new file mode 100644
index 0000000..ee91582
--- /dev/null
+++ b/localedata/unicode-gen/unicode_utils.py
@@ -0,0 +1,502 @@
+# Utilities to generate Unicode data for glibc from upstream Unicode data.
+#
+# Copyright (C) 2014, 2015 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+#
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <http://www.gnu.org/licenses/>.
+
+'''
+This module contains utilities used by the scripts to generate
+Unicode data for glibc from upstream Unicode data files.
+'''
+
+import sys
+import re
+
+# Dictionary holding the entire contents of the UnicodeData.txt file
+#
+# Contents of this dictionary look like this:
+#
+# {0: {'category': 'Cc',
+# 'title': None,
+# 'digit': '',
+# 'name': '<control>',
+# 'bidi': 'BN',
+# 'combining': '0',
+# 'comment': '',
+# 'oldname': 'NULL',
+# 'decomposition': '',
+# 'upper': None,
+# 'mirrored': 'N',
+# 'lower': None,
+# 'decdigit': '',
+# 'numeric': ''},
+# …
+# }
+UNICODE_ATTRIBUTES = {}
+
+# Dictionary holding the entire contents of the DerivedCoreProperties.txt file
+#
+# Contents of this dictionary look like this:
+#
+# {917504: ['Default_Ignorable_Code_Point'],
+# 917505: ['Case_Ignorable', 'Default_Ignorable_Code_Point'],
+# …
+# }
+DERIVED_CORE_PROPERTIES = {}
+
+# Dictionary holding the entire contents of the EastAsianWidths.txt file
+#
+# Contents of this dictionary look like this:
+#
+# {0: 'N', … , 45430: 'W', …}
+EAST_ASIAN_WIDTHS = {}
+
+def fill_attribute(code_point, fields):
+ '''Stores in UNICODE_ATTRIBUTES[code_point] the values from the fields.
+
+ One entry in the UNICODE_ATTRIBUTES dictionary represents one line
+ in the UnicodeData.txt file.
+
+ '''
+ UNICODE_ATTRIBUTES[code_point] = {
+ 'name': fields[1], # Character name
+ 'category': fields[2], # General category
+ 'combining': fields[3], # Canonical combining classes
+ 'bidi': fields[4], # Bidirectional category
+ 'decomposition': fields[5], # Character decomposition mapping
+ 'decdigit': fields[6], # Decimal digit value
+ 'digit': fields[7], # Digit value
+ 'numeric': fields[8], # Numeric value
+ 'mirrored': fields[9], # mirrored
+ 'oldname': fields[10], # Old Unicode 1.0 name
+ 'comment': fields[11], # comment
+ # Uppercase mapping
+ 'upper': int(fields[12], 16) if fields[12] else None,
+ # Lowercase mapping
+ 'lower': int(fields[13], 16) if fields[13] else None,
+ # Titlecase mapping
+ 'title': int(fields[14], 16) if fields[14] else None,
+ }
+
+def fill_attributes(filename):
+ '''Stores the entire contents of the UnicodeData.txt file
+ in the UNICODE_ATTRIBUTES dictionary.
+
+ A typical line for a single code point in UnicodeData.txt looks
+ like this:
+
+ 0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061;
+
+ Code point ranges are indicated by pairs of lines like this:
+
+ 4E00;<CJK Ideograph, First>;Lo;0;L;;;;;N;;;;;
+ 9FCC;<CJK Ideograph, Last>;Lo;0;L;;;;;N;;;;;
+ '''
+ with open(filename, mode='r') as unicode_data_file:
+ fields_start = []
+ for line in unicode_data_file:
+ fields = line.strip().split(';')
+ if len(fields) != 15:
+ sys.stderr.write(
+ 'short line in file "%(f)s": %(l)s\n' %{
+ 'f': filename, 'l': line})
+ exit(1)
+ if fields[2] == 'Cs':
+ # Surrogates are UTF-16 artefacts,
+ # not real characters. Ignore them.
+ fields_start = []
+ continue
+ if fields[1].endswith(', First>'):
+ fields_start = fields
+ fields_start[1] = fields_start[1].split(',')[0][1:]
+ continue
+ if fields[1].endswith(', Last>'):
+ fields[1] = fields[1].split(',')[0][1:]
+ if fields[1:] != fields_start[1:]:
+ sys.stderr.write(
+ 'broken code point range in file "%(f)s": %(l)s\n' %{
+ 'f': filename, 'l': line})
+ exit(1)
+ for code_point in range(
+ int(fields_start[0], 16),
+ int(fields[0], 16)+1):
+ fill_attribute(code_point, fields)
+ fields_start = []
+ continue
+ fill_attribute(int(fields[0], 16), fields)
+ fields_start = []
+
+def fill_derived_core_properties(filename):
+ '''Stores the entire contents of the DerivedCoreProperties.txt file
+ in the DERIVED_CORE_PROPERTIES dictionary.
+
+ Lines in DerivedCoreProperties.txt are either a code point range like
+ this:
+
+ 0061..007A ; Lowercase # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z
+
+ or a single code point like this:
+
+ 00AA ; Lowercase # Lo FEMININE ORDINAL INDICATOR
+
+ '''
+ with open(filename, mode='r') as derived_core_properties_file:
+ for line in derived_core_properties_file:
+ match = re.match(
+ r'^(?P<codepoint1>[0-9A-F]{4,6})'
+ + r'(?:\.\.(?P<codepoint2>[0-9A-F]{4,6}))?'
+ + r'\s*;\s*(?P<property>[a-zA-Z_]+)',
+ line)
+ if not match:
+ continue
+ start = match.group('codepoint1')
+ end = match.group('codepoint2')
+ if not end:
+ end = start
+ for code_point in range(int(start, 16), int(end, 16)+1):
+ prop = match.group('property')
+ if code_point in DERIVED_CORE_PROPERTIES:
+ DERIVED_CORE_PROPERTIES[code_point].append(prop)
+ else:
+ DERIVED_CORE_PROPERTIES[code_point] = [prop]
+
+def fill_east_asian_widths(filename):
+ '''Stores the entire contents of the EastAsianWidths.txt file
+ in the EAST_ASIAN_WIDTHS dictionary.
+
+ Lines in EastAsianWidths.txt are either a code point range like
+ this:
+
+ 9FCD..9FFF;W # Cn [51] <reserved-9FCD>..<reserved-9FFF>
+
+ or a single code point like this:
+
+ A015;W # Lm YI SYLLABLE WU
+ '''
+ with open(filename, mode='r') as east_asian_widths_file:
+ for line in east_asian_widths_file:
+ match = re.match(
+ r'^(?P<codepoint1>[0-9A-F]{4,6})'
+ +r'(?:\.\.(?P<codepoint2>[0-9A-F]{4,6}))?'
+ +r'\s*;\s*(?P<property>[a-zA-Z]+)',
+ line)
+ if not match:
+ continue
+ start = match.group('codepoint1')
+ end = match.group('codepoint2')
+ if not end:
+ end = start
+ for code_point in range(int(start, 16), int(end, 16)+1):
+ EAST_ASIAN_WIDTHS[code_point] = match.group('property')
+
+def to_upper(code_point):
+ '''Returns the code point of the uppercase version
+ of the given code point'''
+ if (UNICODE_ATTRIBUTES[code_point]['name']
+ and UNICODE_ATTRIBUTES[code_point]['upper']):
+ return UNICODE_ATTRIBUTES[code_point]['upper']
+ else:
+ return code_point
+
+def to_lower(code_point):
+ '''Returns the code point of the lowercase version
+ of the given code point'''
+ if (UNICODE_ATTRIBUTES[code_point]['name']
+ and UNICODE_ATTRIBUTES[code_point]['lower']):
+ return UNICODE_ATTRIBUTES[code_point]['lower']
+ else:
+ return code_point
+
+def to_title(code_point):
+ '''Returns the code point of the titlecase version
+ of the given code point'''
+ if (UNICODE_ATTRIBUTES[code_point]['name']
+ and UNICODE_ATTRIBUTES[code_point]['title']):
+ return UNICODE_ATTRIBUTES[code_point]['title']
+ else:
+ return code_point
+
+def is_upper(code_point):
+ '''Checks whether the character with this code point is uppercase'''
+ return (to_lower(code_point) != code_point
+ or (code_point in DERIVED_CORE_PROPERTIES
+ and 'Uppercase' in DERIVED_CORE_PROPERTIES[code_point]))
+
+def is_lower(code_point):
+ '''Checks whether the character with this code point is lowercase'''
+ # Some characters are defined as “Lowercase” in
+ # DerivedCoreProperties.txt but do not have a mapping to upper
+ # case. For example, ꜰ U+A72F “LATIN LETTER SMALL CAPITAL F” is
+ # one of these.
+ return (to_upper(code_point) != code_point
+ # <U00DF> is lowercase, but without simple to_upper mapping.
+ or code_point == 0x00DF
+ or (code_point in DERIVED_CORE_PROPERTIES
+ and 'Lowercase' in DERIVED_CORE_PROPERTIES[code_point]))
+
+def is_alpha(code_point):
+ '''Checks whether the character with this code point is alphabetic'''
+ return ((code_point in DERIVED_CORE_PROPERTIES
+ and
+ 'Alphabetic' in DERIVED_CORE_PROPERTIES[code_point])
+ or
+ # Consider all the non-ASCII digits as alphabetic.
+ # ISO C 99 forbids us to have them in category “digit”,
+ # but we want iswalnum to return true on them.
+ (UNICODE_ATTRIBUTES[code_point]['category'] == 'Nd'
+ and not (code_point >= 0x0030 and code_point <= 0x0039)))
+
+def is_digit(code_point):
+ '''Checks whether the character with this code point is a digit'''
+ if False:
+ return (UNICODE_ATTRIBUTES[code_point]['name']
+ and UNICODE_ATTRIBUTES[code_point]['category'] == 'Nd')
+ # Note: U+0BE7..U+0BEF and U+1369..U+1371 are digit systems without
+ # a zero. Must add <0> in front of them by hand.
+ else:
+ # SUSV2 gives us some freedom for the "digit" category, but ISO C 99
+ # takes it away:
+ # 7.25.2.1.5:
+ # The iswdigit function tests for any wide character that
+ # corresponds to a decimal-digit character (as defined in 5.2.1).
+ # 5.2.1:
+ # the 10 decimal digits 0 1 2 3 4 5 6 7 8 9
+ return (code_point >= 0x0030 and code_point <= 0x0039)
+
+def is_outdigit(code_point):
+ '''Checks whether the character with this code point is outdigit'''
+ return (code_point >= 0x0030 and code_point <= 0x0039)
+
+def is_blank(code_point):
+ '''Checks whether the character with this code point is blank'''
+ return (code_point == 0x0009 # '\t'
+ # Category Zs without mention of '<noBreak>'
+ or (UNICODE_ATTRIBUTES[code_point]['name']
+ and UNICODE_ATTRIBUTES[code_point]['category'] == 'Zs'
+ and '<noBreak>' not in
+ UNICODE_ATTRIBUTES[code_point]['decomposition']))
+
+def is_space(code_point):
+ '''Checks whether the character with this code point is a space'''
+ # Don’t make U+00A0 a space. Non-breaking space means that all programs
+ # should treat it like a punctuation character, not like a space.
+ return (code_point == 0x0020 # ' '
+ or code_point == 0x000C # '\f'
+ or code_point == 0x000A # '\n'
+ or code_point == 0x000D # '\r'
+ or code_point == 0x0009 # '\t'
+ or code_point == 0x000B # '\v'
+ # Categories Zl, Zp, and Zs without mention of "<noBreak>"
+ or (UNICODE_ATTRIBUTES[code_point]['name']
+ and
+ (UNICODE_ATTRIBUTES[code_point]['category'] in ['Zl', 'Zp']
+ or
+ (UNICODE_ATTRIBUTES[code_point]['category'] in ['Zs']
+ and
+ '<noBreak>' not in
+ UNICODE_ATTRIBUTES[code_point]['decomposition']))))
+
+def is_cntrl(code_point):
+ '''Checks whether the character with this code point is
+ a control character'''
+ return (UNICODE_ATTRIBUTES[code_point]['name']
+ and (UNICODE_ATTRIBUTES[code_point]['name'] == '<control>'
+ or
+ UNICODE_ATTRIBUTES[code_point]['category'] in ['Zl', 'Zp']))
+
+def is_xdigit(code_point):
+ '''Checks whether the character with this code point is
+ a hexadecimal digit'''
+ if False:
+ return (is_digit(code_point)
+ or (code_point >= 0x0041 and code_point <= 0x0046)
+ or (code_point >= 0x0061 and code_point <= 0x0066))
+ else:
+ # SUSV2 gives us some freedom for the "xdigit" category, but ISO C 99
+ # takes it away:
+ # 7.25.2.1.12:
+ # The iswxdigit function tests for any wide character that
+ # corresponds to a hexadecimal-digit character (as defined
+ # in 6.4.4.1).
+ # 6.4.4.1:
+ # hexadecimal-digit: one of
+ # 0 1 2 3 4 5 6 7 8 9 a b c d e f A B C D E F
+ return ((code_point >= 0x0030 and code_point <= 0x0039)
+ or (code_point >= 0x0041 and code_point <= 0x0046)
+ or (code_point >= 0x0061 and code_point <= 0x0066))
+
+def is_graph(code_point):
+ '''Checks whether the character with this code point is
+ a graphical character'''
+ return (UNICODE_ATTRIBUTES[code_point]['name']
+ and UNICODE_ATTRIBUTES[code_point]['name'] != '<control>'
+ and not is_space(code_point))
+
+def is_print(code_point):
+ '''Checks whether the character with this code point is printable'''
+ return (UNICODE_ATTRIBUTES[code_point]['name']
+ and UNICODE_ATTRIBUTES[code_point]['name'] != '<control>'
+ and UNICODE_ATTRIBUTES[code_point]['category'] not in ['Zl', 'Zp'])
+
+def is_punct(code_point):
+ '''Checks whether the character with this code point is punctuation'''
+ if False:
+ return (UNICODE_ATTRIBUTES[code_point]['name']
+ and UNICODE_ATTRIBUTES[code_point]['category'].startswith('P'))
+ else:
+ # The traditional POSIX definition of punctuation is every graphic,
+ # non-alphanumeric character.
+ return (is_graph(code_point)
+ and not is_alpha(code_point)
+ and not is_digit(code_point))
+
+def is_combining(code_point):
+ '''Checks whether the character with this code point is
+ a combining character'''
+ # Up to Unicode 3.0.1 we took the Combining property from the PropList.txt
+ # file. In 3.0.1 it was identical to the union of the general categories
+ # "Mn", "Mc", "Me". In Unicode 3.1 this property has been dropped from the
+ # PropList.txt file, so we take the latter definition.
+ return (UNICODE_ATTRIBUTES[code_point]['name']
+ and
+ UNICODE_ATTRIBUTES[code_point]['category'] in ['Mn', 'Mc', 'Me'])
+
+def is_combining_level3(code_point):
+ '''Checks whether the character with this code point is
+ a combining level3 character'''
+ return (is_combining(code_point)
+ and
+ int(UNICODE_ATTRIBUTES[code_point]['combining']) in range(0, 200))
+
+def ucs_symbol(code_point):
+ '''Return the UCS symbol string for a Unicode character.'''
+ if code_point < 0x10000:
+ return '<U{:04X}>'.format(code_point)
+ else:
+ return '<U{:08X}>'.format(code_point)
+
+def ucs_symbol_range(code_point_low, code_point_high):
+ '''Returns a string UCS symbol string for a code point range.
+
+ Example:
+
+ <U0041>..<U005A>
+ '''
+ return ucs_symbol(code_point_low) + '..' + ucs_symbol(code_point_high)
+
+def verifications():
+ '''Tests whether the is_* functions observe the known restrictions'''
+ for code_point in sorted(UNICODE_ATTRIBUTES):
+ # toupper restriction: "Only characters specified for the keywords
+ # lower and upper shall be specified.
+ if (to_upper(code_point) != code_point
+ and not (is_lower(code_point) or is_upper(code_point))):
+ sys.stderr.write(
+ ('%(sym)s is not upper|lower '
+ + 'but toupper(0x%(c)04X) = 0x%(uc)04X\n') %{
+ 'sym': ucs_symbol(code_point),
+ 'c': code_point,
+ 'uc': to_upper(code_point)})
+ # tolower restriction: "Only characters specified for the keywords
+ # lower and upper shall be specified.
+ if (to_lower(code_point) != code_point
+ and not (is_lower(code_point) or is_upper(code_point))):
+ sys.stderr.write(
+ ('%(sym)s is not upper|lower '
+ + 'but tolower(0x%(c)04X) = 0x%(uc)04X\n') %{
+ 'sym': ucs_symbol(code_point),
+ 'c': code_point,
+ 'uc': to_lower(code_point)})
+ # alpha restriction: "Characters classified as either upper or lower
+ # shall automatically belong to this class.
+ if ((is_lower(code_point) or is_upper(code_point))
+ and not is_alpha(code_point)):
+ sys.stderr.write('%(sym)s is upper|lower but not alpha\n' %{
+ 'sym': ucs_symbol(code_point)})
+ # alpha restriction: “No character specified for the keywords cntrl,
+ # digit, punct or space shall be specified.”
+ if (is_alpha(code_point) and is_cntrl(code_point)):
+ sys.stderr.write('%(sym)s is alpha and cntrl\n' %{
+ 'sym': ucs_symbol(code_point)})
+ if (is_alpha(code_point) and is_digit(code_point)):
+ sys.stderr.write('%(sym)s is alpha and digit\n' %{
+ 'sym': ucs_symbol(code_point)})
+ if (is_alpha(code_point) and is_punct(code_point)):
+ sys.stderr.write('%(sym)s is alpha and punct\n' %{
+ 'sym': ucs_symbol(code_point)})
+ if (is_alpha(code_point) and is_space(code_point)):
+ sys.stderr.write('%(sym)s is alpha and space\n' %{
+ 'sym': ucs_symbol(code_point)})
+ # space restriction: “No character specified for the keywords upper,
+ # lower, alpha, digit, graph or xdigit shall be specified.”
+ # upper, lower, alpha already checked above.
+ if (is_space(code_point) and is_digit(code_point)):
+ sys.stderr.write('%(sym)s is space and digit\n' %{
+ 'sym': ucs_symbol(code_point)})
+ if (is_space(code_point) and is_graph(code_point)):
+ sys.stderr.write('%(sym)s is space and graph\n' %{
+ 'sym': ucs_symbol(code_point)})
+ if (is_space(code_point) and is_xdigit(code_point)):
+ sys.stderr.write('%(sym)s is space and xdigit\n' %{
+ 'sym': ucs_symbol(code_point)})
+ # cntrl restriction: “No character specified for the keywords upper,
+ # lower, alpha, digit, punct, graph, print or xdigit shall be
+ # specified.” upper, lower, alpha already checked above.
+ if (is_cntrl(code_point) and is_digit(code_point)):
+ sys.stderr.write('%(sym)s is cntrl and digit\n' %{
+ 'sym': ucs_symbol(code_point)})
+ if (is_cntrl(code_point) and is_punct(code_point)):
+ sys.stderr.write('%(sym)s is cntrl and punct\n' %{
+ 'sym': ucs_symbol(code_point)})
+ if (is_cntrl(code_point) and is_graph(code_point)):
+ sys.stderr.write('%(sym)s is cntrl and graph\n' %{
+ 'sym': ucs_symbol(code_point)})
+ if (is_cntrl(code_point) and is_print(code_point)):
+ sys.stderr.write('%(sym)s is cntrl and print\n' %{
+ 'sym': ucs_symbol(code_point)})
+ if (is_cntrl(code_point) and is_xdigit(code_point)):
+ sys.stderr.write('%(sym)s is cntrl and xdigit\n' %{
+ 'sym': ucs_symbol(code_point)})
+ # punct restriction: “No character specified for the keywords upper,
+ # lower, alpha, digit, cntrl, xdigit or as the <space> character shall
+ # be specified.” upper, lower, alpha, cntrl already checked above.
+ if (is_punct(code_point) and is_digit(code_point)):
+ sys.stderr.write('%(sym)s is punct and digit\n' %{
+ 'sym': ucs_symbol(code_point)})
+ if (is_punct(code_point) and is_xdigit(code_point)):
+ sys.stderr.write('%(sym)s is punct and xdigit\n' %{
+ 'sym': ucs_symbol(code_point)})
+ if (is_punct(code_point) and code_point == 0x0020):
+ sys.stderr.write('%(sym)s is punct\n' %{
+ 'sym': ucs_symbol(code_point)})
+ # graph restriction: “No character specified for the keyword cntrl
+ # shall be specified.” Already checked above.
+
+ # print restriction: “No character specified for the keyword cntrl
+ # shall be specified.” Already checked above.
+
+ # graph - print relation: differ only in the <space> character.
+ # How is this possible if there are more than one space character?!
+ # I think susv2/xbd/locale.html should speak of “space characters”,
+ # not “space character”.
+ if (is_print(code_point)
+ and not (is_graph(code_point) or is_space(code_point))):
+ sys.stderr.write('%(sym)s is print but not graph|<space>\n' %{
+ 'sym': unicode_utils.ucs_symbol(code_point)})
+ if (not is_print(code_point)
+ and (is_graph(code_point) or code_point == 0x0020)):
+ sys.stderr.write('%(sym)s is graph|<space> but not print\n' %{
+ 'sym': unicode_utils.ucs_symbol(code_point)})
diff --git a/localedata/unicode-gen/utf8_compatibility.py b/localedata/unicode-gen/utf8_compatibility.py
index b84a1eb..3b7a94c 100755
--- a/localedata/unicode-gen/utf8_compatibility.py
+++ b/localedata/unicode-gen/utf8_compatibility.py
@@ -30,146 +30,7 @@ To see how this script is used, call it with the “-h” option:
import sys
import re
import argparse
-
-# Dictionary holding the entire contents of the UnicodeData.txt file
-#
-# Contents of this dictionary look like this:
-#
-# {0: {'category': 'Cc',
-# 'title': None,
-# 'digit': '',
-# 'name': '<control>',
-# 'bidi': 'BN',
-# 'combining': '0',
-# 'comment': '',
-# 'oldname': 'NULL',
-# 'decomposition': '',
-# 'upper': None,
-# 'mirrored': 'N',
-# 'lower': None,
-# 'decdigit': '',
-# 'numeric': ''},
-# …
-# }
-UNICODE_ATTRIBUTES = {}
-
-# Dictionary holding the entire contents of the EastAsianWidths.txt file
-#
-# Contents of this dictionary look like this:
-#
-# {0: 'N', … , 45430: 'W', …}
-EAST_ASIAN_WIDTHS = {}
-
-def fill_attribute(code_point, fields):
- '''Stores in UNICODE_ATTRIBUTES[code_point] the values from the fields.
-
- One entry in the UNICODE_ATTRIBUTES dictionary represents one line
- in the UnicodeData.txt file.
-
- '''
- UNICODE_ATTRIBUTES[code_point] = {
- 'name': fields[1], # Character name
- 'category': fields[2], # General category
- 'combining': fields[3], # Canonical combining classes
- 'bidi': fields[4], # Bidirectional category
- 'decomposition': fields[5], # Character decomposition mapping
- 'decdigit': fields[6], # Decimal digit value
- 'digit': fields[7], # Digit value
- 'numeric': fields[8], # Numeric value
- 'mirrored': fields[9], # mirrored
- 'oldname': fields[10], # Old Unicode 1.0 name
- 'comment': fields[11], # comment
- # Uppercase mapping
- 'upper': int(fields[12], 16) if fields[12] else None,
- # Lowercase mapping
- 'lower': int(fields[13], 16) if fields[13] else None,
- # Titlecase mapping
- 'title': int(fields[14], 16) if fields[14] else None,
- }
-
-def fill_attributes(filename):
- '''Stores the entire contents of the UnicodeData.txt file
- in the UNICODE_ATTRIBUTES dictionary.
-
- A typical line for a single code point in UnicodeData.txt looks
- like this:
-
- 0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061;
-
- Code point ranges are indicated by pairs of lines like this:
-
- 4E00;<CJK Ideograph, First>;Lo;0;L;;;;;N;;;;;
- 9FCC;<CJK Ideograph, Last>;Lo;0;L;;;;;N;;;;;
- '''
- with open(filename, mode='r') as unicode_data_file:
- fields_start = []
- for line in unicode_data_file:
- fields = line.strip().split(';')
- if len(fields) != 15:
- sys.stderr.write(
- 'short line in file "%(f)s": %(l)s\n' %{
- 'f': filename, 'l': line})
- exit(1)
- if fields[2] == 'Cs':
- # Surrogates are UTF-16 artefacts,
- # not real characters. Ignore them.
- fields_start = []
- continue
- if fields[1].endswith(', First>'):
- fields_start = fields
- fields_start[1] = fields_start[1].split(',')[0][1:]
- continue
- if fields[1].endswith(', Last>'):
- fields[1] = fields[1].split(',')[0][1:]
- if fields[1:] != fields_start[1:]:
- sys.stderr.write(
- 'broken code point range in file "%(f)s": %(l)s\n' %{
- 'f': filename, 'l': line})
- exit(1)
- for code_point in range(
- int(fields_start[0], 16),
- int(fields[0], 16)+1):
- fill_attribute(code_point, fields)
- fields_start = []
- continue
- fill_attribute(int(fields[0], 16), fields)
- fields_start = []
-
-def fill_east_asian_widths(filename):
- '''Stores the entire contents of the EastAsianWidths.txt file
- in the EAST_ASIAN_WIDTHS dictionary.
-
- Lines in EastAsianWidths.txt are either a code point range like
- this:
-
- 9FCD..9FFF;W # Cn [51] <reserved-9FCD>..<reserved-9FFF>
-
- or a single code point like this:
-
- A015;W # Lm YI SYLLABLE WU
- '''
- with open(filename, mode='r') as east_asian_widths_file:
- for line in east_asian_widths_file:
- match = re.match(
- r'^(?P<codepoint1>[0-9A-F]{4,6})'
- +r'(?:\.\.(?P<codepoint2>[0-9A-F]{4,6}))?'
- +r'\s*;\s*(?P<property>[a-zA-Z]+)',
- line)
- if not match:
- continue
- start = match.group('codepoint1')
- end = match.group('codepoint2')
- if not end:
- end = start
- for code_point in range(int(start, 16), int(end, 16)+1):
- EAST_ASIAN_WIDTHS[code_point] = match.group('property')
-
-def ucs_symbol(code_point):
- '''Return the UCS symbol string for a Unicode character.'''
- if code_point < 0x10000:
- return '<U{:04X}>'.format(code_point)
- else:
- return '<U{:08X}>'.format(code_point)
+import unicode_utils
def create_charmap_dictionary(file_name):
'''Create a dictionary for all code points found in the CHARMAP
@@ -217,10 +78,10 @@ def check_charmap(original_file_name, new_file_name):
if ARGS.show_missing_characters:
for key in sorted(set(ocharmap)-set(ncharmap)):
print('removed: {:s} {:s} {:s}'.format(
- ucs_symbol(key),
+ unicode_utils.ucs_symbol(key),
ocharmap[key],
- UNICODE_ATTRIBUTES[key]['name'] \
- if key in UNICODE_ATTRIBUTES else None))
+ unicode_utils.UNICODE_ATTRIBUTES[key]['name'] \
+ if key in unicode_utils.UNICODE_ATTRIBUTES else 'None'))
print('------------------------------------------------------------')
changed_charmap = {}
for key in set(ocharmap).intersection(set(ncharmap)):
@@ -231,21 +92,21 @@ def check_charmap(original_file_name, new_file_name):
if ARGS.show_changed_characters:
for key in sorted(changed_charmap):
print('changed: {:s} {:s}->{:s} {:s}'.format(
- ucs_symbol(key),
+ unicode_utils.ucs_symbol(key),
changed_charmap[key][0],
changed_charmap[key][1],
- UNICODE_ATTRIBUTES[key]['name'] \
- if key in UNICODE_ATTRIBUTES else None))
+ unicode_utils.UNICODE_ATTRIBUTES[key]['name'] \
+ if key in unicode_utils.UNICODE_ATTRIBUTES else 'None'))
print('------------------------------------------------------------')
print('Total added characters in newly generated CHARMAP: %d'
%len(set(ncharmap)-set(ocharmap)))
if ARGS.show_added_characters:
for key in sorted(set(ncharmap)-set(ocharmap)):
print('added: {:s} {:s} {:s}'.format(
- ucs_symbol(key),
+ unicode_utils.ucs_symbol(key),
ncharmap[key],
- UNICODE_ATTRIBUTES[key]['name'] \
- if key in UNICODE_ATTRIBUTES else None))
+ unicode_utils.UNICODE_ATTRIBUTES[key]['name'] \
+ if key in unicode_utils.UNICODE_ATTRIBUTES else 'None'))
def create_width_dictionary(file_name):
'''Create a dictionary for all code points found in the WIDTH
@@ -290,20 +151,20 @@ def check_width(original_file_name, new_file_name):
+ 'i.e. these have width 1 now.)')
if ARGS.show_missing_characters:
for key in sorted(set(owidth)-set(nwidth)):
- print('removed: {:s} '.format(ucs_symbol(key))
+ print('removed: {:s} '.format(unicode_utils.ucs_symbol(key))
+ '{:d} : '.format(owidth[key])
+ 'eaw={:s} '.format(
- EAST_ASIAN_WIDTHS[key]
- if key in EAST_ASIAN_WIDTHS else None)
+ unicode_utils.EAST_ASIAN_WIDTHS[key]
+ if key in unicode_utils.EAST_ASIAN_WIDTHS else 'None')
+ 'category={:2s} '.format(
- UNICODE_ATTRIBUTES[key]['category']
- if key in UNICODE_ATTRIBUTES else None)
+ unicode_utils.UNICODE_ATTRIBUTES[key]['category']
+ if key in unicode_utils.UNICODE_ATTRIBUTES else 'None')
+ 'bidi={:3s} '.format(
- UNICODE_ATTRIBUTES[key]['bidi']
- if key in UNICODE_ATTRIBUTES else None)
+ unicode_utils.UNICODE_ATTRIBUTES[key]['bidi']
+ if key in unicode_utils.UNICODE_ATTRIBUTES else 'None')
+ 'name={:s}'.format(
- UNICODE_ATTRIBUTES[key]['name']
- if key in UNICODE_ATTRIBUTES else None))
+ unicode_utils.UNICODE_ATTRIBUTES[key]['name']
+ if key in unicode_utils.UNICODE_ATTRIBUTES else 'None'))
print('------------------------------------------------------------')
changed_width = {}
for key in set(owidth).intersection(set(nwidth)):
@@ -313,21 +174,21 @@ def check_width(original_file_name, new_file_name):
%len(changed_width))
if ARGS.show_changed_characters:
for key in sorted(changed_width):
- print('changed width: {:s} '.format(ucs_symbol(key))
+ print('changed width: {:s} '.format(unicode_utils.ucs_symbol(key))
+ '{:d}->{:d} : '.format(changed_width[key][0],
changed_width[key][1])
+ 'eaw={:s} '.format(
- EAST_ASIAN_WIDTHS[key]
- if key in EAST_ASIAN_WIDTHS else None)
+ unicode_utils.EAST_ASIAN_WIDTHS[key]
+ if key in unicode_utils.EAST_ASIAN_WIDTHS else 'None')
+ 'category={:2s} '.format(
- UNICODE_ATTRIBUTES[key]['category']
- if key in UNICODE_ATTRIBUTES else None)
+ unicode_utils.UNICODE_ATTRIBUTES[key]['category']
+ if key in unicode_utils.UNICODE_ATTRIBUTES else 'None')
+ 'bidi={:3s} '.format(
- UNICODE_ATTRIBUTES[key]['bidi']
- if key in UNICODE_ATTRIBUTES else None)
+ unicode_utils.UNICODE_ATTRIBUTES[key]['bidi']
+ if key in unicode_utils.UNICODE_ATTRIBUTES else 'None')
+ 'name={:s}'.format(
- UNICODE_ATTRIBUTES[key]['name']
- if key in UNICODE_ATTRIBUTES else None))
+ unicode_utils.UNICODE_ATTRIBUTES[key]['name']
+ if key in unicode_utils.UNICODE_ATTRIBUTES else 'None'))
print('------------------------------------------------------------')
print('Total added characters in newly generated WIDTH: %d'
%len(set(nwidth)-set(owidth)))
@@ -335,20 +196,20 @@ def check_width(original_file_name, new_file_name):
+ 'i.e. these had width 1 before.)')
if ARGS.show_added_characters:
for key in sorted(set(nwidth)-set(owidth)):
- print('added: {:s} '.format(ucs_symbol(key))
+ print('added: {:s} '.format(unicode_utils.ucs_symbol(key))
+ '{:d} : '.format(nwidth[key])
+ 'eaw={:s} '.format(
- EAST_ASIAN_WIDTHS[key]
- if key in EAST_ASIAN_WIDTHS else None)
+ unicode_utils.EAST_ASIAN_WIDTHS[key]
+ if key in unicode_utils.EAST_ASIAN_WIDTHS else 'None')
+ 'category={:2s} '.format(
- UNICODE_ATTRIBUTES[key]['category']
- if key in UNICODE_ATTRIBUTES else None)
+ unicode_utils.UNICODE_ATTRIBUTES[key]['category']
+ if key in unicode_utils.UNICODE_ATTRIBUTES else 'None')
+ 'bidi={:3s} '.format(
- UNICODE_ATTRIBUTES[key]['bidi']
- if key in UNICODE_ATTRIBUTES else None)
+ unicode_utils.UNICODE_ATTRIBUTES[key]['bidi']
+ if key in unicode_utils.UNICODE_ATTRIBUTES else 'None')
+ 'name={:s}'.format(
- UNICODE_ATTRIBUTES[key]['name']
- if key in UNICODE_ATTRIBUTES else None))
+ unicode_utils.UNICODE_ATTRIBUTES[key]['name']
+ if key in unicode_utils.UNICODE_ATTRIBUTES else 'None'))
if __name__ == "__main__":
PARSER = argparse.ArgumentParser(
@@ -392,8 +253,8 @@ if __name__ == "__main__":
ARGS = PARSER.parse_args()
if ARGS.unicode_data_file:
- fill_attributes(ARGS.unicode_data_file)
+ unicode_utils.fill_attributes(ARGS.unicode_data_file)
if ARGS.east_asian_width_file:
- fill_east_asian_widths(ARGS.east_asian_width_file)
+ unicode_utils.fill_east_asian_widths(ARGS.east_asian_width_file)
check_charmap(ARGS.old_utf8_file, ARGS.new_utf8_file)
check_width(ARGS.old_utf8_file, ARGS.new_utf8_file)
diff --git a/localedata/unicode-gen/utf8_gen.py b/localedata/unicode-gen/utf8_gen.py
index f1b88f5..bc84c07 100755
--- a/localedata/unicode-gen/utf8_gen.py
+++ b/localedata/unicode-gen/utf8_gen.py
@@ -29,6 +29,7 @@ It will output UTF-8 file
import sys
import re
+import unicode_utils
# Auxiliary tables for Hangul syllable names, see the Unicode 3.0 book,
# sections 3.11 and 4.4.
@@ -49,13 +50,6 @@ JAMO_FINAL_SHORT_NAME = (
'P', 'H'
)
-def ucs_symbol(code_point):
- '''Return the UCS symbol string for a Unicode character.'''
- if code_point < 0x10000:
- return '<U{:04X}>'.format(code_point)
- else:
- return '<U{:08X}>'.format(code_point)
-
def process_range(start, end, outfile, name):
'''Writes a range of code points into the CHARMAP section of the
output file
@@ -78,7 +72,7 @@ def process_range(start, end, outfile, name):
+ JAMO_MEDIAL_SHORT_NAME[index2] \
+ JAMO_FINAL_SHORT_NAME[index3]
outfile.write('{:<11s} {:<12s} {:s}\n'.format(
- ucs_symbol(i), convert_to_hex(i),
+ unicode_utils.ucs_symbol(i), convert_to_hex(i),
hangul_syllable_name))
return
# UnicodeData.txt file has contains code point ranges like this:
@@ -95,14 +89,14 @@ def process_range(start, end, outfile, name):
for i in range(int(start, 16), int(end, 16), 64 ):
if i > (int(end, 16)-64):
outfile.write('{:s}..{:s} {:<12s} {:s}\n'.format(
- ucs_symbol(i),
- ucs_symbol(int(end,16)),
+ unicode_utils.ucs_symbol(i),
+ unicode_utils.ucs_symbol(int(end,16)),
convert_to_hex(i),
name))
break
outfile.write('{:s}..{:s} {:<12s} {:s}\n'.format(
- ucs_symbol(i),
- ucs_symbol(i+63),
+ unicode_utils.ucs_symbol(i),
+ unicode_utils.ucs_symbol(i+63),
convert_to_hex(i),
name))
@@ -168,7 +162,7 @@ def process_charmap(flines, outfile):
# comments, so we keep these comment lines.
outfile.write('%')
outfile.write('{:<11s} {:<12s} {:s}\n'.format(
- ucs_symbol(int(fields[0], 16)),
+ unicode_utils.ucs_symbol(int(fields[0], 16)),
convert_to_hex(int(fields[0], 16)),
fields[1]))
@@ -230,7 +224,7 @@ def process_width(outfile, ulines, elines):
for line in ulines:
fields = line.split(";")
if fields[4] == "NSM" or fields[2] == "Cf":
- width_dict[int(fields[0], 16)] = ucs_symbol(
+ width_dict[int(fields[0], 16)] = unicode_utils.ucs_symbol(
int(fields[0], 16)) + '\t0'
for line in elines:
@@ -238,7 +232,7 @@ def process_width(outfile, ulines, elines):
# UnicodeData.txt:
fields = line.split(";")
if not '..' in fields[0]:
- width_dict[int(fields[0], 16)] = ucs_symbol(
+ width_dict[int(fields[0], 16)] = unicode_utils.ucs_symbol(
int(fields[0], 16)) + '\t2'
else:
code_points = fields[0].split("..")
@@ -247,8 +241,8 @@ def process_width(outfile, ulines, elines):
if key in width_dict:
del width_dict[key]
width_dict[int(code_points[0], 16)] = '{:s}...{:s}\t2'.format(
- ucs_symbol(int(code_points[0], 16)),
- ucs_symbol(int(code_points[1], 16)))
+ unicode_utils.ucs_symbol(int(code_points[0], 16)),
+ unicode_utils.ucs_symbol(int(code_points[1], 16)))
for key in sorted(width_dict):
outfile.write(width_dict[key]+'\n')
--
2.4.3
Reference in New Issue Copy Permalink