52b91b4b42
- Improve check whether a phrase is simplified or traditional Chinese The improvement is to ignore all non-Han characters when doing the check. This is to avoid classifying a simplified Chinese string as traditional just because it happens to include some non-Chinese characters, for example box drawing characters, which cannot be converted to gb2312 but happen to be convertible to big5hkscs. This fixes the problem in the emoji-table input method that most phrases cannot be input at all.
70 lines
2.7 KiB
Diff
70 lines
2.7 KiB
Diff
From 7a2eed5fb33098b10d67958472a23bb41a3f7b11 Mon Sep 17 00:00:00 2001
|
||
From: Mike FABIAN <mfabian@redhat.com>
|
||
Date: Thu, 13 Sep 2012 12:43:55 +0200
|
||
Subject: [PATCH] Improve check whether a phrase is simplified or traditional
|
||
Chinese
|
||
|
||
The improvement is to ignore all non-Han characters when
|
||
doing the check.
|
||
|
||
This is to avoid classifying a simplified Chinese string as
|
||
traditional just because it happens to include some non-Chinese
|
||
characters, for example box drawing characters, which cannot be
|
||
converted to gb2312 but happen to be convertible to big5hkscs.
|
||
|
||
This fixes the problem in the emoji-table input method that most
|
||
phrases cannot be input at all.
|
||
|
||
See: https://bugzilla.redhat.com/show_bug.cgi?id=856320
|
||
---
|
||
engine/tabsqlitedb.py | 18 ++++++++++++++----
|
||
1 file changed, 14 insertions(+), 4 deletions(-)
|
||
|
||
diff --git a/engine/tabsqlitedb.py b/engine/tabsqlitedb.py
|
||
index 7606afb..9e3b50c 100644
|
||
--- a/engine/tabsqlitedb.py
|
||
+++ b/engine/tabsqlitedb.py
|
||
@@ -483,17 +483,27 @@ class tabsqlitedb:
|
||
# make sure that we got a unicode string
|
||
if type(phrase) != type(u''):
|
||
phrase = phrase.decode('utf8')
|
||
+ tmp_phrase = ''.join(re.findall(u'['
|
||
+ + u'\u4E00-\u9FCB'
|
||
+ + u'\u3400-\u4DB5'
|
||
+ + u'\uF900-\uFaFF'
|
||
+ + u'\U00020000-\U0002A6D6'
|
||
+ + u'\U0002A700-\U0002B734'
|
||
+ + u'\U0002B740-\U0002B81D'
|
||
+ + u'\U0002F800-\U0002FA1D'
|
||
+ + u']+',
|
||
+ phrase))
|
||
# first whether in gb2312
|
||
try:
|
||
- phrase.encode('gb2312')
|
||
+ tmp_phrase.encode('gb2312')
|
||
category |= 1
|
||
except:
|
||
- if '〇'.decode('utf8') in phrase:
|
||
+ if '〇'.decode('utf8') in tmp_phrase:
|
||
# we add '〇' into SC as well
|
||
category |= 1
|
||
# second check big5-hkscs
|
||
try:
|
||
- phrase.encode('big5hkscs')
|
||
+ tmp_phrase.encode('big5hkscs')
|
||
category |= 1 << 1
|
||
except:
|
||
# then check whether in gbk,
|
||
@@ -503,7 +513,7 @@ class tabsqlitedb:
|
||
else:
|
||
# need to check
|
||
try:
|
||
- phrase.encode('gbk')
|
||
+ tmp_phrase.encode('gbk')
|
||
category |= 1
|
||
except:
|
||
# not in gbk
|
||
--
|
||
1.7.11.4
|
||
|