diff --git a/include/m_ctype.h b/include/m_ctype.h index 279e8a74ddc..e3fd1668eb9 100644 --- a/include/m_ctype.h +++ b/include/m_ctype.h @@ -225,6 +225,7 @@ extern MY_UNI_CTYPE my_uni_ctype[256]; #define MY_CS_NON1TO1 0x40000 /* Has a complex mapping from characters to weights, e.g. contractions, expansions, ignorable characters */ +#define MY_CS_NUMERIC 0x80000 /* Natural sorting of numbers */ #define MY_CHARSET_UNDEFINED 0 /* Character repertoire flags */ diff --git a/strings/ctype-uca.c b/strings/ctype-uca.c index 2bcc16bb734..faf491a3846 100644 --- a/strings/ctype-uca.c +++ b/strings/ctype-uca.c @@ -30527,6 +30527,7 @@ static const char vietnamese[]= http://unicode.org/cldr/trac/browser/trunk/common/collation/my.xml */ static const char myanmar[]= "[shift-after-method expand][version 5.2.0]" + "[numericOrdering on]" /* Tones */ "&\\u108C" "<\\u1037" @@ -31152,7 +31153,7 @@ typedef struct my_uca_scanner_st const uchar *sbeg; /* Beginning of the input string */ const uchar *send; /* End of the input string */ const MY_UCA_WEIGHT_LEVEL *level; - uint16 implicit[2]; + uint16 implicit[6]; int page; int code; CHARSET_INFO *cs; @@ -31178,6 +31179,19 @@ static const uint16 nochar[]= {0,0}; /********** Helper functions to handle contraction ************/ +static inline int +my_space_weight(const MY_UCA_WEIGHT_LEVEL *level) +{ + return level->weights[0][0x20 * level->lengths[0]]; +} + +static inline int +my_digit_zero_weight(const MY_UCA_WEIGHT_LEVEL *level) +{ + return level->weights[0][0x30 * level->lengths[0]]; +} + + /** Mark a character as a contraction part @@ -31423,11 +31437,14 @@ my_uca_can_be_previous_context_tail(const MY_CONTRACTIONS *list, my_wc_t wc) */ static inline my_bool -my_uca_needs_context_handling(const MY_UCA_WEIGHT_LEVEL *level, my_wc_t wc) +my_uca_needs_context_handling(const my_uca_scanner *scanner, my_wc_t wc) { - return level->contractions.nitems > 0 && - level->contractions.flags[wc & MY_UCA_CNT_FLAG_MASK] & - (MY_UCA_PREVIOUS_CONTEXT_TAIL | MY_UCA_CNT_HEAD); + return (wc >= '0' && wc <= '9' && + scanner->cs->state & MY_CS_NUMERIC && + scanner->level->levelno == 0) || + (scanner->level->contractions.nitems > 0 && + scanner->level->contractions.flags[wc & MY_UCA_CNT_FLAG_MASK] & + (MY_UCA_PREVIOUS_CONTEXT_TAIL | MY_UCA_CNT_HEAD)); } @@ -31616,6 +31633,59 @@ my_uca_context_weight_find(my_uca_scanner *scanner, my_wc_t *wc) if ((cweight= my_uca_scanner_contraction_find(scanner, wc))) return cweight; } + /* Handle natural number sorting, on the primary level only */ + if ((scanner->cs->state & MY_CS_NUMERIC) && + scanner->level->levelno == 0 && + wc[0] >= '0' && wc[0] <= '9') + { + /* TODO: add arbitrary length instead of uint32 fitting number only */ + /* TODO: support all Unicode digits from the Nd category, not only ASCII */ + uint32 num= wc[0] - '0'; + const uchar *s; + size_t significant_digit_count= wc[0] > '0'; /* Ignore leading zero */ + for (s= scanner->sbeg; ; ) + { + int mblen; + my_wc_t wc2; + if ((mblen= my_ci_mb_wc(scanner->cs, &wc2, s, scanner->send)) <= 0 || + wc2 < '0' || wc2 > '9') + break; + num= (num * 10) + wc2-'0'; + s+= mblen; + if (significant_digit_count || wc2 > '0') + significant_digit_count++; + } + scanner->sbeg= s; + /* + First, put the weight of the digit zero. + This is needed to have natual sorting sort numbers + near their usual UCA position, + between currency signs and latin letters. + */ + scanner->implicit[0]= my_digit_zero_weight(scanner->level); + /* + Now let's encode the digits using weights from the primary range + (e.g. starting from 0x0200). Lower weights won't work well in + multi-level collations. + TODO: 0x0200 may depend on the UCA version. + */ + /* + First, put the total length of the digit sequence. + TODO: the below may overflow if the digit sequence is too long. + */ + scanner->implicit[1]= 0x0200 + significant_digit_count; + /* + Now encode the number into three weights. + uint32 range: 1024 * 2048 * 2048 + */ + scanner->implicit[2]= 0x0200 + (num / 2048 / 2048); + num= num % (2048*2048); + scanner->implicit[3]= 0x0200 + (num / 2048); + scanner->implicit[4]= 0x0200 + (num % 2048); + scanner->implicit[5]= 0; /* End of the weight string */ + scanner->wbeg= scanner->implicit + 1; + return (uint16*) scanner->implicit; + } return NULL; } @@ -31742,13 +31812,6 @@ my_uca_scanner_init_any(my_uca_scanner *scanner, } -static inline int -my_space_weight(const MY_UCA_WEIGHT_LEVEL *level) -{ - return level->weights[0][0x20 * level->lengths[0]]; -} - - /** Helper function: Find address of weights of the given character. @@ -32495,6 +32558,7 @@ typedef struct my_coll_rules_st MY_COLL_RULE *rule; /* Rule array */ MY_CHARSET_LOADER *loader; my_coll_shift_method shift_after_method; + my_bool numeric; } MY_COLL_RULES; @@ -32774,6 +32838,10 @@ my_coll_parser_scan_setting(MY_COLL_RULE_PARSER *p) rules->strength= 1; else if (!lex_cmp(lexem, C_STRING_WITH_LEN("[strength 2]"))) rules->strength= 2; + else if (!lex_cmp(lexem, C_STRING_WITH_LEN("[numericOrdering on]"))) + rules->numeric= TRUE; + else if (!lex_cmp(lexem, C_STRING_WITH_LEN("[numericOrdering off]"))) + rules->numeric= FALSE; else { return 0; @@ -33863,6 +33931,8 @@ create_tailoring(struct charset_info_st *cs, goto ex; } cs->uca[0]= new_uca; + if (rules.numeric) + cs->state |= MY_CS_NUMERIC; if (cs->levels_for_order > 1) cs->coll= (cs->state & MY_CS_NOPAD) ? &my_uca_collation_handler_nopad_multilevel_generic : diff --git a/strings/ctype-uca.ic b/strings/ctype-uca.ic index e47f1e1fd82..f9a9ba0a246 100644 --- a/strings/ctype-uca.ic +++ b/strings/ctype-uca.ic @@ -63,7 +63,7 @@ MY_FUNCTION_NAME(scanner_next)(my_uca_scanner *scanner) scanner->sbeg+= 1; #if MY_UCA_COMPILE_CONTRACTIONS - if (my_uca_needs_context_handling(scanner->level, wc[0])) + if (my_uca_needs_context_handling(scanner, wc[0])) { uint16 *cweight= my_uca_context_weight_find(scanner, wc); if (cweight) @@ -112,7 +112,7 @@ MY_FUNCTION_NAME(scanner_next)(my_uca_scanner *scanner) } #if MY_UCA_COMPILE_CONTRACTIONS - if (my_uca_needs_context_handling(scanner->level, wc[0])) + if (my_uca_needs_context_handling(scanner, wc[0])) { uint16 *cweight= my_uca_context_weight_find(scanner, wc); if (cweight)