1#
2#   Copyright (C) 2002-2003, International Business Machines Corporation and others.
3#       All Rights Reserved.
4#
5#   file:  count_word.txt
6#
7#   ICU Word Break Rules
8#      See Unicode Standard Annex #29.
9#      These rules are based on Version 4.0.0, dated 2003-04-17
10#
11
12
13
14####################################################################################
15#
16#  Character class definitions from TR 29
17#
18####################################################################################
19$Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND MARK:]
20                                   [:name = HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK:]
21                                   [:name = HALFWIDTH KATAKANA VOICED SOUND MARK:]
22                                   [:name = HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK:]];
23
24$ALetter   = [[:Alphabetic:] [:name= HEBREW PUNCTUATION GERESH:] [:name = HYPHEN-MINUS:]
25                           - $Katakana
26                           - [:Script = Thai:]
27                           - [:Script = Lao:]
28                           - [:Script = Hiragana:]];
29
30$MidLetter = [[:name = APOSTROPHE:] [:name = MIDDLE DOT:]  [:name = HEBREW PUNCTUATION GERSHAYIM:]
31              [:name = RIGHT SINGLE QUOTATION MARK:] [:name = HYPHENATION POINT:] [:name = COLON:]];
32
33$MidNumLet = [:name = FULL STOP:];
34
35$MidNum    = [[:LineBreak = Infix_Numeric:] - $MidNumLet];
36$Numeric   = [:LineBreak = Numeric:];
37
38
39$TheZWSP = \u200b;
40
41#
42#  Character Class Definitions.
43#    The names are those from TR29.
44#
45$CR         = \u000d;
46$LF         = \u000a;
47$Control    = [[[:Zl:] [:Zp:] [:Cc:] [:Cf:]] - $TheZWSP];
48$Extend     = [[:Grapheme_Extend = TRUE:]];
49
50
51
52
53####################################################################################
54#
55#  Word Break Rules.    Definitions and Rules specific to word break begin Here.
56#
57####################################################################################
58
59$Format    = [[:Cf:] - $TheZWSP];
60
61
62
63# Rule 3:  Treat a grapheme cluster as if it were a single character.
64#          Hangul Syllables are easier to deal with here than they are in Grapheme Clusters
65#          because we don't need to find the boundaries between adjacent syllables -
66#          they won't be word boundaries.
67#
68
69
70#
71#  "Extended"  definitions.  Grapheme Cluster + Format Chars, treated like the base char.
72#
73$ALetterEx    = $ALetter   $Extend*;
74$NumericEx    = $Numeric   $Extend*;
75$MidNumEx     = $MidNum    $Extend*;
76$MidNumLetEx  = $MidNumLet $Extend*;
77$MidLetterEx  = $MidLetter $Extend*;
78$KatakanaEx   = $Katakana  $Extend*;
79$FormatEx     = $Format    $Extend*;
80
81$word_pad=[[:P:][:S:][:Z:][:C:]];
82
83#
84#  Numbers.  Rules 8, 11, 12 form the TR.
85#
86$NumberSequence = $NumericEx ($FormatEx* ($MidNumEx | $MidNumLetEx)? $FormatEx* $NumericEx)*;
87$NumberSequence $word_pad* {100};
88
89#
90#  Words.  Alpha-numerics.  Rule 5, 6, 7, 9, 10
91#     - must include at least one letter.
92#     - may include both letters and numbers.
93#     - may include  MideLetter, MidNumber punctuation.
94#
95$LetterSequence = $ALetterEx ($FormatEx* ($MidLetterEx | $MidNumLetEx)? $FormatEx* $ALetterEx)*;     # rules #6, #7
96($NumberSequence $FormatEx*)? $LetterSequence ($FormatEx* ($NumberSequence | $LetterSequence))* $word_pad* {200};
97
98#
99#  Do not break between Katakana.   Rule #13.
100#
101$KatakanaEx ($FormatEx* $KatakanaEx)* {300};
102[:Hiragana:] $Extend* {300};
103
104#
105#  Ideographic Characters.  Stand by themselves as words.
106#                           Separated from the "Everything Else" rule, below, only so that they
107#                           can be tagged with a return value.   TODO:  is this what we want?
108#
109# [:IDEOGRAPHIC:] $Extend* $word_pad* {400};
110
111#
112#  Everything Else, with no tag.
113#                   Non-Control chars combine with $Extend (combining) chars.
114#                   Controls are do not.
115#
116[^$Control [:Ideographic:]] $Extend* $word_pad*;
117$CR $LF;
118
119#
120#  Reverse Rules.   Back up over any of the chars that can group together.
121#                   (Reverse rules do not need to be exact; they can back up  too far,
122#                   but must back up at least enough, and must stop on a boundary.)
123#
124
125# NonStarters are the set of all characters that can appear at the 2nd - nth position of
126#    a word.   (They may also be the first.)   The reverse rule skips over these, until it
127#    reaches something that can only be the start (and probably only) char in a "word".
128#    A space or punctuation meets the test.
129#
130$NonStarters = [$Numeric $ALetter $Katakana $MidLetter $MidNum $MidNumLet $Extend $Format];
131
132#!.*;
133! ($NonStarters* | \n \r) .;
134
135