1#
2#   Copyright (C) 2002-2003, International Business Machines Corporation and others.
3#       All Rights Reserved.
4#
5#   file:  count_word.txt
6#
7#   ICU Word Break Rules
8#      See Unicode Standard Annex #29.
9#      These rules are based on Version 4.0.0, dated 2003-04-17
10#
11
12
13
14####################################################################################
15#
16#  Character class definitions from TR 29
17#
18####################################################################################
19$Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND MARK:]
20                                   [:name = HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK:]
21                                   [:name = HALFWIDTH KATAKANA VOICED SOUND MARK:]
22                                   [:name = HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK:]];
23
24
25$dash = \u002d;
26
27$ALetter   = [[:Alphabetic:] [:name= HEBREW PUNCTUATION GERESH:]
28                           [:P:] [:S:] [:LineBreak = Numeric:]
29                           - $dash
30                           - $Katakana
31                           - [:Script = Thai:]
32                           - [:Script = Lao:]
33                           - [:Script = Hiragana:]];
34
35$TheZWSP = \u200b;
36
37#
38#  Character Class Definitions.
39#    The names are those from TR29.
40#
41$CR         = \u000d;
42$LF         = \u000a;
43$Control    = [[[:Zl:] [:Zp:] [:Cc:] [:Cf:]] - $TheZWSP];
44$Extend     = [[:Grapheme_Extend = TRUE:]];
45
46
47
48
49####################################################################################
50#
51#  Word Break Rules.    Definitions and Rules specific to word break begin Here.
52#
53####################################################################################
54
55$Format    = [[:Cf:] - $TheZWSP];
56
57
58
59# Rule 3:  Treat a grapheme cluster as if it were a single character.
60#          Hangul Syllables are easier to deal with here than they are in Grapheme Clusters
61#          because we don't need to find the boundaries between adjacent syllables -
62#          they won't be word boundaries.
63#
64
65
66#
67#  "Extended"  definitions.  Grapheme Cluster + Format Chars, treated like the base char.
68#
69$ALetterEx    = $ALetter   $Extend*;
70$KatakanaEx   = $Katakana  $Extend*;
71$FormatEx     = $Format    $Extend*;
72
73#
74#  Numbers.  Rules 8, 11, 12 form the TR.
75#
76
77#
78#  Words.  Alpha-numerics.  Rule 5, 6, 7, 9, 10
79#     - must include at least one letter.
80#     - may include both letters and numbers.
81#     - may include  MideLetter, MidNumber punctuation.
82#
83$LetterSequence = $ALetterEx ($FormatEx* $ALetterEx)*;     # rules #6, #7
84$LetterSequence {200};
85
86$ALetterEx* $dash+ {200};
87$ALetterEx* ($dash $LetterSequence)+ $dash* {200};
88
89#
90#  Do not break between Katakana.   Rule #13.
91#
92$KatakanaEx ($FormatEx* $KatakanaEx)* {300};
93[:Hiragana:] $Extend* {300};
94
95#
96#  Ideographic Characters.  Stand by themselves as words.
97#                           Separated from the "Everything Else" rule, below, only so that they
98#                           can be tagged with a return value.   TODO:  is this what we want?
99#
100# [:IDEOGRAPHIC:] $Extend* {400};
101
102#
103#  Everything Else, with no tag.
104#                   Non-Control chars combine with $Extend (combining) chars.
105#                   Controls are do not.
106#
107[^$Control [:Ideographic:]] $Extend*;
108$CR $LF;
109
110#
111#  Reverse Rules.   Back up over any of the chars that can group together.
112#                   (Reverse rules do not need to be exact; they can back up  too far,
113#                   but must back up at least enough, and must stop on a boundary.)
114#
115
116# NonStarters are the set of all characters that can appear at the 2nd - nth position of
117#    a word.   (They may also be the first.)   The reverse rule skips over these, until it
118#    reaches something that can only be the start (and probably only) char in a "word".
119#    A space or punctuation meets the test.
120#
121$NonStarters = [$ALetter $Katakana $Extend $Format];
122
123#!.*;
124! ($NonStarters* | \n \r) .;
125
126