1#
2#   Copyright (C) 2002-2003, International Business Machines Corporation and others.
3#       All Rights Reserved.
4#
5#   file:  dict_word.txt
6#
7#   ICU Word Break Rules
8#      See Unicode Standard Annex #29.
9#      These rules are based on Version 4.0.0, dated 2003-04-17
10#
11
12
13
14####################################################################################
15#
16#  Character class definitions from TR 29
17#
18####################################################################################
19$Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND MARK:]
20                                   [:name = HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK:]
21                                   [:name = HALFWIDTH KATAKANA VOICED SOUND MARK:]
22                                   [:name = HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK:]];
23
24$Ideographic = [:Ideographic:];
25$Hangul = [:Script = HANGUL:];
26
27# list of dashes or hyphens that should be accepted as part of the word if a single one of these
28# pre- or postfixes a word. E.g. in German: "Arbeits-" or "-nehmer" where that hyphen needs to
29# be part of the word in order to have it properly spell checked etc.
30$PrePostDashHyphen = [ [:name = HYPHEN-MINUS:] ];
31
32
33$ALetter   = [\u0002 [:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
34                           - $Ideographic
35                           - $Katakana
36                           - $Hangul
37                           - [:Script = Thai:]
38                           - [:Script = Lao:]
39                           - [:Script = Hiragana:]];
40
41$MidLetter = [[:name = APOSTROPHE:] [:name = GRAVE ACCENT:] \u0084 [:name = SOFT HYPHEN:] [:name = MIDDLE DOT:] [:name = GREEK TONOS:] [:name= FULL STOP:]
42              [:name = HEBREW PUNCTUATION GERSHAYIM:] [:name = DOUBLE VERTICAL LINE:] [:name = LEFT SINGLE QUOTATION MARK:]
43              [:name = RIGHT SINGLE QUOTATION MARK:] [:name = HYPHENATION POINT:] [:name = PRIME:]
44              [:name = HYPHEN-MINUS:] ];
45
46$SufixLetter = [:name= FULL STOP:];
47
48
49$MidNum    = [[:LineBreak = Infix_Numeric:] [:name= COMMERCIAL AT:] \u0084 [:name = GREEK TONOS:] [:name = ARABIC DECIMAL SEPARATOR:]
50             [:name = LEFT SINGLE QUOTATION MARK:] [:name = RIGHT SINGLE QUOTATION MARK:] [:name = SINGLE HIGH-REVERSED-9 QUOTATION MARK:]
51             [:name = PRIME:]];
52$Numeric   = [:LineBreak = Numeric:];
53
54
55$TheZWSP = \u200b;
56
57#
58#  Character Class Definitions.
59#    The names are those from TR29.
60#
61$CR         = \u000d;
62$LF         = \u000a;
63$Control    = [[[:Zl:] [:Zp:] [:Cc:] [:Cf:]] - $TheZWSP];
64$Extend     = [[:Grapheme_Extend = TRUE:]];
65
66
67
68
69####################################################################################
70#
71#  Word Break Rules.    Definitions and Rules specific to word break begin Here.
72#
73####################################################################################
74
75$Format    = [[:Cf:] - $TheZWSP];
76
77
78
79# Rule 3:  Treat a grapheme cluster as if it were a single character.
80#          Hangul Syllables are easier to deal with here than they are in Grapheme Clusters
81#          because we don't need to find the boundaries between adjacent syllables -
82#          they won't be word boundaries.
83#
84
85
86#
87#  "Extended"  definitions.  Grapheme Cluster + Format Chars, treated like the base char.
88#
89$ALetterEx    = $ALetter   $Extend*;
90$NumericEx    = $Numeric   $Extend*;
91$MidNumEx     = $MidNum    $Extend*;
92$MidLetterEx  = $MidLetter $Extend*;
93$SufixLetterEx= $SufixLetter $Extend*;
94$KatakanaEx   = $Katakana  $Extend*;
95$IdeographicEx= $Ideographic  $Extend*;
96$HangulEx = $Hangul  $Extend*;
97$FormatEx     = $Format    $Extend*;
98
99
100#
101#  Numbers.  Rules 8, 11, 12 form the TR.
102#
103$NumberSequence = $NumericEx ($FormatEx* $MidNumEx? $FormatEx* $NumericEx)*;
104$NumberSequence {100};
105
106#
107#  Words.  Alpha-numerics.  Rule 5, 6, 7, 9, 10
108#     - must include at least one letter.
109#     - may include both letters and numbers.
110#     - may include  MideLetter, MidNumber punctuation.
111#
112# At most one leading or trailing dash/hyphen should be accepted as well.
113# E.g. in German: "Arbeits-" or "-nehmer" where that hyphen needs to
114# be part of the word in order to have it properly spell checked etc.
115$LetterSequence = $PrePostDashHyphen? $ALetterEx ($FormatEx* $MidLetterEx? $FormatEx* $ALetterEx)* $PrePostDashHyphen?;     # rules #6, #7
116($NumberSequence $FormatEx*)? $LetterSequence ($FormatEx* ($NumberSequence | $LetterSequence))* $SufixLetterEx? {200};
117
118[[:P:][:S:]]*;
119
120#
121#  Do not break between Katakana.   Rule #13.
122#
123$KatakanaEx ($FormatEx* $KatakanaEx)* {300};
124[:Hiragana:] $Extend* {300};
125
126#
127#  Ideographic Characters.  Stand by themselves as words.
128#                           Separated from the "Everything Else" rule, below, only so that they
129#                           can be tagged with a return value.   TODO:  is this what we want?
130#
131$IdeographicEx ($FormatEx* $IdeographicEx)* {400};
132$HangulEx ($FormatEx* $HangulEx)* {400};
133
134#
135#  Everything Else, with no tag.
136#                   Non-Control chars combine with $Extend (combining) chars.
137#                   Controls are do not.
138#
139[^$Control [:Ideographic:]] $Extend*;
140$CR $LF;
141
142#
143#  Reverse Rules.   Back up over any of the chars that can group together.
144#                   (Reverse rules do not need to be exact; they can back up  too far,
145#                   but must back up at least enough, and must stop on a boundary.)
146#
147
148# NonStarters are the set of all characters that can appear at the 2nd - nth position of
149#    a word.   (They may also be the first.)   The reverse rule skips over these, until it
150#    reaches something that can only be the start (and probably only) char in a "word".
151#    A space or punctuation meets the test.
152#
153$NonStarters = [$Numeric $ALetter $Katakana $Ideographic $Hangul [:P:] [:S:] $MidLetter $MidNum $SufixLetter $Extend $Format];
154
155#!.*;
156! ($NonStarters* | \n \r) .;
157
158