1#
2#   Copyright (C) 2002-2006, International Business Machines Corporation and others.
3#       All Rights Reserved.
4#
5#   file:  sent.txt
6#
7#   ICU Sentence Break Rules
8#      See Unicode Standard Annex #29.
9#      These rules are based on SA 29 version 5.0.0
10#      Includes post 5.0 changes to treat Japanese half width voicing marks
11#        as Grapheme Extend.
12#
13
14
15$VoiceMarks   = [\uff9e\uff9f];
16$Thai         = [:Script = Thai:];
17
18#
19# Character categories as defined in TR 29
20#
21$Sep       = [\p{Sentence_Break = Sep}];
22$Format    = [\p{Sentence_Break = Format}];
23$Sp        = [\p{Sentence_Break = Sp}];
24$Lower     = [\p{Sentence_Break = Lower}];
25$Upper     = [\p{Sentence_Break = Upper}];
26$OLetter   = [\p{Sentence_Break = OLetter}-$VoiceMarks];
27$Numeric   = [\p{Sentence_Break = Numeric}];
28$ATerm     = [\p{Sentence_Break = ATerm}];
29$STerm     = [\p{Sentence_Break = STerm}];
30$Close     = [\p{Sentence_Break = Close}];
31
32#
33# Define extended forms of the character classes,
34#   incorporate grapheme cluster + format chars.
35#   Rules 4 and 5.
36
37
38$CR         = \u000d;
39$LF         = \u000a;
40$Extend     = [[:Grapheme_Extend = TRUE:]$VoiceMarks];
41
42$SpEx       = $Sp      ($Extend | $Format)*;
43$LowerEx    = $Lower   ($Extend | $Format)*;
44$UpperEx    = $Upper   ($Extend | $Format)*;
45$OLetterEx  = $OLetter ($Extend | $Format)*;
46$NumericEx  = $Numeric ($Extend | $Format)*;
47$ATermEx    = $ATerm   ($Extend | $Format)*;
48$STermEx    = $STerm   ($Extend | $Format)*;
49$CloseEx    = $Close   ($Extend | $Format)*;
50
51
52## -------------------------------------------------
53
54!!chain;
55!!forward;
56
57# Rule 3 - break after separators.  Keep CR/LF together.
58#
59$CR $LF;
60
61$LettersEx = [$OLetter $Upper $Lower $Numeric $Close $STerm] ($Extend | $Format)*;
62$LettersEx* $Thai $LettersEx* ($ATermEx | $SpEx)*;
63
64# Rule 4 - Break after $Sep.
65# Rule 5 - Ignore $Format and $Extend
66#
67[^$Sep]? ($Extend | $Format)*;
68
69
70# Rule 6
71$ATermEx $NumericEx;
72
73# Rule 7
74$UpperEx $ATermEx $UpperEx;
75
76#Rule 8
77#  Note:  follows errata for Unicode 5.0 boundary rules.
78$NotLettersEx = [^$OLetter $Upper $Lower $Sep $ATerm $STerm] ($Extend | $Format)*;
79$ATermEx $CloseEx* $SpEx* $NotLettersEx* $Lower;
80
81# Rule 8a
82($STermEx | $ATermEx) $CloseEx* $SpEx* ($STermEx | $ATermEx);
83
84#Rule 9, 10, 11
85($STermEx | $ATermEx) $CloseEx* $SpEx* $Sep?;
86
87#Rule 12
88[[^$STerm $ATerm $Close $Sp $Sep $Format $Extend $Thai]{bof}] ($Extend | $Format | $Close | $Sp)* [^$Thai];
89[[^$STerm $ATerm $Close $Sp $Sep $Format $Extend]{bof}] ($Extend | $Format | $Close | $Sp)* ([$Sep{eof}] | $CR $LF){100};
90
91## -------------------------------------------------
92
93!!reverse;
94
95$SpEx_R       = ($Extend | $Format)* $Sp;
96$ATermEx_R    = ($Extend | $Format)* $ATerm;
97$STermEx_R    = ($Extend | $Format)* $STerm;
98$CloseEx_R    = ($Extend | $Format)* $Close;
99
100#
101#  Reverse rules.
102#     For now, use the old style inexact reverse rules, which are easier
103#     to write, but less efficient.
104#     TODO:  exact reverse rules.  It appears that exact reverse rules
105#            may require improving support for look-ahead breaks in the
106#            builder.  Needs more investigation.
107#
108
109[{bof}] (.? | $LF $CR) [^$Sep]* [$Sep {eof}] ($SpEx_R* $CloseEx_R* ($STermEx_R | $ATermEx_R))*;
110#.*;
111
112# Explanation for this rule:
113#
114#    It needs to back over
115#        The $Sep at which we probably begin
116#        All of the non $Sep chars leading to the preceding $Sep
117#        The preceding $Sep, which will be the second one that the rule matches.
118#        Any immediately preceding STerm or ATerm sequences.  We need to see these
119#              to get the correct rule status when moving forwards again.
120#
121# [{bof}]           inhibit rule chaining.  Without this, rule would loop on itself and match
122#                   the entire string.
123#
124# (.? | $LF $CR)    Match one $Sep instance.  Use .? rather than $Sep because position might be
125#                   at the beginning of the string at this point, and we don't want to fail.
126#                   Can only use {eof} once, and it is used later.
127#
128
129