1*b1cdbd2cSJim Jagielski# 2*b1cdbd2cSJim Jagielski# Copyright (C) 2002-2003, International Business Machines Corporation and others. 3*b1cdbd2cSJim Jagielski# All Rights Reserved. 4*b1cdbd2cSJim Jagielski# 5*b1cdbd2cSJim Jagielski# file: edit_word.txt 6*b1cdbd2cSJim Jagielski# 7*b1cdbd2cSJim Jagielski# ICU Word Break Rules 8*b1cdbd2cSJim Jagielski# See Unicode Standard Annex #29. 9*b1cdbd2cSJim Jagielski# These rules are based on Version 4.0.0, dated 2003-04-17 10*b1cdbd2cSJim Jagielski# 11*b1cdbd2cSJim Jagielski 12*b1cdbd2cSJim Jagielski 13*b1cdbd2cSJim Jagielski 14*b1cdbd2cSJim Jagielski#################################################################################### 15*b1cdbd2cSJim Jagielski# 16*b1cdbd2cSJim Jagielski# Character class definitions from TR 29 17*b1cdbd2cSJim Jagielski# 18*b1cdbd2cSJim Jagielski#################################################################################### 19*b1cdbd2cSJim Jagielski$Katakana = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND MARK:] 20*b1cdbd2cSJim Jagielski [:name = HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK:] 21*b1cdbd2cSJim Jagielski [:name = HALFWIDTH KATAKANA VOICED SOUND MARK:] 22*b1cdbd2cSJim Jagielski [:name = HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK:]]; 23*b1cdbd2cSJim Jagielski 24*b1cdbd2cSJim Jagielski$Ideographic = [:Ideographic:]; 25*b1cdbd2cSJim Jagielski$Hangul = [:Script = HANGUL:]; 26*b1cdbd2cSJim Jagielski 27*b1cdbd2cSJim Jagielski$ALetter = [\u0002 [:Alphabetic:] [:name= NO-BREAK SPACE:] [:name= HEBREW PUNCTUATION GERESH:] 28*b1cdbd2cSJim Jagielski [:name = PERCENT SIGN:] [:name = PER MILLE SIGN:] [:name = PER TEN THOUSAND SIGN:] 29*b1cdbd2cSJim Jagielski [:name = SECTION SIGN:] [:name = DEGREE SIGN:] [:name = EURO SIGN:] 30*b1cdbd2cSJim Jagielski [:name = HYPHEN-MINUS:] [:name = EN DASH:] [:name = EM DASH:] 31*b1cdbd2cSJim Jagielski [:name = DIGIT ZERO:] 32*b1cdbd2cSJim Jagielski [:name = DIGIT ONE:] 33*b1cdbd2cSJim Jagielski [:name = DIGIT TWO:] 34*b1cdbd2cSJim Jagielski [:name = DIGIT THREE:] 35*b1cdbd2cSJim Jagielski [:name = DIGIT FOUR:] 36*b1cdbd2cSJim Jagielski [:name = DIGIT FIVE:] 37*b1cdbd2cSJim Jagielski [:name = DIGIT SIX:] 38*b1cdbd2cSJim Jagielski [:name = DIGIT SEVEN:] 39*b1cdbd2cSJim Jagielski [:name = DIGIT EIGHT:] 40*b1cdbd2cSJim Jagielski [:name = DIGIT NINE:] 41*b1cdbd2cSJim Jagielski - $Ideographic 42*b1cdbd2cSJim Jagielski - $Katakana 43*b1cdbd2cSJim Jagielski - $Hangul 44*b1cdbd2cSJim Jagielski - [:Script = Thai:] 45*b1cdbd2cSJim Jagielski - [:Script = Lao:] 46*b1cdbd2cSJim Jagielski - [:Script = Hiragana:]]; 47*b1cdbd2cSJim Jagielski 48*b1cdbd2cSJim Jagielski$MidLetter = [[:name = APOSTROPHE:] [:name = MIDDLE DOT:] [:name = HEBREW PUNCTUATION GERSHAYIM:] 49*b1cdbd2cSJim Jagielski [:name = RIGHT SINGLE QUOTATION MARK:] [:name = HYPHENATION POINT:] 50*b1cdbd2cSJim Jagielski [:name = HYPHEN-MINUS:] [:name = EURO SIGN:] [:name = PERCENT SIGN:] 51*b1cdbd2cSJim Jagielski [:name = PER MILLE SIGN:] [:name = PER TEN THOUSAND SIGN:] 52*b1cdbd2cSJim Jagielski [:name = EN DASH:] [:name = EM DASH:] 53*b1cdbd2cSJim Jagielski [:name = PERCENT SIGN:] [:name = SECTION SIGN:] [:name = DEGREE SIGN:]]; 54*b1cdbd2cSJim Jagielski 55*b1cdbd2cSJim Jagielski$MidNum = [[:LineBreak = Infix_Numeric:] - [:name = FULL STOP:]]; 56*b1cdbd2cSJim Jagielski$Numeric = [:LineBreak = Numeric:]; 57*b1cdbd2cSJim Jagielski 58*b1cdbd2cSJim Jagielski 59*b1cdbd2cSJim Jagielski$TheZWSP = \u200b; 60*b1cdbd2cSJim Jagielski 61*b1cdbd2cSJim Jagielski# 62*b1cdbd2cSJim Jagielski# Character Class Definitions. 63*b1cdbd2cSJim Jagielski# The names are those from TR29. 64*b1cdbd2cSJim Jagielski# 65*b1cdbd2cSJim Jagielski$CR = \u000d; 66*b1cdbd2cSJim Jagielski$LF = \u000a; 67*b1cdbd2cSJim Jagielski$Control = [[[:Zl:] [:Zp:] [:Cc:] [:Cf:]] - $TheZWSP]; 68*b1cdbd2cSJim Jagielski$Extend = [[:Grapheme_Extend = TRUE:]]; 69*b1cdbd2cSJim Jagielski 70*b1cdbd2cSJim Jagielski 71*b1cdbd2cSJim Jagielski 72*b1cdbd2cSJim Jagielski 73*b1cdbd2cSJim Jagielski#################################################################################### 74*b1cdbd2cSJim Jagielski# 75*b1cdbd2cSJim Jagielski# Word Break Rules. Definitions and Rules specific to word break begin Here. 76*b1cdbd2cSJim Jagielski# 77*b1cdbd2cSJim Jagielski#################################################################################### 78*b1cdbd2cSJim Jagielski 79*b1cdbd2cSJim Jagielski$Format = [[:Cf:] - $TheZWSP]; 80*b1cdbd2cSJim Jagielski 81*b1cdbd2cSJim Jagielski 82*b1cdbd2cSJim Jagielski 83*b1cdbd2cSJim Jagielski# Rule 3: Treat a grapheme cluster as if it were a single character. 84*b1cdbd2cSJim Jagielski# Hangul Syllables are easier to deal with here than they are in Grapheme Clusters 85*b1cdbd2cSJim Jagielski# because we don't need to find the boundaries between adjacent syllables - 86*b1cdbd2cSJim Jagielski# they won't be word boundaries. 87*b1cdbd2cSJim Jagielski# 88*b1cdbd2cSJim Jagielski 89*b1cdbd2cSJim Jagielski 90*b1cdbd2cSJim Jagielski# 91*b1cdbd2cSJim Jagielski# "Extended" definitions. Grapheme Cluster + Format Chars, treated like the base char. 92*b1cdbd2cSJim Jagielski# 93*b1cdbd2cSJim Jagielski$ALetterEx = $ALetter $Extend*; 94*b1cdbd2cSJim Jagielski$NumericEx = $Numeric $Extend*; 95*b1cdbd2cSJim Jagielski$MidNumEx = $MidNum $Extend*; 96*b1cdbd2cSJim Jagielski$MidLetterEx = $MidLetter $Extend*; 97*b1cdbd2cSJim Jagielski$KatakanaEx = $Katakana $Extend*; 98*b1cdbd2cSJim Jagielski$IdeographicEx= $Ideographic $Extend*; 99*b1cdbd2cSJim Jagielski$HangulEx = $Hangul $Extend*; 100*b1cdbd2cSJim Jagielski$FormatEx = $Format $Extend*; 101*b1cdbd2cSJim Jagielski 102*b1cdbd2cSJim Jagielski 103*b1cdbd2cSJim Jagielski# 104*b1cdbd2cSJim Jagielski# Numbers. Rules 8, 11, 12 form the TR. 105*b1cdbd2cSJim Jagielski# 106*b1cdbd2cSJim Jagielski$NumberSequence = $NumericEx ($FormatEx* $MidNumEx? $FormatEx* $NumericEx)*; 107*b1cdbd2cSJim Jagielski$NumberSequence {100}; 108*b1cdbd2cSJim Jagielski 109*b1cdbd2cSJim Jagielski# 110*b1cdbd2cSJim Jagielski# Words. Alpha-numerics. Rule 5, 6, 7, 9, 10 111*b1cdbd2cSJim Jagielski# - must include at least one letter. 112*b1cdbd2cSJim Jagielski# - may include both letters and numbers. 113*b1cdbd2cSJim Jagielski# - may include MideLetter, MidNumber punctuation. 114*b1cdbd2cSJim Jagielski# 115*b1cdbd2cSJim Jagielski$LetterSequence = $ALetterEx ($FormatEx* $MidLetterEx? $FormatEx* $ALetterEx)*; # rules #6, #7 116*b1cdbd2cSJim Jagielski($NumberSequence $FormatEx*)? $LetterSequence ($FormatEx* ($NumberSequence | $LetterSequence))* {200}; 117*b1cdbd2cSJim Jagielski 118*b1cdbd2cSJim Jagielski# Punctuations by themselves 119*b1cdbd2cSJim Jagielski[[:P:][:S:]-[:name = FULL STOP:]]*; 120*b1cdbd2cSJim Jagielski[[:name = FULL STOP:]]*; 121*b1cdbd2cSJim Jagielski 122*b1cdbd2cSJim Jagielski# 123*b1cdbd2cSJim Jagielski# Do not break between Katakana. Rule #13. 124*b1cdbd2cSJim Jagielski# 125*b1cdbd2cSJim Jagielski$KatakanaEx ($FormatEx* $KatakanaEx)* {300}; 126*b1cdbd2cSJim Jagielski[:Hiragana:] $Extend* {300}; 127*b1cdbd2cSJim Jagielski 128*b1cdbd2cSJim Jagielski# 129*b1cdbd2cSJim Jagielski# Ideographic Characters. Stand by themselves as words. 130*b1cdbd2cSJim Jagielski# Separated from the "Everything Else" rule, below, only so that they 131*b1cdbd2cSJim Jagielski# can be tagged with a return value. TODO: is this what we want? 132*b1cdbd2cSJim Jagielski# 133*b1cdbd2cSJim Jagielski$IdeographicEx ($FormatEx* $IdeographicEx)* {400}; 134*b1cdbd2cSJim Jagielski$HangulEx ($FormatEx* $HangulEx)* {400}; 135*b1cdbd2cSJim Jagielski 136*b1cdbd2cSJim Jagielski# 137*b1cdbd2cSJim Jagielski# Everything Else, with no tag. 138*b1cdbd2cSJim Jagielski# Non-Control chars combine with $Extend (combining) chars. 139*b1cdbd2cSJim Jagielski# Controls are do not. 140*b1cdbd2cSJim Jagielski# 141*b1cdbd2cSJim Jagielski[^$Control [:Ideographic:]] $Extend*; 142*b1cdbd2cSJim Jagielski$CR $LF; 143*b1cdbd2cSJim Jagielski 144*b1cdbd2cSJim Jagielski# 145*b1cdbd2cSJim Jagielski# Reverse Rules. Back up over any of the chars that can group together. 146*b1cdbd2cSJim Jagielski# (Reverse rules do not need to be exact; they can back up too far, 147*b1cdbd2cSJim Jagielski# but must back up at least enough, and must stop on a boundary.) 148*b1cdbd2cSJim Jagielski# 149*b1cdbd2cSJim Jagielski 150*b1cdbd2cSJim Jagielski# NonStarters are the set of all characters that can appear at the 2nd - nth position of 151*b1cdbd2cSJim Jagielski# a word. (They may also be the first.) The reverse rule skips over these, until it 152*b1cdbd2cSJim Jagielski# reaches something that can only be the start (and probably only) char in a "word". 153*b1cdbd2cSJim Jagielski# A space or punctuation meets the test. 154*b1cdbd2cSJim Jagielski# 155*b1cdbd2cSJim Jagielski$NonStarters = [$Numeric $ALetter $Katakana $Ideographic $Hangul [:P:] [:S:] $MidLetter $MidNum $Extend $Format]; 156*b1cdbd2cSJim Jagielski 157*b1cdbd2cSJim Jagielski#!.*; 158*b1cdbd2cSJim Jagielski! ($NonStarters* | \n \r) .; 159*b1cdbd2cSJim Jagielski 160