1*b1cdbd2cSJim Jagielski# 2*b1cdbd2cSJim Jagielski# Copyright (C) 2002-2003, International Business Machines Corporation and others. 3*b1cdbd2cSJim Jagielski# All Rights Reserved. 4*b1cdbd2cSJim Jagielski# 5*b1cdbd2cSJim Jagielski# file: dict_word.txt 6*b1cdbd2cSJim Jagielski# 7*b1cdbd2cSJim Jagielski# ICU Word Break Rules 8*b1cdbd2cSJim Jagielski# See Unicode Standard Annex #29. 9*b1cdbd2cSJim Jagielski# These rules are based on Version 4.0.0, dated 2003-04-17 10*b1cdbd2cSJim Jagielski# 11*b1cdbd2cSJim Jagielski 12*b1cdbd2cSJim Jagielski 13*b1cdbd2cSJim Jagielski 14*b1cdbd2cSJim Jagielski#################################################################################### 15*b1cdbd2cSJim Jagielski# 16*b1cdbd2cSJim Jagielski# Character class definitions from TR 29 17*b1cdbd2cSJim Jagielski# 18*b1cdbd2cSJim Jagielski#################################################################################### 19*b1cdbd2cSJim Jagielski$Katakana = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND MARK:] 20*b1cdbd2cSJim Jagielski [:name = HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK:] 21*b1cdbd2cSJim Jagielski [:name = HALFWIDTH KATAKANA VOICED SOUND MARK:] 22*b1cdbd2cSJim Jagielski [:name = HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK:]]; 23*b1cdbd2cSJim Jagielski 24*b1cdbd2cSJim Jagielski$Ideographic = [:Ideographic:]; 25*b1cdbd2cSJim Jagielski$Hangul = [:Script = HANGUL:]; 26*b1cdbd2cSJim Jagielski 27*b1cdbd2cSJim Jagielski$ALetter = [\u0002 [:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:] 28*b1cdbd2cSJim Jagielski - $Ideographic 29*b1cdbd2cSJim Jagielski - $Katakana 30*b1cdbd2cSJim Jagielski - $Hangul 31*b1cdbd2cSJim Jagielski - [:Script = Thai:] 32*b1cdbd2cSJim Jagielski - [:Script = Lao:] 33*b1cdbd2cSJim Jagielski - [:Script = Hiragana:]]; 34*b1cdbd2cSJim Jagielski 35*b1cdbd2cSJim Jagielski$MidLetter = [[:name = APOSTROPHE:] [:name = GRAVE ACCENT:] \u0084 [:name = SOFT HYPHEN:] [:name = MIDDLE DOT:] [:name = GREEK TONOS:] [:name= FULL STOP:] 36*b1cdbd2cSJim Jagielski [:name = HEBREW PUNCTUATION GERSHAYIM:] [:name = DOUBLE VERTICAL LINE:] [:name = LEFT SINGLE QUOTATION MARK:] 37*b1cdbd2cSJim Jagielski [:name = RIGHT SINGLE QUOTATION MARK:] [:name = HYPHENATION POINT:] [:name = PRIME:] ]; 38*b1cdbd2cSJim Jagielski 39*b1cdbd2cSJim Jagielski$SufixLetter = [:name= FULL STOP:]; 40*b1cdbd2cSJim Jagielski 41*b1cdbd2cSJim Jagielski 42*b1cdbd2cSJim Jagielski$MidNum = [[:LineBreak = Infix_Numeric:] [:name= COMMERCIAL AT:] \u0084 [:name = GREEK TONOS:] [:name = ARABIC DECIMAL SEPARATOR:] 43*b1cdbd2cSJim Jagielski [:name = LEFT SINGLE QUOTATION MARK:] [:name = RIGHT SINGLE QUOTATION MARK:] [:name = SINGLE HIGH-REVERSED-9 QUOTATION MARK:] 44*b1cdbd2cSJim Jagielski [:name = PRIME:]]; 45*b1cdbd2cSJim Jagielski$Numeric = [:LineBreak = Numeric:]; 46*b1cdbd2cSJim Jagielski 47*b1cdbd2cSJim Jagielski 48*b1cdbd2cSJim Jagielski$TheZWSP = \u200b; 49*b1cdbd2cSJim Jagielski 50*b1cdbd2cSJim Jagielski# 51*b1cdbd2cSJim Jagielski# Character Class Definitions. 52*b1cdbd2cSJim Jagielski# The names are those from TR29. 53*b1cdbd2cSJim Jagielski# 54*b1cdbd2cSJim Jagielski$CR = \u000d; 55*b1cdbd2cSJim Jagielski$LF = \u000a; 56*b1cdbd2cSJim Jagielski$Control = [[[:Zl:] [:Zp:] [:Cc:] [:Cf:]] - $TheZWSP]; 57*b1cdbd2cSJim Jagielski$Extend = [[:Grapheme_Extend = TRUE:]]; 58*b1cdbd2cSJim Jagielski 59*b1cdbd2cSJim Jagielski 60*b1cdbd2cSJim Jagielski 61*b1cdbd2cSJim Jagielski 62*b1cdbd2cSJim Jagielski#################################################################################### 63*b1cdbd2cSJim Jagielski# 64*b1cdbd2cSJim Jagielski# Word Break Rules. Definitions and Rules specific to word break begin Here. 65*b1cdbd2cSJim Jagielski# 66*b1cdbd2cSJim Jagielski#################################################################################### 67*b1cdbd2cSJim Jagielski 68*b1cdbd2cSJim Jagielski$Format = [[:Cf:] - $TheZWSP]; 69*b1cdbd2cSJim Jagielski 70*b1cdbd2cSJim Jagielski 71*b1cdbd2cSJim Jagielski 72*b1cdbd2cSJim Jagielski# Rule 3: Treat a grapheme cluster as if it were a single character. 73*b1cdbd2cSJim Jagielski# Hangul Syllables are easier to deal with here than they are in Grapheme Clusters 74*b1cdbd2cSJim Jagielski# because we don't need to find the boundaries between adjacent syllables - 75*b1cdbd2cSJim Jagielski# they won't be word boundaries. 76*b1cdbd2cSJim Jagielski# 77*b1cdbd2cSJim Jagielski 78*b1cdbd2cSJim Jagielski 79*b1cdbd2cSJim Jagielski# 80*b1cdbd2cSJim Jagielski# "Extended" definitions. Grapheme Cluster + Format Chars, treated like the base char. 81*b1cdbd2cSJim Jagielski# 82*b1cdbd2cSJim Jagielski$ALetterEx = $ALetter $Extend*; 83*b1cdbd2cSJim Jagielski$NumericEx = $Numeric $Extend*; 84*b1cdbd2cSJim Jagielski$MidNumEx = $MidNum $Extend*; 85*b1cdbd2cSJim Jagielski$MidLetterEx = $MidLetter $Extend*; 86*b1cdbd2cSJim Jagielski$SufixLetterEx= $SufixLetter $Extend*; 87*b1cdbd2cSJim Jagielski$KatakanaEx = $Katakana $Extend*; 88*b1cdbd2cSJim Jagielski$IdeographicEx= $Ideographic $Extend*; 89*b1cdbd2cSJim Jagielski$HangulEx = $Hangul $Extend*; 90*b1cdbd2cSJim Jagielski$FormatEx = $Format $Extend*; 91*b1cdbd2cSJim Jagielski 92*b1cdbd2cSJim Jagielski 93*b1cdbd2cSJim Jagielski# 94*b1cdbd2cSJim Jagielski# Numbers. Rules 8, 11, 12 form the TR. 95*b1cdbd2cSJim Jagielski# 96*b1cdbd2cSJim Jagielski$NumberSequence = $NumericEx ($FormatEx* $MidNumEx? $FormatEx* $NumericEx)*; 97*b1cdbd2cSJim Jagielski$NumberSequence {100}; 98*b1cdbd2cSJim Jagielski 99*b1cdbd2cSJim Jagielski# 100*b1cdbd2cSJim Jagielski# Words. Alpha-numerics. Rule 5, 6, 7, 9, 10 101*b1cdbd2cSJim Jagielski# - must include at least one letter. 102*b1cdbd2cSJim Jagielski# - may include both letters and numbers. 103*b1cdbd2cSJim Jagielski# - may include MideLetter, MidNumber punctuation. 104*b1cdbd2cSJim Jagielski# 105*b1cdbd2cSJim Jagielski$LetterSequence = $ALetterEx ($FormatEx* $MidLetterEx? $FormatEx* $ALetterEx)*; # rules #6, #7 106*b1cdbd2cSJim Jagielski($NumberSequence $FormatEx*)? $LetterSequence ($FormatEx* ($NumberSequence | $LetterSequence))* $SufixLetterEx? {200}; 107*b1cdbd2cSJim Jagielski 108*b1cdbd2cSJim Jagielski[[:P:][:S:]]*; 109*b1cdbd2cSJim Jagielski 110*b1cdbd2cSJim Jagielski# 111*b1cdbd2cSJim Jagielski# Do not break between Katakana. Rule #13. 112*b1cdbd2cSJim Jagielski# 113*b1cdbd2cSJim Jagielski$KatakanaEx ($FormatEx* $KatakanaEx)* {300}; 114*b1cdbd2cSJim Jagielski[:Hiragana:] $Extend* {300}; 115*b1cdbd2cSJim Jagielski 116*b1cdbd2cSJim Jagielski# 117*b1cdbd2cSJim Jagielski# Ideographic Characters. Stand by themselves as words. 118*b1cdbd2cSJim Jagielski# Separated from the "Everything Else" rule, below, only so that they 119*b1cdbd2cSJim Jagielski# can be tagged with a return value. TODO: is this what we want? 120*b1cdbd2cSJim Jagielski# 121*b1cdbd2cSJim Jagielski$IdeographicEx ($FormatEx* $IdeographicEx)* {400}; 122*b1cdbd2cSJim Jagielski$HangulEx ($FormatEx* $HangulEx)* {400}; 123*b1cdbd2cSJim Jagielski 124*b1cdbd2cSJim Jagielski# 125*b1cdbd2cSJim Jagielski# Everything Else, with no tag. 126*b1cdbd2cSJim Jagielski# Non-Control chars combine with $Extend (combining) chars. 127*b1cdbd2cSJim Jagielski# Controls are do not. 128*b1cdbd2cSJim Jagielski# 129*b1cdbd2cSJim Jagielski[^$Control [:Ideographic:]] $Extend*; 130*b1cdbd2cSJim Jagielski$CR $LF; 131*b1cdbd2cSJim Jagielski 132*b1cdbd2cSJim Jagielski# 133*b1cdbd2cSJim Jagielski# Reverse Rules. Back up over any of the chars that can group together. 134*b1cdbd2cSJim Jagielski# (Reverse rules do not need to be exact; they can back up too far, 135*b1cdbd2cSJim Jagielski# but must back up at least enough, and must stop on a boundary.) 136*b1cdbd2cSJim Jagielski# 137*b1cdbd2cSJim Jagielski 138*b1cdbd2cSJim Jagielski# NonStarters are the set of all characters that can appear at the 2nd - nth position of 139*b1cdbd2cSJim Jagielski# a word. (They may also be the first.) The reverse rule skips over these, until it 140*b1cdbd2cSJim Jagielski# reaches something that can only be the start (and probably only) char in a "word". 141*b1cdbd2cSJim Jagielski# A space or punctuation meets the test. 142*b1cdbd2cSJim Jagielski# 143*b1cdbd2cSJim Jagielski$NonStarters = [$Numeric $ALetter $Katakana $Ideographic $Hangul [:P:] [:S:] $MidLetter $MidNum $SufixLetter $Extend $Format]; 144*b1cdbd2cSJim Jagielski 145*b1cdbd2cSJim Jagielski#!.*; 146*b1cdbd2cSJim Jagielski! ($NonStarters* | \n \r) .; 147*b1cdbd2cSJim Jagielski 148