1*b1cdbd2cSJim Jagielski# 2*b1cdbd2cSJim Jagielski# Copyright (C) 2002-2003, International Business Machines Corporation and others. 3*b1cdbd2cSJim Jagielski# All Rights Reserved. 4*b1cdbd2cSJim Jagielski# 5*b1cdbd2cSJim Jagielski# file: dict_word.txt 6*b1cdbd2cSJim Jagielski# 7*b1cdbd2cSJim Jagielski# ICU Word Break Rules 8*b1cdbd2cSJim Jagielski# See Unicode Standard Annex #29. 9*b1cdbd2cSJim Jagielski# These rules are based on Version 4.0.0, dated 2003-04-17 10*b1cdbd2cSJim Jagielski# 11*b1cdbd2cSJim Jagielski 12*b1cdbd2cSJim Jagielski 13*b1cdbd2cSJim Jagielski 14*b1cdbd2cSJim Jagielski#################################################################################### 15*b1cdbd2cSJim Jagielski# 16*b1cdbd2cSJim Jagielski# Character class definitions from TR 29 17*b1cdbd2cSJim Jagielski# 18*b1cdbd2cSJim Jagielski#################################################################################### 19*b1cdbd2cSJim Jagielski$Katakana = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND MARK:] 20*b1cdbd2cSJim Jagielski [:name = HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK:] 21*b1cdbd2cSJim Jagielski [:name = HALFWIDTH KATAKANA VOICED SOUND MARK:] 22*b1cdbd2cSJim Jagielski [:name = HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK:]]; 23*b1cdbd2cSJim Jagielski 24*b1cdbd2cSJim Jagielski$Ideographic = [:Ideographic:]; 25*b1cdbd2cSJim Jagielski$Hangul = [:Script = HANGUL:]; 26*b1cdbd2cSJim Jagielski 27*b1cdbd2cSJim Jagielski# list of dashes or hyphens that should be accepted as part of the word if a single one of these 28*b1cdbd2cSJim Jagielski# pre- or postfixes a word. E.g. in German: "Arbeits-" or "-nehmer" where that hyphen needs to 29*b1cdbd2cSJim Jagielski# be part of the word in order to have it properly spell checked etc. 30*b1cdbd2cSJim Jagielski$PrePostDashHyphen = [ [:name = HYPHEN-MINUS:] ]; 31*b1cdbd2cSJim Jagielski 32*b1cdbd2cSJim Jagielski 33*b1cdbd2cSJim Jagielski$ALetter = [\u0002 [:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:] 34*b1cdbd2cSJim Jagielski - $Ideographic 35*b1cdbd2cSJim Jagielski - $Katakana 36*b1cdbd2cSJim Jagielski - $Hangul 37*b1cdbd2cSJim Jagielski - [:Script = Thai:] 38*b1cdbd2cSJim Jagielski - [:Script = Lao:] 39*b1cdbd2cSJim Jagielski - [:Script = Hiragana:]]; 40*b1cdbd2cSJim Jagielski 41*b1cdbd2cSJim Jagielski$MidLetter = [[:name = APOSTROPHE:] [:name = GRAVE ACCENT:] \u0084 [:name = SOFT HYPHEN:] [:name = MIDDLE DOT:] [:name = GREEK TONOS:] [:name= FULL STOP:] 42*b1cdbd2cSJim Jagielski [:name = HEBREW PUNCTUATION GERSHAYIM:] [:name = DOUBLE VERTICAL LINE:] [:name = LEFT SINGLE QUOTATION MARK:] 43*b1cdbd2cSJim Jagielski [:name = RIGHT SINGLE QUOTATION MARK:] [:name = HYPHENATION POINT:] [:name = PRIME:] 44*b1cdbd2cSJim Jagielski [:name = HYPHEN-MINUS:] ]; 45*b1cdbd2cSJim Jagielski 46*b1cdbd2cSJim Jagielski$SufixLetter = [:name= FULL STOP:]; 47*b1cdbd2cSJim Jagielski 48*b1cdbd2cSJim Jagielski 49*b1cdbd2cSJim Jagielski$MidNum = [[:LineBreak = Infix_Numeric:] [:name= COMMERCIAL AT:] \u0084 [:name = GREEK TONOS:] [:name = ARABIC DECIMAL SEPARATOR:] 50*b1cdbd2cSJim Jagielski [:name = LEFT SINGLE QUOTATION MARK:] [:name = RIGHT SINGLE QUOTATION MARK:] [:name = SINGLE HIGH-REVERSED-9 QUOTATION MARK:] 51*b1cdbd2cSJim Jagielski [:name = PRIME:]]; 52*b1cdbd2cSJim Jagielski$Numeric = [:LineBreak = Numeric:]; 53*b1cdbd2cSJim Jagielski 54*b1cdbd2cSJim Jagielski 55*b1cdbd2cSJim Jagielski$TheZWSP = \u200b; 56*b1cdbd2cSJim Jagielski 57*b1cdbd2cSJim Jagielski# 58*b1cdbd2cSJim Jagielski# Character Class Definitions. 59*b1cdbd2cSJim Jagielski# The names are those from TR29. 60*b1cdbd2cSJim Jagielski# 61*b1cdbd2cSJim Jagielski$CR = \u000d; 62*b1cdbd2cSJim Jagielski$LF = \u000a; 63*b1cdbd2cSJim Jagielski$Control = [[[:Zl:] [:Zp:] [:Cc:] [:Cf:]] - $TheZWSP]; 64*b1cdbd2cSJim Jagielski$Extend = [[:Grapheme_Extend = TRUE:]]; 65*b1cdbd2cSJim Jagielski 66*b1cdbd2cSJim Jagielski 67*b1cdbd2cSJim Jagielski 68*b1cdbd2cSJim Jagielski 69*b1cdbd2cSJim Jagielski#################################################################################### 70*b1cdbd2cSJim Jagielski# 71*b1cdbd2cSJim Jagielski# Word Break Rules. Definitions and Rules specific to word break begin Here. 72*b1cdbd2cSJim Jagielski# 73*b1cdbd2cSJim Jagielski#################################################################################### 74*b1cdbd2cSJim Jagielski 75*b1cdbd2cSJim Jagielski$Format = [[:Cf:] - $TheZWSP]; 76*b1cdbd2cSJim Jagielski 77*b1cdbd2cSJim Jagielski 78*b1cdbd2cSJim Jagielski 79*b1cdbd2cSJim Jagielski# Rule 3: Treat a grapheme cluster as if it were a single character. 80*b1cdbd2cSJim Jagielski# Hangul Syllables are easier to deal with here than they are in Grapheme Clusters 81*b1cdbd2cSJim Jagielski# because we don't need to find the boundaries between adjacent syllables - 82*b1cdbd2cSJim Jagielski# they won't be word boundaries. 83*b1cdbd2cSJim Jagielski# 84*b1cdbd2cSJim Jagielski 85*b1cdbd2cSJim Jagielski 86*b1cdbd2cSJim Jagielski# 87*b1cdbd2cSJim Jagielski# "Extended" definitions. Grapheme Cluster + Format Chars, treated like the base char. 88*b1cdbd2cSJim Jagielski# 89*b1cdbd2cSJim Jagielski$ALetterEx = $ALetter $Extend*; 90*b1cdbd2cSJim Jagielski$NumericEx = $Numeric $Extend*; 91*b1cdbd2cSJim Jagielski$MidNumEx = $MidNum $Extend*; 92*b1cdbd2cSJim Jagielski$MidLetterEx = $MidLetter $Extend*; 93*b1cdbd2cSJim Jagielski$SufixLetterEx= $SufixLetter $Extend*; 94*b1cdbd2cSJim Jagielski$KatakanaEx = $Katakana $Extend*; 95*b1cdbd2cSJim Jagielski$IdeographicEx= $Ideographic $Extend*; 96*b1cdbd2cSJim Jagielski$HangulEx = $Hangul $Extend*; 97*b1cdbd2cSJim Jagielski$FormatEx = $Format $Extend*; 98*b1cdbd2cSJim Jagielski 99*b1cdbd2cSJim Jagielski 100*b1cdbd2cSJim Jagielski# 101*b1cdbd2cSJim Jagielski# Numbers. Rules 8, 11, 12 form the TR. 102*b1cdbd2cSJim Jagielski# 103*b1cdbd2cSJim Jagielski$NumberSequence = $NumericEx ($FormatEx* $MidNumEx? $FormatEx* $NumericEx)*; 104*b1cdbd2cSJim Jagielski$NumberSequence {100}; 105*b1cdbd2cSJim Jagielski 106*b1cdbd2cSJim Jagielski# 107*b1cdbd2cSJim Jagielski# Words. Alpha-numerics. Rule 5, 6, 7, 9, 10 108*b1cdbd2cSJim Jagielski# - must include at least one letter. 109*b1cdbd2cSJim Jagielski# - may include both letters and numbers. 110*b1cdbd2cSJim Jagielski# - may include MideLetter, MidNumber punctuation. 111*b1cdbd2cSJim Jagielski# 112*b1cdbd2cSJim Jagielski# At most one leading or trailing dash/hyphen should be accepted as well. 113*b1cdbd2cSJim Jagielski# E.g. in German: "Arbeits-" or "-nehmer" where that hyphen needs to 114*b1cdbd2cSJim Jagielski# be part of the word in order to have it properly spell checked etc. 115*b1cdbd2cSJim Jagielski$LetterSequence = $PrePostDashHyphen? $ALetterEx ($FormatEx* $MidLetterEx? $FormatEx* $ALetterEx)* $PrePostDashHyphen?; # rules #6, #7 116*b1cdbd2cSJim Jagielski($NumberSequence $FormatEx*)? $LetterSequence ($FormatEx* ($NumberSequence | $LetterSequence))* $SufixLetterEx? {200}; 117*b1cdbd2cSJim Jagielski 118*b1cdbd2cSJim Jagielski[[:P:][:S:]]*; 119*b1cdbd2cSJim Jagielski 120*b1cdbd2cSJim Jagielski# 121*b1cdbd2cSJim Jagielski# Do not break between Katakana. Rule #13. 122*b1cdbd2cSJim Jagielski# 123*b1cdbd2cSJim Jagielski$KatakanaEx ($FormatEx* $KatakanaEx)* {300}; 124*b1cdbd2cSJim Jagielski[:Hiragana:] $Extend* {300}; 125*b1cdbd2cSJim Jagielski 126*b1cdbd2cSJim Jagielski# 127*b1cdbd2cSJim Jagielski# Ideographic Characters. Stand by themselves as words. 128*b1cdbd2cSJim Jagielski# Separated from the "Everything Else" rule, below, only so that they 129*b1cdbd2cSJim Jagielski# can be tagged with a return value. TODO: is this what we want? 130*b1cdbd2cSJim Jagielski# 131*b1cdbd2cSJim Jagielski$IdeographicEx ($FormatEx* $IdeographicEx)* {400}; 132*b1cdbd2cSJim Jagielski$HangulEx ($FormatEx* $HangulEx)* {400}; 133*b1cdbd2cSJim Jagielski 134*b1cdbd2cSJim Jagielski# 135*b1cdbd2cSJim Jagielski# Everything Else, with no tag. 136*b1cdbd2cSJim Jagielski# Non-Control chars combine with $Extend (combining) chars. 137*b1cdbd2cSJim Jagielski# Controls are do not. 138*b1cdbd2cSJim Jagielski# 139*b1cdbd2cSJim Jagielski[^$Control [:Ideographic:]] $Extend*; 140*b1cdbd2cSJim Jagielski$CR $LF; 141*b1cdbd2cSJim Jagielski 142*b1cdbd2cSJim Jagielski# 143*b1cdbd2cSJim Jagielski# Reverse Rules. Back up over any of the chars that can group together. 144*b1cdbd2cSJim Jagielski# (Reverse rules do not need to be exact; they can back up too far, 145*b1cdbd2cSJim Jagielski# but must back up at least enough, and must stop on a boundary.) 146*b1cdbd2cSJim Jagielski# 147*b1cdbd2cSJim Jagielski 148*b1cdbd2cSJim Jagielski# NonStarters are the set of all characters that can appear at the 2nd - nth position of 149*b1cdbd2cSJim Jagielski# a word. (They may also be the first.) The reverse rule skips over these, until it 150*b1cdbd2cSJim Jagielski# reaches something that can only be the start (and probably only) char in a "word". 151*b1cdbd2cSJim Jagielski# A space or punctuation meets the test. 152*b1cdbd2cSJim Jagielski# 153*b1cdbd2cSJim Jagielski$NonStarters = [$Numeric $ALetter $Katakana $Ideographic $Hangul [:P:] [:S:] $MidLetter $MidNum $SufixLetter $Extend $Format]; 154*b1cdbd2cSJim Jagielski 155*b1cdbd2cSJim Jagielski#!.*; 156*b1cdbd2cSJim Jagielski! ($NonStarters* | \n \r) .; 157*b1cdbd2cSJim Jagielski 158