1#
2#   Copyright (C) 2002-2003, International Business Machines Corporation and others.
3#       All Rights Reserved.
4#
5#   file:  edit_word.txt
6#
7#   ICU Word Break Rules
8#      See Unicode Standard Annex #29.
9#      These rules are based on Version 4.0.0, dated 2003-04-17
10#
11
12
13
14####################################################################################
15#
16#  Character class definitions from TR 29
17#
18####################################################################################
19$Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND MARK:]
20                                   [:name = HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK:]
21                                   [:name = HALFWIDTH KATAKANA VOICED SOUND MARK:]
22                                   [:name = HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK:]];
23
24$Ideographic = [:Ideographic:];
25$Hangul = [:Script = HANGUL:];
26
27$ALetter   = [\u0002 [:Alphabetic:] [:name= NO-BREAK SPACE:] [:name= HEBREW PUNCTUATION GERESH:]
28                [:name = PERCENT SIGN:] [:name = PER MILLE SIGN:] [:name = PER TEN THOUSAND SIGN:]
29                [:name = SECTION SIGN:] [:name = DEGREE SIGN:] [:name = EURO SIGN:]
30                [:name = HYPHEN-MINUS:] [:name = EN DASH:] [:name = EM DASH:]
31                [:name = DIGIT ZERO:]
32                [:name = DIGIT ONE:]
33                [:name = DIGIT TWO:]
34                [:name = DIGIT THREE:]
35                [:name = DIGIT FOUR:]
36                [:name = DIGIT FIVE:]
37                [:name = DIGIT SIX:]
38                [:name = DIGIT SEVEN:]
39                [:name = DIGIT EIGHT:]
40                [:name = DIGIT NINE:]
41                           - $Ideographic
42                           - $Katakana
43                           - $Hangul
44                           - [:Script = Thai:]
45                           - [:Script = Lao:]
46                           - [:Script = Hiragana:]];
47
48$MidLetter = [[:name = APOSTROPHE:] [:name = MIDDLE DOT:]  [:name = HEBREW PUNCTUATION GERSHAYIM:]
49              [:name = RIGHT SINGLE QUOTATION MARK:] [:name = HYPHENATION POINT:]
50              [:name = HYPHEN-MINUS:] [:name = EURO SIGN:] [:name = PERCENT SIGN:]
51              [:name = PER MILLE SIGN:] [:name = PER TEN THOUSAND SIGN:]
52              [:name = EN DASH:] [:name = EM DASH:]
53              [:name = PERCENT SIGN:] [:name = SECTION SIGN:] [:name = DEGREE SIGN:]];
54
55$MidNum    = [[:LineBreak = Infix_Numeric:] - [:name = FULL STOP:]];
56$Numeric   = [:LineBreak = Numeric:];
57
58
59$TheZWSP = \u200b;
60
61#
62#  Character Class Definitions.
63#    The names are those from TR29.
64#
65$CR         = \u000d;
66$LF         = \u000a;
67$Control    = [[[:Zl:] [:Zp:] [:Cc:] [:Cf:]] - $TheZWSP];
68$Extend     = [[:Grapheme_Extend = TRUE:]];
69
70
71
72
73####################################################################################
74#
75#  Word Break Rules.    Definitions and Rules specific to word break begin Here.
76#
77####################################################################################
78
79$Format    = [[:Cf:] - $TheZWSP];
80
81
82
83# Rule 3:  Treat a grapheme cluster as if it were a single character.
84#          Hangul Syllables are easier to deal with here than they are in Grapheme Clusters
85#          because we don't need to find the boundaries between adjacent syllables -
86#          they won't be word boundaries.
87#
88
89
90#
91#  "Extended"  definitions.  Grapheme Cluster + Format Chars, treated like the base char.
92#
93$ALetterEx    = $ALetter   $Extend*;
94$NumericEx    = $Numeric   $Extend*;
95$MidNumEx     = $MidNum    $Extend*;
96$MidLetterEx  = $MidLetter $Extend*;
97$KatakanaEx   = $Katakana  $Extend*;
98$IdeographicEx= $Ideographic  $Extend*;
99$HangulEx = $Hangul  $Extend*;
100$FormatEx     = $Format    $Extend*;
101
102
103#
104#  Numbers.  Rules 8, 11, 12 form the TR.
105#
106$NumberSequence = $NumericEx ($FormatEx* $MidNumEx? $FormatEx* $NumericEx)*;
107$NumberSequence {100};
108
109#
110#  Words.  Alpha-numerics.  Rule 5, 6, 7, 9, 10
111#     - must include at least one letter.
112#     - may include both letters and numbers.
113#     - may include  MideLetter, MidNumber punctuation.
114#
115$LetterSequence = $ALetterEx ($FormatEx* $MidLetterEx? $FormatEx* $ALetterEx)*;     # rules #6, #7
116($NumberSequence $FormatEx*)? $LetterSequence ($FormatEx* ($NumberSequence | $LetterSequence))* {200};
117
118# Punctuations by themselves
119[[:P:][:S:]-[:name = FULL STOP:]]*;
120[[:name = FULL STOP:]]*;
121
122#
123#  Do not break between Katakana.   Rule #13.
124#
125$KatakanaEx ($FormatEx* $KatakanaEx)* {300};
126[:Hiragana:] $Extend* {300};
127
128#
129#  Ideographic Characters.  Stand by themselves as words.
130#                           Separated from the "Everything Else" rule, below, only so that they
131#                           can be tagged with a return value.   TODO:  is this what we want?
132#
133$IdeographicEx ($FormatEx* $IdeographicEx)* {400};
134$HangulEx ($FormatEx* $HangulEx)* {400};
135
136#
137#  Everything Else, with no tag.
138#                   Non-Control chars combine with $Extend (combining) chars.
139#                   Controls are do not.
140#
141[^$Control [:Ideographic:]] $Extend*;
142$CR $LF;
143
144#
145#  Reverse Rules.   Back up over any of the chars that can group together.
146#                   (Reverse rules do not need to be exact; they can back up  too far,
147#                   but must back up at least enough, and must stop on a boundary.)
148#
149
150# NonStarters are the set of all characters that can appear at the 2nd - nth position of
151#    a word.   (They may also be the first.)   The reverse rule skips over these, until it
152#    reaches something that can only be the start (and probably only) char in a "word".
153#    A space or punctuation meets the test.
154#
155$NonStarters = [$Numeric $ALetter $Katakana $Ideographic $Hangul [:P:] [:S:] $MidLetter $MidNum $Extend $Format];
156
157#!.*;
158! ($NonStarters* | \n \r) .;
159
160