stem.lua (6279B)
1 --[[ 2 This software is based on the Porter Stemmer which can be found here: 3 http://tartarus.org/~martin/PorterStemmer/ 4 5 The general algorithm is based on the Javascript version from the above page 6 (by 'Andargor'?) and then verified against the C reference source. 7 8 MIT LICENSE 9 10 Copyright (C) 2012 Chris Osgood <chris at luadev.com> 11 12 Permission is hereby granted, free of charge, to any person obtaining a copy 13 of this software and associated documentation files (the "Software"), to 14 deal in the Software without restriction, including without limitation the 15 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 16 sell copies of the Software, and to permit persons to whom the Software is 17 furnished to do so, subject to the following conditions: 18 19 The above copyright notice and this permission notice shall be included in 20 all copies or substantial portions of the Software. 21 22 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 26 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 27 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 28 IN THE SOFTWARE. 29 --]] 30 -- module("stem", package.seeall) 31 32 local lpeg = require("lpeg") 33 34 -- Makes pattern match at end of string 35 local function eos(p) 36 return (1 - (p * -1))^0 * lpeg.C(p) * -1 37 end 38 39 -- Word patterns 40 local c = 1 - lpeg.S("aeiou") -- consonant 41 local v = lpeg.S("aeiouy") -- vowel 42 local C = c * (1 - v)^0 -- consonant sequence 43 local V = v * lpeg.S("aeiou")^0 -- vowel sequence 44 45 local mgr0 = C^-1 * V * C -- [C]VC... is m>0 46 local meq1 = C^-1 * V * C * V^-1 * -1 -- [C]VC[V] is m=1 47 local mgr1 = C^-1 * V * C * V * C -- [C]VCVC... is m>1 48 local s_v = C^-1 * v -- vowel in stem 49 local e_v = C * v * (1 - lpeg.S("aeiouwxy")) * -1 50 51 -- Suffix patterns 52 local step2map = { 53 ["ational"] = "ate", ["ization"] = "ize", ["iveness"] = "ive", 54 ["fulness"] = "ful", ["ousness"] = "ous", ["tional"] = "tion", 55 ["biliti"] = "ble", ["entli"] = "ent", ["ousli"] = "ous", 56 ["ation"] = "ate", ["alism"] = "al", ["aliti"] = "al", 57 ["iviti"] = "ive", ["enci"] = "ence", ["anci"] = "ance", 58 ["izer"] = "ize", ["alli"] = "al", ["ator"] = "ate", 59 ["logi"] = "log", ["bli"] = "ble", ["eli"] = "e" 60 } 61 62 local step3map = { 63 ["icate"] = "ic", ["ative"] = "", ["alize"] = "al", 64 ["iciti"] = "ic", ["ical"] = "ic", ["ful"] = "", 65 ["ness"] = "" 66 } 67 68 local step1a1 = eos(lpeg.P("sses") + "ies") 69 local step1a2 = eos((1 - lpeg.P("s")) * "s") 70 local step1b1 = eos(lpeg.P("eed")) 71 local step1b2 = eos(lpeg.P("ed") + "ing") 72 local step1b3 = eos(lpeg.P("at") + "bl" + "iz") 73 local step1c = eos(lpeg.P("y")) 74 75 local step2 = eos(lpeg.P("ational") + "ization" + "iveness" + "fulness" + 76 "ousness" + "tional" + "biliti" + "entli" + "ousli" + 77 "ation" + "alism" + "aliti" + "iviti" + "enci" + "anci" + 78 "izer" + "alli" + "ator" + "logi" + "bli" + "eli") 79 80 local step3 = eos(lpeg.P("icate") + "ative" + "alize" + 81 "iciti" + "ical" + "ness" + "ful") 82 83 local step4a = eos(lpeg.P("ement") + "ance" + "ence" + "able" + "ible" + 84 "ment" + "ant" + "ent" + "ous" + "ism" + "ate" + "iti" + 85 "ive" + "ize" + "al" + "er" + "ic" + "ou") 86 87 local step4b = eos(lpeg.P("sion") + "tion") 88 89 local step5a = eos(lpeg.P("e")) 90 local step5b = eos(lpeg.P("ll")) 91 92 -- Stemming function 93 function stem(w) 94 if #w < 3 then return w end 95 96 local firstch = w:sub(1,1) 97 if firstch == "y" then 98 w = "Y" .. w:sub(2) 99 end 100 101 -- Step 1a 102 if lpeg.match(step1a1, w) then 103 w = w:sub(1, -3) 104 elseif lpeg.match(step1a2, w) then 105 w = w:sub(1, -2) 106 end 107 108 -- Step 1b 109 if lpeg.match(step1b1, w) then 110 if lpeg.match(mgr0, w:sub(1, -4)) then 111 w = w:sub(1, -2) 112 end 113 else 114 local suffix = lpeg.match(step1b2, w) 115 116 if suffix then 117 local stem = w:sub(1, -#suffix - 1) 118 if lpeg.match(s_v, stem) then 119 w = stem 120 if lpeg.match(step1b3, w) then 121 w = w .."e" 122 -- Don't use lpeg, it's slower with backreferences 123 elseif w:find("([^aeiouylsz])%1$") then 124 w = w:sub(1, -2) 125 elseif lpeg.match(e_v, w) then 126 w = w .. "e" 127 end 128 end 129 end 130 end 131 132 -- Step 1c 133 if lpeg.match(step1c, w) then 134 local stem = w:sub(1, -2) 135 if lpeg.match(s_v, stem) then 136 w = stem .. "i" 137 end 138 end 139 140 -- Step 2 141 local suffix = lpeg.match(step2, w) 142 143 if suffix then 144 local stem = w:sub(1, -#suffix - 1) 145 if lpeg.match(mgr0, stem) then 146 w = stem .. step2map[suffix] 147 end 148 end 149 150 -- Step 3 151 suffix = lpeg.match(step3, w) 152 153 if suffix then 154 local stem = w:sub(1, -#suffix - 1) 155 if lpeg.match(mgr0, stem) then 156 w = stem .. step3map[suffix] 157 end 158 end 159 160 -- Step 4 161 suffix = lpeg.match(step4a, w) 162 163 if suffix then 164 local stem = w:sub(1, -#suffix - 1) 165 if lpeg.match(mgr1, stem) then 166 w = stem 167 end 168 elseif lpeg.match(step4b, w) then 169 local stem = w:sub(1, -4) 170 if lpeg.match(mgr1, stem) then 171 w = stem 172 end 173 end 174 175 -- Step 5 176 if lpeg.match(step5a, w) then 177 local stem = w:sub(1, -2) 178 if lpeg.match(mgr1, stem) or (lpeg.match(meq1, stem) and not lpeg.match(e_v, stem)) then 179 w = stem 180 end 181 end 182 183 if lpeg.match(step5b, w) and lpeg.match(mgr1, w) then 184 w = w:sub(1, -2) 185 end 186 187 if firstch == "y" then 188 w = "y" .. w:sub(2) 189 end 190 191 return w 192 end 193 194 return { stem = stem } 195 196 --[[ 197 function file(filename) 198 local fp = io.open(filename, "r") 199 if not fp then error("error opening file `"..tostring(filename).."'") end 200 for l in fp:lines() do 201 print(stem(l)) 202 end 203 fp:close() 204 end 205 206 local args = {...} 207 if #args > 0 then 208 for _, filename in ipairs(args) do 209 file(filename) 210 end 211 end 212 --]] 213