首页 > 代码库 > Lucene5.x 中文 同义词
Lucene5.x 中文 同义词
查询好好多资料,英文同义词好好的,中文就不行,多谢网友支持,拼接了好多代码,然后修改了一些,不足之处,多谢指正。
直接上代码吧,在代码中了解怎么分词的最好
1,创建分词引擎
1 public interface SamewordContext { 2 String[] getSamewords(String name); 3 }
2,同义词
1 import java.util.HashMap; 2 import java.util.Map; 3 4 public class SimpleSamewordContext implements SamewordContext { 5 Map<String,String[]> maps = new HashMap<String,String[]>(); 6 public SimpleSamewordContext() { 7 maps.put("中国",new String[]{"天朝","大陆"}); 8 maps.put("我家",new String[]{"family","伐木累"}); 9 } 10 @Override 11 public String[] getSamewords(String name) { 12 // TODO Auto-generated method stub 13 return maps.get(name); 14 } 15 }
3,TokenFilter
import java.io.IOException; import java.util.Stack; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.util.AttributeSource; public class MySameTokenFilter extends TokenFilter { private CharTermAttribute cta = null; private PositionIncrementAttribute pia = null; private AttributeSource.State current; private Stack<String> sames = null; private SamewordContext samewordContext; protected MySameTokenFilter(TokenStream input,SamewordContext samewordContext) { super(input); cta = this.addAttribute(CharTermAttribute.class); pia = this.addAttribute(PositionIncrementAttribute.class); sames = new Stack<String>(); this.samewordContext = samewordContext; } @Override public boolean incrementToken() throws IOException { if(sames.size()>0) { //将元素出栈,并且获取这个同义词 String str = sames.pop(); //还原状态 restoreState(current); cta.setEmpty(); cta.append(str); //设置位置0 pia.setPositionIncrement(0); return true; } if(!this.input.incrementToken()) return false; if(addSames(cta.toString())) { //如果有同义词将当前状态先保存 current = captureState(); } return true; } private boolean addSames(String name) { String[] sws = samewordContext.getSamewords(name); if(sws!=null) { for(String str:sws) { sames.push(str); } return true; } return false; } }
4,Analyzer
import java.io.Reader; import java.io.StringReader; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.core.LowerCaseFilter; import org.apache.lucene.analysis.core.StopAnalyzer; import org.apache.lucene.analysis.core.StopFilter; import org.wltea.analyzer.lucene.IKTokenizer; import com.chenlb.mmseg4j.analysis.MMSegAnalyzer; public class MySameworkAnalyzer extends MMSegAnalyzer { private SamewordContext samewordContext; public MySameworkAnalyzer(SamewordContext samewordContext) { // TODO Auto-generated constructor stub this.samewordContext = samewordContext; } @Override protected TokenStreamComponents createComponents(String text) { Reader in = new StringReader(text); IKTokenizer tokenizer = new IKTokenizer(in , true); TokenStream tokenStream = new MySameTokenFilter(tokenizer, samewordContext); tokenStream = new LowerCaseFilter(tokenStream); tokenStream = new StopFilter(tokenStream, StopAnalyzer.ENGLISH_STOP_WORDS_SET); return new TokenStreamComponents(tokenizer, tokenStream); } }
5,测试
@Test public void test01() { String text = "我家在中国"; Analyzer analyzer = new MySameworkAnalyzer(new SimpleSamewordContext()); AnalyzerUtils.displayAllToken(text,analyzer); }
运行结果:
Lucene5.x 中文 同义词
声明:以上内容来自用户投稿及互联网公开渠道收集整理发布,本网站不拥有所有权,未作人工编辑处理,也不承担相关法律责任,若内容有误或涉及侵权可进行投诉: 投诉/举报 工作人员会在5个工作日内联系你,一经查实,本站将立刻删除涉嫌侵权内容。