Skip to content

GitLab

商
商学院分词
Commit e1ac0544 authored by 田磊's avatar 田磊
Browse files
Options

push

parents
Expand all Show whitespace changes
Inline Side-by-side
Showing with 3847 additions and 0 deletions
.gitignore 0 → 100644
View file @ e1ac0544
.idea
\ No newline at end of file
README.md 0 → 100644
View file @ e1ac0544
# IKAnalyzer-Mirror
For IKAnalyzer has stopped maintaining, this repository helps it works with higher version of lucene. Such as 6.x.
# How to use
* git clone https://github.com/sea-boat/IKAnalyzer-Mirror.git
* mvn install
* pom.xml
```
<dependency>
<groupId>com.seaboat</groupId>
<artifactId>IKAnalyzer</artifactId>
<version>6.4.0</version>
</dependency>
```
pom.xml 0 → 100644
View file @ e1ac0544
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.seaboat</groupId>
<artifactId>IKAnalyzer</artifactId>
<version>6.4.0</version>
<name>IKAnalyzer</name>
<description>IKAnalyzer</description>
<dependencies>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>6.4.0</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
<version>6.4.0</version>
</dependency>
</dependencies>
<distributionManagement>
<repository>
<id>rdc-releases</id>
<url>https://repo.rdc.aliyun.com/repository/51000-release-DPmfIO/</url>
</repository>
<snapshotRepository>
<id>rdc-snapshots</id>
<url>https://repo.rdc.aliyun.com/repository/51000-snapshot-oQlu1r/</url>
</snapshotRepository>
</distributionManagement>
</project>
\ No newline at end of file
settings.xml 0 → 100644
View file @ e1ac0544
<?xml version="1.0" encoding="UTF-8"?>
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<mirrors>
<mirror>
<id>alimaven</id>
<name>aliyun maven</name>
<url>http://maven.aliyun.com/nexus/content/groups/public/</url>
<mirrorOf>central</mirrorOf>
</mirror>
</mirrors>
<servers>
<server>
<id>rdc-releases</id>
<username>aGnXSC</username>
<password>tY7JBof4iN</password>
</server>
<server>
<id>rdc-snapshots</id>
<username>aGnXSC</username>
<password>tY7JBof4iN</password>
</server>
</servers>
<profiles>
<profile>
<id>nexus</id>
<repositories>
<repository>
<id>central</id>
<url>http://maven.aliyun.com/nexus/content/groups/public</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>snapshots</id>
<url>http://maven.aliyun.com/nexus/content/groups/public</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
<repository>
<id>rdc-releases</id>
<url>https://repo.rdc.aliyun.com/repository/51000-release-DPmfIO/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>rdc-snapshots</id>
<url>https://repo.rdc.aliyun.com/repository/51000-snapshot-oQlu1r/</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
<pluginRepositories>
<pluginRepository>
<id>central</id>
<url>http://maven.aliyun.com/nexus/content/groups/public</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</pluginRepository>
<pluginRepository>
<id>snapshots</id>
<url>http://maven.aliyun.com/nexus/content/groups/public</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</pluginRepository>
<pluginRepository>
<id>rdc-releases</id>
<url>https://repo.rdc.aliyun.com/repository/51000-release-DPmfIO/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</pluginRepository>
<pluginRepository>
<id>rdc-snapshots</id>
<url>https://repo.rdc.aliyun.com/repository/51000-snapshot-oQlu1r/</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</pluginRepository>
</pluginRepositories>
</profile>
</profiles>
<activeProfiles>
<activeProfile>nexus</activeProfile>
</activeProfiles>
</settings>
\ No newline at end of file
src/main/java/org/wltea/analyzer/cfg/Configuration.java 0 → 100644
View file @ e1ac0544
/**
* IK 中文分词 版本 5.0
* IK Analyzer release 5.0
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* 源代码由林良益(linliangyi2005@gmail.com)提供
* 版权声明 2012，乌龙茶工作室
* provided by Linliangyi and copyright 2012 by Oolong studio
*
*/
package org.wltea.analyzer.cfg;
import java.util.List;
/**
*
* 配置管理类接口
*
*/
public interface Configuration {
/**
* 返回useSmart标志位 useSmart =true ，分词器使用智能切分策略， =false则使用细粒度切分
*
* @return useSmart
*/
public boolean useSmart();
/**
* 设置useSmart标志位 useSmart =true ，分词器使用智能切分策略， =false则使用细粒度切分
*
* @param useSmart
*/
public void setUseSmart(boolean useSmart);
/**
* 获取主词典路径
*
* @return String 主词典路径
*/
public String getMainDictionary();
/**
* 获取量词词典路径
*
* @return String 量词词典路径
*/
public String getQuantifierDicionary();
/**
* 获取扩展字典配置路径
*
* @return List<String> 相对类加载器的路径
*/
public List<String> getExtDictionarys();
/**
* 获取扩展停止词典配置路径
*
* @return List<String> 相对类加载器的路径
*/
public List<String> getExtStopWordDictionarys();
}
src/main/java/org/wltea/analyzer/cfg/DefaultConfig.java 0 → 100644
View file @ e1ac0544
/**
* IK 中文分词 版本 5.0
* IK Analyzer release 5.0
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* 源代码由林良益(linliangyi2005@gmail.com)提供
* 版权声明 2012，乌龙茶工作室
* provided by Linliangyi and copyright 2012 by Oolong studio
*
*
*/
package org.wltea.analyzer.cfg;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.InvalidPropertiesFormatException;
import java.util.List;
import java.util.Properties;
/**
* Configuration 默认实现 2012-5-8
*
*/
public class DefaultConfig implements Configuration {
/*
* 分词器默认字典路径
*/
private static final String PATH_DIC_MAIN = "org/wltea/analyzer/dic/main2012.dic";
private static final String PATH_DIC_QUANTIFIER = "org/wltea/analyzer/dic/quantifier.dic";
/*
* 分词器配置文件路径
*/
private static final String FILE_NAME = "IKAnalyzer.cfg.xml";
// 配置属性——扩展字典
private static final String EXT_DICT = "ext_dict";
// 配置属性——扩展停止词典
private static final String EXT_STOP = "ext_stopwords";
private Properties props;
/*
* 是否使用smart方式分词
*/
private boolean useSmart;
/**
* 返回单例
*
* @return Configuration单例
*/
public static Configuration getInstance() {
return new DefaultConfig();
}
/*
* 初始化配置文件
*/
private DefaultConfig() {
props = new Properties();
InputStream input = this.getClass().getClassLoader()
.getResourceAsStream(FILE_NAME);
if (input != null) {
try {
props.loadFromXML(input);
} catch (InvalidPropertiesFormatException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* 返回useSmart标志位 useSmart =true ，分词器使用智能切分策略， =false则使用细粒度切分
*
* @return useSmart
*/
public boolean useSmart() {
return useSmart;
}
/**
* 设置useSmart标志位 useSmart =true ，分词器使用智能切分策略， =false则使用细粒度切分
*
* @param useSmart
*/
public void setUseSmart(boolean useSmart) {
this.useSmart = useSmart;
}
/**
* 获取主词典路径
*
* @return String 主词典路径
*/
public String getMainDictionary() {
return PATH_DIC_MAIN;
}
/**
* 获取量词词典路径
*
* @return String 量词词典路径
*/
public String getQuantifierDicionary() {
return PATH_DIC_QUANTIFIER;
}
/**
* 获取扩展字典配置路径
*
* @return List<String> 相对类加载器的路径
*/
public List<String> getExtDictionarys() {
List<String> extDictFiles = new ArrayList<String>(2);
String extDictCfg = props.getProperty(EXT_DICT);
if (extDictCfg != null) {
// 使用;分割多个扩展字典配置
String[] filePaths = extDictCfg.split(";");
if (filePaths != null) {
for (String filePath : filePaths) {
if (filePath != null && !"".equals(filePath.trim())) {
extDictFiles.add(filePath.trim());
}
}
}
}
return extDictFiles;
}
/**
* 获取扩展停止词典配置路径
*
* @return List<String> 相对类加载器的路径
*/
public List<String> getExtStopWordDictionarys() {
List<String> extStopWordDictFiles = new ArrayList<String>(2);
String extStopWordDictCfg = props.getProperty(EXT_STOP);
if (extStopWordDictCfg != null) {
// 使用;分割多个扩展字典配置
String[] filePaths = extStopWordDictCfg.split(";");
if (filePaths != null) {
for (String filePath : filePaths) {
if (filePath != null && !"".equals(filePath.trim())) {
extStopWordDictFiles.add(filePath.trim());
}
}
}
}
return extStopWordDictFiles;
}
}
src/main/java/org/wltea/analyzer/core/AnalyzeContext.java 0 → 100644
View file @ e1ac0544
This diff is collapsed.
src/main/java/org/wltea/analyzer/core/CJKSegmenter.java 0 → 100644
View file @ e1ac0544
/**
* IK 中文分词 版本 5.0
* IK Analyzer release 5.0
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* 源代码由林良益(linliangyi2005@gmail.com)提供
* 版权声明 2012，乌龙茶工作室
* provided by Linliangyi and copyright 2012 by Oolong studio
*
*/
package org.wltea.analyzer.core;
import java.util.LinkedList;
import java.util.List;
import org.wltea.analyzer.dic.Dictionary;
import org.wltea.analyzer.dic.Hit;
/**
* 中文-日韩文子分词器
*/
class CJKSegmenter implements ISegmenter {
// 子分词器标签
static final String SEGMENTER_NAME = "CJK_SEGMENTER";
// 待处理的分词hit队列
private List<Hit> tmpHits;
CJKSegmenter() {
this.tmpHits = new LinkedList<Hit>();
}
/*
* (non-Javadoc)
*
* @see org.wltea.analyzer.core.ISegmenter#analyze(org.wltea.analyzer.core.
* AnalyzeContext)
*/
public void analyze(AnalyzeContext context) {
if (CharacterUtil.CHAR_USELESS != context.getCurrentCharType()) {
// 优先处理tmpHits中的hit
if (!this.tmpHits.isEmpty()) {
// 处理词段队列
Hit[] tmpArray = this.tmpHits.toArray(new Hit[this.tmpHits
.size()]);
for (Hit hit : tmpArray) {
hit = Dictionary.getSingleton().matchWithHit(
context.getSegmentBuff(), context.getCursor(), hit);
if (hit.isMatch()) {
// 输出当前的词
Lexeme newLexeme = new Lexeme(
context.getBufferOffset(), hit.getBegin(),
context.getCursor() - hit.getBegin() + 1,
Lexeme.TYPE_CNWORD);
context.addLexeme(newLexeme);
if (!hit.isPrefix()) {// 不是词前缀，hit不需要继续匹配，移除
this.tmpHits.remove(hit);
}
} else if (hit.isUnmatch()) {
// hit不是词，移除
this.tmpHits.remove(hit);
}
}
}
// *********************************
// 再对当前指针位置的字符进行单字匹配
Hit singleCharHit = Dictionary.getSingleton().matchInMainDict(
context.getSegmentBuff(), context.getCursor(), 1);
if (singleCharHit.isMatch()) {// 首字成词
// 输出当前的词
Lexeme newLexeme = new Lexeme(context.getBufferOffset(),
context.getCursor(), 1, Lexeme.TYPE_CNWORD);
context.addLexeme(newLexeme);
// 同时也是词前缀
if (singleCharHit.isPrefix()) {
// 前缀匹配则放入hit列表
this.tmpHits.add(singleCharHit);
}
} else if (singleCharHit.isPrefix()) {// 首字为词前缀
// 前缀匹配则放入hit列表
this.tmpHits.add(singleCharHit);
}
} else {
// 遇到CHAR_USELESS字符
// 清空队列
this.tmpHits.clear();
}
// 判断缓冲区是否已经读完
if (context.isBufferConsumed()) {
// 清空队列
this.tmpHits.clear();
}
// 判断是否锁定缓冲区
if (this.tmpHits.size() == 0) {
context.unlockBuffer(SEGMENTER_NAME);
} else {
context.lockBuffer(SEGMENTER_NAME);
}
}
/*
* (non-Javadoc)
*
* @see org.wltea.analyzer.core.ISegmenter#reset()
*/
public void reset() {
// 清空队列
this.tmpHits.clear();
}
}
src/main/java/org/wltea/analyzer/core/CN_QuantifierSegmenter.java 0 → 100644
View file @ e1ac0544
/**
* IK 中文分词 版本 5.0
* IK Analyzer release 5.0
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* 源代码由林良益(linliangyi2005@gmail.com)提供
* 版权声明 2012，乌龙茶工作室
* provided by Linliangyi and copyright 2012 by Oolong studio
*
*/
package org.wltea.analyzer.core;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import org.wltea.analyzer.dic.Dictionary;
import org.wltea.analyzer.dic.Hit;
/**
*
* 中文数量词子分词器
*/
class CN_QuantifierSegmenter implements ISegmenter {
// 子分词器标签
static final String SEGMENTER_NAME = "QUAN_SEGMENTER";
// 中文数词
private static String Chn_Num = "一二两三四五六七八九十零壹贰叁肆伍陆柒捌玖拾百千万亿拾佰仟萬億兆卅廿";// Cnum
private static Set<Character> ChnNumberChars = new HashSet<Character>();
static {
char[] ca = Chn_Num.toCharArray();
for (char nChar : ca) {
ChnNumberChars.add(nChar);
}
}
/*
* 词元的开始位置， 同时作为子分词器状态标识 当start > -1 时，标识当前的分词器正在处理字符
*/
private int nStart;
/*
* 记录词元结束位置 end记录的是在词元中最后一个出现的合理的数词结束
*/
private int nEnd;
// 待处理的量词hit队列
private List<Hit> countHits;
CN_QuantifierSegmenter() {
nStart = -1;
nEnd = -1;
this.countHits = new LinkedList<Hit>();
}
/**
* 分词
*/
public void analyze(AnalyzeContext context) {
// 处理中文数词
this.processCNumber(context);
// 处理中文量词
this.processCount(context);
// 判断是否锁定缓冲区
if (this.nStart == -1 && this.nEnd == -1 && countHits.isEmpty()) {
// 对缓冲区解锁
context.unlockBuffer(SEGMENTER_NAME);
} else {
context.lockBuffer(SEGMENTER_NAME);
}
}
/**
* 重置子分词器状态
*/
public void reset() {
nStart = -1;
nEnd = -1;
countHits.clear();
}
/**
* 处理数词
*/
private void processCNumber(AnalyzeContext context) {
if (nStart == -1 && nEnd == -1) {// 初始状态
if (CharacterUtil.CHAR_CHINESE == context.getCurrentCharType()
&& ChnNumberChars.contains(context.getCurrentChar())) {
// 记录数词的起始、结束位置
nStart = context.getCursor();
nEnd = context.getCursor();
}
} else {// 正在处理状态
if (CharacterUtil.CHAR_CHINESE == context.getCurrentCharType()
&& ChnNumberChars.contains(context.getCurrentChar())) {
// 记录数词的结束位置
nEnd = context.getCursor();
} else {
// 输出数词
this.outputNumLexeme(context);
// 重置头尾指针
nStart = -1;
nEnd = -1;
}
}
// 缓冲区已经用完，还有尚未输出的数词
if (context.isBufferConsumed()) {
if (nStart != -1 && nEnd != -1) {
// 输出数词
outputNumLexeme(context);
// 重置头尾指针
nStart = -1;
nEnd = -1;
}
}
}
/**
* 处理中文量词
*
* @param context
*/
private void processCount(AnalyzeContext context) {
// 判断是否需要启动量词扫描
if (!this.needCountScan(context)) {
return;
}
if (CharacterUtil.CHAR_CHINESE == context.getCurrentCharType()) {
// 优先处理countHits中的hit
if (!this.countHits.isEmpty()) {
// 处理词段队列
Hit[] tmpArray = this.countHits.toArray(new Hit[this.countHits
.size()]);
for (Hit hit : tmpArray) {
hit = Dictionary.getSingleton().matchWithHit(
context.getSegmentBuff(), context.getCursor(), hit);
if (hit.isMatch()) {
// 输出当前的词
Lexeme newLexeme = new Lexeme(
context.getBufferOffset(), hit.getBegin(),
context.getCursor() - hit.getBegin() + 1,
Lexeme.TYPE_COUNT);
context.addLexeme(newLexeme);
if (!hit.isPrefix()) {// 不是词前缀，hit不需要继续匹配，移除
this.countHits.remove(hit);
}
} else if (hit.isUnmatch()) {
// hit不是词，移除
this.countHits.remove(hit);
}
}
}
// *********************************
// 对当前指针位置的字符进行单字匹配
Hit singleCharHit = Dictionary.getSingleton()
.matchInQuantifierDict(context.getSegmentBuff(),
context.getCursor(), 1);
if (singleCharHit.isMatch()) {// 首字成量词词
// 输出当前的词
Lexeme newLexeme = new Lexeme(context.getBufferOffset(),
context.getCursor(), 1, Lexeme.TYPE_COUNT);
context.addLexeme(newLexeme);
// 同时也是词前缀
if (singleCharHit.isPrefix()) {
// 前缀匹配则放入hit列表
this.countHits.add(singleCharHit);
}
} else if (singleCharHit.isPrefix()) {// 首字为量词前缀
// 前缀匹配则放入hit列表
this.countHits.add(singleCharHit);
}
} else {
// 输入的不是中文字符
// 清空未成形的量词
this.countHits.clear();
}
// 缓冲区数据已经读完，还有尚未输出的量词
if (context.isBufferConsumed()) {
// 清空未成形的量词
this.countHits.clear();
}
}
/**
* 判断是否需要扫描量词
*
* @return
*/
private boolean needCountScan(AnalyzeContext context) {
if ((nStart != -1 && nEnd != -1) || !countHits.isEmpty()) {
// 正在处理中文数词,或者正在处理量词
return true;
} else {
// 找到一个相邻的数词
if (!context.getOrgLexemes().isEmpty()) {
Lexeme l = context.getOrgLexemes().peekLast();
if (Lexeme.TYPE_CNUM == l.getLexemeType()
|| Lexeme.TYPE_ARABIC == l.getLexemeType()) {
if (l.getBegin() + l.getLength() == context.getCursor()) {
return true;
}
}
}
}
return false;
}
/**
* 添加数词词元到结果集
*
* @param context
*/
private void outputNumLexeme(AnalyzeContext context) {
if (nStart > -1 && nEnd > -1) {
// 输出数词
Lexeme newLexeme = new Lexeme(context.getBufferOffset(), nStart,
nEnd - nStart + 1, Lexeme.TYPE_CNUM);
context.addLexeme(newLexeme);
}
}
}
src/main/java/org/wltea/analyzer/core/CharacterUtil.java 0 → 100644
View file @ e1ac0544
/**
* IK 中文分词 版本 5.0
* IK Analyzer release 5.0
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* 源代码由林良益(linliangyi2005@gmail.com)提供
* 版权声明 2012，乌龙茶工作室
* provided by Linliangyi and copyright 2012 by Oolong studio
*
* 字符集识别工具类
*/
package org.wltea.analyzer.core;
/**
*
* 字符集识别工具类
*/
class CharacterUtil {
public static final int CHAR_USELESS = 0;
public static final int CHAR_ARABIC = 0X00000001;
public static final int CHAR_ENGLISH = 0X00000002;
public static final int CHAR_CHINESE = 0X00000004;
public static final int CHAR_OTHER_CJK = 0X00000008;
/**
* 识别字符类型
*
* @param input
* @return int CharacterUtil定义的字符类型常量
*/
static int identifyCharType(char input) {
if (input >= '0' && input <= '9') {
return CHAR_ARABIC;
} else if ((input >= 'a' && input <= 'z')
|| (input >= 'A' && input <= 'Z')) {
return CHAR_ENGLISH;
} else {
Character.UnicodeBlock ub = Character.UnicodeBlock.of(input);
if (ub == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS
|| ub == Character.UnicodeBlock.CJK_COMPATIBILITY_IDEOGRAPHS
|| ub == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A) {
// 目前已知的中文字符UTF-8集合
return CHAR_CHINESE;
} else if (ub == Character.UnicodeBlock.HALFWIDTH_AND_FULLWIDTH_FORMS // 全角数字字符和日韩字符
// 韩文字符集
|| ub == Character.UnicodeBlock.HANGUL_SYLLABLES
|| ub == Character.UnicodeBlock.HANGUL_JAMO
|| ub == Character.UnicodeBlock.HANGUL_COMPATIBILITY_JAMO
// 日文字符集
|| ub == Character.UnicodeBlock.HIRAGANA // 平假名
|| ub == Character.UnicodeBlock.KATAKANA // 片假名
|| ub == Character.UnicodeBlock.KATAKANA_PHONETIC_EXTENSIONS) {
return CHAR_OTHER_CJK;
}
}
// 其他的不做处理的字符
return CHAR_USELESS;
}
/**
* 进行字符规格化（全角转半角，大写转小写处理）
*
* @param input
* @return char
*/
static char regularize(char input) {
if (input == 12288) {
input = (char) 32;
} else if (input > 65280 && input < 65375) {
input = (char) (input - 65248);
} else if (input >= 'A' && input <= 'Z') {
input += 32;
}
return input;
}
}
src/main/java/org/wltea/analyzer/core/IKArbitrator.java 0 → 100644
View file @ e1ac0544
/**
* IK 中文分词 版本 5.0
* IK Analyzer release 5.0
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* 源代码由林良益(linliangyi2005@gmail.com)提供
* 版权声明 2012，乌龙茶工作室
* provided by Linliangyi and copyright 2012 by Oolong studio
*
*/
package org.wltea.analyzer.core;
import java.util.Stack;
import java.util.TreeSet;
/**
* IK分词歧义裁决器
*/
class IKArbitrator {
IKArbitrator() {
}
/**
* 分词歧义处理
*
* @param orgLexemes
* @param useSmart
*/
void process(AnalyzeContext context, boolean useSmart) {
QuickSortSet orgLexemes = context.getOrgLexemes();
Lexeme orgLexeme = orgLexemes.pollFirst();
LexemePath crossPath = new LexemePath();
while (orgLexeme != null) {
if (!crossPath.addCrossLexeme(orgLexeme)) {
// 找到与crossPath不相交的下一个crossPath
if (crossPath.size() == 1 || !useSmart) {
// crossPath没有歧义 或者 不做歧义处理
// 直接输出当前crossPath
context.addLexemePath(crossPath);
} else {
// 对当前的crossPath进行歧义处理
QuickSortSet.Cell headCell = crossPath.getHead();
LexemePath judgeResult = this.judge(headCell,
crossPath.getPathLength());
// 输出歧义处理结果judgeResult
context.addLexemePath(judgeResult);
}
// 把orgLexeme加入新的crossPath中
crossPath = new LexemePath();
crossPath.addCrossLexeme(orgLexeme);
}
orgLexeme = orgLexemes.pollFirst();
}
// 处理最后的path
if (crossPath.size() == 1 || !useSmart) {
// crossPath没有歧义 或者 不做歧义处理
// 直接输出当前crossPath
context.addLexemePath(crossPath);
} else {
// 对当前的crossPath进行歧义处理
QuickSortSet.Cell headCell = crossPath.getHead();
LexemePath judgeResult = this.judge(headCell,
crossPath.getPathLength());
// 输出歧义处理结果judgeResult
context.addLexemePath(judgeResult);
}
}
/**
* 歧义识别
*
* @param lexemeCell
* 歧义路径链表头
* @param fullTextLength
* 歧义路径文本长度
* @param option
* 候选结果路径
* @return
*/
private LexemePath judge(QuickSortSet.Cell lexemeCell, int fullTextLength) {
// 候选路径集合
TreeSet<LexemePath> pathOptions = new TreeSet<LexemePath>();
// 候选结果路径
LexemePath option = new LexemePath();
// 对crossPath进行一次遍历,同时返回本次遍历中有冲突的Lexeme栈
Stack<QuickSortSet.Cell> lexemeStack = this.forwardPath(lexemeCell,
option);
// 当前词元链并非最理想的，加入候选路径集合
pathOptions.add(option.copy());
// 存在歧义词，处理
QuickSortSet.Cell c = null;
while (!lexemeStack.isEmpty()) {
c = lexemeStack.pop();
// 回滚词元链
this.backPath(c.getLexeme(), option);
// 从歧义词位置开始，递归，生成可选方案
this.forwardPath(c, option);
pathOptions.add(option.copy());
}
// 返回集合中的最优方案
return pathOptions.first();
}
/**
* 向前遍历，添加词元，构造一个无歧义词元组合
*
* @param LexemePath
* path
* @return
*/
private Stack<QuickSortSet.Cell> forwardPath(QuickSortSet.Cell lexemeCell,
LexemePath option) {
// 发生冲突的Lexeme栈
Stack<QuickSortSet.Cell> conflictStack = new Stack<QuickSortSet.Cell>();
QuickSortSet.Cell c = lexemeCell;
// 迭代遍历Lexeme链表
while (c != null && c.getLexeme() != null) {
if (!option.addNotCrossLexeme(c.getLexeme())) {
// 词元交叉，添加失败则加入lexemeStack栈
conflictStack.push(c);
}
c = c.getNext();
}
return conflictStack;
}
/**
* 回滚词元链，直到它能够接受指定的词元
*
* @param lexeme
* @param l
*/
private void backPath(Lexeme l, LexemePath option) {
while (option.checkCross(l)) {
option.removeTail();
}
}
}
src/main/java/org/wltea/analyzer/core/IKSegmenter.java 0 → 100644
View file @ e1ac0544
/**
* IK 中文分词 版本 5.0
* IK Analyzer release 5.0
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* 源代码由林良益(linliangyi2005@gmail.com)提供
* 版权声明 2012，乌龙茶工作室
* provided by Linliangyi and copyright 2012 by Oolong studio
*/
package org.wltea.analyzer.core;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.List;
import org.wltea.analyzer.cfg.Configuration;
import org.wltea.analyzer.cfg.DefaultConfig;
import org.wltea.analyzer.dic.Dictionary;
/**
* IK分词器主类
*
*/
public final class IKSegmenter {
// 字符窜reader
private Reader input;
// 分词器配置项
private Configuration cfg;
// 分词器上下文
private AnalyzeContext context;
// 分词处理器列表
private List<ISegmenter> segmenters;
// 分词歧义裁决器
private IKArbitrator arbitrator;
/**
* IK分词器构造函数
*
* @param input
* @param useSmart
* 为true，使用智能分词策略
*
* 非智能分词：细粒度输出所有可能的切分结果 智能分词： 合并数词和量词，对分词结果进行歧义判断
*/
public IKSegmenter(Reader input, boolean useSmart) {
this.input = input;
this.cfg = DefaultConfig.getInstance();
this.cfg.setUseSmart(useSmart);
this.init();
}
/**
* IK分词器构造函数
*
* @param input
* @param cfg
* 使用自定义的Configuration构造分词器
*
*/
public IKSegmenter(Reader input, Configuration cfg) {
this.input = input;
this.cfg = cfg;
this.init();
}
/**
* 初始化
*/
private void init() {
// 初始化词典单例
Dictionary.initial(this.cfg);
// 初始化分词上下文
this.context = new AnalyzeContext(this.cfg);
// 加载子分词器
this.segmenters = this.loadSegmenters();
// 加载歧义裁决器
this.arbitrator = new IKArbitrator();
}
/**
* 初始化词典，加载子分词器实现
*
* @return List<ISegmenter>
*/
private List<ISegmenter> loadSegmenters() {
List<ISegmenter> segmenters = new ArrayList<ISegmenter>(4);
// 处理字母的子分词器
segmenters.add(new LetterSegmenter());
// 处理中文数量词的子分词器
segmenters.add(new CN_QuantifierSegmenter());
// 处理中文词的子分词器
segmenters.add(new CJKSegmenter());
return segmenters;
}
/**
* 分词，获取下一个词元
*
* @return Lexeme 词元对象
* @throws IOException
*/
public synchronized Lexeme next() throws IOException {
Lexeme l = null;
while ((l = context.getNextLexeme()) == null) {
/*
* 从reader中读取数据，填充buffer 如果reader是分次读入buffer的，那么buffer要 进行移位处理
* 移位处理上次读入的但未处理的数据
*/
int available = context.fillBuffer(this.input);
if (available <= 0) {
// reader已经读完
context.reset();
return null;
} else {
// 初始化指针
context.initCursor();
do {
// 遍历子分词器
for (ISegmenter segmenter : segmenters) {
segmenter.analyze(context);
}
// 字符缓冲区接近读完，需要读入新的字符
if (context.needRefillBuffer()) {
break;
}
// 向前移动指针
} while (context.moveCursor());
// 重置子分词器，为下轮循环进行初始化
for (ISegmenter segmenter : segmenters) {
segmenter.reset();
}
}
// 对分词进行歧义处理
this.arbitrator.process(context, this.cfg.useSmart());
// 将分词结果输出到结果集，并处理未切分的单个CJK字符
context.outputToResult();
// 记录本次分词的缓冲区位移
context.markBufferOffset();
}
return l;
}
/**
* 重置分词器到初始状态
*
* @param input
*/
public synchronized void reset(Reader input) {
this.input = input;
context.reset();
for (ISegmenter segmenter : segmenters) {
segmenter.reset();
}
}
}
src/main/java/org/wltea/analyzer/core/ISegmenter.java 0 → 100644
View file @ e1ac0544
/**
* IK 中文分词 版本 5.0
* IK Analyzer release 5.0
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* 源代码由林良益(linliangyi2005@gmail.com)提供
* 版权声明 2012，乌龙茶工作室
* provided by Linliangyi and copyright 2012 by Oolong studio
*
*/
package org.wltea.analyzer.core;
/**
*
* 子分词器接口
*/
interface ISegmenter {
/**
* 从分析器读取下一个可能分解的词元对象
*
* @param context
* 分词算法上下文
*/
void analyze(AnalyzeContext context);
/**
* 重置子分析器状态
*/
void reset();
}
src/main/java/org/wltea/analyzer/core/LetterSegmenter.java 0 → 100644
View file @ e1ac0544
/**
* IK 中文分词 版本 5.0
* IK Analyzer release 5.0
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* 源代码由林良益(linliangyi2005@gmail.com)提供
* 版权声明 2012，乌龙茶工作室
* provided by Linliangyi and copyright 2012 by Oolong studio
*
*/
package org.wltea.analyzer.core;
import java.util.Arrays;
/**
*
* 英文字符及阿拉伯数字子分词器
*/
class LetterSegmenter implements ISegmenter {
// 子分词器标签
static final String SEGMENTER_NAME = "LETTER_SEGMENTER";
// 链接符号
private static final char[] Letter_Connector = new char[] { '#', '&', '+',
'-', '.', '@', '_' };
// 数字符号
private static final char[] Num_Connector = new char[] { ',', '.' };
/*
* 词元的开始位置， 同时作为子分词器状态标识 当start > -1 时，标识当前的分词器正在处理字符
*/
private int start;
/*
* 记录词元结束位置 end记录的是在词元中最后一个出现的Letter但非Sign_Connector的字符的位置
*/
private int end;
/*
* 字母起始位置
*/
private int englishStart;
/*
* 字母结束位置
*/
private int englishEnd;
/*
* 阿拉伯数字起始位置
*/
private int arabicStart;
/*
* 阿拉伯数字结束位置
*/
private int arabicEnd;
LetterSegmenter() {
Arrays.sort(Letter_Connector);
Arrays.sort(Num_Connector);
this.start = -1;
this.end = -1;
this.englishStart = -1;
this.englishEnd = -1;
this.arabicStart = -1;
this.arabicEnd = -1;
}
/*
* (non-Javadoc)
*
* @see org.wltea.analyzer.core.ISegmenter#analyze(org.wltea.analyzer.core.
* AnalyzeContext)
*/
public void analyze(AnalyzeContext context) {
boolean bufferLockFlag = false;
// 处理英文字母
bufferLockFlag = this.processEnglishLetter(context) || bufferLockFlag;
// 处理阿拉伯字母
bufferLockFlag = this.processArabicLetter(context) || bufferLockFlag;
// 处理混合字母(这个要放最后处理，可以通过QuickSortSet排除重复)
bufferLockFlag = this.processMixLetter(context) || bufferLockFlag;
// 判断是否锁定缓冲区
if (bufferLockFlag) {
context.lockBuffer(SEGMENTER_NAME);
} else {
// 对缓冲区解锁
context.unlockBuffer(SEGMENTER_NAME);
}
}
/*
* (non-Javadoc)
*
* @see org.wltea.analyzer.core.ISegmenter#reset()
*/
public void reset() {
this.start = -1;
this.end = -1;
this.englishStart = -1;
this.englishEnd = -1;
this.arabicStart = -1;
this.arabicEnd = -1;
}
/**
* 处理数字字母混合输出 如：windos2000 | linliangyi2005@gmail.com
*
* @param input
* @param context
* @return
*/
private boolean processMixLetter(AnalyzeContext context) {
boolean needLock = false;
if (this.start == -1) {// 当前的分词器尚未开始处理字符
if (CharacterUtil.CHAR_ARABIC == context.getCurrentCharType()
|| CharacterUtil.CHAR_ENGLISH == context
.getCurrentCharType()) {
// 记录起始指针的位置,标明分词器进入处理状态
this.start = context.getCursor();
this.end = start;
}
} else {// 当前的分词器正在处理字符
if (CharacterUtil.CHAR_ARABIC == context.getCurrentCharType()
|| CharacterUtil.CHAR_ENGLISH == context
.getCurrentCharType()) {
// 记录下可能的结束位置
this.end = context.getCursor();
} else if (CharacterUtil.CHAR_USELESS == context
.getCurrentCharType()
&& this.isLetterConnector(context.getCurrentChar())) {
// 记录下可能的结束位置
this.end = context.getCursor();
} else {
// 遇到非Letter字符，输出词元
Lexeme newLexeme = new Lexeme(context.getBufferOffset(),
this.start, this.end - this.start + 1,
Lexeme.TYPE_LETTER);
context.addLexeme(newLexeme);
this.start = -1;
this.end = -1;
}
}
// 判断缓冲区是否已经读完
if (context.isBufferConsumed()) {
if (this.start != -1 && this.end != -1) {
// 缓冲以读完，输出词元
Lexeme newLexeme = new Lexeme(context.getBufferOffset(),
this.start, this.end - this.start + 1,
Lexeme.TYPE_LETTER);
context.addLexeme(newLexeme);
this.start = -1;
this.end = -1;
}
}
// 判断是否锁定缓冲区
if (this.start == -1 && this.end == -1) {
// 对缓冲区解锁
needLock = false;
} else {
needLock = true;
}
return needLock;
}
/**
* 处理纯英文字母输出
*
* @param context
* @return
*/
private boolean processEnglishLetter(AnalyzeContext context) {
boolean needLock = false;
if (this.englishStart == -1) {// 当前的分词器尚未开始处理英文字符
if (CharacterUtil.CHAR_ENGLISH == context.getCurrentCharType()) {
// 记录起始指针的位置,标明分词器进入处理状态
this.englishStart = context.getCursor();
this.englishEnd = this.englishStart;
}
} else {// 当前的分词器正在处理英文字符
if (CharacterUtil.CHAR_ENGLISH == context.getCurrentCharType()) {
// 记录当前指针位置为结束位置
this.englishEnd = context.getCursor();
} else {
// 遇到非English字符,输出词元
Lexeme newLexeme = new Lexeme(context.getBufferOffset(),
this.englishStart, this.englishEnd - this.englishStart
+ 1, Lexeme.TYPE_ENGLISH);
context.addLexeme(newLexeme);
this.englishStart = -1;
this.englishEnd = -1;
}
}
// 判断缓冲区是否已经读完
if (context.isBufferConsumed()) {
if (this.englishStart != -1 && this.englishEnd != -1) {
// 缓冲以读完，输出词元
Lexeme newLexeme = new Lexeme(context.getBufferOffset(),
this.englishStart, this.englishEnd - this.englishStart
+ 1, Lexeme.TYPE_ENGLISH);
context.addLexeme(newLexeme);
this.englishStart = -1;
this.englishEnd = -1;
}
}
// 判断是否锁定缓冲区
if (this.englishStart == -1 && this.englishEnd == -1) {
// 对缓冲区解锁
needLock = false;
} else {
needLock = true;
}
return needLock;
}
/**
* 处理阿拉伯数字输出
*
* @param context
* @return
*/
private boolean processArabicLetter(AnalyzeContext context) {
boolean needLock = false;
if (this.arabicStart == -1) {// 当前的分词器尚未开始处理数字字符
if (CharacterUtil.CHAR_ARABIC == context.getCurrentCharType()) {
// 记录起始指针的位置,标明分词器进入处理状态
this.arabicStart = context.getCursor();
this.arabicEnd = this.arabicStart;
}
} else {// 当前的分词器正在处理数字字符
if (CharacterUtil.CHAR_ARABIC == context.getCurrentCharType()) {
// 记录当前指针位置为结束位置
this.arabicEnd = context.getCursor();
} else if (CharacterUtil.CHAR_USELESS == context
.getCurrentCharType()
&& this.isNumConnector(context.getCurrentChar())) {
// 不输出数字，但不标记结束
} else {
// //遇到非Arabic字符,输出词元
Lexeme newLexeme = new Lexeme(context.getBufferOffset(),
this.arabicStart,
this.arabicEnd - this.arabicStart + 1,
Lexeme.TYPE_ARABIC);
context.addLexeme(newLexeme);
this.arabicStart = -1;
this.arabicEnd = -1;
}
}
// 判断缓冲区是否已经读完
if (context.isBufferConsumed()) {
if (this.arabicStart != -1 && this.arabicEnd != -1) {
// 生成已切分的词元
Lexeme newLexeme = new Lexeme(context.getBufferOffset(),
this.arabicStart,
this.arabicEnd - this.arabicStart + 1,
Lexeme.TYPE_ARABIC);
context.addLexeme(newLexeme);
this.arabicStart = -1;
this.arabicEnd = -1;
}
}
// 判断是否锁定缓冲区
if (this.arabicStart == -1 && this.arabicEnd == -1) {
// 对缓冲区解锁
needLock = false;
} else {
needLock = true;
}
return needLock;
}
/**
* 判断是否是字母连接符号
*
* @param input
* @return
*/
private boolean isLetterConnector(char input) {
int index = Arrays.binarySearch(Letter_Connector, input);
return index >= 0;
}
/**
* 判断是否是数字连接符号
*
* @param input
* @return
*/
private boolean isNumConnector(char input) {
int index = Arrays.binarySearch(Num_Connector, input);
return index >= 0;
}
}
src/main/java/org/wltea/analyzer/core/Lexeme.java 0 → 100644
View file @ e1ac0544
/**
* IK 中文分词 版本 5.0
* IK Analyzer release 5.0
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* 源代码由林良益(linliangyi2005@gmail.com)提供
* 版权声明 2012，乌龙茶工作室
* provided by Linliangyi and copyright 2012 by Oolong studio
*
*/
package org.wltea.analyzer.core;
/**
* IK词元对象
*/
public class Lexeme implements Comparable<Lexeme> {
// lexemeType常量
// 未知
public static final int TYPE_UNKNOWN = 0;
// 英文
public static final int TYPE_ENGLISH = 1;
// 数字
public static final int TYPE_ARABIC = 2;
// 英文数字混合
public static final int TYPE_LETTER = 3;
// 中文词元
public static final int TYPE_CNWORD = 4;
// 中文单字
public static final int TYPE_CNCHAR = 64;
// 日韩文字
public static final int TYPE_OTHER_CJK = 8;
// 中文数词
public static final int TYPE_CNUM = 16;
// 中文量词
public static final int TYPE_COUNT = 32;
// 中文数量词
public static final int TYPE_CQUAN = 48;
// 词元的起始位移
private int offset;
// 词元的相对起始位置
private int begin;
// 词元的长度
private int length;
// 词元文本
private String lexemeText;
// 词元类型
private int lexemeType;
public Lexeme(int offset, int begin, int length, int lexemeType) {
this.offset = offset;
this.begin = begin;
if (length < 0) {
throw new IllegalArgumentException("length < 0");
}
this.length = length;
this.lexemeType = lexemeType;
}
/*
* 判断词元相等算法 起始位置偏移、起始位置、终止位置相同
*
* @see java.lang.Object#equals(Object o)
*/
public boolean equals(Object o) {
if (o == null) {
return false;
}
if (this == o) {
return true;
}
if (o instanceof Lexeme) {
Lexeme other = (Lexeme) o;
if (this.offset == other.getOffset()
&& this.begin == other.getBegin()
&& this.length == other.getLength()) {
return true;
} else {
return false;
}
} else {
return false;
}
}
/*
* 词元哈希编码算法
*
* @see java.lang.Object#hashCode()
*/
public int hashCode() {
int absBegin = getBeginPosition();
int absEnd = getEndPosition();
return (absBegin * 37) + (absEnd * 31)
+ ((absBegin * absEnd) % getLength()) * 11;
}
/*
* 词元在排序集合中的比较算法
*
* @see java.lang.Comparable#compareTo(java.lang.Object)
*/
public int compareTo(Lexeme other) {
// 起始位置优先
if (this.begin < other.getBegin()) {
return -1;
} else if (this.begin == other.getBegin()) {
// 词元长度优先
if (this.length > other.getLength()) {
return -1;
} else if (this.length == other.getLength()) {
return 0;
} else {// this.length < other.getLength()
return 1;
}
} else {// this.begin > other.getBegin()
return 1;
}
}
public int getOffset() {
return offset;
}
public void setOffset(int offset) {
this.offset = offset;
}
public int getBegin() {
return begin;
}
/**
* 获取词元在文本中的起始位置
*
* @return int
*/
public int getBeginPosition() {
return offset + begin;
}
public void setBegin(int begin) {
this.begin = begin;
}
/**
* 获取词元在文本中的结束位置
*
* @return int
*/
public int getEndPosition() {
return offset + begin + length;
}
/**
* 获取词元的字符长度
*
* @return int
*/
public int getLength() {
return this.length;
}
public void setLength(int length) {
if (this.length < 0) {
throw new IllegalArgumentException("length < 0");
}
this.length = length;
}
/**
* 获取词元的文本内容
*
* @return String
*/
public String getLexemeText() {
if (lexemeText == null) {
return "";
}
return lexemeText;
}
public void setLexemeText(String lexemeText) {
if (lexemeText == null) {
this.lexemeText = "";
this.length = 0;
} else {
this.lexemeText = lexemeText;
this.length = lexemeText.length();
}
}
/**
* 获取词元类型
*
* @return int
*/
public int getLexemeType() {
return lexemeType;
}
/**
* 获取词元类型标示字符串
*
* @return String
*/
public String getLexemeTypeString() {
switch (lexemeType) {
case TYPE_ENGLISH:
return "ENGLISH";
case TYPE_ARABIC:
return "ARABIC";
case TYPE_LETTER:
return "LETTER";
case TYPE_CNWORD:
return "CN_WORD";
case TYPE_CNCHAR:
return "CN_CHAR";
case TYPE_OTHER_CJK:
return "OTHER_CJK";
case TYPE_COUNT:
return "COUNT";
case TYPE_CNUM:
return "TYPE_CNUM";
case TYPE_CQUAN:
return "TYPE_CQUAN";
default:
return "UNKONW";
}
}
public void setLexemeType(int lexemeType) {
this.lexemeType = lexemeType;
}
/**
* 合并两个相邻的词元
*
* @param l
* @param lexemeType
* @return boolean 词元是否成功合并
*/
public boolean append(Lexeme l, int lexemeType) {
if (l != null && this.getEndPosition() == l.getBeginPosition()) {
this.length += l.getLength();
this.lexemeType = lexemeType;
return true;
} else {
return false;
}
}
/**
*
*/
public String toString() {
StringBuffer strbuf = new StringBuffer();
strbuf.append(this.getBeginPosition()).append("-")
.append(this.getEndPosition());
strbuf.append(" : ").append(this.lexemeText).append(" : \t");
strbuf.append(this.getLexemeTypeString());
return strbuf.toString();
}
}
src/main/java/org/wltea/analyzer/core/LexemePath.java 0 → 100644
View file @ e1ac0544
/**
* IK 中文分词 版本 5.0
* IK Analyzer release 5.0
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* 源代码由林良益(linliangyi2005@gmail.com)提供
* 版权声明 2012，乌龙茶工作室
* provided by Linliangyi and copyright 2012 by Oolong studio
*
*/
package org.wltea.analyzer.core;
/**
* Lexeme链（路径）
*/
class LexemePath extends QuickSortSet implements Comparable<LexemePath> {
// 起始位置
private int pathBegin;
// 结束
private int pathEnd;
// 词元链的有效字符长度
private int payloadLength;
LexemePath() {
this.pathBegin = -1;
this.pathEnd = -1;
this.payloadLength = 0;
}
/**
* 向LexemePath追加相交的Lexeme
*
* @param lexeme
* @return
*/
boolean addCrossLexeme(Lexeme lexeme) {
if (this.isEmpty()) {
this.addLexeme(lexeme);
this.pathBegin = lexeme.getBegin();
this.pathEnd = lexeme.getBegin() + lexeme.getLength();
this.payloadLength += lexeme.getLength();
return true;
} else if (this.checkCross(lexeme)) {
this.addLexeme(lexeme);
if (lexeme.getBegin() + lexeme.getLength() > this.pathEnd) {
this.pathEnd = lexeme.getBegin() + lexeme.getLength();
}
this.payloadLength = this.pathEnd - this.pathBegin;
return true;
} else {
return false;
}
}
/**
* 向LexemePath追加不相交的Lexeme
*
* @param lexeme
* @return
*/
boolean addNotCrossLexeme(Lexeme lexeme) {
if (this.isEmpty()) {
this.addLexeme(lexeme);
this.pathBegin = lexeme.getBegin();
this.pathEnd = lexeme.getBegin() + lexeme.getLength();
this.payloadLength += lexeme.getLength();
return true;
} else if (this.checkCross(lexeme)) {
return false;
} else {
this.addLexeme(lexeme);
this.payloadLength += lexeme.getLength();
Lexeme head = this.peekFirst();
this.pathBegin = head.getBegin();
Lexeme tail = this.peekLast();
this.pathEnd = tail.getBegin() + tail.getLength();
return true;
}
}
/**
* 移除尾部的Lexeme
*
* @return
*/
Lexeme removeTail() {
Lexeme tail = this.pollLast();
if (this.isEmpty()) {
this.pathBegin = -1;
this.pathEnd = -1;
this.payloadLength = 0;
} else {
this.payloadLength -= tail.getLength();
Lexeme newTail = this.peekLast();
this.pathEnd = newTail.getBegin() + newTail.getLength();
}
return tail;
}
/**
* 检测词元位置交叉（有歧义的切分）
*
* @param lexeme
* @return
*/
boolean checkCross(Lexeme lexeme) {
return (lexeme.getBegin() >= this.pathBegin && lexeme.getBegin() < this.pathEnd)
|| (this.pathBegin >= lexeme.getBegin() && this.pathBegin < lexeme
.getBegin() + lexeme.getLength());
}
int getPathBegin() {
return pathBegin;
}
int getPathEnd() {
return pathEnd;
}
/**
* 获取Path的有效词长
*
* @return
*/
int getPayloadLength() {
return this.payloadLength;
}
/**
* 获取LexemePath的路径长度
*
* @return
*/
int getPathLength() {
return this.pathEnd - this.pathBegin;
}
/**
* X权重（词元长度积）
*
* @return
*/
int getXWeight() {
int product = 1;
Cell c = this.getHead();
while (c != null && c.getLexeme() != null) {
product *= c.getLexeme().getLength();
c = c.getNext();
}
return product;
}
/**
* 词元位置权重
*
* @return
*/
int getPWeight() {
int pWeight = 0;
int p = 0;
Cell c = this.getHead();
while (c != null && c.getLexeme() != null) {
p++;
pWeight += p * c.getLexeme().getLength();
c = c.getNext();
}
return pWeight;
}
LexemePath copy() {
LexemePath theCopy = new LexemePath();
theCopy.pathBegin = this.pathBegin;
theCopy.pathEnd = this.pathEnd;
theCopy.payloadLength = this.payloadLength;
Cell c = this.getHead();
while (c != null && c.getLexeme() != null) {
theCopy.addLexeme(c.getLexeme());
c = c.getNext();
}
return theCopy;
}
public int compareTo(LexemePath o) {
// 比较有效文本长度
if (this.payloadLength > o.payloadLength) {
return -1;
} else if (this.payloadLength < o.payloadLength) {
return 1;
} else {
// 比较词元个数，越少越好
if (this.size() < o.size()) {
return -1;
} else if (this.size() > o.size()) {
return 1;
} else {
// 路径跨度越大越好
if (this.getPathLength() > o.getPathLength()) {
return -1;
} else if (this.getPathLength() < o.getPathLength()) {
return 1;
} else {
// 根据统计学结论，逆向切分概率高于正向切分，因此位置越靠后的优先
if (this.pathEnd > o.pathEnd) {
return -1;
} else if (pathEnd < o.pathEnd) {
return 1;
} else {
// 词长越平均越好
if (this.getXWeight() > o.getXWeight()) {
return -1;
} else if (this.getXWeight() < o.getXWeight()) {
return 1;
} else {
// 词元位置权重比较
if (this.getPWeight() > o.getPWeight()) {
return -1;
} else if (this.getPWeight() < o.getPWeight()) {
return 1;
}
}
}
}
}
}
return 0;
}
public String toString() {
StringBuffer sb = new StringBuffer();
sb.append("pathBegin : ").append(pathBegin).append("\r\n");
sb.append("pathEnd : ").append(pathEnd).append("\r\n");
sb.append("payloadLength : ").append(payloadLength).append("\r\n");
Cell head = this.getHead();
while (head != null) {
sb.append("lexeme : ").append(head.getLexeme()).append("\r\n");
head = head.getNext();
}
return sb.toString();
}
}
src/main/java/org/wltea/analyzer/core/QuickSortSet.java 0 → 100644
View file @ e1ac0544
/**
* IK 中文分词 版本 5.0
* IK Analyzer release 5.0
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* 源代码由林良益(linliangyi2005@gmail.com)提供
* 版权声明 2012，乌龙茶工作室
* provided by Linliangyi and copyright 2012 by Oolong studio
*
*/
package org.wltea.analyzer.core;
/**
* IK分词器专用的Lexem快速排序集合
*/
class QuickSortSet {
// 链表头
private Cell head;
// 链表尾
private Cell tail;
// 链表的实际大小
private int size;
QuickSortSet() {
this.size = 0;
}
/**
* 向链表集合添加词元
*
* @param lexeme
*/
boolean addLexeme(Lexeme lexeme) {
Cell newCell = new Cell(lexeme);
if (this.size == 0) {
this.head = newCell;
this.tail = newCell;
this.size++;
return true;
} else {
if (this.tail.compareTo(newCell) == 0) {// 词元与尾部词元相同，不放入集合
return false;
} else if (this.tail.compareTo(newCell) < 0) {// 词元接入链表尾部
this.tail.next = newCell;
newCell.prev = this.tail;
this.tail = newCell;
this.size++;
return true;
} else if (this.head.compareTo(newCell) > 0) {// 词元接入链表头部
this.head.prev = newCell;
newCell.next = this.head;
this.head = newCell;
this.size++;
return true;
} else {
// 从尾部上逆
Cell index = this.tail;
while (index != null && index.compareTo(newCell) > 0) {
index = index.prev;
}
if (index.compareTo(newCell) == 0) {// 词元与集合中的词元重复，不放入集合
return false;
} else if (index.compareTo(newCell) < 0) {// 词元插入链表中的某个位置
newCell.prev = index;
newCell.next = index.next;
index.next.prev = newCell;
index.next = newCell;
this.size++;
return true;
}
}
}
return false;
}
/**
* 返回链表头部元素
*
* @return
*/
Lexeme peekFirst() {
if (this.head != null) {
return this.head.lexeme;
}
return null;
}
/**
* 取出链表集合的第一个元素
*
* @return Lexeme
*/
Lexeme pollFirst() {
if (this.size == 1) {
Lexeme first = this.head.lexeme;
this.head = null;
this.tail = null;
this.size--;
return first;
} else if (this.size > 1) {
Lexeme first = this.head.lexeme;
this.head = this.head.next;
this.size--;
return first;
} else {
return null;
}
}
/**
* 返回链表尾部元素
*
* @return
*/
Lexeme peekLast() {
if (this.tail != null) {
return this.tail.lexeme;
}
return null;
}
/**
* 取出链表集合的最后一个元素
*
* @return Lexeme
*/
Lexeme pollLast() {
if (this.size == 1) {
Lexeme last = this.head.lexeme;
this.head = null;
this.tail = null;
this.size--;
return last;
} else if (this.size > 1) {
Lexeme last = this.tail.lexeme;
this.tail = this.tail.prev;
this.size--;
return last;
} else {
return null;
}
}
/**
* 返回集合大小
*
* @return
*/
int size() {
return this.size;
}
/**
* 判断集合是否为空
*
* @return
*/
boolean isEmpty() {
return this.size == 0;
}
/**
* 返回lexeme链的头部
*
* @return
*/
Cell getHead() {
return this.head;
}
/**
*
* IK 中文分词 版本 5.0 IK Analyzer release 5.0
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the
* License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
* 源代码由林良益(linliangyi2005@gmail.com)提供 版权声明 2012，乌龙茶工作室 provided by
* Linliangyi and copyright 2012 by Oolong studio
*
* QuickSortSet集合单元
*
*/
class Cell implements Comparable<Cell> {
private Cell prev;
private Cell next;
private Lexeme lexeme;
Cell(Lexeme lexeme) {
if (lexeme == null) {
throw new IllegalArgumentException("lexeme must not be null");
}
this.lexeme = lexeme;
}
public int compareTo(Cell o) {
return this.lexeme.compareTo(o.lexeme);
}
public Cell getPrev() {
return this.prev;
}
public Cell getNext() {
return this.next;
}
public Lexeme getLexeme() {
return this.lexeme;
}
}
}
src/main/java/org/wltea/analyzer/dic/DictSegment.java 0 → 100644
View file @ e1ac0544
/**
*
* IK 中文分词 版本 5.0
* IK Analyzer release 5.0
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* 源代码由林良益(linliangyi2005@gmail.com)提供
* 版权声明 2012，乌龙茶工作室
* provided by Linliangyi and copyright 2012 by Oolong studio
*
*/
package org.wltea.analyzer.dic;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
/**
* 词典树分段，表示词典树的一个分枝
*/
class DictSegment implements Comparable<DictSegment> {
// 公用字典表，存储汉字
private static final Map<Character, Character> charMap = new HashMap<Character, Character>(
16, 0.95f);
// 数组大小上限
private static final int ARRAY_LENGTH_LIMIT = 3;
// Map存储结构
private Map<Character, DictSegment> childrenMap;
// 数组方式存储结构
private DictSegment[] childrenArray;
// 当前节点上存储的字符
private Character nodeChar;
// 当前节点存储的Segment数目
// storeSize <=ARRAY_LENGTH_LIMIT ，使用数组存储， storeSize >ARRAY_LENGTH_LIMIT
// ,则使用Map存储
private int storeSize = 0;
// 当前DictSegment状态 ,默认 0 , 1表示从根节点到当前节点的路径表示一个词
private int nodeState = 0;
DictSegment(Character nodeChar) {
if (nodeChar == null) {
throw new IllegalArgumentException("参数为空异常，字符不能为空");
}
this.nodeChar = nodeChar;
}
Character getNodeChar() {
return nodeChar;
}
/*
* 判断是否有下一个节点
*/
boolean hasNextNode() {
return this.storeSize > 0;
}
/**
* 匹配词段
*
* @param charArray
* @return Hit
*/
Hit match(char[] charArray) {
return this.match(charArray, 0, charArray.length, null);
}
/**
* 匹配词段
*
* @param charArray
* @param begin
* @param length
* @return Hit
*/
Hit match(char[] charArray, int begin, int length) {
return this.match(charArray, begin, length, null);
}
/**
* 匹配词段
*
* @param charArray
* @param begin
* @param length
* @param searchHit
* @return Hit
*/
Hit match(char[] charArray, int begin, int length, Hit searchHit) {
if (searchHit == null) {
// 如果hit为空，新建
searchHit = new Hit();
// 设置hit的其实文本位置
searchHit.setBegin(begin);
} else {
// 否则要将HIT状态重置
searchHit.setUnmatch();
}
// 设置hit的当前处理位置
searchHit.setEnd(begin);
Character keyChar = new Character(charArray[begin]);
DictSegment ds = null;
// 引用实例变量为本地变量，避免查询时遇到更新的同步问题
DictSegment[] segmentArray = this.childrenArray;
Map<Character, DictSegment> segmentMap = this.childrenMap;
// STEP1 在节点中查找keyChar对应的DictSegment
if (segmentArray != null) {
// 在数组中查找
DictSegment keySegment = new DictSegment(keyChar);
int position = Arrays.binarySearch(segmentArray, 0, this.storeSize,
keySegment);
if (position >= 0) {
ds = segmentArray[position];
}
} else if (segmentMap != null) {
// 在map中查找
ds = (DictSegment) segmentMap.get(keyChar);
}
// STEP2 找到DictSegment，判断词的匹配状态，是否继续递归，还是返回结果
if (ds != null) {
if (length > 1) {
// 词未匹配完，继续往下搜索
return ds.match(charArray, begin + 1, length - 1, searchHit);
} else if (length == 1) {
// 搜索最后一个char
if (ds.nodeState == 1) {
// 添加HIT状态为完全匹配
searchHit.setMatch();
}
if (ds.hasNextNode()) {
// 添加HIT状态为前缀匹配
searchHit.setPrefix();
// 记录当前位置的DictSegment
searchHit.setMatchedDictSegment(ds);
}
return searchHit;
}
}
// STEP3 没有找到DictSegment， 将HIT设置为不匹配
return searchHit;
}
/**
* 加载填充词典片段
*
* @param charArray
*/
void fillSegment(char[] charArray) {
this.fillSegment(charArray, 0, charArray.length, 1);
}
/**
* 屏蔽词典中的一个词
*
* @param charArray
*/
void disableSegment(char[] charArray) {
this.fillSegment(charArray, 0, charArray.length, 0);
}
/**
* 加载填充词典片段
*
* @param charArray
* @param begin
* @param length
* @param enabled
*/
private synchronized void fillSegment(char[] charArray, int begin,
int length, int enabled) {
// 获取字典表中的汉字对象
Character beginChar = new Character(charArray[begin]);
Character keyChar = charMap.get(beginChar);
// 字典中没有该字，则将其添加入字典
if (keyChar == null) {
charMap.put(beginChar, beginChar);
keyChar = beginChar;
}
// 搜索当前节点的存储，查询对应keyChar的keyChar，如果没有则创建
DictSegment ds = lookforSegment(keyChar, enabled);
if (ds != null) {
// 处理keyChar对应的segment
if (length > 1) {
// 词元还没有完全加入词典树
ds.fillSegment(charArray, begin + 1, length - 1, enabled);
} else if (length == 1) {
// 已经是词元的最后一个char,设置当前节点状态为enabled，
// enabled=1表明一个完整的词，enabled=0表示从词典中屏蔽当前词
ds.nodeState = enabled;
}
}
}
/**
* 查找本节点下对应的keyChar的segment *
*
* @param keyChar
* @param create
* =1如果没有找到，则创建新的segment ; =0如果没有找到，不创建，返回null
* @return
*/
private DictSegment lookforSegment(Character keyChar, int create) {
DictSegment ds = null;
if (this.storeSize <= ARRAY_LENGTH_LIMIT) {
// 获取数组容器，如果数组未创建则创建数组
DictSegment[] segmentArray = getChildrenArray();
// 搜寻数组
DictSegment keySegment = new DictSegment(keyChar);
int position = Arrays.binarySearch(segmentArray, 0, this.storeSize,
keySegment);
if (position >= 0) {
ds = segmentArray[position];
}
// 遍历数组后没有找到对应的segment
if (ds == null && create == 1) {
ds = keySegment;
if (this.storeSize < ARRAY_LENGTH_LIMIT) {
// 数组容量未满，使用数组存储
segmentArray[this.storeSize] = ds;
// segment数目+1
this.storeSize++;
Arrays.sort(segmentArray, 0, this.storeSize);
} else {
// 数组容量已满，切换Map存储
// 获取Map容器，如果Map未创建,则创建Map
Map<Character, DictSegment> segmentMap = getChildrenMap();
// 将数组中的segment迁移到Map中
migrate(segmentArray, segmentMap);
// 存储新的segment
segmentMap.put(keyChar, ds);
// segment数目+1 ， 必须在释放数组前执行storeSize++ ， 确保极端情况下，不会取到空的数组
this.storeSize++;
// 释放当前的数组引用
this.childrenArray = null;
}
}
} else {
// 获取Map容器，如果Map未创建,则创建Map
Map<Character, DictSegment> segmentMap = getChildrenMap();
// 搜索Map
ds = (DictSegment) segmentMap.get(keyChar);
if (ds == null && create == 1) {
// 构造新的segment
ds = new DictSegment(keyChar);
segmentMap.put(keyChar, ds);
// 当前节点存储segment数目+1
this.storeSize++;
}
}
return ds;
}
/**
* 获取数组容器 线程同步方法
*/
private DictSegment[] getChildrenArray() {
if (this.childrenArray == null) {
synchronized (this) {
if (this.childrenArray == null) {
this.childrenArray = new DictSegment[ARRAY_LENGTH_LIMIT];
}
}
}
return this.childrenArray;
}
/**
* 获取Map容器 线程同步方法
*/
private Map<Character, DictSegment> getChildrenMap() {
if (this.childrenMap == null) {
synchronized (this) {
if (this.childrenMap == null) {
this.childrenMap = new HashMap<Character, DictSegment>(
ARRAY_LENGTH_LIMIT * 2, 0.8f);
}
}
}
return this.childrenMap;
}
/**
* 将数组中的segment迁移到Map中
*
* @param segmentArray
*/
private void migrate(DictSegment[] segmentArray,
Map<Character, DictSegment> segmentMap) {
for (DictSegment segment : segmentArray) {
if (segment != null) {
segmentMap.put(segment.nodeChar, segment);
}
}
}
/**
* 实现Comparable接口
*
* @param o
* @return int
*/
public int compareTo(DictSegment o) {
// 对当前节点存储的char进行比较
return this.nodeChar.compareTo(o.nodeChar);
}
}
src/main/java/org/wltea/analyzer/dic/Dictionary.java 0 → 100644
View file @ e1ac0544
/**
* IK 中文分词 版本 5.0
* IK Analyzer release 5.0
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* 源代码由林良益(linliangyi2005@gmail.com)提供
* 版权声明 2012，乌龙茶工作室
* provided by Linliangyi and copyright 2012 by Oolong studio
*
*
*/
package org.wltea.analyzer.dic;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Collection;
import java.util.List;
import org.wltea.analyzer.cfg.Configuration;
/**
* 词典管理类,单子模式
*/
public class Dictionary {
/*
* 词典单子实例
*/
private static Dictionary singleton;
/*
* 主词典对象
*/
private DictSegment _MainDict;
/*
* 停止词词典
*/
private DictSegment _StopWordDict;
/*
* 量词词典
*/
private DictSegment _QuantifierDict;
/**
* 配置对象
*/
private Configuration cfg;
private Dictionary(Configuration cfg) {
this.cfg = cfg;
this.loadMainDict();
this.loadStopWordDict();
this.loadQuantifierDict();
}
/**
* 词典初始化 由于IK Analyzer的词典采用Dictionary类的静态方法进行词典初始化
* 只有当Dictionary类被实际调用时，才会开始载入词典， 这将延长首次分词操作的时间 该方法提供了一个在应用加载阶段就初始化字典的手段
*
* @return Dictionary
*/
public static Dictionary initial(Configuration cfg) {
if (singleton == null) {
synchronized (Dictionary.class) {
if (singleton == null) {
singleton = new Dictionary(cfg);
return singleton;
}
}
}
return singleton;
}
/**
* 获取词典单子实例
*
* @return Dictionary 单例对象
*/
public static Dictionary getSingleton() {
if (singleton == null) {
throw new IllegalStateException("词典尚未初始化，请先调用initial方法");
}
return singleton;
}
/**
* 批量加载新词条
*
* @param words
* Collection<String>词条列表
*/
public void addWords(Collection<String> words) {
if (words != null) {
for (String word : words) {
if (word != null) {
// 批量加载词条到主内存词典中
singleton._MainDict.fillSegment(word.trim().toLowerCase()
.toCharArray());
}
}
}
}
/**
* 批量移除（屏蔽）词条
*
* @param words
*/
public void disableWords(Collection<String> words) {
if (words != null) {
for (String word : words) {
if (word != null) {
// 批量屏蔽词条
singleton._MainDict.disableSegment(word.trim()
.toLowerCase().toCharArray());
}
}
}
}
/**
* 检索匹配主词典
*
* @param charArray
* @return Hit 匹配结果描述
*/
public Hit matchInMainDict(char[] charArray) {
return singleton._MainDict.match(charArray);
}
/**
* 检索匹配主词典
*
* @param charArray
* @param begin
* @param length
* @return Hit 匹配结果描述
*/
public Hit matchInMainDict(char[] charArray, int begin, int length) {
return singleton._MainDict.match(charArray, begin, length);
}
/**
* 检索匹配量词词典
*
* @param charArray
* @param begin
* @param length
* @return Hit 匹配结果描述
*/
public Hit matchInQuantifierDict(char[] charArray, int begin, int length) {
return singleton._QuantifierDict.match(charArray, begin, length);
}
/**
* 从已匹配的Hit中直接取出DictSegment，继续向下匹配
*
* @param charArray
* @param currentIndex
* @param matchedHit
* @return Hit
*/
public Hit matchWithHit(char[] charArray, int currentIndex, Hit matchedHit) {
DictSegment ds = matchedHit.getMatchedDictSegment();
return ds.match(charArray, currentIndex, 1, matchedHit);
}
/**
* 判断是否是停止词
*
* @param charArray
* @param begin
* @param length
* @return boolean
*/
public boolean isStopWord(char[] charArray, int begin, int length) {
return singleton._StopWordDict.match(charArray, begin, length)
.isMatch();
}
/**
* 加载主词典及扩展词典
*/
private void loadMainDict() {
// 建立一个主词典实例
_MainDict = new DictSegment((char) 0);
// 读取主词典文件
InputStream is = this.getClass().getClassLoader()
.getResourceAsStream(cfg.getMainDictionary());
if (is == null) {
throw new RuntimeException("Main Dictionary not found!!!");
}
try {
BufferedReader br = new BufferedReader(new InputStreamReader(is,
"UTF-8"), 512);
String theWord = null;
do {
theWord = br.readLine();
if (theWord != null && !"".equals(theWord.trim())) {
_MainDict.fillSegment(theWord.trim().toLowerCase()
.toCharArray());
}
} while (theWord != null);
} catch (IOException ioe) {
System.err.println("Main Dictionary loading exception.");
ioe.printStackTrace();
} finally {
try {
if (is != null) {
is.close();
is = null;
}
} catch (IOException e) {
e.printStackTrace();
}
}
// 加载扩展词典
this.loadExtDict();
}
/**
* 加载用户配置的扩展词典到主词库表
*/
private void loadExtDict() {
// 加载扩展词典配置
List<String> extDictFiles = cfg.getExtDictionarys();
if (extDictFiles != null) {
InputStream is = null;
for (String extDictName : extDictFiles) {
// 读取扩展词典文件
System.out.println("加载扩展词典：" + extDictName);
is = this.getClass().getClassLoader()
.getResourceAsStream(extDictName);
// 如果找不到扩展的字典，则忽略
if (is == null) {
continue;
}
try {
BufferedReader br = new BufferedReader(
new InputStreamReader(is, "UTF-8"), 512);
String theWord = null;
do {
theWord = br.readLine();
if (theWord != null && !"".equals(theWord.trim())) {
// 加载扩展词典数据到主内存词典中
// System.out.println(theWord);
_MainDict.fillSegment(theWord.trim().toLowerCase()
.toCharArray());
}
} while (theWord != null);
} catch (IOException ioe) {
System.err
.println("Extension Dictionary loading exception.");
ioe.printStackTrace();
} finally {
try {
if (is != null) {
is.close();
is = null;
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
}
/**
* 加载用户扩展的停止词词典
*/
private void loadStopWordDict() {
// 建立一个主词典实例
_StopWordDict = new DictSegment((char) 0);
// 加载扩展停止词典
List<String> extStopWordDictFiles = cfg.getExtStopWordDictionarys();
if (extStopWordDictFiles != null) {
InputStream is = null;
for (String extStopWordDictName : extStopWordDictFiles) {
System.out.println("加载扩展停止词典：" + extStopWordDictName);
// 读取扩展词典文件
is = this.getClass().getClassLoader()
.getResourceAsStream(extStopWordDictName);
// 如果找不到扩展的字典，则忽略
if (is == null) {
continue;
}
try {
BufferedReader br = new BufferedReader(
new InputStreamReader(is, "UTF-8"), 512);
String theWord = null;
do {
theWord = br.readLine();
if (theWord != null && !"".equals(theWord.trim())) {
// System.out.println(theWord);
// 加载扩展停止词典数据到内存中
_StopWordDict.fillSegment(theWord.trim()
.toLowerCase().toCharArray());
}
} while (theWord != null);
} catch (IOException ioe) {
System.err
.println("Extension Stop word Dictionary loading exception.");
ioe.printStackTrace();
} finally {
try {
if (is != null) {
is.close();
is = null;
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
}
/**
* 加载量词词典
*/
private void loadQuantifierDict() {
// 建立一个量词典实例
_QuantifierDict = new DictSegment((char) 0);
// 读取量词词典文件
InputStream is = this.getClass().getClassLoader()
.getResourceAsStream(cfg.getQuantifierDicionary());
if (is == null) {
throw new RuntimeException("Quantifier Dictionary not found!!!");
}
try {
BufferedReader br = new BufferedReader(new InputStreamReader(is,
"UTF-8"), 512);
String theWord = null;
do {
theWord = br.readLine();
if (theWord != null && !"".equals(theWord.trim())) {
_QuantifierDict.fillSegment(theWord.trim().toLowerCase()
.toCharArray());
}
} while (theWord != null);
} catch (IOException ioe) {
System.err.println("Quantifier Dictionary loading exception.");
ioe.printStackTrace();
} finally {
try {
if (is != null) {
is.close();
is = null;
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
src/main/java/org/wltea/analyzer/dic/Hit.java 0 → 100644
View file @ e1ac0544
/**
*
* IK 中文分词 版本 5.0
* IK Analyzer release 5.0
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* 源代码由林良益(linliangyi2005@gmail.com)提供
* 版权声明 2012，乌龙茶工作室
* provided by Linliangyi and copyright 2012 by Oolong studio
*
*/
package org.wltea.analyzer.dic;
/**
* 表示一次词典匹配的命中
*/
public class Hit {
// Hit不匹配
private static final int UNMATCH = 0x00000000;
// Hit完全匹配
private static final int MATCH = 0x00000001;
// Hit前缀匹配
private static final int PREFIX = 0x00000010;
// 该HIT当前状态，默认未匹配
private int hitState = UNMATCH;
// 记录词典匹配过程中，当前匹配到的词典分支节点
private DictSegment matchedDictSegment;
/*
* 词段开始位置
*/
private int begin;
/*
* 词段的结束位置
*/
private int end;
/**
* 判断是否完全匹配
*/
public boolean isMatch() {
return (this.hitState & MATCH) > 0;
}
/**
*
*/
public void setMatch() {
this.hitState = this.hitState | MATCH;
}
/**
* 判断是否是词的前缀
*/
public boolean isPrefix() {
return (this.hitState & PREFIX) > 0;
}
/**
*
*/
public void setPrefix() {
this.hitState = this.hitState | PREFIX;
}
/**
* 判断是否是不匹配
*/
public boolean isUnmatch() {
return this.hitState == UNMATCH;
}
/**
*
*/
public void setUnmatch() {
this.hitState = UNMATCH;
}
public DictSegment getMatchedDictSegment() {
return matchedDictSegment;
}
public void setMatchedDictSegment(DictSegment matchedDictSegment) {
this.matchedDictSegment = matchedDictSegment;
}
public int getBegin() {
return begin;
}
public void setBegin(int begin) {
this.begin = begin;
}
public int getEnd() {
return end;
}
public void setEnd(int end) {
this.end = end;
}
}
src/main/java/org/wltea/analyzer/dic/main2012.dic 0 → 100644
View file @ e1ac0544
This diff is collapsed.
src/main/java/org/wltea/analyzer/dic/quantifier.dic 0 → 100644
View file @ e1ac0544
丈
下
世
世纪
两
个
中
串
亩
人
介
付
代
件
任
份
伏
伙
位
位数
例
倍
像素
元
克
克拉
公亩
公克
公分
公升
公尺
公担
公斤
公里
公顷
具
册
出
刀
分
分钟
分米
划
列
则
刻
剂
剑
副
加仑
勺
包
匙
匹
区
千克
千米
升
卷
厅
厘
厘米
双
发
口
句
只
台
叶
号
名
吨
听
员
周
周年
品
回
团
圆
圈
地
场
块
坪
堆
声
壶
处
夜
大
天
头
套
女
孔
字
宗
室
家
寸
对
封
尊
小时
尺
尾
局
层
届
岁
师
帧
幅
幕
幢
平方
平方公尺
平方公里
平方分米
平方厘米
平方码
平方米
平方英寸
平方英尺
平方英里
平米
年
年代
年级
度
座
式
引
张
成
战
截
户
房
所
扇
手
打
批
把
折
担
拍
招
拨
拳
指
掌
排
撮
支
文
斗
斤
方
族
日
时
曲
月
月份
期
本
朵
村
束
条
来
杯
枚
枝
枪
架
柄
柜
栋
栏
株
样
根
格
案
桌
档
桩
桶
梯
棵
楼
次
款
步
段
毛
毫
毫升
毫米
毫克
池
洲
派
海里
滴
炮
点
点钟
片
版
环
班
瓣
瓶
生
男
画
界
盆
盎司
盏
盒
盘
相
眼
石
码
碗
碟
磅
种
科
秒
秒钟
窝
立方公尺
立方分米
立方厘米
立方码
立方米
立方英寸
立方英尺
站
章
笔
等
筐
筒
箱
篇
篓
篮
簇
米
类
粒
级
组
维
缕
缸
罐
网
群
股
脚
船
艇
艘
色
节
英亩
英寸
英尺
英里
行
袋
角
言
课
起
趟
路
车
转
轮
辆
辈
连
通
遍
部
里
重
针
钟
钱
锅
门
间
队
阶段
隅
集
页
顶
顷
项
顿
颗
餐
首
\ No newline at end of file
src/main/java/org/wltea/analyzer/lucene/IKAnalyzer.java 0 → 100644
View file @ e1ac0544
/**
* IK 中文分词 版本 5.0.1
* IK Analyzer release 5.0.1
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* 源代码由林良益(linliangyi2005@gmail.com)提供
* 版权声明 2012，乌龙茶工作室
* provided by Linliangyi and copyright 2012 by Oolong studio
*
*/
package org.wltea.analyzer.lucene;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
/**
* IK分词器，Lucene Analyzer接口实现 兼容Lucene 6.x版本
*/
public final class IKAnalyzer extends Analyzer {
private boolean useSmart;
public boolean useSmart() {
return useSmart;
}
public void setUseSmart(boolean useSmart) {
this.useSmart = useSmart;
}
/**
* IK分词器Lucene Analyzer接口实现类
*
* 默认细粒度切分算法
*/
public IKAnalyzer() {
this(false);
}
/**
* IK分词器Lucene Analyzer接口实现类
*
* @param useSmart
* 当为true时，分词器进行智能切分
*/
public IKAnalyzer(boolean useSmart) {
super();
this.useSmart = useSmart;
}
/**
* 重载Analyzer接口，构造分词组件
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer _IKTokenizer = new IKTokenizer(this.useSmart());
return new TokenStreamComponents(_IKTokenizer);
}
}
src/main/java/org/wltea/analyzer/lucene/IKTokenizer.java 0 → 100644
View file @ e1ac0544
/**
* IK 中文分词 版本 5.0.1
* IK Analyzer release 5.0.1
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* 源代码由林良益(linliangyi2005@gmail.com)提供
* 版权声明 2012，乌龙茶工作室
* provided by Linliangyi and copyright 2012 by Oolong studio
*
*
*/
package org.wltea.analyzer.lucene;
import java.io.IOException;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme;
/**
* IK分词器 Lucene Tokenizer适配器类 兼容Lucene 6.x版本
*/
public final class IKTokenizer extends Tokenizer {
// IK分词器实现
private IKSegmenter _IKImplement;
// 词元文本属性
private final CharTermAttribute termAtt;
// 词元位移属性
private final OffsetAttribute offsetAtt;
// 词元分类属性（该属性分类参考org.wltea.analyzer.core.Lexeme中的分类常量）
private final TypeAttribute typeAtt;
// 记录最后一个词元的结束位置
private int endPosition;
/**
* Lucene 6.x Tokenizer适配器类构造函数
*
* @param in
* @param useSmart
*/
public IKTokenizer(boolean useSmart) {
super();
offsetAtt = addAttribute(OffsetAttribute.class);
termAtt = addAttribute(CharTermAttribute.class);
typeAtt = addAttribute(TypeAttribute.class);
_IKImplement = new IKSegmenter(input, useSmart);
}
/*
* (non-Javadoc)
*
* @see org.apache.lucene.analysis.TokenStream#incrementToken()
*/
@Override
public boolean incrementToken() throws IOException {
// 清除所有的词元属性
clearAttributes();
Lexeme nextLexeme = _IKImplement.next();
if (nextLexeme != null) {
// 将Lexeme转成Attributes
// 设置词元文本
termAtt.append(nextLexeme.getLexemeText());
// 设置词元长度
termAtt.setLength(nextLexeme.getLength());
// 设置词元位移
offsetAtt.setOffset(nextLexeme.getBeginPosition(),
nextLexeme.getEndPosition());
// 记录分词的最后位置
endPosition = nextLexeme.getEndPosition();
// 记录词元分类
typeAtt.setType(nextLexeme.getLexemeTypeString());
// 返会true告知还有下个词元
return true;
}
// 返会false告知词元输出完毕
return false;
}
/*
* (non-Javadoc)
*
* @see org.apache.lucene.analysis.Tokenizer#reset(java.io.Reader)
*/
@Override
public void reset() throws IOException {
super.reset();
_IKImplement.reset(input);
}
@Override
public final void end() {
// set final offset
int finalOffset = correctOffset(this.endPosition);
offsetAtt.setOffset(finalOffset, finalOffset);
}
}
src/main/java/org/wltea/analyzer/query/SWMCQueryBuilder.java 0 → 100644
View file @ e1ac0544
This diff is collapsed.
src/main/java/org/wltea/analyzer/sample/IKAnalzyerDemo.java 0 → 100644
View file @ e1ac0544
This diff is collapsed.
src/main/resources/IKAnalyzer.cfg.xml 0 → 100644
View file @ e1ac0544
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>IK Analyzer 扩展配置</comment>
<!--用户可以在这里配置自己的扩展字典 -->
<entry key="ext_dict">ext.dic;</entry>
<!--用户可以在这里配置自己的扩展停止词字典-->
<entry key="ext_stopwords">stopword.dic;</entry>
</properties>
\ No newline at end of file
src/main/resources/ext.dic 0 → 100644
View file @ e1ac0544
This diff is collapsed.
src/main/resources/org/wltea/analyzer/dic/main2012.dic 0 → 100644
View file @ e1ac0544
This diff is collapsed.
src/main/resources/org/wltea/analyzer/dic/quantifier.dic 0 → 100644
View file @ e1ac0544
This diff is collapsed.
src/main/resources/stopword.dic 0 → 100644
View file @ e1ac0544
a
an
and
are
as
at
be
but
by
for
if
in
into
is
it
no
not
of
on
or
such
that
the
their
then
there
these
they
this
to
was
will
with
也
了
仍
从
以
使
则
却
又
及
对
就
并
很
或
把
是
的
着
给
而
被
让
在
还
比
等
当
与
于
但
\ No newline at end of file
target/classes/IKAnalyzer.cfg.xml 0 → 100644
View file @ e1ac0544
This diff is collapsed.
target/classes/ext.dic 0 → 100644
View file @ e1ac0544
This diff is collapsed.
target/classes/org/wltea/analyzer/dic/main2012.dic 0 → 100644
View file @ e1ac0544
This diff is collapsed.
target/classes/org/wltea/analyzer/dic/quantifier.dic 0 → 100644
View file @ e1ac0544
This diff is collapsed.
target/classes/stopword.dic 0 → 100644
View file @ e1ac0544
This diff is collapsed.
target/maven-archiver/pom.properties 0 → 100644
View file @ e1ac0544
#Generated by Maven
#Tue Aug 04 10:03:10 CST 2020
version=6.4.0
groupId=com.seaboat
artifactId=IKAnalyzer
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment