[分词] 基于 Lucene8 版本的逆向最大匹配分词器 (依赖本地词典)

import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.TokenFilter;import org.apache.lucene.analysis.Tokenizer;import org.apache.lucene.analysis.core.LowerCaseFilter;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import java.io.BufferedReader;import java.io.IOException;import java.io.InputStream;import java.io.InputStreamReader;import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.nio.file.Path;import java.util.HashSet;import java.util.Set;
public class ReverseMaximumMatchingAnalyzer extends Analyzer {    private Set<String> dictionary;
    public ReverseMaximumMatchingAnalyzer(Path dictionaryPath) {        this.dictionary = loadDictionary(dictionaryPath);    }
    private Set<String> loadDictionary(Path dictionaryPath) {        Set<String> dictionary = new HashSet<>();        try (InputStream inputStream = Files.newInputStream(dictionaryPath);             InputStreamReader streamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);             BufferedReader reader = new BufferedReader(streamReader)) {            String line;            while ((line = reader.readLine()) != null) {                dictionary.add(line.trim());            }        } catch (Exception e) {        }        return dictionary;    }
    @Override    protected TokenStreamComponents createComponents(String fieldName) {        Tokenizer tokenizer = new ReverseMaximumMatchingTokenizer(dictionary);        TokenFilter filter = new LowerCaseFilter(tokenizer);        return new TokenStreamComponents(tokenizer, filter);    }
    private static class ReverseMaximumMatchingTokenizer extends Tokenizer {        private char[] buffer = new char[256];        private int length = 0;        private int offset = 0;
        private CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);        private OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
        private Set<String> dictionary;
        public ReverseMaximumMatchingTokenizer(Set<String> dictionary) {            this.dictionary = dictionary;        }
        @Override        public boolean incrementToken() throws IOException {            while (true) {                if (offset < length) {                    clearAttributes();                    int start = offset;                    while (offset < length) {                        offset++;                        int end = offset;                        while (end >= start) {                            String token = new String(buffer, start, end - start);                            if (dictionary.contains(token)) {                                termAttribute.copyBuffer(buffer, start, end - start);                                offsetAttribute.setOffset(correctOffset(start), correctOffset(end));                                return true;                            }                            end--;                        }                    }                } else if (input.read(buffer) != -1) {                    length = buffer.length;                    offset = 0;                } else {                    return false;                }            }        }
        @Override        public void reset() throws IOException {            super.reset();            offset = length = 0;        }
        @Override        public void end() throws IOException {            super.end();            offset = length;        }    }}