mirror of
https://github.com/Alfresco/SearchServices.git
synced 2026-09-16 18:12:56 +00:00
Merge branch 'fix/SEARCH_2244' into 'master'
Revert "[ SEARCH-1369 ] Custom StopFilter for detecting synonym tokens + UT" See merge request search_discovery/insightengine!492
This commit is contained in:
Andrea Gazzarini
-103
@@ -1,103 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2005-2010 Alfresco Software Limited.
|
||||
*
|
||||
* This file is part of Alfresco
|
||||
*
|
||||
* Alfresco is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Alfresco is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package org.alfresco.repo.search.impl.lucene.analysis;
|
||||
|
||||
import org.apache.lucene.analysis.CharArraySet;
|
||||
import org.apache.lucene.analysis.StopFilter;
|
||||
import org.apache.lucene.analysis.TokenStream;
|
||||
import org.apache.lucene.analysis.core.StopFilterFactory;
|
||||
import org.apache.lucene.analysis.synonym.SynonymGraphFilter;
|
||||
import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
|
||||
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* A {@link StopFilter} which doesn't remove stop tokens previously marked as part of a synonym.
|
||||
* When using a stop filter in a query analyzer which already includes a synonym filter, there are two options:
|
||||
* the filter could be defined before or after the synonym filter.
|
||||
*
|
||||
* However, the first way (before) doesn’t make so much sense, because terms that are stopwords and that are, at the same
|
||||
* time, part of a synonym will be removed before the synonym detection. As consequence of that no synonym detection will happen.
|
||||
*
|
||||
* If we have
|
||||
*
|
||||
* <ul>
|
||||
* <li>a stopwords list consisting of one term (of)</li>
|
||||
* <li>and a single synonym definition like "out of warranty,oow"</li>
|
||||
* <li>a query which should match the synonym: q=out of warranty</li>
|
||||
* </ul>
|
||||
*
|
||||
* a stop filter defined before the synonym filter would remove the "of" term from the query, therefore causing the missing
|
||||
* synonym detection. If we postpone the stop filter after the synonym filter, the synonym will be detected, but the
|
||||
* stop filter later would still remove the "of" token causing the open issue well described in https://issues.apache.org/jira/browse/LUCENE-4065
|
||||
*
|
||||
* Being related with FilteringTokenFilter (the StopFilter superclass) LUCENE-4065 has a wider scope which doesn't affect only
|
||||
* stopwords removal. However, in order to mitigate the issue above, this custom stop filter takes in account the typeAttribute of the
|
||||
* incoming tokens; it removes them only if they are stopwords and if they haven't been marked as part of a synonym ({@link SynonymGraphFilter#TYPE_SYNONYM})
|
||||
*
|
||||
* @see SynonymGraphFilter
|
||||
* @author Andrea Gazzarini
|
||||
*/
|
||||
public class SynonymAwareStopFilterFactory extends StopFilterFactory
|
||||
{
|
||||
class SynonymAwareStopFilter extends StopFilter
|
||||
{
|
||||
|
||||
private TypeAttribute typeAttribute = addAttribute(TypeAttribute.class);
|
||||
private PositionLengthAttribute positionLengthAttribute = addAttribute(PositionLengthAttribute.class);
|
||||
|
||||
private int synonymSpans;
|
||||
|
||||
SynonymAwareStopFilter(TokenStream in, CharArraySet stopwords)
|
||||
{
|
||||
super(in, stopwords);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean accept()
|
||||
{
|
||||
if (isSynonymToken())
|
||||
{
|
||||
synonymSpans = positionLengthAttribute.getPositionLength() > 1
|
||||
? positionLengthAttribute.getPositionLength()
|
||||
: 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
return (--synonymSpans > 0) || super.accept();
|
||||
}
|
||||
|
||||
private boolean isSynonymToken()
|
||||
{
|
||||
return SynonymGraphFilter.TYPE_SYNONYM.equals(typeAttribute.type());
|
||||
}
|
||||
}
|
||||
|
||||
public SynonymAwareStopFilterFactory(Map<String, String> args)
|
||||
{
|
||||
super(args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public TokenStream create(TokenStream input)
|
||||
{
|
||||
return new SynonymAwareStopFilter(input, getStopWords());
|
||||
}
|
||||
}
|
||||
-278
@@ -1,278 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2005-2014 Alfresco Software Limited.
|
||||
*
|
||||
* This file is part of Alfresco
|
||||
*
|
||||
* Alfresco is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Alfresco is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package org.alfresco.repo.search.impl.lucene.analysis;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.lucene.analysis.Analyzer;
|
||||
import org.apache.lucene.analysis.TokenStream;
|
||||
import org.apache.lucene.analysis.Tokenizer;
|
||||
import org.apache.lucene.analysis.icu.segmentation.ICUTokenizerFactory;
|
||||
import org.apache.lucene.analysis.standard.StandardTokenizer;
|
||||
import org.apache.lucene.analysis.synonym.SynonymGraphFilter;
|
||||
import org.apache.lucene.analysis.synonym.SynonymGraphFilterFactory;
|
||||
import org.apache.lucene.analysis.tokenattributes.*;
|
||||
import org.apache.solr.core.SolrResourceLoader;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TemporaryFolder;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.junit.MockitoJUnitRunner;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.io.StringReader;
|
||||
import java.util.*;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static java.util.Arrays.asList;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
/**
|
||||
* Test case which illustrates some multi-term synonyms scenarios.
|
||||
*
|
||||
* @author Andrea Gazzarini
|
||||
*/
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class MultiTermSynonymsTest
|
||||
{
|
||||
private Analyzer analyzer;
|
||||
|
||||
private SolrResourceLoader resourceLoader;
|
||||
private final String ALPHANUM_TYPE = StandardTokenizer.TOKEN_TYPES[StandardTokenizer.ALPHANUM];
|
||||
private final String SYNONYM_TYPE = SynonymGraphFilter.TYPE_SYNONYM;
|
||||
|
||||
@Rule
|
||||
public TemporaryFolder testFolder = new TemporaryFolder();
|
||||
|
||||
/**
|
||||
* The Unit test makes use of the same filter chain used within the solrconfig.xml,
|
||||
* see the query analyzer of "text_en" field type:
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@link org.apache.lucene.analysis.synonym.SynonymGraphFilter}</li>
|
||||
* <li>{@link org.alfresco.repo.search.impl.lucene.analysis.SynonymAwareStopFilterFactory.SynonymAwareStopFilter}</li>
|
||||
* </ul>
|
||||
*
|
||||
*/
|
||||
@Before
|
||||
public void setUp() throws Exception
|
||||
{
|
||||
File stopwordsFile = testFolder.newFile("stopwords.txt");
|
||||
File synonyms = testFolder.newFile("synonyms.txt");
|
||||
|
||||
FileUtils.writeLines(stopwordsFile, asList("of", "my"));
|
||||
FileUtils.writeLines(synonyms, asList(
|
||||
"OOW,Out of Warranty",
|
||||
"Apache Solr,Apache Solr")); // this is how we define a multi-term concept without any synonym
|
||||
|
||||
System.setProperty("solr.solr.home", stopwordsFile.getParentFile().getAbsolutePath());
|
||||
resourceLoader = new SolrResourceLoader();
|
||||
|
||||
analyzer = new Analyzer()
|
||||
{
|
||||
@Override
|
||||
protected TokenStreamComponents createComponents(String fieldName)
|
||||
{
|
||||
try
|
||||
{
|
||||
ICUTokenizerFactory tokenizerFactory = new ICUTokenizerFactory(Collections.emptyMap());
|
||||
tokenizerFactory.inform(resourceLoader);
|
||||
Tokenizer tokenizer = tokenizerFactory.create();
|
||||
|
||||
Map<String, String> synonymsConfig = new HashMap<>();
|
||||
synonymsConfig.put("synonyms", "synonyms.txt");
|
||||
synonymsConfig.put("ignoreCase", "true");
|
||||
|
||||
SynonymGraphFilterFactory synonymsFactory = new SynonymGraphFilterFactory(synonymsConfig);
|
||||
synonymsFactory.inform(resourceLoader);
|
||||
TokenStream synonymsFilter = synonymsFactory.create(tokenizer);
|
||||
|
||||
Map<String, String> stopFilterConfig = new HashMap<>();
|
||||
stopFilterConfig.put("words", "stopwords.txt");
|
||||
stopFilterConfig.put("ignoreCase", "true");
|
||||
|
||||
SynonymAwareStopFilterFactory stopWordsFactory = new SynonymAwareStopFilterFactory(stopFilterConfig);
|
||||
stopWordsFactory.inform(resourceLoader);
|
||||
TokenStream sinkStream = stopWordsFactory.create(synonymsFilter);
|
||||
|
||||
return new TokenStreamComponents(tokenizer, sinkStream);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
throw new IllegalArgumentException(exception);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* No synonyms and no stopwords (plain and subsequent tokens with no removal).
|
||||
* No Graph is being generated from the outcoming stream.
|
||||
*/
|
||||
@Test
|
||||
public void noSynonmsNoStopwords() throws Exception
|
||||
{
|
||||
final String text = "no synonyms and no stopwords";
|
||||
final List<PackedTokenAttributeImpl> expected = asList(
|
||||
token("no", 0, 2, 1, 1, ALPHANUM_TYPE),
|
||||
token("synonyms", 3, 11, 1, 1, ALPHANUM_TYPE),
|
||||
token("and", 12, 15, 1, 1, ALPHANUM_TYPE),
|
||||
token("no", 16, 18, 1, 1, ALPHANUM_TYPE),
|
||||
token("stopwords", 19, 28, 1, 1, ALPHANUM_TYPE));
|
||||
|
||||
assertAnalysisCorrectness(text, expected);
|
||||
}
|
||||
|
||||
/**
|
||||
* A synonym is detected and the correct graph is generated.
|
||||
* Note the original text (out *of* warranty) contains a stopwords but being that part of a synonym, it won't be
|
||||
* removed so a the graph will later generate a correct phrase query (i.e. "out of warranty")
|
||||
*/
|
||||
@Test
|
||||
public void multiTermConceptWithSynonyms() throws Exception
|
||||
{
|
||||
final String text = "Car is Out of warranty";
|
||||
final List<PackedTokenAttributeImpl> expected = asList(
|
||||
token("Car", 0, 3, 1, 1, ALPHANUM_TYPE),
|
||||
token("is", 4, 6, 1, 1, ALPHANUM_TYPE),
|
||||
token("oow", 7, 22, 1, 3, SYNONYM_TYPE),
|
||||
token("Out", 7, 10, 0, 1, ALPHANUM_TYPE),
|
||||
token("of", 11, 13, 1, 1, ALPHANUM_TYPE),
|
||||
token("warranty", 14, 22, 1, 1, ALPHANUM_TYPE));
|
||||
|
||||
assertAnalysisCorrectness(text, expected);
|
||||
}
|
||||
|
||||
/**
|
||||
* This is the same test as above but instead of starting from the expanded form, the original text contains the
|
||||
* acronym (oow). Expected result is the same, the only difference is in the different tokens types.
|
||||
*/
|
||||
@Test
|
||||
public void acronymsIsExpandedAndContainsStopwords() throws Exception
|
||||
{
|
||||
final String text = "Car is OOW";
|
||||
final List<PackedTokenAttributeImpl> expected = asList(
|
||||
token("Car", 0, 3, 1, 1, ALPHANUM_TYPE),
|
||||
token("is", 4, 6, 1, 1, ALPHANUM_TYPE),
|
||||
token("out", 7, 10, 1, 1, SYNONYM_TYPE),
|
||||
token("OOW", 7, 10, 0, 3, ALPHANUM_TYPE),
|
||||
token("of", 7, 10, 1, 1, SYNONYM_TYPE),
|
||||
token("warranty", 7, 10, 1, 1, SYNONYM_TYPE));
|
||||
|
||||
assertAnalysisCorrectness(text, expected);
|
||||
}
|
||||
|
||||
/**
|
||||
* If a multi-term concept needs to be captured, then its definition must be double in the synonyms file.
|
||||
*
|
||||
*/
|
||||
@Test
|
||||
public void multiTermConceptWithoutSynonyms() throws Exception
|
||||
{
|
||||
final String text = "apache solr, the enterprise search platform";
|
||||
final List<PackedTokenAttributeImpl> expected = asList(
|
||||
token("apache", 0, 11, 1, 1, SYNONYM_TYPE),
|
||||
token("apache", 0, 6, 0, 2, ALPHANUM_TYPE),
|
||||
token("solr", 0, 11, 1, 2, SYNONYM_TYPE),
|
||||
token("solr", 7, 11, 1, 1, ALPHANUM_TYPE),
|
||||
token("the", 13, 16, 1, 1, ALPHANUM_TYPE),
|
||||
token("enterprise", 17, 27, 1, 1, ALPHANUM_TYPE),
|
||||
token("search", 28, 34, 1, 1, ALPHANUM_TYPE),
|
||||
token("platform", 35, 43, 1, 1, ALPHANUM_TYPE));
|
||||
|
||||
assertAnalysisCorrectness(text, expected);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the text analysis produces the expected tokens graph.
|
||||
*
|
||||
* @param text the input text.
|
||||
* @param expectedTokens the list of tokens in the order expected from the stream.
|
||||
*/
|
||||
private void assertAnalysisCorrectness(String text, List<PackedTokenAttributeImpl> expectedTokens) throws IOException
|
||||
{
|
||||
final List<PackedTokenAttributeImpl> actualTokens = new ArrayList<>();
|
||||
|
||||
try (Reader reader = new StringReader(text);
|
||||
TokenStream tokenStream = analyzer.tokenStream("dummy_field", reader))
|
||||
{
|
||||
CharTermAttribute termAtt = tokenStream.addAttribute(CharTermAttribute.class);
|
||||
TypeAttribute typeAtt = tokenStream.addAttribute(TypeAttribute.class);
|
||||
PositionIncrementAttribute positionIncrementAtt = tokenStream.addAttribute(PositionIncrementAttribute.class);
|
||||
PositionLengthAttribute positionLengthAtt = tokenStream.addAttribute(PositionLengthAttribute.class);
|
||||
OffsetAttribute offsetAtt = tokenStream.addAttribute(OffsetAttribute.class);
|
||||
|
||||
tokenStream.reset();
|
||||
while (tokenStream.incrementToken())
|
||||
{
|
||||
actualTokens.add(
|
||||
token(new String(termAtt.buffer(), 0, termAtt.length()),
|
||||
offsetAtt.startOffset(),
|
||||
offsetAtt.endOffset(),
|
||||
positionIncrementAtt.getPositionIncrement(),
|
||||
positionLengthAtt.getPositionLength(),
|
||||
typeAtt.type()));
|
||||
}
|
||||
tokenStream.end();
|
||||
}
|
||||
|
||||
assertEquals(mismatchTokensListMessage(expectedTokens, actualTokens), expectedTokens, actualTokens);
|
||||
}
|
||||
|
||||
private String mismatchTokensListMessage(List<PackedTokenAttributeImpl> expected, List<PackedTokenAttributeImpl> actual)
|
||||
{
|
||||
Function<PackedTokenAttributeImpl, String> toString =
|
||||
token -> "term=" + token.toString() +
|
||||
",startOffset=" + token.startOffset() +
|
||||
",endOffset=" + token.endOffset() +
|
||||
",posInc=" + token.getPositionIncrement() +
|
||||
",posLength=" + token.getPositionLength() +
|
||||
",type=" + token.type();
|
||||
|
||||
String expectedMessage = expected.stream().map(toString).collect(Collectors.joining("\n"));
|
||||
String actualMessage = actual.stream().map(toString).collect(Collectors.joining("\n"));
|
||||
|
||||
return "*** EXPECTED ***\n" + expectedMessage + "\n*** ACTUAL ***\n" + actualMessage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a test {@link PackedTokenAttributeImpl} used for test verifications.
|
||||
*
|
||||
* @param text the token content.
|
||||
* @param positionLength the token position length.
|
||||
* @param type the token type.
|
||||
* @param startOffset the token start offset.
|
||||
* @param endOffset the token end offset.
|
||||
* @param positionIncrement the token position increment.
|
||||
* @return a test {@link PackedTokenAttributeImpl} used for test verifications.
|
||||
*/
|
||||
private PackedTokenAttributeImpl token(String text, int startOffset, int endOffset, int positionIncrement, int positionLength, String type)
|
||||
{
|
||||
PackedTokenAttributeImpl token = new PackedTokenAttributeImpl();
|
||||
token.setPositionLength(positionLength);
|
||||
token.setType(type);
|
||||
token.setOffset(startOffset, endOffset);
|
||||
token.setPositionIncrement(positionIncrement);
|
||||
token.copyBuffer(text.toCharArray(), 0, text.length());
|
||||
return token;
|
||||
}
|
||||
}
|
||||
-236
@@ -1,236 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2005-2014 Alfresco Software Limited.
|
||||
*
|
||||
* This file is part of Alfresco
|
||||
*
|
||||
* Alfresco is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Alfresco is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package org.alfresco.repo.search.impl.lucene.analysis;
|
||||
|
||||
import static java.util.Arrays.asList;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.lucene.analysis.Analyzer;
|
||||
import org.apache.lucene.analysis.TokenStream;
|
||||
import org.apache.lucene.analysis.Tokenizer;
|
||||
import org.apache.lucene.analysis.standard.StandardTokenizerFactory;
|
||||
import org.apache.lucene.analysis.synonym.SynonymGraphFilterFactory;
|
||||
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
|
||||
import org.apache.solr.core.SolrResourceLoader;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TemporaryFolder;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.junit.MockitoJUnitRunner;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.io.StringReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Test case for {@link SynonymAwareStopFilterFactory.SynonymAwareStopFilter}
|
||||
*
|
||||
* @author Andrea Gazzarini
|
||||
*/
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class SynonymAwareStopFilterTest
|
||||
{
|
||||
|
||||
private Analyzer analyzer;
|
||||
|
||||
private SolrResourceLoader resourceLoader;
|
||||
|
||||
private final List<String> stopwords = asList("of", "my");
|
||||
|
||||
@Rule
|
||||
public TemporaryFolder testFolder = new TemporaryFolder();
|
||||
|
||||
/**
|
||||
* Setup fixture for this test case.
|
||||
* Specifically:
|
||||
*
|
||||
* <ul>
|
||||
* <li>A test analyzer composed by a StandardTokenizer, SynonymsGraphFilter and the class under test (SynonymAwareStopFilter)</li>
|
||||
* <li>a synonyms file containing two rules: ["OOW,out of warranty","transfer phone number,port number"]</li>
|
||||
* <li>a stopwords file with two stopwords: ["of","my"]</li>
|
||||
* </ul>
|
||||
*/
|
||||
@Before
|
||||
public void setUp() throws Exception
|
||||
{
|
||||
File stopwordsFile = testFolder.newFile("stopwords.txt");
|
||||
File synonyms = testFolder.newFile("synonyms.txt");
|
||||
|
||||
FileUtils.writeLines(stopwordsFile, stopwords);
|
||||
FileUtils.writeLines(synonyms, asList("OOW,Out of Warranty","transfer Phone Number,Port Number"));
|
||||
|
||||
System.setProperty("solr.solr.home", stopwordsFile.getParentFile().getAbsolutePath());
|
||||
resourceLoader = new SolrResourceLoader();
|
||||
|
||||
analyzer = new Analyzer()
|
||||
{
|
||||
@Override
|
||||
protected TokenStreamComponents createComponents(String fieldName)
|
||||
{
|
||||
try
|
||||
{
|
||||
Tokenizer tokenizer = new StandardTokenizerFactory(Collections.emptyMap()).create();
|
||||
|
||||
Map<String, String> synonymsConfig = new HashMap<>();
|
||||
synonymsConfig.put("synonyms", "synonyms.txt");
|
||||
synonymsConfig.put("ignoreCase", "true");
|
||||
|
||||
SynonymGraphFilterFactory synonymsFactory = new SynonymGraphFilterFactory(synonymsConfig);
|
||||
synonymsFactory.inform(resourceLoader);
|
||||
TokenStream synonymsFilter = synonymsFactory.create(tokenizer);
|
||||
|
||||
Map<String, String> stopFilterConfig = new HashMap<>();
|
||||
stopFilterConfig.put("words", "stopwords.txt");
|
||||
stopFilterConfig.put("ignoreCase", "true");
|
||||
|
||||
SynonymAwareStopFilterFactory stopWordsFactory = new SynonymAwareStopFilterFactory(stopFilterConfig);
|
||||
stopWordsFactory.inform(resourceLoader);
|
||||
|
||||
TokenStream sinkStream = stopWordsFactory.create(synonymsFilter);
|
||||
|
||||
return new TokenStreamComponents(tokenizer, sinkStream);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
throw new IllegalArgumentException(exception);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* In case we have no synonyms and no stopwords then the input should be left untouched.
|
||||
*/
|
||||
@Test
|
||||
public void noSynonmsNoStopwords() throws Exception
|
||||
{
|
||||
String text = "no synonyms and no stopwords here";
|
||||
List<String> expected = asList(text.split("\\W"));
|
||||
|
||||
assertAnalysisCorrectness(text, expected);
|
||||
}
|
||||
|
||||
/**
|
||||
* If a stream doesn't contain any SYNONYM token, the custom stop filter behaves like its superclass (StopFilter)
|
||||
*/
|
||||
@Test
|
||||
public void noSynonymsOnlyStopwords() throws Exception
|
||||
{
|
||||
String text = "No synonyms detection. However we have two stopwords of and my which will be removed";
|
||||
List<String> expectedTokens = asList("No", "synonyms", "detection","However", "we", "have", "two", "stopwords", "and", "which", "will", "be", "removed");
|
||||
|
||||
assertAnalysisCorrectness(text, expectedTokens);
|
||||
}
|
||||
|
||||
/**
|
||||
* A synonym is detected but it doesn't contain any stopwords
|
||||
*/
|
||||
@Test
|
||||
public void synonymsDetectionWithoutStopwordTokens() throws Exception {
|
||||
String text = "How do I transfer phone number?";
|
||||
List<String> expectedTokens =
|
||||
asList(
|
||||
"How",
|
||||
"do",
|
||||
"I",
|
||||
"port", "transfer", "number", "phone", "number"); // detected synonym
|
||||
|
||||
assertAnalysisCorrectness(text, expectedTokens);
|
||||
}
|
||||
|
||||
/**
|
||||
* A synonym is detected; there are also stopwords which are not part of the synonym so the filter will be remove it.
|
||||
*/
|
||||
@Test
|
||||
public void synonymsDetectionWithStopwordTokensOutside() throws Exception
|
||||
{
|
||||
String text = "How do I transfer phone number, in my new device, of course?";
|
||||
List<String> expectedTokens =
|
||||
asList(
|
||||
"How",
|
||||
"do",
|
||||
"I",
|
||||
"port", "transfer", "number", "phone", "number", // detected synonym
|
||||
"in",
|
||||
// "my" is removed as it is a stopword outside the synonym detection
|
||||
"new",
|
||||
"device",
|
||||
// "of" is removed as it is a stopword outside the synonym detection
|
||||
"course");
|
||||
|
||||
assertAnalysisCorrectness(text, expectedTokens);
|
||||
}
|
||||
|
||||
/**
|
||||
* A synonym is detected (oow => out of warranty) but it contains a stopword token ("of")
|
||||
* The filter won't remove it.
|
||||
*/
|
||||
@Test
|
||||
public void synonymsDetectionWithStopwordTokensInside() throws Exception
|
||||
{
|
||||
String text = "My car is out of warranty and it's broken. What should I do?";
|
||||
List<String> expectedTokens =
|
||||
asList(
|
||||
// "My" is removed because outside the detected synonym
|
||||
"car",
|
||||
"is",
|
||||
"oow", "out", "of", "warranty", // detected synonym: of, although a stopword, is not removed
|
||||
"and",
|
||||
"it's",
|
||||
"broken",
|
||||
"What",
|
||||
"should",
|
||||
"I", "do");
|
||||
|
||||
assertAnalysisCorrectness(text, expectedTokens);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the text analysis produces the expected token sequence.
|
||||
*
|
||||
* @param text the input text.
|
||||
* @param expectedTokens the list of tokens in the order expected from the stream.
|
||||
*/
|
||||
private void assertAnalysisCorrectness(String text, List<String> expectedTokens) throws IOException
|
||||
{
|
||||
List<String> actualTokens = new ArrayList<>();
|
||||
|
||||
try (Reader reader = new StringReader(text);
|
||||
TokenStream tokenStream = analyzer.tokenStream("dummy_field", reader))
|
||||
{
|
||||
CharTermAttribute termAtt = tokenStream.addAttribute(CharTermAttribute.class);
|
||||
tokenStream.reset();
|
||||
while (tokenStream.incrementToken())
|
||||
{
|
||||
actualTokens.add(new String(termAtt.buffer(), 0, termAtt.length()));
|
||||
}
|
||||
tokenStream.end();
|
||||
}
|
||||
|
||||
assertEquals("Expected tokens don't match", expectedTokens, actualTokens);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user