diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/repo/search/impl/lucene/analysis/SynonymAwareStopFilterFactory.java b/search-services/alfresco-search/src/main/java/org/alfresco/repo/search/impl/lucene/analysis/SynonymAwareStopFilterFactory.java deleted file mode 100644 index e85688b6f..000000000 --- a/search-services/alfresco-search/src/main/java/org/alfresco/repo/search/impl/lucene/analysis/SynonymAwareStopFilterFactory.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (C) 2005-2010 Alfresco Software Limited. - * - * This file is part of Alfresco - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.repo.search.impl.lucene.analysis; - -import org.apache.lucene.analysis.CharArraySet; -import org.apache.lucene.analysis.StopFilter; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.core.StopFilterFactory; -import org.apache.lucene.analysis.synonym.SynonymGraphFilter; -import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute; -import org.apache.lucene.analysis.tokenattributes.TypeAttribute; - -import java.util.Map; - -/** - * A {@link StopFilter} which doesn't remove stop tokens previously marked as part of a synonym. - * When using a stop filter in a query analyzer which already includes a synonym filter, there are two options: - * the filter could be defined before or after the synonym filter. - * - * However, the first way (before) doesn’t make so much sense, because terms that are stopwords and that are, at the same - * time, part of a synonym will be removed before the synonym detection. As consequence of that no synonym detection will happen. - * - * If we have - * - * - * - * a stop filter defined before the synonym filter would remove the "of" term from the query, therefore causing the missing - * synonym detection. If we postpone the stop filter after the synonym filter, the synonym will be detected, but the - * stop filter later would still remove the "of" token causing the open issue well described in https://issues.apache.org/jira/browse/LUCENE-4065 - * - * Being related with FilteringTokenFilter (the StopFilter superclass) LUCENE-4065 has a wider scope which doesn't affect only - * stopwords removal. However, in order to mitigate the issue above, this custom stop filter takes in account the typeAttribute of the - * incoming tokens; it removes them only if they are stopwords and if they haven't been marked as part of a synonym ({@link SynonymGraphFilter#TYPE_SYNONYM}) - * - * @see SynonymGraphFilter - * @author Andrea Gazzarini - */ -public class SynonymAwareStopFilterFactory extends StopFilterFactory -{ - class SynonymAwareStopFilter extends StopFilter - { - - private TypeAttribute typeAttribute = addAttribute(TypeAttribute.class); - private PositionLengthAttribute positionLengthAttribute = addAttribute(PositionLengthAttribute.class); - - private int synonymSpans; - - SynonymAwareStopFilter(TokenStream in, CharArraySet stopwords) - { - super(in, stopwords); - } - - @Override - protected boolean accept() - { - if (isSynonymToken()) - { - synonymSpans = positionLengthAttribute.getPositionLength() > 1 - ? positionLengthAttribute.getPositionLength() - : 0; - return true; - } - - return (--synonymSpans > 0) || super.accept(); - } - - private boolean isSynonymToken() - { - return SynonymGraphFilter.TYPE_SYNONYM.equals(typeAttribute.type()); - } - } - - public SynonymAwareStopFilterFactory(Map args) - { - super(args); - } - - @Override - public TokenStream create(TokenStream input) - { - return new SynonymAwareStopFilter(input, getStopWords()); - } -} \ No newline at end of file diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/repo/search/impl/lucene/analysis/MultiTermSynonymsTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/repo/search/impl/lucene/analysis/MultiTermSynonymsTest.java deleted file mode 100644 index e7b0f5901..000000000 --- a/search-services/alfresco-search/src/test/java/org/alfresco/repo/search/impl/lucene/analysis/MultiTermSynonymsTest.java +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Copyright (C) 2005-2014 Alfresco Software Limited. - * - * This file is part of Alfresco - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.repo.search.impl.lucene.analysis; - -import org.apache.commons.io.FileUtils; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.icu.segmentation.ICUTokenizerFactory; -import org.apache.lucene.analysis.standard.StandardTokenizer; -import org.apache.lucene.analysis.synonym.SynonymGraphFilter; -import org.apache.lucene.analysis.synonym.SynonymGraphFilterFactory; -import org.apache.lucene.analysis.tokenattributes.*; -import org.apache.solr.core.SolrResourceLoader; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.mockito.junit.MockitoJUnitRunner; - -import java.io.File; -import java.io.IOException; -import java.io.Reader; -import java.io.StringReader; -import java.util.*; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static java.util.Arrays.asList; -import static org.junit.Assert.assertEquals; - -/** - * Test case which illustrates some multi-term synonyms scenarios. - * - * @author Andrea Gazzarini - */ -@RunWith(MockitoJUnitRunner.class) -public class MultiTermSynonymsTest -{ - private Analyzer analyzer; - - private SolrResourceLoader resourceLoader; - private final String ALPHANUM_TYPE = StandardTokenizer.TOKEN_TYPES[StandardTokenizer.ALPHANUM]; - private final String SYNONYM_TYPE = SynonymGraphFilter.TYPE_SYNONYM; - - @Rule - public TemporaryFolder testFolder = new TemporaryFolder(); - - /** - * The Unit test makes use of the same filter chain used within the solrconfig.xml, - * see the query analyzer of "text_en" field type: - * - *
    - *
  • {@link org.apache.lucene.analysis.synonym.SynonymGraphFilter}
    • - *
  • {@link org.alfresco.repo.search.impl.lucene.analysis.SynonymAwareStopFilterFactory.SynonymAwareStopFilter}
    • - *
- * - */ - @Before - public void setUp() throws Exception - { - File stopwordsFile = testFolder.newFile("stopwords.txt"); - File synonyms = testFolder.newFile("synonyms.txt"); - - FileUtils.writeLines(stopwordsFile, asList("of", "my")); - FileUtils.writeLines(synonyms, asList( - "OOW,Out of Warranty", - "Apache Solr,Apache Solr")); // this is how we define a multi-term concept without any synonym - - System.setProperty("solr.solr.home", stopwordsFile.getParentFile().getAbsolutePath()); - resourceLoader = new SolrResourceLoader(); - - analyzer = new Analyzer() - { - @Override - protected TokenStreamComponents createComponents(String fieldName) - { - try - { - ICUTokenizerFactory tokenizerFactory = new ICUTokenizerFactory(Collections.emptyMap()); - tokenizerFactory.inform(resourceLoader); - Tokenizer tokenizer = tokenizerFactory.create(); - - Map synonymsConfig = new HashMap<>(); - synonymsConfig.put("synonyms", "synonyms.txt"); - synonymsConfig.put("ignoreCase", "true"); - - SynonymGraphFilterFactory synonymsFactory = new SynonymGraphFilterFactory(synonymsConfig); - synonymsFactory.inform(resourceLoader); - TokenStream synonymsFilter = synonymsFactory.create(tokenizer); - - Map stopFilterConfig = new HashMap<>(); - stopFilterConfig.put("words", "stopwords.txt"); - stopFilterConfig.put("ignoreCase", "true"); - - SynonymAwareStopFilterFactory stopWordsFactory = new SynonymAwareStopFilterFactory(stopFilterConfig); - stopWordsFactory.inform(resourceLoader); - TokenStream sinkStream = stopWordsFactory.create(synonymsFilter); - - return new TokenStreamComponents(tokenizer, sinkStream); - } - catch (Exception exception) - { - throw new IllegalArgumentException(exception); - } - } - }; - } - - /** - * No synonyms and no stopwords (plain and subsequent tokens with no removal). - * No Graph is being generated from the outcoming stream. - */ - @Test - public void noSynonmsNoStopwords() throws Exception - { - final String text = "no synonyms and no stopwords"; - final List expected = asList( - token("no", 0, 2, 1, 1, ALPHANUM_TYPE), - token("synonyms", 3, 11, 1, 1, ALPHANUM_TYPE), - token("and", 12, 15, 1, 1, ALPHANUM_TYPE), - token("no", 16, 18, 1, 1, ALPHANUM_TYPE), - token("stopwords", 19, 28, 1, 1, ALPHANUM_TYPE)); - - assertAnalysisCorrectness(text, expected); - } - - /** - * A synonym is detected and the correct graph is generated. - * Note the original text (out *of* warranty) contains a stopwords but being that part of a synonym, it won't be - * removed so a the graph will later generate a correct phrase query (i.e. "out of warranty") - */ - @Test - public void multiTermConceptWithSynonyms() throws Exception - { - final String text = "Car is Out of warranty"; - final List expected = asList( - token("Car", 0, 3, 1, 1, ALPHANUM_TYPE), - token("is", 4, 6, 1, 1, ALPHANUM_TYPE), - token("oow", 7, 22, 1, 3, SYNONYM_TYPE), - token("Out", 7, 10, 0, 1, ALPHANUM_TYPE), - token("of", 11, 13, 1, 1, ALPHANUM_TYPE), - token("warranty", 14, 22, 1, 1, ALPHANUM_TYPE)); - - assertAnalysisCorrectness(text, expected); - } - - /** - * This is the same test as above but instead of starting from the expanded form, the original text contains the - * acronym (oow). Expected result is the same, the only difference is in the different tokens types. - */ - @Test - public void acronymsIsExpandedAndContainsStopwords() throws Exception - { - final String text = "Car is OOW"; - final List expected = asList( - token("Car", 0, 3, 1, 1, ALPHANUM_TYPE), - token("is", 4, 6, 1, 1, ALPHANUM_TYPE), - token("out", 7, 10, 1, 1, SYNONYM_TYPE), - token("OOW", 7, 10, 0, 3, ALPHANUM_TYPE), - token("of", 7, 10, 1, 1, SYNONYM_TYPE), - token("warranty", 7, 10, 1, 1, SYNONYM_TYPE)); - - assertAnalysisCorrectness(text, expected); - } - - /** - * If a multi-term concept needs to be captured, then its definition must be double in the synonyms file. - * - */ - @Test - public void multiTermConceptWithoutSynonyms() throws Exception - { - final String text = "apache solr, the enterprise search platform"; - final List expected = asList( - token("apache", 0, 11, 1, 1, SYNONYM_TYPE), - token("apache", 0, 6, 0, 2, ALPHANUM_TYPE), - token("solr", 0, 11, 1, 2, SYNONYM_TYPE), - token("solr", 7, 11, 1, 1, ALPHANUM_TYPE), - token("the", 13, 16, 1, 1, ALPHANUM_TYPE), - token("enterprise", 17, 27, 1, 1, ALPHANUM_TYPE), - token("search", 28, 34, 1, 1, ALPHANUM_TYPE), - token("platform", 35, 43, 1, 1, ALPHANUM_TYPE)); - - assertAnalysisCorrectness(text, expected); - } - - /** - * Check the text analysis produces the expected tokens graph. - * - * @param text the input text. - * @param expectedTokens the list of tokens in the order expected from the stream. - */ - private void assertAnalysisCorrectness(String text, List expectedTokens) throws IOException - { - final List actualTokens = new ArrayList<>(); - - try (Reader reader = new StringReader(text); - TokenStream tokenStream = analyzer.tokenStream("dummy_field", reader)) - { - CharTermAttribute termAtt = tokenStream.addAttribute(CharTermAttribute.class); - TypeAttribute typeAtt = tokenStream.addAttribute(TypeAttribute.class); - PositionIncrementAttribute positionIncrementAtt = tokenStream.addAttribute(PositionIncrementAttribute.class); - PositionLengthAttribute positionLengthAtt = tokenStream.addAttribute(PositionLengthAttribute.class); - OffsetAttribute offsetAtt = tokenStream.addAttribute(OffsetAttribute.class); - - tokenStream.reset(); - while (tokenStream.incrementToken()) - { - actualTokens.add( - token(new String(termAtt.buffer(), 0, termAtt.length()), - offsetAtt.startOffset(), - offsetAtt.endOffset(), - positionIncrementAtt.getPositionIncrement(), - positionLengthAtt.getPositionLength(), - typeAtt.type())); - } - tokenStream.end(); - } - - assertEquals(mismatchTokensListMessage(expectedTokens, actualTokens), expectedTokens, actualTokens); - } - - private String mismatchTokensListMessage(List expected, List actual) - { - Function toString = - token -> "term=" + token.toString() + - ",startOffset=" + token.startOffset() + - ",endOffset=" + token.endOffset() + - ",posInc=" + token.getPositionIncrement() + - ",posLength=" + token.getPositionLength() + - ",type=" + token.type(); - - String expectedMessage = expected.stream().map(toString).collect(Collectors.joining("\n")); - String actualMessage = actual.stream().map(toString).collect(Collectors.joining("\n")); - - return "*** EXPECTED ***\n" + expectedMessage + "\n*** ACTUAL ***\n" + actualMessage; - } - - /** - * Creates a test {@link PackedTokenAttributeImpl} used for test verifications. - * - * @param text the token content. - * @param positionLength the token position length. - * @param type the token type. - * @param startOffset the token start offset. - * @param endOffset the token end offset. - * @param positionIncrement the token position increment. - * @return a test {@link PackedTokenAttributeImpl} used for test verifications. - */ - private PackedTokenAttributeImpl token(String text, int startOffset, int endOffset, int positionIncrement, int positionLength, String type) - { - PackedTokenAttributeImpl token = new PackedTokenAttributeImpl(); - token.setPositionLength(positionLength); - token.setType(type); - token.setOffset(startOffset, endOffset); - token.setPositionIncrement(positionIncrement); - token.copyBuffer(text.toCharArray(), 0, text.length()); - return token; - } -} \ No newline at end of file diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/repo/search/impl/lucene/analysis/SynonymAwareStopFilterTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/repo/search/impl/lucene/analysis/SynonymAwareStopFilterTest.java deleted file mode 100644 index c1ff9cc3d..000000000 --- a/search-services/alfresco-search/src/test/java/org/alfresco/repo/search/impl/lucene/analysis/SynonymAwareStopFilterTest.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright (C) 2005-2014 Alfresco Software Limited. - * - * This file is part of Alfresco - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.repo.search.impl.lucene.analysis; - -import static java.util.Arrays.asList; -import static org.junit.Assert.assertEquals; - -import org.apache.commons.io.FileUtils; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.standard.StandardTokenizerFactory; -import org.apache.lucene.analysis.synonym.SynonymGraphFilterFactory; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.solr.core.SolrResourceLoader; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.mockito.junit.MockitoJUnitRunner; - -import java.io.File; -import java.io.IOException; -import java.io.Reader; -import java.io.StringReader; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Test case for {@link SynonymAwareStopFilterFactory.SynonymAwareStopFilter} - * - * @author Andrea Gazzarini - */ -@RunWith(MockitoJUnitRunner.class) -public class SynonymAwareStopFilterTest -{ - - private Analyzer analyzer; - - private SolrResourceLoader resourceLoader; - - private final List stopwords = asList("of", "my"); - - @Rule - public TemporaryFolder testFolder = new TemporaryFolder(); - - /** - * Setup fixture for this test case. - * Specifically: - * - *
    - *
  • A test analyzer composed by a StandardTokenizer, SynonymsGraphFilter and the class under test (SynonymAwareStopFilter)
    • - *
  • a synonyms file containing two rules: ["OOW,out of warranty","transfer phone number,port number"]
    • - *
  • a stopwords file with two stopwords: ["of","my"]
    • - *
- */ - @Before - public void setUp() throws Exception - { - File stopwordsFile = testFolder.newFile("stopwords.txt"); - File synonyms = testFolder.newFile("synonyms.txt"); - - FileUtils.writeLines(stopwordsFile, stopwords); - FileUtils.writeLines(synonyms, asList("OOW,Out of Warranty","transfer Phone Number,Port Number")); - - System.setProperty("solr.solr.home", stopwordsFile.getParentFile().getAbsolutePath()); - resourceLoader = new SolrResourceLoader(); - - analyzer = new Analyzer() - { - @Override - protected TokenStreamComponents createComponents(String fieldName) - { - try - { - Tokenizer tokenizer = new StandardTokenizerFactory(Collections.emptyMap()).create(); - - Map synonymsConfig = new HashMap<>(); - synonymsConfig.put("synonyms", "synonyms.txt"); - synonymsConfig.put("ignoreCase", "true"); - - SynonymGraphFilterFactory synonymsFactory = new SynonymGraphFilterFactory(synonymsConfig); - synonymsFactory.inform(resourceLoader); - TokenStream synonymsFilter = synonymsFactory.create(tokenizer); - - Map stopFilterConfig = new HashMap<>(); - stopFilterConfig.put("words", "stopwords.txt"); - stopFilterConfig.put("ignoreCase", "true"); - - SynonymAwareStopFilterFactory stopWordsFactory = new SynonymAwareStopFilterFactory(stopFilterConfig); - stopWordsFactory.inform(resourceLoader); - - TokenStream sinkStream = stopWordsFactory.create(synonymsFilter); - - return new TokenStreamComponents(tokenizer, sinkStream); - } - catch (Exception exception) - { - throw new IllegalArgumentException(exception); - } - } - }; - } - - /** - * In case we have no synonyms and no stopwords then the input should be left untouched. - */ - @Test - public void noSynonmsNoStopwords() throws Exception - { - String text = "no synonyms and no stopwords here"; - List expected = asList(text.split("\\W")); - - assertAnalysisCorrectness(text, expected); - } - - /** - * If a stream doesn't contain any SYNONYM token, the custom stop filter behaves like its superclass (StopFilter) - */ - @Test - public void noSynonymsOnlyStopwords() throws Exception - { - String text = "No synonyms detection. However we have two stopwords of and my which will be removed"; - List expectedTokens = asList("No", "synonyms", "detection","However", "we", "have", "two", "stopwords", "and", "which", "will", "be", "removed"); - - assertAnalysisCorrectness(text, expectedTokens); - } - - /** - * A synonym is detected but it doesn't contain any stopwords - */ - @Test - public void synonymsDetectionWithoutStopwordTokens() throws Exception { - String text = "How do I transfer phone number?"; - List expectedTokens = - asList( - "How", - "do", - "I", - "port", "transfer", "number", "phone", "number"); // detected synonym - - assertAnalysisCorrectness(text, expectedTokens); - } - - /** - * A synonym is detected; there are also stopwords which are not part of the synonym so the filter will be remove it. - */ - @Test - public void synonymsDetectionWithStopwordTokensOutside() throws Exception - { - String text = "How do I transfer phone number, in my new device, of course?"; - List expectedTokens = - asList( - "How", - "do", - "I", - "port", "transfer", "number", "phone", "number", // detected synonym - "in", - // "my" is removed as it is a stopword outside the synonym detection - "new", - "device", - // "of" is removed as it is a stopword outside the synonym detection - "course"); - - assertAnalysisCorrectness(text, expectedTokens); - } - - /** - * A synonym is detected (oow => out of warranty) but it contains a stopword token ("of") - * The filter won't remove it. - */ - @Test - public void synonymsDetectionWithStopwordTokensInside() throws Exception - { - String text = "My car is out of warranty and it's broken. What should I do?"; - List expectedTokens = - asList( - // "My" is removed because outside the detected synonym - "car", - "is", - "oow", "out", "of", "warranty", // detected synonym: of, although a stopword, is not removed - "and", - "it's", - "broken", - "What", - "should", - "I", "do"); - - assertAnalysisCorrectness(text, expectedTokens); - } - - /** - * Check the text analysis produces the expected token sequence. - * - * @param text the input text. - * @param expectedTokens the list of tokens in the order expected from the stream. - */ - private void assertAnalysisCorrectness(String text, List expectedTokens) throws IOException - { - List actualTokens = new ArrayList<>(); - - try (Reader reader = new StringReader(text); - TokenStream tokenStream = analyzer.tokenStream("dummy_field", reader)) - { - CharTermAttribute termAtt = tokenStream.addAttribute(CharTermAttribute.class); - tokenStream.reset(); - while (tokenStream.incrementToken()) - { - actualTokens.add(new String(termAtt.buffer(), 0, termAtt.length())); - } - tokenStream.end(); - } - - assertEquals("Expected tokens don't match", expectedTokens, actualTokens); - } -} \ No newline at end of file