Fix for MOB-1185: Alfresco FTS does not respect dual tokenisation when ordering (and locale etc etc)

git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@15300 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

source/java/org/alfresco/cmis/search/CmisFunctionEvaluationContext.java

@@ -265,7 +265,7 @@ public class CmisFunctionEvaluationContext implements FunctionEvaluationContext
     /* (non-Javadoc)
      * @see org.alfresco.repo.search.impl.querymodel.FunctionEvaluationContext#getLuceneSortField(org.alfresco.service.namespace.QName)
      */
-    public String getLuceneSortField(String propertyName)
+    public String getLuceneSortField(LuceneQueryParser lqp, String propertyName)
     {
         CMISPropertyDefinition propertyDef = cmisDictionaryService.findProperty(propertyName, null);
         return propertyDef.getPropertyLuceneBuilder().getLuceneSortField();

source/java/org/alfresco/repo/search/impl/lucene/ADMLuceneTest.java

@@ -931,6 +931,31 @@ public class ADMLuceneTest extends TestCase
         
         results.close();
         
+        sp = new SearchParameters();
+        sp.setLanguage(SearchService.LANGUAGE_FTS_ALFRESCO);
+        sp.addStore(rootNodeRef.getStoreRef());
+        sp.setQuery("-eager or -dog");
+        sp.addQueryTemplate("ANDY", "%cm:content");
+        sp.setNamespace(NamespaceService.CONTENT_MODEL_1_0_URI);
+        sp.excludeDataInTheCurrentTransaction(true);
+        sp.addSort("cm:name", false);
+        results = searcher.query(sp);
+        assertEquals(15, results.length());
+        
+        f = null;
+        for (ResultSetRow row : results)
+        {
+            String currentBun = DefaultTypeConverter.INSTANCE.convert(String.class, nodeService.getProperty(row.getNodeRef(), ContentModel.PROP_NODE_UUID));
+            // System.out.println( (currentBun == null ? "null" : NumericEncoder.encode(currentBun))+ " "+currentBun);
+            if (f != null)
+            {
+                assertTrue(f.compareTo(currentBun) >= 0);
+            }
+            f = currentBun;
+        }
+        
+        results.close();
+        
        
     }
     

source/java/org/alfresco/repo/search/impl/lucene/LuceneQueryParser.java

@@ -133,13 +133,13 @@ public class LuceneQueryParser extends QueryParser
      *            the default field for query terms.
      * @param analyzer
      *            used to find terms in the query text.
-     * @param namespacePrefixResolver 
+     * @param namespacePrefixResolver
-     * @param dictionaryService 
+     * @param dictionaryService
-     * @param tenantService 
+     * @param tenantService
-     * @param defaultOperator 
+     * @param defaultOperator
-     * @param searchParameters 
+     * @param searchParameters
      * @param config
-     * @param indexReader 
+     * @param indexReader
      * @return - the query
      * @throws ParseException
      *             if the parsing fails
@@ -209,8 +209,24 @@ public class LuceneQueryParser extends QueryParser
         this.tenantService = tenantService;
     }
 
+    public SearchParameters getSearchParameters()
+    {
+        return searchParameters;
+    }
+
+    public IndexReader getIndexReader()
+    {
+        return indexReader;
+    }
+
+    public LuceneConfig getConfig()
+    {
+        return config;
+    }
+
     /**
      * Lucene default constructor
+     * 
      * @param arg0
      * @param arg1
      */
@@ -225,6 +241,7 @@ public class LuceneQueryParser extends QueryParser
 
     /**
      * Lucene default constructor
+     * 
      * @param arg0
      */
     public LuceneQueryParser(CharStream arg0)
@@ -234,6 +251,7 @@ public class LuceneQueryParser extends QueryParser
 
     /**
      * Lucene default constructor
+     * 
      * @param arg0
      */
     public LuceneQueryParser(QueryParserTokenManager arg0)
@@ -1497,14 +1515,14 @@ public class LuceneQueryParser extends QueryParser
     }
 
     /**
-     * @param field 
+     * @param field
-     * @param part1 
+     * @param part1
      * @param part2
-     * @param includeLower 
+     * @param includeLower
-     * @param includeUpper 
+     * @param includeUpper
-     * @param analysisMode 
+     * @param analysisMode
-     * @param luceneFunction 
+     * @param luceneFunction
-     * @return the query 
+     * @return the query
      * @exception ParseException
      *                throw in overridden method to disallow
      */
@@ -1558,7 +1576,8 @@ public class LuceneQueryParser extends QueryParser
                                 textFieldName = textFieldName + "." + locale + ".sort";
                             }
 
-                            addLocaleSpecificUntokenisedTextRangeFunction(field, part1, part2, includeLower, includeUpper, luceneFunction, booleanQuery, mlAnalysisMode, locale, textFieldName);
+                            addLocaleSpecificUntokenisedTextRangeFunction(field, part1, part2, includeLower, includeUpper, luceneFunction, booleanQuery, mlAnalysisMode, locale,
+                                    textFieldName);
 
                         }
                         return booleanQuery;
@@ -1756,8 +1775,8 @@ public class LuceneQueryParser extends QueryParser
         }
     }
 
-    private void addLocaleSpecificUntokenisedTextRangeFunction(String expandedFieldName, String lower, String upper, boolean includeLower, boolean includeUpper, LuceneFunction luceneFunction, BooleanQuery booleanQuery,
+    private void addLocaleSpecificUntokenisedTextRangeFunction(String expandedFieldName, String lower, String upper, boolean includeLower, boolean includeUpper,
-            MLAnalysisMode mlAnalysisMode, Locale locale, String textFieldName)
+            LuceneFunction luceneFunction, BooleanQuery booleanQuery, MLAnalysisMode mlAnalysisMode, Locale locale, String textFieldName)
     {
         String lowerTermText = lower;
         if (locale.toString().length() > 0)
@@ -1769,7 +1788,7 @@ public class LuceneQueryParser extends QueryParser
         {
             upperTermText = "{" + locale + "}" + upper;
         }
-        Query subQuery = buildRangeFunctionQuery(textFieldName, lowerTermText,  upperTermText, includeLower, includeUpper, luceneFunction);
+        Query subQuery = buildRangeFunctionQuery(textFieldName, lowerTermText, upperTermText, includeLower, includeUpper, luceneFunction);
         booleanQuery.add(subQuery, Occur.SHOULD);
 
         if (booleanQuery.getClauses().length == 0)
@@ -1778,7 +1797,8 @@ public class LuceneQueryParser extends QueryParser
         }
     }
 
-    private Query buildRangeFunctionQuery(String expandedFieldName, String lowerTermText, String upperTermText, boolean includeLower, boolean includeUpper, LuceneFunction luceneFunction)
+    private Query buildRangeFunctionQuery(String expandedFieldName, String lowerTermText, String upperTermText, boolean includeLower, boolean includeUpper,
+            LuceneFunction luceneFunction)
     {
         String testLowerTermText = lowerTermText;
         if (testLowerTermText.startsWith("{"))
@@ -1786,14 +1806,14 @@ public class LuceneQueryParser extends QueryParser
             int index = lowerTermText.indexOf("}");
             testLowerTermText = lowerTermText.substring(index + 1);
         }
-        
+
         String testUpperTermText = upperTermText;
         if (testUpperTermText.startsWith("{"))
         {
             int index = upperTermText.indexOf("}");
             testUpperTermText = upperTermText.substring(index + 1);
         }
-        
+
         switch (luceneFunction)
         {
         case LOWER:
@@ -3088,8 +3108,7 @@ public class LuceneQueryParser extends QueryParser
         }
         else if ((propertyDef != null) && (propertyDef.getDataType().getName().equals(DataTypeDefinition.TEXT)))
         {
-            if (propertyQName.equals(ContentModel.PROP_USER_USERNAME)
+            if (propertyQName.equals(ContentModel.PROP_USER_USERNAME) || propertyQName.equals(ContentModel.PROP_USERNAME) || propertyQName.equals(ContentModel.PROP_AUTHORITY_NAME))
-                    || propertyQName.equals(ContentModel.PROP_USERNAME) || propertyQName.equals(ContentModel.PROP_AUTHORITY_NAME))
             {
                 return subQueryBuilder.getQuery(expandedFieldName, queryText, analysisMode, luceneFunction);
             }
@@ -3281,8 +3300,7 @@ public class LuceneQueryParser extends QueryParser
         }
         else if ((propertyDef != null) && (propertyDef.getDataType().getName().equals(DataTypeDefinition.TEXT)))
         {
-            if (propertyQName.equals(ContentModel.PROP_USER_USERNAME)
+            if (propertyQName.equals(ContentModel.PROP_USER_USERNAME) || propertyQName.equals(ContentModel.PROP_USERNAME) || propertyQName.equals(ContentModel.PROP_AUTHORITY_NAME))
-                    || propertyQName.equals(ContentModel.PROP_USERNAME) || propertyQName.equals(ContentModel.PROP_AUTHORITY_NAME))
             {
                 throw new UnsupportedOperationException("Functions are not supported agaisnt special text fields");
             }

source/java/org/alfresco/repo/search/impl/parsers/AlfrescoFunctionEvaluationContext.java

@@ -26,21 +26,34 @@ package org.alfresco.repo.search.impl.parsers;
 
 import java.io.Serializable;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 
+import org.alfresco.i18n.I18NUtil;
+import org.alfresco.repo.search.MLAnalysisMode;
+import org.alfresco.repo.search.SearcherException;
 import org.alfresco.repo.search.impl.lucene.LuceneFunction;
 import org.alfresco.repo.search.impl.lucene.LuceneQueryParser;
+import org.alfresco.repo.search.impl.lucene.analysis.DateTimeAnalyser;
 import org.alfresco.repo.search.impl.querymodel.FunctionArgument;
 import org.alfresco.repo.search.impl.querymodel.FunctionEvaluationContext;
 import org.alfresco.repo.search.impl.querymodel.PredicateMode;
+import org.alfresco.repo.search.impl.querymodel.impl.lucene.LuceneQueryBuilderContext;
+import org.alfresco.service.cmr.dictionary.DataTypeDefinition;
 import org.alfresco.service.cmr.dictionary.DictionaryService;
+import org.alfresco.service.cmr.dictionary.PropertyDefinition;
 import org.alfresco.service.cmr.repository.NodeRef;
 import org.alfresco.service.cmr.repository.NodeService;
 import org.alfresco.service.namespace.NamespacePrefixResolver;
 import org.alfresco.service.namespace.QName;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.FieldOption;
 import org.apache.lucene.queryParser.ParseException;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SortField;
 
 /**
  * Alfrecso function evaluation context for evaluating FTS expressions against lucene.
@@ -90,6 +103,7 @@ public class AlfrescoFunctionEvaluationContext implements FunctionEvaluationCont
         this.dictionaryService = dictionaryService;
         this.defaultNamespace = defaultNamespace;
     }
+   
 
     public Query buildLuceneEquality(LuceneQueryParser lqp, String propertyName, Serializable value, PredicateMode mode, LuceneFunction luceneFunction) throws ParseException
     {
@@ -136,9 +150,96 @@ public class AlfrescoFunctionEvaluationContext implements FunctionEvaluationCont
         throw new UnsupportedOperationException();
     }
 
-    public String getLuceneSortField(String propertyName)
+    public String getLuceneSortField(LuceneQueryParser lqp, String propertyName)
     {
-        return getLuceneFieldName(propertyName);
+        // Score is special
+        if(propertyName.equalsIgnoreCase("Score"))
+        {
+            return "Score";
+        }
+        String field =  getLuceneFieldName(propertyName);
+        // need to find the real field to use
+        Locale sortLocale = null;
+        if (field.startsWith("@"))
+        {
+            PropertyDefinition propertyDef = dictionaryService.getProperty(QName.createQName(field.substring(1)));
+
+            if (propertyDef.getDataType().getName().equals(DataTypeDefinition.CONTENT))
+            {
+                throw new SearcherException("Order on content properties is not curently supported");
+            }
+            else if ((propertyDef.getDataType().getName().equals(DataTypeDefinition.MLTEXT))
+                    || (propertyDef.getDataType().getName().equals(DataTypeDefinition.TEXT)))
+            {
+                List<Locale> locales = lqp.getSearchParameters().getLocales();
+                if (((locales == null) || (locales.size() == 0)))
+                {
+                    locales = Collections.singletonList(I18NUtil.getLocale());
+                }
+
+                if (locales.size() > 1)
+                {
+                    throw new SearcherException("Order on text/mltext properties with more than one locale is not curently supported");
+                }
+
+                sortLocale = locales.get(0);
+                // find best field match
+
+                HashSet<String> allowableLocales = new HashSet<String>();
+                MLAnalysisMode analysisMode = lqp.getConfig().getDefaultMLSearchAnalysisMode();
+                for (Locale l : MLAnalysisMode.getLocales(analysisMode, sortLocale, false))
+                {
+                    allowableLocales.add(l.toString());
+                }
+
+                String sortField = field;
+
+                for (Object current :  lqp.getIndexReader().getFieldNames(FieldOption.INDEXED))
+                {
+                    String currentString = (String) current;
+                    if (currentString.startsWith(field) && currentString.endsWith(".sort"))
+                    {
+                        String fieldLocale = currentString.substring(field.length() + 1, currentString.length() - 5);
+                        if (allowableLocales.contains(fieldLocale))
+                        {
+                            if (fieldLocale.equals(sortLocale.toString()))
+                            {
+                                sortField = currentString;
+                                break;
+                            }
+                            else if (sortLocale.toString().startsWith(fieldLocale))
+                            {
+                                if (sortField.equals(field) || (currentString.length() < sortField.length()))
+                                {
+                                    sortField = currentString;
+                                }
+                            }
+                            else if (fieldLocale.startsWith(sortLocale.toString()))
+                            {
+                                if (sortField.equals(field) || (currentString.length() < sortField.length()))
+                                {
+                                    sortField = currentString;
+                                }
+                            }
+                        }
+                    }
+                }
+
+                field = sortField;
+
+            }
+            else if (propertyDef.getDataType().getName().equals(DataTypeDefinition.DATETIME))
+            {
+                DataTypeDefinition dataType = propertyDef.getDataType();
+                String analyserClassName = dataType.getAnalyserClassName();
+                if (analyserClassName.equals(DateTimeAnalyser.class.getCanonicalName()))
+                {
+                    field = field + ".sort";
+                }
+            }
+
+        }
+        return field;
     }
 
     public Map<String, NodeRef> getNodeRefs()

source/java/org/alfresco/repo/search/impl/querymodel/FunctionEvaluationContext.java

@@ -26,12 +26,17 @@ package org.alfresco.repo.search.impl.querymodel;
 
 import java.io.Serializable;
 import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 
+import org.alfresco.repo.search.MLAnalysisMode;
 import org.alfresco.repo.search.impl.lucene.LuceneFunction;
 import org.alfresco.repo.search.impl.lucene.LuceneQueryParser;
+import org.alfresco.repo.search.impl.querymodel.impl.lucene.LuceneQueryBuilderContext;
 import org.alfresco.service.cmr.repository.NodeRef;
 import org.alfresco.service.cmr.repository.NodeService;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.queryParser.ParseException;
 import org.apache.lucene.search.Query;
 
@@ -173,9 +178,13 @@ public interface FunctionEvaluationContext
 
     /**
      * @param propertyName
+     * @param luceneContext 
+     * @param locales 
+     * @param analysisMode 
+     * @param reader 
      * @return the field used for sorting the given property
      */
-    public String getLuceneSortField(String propertyName);
+    public String getLuceneSortField(LuceneQueryParser lqp, String propertyName);
     
     /**
      * @param propertyName

source/java/org/alfresco/repo/search/impl/querymodel/impl/lucene/LuceneQuery.java

@@ -178,7 +178,7 @@ public class LuceneQuery extends BaseQuery implements LuceneQueryBuilder
 
                 String propertyName = property.getPropertyName();
 
-                String luceneField = functionContext.getLuceneSortField(propertyName);
+                String luceneField = functionContext.getLuceneSortField(luceneContext.getLuceneQueryParser(), propertyName);
 
                 if (luceneField != null)
                 {