Next stage of multi lingual searches

git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@4609 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

source/java/org/alfresco/repo/search/MLAnalysisMode.java

@@ -13,30 +13,144 @@ public enum MLAnalysisMode
     /**
      * Only exact locale is used.
      */
-    LOCALE_ONLY, 
+    LOCALE_ONLY
+    {
+        public boolean includesAll()
+        {
+            return false;
+        }
+        public boolean includesContained()
+        {
+            return false;
+        }
+        public boolean includesContaining()
+        {
+            return false;
+        }
+        public boolean includesExact()
+        {
+            return true;
+        }
+        
+    },
     
     /**
      * Only the exact locale and no local === all lnaguages 
      */
-    LOCALE_AND_ALL, 
+    LOCALE_AND_ALL
+    {
+        public boolean includesAll()
+        {
+            return true;
+        }
+        public boolean includesContained()
+        {
+            return false;
+        }
+        public boolean includesContaining()
+        {
+            return false;
+        }
+        public boolean includesExact()
+        {
+            return true;
+        }
+    },
     
     /**
      * Expand the locale to include all the locales that contain it.
      * en_GB would be en_GB, en, but not all languages
      */
-    LOCALE_AND_ALL_CONTAINING_LOCALES, 
+    LOCALE_AND_ALL_CONTAINING_LOCALES
+    {
+        public boolean includesAll()
+        {
+            return false;
+        }
+        public boolean includesContained()
+        {
+            return false;
+        }
+        public boolean includesContaining()
+        {
+            return true;
+        }
+        public boolean includesExact()
+        {
+            return true;
+        }
+    },
     
     /**
      * Expand the locale to include all the locales that contain it.
      * en_GB would be en_GB, en, and all.
      */
-    LOCALE_AND_ALL_CONTAINING_LOCALES_AND_ALL, 
+    LOCALE_AND_ALL_CONTAINING_LOCALES_AND_ALL
+    {
+        public boolean includesAll()
+        {
+            return true;
+        }
+        public boolean includesContained()
+        {
+            return false;
+        }
+        public boolean includesContaining()
+        {
+            return true;
+        }
+        public boolean includesExact()
+        {
+            return true;
+        }
+    },
     
     /**
      * Expand to all the locales that are contained by this.
      * en would expand to en, en_GB, en_US, ....
      */
-    LOCAL_AND_ALL_CONTAINED_LOCALES;
+    LOCALE_AND_ALL_CONTAINED_LOCALES
+    {
+        public boolean includesAll()
+        {
+            return false;
+        }
+        public boolean includesContained()
+        {
+            return true;
+        }
+        public boolean includesContaining()
+        {
+            return false;
+        }
+        public boolean includesExact()
+        {
+            return true;
+        }
+    },
+    
+    /**
+     * No prefix only
+     */
+    ALL_ONLY
+    {
+        public boolean includesAll()
+        {
+            return true;
+        }
+        public boolean includesContained()
+        {
+            return false;
+        }
+        public boolean includesContaining()
+        {
+            return false;
+        }
+        public boolean includesExact()
+        {
+            return false;
+        }
+    };
     
     public static MLAnalysisMode getMLAnalysisMode(String mode)
     {
@@ -49,4 +163,13 @@ public enum MLAnalysisMode
         }
         throw new AlfrescoRuntimeException("Unknown ML Analysis mode "+mode);
     }
+    
+    public abstract boolean includesAll();
+    
+    public abstract boolean includesContained();
+    
+    public abstract boolean includesContaining();
+    
+    public abstract boolean includesExact();
+    
 }

source/java/org/alfresco/repo/search/impl/lucene/FastCharStream.java

@@ -17,7 +17,8 @@ package org.alfresco.repo.search.impl.lucene;
  * limitations under the License.
  */
 
-import java.io.*;
+import java.io.IOException;
+import java.io.Reader;
 
 /** An efficient implementation of JavaCC's CharStream interface.  <p>Note that
  * this does not do line-number counting, but instead keeps track of the

source/java/org/alfresco/repo/search/impl/lucene/LuceneAnalyser.java

@@ -21,21 +21,23 @@ import java.util.HashMap;
 import java.util.Map;
 
 import org.alfresco.repo.search.MLAnalysisMode;
+import org.alfresco.repo.search.impl.lucene.analysis.AlfrescoStandardAnalyser;
+import org.alfresco.repo.search.impl.lucene.analysis.LongAnalyser;
 import org.alfresco.repo.search.impl.lucene.analysis.MLAnalayser;
 import org.alfresco.repo.search.impl.lucene.analysis.PathAnalyser;
 import org.alfresco.repo.search.impl.lucene.analysis.VerbatimAnalyser;
+import org.alfresco.service.cmr.dictionary.DataTypeDefinition;
 import org.alfresco.service.cmr.dictionary.DictionaryService;
 import org.alfresco.service.cmr.dictionary.PropertyDefinition;
-import org.alfresco.service.cmr.dictionary.DataTypeDefinition;
 import org.alfresco.service.namespace.QName;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.WhitespaceAnalyzer;
-import org.alfresco.repo.search.impl.lucene.analysis.AlfrescoStandardAnalyser;
 
 /**
- * Analyse properties according to the property definition. The default is to use the standard tokeniser. The tokeniser should not have been called when indexing properties that
+ * Analyse properties according to the property definition. The default is to use the standard tokeniser. The tokeniser
- * require no tokenisation. (tokenise should be set to false when adding the field to the document)
+ * should not have been called when indexing properties that require no tokenisation. (tokenise should be set to false
+ * when adding the field to the document)
  * 
  * @author andyh
  */
@@ -82,7 +84,7 @@ public class LuceneAnalyser extends Analyzer
         // Treat multilingual as a special case.
         // If multilingual then we need to find the correct tokeniser.
         // This is done dynamically by reading a language code at the start of the reader.
-        if (fieldName.startsWith("@") && !fieldName.endsWith(".mimetype"))
+        if (fieldName.startsWith("@"))
         {
             QName propertyQName = QName.createQName(fieldName.substring(1));
             PropertyDefinition propertyDef = dictionaryService.getProperty(propertyQName);
@@ -90,7 +92,7 @@ public class LuceneAnalyser extends Analyzer
             {
                 if (propertyDef.getDataType().getName().equals(DataTypeDefinition.MLTEXT))
                 {
-                    MLAnalayser analyser = new MLAnalayser(dictionaryService);
+                    MLAnalayser analyser = new MLAnalayser(dictionaryService, mlAlaysisMode);
                     return analyser.tokenStream(fieldName, reader);
                 }
             }
@@ -133,17 +135,20 @@ public class LuceneAnalyser extends Analyzer
         {
             analyser = new WhitespaceAnalyzer();
         }
-        else if (fieldName.equals("TEXT"))
-        {
-            DataTypeDefinition dataType = dictionaryService.getDataType(DataTypeDefinition.TEXT);
-            analyser = loadAnalyzer(dataType);
-        }
         else if (fieldName.startsWith("@"))
         {
             if (fieldName.endsWith(".mimetype"))
             {
                 analyser = new VerbatimAnalyser();
             }
+            else if (fieldName.endsWith(".size"))
+            {
+                analyser = new LongAnalyser();
+            }
+            else if (fieldName.endsWith(".locale"))
+            {
+                analyser = new VerbatimAnalyser(true);
+            }
             else
             {
                 QName propertyQName = QName.createQName(fieldName.substring(1));
@@ -153,8 +158,19 @@ public class LuceneAnalyser extends Analyzer
                     if (propertyDef.isTokenisedInIndex())
                     {
                         DataTypeDefinition dataType = propertyDef.getDataType();
+                        if (dataType.getName().equals(DataTypeDefinition.CONTENT))
+                        {
+                            analyser = new MLAnalayser(dictionaryService, MLAnalysisMode.ALL_ONLY);
+                        }
+                        else if (dataType.getName().equals(DataTypeDefinition.TEXT))
+                        {
+                            analyser = new MLAnalayser(dictionaryService, MLAnalysisMode.ALL_ONLY);
+                        }
+                        else
+                        {
                             analyser = loadAnalyzer(dataType);
                         }
+                    }
                     else
                     {
                         analyser = new VerbatimAnalyser();
@@ -208,7 +224,8 @@ public class LuceneAnalyser extends Analyzer
     }
 
     /**
-     * For multilingual fields we separate the tokens for each instance to break phrase queries spanning different languages etc.
+     * For multilingual fields we separate the tokens for each instance to break phrase queries spanning different
+     * languages etc.
      */
     @Override
     public int getPositionIncrementGap(String fieldName)

source/java/org/alfresco/repo/search/impl/lucene/LuceneBase2.java

@@ -21,7 +21,6 @@ import java.io.IOException;
 import java.util.Set;
 
 import org.alfresco.repo.search.IndexerException;
-import org.alfresco.repo.search.MLAnalysisMode;
 import org.alfresco.repo.search.impl.lucene.index.IndexInfo;
 import org.alfresco.repo.search.impl.lucene.index.TransactionStatus;
 import org.alfresco.repo.search.impl.lucene.index.IndexInfo.LockWork;
@@ -32,7 +31,6 @@ import org.apache.log4j.Logger;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Searcher;
 
 /**
  * Common support for abstracting the lucene indexer from its configuration and management requirements.

source/java/org/alfresco/repo/search/impl/lucene/LuceneCategoryServiceImpl.java

@@ -40,7 +40,6 @@ import org.alfresco.service.cmr.search.ResultSetRow;
 import org.alfresco.service.namespace.NamespacePrefixResolver;
 import org.alfresco.service.namespace.QName;
 import org.alfresco.util.ISO9075;
-import org.bouncycastle.crypto.paddings.ISO7816d4Padding;
 
 public class LuceneCategoryServiceImpl implements CategoryService
 {

source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexerAndSearcherFactory2.java

@@ -131,9 +131,9 @@ public class LuceneIndexerAndSearcherFactory2 implements LuceneIndexerAndSearche
 
     private String lockDirectory;
 
-    private MLAnalysisMode defaultMLIndexAnalysisMode = MLAnalysisMode.LOCALE_ONLY;
+    private MLAnalysisMode defaultMLIndexAnalysisMode = MLAnalysisMode.LOCALE_AND_ALL;
 
-    private MLAnalysisMode defaultMLSearchAnalysisMode = MLAnalysisMode.LOCALE_AND_ALL;
+    private MLAnalysisMode defaultMLSearchAnalysisMode = MLAnalysisMode.LOCALE_AND_ALL_CONTAINING_LOCALES_AND_ALL;
 
     /**
      * Private constructor for the singleton TODO: FIt in with IOC

source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexerImpl2.java

@@ -19,7 +19,9 @@ package org.alfresco.repo.search.impl.lucene;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.io.Reader;
 import java.io.Serializable;
+import java.io.StringReader;
 import java.io.UnsupportedEncodingException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -85,7 +87,6 @@ import org.apache.lucene.search.BooleanClause.Occur;
  * The implementation of the lucene based indexer. Supports basic transactional behaviour if used on its own.
  * 
  * @author andyh
- * 
  */
 public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
 {
@@ -119,9 +120,8 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
     private long maxAtomicTransformationTime = 20;
 
     /**
-     * A list of all deletions we have made - at merge these deletions need to be made against the main index.
+     * A list of all deletions we have made - at merge these deletions need to be made against the main index. TODO:
-     * 
+     * Consider if this information needs to be persisted for recovery
-     * TODO: Consider if this information needs to be persisted for recovery
      */
     private Set<NodeRef> deletions = new LinkedHashSet<NodeRef>();
 
@@ -140,8 +140,8 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
     private boolean isModified = false;
 
     /**
-     * Flag to indicte if we are doing an in transactional delta or a batch update to the index. If true, we are just fixing up non atomically indexed things from one or more other
+     * Flag to indicte if we are doing an in transactional delta or a batch update to the index. If true, we are just
-     * updates.
+     * fixing up non atomically indexed things from one or more other updates.
      */
 
     private Boolean isFTSUpdate = null;
@@ -168,7 +168,6 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
 
     /**
      * Default construction
-     * 
      */
     LuceneIndexerImpl2()
     {
@@ -216,7 +215,6 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
 
     /**
      * Utility method to check we are in the correct state to do work Also keeps track of the dirty flag.
-     * 
      */
 
     private void checkAbleToDoWork(boolean isFTS, boolean isModified)
@@ -592,7 +590,6 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
 
                     deletions.add(helper.nodeRef);
 
-                    
                     // try
                     // {
                     // Hits hits = mainSearcher.search(query);
@@ -688,11 +685,8 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
     }
 
     /**
-     * Prepare to commit
+     * Prepare to commit At the moment this makes sure we have all the locks TODO: This is not doing proper
-     * 
+     * serialisation against the index as would a data base transaction.
-     * At the moment this makes sure we have all the locks
-     * 
-     * TODO: This is not doing proper serialisation against the index as would a data base transaction.
      * 
      * @return
      */
@@ -766,7 +760,6 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
 
     /**
      * Roll back the index changes (this just means they are never added)
-     * 
      */
 
     public void rollback() throws LuceneIndexException
@@ -807,8 +800,8 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
     }
 
     /**
-     * Mark this index for roll back only. This action can not be reversed. It will reject all other work and only allow roll back.
+     * Mark this index for roll back only. This action can not be reversed. It will reject all other work and only allow
-     * 
+     * roll back.
      */
 
     public void setRollbackOnly()
@@ -1242,7 +1235,8 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
 
         Document xdoc = new Document();
         xdoc.add(new Field("ID", nodeRef.toString(), Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
-        xdoc.add(new Field("TX", nodeStatus.getChangeTxnId(), Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+        xdoc.add(new Field("TX", nodeStatus.getChangeTxnId(), Field.Store.YES, Field.Index.UN_TOKENIZED,
+                Field.TermVector.NO));
         boolean isAtomic = true;
         for (QName propertyName : properties.keySet())
         {
@@ -1298,8 +1292,10 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
                             qNameBuffer.append(";/");
                         }
                         qNameBuffer.append(ISO9075.getXPathName(qNameRef.getQName()));
-                        xdoc.add(new Field("PARENT", qNameRef.getParentRef().toString(), Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+                        xdoc.add(new Field("PARENT", qNameRef.getParentRef().toString(), Field.Store.YES,
-                        xdoc.add(new Field("ASSOCTYPEQNAME", ISO9075.getXPathName(qNameRef.getTypeQName()), Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
+                                Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+                        xdoc.add(new Field("ASSOCTYPEQNAME", ISO9075.getXPathName(qNameRef.getTypeQName()),
+                                Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
                         xdoc.add(new Field("LINKASPECT", (pair.getSecond() == null) ? "" : ISO9075.getXPathName(pair
                                 .getSecond()), Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
                     }
@@ -1322,17 +1318,22 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
                         if (directPaths.contains(pair.getFirst()))
                         {
                             Document directoryEntry = new Document();
-                            directoryEntry.add(new Field("ID", nodeRef.toString(), Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+                            directoryEntry.add(new Field("ID", nodeRef.toString(), Field.Store.YES,
-                            directoryEntry.add(new Field("PATH", pathString, Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.NO));
+                                    Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+                            directoryEntry.add(new Field("PATH", pathString, Field.Store.YES, Field.Index.TOKENIZED,
+                                    Field.TermVector.NO));
                             for (NodeRef parent : getParents(pair.getFirst()))
                             {
-                                directoryEntry.add(new Field("ANCESTOR", parent.toString(), Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+                                directoryEntry.add(new Field("ANCESTOR", parent.toString(), Field.Store.NO,
+                                        Field.Index.UN_TOKENIZED, Field.TermVector.NO));
                             }
-                            directoryEntry.add(new Field("ISCONTAINER", "T", Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+                            directoryEntry.add(new Field("ISCONTAINER", "T", Field.Store.YES, Field.Index.UN_TOKENIZED,
+                                    Field.TermVector.NO));
 
                             if (isCategory(getDictionaryService().getType(nodeService.getType(nodeRef))))
                             {
-                                directoryEntry.add(new Field("ISCATEGORY", "T", Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+                                directoryEntry.add(new Field("ISCATEGORY", "T", Field.Store.YES,
+                                        Field.Index.UN_TOKENIZED, Field.TermVector.NO));
                             }
 
                             docs.add(directoryEntry);
@@ -1350,7 +1351,8 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
             xdoc.add(new Field("PATH", "", Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.NO));
             xdoc.add(new Field("QNAME", "", Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.NO));
             xdoc.add(new Field("ISROOT", "T", Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
-            xdoc.add(new Field("PRIMARYASSOCTYPEQNAME", ISO9075.getXPathName(ContentModel.ASSOC_CHILDREN), Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
+            xdoc.add(new Field("PRIMARYASSOCTYPEQNAME", ISO9075.getXPathName(ContentModel.ASSOC_CHILDREN),
+                    Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
             xdoc.add(new Field("ISNODE", "T", Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
             docs.add(xdoc);
 
@@ -1358,36 +1360,45 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
         else
         // not a root node
         {
-            xdoc.add(new Field("QNAME", qNameBuffer.toString(),Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.NO));
+            xdoc.add(new Field("QNAME", qNameBuffer.toString(), Field.Store.YES, Field.Index.TOKENIZED,
+                    Field.TermVector.NO));
             // xdoc.add(new Field("PARENT", parentBuffer.toString(), true, true,
             // true));
 
             ChildAssociationRef primary = nodeService.getPrimaryParent(nodeRef);
-            xdoc.add(new Field("PRIMARYPARENT", primary.getParentRef().toString(), Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+            xdoc.add(new Field("PRIMARYPARENT", primary.getParentRef().toString(), Field.Store.YES,
-            xdoc.add(new Field("PRIMARYASSOCTYPEQNAME", ISO9075.getXPathName(primary.getTypeQName()), Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
+                    Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+            xdoc.add(new Field("PRIMARYASSOCTYPEQNAME", ISO9075.getXPathName(primary.getTypeQName()), Field.Store.YES,
+                    Field.Index.NO, Field.TermVector.NO));
             QName typeQName = nodeService.getType(nodeRef);
 
-            xdoc.add(new Field("TYPE", ISO9075.getXPathName(typeQName), Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+            xdoc.add(new Field("TYPE", ISO9075.getXPathName(typeQName), Field.Store.YES, Field.Index.UN_TOKENIZED,
+                    Field.TermVector.NO));
             for (QName classRef : nodeService.getAspects(nodeRef))
             {
-                xdoc.add(new Field("ASPECT", ISO9075.getXPathName(classRef), Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+                xdoc.add(new Field("ASPECT", ISO9075.getXPathName(classRef), Field.Store.YES, Field.Index.UN_TOKENIZED,
+                        Field.TermVector.NO));
             }
 
             xdoc.add(new Field("ISROOT", "F", Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
             xdoc.add(new Field("ISNODE", "T", Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
             if (isAtomic || indexAllProperties)
             {
-                xdoc.add(new Field("FTSSTATUS", "Clean", Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+                xdoc
+                        .add(new Field("FTSSTATUS", "Clean", Field.Store.NO, Field.Index.UN_TOKENIZED,
+                                Field.TermVector.NO));
             }
             else
             {
                 if (isNew)
                 {
-                    xdoc.add(new Field("FTSSTATUS", "New", Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+                    xdoc.add(new Field("FTSSTATUS", "New", Field.Store.NO, Field.Index.UN_TOKENIZED,
+                            Field.TermVector.NO));
                 }
                 else
                 {
-                    xdoc.add(new Field("FTSSTATUS", "Dirty", Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+                    xdoc.add(new Field("FTSSTATUS", "Dirty", Field.Store.NO, Field.Index.UN_TOKENIZED,
+                            Field.TermVector.NO));
                 }
             }
 
@@ -1446,6 +1457,7 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
         boolean atomic = true;
         boolean isContent = false;
         boolean isMultiLingual = false;
+        boolean isText = false;
 
         PropertyDefinition propertyDef = getDictionaryService().getProperty(propertyName);
         if (propertyDef != null)
@@ -1456,6 +1468,7 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
             atomic = propertyDef.isIndexedAtomically();
             isContent = propertyDef.getDataType().getName().equals(DataTypeDefinition.CONTENT);
             isMultiLingual = propertyDef.getDataType().getName().equals(DataTypeDefinition.MLTEXT);
+            isText = propertyDef.getDataType().getName().equals(DataTypeDefinition.TEXT);
         }
         if (value == null)
         {
@@ -1493,7 +1506,19 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
                     continue;
                 }
                 // store mimetype in index - even if content does not index it is useful
-                doc.add(new Field(attributeName + ".mimetype", contentData.getMimetype(), Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+                // Added szie and locale - size needs to be tokenised correctly
+                doc.add(new Field(attributeName + ".mimetype", contentData.getMimetype(), Field.Store.NO,
+                        Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+                doc.add(new Field(attributeName + ".size", Long.toString(contentData.getSize()), Field.Store.NO,
+                        Field.Index.TOKENIZED, Field.TermVector.NO));
+                // TODO: Use the node locale in preferanced to the system locale
+                Locale locale = contentData.getLocale();
+                if (locale == null)
+                {
+                    locale = Locale.getDefault();
+                }
+                doc.add(new Field(attributeName + ".locale", locale.toString().toLowerCase(), Field.Store.NO,
+                        Field.Index.UN_TOKENIZED, Field.TermVector.NO));
 
                 ContentReader reader = contentService.getReader(nodeRef, propertyName);
                 if (reader != null && reader.exists())
@@ -1519,8 +1544,10 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
                             // don't index from the reader
                             readerReady = false;
                             // not indexed: no transformation
-                            //doc.add(new Field("TEXT", NOT_INDEXED_NO_TRANSFORMATION, Field.Store.NO, Field.Index.TOKENIZED, Field.TermVector.NO));
+                            // doc.add(new Field("TEXT", NOT_INDEXED_NO_TRANSFORMATION, Field.Store.NO,
-                            doc.add(new Field(attributeName, NOT_INDEXED_NO_TRANSFORMATION, Field.Store.NO, Field.Index.TOKENIZED, Field.TermVector.NO));
+                            // Field.Index.TOKENIZED, Field.TermVector.NO));
+                            doc.add(new Field(attributeName, NOT_INDEXED_NO_TRANSFORMATION, Field.Store.NO,
+                                    Field.Index.TOKENIZED, Field.TermVector.NO));
                         }
                         else if (indexAtomicPropertiesOnly
                                 && transformer.getTransformationTime() > maxAtomicTransformationTime)
@@ -1554,8 +1581,10 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
                                 readerReady = false;
                                 // not indexed: transformation
                                 // failed
-                                //doc.add(new Field("TEXT", NOT_INDEXED_TRANSFORMATION_FAILED, Field.Store.NO, Field.Index.TOKENIZED, Field.TermVector.NO));
+                                // doc.add(new Field("TEXT", NOT_INDEXED_TRANSFORMATION_FAILED, Field.Store.NO,
-                                doc.add(new Field(attributeName, NOT_INDEXED_TRANSFORMATION_FAILED, Field.Store.NO, Field.Index.TOKENIZED, Field.TermVector.NO));
+                                // Field.Index.TOKENIZED, Field.TermVector.NO));
+                                doc.add(new Field(attributeName, NOT_INDEXED_TRANSFORMATION_FAILED, Field.Store.NO,
+                                        Field.Index.TOKENIZED, Field.TermVector.NO));
                             }
                         }
                     }
@@ -1584,10 +1613,11 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
                         {
                             isr = new InputStreamReader(ris);
                         }
-
+                        StringBuilder builder = new StringBuilder();
-                        doc.add(new Field("@"
+                        builder.append("\u0000").append(locale.toString()).append("\u0000");
-                                + QName.createQName(propertyName.getNamespaceURI(), ISO9075.encode(propertyName
+                        StringReader prefix = new StringReader(builder.toString());
-                                        .getLocalName())), isr, Field.TermVector.NO));
+                        Reader multiReader = new MultiReader(prefix, isr);
+                        doc.add(new Field(attributeName, multiReader, Field.TermVector.NO));
                     }
                 }
                 else
@@ -1601,8 +1631,10 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
                                 + (reader == null ? " --- " : Boolean.toString(reader.exists())));
                     }
                     // not indexed: content missing
-                    doc.add(new Field("TEXT", NOT_INDEXED_CONTENT_MISSING, Field.Store.NO, Field.Index.TOKENIZED, Field.TermVector.NO));
+                    doc.add(new Field("TEXT", NOT_INDEXED_CONTENT_MISSING, Field.Store.NO, Field.Index.TOKENIZED,
-                    doc.add(new Field(attributeName, NOT_INDEXED_CONTENT_MISSING, Field.Store.NO, Field.Index.TOKENIZED, Field.TermVector.NO));
+                            Field.TermVector.NO));
+                    doc.add(new Field(attributeName, NOT_INDEXED_CONTENT_MISSING, Field.Store.NO,
+                            Field.Index.TOKENIZED, Field.TermVector.NO));
                 }
             }
             else
@@ -1631,9 +1663,21 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
                     for (Locale locale : mlText.getLocales())
                     {
                         String localeString = mlText.getValue(locale);
-                        doc.add(new Field(attributeName, "\u0000" + locale.toString() +"\u0000" + localeString, fieldStore, fieldIndex, Field.TermVector.NO));
+                        StringBuilder builder = new StringBuilder();
+                        builder.append("\u0000").append(locale.toString()).append("\u0000").append(localeString);
+                        doc.add(new Field(attributeName, builder.toString(), fieldStore, fieldIndex,
+                                Field.TermVector.NO));
                     }
                 }
+                else if(isText)
+                {
+                    // TODO: Use the node locale in preferanced to the system locale
+                    Locale locale = Locale.getDefault();
+                    StringBuilder builder = new StringBuilder();
+                    builder.append("\u0000").append(locale.toString()).append("\u0000").append(strValue);
+                    doc.add(new Field(attributeName, builder.toString(), fieldStore, fieldIndex,
+                            Field.TermVector.NO));
+                }
                 else
                 {
                     doc.add(new Field(attributeName, strValue, fieldStore, fieldIndex, Field.TermVector.NO));
@@ -2010,9 +2054,4 @@ public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2
         }
     }
 
-   
-
-
-
-  
 }

source/java/org/alfresco/repo/search/impl/lucene/LuceneQueryParser.java

@@ -23,6 +23,7 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Set;
 
 import org.alfresco.i18n.I18NUtil;
 import org.alfresco.repo.search.SearcherException;
@@ -32,6 +33,7 @@ import org.alfresco.service.cmr.dictionary.DataTypeDefinition;
 import org.alfresco.service.cmr.dictionary.DictionaryService;
 import org.alfresco.service.cmr.dictionary.PropertyDefinition;
 import org.alfresco.service.cmr.dictionary.TypeDefinition;
+import org.alfresco.service.cmr.search.SearchParameters;
 import org.alfresco.service.namespace.NamespacePrefixResolver;
 import org.alfresco.service.namespace.QName;
 import org.apache.log4j.Logger;
@@ -55,7 +57,7 @@ public class LuceneQueryParser extends QueryParser
 
     private DictionaryService dictionaryService;
 
-    private List<Locale> locales;
+    private SearchParameters searchParameters;
 
     /**
      * Parses a query string, returning a {@link org.apache.lucene.search.Query}.
@@ -71,7 +73,7 @@ public class LuceneQueryParser extends QueryParser
      */
     static public Query parse(String query, String field, Analyzer analyzer,
             NamespacePrefixResolver namespacePrefixResolver, DictionaryService dictionaryService,
-            Operator defaultOperator, List<Locale> locales) throws ParseException
+            Operator defaultOperator, SearchParameters searchParameters) throws ParseException
     {
         if (s_logger.isDebugEnabled())
         {
@@ -81,14 +83,19 @@ public class LuceneQueryParser extends QueryParser
         parser.setDefaultOperator(defaultOperator);
         parser.setNamespacePrefixResolver(namespacePrefixResolver);
         parser.setDictionaryService(dictionaryService);
-        parser.setLocales(locales);
+        parser.setSearchParameters(searchParameters);
         // TODO: Apply locale contstraints at the top level if required for the non ML doc types.
-        return parser.parse(query);
+        Query result = parser.parse(query);
+        if (s_logger.isDebugEnabled())
+        {
+            s_logger.debug("Query " + query + "                             is\n\t" + result.toString());
+        }
+        return result;
     }
 
-    private void setLocales(List<Locale> locales)
+    private void setSearchParameters(SearchParameters searchParameters)
     {
-        this.locales = locales;
+        this.searchParameters = searchParameters;
     }
 
     public void setNamespacePrefixResolver(NamespacePrefixResolver namespacePrefixResolver)
@@ -141,16 +148,32 @@ public class LuceneQueryParser extends QueryParser
             }
             else if (field.equals("TEXT"))
             {
-                Collection<QName> contentAttributes = dictionaryService.getAllProperties(DataTypeDefinition.CONTENT);
+                Set<String> text = searchParameters.getTextAttributes();
+                if ((text == null) || (text.size() == 0))
+                {
+                    Collection<QName> contentAttributes = dictionaryService
+                            .getAllProperties(DataTypeDefinition.CONTENT);
                     BooleanQuery query = new BooleanQuery();
                     for (QName qname : contentAttributes)
                     {
                         // The super implementation will create phrase queries etc if required
-                    Query part = super.getFieldQuery("@" + qname.toString(), queryText);
+                        Query part = getFieldQuery("@" + qname.toString(), queryText);
                         query.add(part, Occur.SHOULD);
                     }
                     return query;
                 }
+                else
+                {
+                    BooleanQuery query = new BooleanQuery();
+                    for (String fieldName : text)
+                    {
+                        Query part = getFieldQuery(fieldName, queryText);
+                        query.add(part, Occur.SHOULD);
+                    }
+                    return query;
+                }
+
+            }
             else if (field.equals("ID"))
             {
                 TermQuery termQuery = new TermQuery(new Term(field, queryText));
@@ -232,6 +255,39 @@ public class LuceneQueryParser extends QueryParser
                 }
                 return booleanQuery;
             }
+            else if (field.equals("EXACTTYPE"))
+            {
+                TypeDefinition target;
+                if (queryText.startsWith("{"))
+                {
+                    target = dictionaryService.getType(QName.createQName(queryText));
+                }
+                else
+                {
+                    int colonPosition = queryText.indexOf(':');
+                    if (colonPosition == -1)
+                    {
+                        // use the default namespace
+                        target = dictionaryService.getType(QName.createQName(namespacePrefixResolver
+                                .getNamespaceURI(""), queryText));
+                    }
+                    else
+                    {
+                        // find the prefix
+                        target = dictionaryService.getType(QName.createQName(namespacePrefixResolver
+                                .getNamespaceURI(queryText.substring(0, colonPosition)), queryText
+                                .substring(colonPosition + 1)));
+                    }
+                }
+                if (target == null)
+                {
+                    throw new SearcherException("Invalid type: " + queryText);
+                }
+                QName targetQName = target.getName();
+                TermQuery termQuery = new TermQuery(new Term("TYPE", targetQName.toString()));
+                return termQuery;
+
+            }
             else if (field.equals("ASPECT"))
             {
                 AspectDefinition target;
@@ -281,100 +337,133 @@ public class LuceneQueryParser extends QueryParser
                 }
                 return booleanQuery;
             }
-            else if (field.startsWith("@"))
+            else if (field.equals("EXACTASPECT"))
             {
-                // Expand prefixes
+                AspectDefinition target;
-
+                if (queryText.startsWith("{"))
-                String expandedFieldName = field;
-                // Check for any prefixes and expand to the full uri
-                if (field.charAt(1) != '{')
                 {
-                    int colonPosition = field.indexOf(':');
+                    target = dictionaryService.getAspect(QName.createQName(queryText));
+                }
+                else
+                {
+                    int colonPosition = queryText.indexOf(':');
                     if (colonPosition == -1)
                     {
                         // use the default namespace
-                        expandedFieldName = "@{"
+                        target = dictionaryService.getAspect(QName.createQName(namespacePrefixResolver
-                                + namespacePrefixResolver.getNamespaceURI("") + "}" + field.substring(1);
+                                .getNamespaceURI(""), queryText));
                     }
                     else
                     {
                         // find the prefix
-                        expandedFieldName = "@{"
+                        target = dictionaryService.getAspect(QName.createQName(namespacePrefixResolver
-                                + namespacePrefixResolver.getNamespaceURI(field.substring(1, colonPosition)) + "}"
+                                .getNamespaceURI(queryText.substring(0, colonPosition)), queryText
-                                + field.substring(colonPosition + 1);
+                                .substring(colonPosition + 1)));
                     }
                 }
 
-                // Mime type
+                QName targetQName = target.getName();
-                if (expandedFieldName.endsWith(".mimetype"))
+                TermQuery termQuery = new TermQuery(new Term("ASPECT", targetQName.toString()));
-                {
-                    QName propertyQName = QName.createQName(expandedFieldName.substring(1,
-                            expandedFieldName.length() - 9));
-                    PropertyDefinition propertyDef = dictionaryService.getProperty(propertyQName);
-                    if ((propertyDef != null)
-                            && (propertyDef.getDataType().getName().equals(DataTypeDefinition.CONTENT)))
-                    {
-                        return super.getFieldQuery(expandedFieldName, queryText);
-                    }
 
+                return termQuery;
             }
-                else if (expandedFieldName.endsWith(".size"))
+            else if (field.startsWith("@"))
             {
-                    QName propertyQName = QName.createQName(expandedFieldName.substring(1,
+                return attributeQueryBuilder(field, queryText, new FieldQuery());
-                            expandedFieldName.length() - 5));
-                    PropertyDefinition propertyDef = dictionaryService.getProperty(propertyQName);
-                    if ((propertyDef != null)
-                            && (propertyDef.getDataType().getName().equals(DataTypeDefinition.CONTENT)))
-                    {
-                        return super.getFieldQuery(expandedFieldName, queryText);
             }
-
+            else if (field.equals("ALL"))
-                }
-                else if (expandedFieldName.endsWith(".locale"))
             {
-                    QName propertyQName = QName.createQName(expandedFieldName.substring(1,
+                Set<String> all = searchParameters.getAllAttributes();
-                            expandedFieldName.length() - 7));
+                if ((all == null) || (all.size() == 0))
-                    PropertyDefinition propertyDef = dictionaryService.getProperty(propertyQName);
-                    if ((propertyDef != null)
-                            && (propertyDef.getDataType().getName().equals(DataTypeDefinition.CONTENT)))
                 {
-                        return super.getFieldQuery(expandedFieldName, queryText);
+                    Collection<QName> contentAttributes = dictionaryService.getAllProperties(null);
-                    }
+                    BooleanQuery query = new BooleanQuery();
-
+                    for (QName qname : contentAttributes)
-                }
-
-
-                // Already in expanded form
-
-                // ML
-
-                QName propertyQName = QName.createQName(expandedFieldName.substring(1));
-                PropertyDefinition propertyDef = dictionaryService.getProperty(propertyQName);
-                if ((propertyDef != null) && (propertyDef.getDataType().getName().equals(DataTypeDefinition.MLTEXT)))
                     {
-                    // Build a sub query for each locale and or the results together - the analysis will take care of
+                        // The super implementation will create phrase queries etc if required
-                    // cross language matching for each entry
+                        Query part = getFieldQuery("@" + qname.toString(), queryText);
-                    BooleanQuery booleanQuery = new BooleanQuery();
+                        if (part != null)
-                    for (Locale locale : (((locales == null) || (locales.size() == 0)) ? Collections
-                            .singletonList(I18NUtil.getLocale()) : locales))
                         {
-                        StringBuilder builder = new StringBuilder(queryText.length() + 10);
+                            query.add(part, Occur.SHOULD);
-                        builder.append("\u0000").append(locale.toString()).append("\u0000").append(queryText);
-                        Query subQuery = super.getFieldQuery(expandedFieldName, builder.toString());
-                        booleanQuery.add(subQuery, Occur.SHOULD);
                         }
-                    return booleanQuery;
+                    }
+                    return query;
                 }
                 else
                 {
-                    return super.getFieldQuery(expandedFieldName, queryText);
+                    BooleanQuery query = new BooleanQuery();
+                    for (String fieldName : all)
+                    {
+                        Query part = getFieldQuery(fieldName, queryText);
+                        if (part != null)
+                        {
+                            query.add(part, Occur.SHOULD);
+                        }
+                    }
+                    return query;
                 }
 
             }
+            else if (field.equals("ISNULL"))
+            {
+                String qnameString = expandFieldName(queryText);
+                QName qname = QName.createQName(qnameString);
+                PropertyDefinition pd = dictionaryService.getProperty(qname);
+                if (pd != null)
+                {
+                    QName container = pd.getContainerClass().getName();
+                    BooleanQuery query = new BooleanQuery();
+                    Query typeQuery = getFieldQuery("TYPE", container.toString());
+                    query.add(typeQuery, Occur.MUST);
+                    Query presenceQuery = getWildcardQuery("@" + qname.toString(), "*");
+                    query.add(presenceQuery, Occur.MUST_NOT);
+                    return query;
+                }
                 else
                 {
                     return super.getFieldQuery(field, queryText);
                 }
+
+            }
+            else if (field.equals("ISNOTNULL"))
+            {
+                String qnameString = expandFieldName(queryText);
+                QName qname = QName.createQName(qnameString);
+                PropertyDefinition pd = dictionaryService.getProperty(qname);
+                if (pd != null)
+                {
+                    QName container = pd.getContainerClass().getName();
+                    BooleanQuery query = new BooleanQuery();
+                    Query typeQuery = getFieldQuery("TYPE", container.toString());
+                    query.add(typeQuery, Occur.MUST);
+                    Query presenceQuery = getWildcardQuery("@" + qname.toString(), "*");
+                    query.add(presenceQuery, Occur.MUST);
+                    return query;
+                }
+                else
+                {
+                    return super.getFieldQuery(field, queryText);
+                }
+
+            }
+            else if (dictionaryService.getDataType(QName.createQName(expandFieldName(field))) != null)
+            {
+                Collection<QName> contentAttributes = dictionaryService.getAllProperties(dictionaryService.getDataType(
+                        QName.createQName(expandFieldName(field))).getName());
+                BooleanQuery query = new BooleanQuery();
+                for (QName qname : contentAttributes)
+                {
+                    // The super implementation will create phrase queries etc if required
+                    Query part = getFieldQuery("@" + qname.toString(), queryText);
+                    query.add(part, Occur.SHOULD);
+                }
+                return query;
+            }
+            else
+            {
+                return super.getFieldQuery(field, queryText);
+            }
+
         }
         catch (SAXPathException e)
         {
@@ -390,6 +479,20 @@ public class LuceneQueryParser extends QueryParser
     protected Query getRangeQuery(String field, String part1, String part2, boolean inclusive) throws ParseException
     {
         if (field.startsWith("@"))
+        {
+            String fieldName = expandAttributeFieldName(field);
+            return new RangeQuery(new Term(fieldName, getToken(fieldName, part1)), new Term(fieldName, getToken(
+                    fieldName, part2)), inclusive);
+
+        }
+        else
+        {
+            return super.getRangeQuery(field, part1, part2, inclusive);
+        }
+
+    }
+
+    private String expandAttributeFieldName(String field)
     {
         String fieldName = field;
         // Check for any prefixes and expand to the full uri
@@ -409,15 +512,30 @@ public class LuceneQueryParser extends QueryParser
                         + field.substring(colonPosition + 1);
             }
         }
-            return new RangeQuery(new Term(fieldName, getToken(fieldName, part1)), new Term(fieldName, getToken(
+        return fieldName;
-                    fieldName, part2)), inclusive);
+    }
 
+    private String expandFieldName(String field)
+    {
+        String fieldName = field;
+        // Check for any prefixes and expand to the full uri
+        if (field.charAt(0) != '{')
+        {
+            int colonPosition = field.indexOf(':');
+            if (colonPosition == -1)
+            {
+                // use the default namespace
+                fieldName = "{" + namespacePrefixResolver.getNamespaceURI("") + "}" + field;
             }
             else
             {
-            return super.getRangeQuery(field, part1, part2, inclusive);
+                // find the prefix
+                fieldName = "{"
+                        + namespacePrefixResolver.getNamespaceURI(field.substring(0, colonPosition)) + "}"
+                        + field.substring(colonPosition + 1);
             }
-
+        }
+        return fieldName;
     }
 
     private String getToken(String field, String value)
@@ -457,67 +575,8 @@ public class LuceneQueryParser extends QueryParser
     {
         if (field.startsWith("@"))
         {
-            // Expand prefixes
+            return attributeQueryBuilder(field, termStr, new PrefixQuery());
-
-            String expandedFieldName = field;
-            // Check for any prefixes and expand to the full uri
-            if (field.charAt(1) != '{')
-            {
-                int colonPosition = field.indexOf(':');
-                if (colonPosition == -1)
-                {
-                    // use the default namespace
-                    expandedFieldName = "@{" + namespacePrefixResolver.getNamespaceURI("") + "}" + field.substring(1);
         }
-                else
-                {
-                    // find the prefix
-                    expandedFieldName = "@{"
-                            + namespacePrefixResolver.getNamespaceURI(field.substring(1, colonPosition)) + "}"
-                            + field.substring(colonPosition + 1);
-                }
-            }
-
-            // Mime type
-            if (expandedFieldName.endsWith(".mimetype"))
-            {
-                QName propertyQName = QName.createQName(expandedFieldName.substring(1, expandedFieldName.length() - 9));
-                PropertyDefinition propertyDef = dictionaryService.getProperty(propertyQName);
-                if ((propertyDef != null) && (propertyDef.getDataType().getName().equals(DataTypeDefinition.CONTENT)))
-                {
-                    return super.getPrefixQuery(expandedFieldName, termStr);
-                }
-
-            }
-
-            // Already in expanded form
-
-            // ML
-
-            QName propertyQName = QName.createQName(expandedFieldName.substring(1));
-            PropertyDefinition propertyDef = dictionaryService.getProperty(propertyQName);
-            if ((propertyDef != null) && (propertyDef.getDataType().getName().equals(DataTypeDefinition.MLTEXT)))
-            {
-                // Build a sub query for each locale and or the results together - the analysis will take care of
-                // cross language matching for each entry
-                BooleanQuery booleanQuery = new BooleanQuery();
-                for (Locale locale : (((locales == null) || (locales.size() == 0)) ? Collections.singletonList(I18NUtil
-                        .getLocale()) : locales))
-                {
-                    StringBuilder builder = new StringBuilder(termStr.length() + 10);
-                    builder.append("\u0000").append(locale.toString()).append("\u0000").append(termStr);
-                    Query subQuery = super.getPrefixQuery(expandedFieldName, builder.toString());
-                    booleanQuery.add(subQuery, Occur.SHOULD);
-                }
-                return booleanQuery;
-            }
-            else
-            {
-                return super.getPrefixQuery(expandedFieldName, termStr);
-            }
-
-        }
-
         else if (field.equals("TEXT"))
         {
             Collection<QName> contentAttributes = dictionaryService.getAllProperties(DataTypeDefinition.CONTENT);
@@ -525,15 +584,14 @@ public class LuceneQueryParser extends QueryParser
             for (QName qname : contentAttributes)
             {
                 // The super implementation will create phrase queries etc if required
-                Query part = super.getPrefixQuery("@" + qname.toString(), termStr);
+                Query part = getPrefixQuery("@" + qname.toString(), termStr);
                 query.add(part, Occur.SHOULD);
             }
             return query;
-
         }
         else
         {
-            return super.getFieldQuery(field, termStr);
+            return super.getPrefixQuery(field, termStr);
         }
     }
 
@@ -542,65 +600,7 @@ public class LuceneQueryParser extends QueryParser
     {
         if (field.startsWith("@"))
         {
-            // Expand prefixes
+            return attributeQueryBuilder(field, termStr, new WildcardQuery());
-
-            String expandedFieldName = field;
-            // Check for any prefixes and expand to the full uri
-            if (field.charAt(1) != '{')
-            {
-                int colonPosition = field.indexOf(':');
-                if (colonPosition == -1)
-                {
-                    // use the default namespace
-                    expandedFieldName = "@{" + namespacePrefixResolver.getNamespaceURI("") + "}" + field.substring(1);
-                }
-                else
-                {
-                    // find the prefix
-                    expandedFieldName = "@{"
-                            + namespacePrefixResolver.getNamespaceURI(field.substring(1, colonPosition)) + "}"
-                            + field.substring(colonPosition + 1);
-                }
-            }
-
-            // Mime type
-            if (expandedFieldName.endsWith(".mimetype"))
-            {
-                QName propertyQName = QName.createQName(expandedFieldName.substring(1, expandedFieldName.length() - 9));
-                PropertyDefinition propertyDef = dictionaryService.getProperty(propertyQName);
-                if ((propertyDef != null) && (propertyDef.getDataType().getName().equals(DataTypeDefinition.CONTENT)))
-                {
-                    return super.getWildcardQuery(expandedFieldName, termStr);
-                }
-
-            }
-
-            // Already in expanded form
-
-            // ML
-
-            QName propertyQName = QName.createQName(expandedFieldName.substring(1));
-            PropertyDefinition propertyDef = dictionaryService.getProperty(propertyQName);
-            if ((propertyDef != null) && (propertyDef.getDataType().getName().equals(DataTypeDefinition.MLTEXT)))
-            {
-                // Build a sub query for each locale and or the results together - the analysis will take care of
-                // cross language matching for each entry
-                BooleanQuery booleanQuery = new BooleanQuery();
-                for (Locale locale : (((locales == null) || (locales.size() == 0)) ? Collections.singletonList(I18NUtil
-                        .getLocale()) : locales))
-                {
-                    StringBuilder builder = new StringBuilder(termStr.length() + 10);
-                    builder.append("\u0000").append(locale.toString()).append("\u0000").append(termStr);
-                    Query subQuery = super.getWildcardQuery(expandedFieldName, builder.toString());
-                    booleanQuery.add(subQuery, Occur.SHOULD);
-                }
-                return booleanQuery;
-            }
-            else
-            {
-                return super.getWildcardQuery(expandedFieldName, termStr);
-            }
-
         }
 
         else if (field.equals("TEXT"))
@@ -610,11 +610,10 @@ public class LuceneQueryParser extends QueryParser
             for (QName qname : contentAttributes)
             {
                 // The super implementation will create phrase queries etc if required
-                Query part = super.getWildcardQuery("@" + qname.toString(), termStr);
+                Query part = getWildcardQuery("@" + qname.toString(), termStr);
                 query.add(part, Occur.SHOULD);
             }
             return query;
-
         }
         else
         {
@@ -627,27 +626,102 @@ public class LuceneQueryParser extends QueryParser
     {
         if (field.startsWith("@"))
         {
-            // Expand prefixes
+            return attributeQueryBuilder(field, termStr, new FuzzyQuery(minSimilarity));
+        }
 
-            String expandedFieldName = field;
+        else if (field.equals("TEXT"))
-            // Check for any prefixes and expand to the full uri
-            if (field.charAt(1) != '{')
         {
-                int colonPosition = field.indexOf(':');
+            Collection<QName> contentAttributes = dictionaryService.getAllProperties(DataTypeDefinition.CONTENT);
-                if (colonPosition == -1)
+            BooleanQuery query = new BooleanQuery();
+            for (QName qname : contentAttributes)
             {
-                    // use the default namespace
+                // The super implementation will create phrase queries etc if required
-                    expandedFieldName = "@{" + namespacePrefixResolver.getNamespaceURI("") + "}" + field.substring(1);
+                Query part = getFuzzyQuery("@" + qname.toString(), termStr, minSimilarity);
+                query.add(part, Occur.SHOULD);
+            }
+            return query;
         }
         else
         {
-                    // find the prefix
+            return super.getFuzzyQuery(field, termStr, minSimilarity);
-                    expandedFieldName = "@{"
-                            + namespacePrefixResolver.getNamespaceURI(field.substring(1, colonPosition)) + "}"
-                            + field.substring(colonPosition + 1);
         }
     }
 
+    public void setDictionaryService(DictionaryService dictionaryService)
+    {
+        this.dictionaryService = dictionaryService;
+    }
+
+    public Query getSuperFieldQuery(String field, String queryText) throws ParseException
+    {
+        return super.getFieldQuery(field, queryText);
+    }
+
+    public Query getSuperFuzzyQuery(String field, String termStr, float minSimilarity) throws ParseException
+    {
+        return super.getFuzzyQuery(field, termStr, minSimilarity);
+    }
+
+    public Query getSuperPrefixQuery(String field, String termStr) throws ParseException
+    {
+        return super.getPrefixQuery(field, termStr);
+    }
+
+    public Query getSuperWildcardQuery(String field, String termStr) throws ParseException
+    {
+        return super.getWildcardQuery(field, termStr);
+    }
+
+    interface SubQuery
+    {
+        Query getQuery(String field, String queryText) throws ParseException;
+    }
+
+    class FieldQuery implements SubQuery
+    {
+        public Query getQuery(String field, String queryText) throws ParseException
+        {
+            return getSuperFieldQuery(field, queryText);
+        }
+    }
+
+    class FuzzyQuery implements SubQuery
+    {
+        float minSimilarity;
+
+        FuzzyQuery(float minSimilarity)
+        {
+            this.minSimilarity = minSimilarity;
+        }
+
+        public Query getQuery(String field, String termStr) throws ParseException
+        {
+            return getSuperFuzzyQuery(field, termStr, minSimilarity);
+        }
+    }
+
+    class PrefixQuery implements SubQuery
+    {
+        public Query getQuery(String field, String termStr) throws ParseException
+        {
+            return getSuperPrefixQuery(field, termStr);
+        }
+    }
+
+    class WildcardQuery implements SubQuery
+    {
+        public Query getQuery(String field, String termStr) throws ParseException
+        {
+            return getSuperWildcardQuery(field, termStr);
+        }
+    }
+
+    private Query attributeQueryBuilder(String field, String queryText, SubQuery subQueryBuilder) throws ParseException
+    {
+        // Expand prefixes
+
+        String expandedFieldName = expandAttributeFieldName(field);
+
         // Mime type
         if (expandedFieldName.endsWith(".mimetype"))
         {
@@ -655,7 +729,27 @@ public class LuceneQueryParser extends QueryParser
             PropertyDefinition propertyDef = dictionaryService.getProperty(propertyQName);
             if ((propertyDef != null) && (propertyDef.getDataType().getName().equals(DataTypeDefinition.CONTENT)))
             {
-                    return super.getFuzzyQuery(expandedFieldName, termStr, minSimilarity);
+                return subQueryBuilder.getQuery(expandedFieldName, queryText);
+            }
+
+        }
+        else if (expandedFieldName.endsWith(".size"))
+        {
+            QName propertyQName = QName.createQName(expandedFieldName.substring(1, expandedFieldName.length() - 5));
+            PropertyDefinition propertyDef = dictionaryService.getProperty(propertyQName);
+            if ((propertyDef != null) && (propertyDef.getDataType().getName().equals(DataTypeDefinition.CONTENT)))
+            {
+                return subQueryBuilder.getQuery(expandedFieldName, queryText);
+            }
+
+        }
+        else if (expandedFieldName.endsWith(".locale"))
+        {
+            QName propertyQName = QName.createQName(expandedFieldName.substring(1, expandedFieldName.length() - 7));
+            PropertyDefinition propertyDef = dictionaryService.getProperty(propertyQName);
+            if ((propertyDef != null) && (propertyDef.getDataType().getName().equals(DataTypeDefinition.CONTENT)))
+            {
+                return subQueryBuilder.getQuery(expandedFieldName, queryText);
             }
 
         }
@@ -671,45 +765,42 @@ public class LuceneQueryParser extends QueryParser
             // Build a sub query for each locale and or the results together - the analysis will take care of
             // cross language matching for each entry
             BooleanQuery booleanQuery = new BooleanQuery();
+            List<Locale> locales = searchParameters.getLocales();
             for (Locale locale : (((locales == null) || (locales.size() == 0)) ? Collections.singletonList(I18NUtil
                     .getLocale()) : locales))
             {
-                    StringBuilder builder = new StringBuilder(termStr.length() + 10);
+                StringBuilder builder = new StringBuilder(queryText.length() + 10);
-                    builder.append("\u0000").append(locale.toString()).append("\u0000").append(termStr);
+                builder.append("\u0000").append(locale.toString()).append("\u0000").append(queryText);
-                    Query subQuery = super.getFuzzyQuery(expandedFieldName, builder.toString(), minSimilarity);
+                Query subQuery = subQueryBuilder.getQuery(expandedFieldName, builder.toString());
+                booleanQuery.add(subQuery, Occur.SHOULD);
+            }
+            return booleanQuery;
+        }
+        // Content
+        else if ((propertyDef != null) && (propertyDef.getDataType().getName().equals(DataTypeDefinition.CONTENT)))
+        {
+            // Build a sub query for each locale and or the results together -
+            // - add an explicit condition for the locale
+            BooleanQuery booleanQuery = new BooleanQuery();
+            List<Locale> locales = searchParameters.getLocales();
+            for (Locale locale : (((locales == null) || (locales.size() == 0)) ? Collections.singletonList(I18NUtil
+                    .getLocale()) : locales))
+            {
+                BooleanQuery subQuery = new BooleanQuery();
+                Query contentQuery = subQueryBuilder.getQuery(expandedFieldName, queryText);
+                subQuery.add(contentQuery, Occur.MUST);
+                StringBuilder builder = new StringBuilder();
+                builder.append(expandedFieldName).append(".locale");
+                Query localeQuery = getFieldQuery(builder.toString(), locale.toString());
+                subQuery.add(localeQuery, Occur.MUST);
                 booleanQuery.add(subQuery, Occur.SHOULD);
             }
             return booleanQuery;
         }
         else
         {
-                return super.getFuzzyQuery(expandedFieldName, termStr, minSimilarity);
+            return subQueryBuilder.getQuery(expandedFieldName, queryText);
-            }
-
-        }
-
-        else if (field.equals("TEXT"))
-        {
-            Collection<QName> contentAttributes = dictionaryService.getAllProperties(DataTypeDefinition.CONTENT);
-            BooleanQuery query = new BooleanQuery();
-            for (QName qname : contentAttributes)
-            {
-                // The super implementation will create phrase queries etc if required
-                Query part = super.getFuzzyQuery("@" + qname.toString(), termStr, minSimilarity);
-                query.add(part, Occur.SHOULD);
-            }
-            return query;
-
-        }
-        else
-        {
-            return super.getFuzzyQuery(field, termStr, minSimilarity);
         }
     }
 
-    public void setDictionaryService(DictionaryService dictionaryService)
-    {
-        this.dictionaryService = dictionaryService;
-    }
-
 }

source/java/org/alfresco/repo/search/impl/lucene/LuceneSearcherImpl2.java

@@ -215,7 +215,7 @@ public class LuceneSearcherImpl2 extends LuceneBase2 implements LuceneSearcher2
                 }
 
                 Query query = LuceneQueryParser.parse(parameterisedQueryString, DEFAULT_FIELD, new LuceneAnalyser(
-                        dictionaryService, searchParameters.getMlAnalaysisMode() == null ? getLuceneConfig().getDefaultMLSearchAnalysisMode() : searchParameters.getMlAnalaysisMode()), namespacePrefixResolver, dictionaryService, defaultOperator, searchParameters.getLocales());
+                        dictionaryService, searchParameters.getMlAnalaysisMode() == null ? getLuceneConfig().getDefaultMLSearchAnalysisMode() : searchParameters.getMlAnalaysisMode()), namespacePrefixResolver, dictionaryService, defaultOperator, searchParameters);
                 ClosingIndexSearcher searcher = getSearcher(indexer);
                 if (searcher == null)
                 {

source/java/org/alfresco/repo/search/impl/lucene/LuceneTest2.java

@@ -42,6 +42,7 @@ import org.alfresco.repo.dictionary.DictionaryNamespaceComponent;
 import org.alfresco.repo.dictionary.M2Model;
 import org.alfresco.repo.dictionary.NamespaceDAOImpl;
 import org.alfresco.repo.node.BaseNodeServiceTest;
+import org.alfresco.repo.search.MLAnalysisMode;
 import org.alfresco.repo.search.QueryParameterDefImpl;
 import org.alfresco.repo.search.QueryRegisterComponent;
 import org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer;
@@ -1896,6 +1897,16 @@ public class LuceneTest2 extends TestCase
         assertEquals(1, results.length());
         results.close();
         
+        results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "EXACTTYPE:\"" + testType.toString() + "\"", null,
+                null);
+        assertEquals(1, results.length());
+        results.close();
+
+        results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "EXACTTYPE:\""
+                + testType.toPrefixString(namespacePrefixResolver) + "\"", null, null);
+        assertEquals(1, results.length());
+        results.close();
+
         results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "TYPE:\"" + testSuperType.toString() + "\"",
                 null, null);
         assertEquals(13, results.length());
@@ -1906,6 +1917,16 @@ public class LuceneTest2 extends TestCase
         assertEquals(13, results.length());
         results.close();
         
+        results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "EXACTTYPE:\"" + testSuperType.toString() + "\"",
+                null, null);
+        assertEquals(12, results.length());
+        results.close();
+
+        results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "EXACTTYPE:\""
+                + testSuperType.toPrefixString(namespacePrefixResolver) + "\"", null, null);
+        assertEquals(12, results.length());
+        results.close();
+
         results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "ASPECT:\"" + testAspect.toString() + "\"", null,
                 null);
         assertEquals(1, results.length());
@@ -1926,6 +1947,28 @@ public class LuceneTest2 extends TestCase
         assertEquals(1, results.length());
         results.close();
 
+        // Test for AR-384
+
+        results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "TEXT:fox AND TYPE:\""
+                + ContentModel.PROP_CONTENT.toString() + "\"", null, null);
+        assertEquals(1, results.length());
+        results.close();
+
+        results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "TEXT:fo AND TYPE:\""
+                + ContentModel.PROP_CONTENT.toString() + "\"", null, null);
+        assertEquals(0, results.length());
+        results.close();
+
+        // Test stop words are equivalent
+
+        results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "TEXT:\"over the lazy\"", null, null);
+        assertEquals(1, results.length());
+        results.close();
+
+        results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "TEXT:\"over a lazy\"", null, null);
+        assertEquals(1, results.length());
+        results.close();
+
         // FTS test
 
         results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "TEXT:\"fox\"", null, null);
@@ -1943,16 +1986,127 @@ public class LuceneTest2 extends TestCase
         assertEquals(1, results.length());
         results.close();
 
+        results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "@"
+                + LuceneQueryParser.escape(ContentModel.PROP_CONTENT.toString()) + ".locale:\"en_GB\"", null, null);
+        assertEquals(1, results.length());
+        results.close();
+
+        results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "@"
+                + LuceneQueryParser.escape(ContentModel.PROP_CONTENT.toString()) + ".locale:en_*", null, null);
+        assertEquals(1, results.length());
+        results.close();
+
+        results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "@"
+                + LuceneQueryParser.escape(ContentModel.PROP_CONTENT.toString()) + ".locale:e*_GB", null, null);
+        assertEquals(1, results.length());
+        results.close();
+
+        results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "@"
+                + LuceneQueryParser.escape(ContentModel.PROP_CONTENT.toString()) + ".size:\"90\"", null, null);
+        assertEquals(1, results.length());
+        results.close();
+
         QName queryQName = QName.createQName("alf:test1", namespacePrefixResolver);
         results = searcher.query(rootNodeRef.getStoreRef(), queryQName, null);
         assertEquals(1, results.length());
         results.close();
 
+        // Configuration of TEXT
+
+        SearchParameters sp = new SearchParameters();
+        sp.addStore(rootNodeRef.getStoreRef());
+        sp.setLanguage("lucene");
+        sp.setQuery("@" + LuceneQueryParser.escape(ContentModel.PROP_CONTENT.toString()) + ":\"fox\"");
+        results = searcher.query(sp);
+        assertEquals(1, results.length());
+        results.close();
+        
+        sp = new SearchParameters();
+        sp.addStore(rootNodeRef.getStoreRef());
+        sp.setLanguage("lucene");
+        sp.setQuery("TEXT:\"fox\"");
+        results = searcher.query(sp);
+        assertEquals(1, results.length());
+        results.close();
+        
+        sp = new SearchParameters();
+        sp.addStore(rootNodeRef.getStoreRef());
+        sp.setLanguage("lucene");
+        sp.setQuery("TEXT:\"fox\"");
+        sp.addTextAttribute("@"+ContentModel.PROP_NAME.toString());
+        results = searcher.query(sp);
+        assertEquals(0, results.length());
+        results.close();
+        
+        sp.addTextAttribute("@"+ContentModel.PROP_CONTENT.toString());
+        results = searcher.query(sp);
+        assertEquals(1, results.length());
+        results.close();
+
+        // ALL and its configuration
+        
+        sp = new SearchParameters();
+        sp.addStore(rootNodeRef.getStoreRef());
+        sp.setLanguage("lucene");
+        sp.setQuery("ALL:\"fox\"");
+        results = searcher.query(sp);
+        assertEquals(1, results.length());
+        results.close();
+        
+        sp = new SearchParameters();
+        sp.addStore(rootNodeRef.getStoreRef());
+        sp.setLanguage("lucene");
+        sp.setQuery("ALL:\"fox\"");
+        sp.addAllAttribute("@"+ContentModel.PROP_NAME.toString());
+        results = searcher.query(sp);
+        assertEquals(0, results.length());
+        results.close();
+        
+        sp.addAllAttribute("@"+ContentModel.PROP_CONTENT.toString());
+        results = searcher.query(sp);
+        assertEquals(1, results.length());
+        results.close();
+        
+        sp = new SearchParameters();
+        sp.addStore(rootNodeRef.getStoreRef());
+        sp.setLanguage("lucene");
+        sp.setQuery("ALL:\"5.6\"");
+        results = searcher.query(sp);
+        assertEquals(1, results.length());
+        results.close();
+        
+        // Search by data type
+        
+        sp = new SearchParameters();
+        sp.addStore(rootNodeRef.getStoreRef());
+        sp.setLanguage("lucene");
+        sp.setQuery("d\\:double:\"5.6\"");
+        results = searcher.query(sp);
+        assertEquals(1, results.length());
+        results.close();
+        
+        sp = new SearchParameters();
+        sp.addStore(rootNodeRef.getStoreRef());
+        sp.setLanguage("lucene");
+        sp.setQuery("d\\:content:\"fox\"");
+        results = searcher.query(sp);
+        assertEquals(1, results.length());
+        results.close();
+        
         // Direct ML tests
 
         QName mlQName = QName.createQName(TEST_NAMESPACE, "ml");
 
-        SearchParameters sp = new SearchParameters();
+        sp = new SearchParameters();
+        sp.addStore(rootNodeRef.getStoreRef());
+        sp.setLanguage("lucene");
+        sp.setMlAnalaysisMode(MLAnalysisMode.ALL_ONLY);
+        sp.setQuery("@" + LuceneQueryParser.escape(mlQName.toString()) + ":banana");
+        results = searcher.query(sp);
+        assertEquals(1, results.length());
+        results.close();
+
+        sp = new SearchParameters();
         sp.addStore(rootNodeRef.getStoreRef());
         sp.setLanguage("lucene");
         sp.setQuery("@" + LuceneQueryParser.escape(mlQName.toString()) + ":banana");
@@ -1960,6 +2114,20 @@ public class LuceneTest2 extends TestCase
         assertEquals(1, results.length());
         results.close();
 
+        sp = new SearchParameters();
+        sp.addStore(rootNodeRef.getStoreRef());
+        sp.setLanguage("lucene");
+        sp.setQuery("@" + LuceneQueryParser.escape(mlQName.toString()) + ":banana");
+        sp.addLocale(Locale.UK);
+        results = searcher.query(sp);
+        assertEquals(1, results.length());
+        results.close();
+
+        sp = new SearchParameters();
+        sp.addStore(rootNodeRef.getStoreRef());
+        sp.setLanguage("lucene");
+        sp.setQuery("@" + LuceneQueryParser.escape(mlQName.toString()) + ":banana");
+        sp.setMlAnalaysisMode(MLAnalysisMode.LOCALE_AND_ALL_CONTAINING_LOCALES);
         sp.addLocale(Locale.UK);
         results = searcher.query(sp);
         assertEquals(1, results.length());
@@ -2073,6 +2241,42 @@ public class LuceneTest2 extends TestCase
         assertEquals(1, results.length());
         results.close();
 
+        // Test ISNULL/ISNOTNULL
+        
+        sp = new SearchParameters();
+        sp.addStore(rootNodeRef.getStoreRef());
+        sp.setLanguage("lucene");
+        sp.setQuery("ISNULL:\"" + QName.createQName(TEST_NAMESPACE, "null").toString() + "\"");
+        results = searcher.query(sp);
+        assertEquals(1, results.length());
+        results.close();
+        
+        sp = new SearchParameters();
+        sp.addStore(rootNodeRef.getStoreRef());
+        sp.setLanguage("lucene");
+        sp.setQuery("ISNULL:\"" + QName.createQName(TEST_NAMESPACE, "path-ista").toString() + "\"");
+        results = searcher.query(sp);
+        assertEquals(0, results.length());
+        results.close();
+        
+        sp = new SearchParameters();
+        sp.addStore(rootNodeRef.getStoreRef());
+        sp.setLanguage("lucene");
+        sp.setQuery("ISNOTNULL:\"" + QName.createQName(TEST_NAMESPACE, "null").toString() + "\"");
+        results = searcher.query(sp);
+        assertEquals(0, results.length());
+        results.close();
+        
+        sp = new SearchParameters();
+        sp.addStore(rootNodeRef.getStoreRef());
+        sp.setLanguage("lucene");
+        sp.setQuery("ISNOTNULL:\"" + QName.createQName(TEST_NAMESPACE, "path-ista").toString() + "\"");
+        results = searcher.query(sp);
+        assertEquals(1, results.length());
+        results.close();
+        
+        
+        
         // Test non field queries
 
         results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "TEXT:fox", null, null);
@@ -2112,29 +2316,29 @@ public class LuceneTest2 extends TestCase
         results.close();
 
         results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "@"
-                + LuceneQueryParser.escape(ContentModel.PROP_CONTENT.toPrefixString(namespacePrefixResolver)) +":fox", null, null);
+                + LuceneQueryParser.escape(ContentModel.PROP_CONTENT.toPrefixString(namespacePrefixResolver)) + ":fox",
+                null, null);
         assertEquals(1, results.length());
         results.close();
 
         results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "@"
-                + LuceneQueryParser.escape(ContentModel.PROP_CONTENT.toPrefixString(namespacePrefixResolver)) +":fo*", null, null);
+                + LuceneQueryParser.escape(ContentModel.PROP_CONTENT.toPrefixString(namespacePrefixResolver)) + ":fo*",
+                null, null);
         assertEquals(1, results.length());
         results.close();
 
         results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "@"
-                + LuceneQueryParser.escape(ContentModel.PROP_CONTENT.toPrefixString(namespacePrefixResolver)) +":f*x", null, null);
+                + LuceneQueryParser.escape(ContentModel.PROP_CONTENT.toPrefixString(namespacePrefixResolver)) + ":f*x",
+                null, null);
         assertEquals(1, results.length());
         results.close();
 
         results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "@"
-                + LuceneQueryParser.escape(ContentModel.PROP_CONTENT.toPrefixString(namespacePrefixResolver)) +":*ox", null, null);
+                + LuceneQueryParser.escape(ContentModel.PROP_CONTENT.toPrefixString(namespacePrefixResolver)) + ":*ox",
+                null, null);
         assertEquals(1, results.length());
         results.close();
 
-       
-       
-
-
         // Parameters
 
         queryQName = QName.createQName("alf:test2", namespacePrefixResolver);
@@ -3661,6 +3865,7 @@ public class LuceneTest2 extends TestCase
         DynamicNamespacePrefixResolver nspr = new DynamicNamespacePrefixResolver(null);
         nspr.registerNamespace(NamespaceService.ALFRESCO_PREFIX, NamespaceService.ALFRESCO_URI);
         nspr.registerNamespace(NamespaceService.CONTENT_MODEL_PREFIX, NamespaceService.CONTENT_MODEL_1_0_URI);
+        nspr.registerNamespace(NamespaceService.DICTIONARY_MODEL_PREFIX, NamespaceService.DICTIONARY_MODEL_1_0_URI);
         nspr.registerNamespace("namespace", "namespace");
         nspr.registerNamespace("test", TEST_NAMESPACE);
         nspr.registerNamespace(NamespaceService.DEFAULT_PREFIX, defaultURI);

source/java/org/alfresco/repo/search/impl/lucene/LuceneTest_model.xml

@@ -238,6 +238,26 @@
                         <stored>true</stored>
                         <tokenised>true</tokenised>
                     </index>
+                </property>
+				<property name="test:null">
+                    <type>d:text</type>
+                    <mandatory>false</mandatory>
+                    <multiple>false</multiple>
+                    <index enabled="true">
+                        <atomic>true</atomic>
+                        <stored>true</stored>
+                        <tokenised>true</tokenised>
+                    </index>
+                </property>
+				<property name="test:path-ista">
+                    <type>d:path</type>
+                    <mandatory>false</mandatory>
+                    <multiple>false</multiple>
+                    <index enabled="true">
+                        <atomic>true</atomic>
+                        <stored>true</stored>
+                        <tokenised>true</tokenised>
+                    </index>
                 </property>
             </properties>
             <mandatory-aspects>

source/java/org/alfresco/repo/search/impl/lucene/MultiFieldQueryParser.java

@@ -16,6 +16,8 @@ package org.alfresco.repo.search.impl.lucene;
  * limitations under the License.
  */
 
+import java.util.Vector;
+
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
@@ -23,8 +25,6 @@ import org.apache.lucene.search.MultiPhraseQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
 
-import java.util.Vector;
-
 /**
  * A QueryParser which constructs queries to search multiple fields.
  *

source/java/org/alfresco/repo/search/impl/lucene/MultiReader.java

@@ -0,0 +1,93 @@
+/**
+ * 
+ */
+package org.alfresco.repo.search.impl.lucene;
+
+import java.io.IOException;
+import java.io.Reader;
+
+class MultiReader extends Reader
+{
+    Reader first;
+
+    Reader second;
+
+    boolean firstActive = true;
+
+    MultiReader(Reader first, Reader second)
+    {
+        this.first = first;
+        this.second = second;
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        IOException ioe = null;
+        try
+        {
+            first.close();
+        }
+        catch (IOException e)
+        {
+            ioe = e;
+        }
+
+        second.close();
+        if (ioe != null)
+        {
+            throw ioe;
+        }
+
+    }
+
+    @Override
+    public int read(char[] cbuf, int off, int len) throws IOException
+    {
+        synchronized (lock)
+        {
+            if ((off < 0) || (off > cbuf.length) || (len < 0) || ((off + len) > cbuf.length) || ((off + len) < 0))
+            {
+                throw new IndexOutOfBoundsException();
+            }
+            else if (len == 0)
+            {
+                return 0;
+            }
+            for(int i = 0; i < len; i++)
+            {
+                int c; 
+                if(firstActive)
+                {
+                    c = first.read();
+                    if(c == -1)
+                    {
+                        firstActive = false;
+                        c = second.read();
+                    }
+                }
+                else
+                {
+                    c = second.read();
+                }
+                if(c == -1)
+                {
+                    if(i == 0)
+                    {
+                        return -1; 
+                    }
+                    else
+                    {
+                        return i;
+                    }
+                }
+                else
+                {
+                    cbuf[off+i] = (char)c;
+                }
+            }
+            return len;
+        }
+    }
+
+}

source/java/org/alfresco/repo/search/impl/lucene/MultiReaderTest.java

@@ -0,0 +1,85 @@
+package org.alfresco.repo.search.impl.lucene;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+
+import junit.framework.TestCase;
+
+public class MultiReaderTest extends TestCase
+{
+
+    public MultiReaderTest()
+    {
+        super();
+    }
+
+    public MultiReaderTest(String arg0)
+    {
+        super(arg0);
+    }
+
+    public void testMultiReader_single() throws IOException
+    {
+        String first = "my first string";
+        String second = "another little string";
+
+        StringReader one = new StringReader(first);
+        StringReader two = new StringReader(second);
+
+        Reader multiReader = new MultiReader(one, two);
+        StringBuilder builder = new StringBuilder();
+        int c;
+        while ((c = multiReader.read()) != -1)
+        {
+            builder.append((char) c);
+        }
+        assertEquals(builder.toString(), first + second);
+
+    }
+
+    public void testMultiReader_bits() throws IOException
+    {
+        String first = "my first string";
+        String second = "another little string";
+
+        StringReader one = new StringReader(first);
+        StringReader two = new StringReader(second);
+
+        Reader multiReader = new MultiReader(one, two);
+        StringBuilder builder = new StringBuilder();
+        for (int chunk = 1; chunk < 100; chunk++)
+        {
+            char[] c = new char[chunk];
+            int i = 0;
+            while (i != -1)
+            {
+                i = multiReader.read(c);
+                for (int j = 0; j < i; j++)
+                {
+                    builder.append(c[j]);
+                }
+            }
+            assertEquals(builder.toString(), first + second);
+        }
+    }
+    
+    public void testSkip() throws IOException
+    {
+        String first = "my first string";
+        String second = "another little string";
+
+        StringReader one = new StringReader(first);
+        StringReader two = new StringReader(second);
+
+        Reader multiReader = new MultiReader(one, two);
+        
+        multiReader.skip(3);
+        String all = first + second;
+        assertEquals((char)multiReader.read(), all.charAt(3));
+        
+        multiReader.skip(15);
+        assertEquals((char)multiReader.read(), all.charAt(3+15+1));
+    }
+
+}

source/java/org/alfresco/repo/search/impl/lucene/QueryParser.java

@@ -1,14 +1,31 @@
 /* Generated By:JavaCC: Do not edit this line. QueryParser.java */
 package org.alfresco.repo.search.impl.lucene;
 
+import java.io.IOException;
+import java.io.StringReader;
+import java.text.DateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.Locale;
 import java.util.Vector;
-import java.io.*;
+
-import java.text.*;
+import org.apache.lucene.analysis.Analyzer;
-import java.util.*;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.document.DateField;
+import org.apache.lucene.document.DateTools;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.analysis.*;
+import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.document.*;
+import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.*;
+import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.MultiPhraseQuery;
+import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.PrefixQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.RangeQuery;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.util.Parameter;
 
 /**

source/java/org/alfresco/repo/search/impl/lucene/QueryParserTokenManager.java

@@ -1,14 +1,5 @@
 /* Generated By:JavaCC: Do not edit this line. QueryParserTokenManager.java */
 package org.alfresco.repo.search.impl.lucene;
-import java.util.Vector;
-import java.io.*;
-import java.text.*;
-import java.util.*;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.analysis.*;
-import org.apache.lucene.document.*;
-import org.apache.lucene.search.*;
-import org.apache.lucene.util.Parameter;
 
 public class QueryParserTokenManager implements QueryParserConstants
 {

source/java/org/alfresco/repo/search/impl/lucene/analysis/AlfrescoStandardFilter.java

@@ -18,7 +18,6 @@ package org.alfresco.repo.search.impl.lucene.analysis;
 
 import java.util.LinkedList;
 import java.util.Queue;
-import java.util.Stack;
 import java.util.StringTokenizer;
 
 import org.apache.lucene.analysis.TokenFilter;

source/java/org/alfresco/repo/search/impl/lucene/analysis/FloatTokenFilter.java

@@ -22,7 +22,6 @@ import java.io.Reader;
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.WhitespaceTokenizer;
-import org.apache.lucene.analysis.standard.StandardTokenizer;
 
 /**
  * Simple tokeniser for floats.

source/java/org/alfresco/repo/search/impl/lucene/analysis/IntegerTokenFilter.java

@@ -22,7 +22,6 @@ import java.io.Reader;
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.WhitespaceTokenizer;
-import org.apache.lucene.analysis.standard.StandardTokenizer;
 
 /**
  * Simple tokeniser for integers.

source/java/org/alfresco/repo/search/impl/lucene/analysis/LongTokenFilter.java

@@ -19,11 +19,9 @@ package org.alfresco.repo.search.impl.lucene.analysis;
 import java.io.IOException;
 import java.io.Reader;
 
-import org.alfresco.error.AlfrescoRuntimeException;
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.WhitespaceTokenizer;
-import org.apache.lucene.analysis.standard.StandardTokenizer;
 
 /**
  * Simple tokeniser for longs.

source/java/org/alfresco/repo/search/impl/lucene/analysis/LowerCaseVerbatimAnalyser.java

@@ -0,0 +1,9 @@
+package org.alfresco.repo.search.impl.lucene.analysis;
+
+public class LowerCaseVerbatimAnalyser extends VerbatimAnalyser
+{
+    public LowerCaseVerbatimAnalyser()
+    {
+        super(true);
+    }
+}

source/java/org/alfresco/repo/search/impl/lucene/analysis/MLAnalayser.java

@@ -7,7 +7,7 @@ import java.util.HashMap;
 import java.util.Locale;
 
 import org.alfresco.i18n.I18NUtil;
-import org.alfresco.repo.search.impl.lucene.LuceneQueryParser;
+import org.alfresco.repo.search.MLAnalysisMode;
 import org.alfresco.service.cmr.dictionary.DataTypeDefinition;
 import org.alfresco.service.cmr.dictionary.DictionaryService;
 import org.apache.log4j.Logger;
@@ -22,9 +22,12 @@ public class MLAnalayser extends Analyzer
 
     private HashMap<Locale, Analyzer> analysers = new HashMap<Locale, Analyzer>();
     
-    public MLAnalayser(DictionaryService dictionaryService)
+    private MLAnalysisMode mlAnalaysisMode;
+
+    public MLAnalayser(DictionaryService dictionaryService, MLAnalysisMode mlAnalaysisMode)
     {
         this.dictionaryService = dictionaryService;
+        this.mlAnalaysisMode = mlAnalaysisMode;
     }
 
     @Override
@@ -107,7 +110,7 @@ public class MLAnalayser extends Analyzer
                     }
                     Locale locale = new Locale(language, country, varient);
                     // leave the reader where it is ....
-                    return new MLTokenDuplicator(getAnalyser(locale).tokenStream(fieldName, breader), locale, breader);
+                    return new MLTokenDuplicator(getAnalyser(locale).tokenStream(fieldName, breader), locale, breader, mlAnalaysisMode);
                 }
                 else
                 {

source/java/org/alfresco/repo/search/impl/lucene/analysis/MLTokenDuplicator.java

@@ -3,39 +3,36 @@ package org.alfresco.repo.search.impl.lucene.analysis;
 import java.io.IOException;
 import java.io.Reader;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Locale;
 
+import org.alfresco.repo.search.MLAnalysisMode;
+import org.apache.log4j.Logger;
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 
 /**
- * Create duplicate tokens for multilingual varients
+ * Create duplicate tokens for multilingual varients The forms are Tokens: Token - all languages {fr}Token - if a
- * 
+ * language is specified {fr_CA}Token - if a language and country is specified {fr_CA_Varient}Token - for all three
- * The forms are 
- * 
- * Tokens:
- * Token - all languages
- * {fr}Token - if a language is specified 
- * {fr_CA}Token - if a language and country is specified
- * {fr_CA_Varient}Token - for all three
  * {fr__Varient}Token - for a language varient with no country
  * 
  * @author andyh
- *
  */
 public class MLTokenDuplicator extends Tokenizer
 {
+    private static Logger s_logger = Logger.getLogger(MLTokenDuplicator.class);
+    
     TokenStream source;
 
     Locale locale;
 
     Iterator<Token> it;
 
-    ArrayList<String> prefixes;
+    HashSet<String> prefixes;
 
-    public MLTokenDuplicator(TokenStream source, Locale locale, Reader reader)
+    public MLTokenDuplicator(TokenStream source, Locale locale, Reader reader, MLAnalysisMode mlAnalaysisMode)
     {
         super(reader);
         this.source = source;
@@ -45,27 +42,92 @@ public class MLTokenDuplicator extends Tokenizer
         boolean c = locale.getCountry().length() != 0;
         boolean v = locale.getVariant().length() != 0;
 
-        prefixes = new ArrayList<String>(4);
+        prefixes = new HashSet<String>(4);
+        if (mlAnalaysisMode.includesAll())
+        {
             prefixes.add("");
+        }
 
-        if (l)
+        if (mlAnalaysisMode.includesExact())
         {
             StringBuffer result = new StringBuffer();
-            result.append("{").append(locale.getLanguage()).append("}");
+            result.append("{").append(locale.toString()).append("}");
             prefixes.add(result.toString());
-            result.deleteCharAt(result.length()-1);
+        }
 
-            if (c || (l && v))
+        if (mlAnalaysisMode.includesContaining())
         {
-                result.append('_').append(locale.getCountry()).append("}"); 
+            if (v)
+            {
+                Locale noVarient = new Locale(locale.getLanguage(), locale.getCountry(), "");
+                StringBuffer result = new StringBuffer();
+                result.append("{").append(noVarient.toString()).append("}");
                 prefixes.add(result.toString());
-                result.deleteCharAt(result.length()-1);
+
-            }
+                Locale noCountry = new Locale(locale.getLanguage(), "", "");
-            if (v && (l || c))
+                result = new StringBuffer();
-            {
+                result.append("{").append(noCountry.toString()).append("}");
-                result.append('_').append(locale.getVariant()).append("}");
                 prefixes.add(result.toString());
             }
+            if (c)
+            {
+                Locale noCountry = new Locale(locale.getLanguage(), "", "");
+                StringBuffer result = new StringBuffer();
+                result.append("{").append(noCountry.toString()).append("}");
+                prefixes.add(result.toString());
+            }
+        }
+
+        if (mlAnalaysisMode.includesContained())
+        {
+            // varients have not contained
+            if (!v)
+            {
+                if (!c)
+                {
+                    if (!l)
+                    {
+                        // All
+                        for (Locale toAdd : Locale.getAvailableLocales())
+                        {
+                            StringBuffer result = new StringBuffer();
+                            result.append("{").append(toAdd.toString()).append("}");
+                            prefixes.add(result.toString());
+                        }
+                    }
+                    else
+                    {
+                        // All that match language
+                        for (Locale toAdd : Locale.getAvailableLocales())
+                        {
+                            if (locale.getLanguage().equals(toAdd.getLanguage()))
+                            {
+                                StringBuffer result = new StringBuffer();
+                                result.append("{").append(toAdd.toString()).append("}");
+                                prefixes.add(result.toString());
+                            }
+                        }
+                    }
+                }
+                else
+                {
+                    // All that match language and country
+                    for (Locale toAdd : Locale.getAvailableLocales())
+                    {
+                        if ((locale.getLanguage().equals(toAdd.getLanguage()))
+                                && (locale.getCountry().equals(toAdd.getCountry())))
+                        {
+                            StringBuffer result = new StringBuffer();
+                            result.append("{").append(toAdd.toString()).append("}");
+                            prefixes.add(result.toString());
+                        }
+                    }
+                }
+            }
+        }
+        if(s_logger.isDebugEnabled())
+        {
+            s_logger.debug("Locale "+ locale +" using "+mlAnalaysisMode+" is "+prefixes);
         }
 
     }
@@ -118,5 +180,4 @@ public class MLTokenDuplicator extends Tokenizer
 
     }
 
-    
 }

source/java/org/alfresco/repo/search/impl/lucene/analysis/PathTokenFilterTest.java

@@ -19,11 +19,11 @@ package org.alfresco.repo.search.impl.lucene.analysis;
 import java.io.IOException;
 import java.io.StringReader;
 
+import junit.framework.TestCase;
+
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
 
-import junit.framework.TestCase;
-
 public class PathTokenFilterTest extends TestCase
 {
 

source/java/org/alfresco/repo/search/impl/lucene/analysis/VerbatimAnalyser.java

@@ -5,18 +5,23 @@ import java.io.Reader;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 
-public class VerbatimAnalyser
+public class VerbatimAnalyser extends Analyzer
-
-extends Analyzer
 {
+    boolean lowerCase;
     
     public VerbatimAnalyser()
+    {
+        lowerCase = false;
+    }
+    
+    public VerbatimAnalyser(boolean lowerCase)
     {
         super();
+        this.lowerCase = lowerCase;
     }
 
     public TokenStream tokenStream(String fieldName, Reader reader)
     {
-        return new VerbatimTokenFilter(reader);
+        return new VerbatimTokenFilter(reader, lowerCase);
     }
 }

source/java/org/alfresco/repo/search/impl/lucene/analysis/VerbatimTokenFilter.java

@@ -10,9 +10,12 @@ public class VerbatimTokenFilter extends Tokenizer
 {
     boolean readInput = true;
 
-    VerbatimTokenFilter(Reader in)
+    boolean lowerCase;
+    
+    VerbatimTokenFilter(Reader in, boolean lowerCase)
     {
         super(in);
+        this.lowerCase = lowerCase;
     }
 
     @Override
@@ -31,6 +34,10 @@ public class VerbatimTokenFilter extends Tokenizer
             }
 
             String token = buffer.toString();
+            if(lowerCase)
+            {
+                token = token.toLowerCase();
+            }
             return new Token(token, 0, token.length() - 1, "VERBATIM");
         }
         else

source/java/org/alfresco/service/cmr/search/SearchParameters.java

@@ -18,8 +18,10 @@ package org.alfresco.service.cmr.search;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Set;
 
 import org.alfresco.repo.search.MLAnalysisMode;
 import org.alfresco.service.cmr.repository.Path;
@@ -95,6 +97,10 @@ public class SearchParameters extends SearchStatement
 
     private int limit = DEFAULT_LIMIT;
     
+    private HashSet<String> allAttributes = new HashSet<String>();
+    
+    private HashSet<String> textAttributes = new HashSet<String>();
+
     /**
      * Default constructor
      */
@@ -352,6 +358,52 @@ public class SearchParameters extends SearchStatement
         return Collections.unmodifiableList(locales);
     }
     
+    
+    
+
+    /**
+     * Add a locale to include for multi-lingual text searches.
+     * If non are set, the default is to use the user's locale.
+     *  
+     * @param locale
+     */
+    public void addTextAttribute(String attribute)
+    {
+        textAttributes.add(attribute);
+    }
+
+    /**
+     * Get the locales used for multi-lingual text searches.
+     * 
+     * @return
+     */
+    public Set<String> getTextAttributes()
+    {
+        return Collections.unmodifiableSet(textAttributes);
+    }
+    
+    /**
+     * Add a locale to include for multi-lingual text searches.
+     * If non are set, the default is to use the user's locale.
+     *  
+     * @param locale
+     */
+    public void addAllAttribute(String attribute)
+    {
+        allAttributes.add(attribute);
+    }
+
+    /**
+     * Get the locales used for multi-lingual text searches.
+     * 
+     * @return
+     */
+    public Set<String> getAllAttributes()
+    {
+        return Collections.unmodifiableSet(allAttributes);
+    }
+    
+
     /**
      * A helper class for sort definition. Encapsulated using the lucene sortType, field name and a flag for
      * ascending/descending.