Merged V2.1 to HEAD

6361: Word 2007 Add-In and Installer - First version, untested
   6362: Adds a scaling test that a number of customers could [use]
   6363: Build fix
   6365: Submit workflow to always clean up virtualized workflow, etc
   6367: Support for wild cards within queries AR-615
            Conflicts with newer Tenant code
            Resolved conflicted state of 'root\projects\repository\source\java\org\alfresco\repo\search\impl\lucene\ADMLuceneSearcherImpl.java'
            Resolved conflicted state of 'root\projects\repository\source\java\org\alfresco\repo\search\impl\lucene\ADMLuceneTest.java'
            Resolved conflicted state of 'root\projects\repository\source\java\org\alfresco\repo\search\impl\lucene\LuceneQueryParser.java'


git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@6724 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

config/alfresco/rule-services-context.xml

@@ -60,7 +60,7 @@
 	     <ref bean="ruleService"/>
 	  </property>  	
 	  <property name="nodeService">
-		 <ref bean="NodeService"/>
+		 <ref bean="nodeService"/>
 	  </property>					
    </bean>
    

config/alfresco/workflow/submit_processdefinition.xml

@@ -234,7 +234,8 @@
     
     <event type="process-end">
         <action class="org.alfresco.repo.avm.wf.AVMClearSubmittedHandler"/>
+        <action class="org.alfresco.repo.avm.wf.AVMRemoveAllWebappsHandler"/>
         <action class="org.alfresco.repo.avm.wf.AVMRemoveWFStoreHandler"/>
     </event>
 
-</process-definition>
+</process-definition>

source/java/org/alfresco/repo/avm/AVMScaleTestP.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2005-2007 Alfresco Software Limited.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+ * As a special exception to the terms and conditions of version 2.0 of 
+ * the GPL, you may redistribute this Program in connection with Free/Libre 
+ * and Open Source Software ("FLOSS") applications as described in Alfresco's 
+ * FLOSS exception.  You should have recieved a copy of the text describing 
+ * the FLOSS exception, and it is also available here: 
+ * http://www.alfresco.com/legal/licensing
+ */
+
+package org.alfresco.repo.avm;
+
+import org.alfresco.repo.avm.util.BulkLoader;
+import org.alfresco.repo.avm.util.BulkReader;
+
+/**
+ * Test of scaling out to large numbers of files.
+ * @author britt
+ */
+public class AVMScaleTestP extends AVMServiceTestBase
+{
+    public void testScaling()
+    {
+        int n = 4; // The number of BulkLoads to do.
+        int futzCount = 10; // The number of post snapshot modifications to make after each load.
+        String load = "/Users/britt/hibernate-3.1"; // The tree of stuff to load.
+        BulkLoader loader = new BulkLoader();
+        loader.setAvmService(fService);
+        loader.setPropertyCount(5);
+        BulkReader reader = new BulkReader();
+        reader.setAvmService(fService);
+        long lastTime = System.currentTimeMillis();
+        for (int i = 0; i < n; i++)
+        {
+            System.out.println("Round " + (i + 1));
+            fService.createStore("store" + i);
+            loader.recursiveLoad(load, "store" + i + ":/");
+            fService.createSnapshot("store" + i, null, null);
+            long now = System.currentTimeMillis();
+            System.out.println("Load Time: " + (now - lastTime) + "ms");
+            lastTime = now;
+            reader.recursiveFutz("store" + i, "store" + i + ":/", futzCount);
+            now = System.currentTimeMillis();
+            System.out.println("Read Time: " + (now - lastTime) + "ms");
+            System.out.flush();
+            lastTime = now;
+        }
+    }
+}

source/java/org/alfresco/repo/avm/util/BulkLoader.java

@@ -28,9 +28,13 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
 
+import org.alfresco.repo.domain.PropertyValue;
 import org.alfresco.service.cmr.avm.AVMException;
 import org.alfresco.service.cmr.avm.AVMService;
+import org.alfresco.service.namespace.QName;
 
 /**
  * This takes a filesystem directory path and a repository path and name
@@ -41,6 +45,8 @@ public class BulkLoader
 {
     private AVMService fService;
     
+    private int fPropertyCount = 0;
+    
     /**
      * Create a new one.
      */
@@ -57,6 +63,11 @@ public class BulkLoader
         fService = service;
     }
     
+    public void setPropertyCount(int propCount)
+    {
+        fPropertyCount = propCount;
+    }
+    
     /**
      * Recursively load content.
      * @param fsPath The path in the filesystem.
@@ -64,6 +75,11 @@ public class BulkLoader
      */
     public void recursiveLoad(String fsPath, String repPath)
     {
+        Map<QName, PropertyValue> props = new HashMap<QName, PropertyValue>();
+        for (int i = 0; i < fPropertyCount; i++)
+        {
+            props.put(QName.createQName("silly", "prop" + i), new PropertyValue(null, "I am property " + i));
+        }
         File file = new File(fsPath);
         String name = file.getName();
         if (file.isDirectory())
@@ -71,6 +87,7 @@ public class BulkLoader
             fService.createDirectory(repPath, name);
             String[] children = file.list();
             String baseName = repPath.endsWith("/") ? repPath + name : repPath + "/" + name;
+            fService.setNodeProperties(baseName, props);
             for (String child : children)
             {
                 recursiveLoad(fsPath + "/" + child, baseName);
@@ -82,6 +99,7 @@ public class BulkLoader
             {
                 InputStream in = new FileInputStream(file);
                 OutputStream out = fService.createFile(repPath, name);
+                fService.setNodeProperties(repPath + "/" + name, props);
                 byte[] buff = new byte[8192];
                 int read = 0;
                 while ((read = in.read(buff)) != -1)

source/java/org/alfresco/repo/avm/util/BulkReader.java

@@ -0,0 +1,106 @@
+/*
+ * Copyright (C) 2005-2007 Alfresco Software Limited.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+ * As a special exception to the terms and conditions of version 2.0 of 
+ * the GPL, you may redistribute this Program in connection with Free/Libre 
+ * and Open Source Software ("FLOSS") applications as described in Alfresco's 
+ * FLOSS exception.  You should have recieved a copy of the text describing 
+ * the FLOSS exception, and it is also available here: 
+ * http://www.alfresco.com/legal/licensing
+ */
+
+package org.alfresco.repo.avm.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.alfresco.service.cmr.avm.AVMNodeDescriptor;
+import org.alfresco.service.cmr.avm.AVMService;
+
+/**
+ * Testing utility class. Reads a tree recursively.
+ * @author britt
+ */
+public class BulkReader
+{
+    private AVMService fService;
+    
+    public BulkReader()
+    {
+    }
+    
+    public void setAvmService(AVMService service)
+    {
+        fService = service;
+    }
+    
+    public void recursiveFutz(String store, String path, int futz)
+    {
+        List<String> paths = new ArrayList<String>();
+        recursiveRead(path, paths);
+        Random random = new Random(System.currentTimeMillis());
+        int futzed = 0;
+        while (futzed < futz)
+        {
+            String futzPath = paths.get(random.nextInt(paths.size()));
+            AVMNodeDescriptor desc = fService.lookup(-1, futzPath);
+            if (desc.isFile())
+            {
+                try
+                {
+                    fService.getFileOutputStream(futzPath).close();
+                }
+                catch (IOException e)
+                {
+                    // Do nothing.
+                }
+                futzed++;
+            }
+        }
+        fService.createSnapshot(store, null, null);
+    }
+    
+    public void recursiveRead(String path, List<String> paths)
+    {
+        AVMNodeDescriptor desc = fService.lookup(-1, path);
+        paths.add(desc.getPath());
+        if (desc.isFile())
+        {
+            InputStream in = fService.getFileInputStream(desc);
+            try
+            {
+                byte[] buff = new byte[8192];
+                while (in.read(buff) != -1);
+                in.close();
+            }
+            catch (IOException e)
+            {
+                e.printStackTrace();
+            }
+            return;
+        }
+        Map<String, AVMNodeDescriptor> listing = fService.getDirectoryListing(desc);
+        for (Map.Entry<String, AVMNodeDescriptor> entry : listing.entrySet())
+        {
+            recursiveRead(entry.getValue().getPath(), paths);
+        }
+    }
+}

source/java/org/alfresco/repo/avm/wf/AVMRemoveAllWebappsHandler.java

@@ -0,0 +1,112 @@
+/*-----------------------------------------------------------------------------
+*  Copyright 2007 Alfresco Inc.
+*  
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License as published by
+*  the Free Software Foundation; either version 2 of the License, or
+*  (at your option) any later version.
+*  
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+*  or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+*  for more details.
+*  
+*  You should have received a copy of the GNU General Public License along
+*  with this program; if not, write to the Free Software Foundation, Inc.,
+*  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.  As a special
+*  exception to the terms and conditions of version 2.0 of the GPL, you may
+*  redistribute this Program in connection with Free/Libre and Open Source
+*  Software ("FLOSS") applications as described in Alfresco's FLOSS exception.
+*  You should have received a copy of the text describing the FLOSS exception,
+*  and it is also available here:   http://www.alfresco.com/legal/licensing
+*  
+*  
+*  Author  Jon Cox  <jcox@alfresco.com>
+*  File    AVMRemoveAllWebappsHandler.java
+*----------------------------------------------------------------------------*/
+
+package org.alfresco.repo.avm.wf;
+
+import java.util.Map;
+import org.alfresco.config.JNDIConstants;
+import org.alfresco.mbeans.VirtServerRegistry;
+import org.alfresco.repo.avm.AVMNodeConverter;
+import org.alfresco.repo.avm.util.RawServices;
+import org.alfresco.repo.domain.PropertyValue;
+import org.alfresco.repo.workflow.jbpm.JBPMNode;
+import org.alfresco.repo.workflow.jbpm.JBPMSpringActionHandler;
+import org.alfresco.service.cmr.avm.AVMService;
+import org.alfresco.service.cmr.repository.NodeRef;
+import org.alfresco.service.namespace.QName;
+import org.alfresco.util.Pair;
+import org.apache.log4j.Logger;
+import org.jbpm.graph.exe.ExecutionContext;
+import org.springframework.beans.factory.BeanFactory;
+import org.springframework.context.ApplicationContext;
+
+/**
+ * Remove all webapps in a sandbox
+ * 
+ * @author Jon Cox
+ */
+public class AVMRemoveAllWebappsHandler extends JBPMSpringActionHandler 
+{
+    static final long serialVersionUID = 3004374776252613278L;
+
+    private static Logger log = 
+        Logger.getLogger(AVMRemoveAllWebappsHandler.class);
+
+    /**
+     * The AVMService instance.
+     */
+    private AVMService fAVMService;    
+
+    
+    /**
+     * Initialize service references.
+     * @param factory The BeanFactory to get references from.
+     */
+    @Override
+    protected void initialiseHandler(BeanFactory factory) 
+    {
+        fAVMService = (AVMService)factory.getBean("AVMService");
+    }
+
+    /**
+     * Do the actual work.
+     * @param executionContext The context to get stuff from.
+     */
+    public void execute(ExecutionContext executionContext) throws Exception 
+    {
+        if (log.isDebugEnabled())
+            log.debug("AVMRemoveAllWebappsHandler.execute()");
+
+        // retrieve submitted package
+        NodeRef pkg = ((JBPMNode)executionContext.getContextInstance().
+                                 getVariable("bpm_package")).getNodeRef();
+
+        Pair<Integer, String> pkgPath = AVMNodeConverter.ToAVMVersionPath(pkg);
+
+        Integer version     =  pkgPath.getFirst();
+        String  www_dir     =  pkgPath.getSecond();
+        String  appbase_dir =  www_dir + "/" + JNDIConstants.DIR_DEFAULT_APPBASE;
+
+        if (log.isDebugEnabled())
+        {
+            log.debug("version:     " +  version );
+            log.debug("appbase_dir: " +  appbase_dir );
+        }
+
+        ApplicationContext springContext   = RawServices.Instance().getContext();
+        VirtServerRegistry vServerRegistry = (VirtServerRegistry) 
+                                             springContext.getBean("VirtServerRegistry");
+
+        if (log.isDebugEnabled())
+            log.debug("Sending JMX message to shut down workflow webapps");
+
+        vServerRegistry.removeAllWebapps( version,  appbase_dir, true );
+
+        if (log.isDebugEnabled())
+            log.debug("Sent JMX message to shut down workflow webapps");
+    }
+}

source/java/org/alfresco/repo/avm/wf/AVMSubmitTransactionListener.java

@@ -36,6 +36,7 @@ import org.alfresco.repo.transaction.TransactionListenerAdapter;
 import org.alfresco.service.cmr.avmsync.AVMDifference;
 import org.alfresco.util.VirtServerUtils;
 import org.springframework.context.ApplicationContext;
+import org.apache.log4j.Logger;
 
 /**
 *  Gets callbacks at critical moments within a transaction
@@ -44,8 +45,10 @@ import org.springframework.context.ApplicationContext;
 */
 public class AVMSubmitTransactionListener extends TransactionListenerAdapter
 {
+    private static Logger log = 
+        Logger.getLogger(AVMSubmitTransactionListener.class);
+
     public AVMSubmitTransactionListener() { }
-   
 
     /**
     *  Notify virtualization server that webapps in workflow sandbox
@@ -104,22 +107,31 @@ public class AVMSubmitTransactionListener extends TransactionListenerAdapter
                                               requiresUpdate.getDestinationPath(),
                                               true
                                              );
+            if (log.isDebugEnabled())
+                log.debug("JMX update to virt server called after commit");
         }
 
-        // Remove virtual weapps from workflow sandbox
+        // Remove virtual webapps from workflow sandbox prior to 
-
+        // AVMRemoveWFStoreHandler in the "process-end" clause.
-        if ( ! stagingDiffs.isEmpty() )
+        // This way, even if the workflow is aborted, the JMX message
-        {
+        // to the virt server is still sent.  Therefore, no longer 
-            // All the files are from the same workflow sandbox;
+        // doing this here:
-            // so to remove all the webapps, you just need to
+        //
-            // look at the 1st difference
+        //  if ( ! stagingDiffs.isEmpty() )
-
+        //  {
-            AVMDifference d = stagingDiffs.iterator().next();
+        //      // All the files are from the same workflow sandbox;
-            vServerRegistry.removeAllWebapps( d.getSourceVersion(), d.getSourcePath(), true );
+        //      // so to remove all the webapps, you just need to
-        }
+        //      // look at the 1st difference
-
+        //
+        //      AVMDifference d = stagingDiffs.iterator().next();
+        //      vServerRegistry.removeAllWebapps( d.getSourceVersion(), 
+        //                                        d.getSourcePath(), true );
+        //  }
 
         AlfrescoTransactionSupport.unbindResource("staging_diffs");
+
+        if (log.isDebugEnabled())
+            log.debug("staging_diff resource unbound after commit");
     }
 
 
@@ -130,5 +142,8 @@ public class AVMSubmitTransactionListener extends TransactionListenerAdapter
     public void afterRollback()
     {
         AlfrescoTransactionSupport.unbindResource("staging_diffs");
+
+        if (log.isDebugEnabled())
+            log.debug("staging_diff resource unbound after rollback");
     }
 }

source/java/org/alfresco/repo/search/MLAnalysisMode.java

@@ -1,3 +1,27 @@
+/*
+ * Copyright (C) 2005-2007 Alfresco Software Limited.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+ * As a special exception to the terms and conditions of version 2.0 of 
+ * the GPL, you may redistribute this Program in connection with Free/Libre 
+ * and Open Source Software ("FLOSS") applications as described in Alfresco's 
+ * FLOSS exception.  You should have recieved a copy of the text describing 
+ * the FLOSS exception, and it is also available here: 
+ * http://www.alfresco.com/legal/licensing"
+ */
 package org.alfresco.repo.search;
 
 import java.util.Collection;

source/java/org/alfresco/repo/search/impl/lucene/ADMLuceneSearcherImpl.java

@@ -58,6 +58,7 @@ import org.alfresco.service.namespace.NamespacePrefixResolver;
 import org.alfresco.service.namespace.QName;
 import org.alfresco.util.ISO9075;
 import org.alfresco.util.SearchLanguageConversion;
+import org.apache.log4j.Logger;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermEnum;
@@ -71,12 +72,14 @@ import org.saxpath.SAXPathException;
 import com.werken.saxpath.XPathReader;
 
 /**
- * The Lucene implementation of Searcher At the moment we support only lucene based queries. TODO: Support for other query languages
+ * The Lucene implementation of Searcher At the moment we support only lucene based queries. TODO: Support for other
+ * query languages
  * 
  * @author andyh
  */
 public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneSearcher
 {
+    static Logger s_logger = Logger.getLogger(ADMLuceneSearcherImpl.class);
 
     /**
      * Default field name
@@ -100,11 +103,12 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
      */
 
     /**
-     * Get an initialised searcher for the store and transaction Normally we do not search against a a store and delta. Currently only gets the searcher against the main index.
+     * Get an initialised searcher for the store and transaction Normally we do not search against a a store and delta.
+     * Currently only gets the searcher against the main index.
      * 
      * @param storeRef
-     * @param indexer 
+     * @param indexer
-     * @param config 
+     * @param config
      * @return - the searcher implementation
      */
     public static ADMLuceneSearcherImpl getSearcher(StoreRef storeRef, LuceneIndexer indexer, LuceneConfig config)
@@ -127,7 +131,7 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
      * Get an intialised searcher for the store. No transactional ammendsmends are searched.
      * 
      * @param storeRef
-     * @param config 
+     * @param config
      * @return the searcher
      */
     public static ADMLuceneSearcherImpl getSearcher(StoreRef storeRef, LuceneConfig config)
@@ -163,6 +167,7 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
 
     /**
      * Set the query register
+     * 
      * @param queryRegister
      */
     public void setQueryRegister(QueryRegisterComponent queryRegister)
@@ -170,8 +175,7 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
         this.queryRegister = queryRegister;
     }
 
-    public ResultSet query(StoreRef store, String language, String queryString, Path[] queryOptions,
+    public ResultSet query(StoreRef store, String language, String queryString, Path[] queryOptions, QueryParameterDefinition[] queryParameterDefinitions) throws SearcherException
-            QueryParameterDefinition[] queryParameterDefinitions) throws SearcherException
     {
     	store = tenantService.getName(store);
         
@@ -240,11 +244,23 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
                     defaultOperator = LuceneQueryParser.OR_OPERATOR;
                 }
 
-                Query query = LuceneQueryParser.parse(parameterisedQueryString, DEFAULT_FIELD, 
-                        new LuceneAnalyser(dictionaryService, searchParameters.getMlAnalaysisMode() == null ? getLuceneConfig().getDefaultMLSearchAnalysisMode() : searchParameters.getMlAnalaysisMode()), 
-                        namespacePrefixResolver, dictionaryService, tenantService, defaultOperator, searchParameters, getLuceneConfig());
-                
                 ClosingIndexSearcher searcher = getSearcher(indexer);
+                Query query = LuceneQueryParser.parse(
+                        parameterisedQueryString, DEFAULT_FIELD,
+                        new LuceneAnalyser(
+                                dictionaryService,
+                                searchParameters.getMlAnalaysisMode() == null ? getLuceneConfig().getDefaultMLSearchAnalysisMode() : searchParameters.getMlAnalaysisMode()),
+                        namespacePrefixResolver,
+                        dictionaryService,
+                        tenantService,
+                        defaultOperator,
+                        searchParameters,
+                        getLuceneConfig(),
+                        searcher.getIndexReader());
+                if (s_logger.isDebugEnabled())
+                {
+                    s_logger.debug("Query is " + query.toString());
+                }
                 if (searcher == null)
                 {
                     // no index return an empty result set
@@ -288,6 +304,7 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
                 }
 
                 Path[] paths = searchParameters.getAttributePaths().toArray(new Path[0]);
+<<<<<<< .working
                 return new LuceneResultSet(
                         hits,
                         searcher,
@@ -295,6 +312,9 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
                         tenantService,
                         paths,
                         searchParameters);
+=======
+                return new LuceneResultSet(hits, searcher, nodeService, paths, searchParameters);
+>>>>>>> .merge-right.r6367
 
             }
             catch (ParseException e)
@@ -328,8 +348,12 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
                     return new EmptyResultSet();
                 }
                 Hits hits = searcher.search(query);
+<<<<<<< .working
                 return new LuceneResultSet(hits, searcher, nodeService, tenantService, searchParameters.getAttributePaths().toArray(
                         new Path[0]), searchParameters);
+=======
+                return new LuceneResultSet(hits, searcher, nodeService, searchParameters.getAttributePaths().toArray(new Path[0]), searchParameters);
+>>>>>>> .merge-right.r6367
             }
             catch (SAXPathException e)
             {
@@ -380,8 +404,7 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
         return query(store, language, query, null, null);
     }
 
-    public ResultSet query(StoreRef store, String language, String query,
+    public ResultSet query(StoreRef store, String language, String query, QueryParameterDefinition[] queryParameterDefintions)
-            QueryParameterDefinition[] queryParameterDefintions)
     {
         return query(store, language, query, null, queryParameterDefintions);
     }
@@ -407,8 +430,7 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
 
         checkParameters(definition, queryParameters);
 
-        String queryString = parameterise(definition.getQuery(), definition.getQueryParameterMap(), queryParameters,
+        String queryString = parameterise(definition.getQuery(), definition.getQueryParameterMap(), queryParameters, definition.getNamespacePrefixResolver());
-                definition.getNamespacePrefixResolver());
 
         return query(store, definition.getLanguage(), queryString, null, null);
     }
@@ -420,8 +442,7 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
      * @param queryParameters
      * @throws QueryParameterisationException
      */
-    private void checkParameters(CannedQueryDef definition, QueryParameter[] queryParameters)
+    private void checkParameters(CannedQueryDef definition, QueryParameter[] queryParameters) throws QueryParameterisationException
-            throws QueryParameterisationException
     {
         List<QName> missing = new ArrayList<QName>();
 
@@ -461,12 +482,13 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
     }
 
     /*
-     * Parameterise the query string - not sure if it is required to escape lucence spacials chars The parameters could be used to build the query - the contents of parameters
+     * Parameterise the query string - not sure if it is required to escape lucence spacials chars The parameters could
-     * should alread have been escaped if required. ... mush better to provide the parameters and work out what to do TODO: conditional query escapement - may be we should have a
+     * be used to build the query - the contents of parameters should alread have been escaped if required. ... mush
-     * parameter type that is not escaped
+     * better to provide the parameters and work out what to do TODO: conditional query escapement - may be we should
+     * have a parameter type that is not escaped
      */
-    private String parameterise(String unparameterised, Map<QName, QueryParameterDefinition> map,
+    private String parameterise(String unparameterised, Map<QName, QueryParameterDefinition> map, QueryParameter[] queryParameters, NamespacePrefixResolver nspr)
-            QueryParameter[] queryParameters, NamespacePrefixResolver nspr) throws QueryParameterisationException
+            throws QueryParameterisationException
     {
 
         Map<QName, List<Serializable>> valueMap = new HashMap<QName, List<Serializable>>();
@@ -547,28 +569,29 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
     /**
      * @see org.alfresco.repo.search.impl.NodeSearcher
      */
-    public List<NodeRef> selectNodes(NodeRef contextNodeRef, String xpath, QueryParameterDefinition[] parameters,
+    public List<NodeRef> selectNodes(NodeRef contextNodeRef, String xpath, QueryParameterDefinition[] parameters, NamespacePrefixResolver namespacePrefixResolver,
-            NamespacePrefixResolver namespacePrefixResolver, boolean followAllParentLinks, String language)
+            boolean followAllParentLinks, String language) throws InvalidNodeRefException, XPathException
-            throws InvalidNodeRefException, XPathException
     {
         NodeSearcher nodeSearcher = new NodeSearcher(nodeService, dictionaryService, this);
+<<<<<<< .working
         
         contextNodeRef = tenantService.getName(contextNodeRef);
         
         return nodeSearcher.selectNodes(contextNodeRef, xpath, parameters, namespacePrefixResolver,
                 followAllParentLinks, language);
+=======
+        return nodeSearcher.selectNodes(contextNodeRef, xpath, parameters, namespacePrefixResolver, followAllParentLinks, language);
+>>>>>>> .merge-right.r6367
     }
 
     /**
      * @see org.alfresco.repo.search.impl.NodeSearcher
      */
-    public List<Serializable> selectProperties(NodeRef contextNodeRef, String xpath,
+    public List<Serializable> selectProperties(NodeRef contextNodeRef, String xpath, QueryParameterDefinition[] parameters, NamespacePrefixResolver namespacePrefixResolver,
-            QueryParameterDefinition[] parameters, NamespacePrefixResolver namespacePrefixResolver,
             boolean followAllParentLinks, String language) throws InvalidNodeRefException, XPathException
     {
         NodeSearcher nodeSearcher = new NodeSearcher(nodeService, dictionaryService, this);
-        return nodeSearcher.selectProperties(contextNodeRef, xpath, parameters, namespacePrefixResolver,
+        return nodeSearcher.selectProperties(contextNodeRef, xpath, parameters, namespacePrefixResolver, followAllParentLinks, language);
-                followAllParentLinks, language);
     }
 
     /**
@@ -582,30 +605,24 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
     /**
      * @return Returns true if the pattern is present, otherwise false.
      */
-    public boolean contains(NodeRef nodeRef, QName propertyQName, String googleLikePattern,
+    public boolean contains(NodeRef nodeRef, QName propertyQName, String googleLikePattern, SearchParameters.Operator defaultOperator)
-            SearchParameters.Operator defaultOperator)
     {
         ResultSet resultSet = null;
         try
         {
             // build Lucene search string specific to the node
             StringBuilder sb = new StringBuilder();
-            sb.append("+ID:\"").append(nodeRef.toString()).append("\" +(TEXT:(")
+            sb.append("+ID:\"").append(nodeRef.toString()).append("\" +(TEXT:(").append(googleLikePattern.toLowerCase()).append(") ");
-                    .append(googleLikePattern.toLowerCase()).append(") ");
             if (propertyQName != null)
             {
-                sb.append(" OR @").append(
+                sb.append(" OR @").append(LuceneQueryParser.escape(QName.createQName(propertyQName.getNamespaceURI(), ISO9075.encode(propertyQName.getLocalName())).toString()));
-                        LuceneQueryParser.escape(QName.createQName(propertyQName.getNamespaceURI(),
-                                ISO9075.encode(propertyQName.getLocalName())).toString()));
                 sb.append(":(").append(googleLikePattern.toLowerCase()).append(")");
             }
             else
             {
                 for (QName key : nodeService.getProperties(nodeRef).keySet())
                 {
-                    sb.append(" OR @").append(
+                    sb.append(" OR @").append(LuceneQueryParser.escape(QName.createQName(key.getNamespaceURI(), ISO9075.encode(key.getLocalName())).toString()));
-                            LuceneQueryParser.escape(QName.createQName(key.getNamespaceURI(),
-                                    ISO9075.encode(key.getLocalName())).toString()));
                     sb.append(":(").append(googleLikePattern.toLowerCase()).append(")");
                 }
             }
@@ -657,10 +674,8 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
             }
             if (propertyQName != null)
             {
-                sb.append(" @").append(
+                sb.append(" @").append(LuceneQueryParser.escape(QName.createQName(propertyQName.getNamespaceURI(), ISO9075.encode(propertyQName.getLocalName())).toString()))
-                        LuceneQueryParser.escape(QName.createQName(propertyQName.getNamespaceURI(),
+                        .append(":(").append(pattern).append(")");
-                                ISO9075.encode(propertyQName.getLocalName())).toString())).append(":(").append(pattern)
-                        .append(")");
             }
             sb.append(")");
 
@@ -691,26 +706,21 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
             }
             else
             {
-                String propertyString = DefaultTypeConverter.INSTANCE.convert(String.class, nodeService.getProperty(
+                String propertyString = DefaultTypeConverter.INSTANCE.convert(String.class, nodeService.getProperty(nodeRef, propertyQName));
-                        nodeRef, propertyQName));
                 return propertyString.toLowerCase().matches(pattern);
             }
         }
     }
 
-    public List<NodeRef> selectNodes(NodeRef contextNodeRef, String xpath, QueryParameterDefinition[] parameters,
+    public List<NodeRef> selectNodes(NodeRef contextNodeRef, String xpath, QueryParameterDefinition[] parameters, NamespacePrefixResolver namespacePrefixResolver,
-            NamespacePrefixResolver namespacePrefixResolver, boolean followAllParentLinks)
-            throws InvalidNodeRefException, XPathException
-    {
-        return selectNodes(contextNodeRef, xpath, parameters, namespacePrefixResolver, followAllParentLinks,
-                SearchService.LANGUAGE_XPATH);
-    }
-
-    public List<Serializable> selectProperties(NodeRef contextNodeRef, String xpath,
-            QueryParameterDefinition[] parameters, NamespacePrefixResolver namespacePrefixResolver,
             boolean followAllParentLinks) throws InvalidNodeRefException, XPathException
     {
-        return selectProperties(contextNodeRef, xpath, parameters, namespacePrefixResolver, followAllParentLinks,
+        return selectNodes(contextNodeRef, xpath, parameters, namespacePrefixResolver, followAllParentLinks, SearchService.LANGUAGE_XPATH);
-                SearchService.LANGUAGE_XPATH);
+    }
+
+    public List<Serializable> selectProperties(NodeRef contextNodeRef, String xpath, QueryParameterDefinition[] parameters, NamespacePrefixResolver namespacePrefixResolver,
+            boolean followAllParentLinks) throws InvalidNodeRefException, XPathException
+    {
+        return selectProperties(contextNodeRef, xpath, parameters, namespacePrefixResolver, followAllParentLinks, SearchService.LANGUAGE_XPATH);
     }
 }

source/java/org/alfresco/repo/search/impl/lucene/LuceneQueryParser.java

@@ -29,10 +29,13 @@ import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
+import java.util.Vector;
 
 import org.alfresco.i18n.I18NUtil;
 import org.alfresco.repo.search.MLAnalysisMode;
@@ -55,11 +58,17 @@ import org.apache.log4j.Logger;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.MultiPhraseQuery;
+import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.RangeQuery;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.search.WildcardTermEnum;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.saxpath.SAXPathException;
 
@@ -79,6 +88,8 @@ public class LuceneQueryParser extends QueryParser
 
     private LuceneConfig config;
 
+    private IndexReader indexReader;
+
     /**
      * Parses a query string, returning a {@link org.apache.lucene.search.Query}.
      * 
@@ -94,7 +105,7 @@ public class LuceneQueryParser extends QueryParser
      */
     static public Query parse(String query, String field, Analyzer analyzer,
             NamespacePrefixResolver namespacePrefixResolver, DictionaryService dictionaryService, TenantService tenantService,
-            Operator defaultOperator, SearchParameters searchParameters, LuceneConfig config) throws ParseException
+            Operator defaultOperator, SearchParameters searchParameters, LuceneConfig config, IndexReader indexReader) throws ParseException
     {
         if (s_logger.isDebugEnabled())
         {
@@ -107,6 +118,7 @@ public class LuceneQueryParser extends QueryParser
         parser.setTenantService(tenantService);
         parser.setSearchParameters(searchParameters);
         parser.setLuceneConfig(config);
+        parser.setIndexReader(indexReader);
         // TODO: Apply locale contstraints at the top level if required for the non ML doc types.
         Query result = parser.parse(query);
         if (s_logger.isDebugEnabled())
@@ -121,6 +133,11 @@ public class LuceneQueryParser extends QueryParser
         this.config = config;
     }
 
+    private void setIndexReader(IndexReader indexReader)
+    {
+        this.indexReader = indexReader;
+    }
+
     private void setSearchParameters(SearchParameters searchParameters)
     {
         this.searchParameters = searchParameters;
@@ -184,8 +201,7 @@ public class LuceneQueryParser extends QueryParser
                 Set<String> text = searchParameters.getTextAttributes();
                 if ((text == null) || (text.size() == 0))
                 {
-                    Collection<QName> contentAttributes = dictionaryService
+                    Collection<QName> contentAttributes = dictionaryService.getAllProperties(DataTypeDefinition.CONTENT);
-                            .getAllProperties(DataTypeDefinition.CONTENT);
                     BooleanQuery query = new BooleanQuery();
                     for (QName qname : contentAttributes)
                     {
@@ -294,14 +310,12 @@ public class LuceneQueryParser extends QueryParser
                     if (colonPosition == -1)
                     {
                         // use the default namespace
-                        target = dictionaryService.getType(QName.createQName(namespacePrefixResolver
+                        target = dictionaryService.getType(QName.createQName(namespacePrefixResolver.getNamespaceURI(""), queryText));
-                                .getNamespaceURI(""), queryText));
                     }
                     else
                     {
                         // find the prefix
-                        target = dictionaryService.getType(QName.createQName(namespacePrefixResolver
+                        target = dictionaryService.getType(QName.createQName(namespacePrefixResolver.getNamespaceURI(queryText.substring(0, colonPosition)), queryText
-                                .getNamespaceURI(queryText.substring(0, colonPosition)), queryText
                                 .substring(colonPosition + 1)));
                     }
                 }
@@ -316,8 +330,7 @@ public class LuceneQueryParser extends QueryParser
                     TypeDefinition current = dictionaryService.getType(classRef);
                     while ((current != null) && !current.getName().equals(targetQName))
                     {
-                        current = (current.getParentName() == null) ? null : dictionaryService.getType(current
+                        current = (current.getParentName() == null) ? null : dictionaryService.getType(current.getParentName());
-                                .getParentName());
                     }
                     if (current != null)
                     {
@@ -348,14 +361,12 @@ public class LuceneQueryParser extends QueryParser
                     if (colonPosition == -1)
                     {
                         // use the default namespace
-                        target = dictionaryService.getType(QName.createQName(namespacePrefixResolver
+                        target = dictionaryService.getType(QName.createQName(namespacePrefixResolver.getNamespaceURI(""), queryText));
-                                .getNamespaceURI(""), queryText));
                     }
                     else
                     {
                         // find the prefix
-                        target = dictionaryService.getType(QName.createQName(namespacePrefixResolver
+                        target = dictionaryService.getType(QName.createQName(namespacePrefixResolver.getNamespaceURI(queryText.substring(0, colonPosition)), queryText
-                                .getNamespaceURI(queryText.substring(0, colonPosition)), queryText
                                 .substring(colonPosition + 1)));
                     }
                 }
@@ -381,14 +392,12 @@ public class LuceneQueryParser extends QueryParser
                     if (colonPosition == -1)
                     {
                         // use the default namespace
-                        target = dictionaryService.getAspect(QName.createQName(namespacePrefixResolver
+                        target = dictionaryService.getAspect(QName.createQName(namespacePrefixResolver.getNamespaceURI(""), queryText));
-                                .getNamespaceURI(""), queryText));
                     }
                     else
                     {
                         // find the prefix
-                        target = dictionaryService.getAspect(QName.createQName(namespacePrefixResolver
+                        target = dictionaryService.getAspect(QName.createQName(namespacePrefixResolver.getNamespaceURI(queryText.substring(0, colonPosition)), queryText
-                                .getNamespaceURI(queryText.substring(0, colonPosition)), queryText
                                 .substring(colonPosition + 1)));
                     }
                 }
@@ -400,8 +409,7 @@ public class LuceneQueryParser extends QueryParser
                     AspectDefinition current = dictionaryService.getAspect(classRef);
                     while ((current != null) && !current.getName().equals(targetQName))
                     {
-                        current = (current.getParentName() == null) ? null : dictionaryService.getAspect(current
+                        current = (current.getParentName() == null) ? null : dictionaryService.getAspect(current.getParentName());
-                                .getParentName());
                     }
                     if (current != null)
                     {
@@ -433,14 +441,12 @@ public class LuceneQueryParser extends QueryParser
                     if (colonPosition == -1)
                     {
                         // use the default namespace
-                        target = dictionaryService.getAspect(QName.createQName(namespacePrefixResolver
+                        target = dictionaryService.getAspect(QName.createQName(namespacePrefixResolver.getNamespaceURI(""), queryText));
-                                .getNamespaceURI(""), queryText));
                     }
                     else
                     {
                         // find the prefix
-                        target = dictionaryService.getAspect(QName.createQName(namespacePrefixResolver
+                        target = dictionaryService.getAspect(QName.createQName(namespacePrefixResolver.getNamespaceURI(queryText.substring(0, colonPosition)), queryText
-                                .getNamespaceURI(queryText.substring(0, colonPosition)), queryText
                                 .substring(colonPosition + 1)));
                     }
                 }
@@ -516,7 +522,7 @@ public class LuceneQueryParser extends QueryParser
                 }
                 else
                 {
-                    return super.getFieldQuery(field, queryText);
+                    return getFieldQueryImpl(field, queryText);
                 }
 
             }
@@ -540,14 +546,13 @@ public class LuceneQueryParser extends QueryParser
                 }
                 else
                 {
-                    return super.getFieldQuery(field, queryText);
+                    return getFieldQueryImpl(field, queryText);
                 }
 
             }
             else if (dictionaryService.getDataType(QName.createQName(expandFieldName(field))) != null)
             {
-                Collection<QName> contentAttributes = dictionaryService.getAllProperties(dictionaryService.getDataType(
+                Collection<QName> contentAttributes = dictionaryService.getAllProperties(dictionaryService.getDataType(QName.createQName(expandFieldName(field))).getName());
-                        QName.createQName(expandFieldName(field))).getName());
                 BooleanQuery query = new BooleanQuery();
                 for (QName qname : contentAttributes)
                 {
@@ -566,7 +571,7 @@ public class LuceneQueryParser extends QueryParser
             }
             else
             {
-                return super.getFieldQuery(field, queryText);
+                return getFieldQueryImpl(field, queryText);
             }
 
         }
@@ -577,6 +582,542 @@ public class LuceneQueryParser extends QueryParser
 
     }
 
+    private Query getFieldQueryImpl(String field, String queryText) throws ParseException
+    {
+        // Use the analyzer to get all the tokens, and then build a TermQuery,
+        // PhraseQuery, or nothing based on the term count
+
+        boolean isMlText = false;
+        String testText = queryText;
+        String localeString = null;
+        if (field.startsWith("@"))
+        {
+            String expandedFieldName = expandAttributeFieldName(field);
+            QName propertyQName = QName.createQName(expandedFieldName.substring(1));
+            PropertyDefinition propertyDef = dictionaryService.getProperty(propertyQName);
+            if ((propertyDef != null) && (propertyDef.getDataType().getName().equals(DataTypeDefinition.MLTEXT)))
+            {
+                int position = queryText.indexOf("\u0000", 1);
+                testText = queryText.substring(position + 1);
+                isMlText = true;
+                localeString = queryText.substring(1, position);
+            }
+        }
+
+        TokenStream source = analyzer.tokenStream(field, new StringReader(queryText));
+        ArrayList<org.apache.lucene.analysis.Token> v = new ArrayList<org.apache.lucene.analysis.Token>();
+        org.apache.lucene.analysis.Token t;
+        int positionCount = 0;
+        boolean severalTokensAtSamePosition = false;
+
+        while (true)
+        {
+            try
+            {
+                t = source.next();
+            }
+            catch (IOException e)
+            {
+                t = null;
+            }
+            if (t == null)
+                break;
+            v.add(t);
+            if (t.getPositionIncrement() != 0)
+                positionCount += t.getPositionIncrement();
+            else
+                severalTokensAtSamePosition = true;
+        }
+        try
+        {
+            source.close();
+        }
+        catch (IOException e)
+        {
+            // ignore
+        }
+
+        // add any alpha numeric wildcards that have been missed
+        // Fixes most stop word and wild card issues
+
+        for (int index = 0; index < testText.length(); index++)
+        {
+            char current = testText.charAt(index);
+            if ((current == '*') || (current == '?'))
+            {
+                StringBuilder pre = new StringBuilder(10);
+                if (index > 0)
+                {
+                    for (int i = index - 1; i >= 0; i--)
+                    {
+                        char c = testText.charAt(i);
+                        if (Character.isLetterOrDigit(c))
+                        {
+                            boolean found = false;
+                            for (int j = 0; j < v.size(); j++)
+                            {
+                                org.apache.lucene.analysis.Token test = v.get(j);
+                                if ((test.startOffset() <= i) && (i < test.endOffset()))
+                                {
+                                    found = true;
+                                    break;
+                                }
+                            }
+                            if (found)
+                            {
+                                break;
+                            }
+                            else
+                            {
+                                pre.insert(0, c);
+                            }
+                        }
+                    }
+                    if (pre.length() > 0)
+                    {
+                        // Add new token followed by * not given by the tokeniser
+                        org.apache.lucene.analysis.Token newToken = new org.apache.lucene.analysis.Token(pre.toString(), index - pre.length(), index, "ALPHANUM");
+                        if (isMlText)
+                        {
+                            Locale locale = I18NUtil.parseLocale(localeString);
+                            MLAnalysisMode analysisMode = searchParameters.getMlAnalaysisMode() == null ? config.getDefaultMLSearchAnalysisMode() : searchParameters
+                                    .getMlAnalaysisMode();
+                            MLTokenDuplicator duplicator = new MLTokenDuplicator(locale, analysisMode);
+                            Iterator<org.apache.lucene.analysis.Token> it = duplicator.buildIterator(newToken);
+                            if (it != null)
+                            {
+                                int count = 0;
+                                while (it.hasNext())
+                                {
+                                    v.add(it.next());
+                                    count++;
+                                    if(count > 1)
+                                    {
+                                        severalTokensAtSamePosition = true;
+                                    }
+                                }
+                            }
+                        }
+                        // content
+                        else
+                        {
+                            v.add(newToken);
+                        }
+                    }
+                }
+
+                StringBuilder post = new StringBuilder(10);
+                if (index > 0)
+                {
+                    for (int i = index + 1; i < testText.length(); i++)
+                    {
+                        char c = testText.charAt(i);
+                        if (Character.isLetterOrDigit(c))
+                        {
+                            boolean found = false;
+                            for (int j = 0; j < v.size(); j++)
+                            {
+                                org.apache.lucene.analysis.Token test = v.get(j);
+                                if ((test.startOffset() <= i) && (i < test.endOffset()))
+                                {
+                                    found = true;
+                                    break;
+                                }
+                            }
+                            if (found)
+                            {
+                                break;
+                            }
+                            else
+                            {
+                                post.append(c);
+                            }
+                        }
+                    }
+                    if (post.length() > 0)
+                    {
+                        // Add new token followed by * not given by the tokeniser
+                        org.apache.lucene.analysis.Token newToken = new org.apache.lucene.analysis.Token(post.toString(), index - post.length(), index, "ALPHANUM");
+                        if (isMlText)
+                        {
+                            Locale locale = I18NUtil.parseLocale(localeString);
+                            MLAnalysisMode analysisMode = searchParameters.getMlAnalaysisMode() == null ? config.getDefaultMLSearchAnalysisMode() : searchParameters
+                                    .getMlAnalaysisMode();
+                            MLTokenDuplicator duplicator = new MLTokenDuplicator(locale, analysisMode);
+                            Iterator<org.apache.lucene.analysis.Token> it = duplicator.buildIterator(newToken);
+                            if (it != null)
+                            {
+                                int count = 0;
+                                while (it.hasNext())
+                                {
+                                    v.add(it.next());
+                                    count++;
+                                    if(count > 1)
+                                    {
+                                        severalTokensAtSamePosition = true;
+                                    }
+                                }
+                            }
+                        }
+                        // content
+                        else
+                        {
+                            v.add(newToken);
+                        }
+                    }
+                }
+
+            }
+        }
+
+        Collections.sort(v, new Comparator<org.apache.lucene.analysis.Token>()
+        {
+
+            public int compare(Token o1, Token o2)
+            {
+                int dif = o1.startOffset() - o2.startOffset();
+                if (dif != 0)
+                {
+                    return dif;
+                }
+                else
+                {
+                    return o2.getPositionIncrement() - o1.getPositionIncrement();
+                }
+            }
+        });
+
+        // Combined * and ? based strings - should redo the tokeniser
+
+        // Assue we only string together tokens for the same postion
+
+        int max = 0;
+        int current = 0;
+        for (org.apache.lucene.analysis.Token c : v)
+        {
+            if (c.getPositionIncrement() == 0)
+            {
+                current++;
+            }
+            else
+            {
+                if (current > max)
+                {
+                    max = current;
+                }
+                current = 0;
+            }
+        }
+        if (current > max)
+        {
+            max = current;
+        }
+
+        ArrayList<org.apache.lucene.analysis.Token> fixed = new ArrayList<org.apache.lucene.analysis.Token>();
+        for (int repeat = 0; repeat <= max; repeat++)
+        {
+            org.apache.lucene.analysis.Token replace = null;
+            current = 0;
+            for (org.apache.lucene.analysis.Token c : v)
+            {
+                if (c.getPositionIncrement() == 0)
+                {
+                    current++;
+                }
+                else
+                {
+                    current = 0;
+                }
+
+                if (current == repeat)
+                {
+
+                    if (replace == null)
+                    {
+                        StringBuilder prefix = new StringBuilder();
+                        for (int i = c.startOffset() - 1; i >= 0; i--)
+                        {
+                            char test = testText.charAt(i);
+                            if ((test == '*') || (test == '?'))
+                            {
+                                prefix.insert(0, test);
+                            }
+                            else
+                            {
+                                break;
+                            }
+                        }
+                        String pre = prefix.toString();
+                        if (isMlText)
+                        {
+                            int position = c.termText().indexOf("}");
+                            String language = c.termText().substring(0, position + 1);
+                            String token = c.termText().substring(position + 1);
+                            replace = new org.apache.lucene.analysis.Token(language + pre + token, c.startOffset() - pre.length(), c.endOffset(), c.type());
+                            replace.setPositionIncrement(c.getPositionIncrement());
+                        }
+                        else
+                        {
+                            replace = new org.apache.lucene.analysis.Token(pre + c.termText(), c.startOffset() - pre.length(), c.endOffset(), c.type());
+                            replace.setPositionIncrement(c.getPositionIncrement());
+                        }
+                    }
+                    else
+                    {
+                        StringBuilder prefix = new StringBuilder();
+                        StringBuilder postfix = new StringBuilder();
+                        StringBuilder builder = prefix;
+                        for (int i = c.startOffset() - 1; i >= replace.endOffset(); i--)
+                        {
+                            char test = testText.charAt(i);
+                            if ((test == '*') || (test == '?'))
+                            {
+                                builder.insert(0, test);
+                            }
+                            else
+                            {
+                                builder = postfix;
+                                postfix.setLength(0);
+                            }
+                        }
+                        String pre = prefix.toString();
+                        String post = postfix.toString();
+
+                        // Does it bridge?
+                        if ((pre.length() > 0) && (replace.endOffset() + pre.length()) == c.startOffset())
+                        {
+                            if (isMlText)
+                            {
+                                int position = c.termText().indexOf("}");
+                                String language = c.termText().substring(0, position + 1);
+                                String token = c.termText().substring(position + 1);
+                                int oldPositionIncrement = replace.getPositionIncrement();
+                                replace = new org.apache.lucene.analysis.Token(replace.termText() + pre + token, replace.startOffset(), c.endOffset(), replace.type());
+                                replace.setPositionIncrement(oldPositionIncrement);
+                            }
+                            else
+                            {
+                                int oldPositionIncrement = replace.getPositionIncrement();
+                                replace = new org.apache.lucene.analysis.Token(replace.termText() + pre + c.termText(), replace.startOffset(), c.endOffset(), replace.type());
+                                replace.setPositionIncrement(oldPositionIncrement);
+                            }
+                        }
+                        else
+                        {
+                            if (isMlText)
+                            {
+                                int position = c.termText().indexOf("}");
+                                String language = c.termText().substring(0, position + 1);
+                                String token = c.termText().substring(position + 1);
+                                org.apache.lucene.analysis.Token last = new org.apache.lucene.analysis.Token(replace.termText() + post, replace.startOffset(), replace.endOffset()
+                                        + post.length(), replace.type());
+                                last.setPositionIncrement(replace.getPositionIncrement());
+                                fixed.add(last);
+                                replace = new org.apache.lucene.analysis.Token(language + pre + token, c.startOffset() - pre.length(), c.endOffset(), c.type());
+                                replace.setPositionIncrement(c.getPositionIncrement());
+                            }
+                            else
+                            {
+                                org.apache.lucene.analysis.Token last = new org.apache.lucene.analysis.Token(replace.termText() + post, replace.startOffset(), replace.endOffset()
+                                        + post.length(), replace.type());
+                                last.setPositionIncrement(replace.getPositionIncrement());
+                                fixed.add(last);
+                                replace = new org.apache.lucene.analysis.Token(pre + c.termText(), c.startOffset() - pre.length(), c.endOffset(), c.type());
+                                replace.setPositionIncrement(c.getPositionIncrement());
+                            }
+                        }
+                    }
+                }
+            }
+            // finish last
+            if (replace != null)
+            {
+                StringBuilder postfix = new StringBuilder();
+                for (int i = replace.endOffset(); i < testText.length(); i++)
+                {
+                    char test = testText.charAt(i);
+                    if ((test == '*') || (test == '?'))
+                    {
+                        postfix.append(test);
+                    }
+                    else
+                    {
+                        break;
+                    }
+                }
+                String post = postfix.toString();
+                int oldPositionIncrement = replace.getPositionIncrement();
+                replace = new org.apache.lucene.analysis.Token(replace.termText() + post, replace.startOffset(), replace.endOffset() + post.length(), replace.type());
+                replace.setPositionIncrement(oldPositionIncrement);
+                fixed.add(replace);
+
+            }
+        }
+
+        // Add in any missing words containsing * and ?
+
+        // reorder by start position and increment
+
+        Collections.sort(fixed, new Comparator<org.apache.lucene.analysis.Token>()
+        {
+
+            public int compare(Token o1, Token o2)
+            {
+                int dif = o1.startOffset() - o2.startOffset();
+                if (dif != 0)
+                {
+                    return dif;
+                }
+                else
+                {
+                    return o2.getPositionIncrement() - o1.getPositionIncrement();
+                }
+            }
+        });
+
+        v = fixed;
+
+        if (v.size() == 0)
+            return null;
+        else if (v.size() == 1)
+        {
+            t = (org.apache.lucene.analysis.Token) v.get(0);
+            if (t.termText().contains("*") || t.termText().contains("?"))
+            {
+                return new org.apache.lucene.search.WildcardQuery(new Term(field, t.termText()));
+            }
+            else
+            {
+                return new TermQuery(new Term(field, t.termText()));
+            }
+        }
+        else
+        {
+            if (severalTokensAtSamePosition)
+            {
+                if (positionCount == 1)
+                {
+                    // no phrase query:
+                    BooleanQuery q = new BooleanQuery(true);
+                    for (int i = 0; i < v.size(); i++)
+                    {
+                        t = (org.apache.lucene.analysis.Token) v.get(i);
+                        if (t.termText().contains("*") || t.termText().contains("?"))
+                        {
+                            org.apache.lucene.search.WildcardQuery currentQuery = new org.apache.lucene.search.WildcardQuery(new Term(field, t.termText()));
+                            q.add(currentQuery, BooleanClause.Occur.SHOULD);
+                        }
+                        else
+                        {
+                            TermQuery currentQuery = new TermQuery(new Term(field, t.termText()));
+                            q.add(currentQuery, BooleanClause.Occur.SHOULD);
+                        }
+                    }
+                    return q;
+                }
+                else
+                {
+                    // phrase query:
+                    MultiPhraseQuery mpq = new MultiPhraseQuery();
+                    mpq.setSlop(phraseSlop);
+                    ArrayList<Term> multiTerms = new ArrayList<Term>();
+                    for (int i = 0; i < v.size(); i++)
+                    {
+                        t = (org.apache.lucene.analysis.Token) v.get(i);
+                        if (t.getPositionIncrement() == 1 && multiTerms.size() > 0)
+                        {
+                            mpq.add((Term[]) multiTerms.toArray(new Term[0]));
+                            multiTerms.clear();
+                        }
+                        Term term = new Term(field, t.termText());
+                        if ((t.termText() != null) && (t.termText().contains("*") || t.termText().contains("?")))
+                        {
+                            addWildcardTerms(multiTerms, term);
+                        }
+                        else
+                        {
+                            multiTerms.add(term);
+                        }
+                    }
+                    if (multiTerms.size() > 0)
+                    {
+                        mpq.add((Term[]) multiTerms.toArray(new Term[0]));
+                    }
+                    else
+                    {
+                        mpq.add(new Term[] { new Term(field, "\u0000") });
+                    }
+                    return mpq;
+                }
+            }
+            else
+            {
+                MultiPhraseQuery q = new MultiPhraseQuery();
+                q.setSlop(phraseSlop);
+                for (int i = 0; i < v.size(); i++)
+                {
+                    t = (org.apache.lucene.analysis.Token) v.get(i);
+                    Term term = new Term(field, t.termText());
+                    if ((t.termText() != null) && (t.termText().contains("*") || t.termText().contains("?")))
+                    {
+                        q.add(getMatchingTerms(field, term));
+                    }
+                    else
+                    {
+                        q.add(term);
+                    }
+                }
+                return q;
+            }
+        }
+    }
+
+    private Term[] getMatchingTerms(String field, Term term) throws ParseException
+    {
+        ArrayList<Term> terms = new ArrayList<Term>();
+        addWildcardTerms(terms, term);
+        if (terms.size() == 0)
+        {
+            return new Term[] { new Term(field, "\u0000") };
+        }
+        else
+        {
+            return terms.toArray(new Term[0]);
+        }
+
+    }
+
+    private void addWildcardTerms(ArrayList<Term> terms, Term term) throws ParseException
+    {
+        try
+        {
+            WildcardTermEnum wcte = new WildcardTermEnum(indexReader, term);
+
+            while (!wcte.endEnum())
+            {
+                Term current = wcte.term();
+                if((current.text() != null) && (current.text().length() > 0) && (current.text().charAt(0) == '{'))
+                {
+                    if((term != null) && (term.text().length() > 0) && (term.text().charAt(0) == '{'))
+                    {
+                        terms.add(current);
+                    }
+                    // If not, we cod not add so wildcards do not match the locale prefix
+                }
+                else
+                {
+                    terms.add(current);
+                }
+               
+                wcte.next();
+            }
+        }
+        catch (IOException e)
+        {
+            throw new ParseException("IO error generating phares wildcards " + e.getMessage());
+        }
+    }
+
     /**
      * @exception ParseException
      *                throw in overridden method to disallow
@@ -586,8 +1127,7 @@ public class LuceneQueryParser extends QueryParser
         if (field.startsWith("@"))
         {
             String fieldName = expandAttributeFieldName(field);
-            return new RangeQuery(new Term(fieldName, getToken(fieldName, part1)), new Term(fieldName, getToken(
+            return new RangeQuery(new Term(fieldName, getToken(fieldName, part1)), new Term(fieldName, getToken(fieldName, part2)), inclusive);
-                    fieldName, part2)), inclusive);
 
         }
         else
@@ -612,9 +1152,7 @@ public class LuceneQueryParser extends QueryParser
             else
             {
                 // find the prefix
-                fieldName = "@{"
+                fieldName = "@{" + namespacePrefixResolver.getNamespaceURI(field.substring(1, colonPosition)) + "}" + field.substring(colonPosition + 1);
-                        + namespacePrefixResolver.getNamespaceURI(field.substring(1, colonPosition)) + "}"
-                        + field.substring(colonPosition + 1);
             }
         }
         return fieldName;
@@ -635,9 +1173,7 @@ public class LuceneQueryParser extends QueryParser
             else
             {
                 // find the prefix
-                fieldName = "{"
+                fieldName = "{" + namespacePrefixResolver.getNamespaceURI(field.substring(0, colonPosition)) + "}" + field.substring(colonPosition + 1);
-                        + namespacePrefixResolver.getNamespaceURI(field.substring(0, colonPosition)) + "}"
-                        + field.substring(colonPosition + 1);
             }
         }
         return fieldName;
@@ -780,7 +1316,7 @@ public class LuceneQueryParser extends QueryParser
 
     public Query getSuperFieldQuery(String field, String queryText) throws ParseException
     {
-        return super.getFieldQuery(field, queryText);
+        return getFieldQueryImpl(field, queryText);
     }
 
     public Query getSuperFuzzyQuery(String field, String termStr, float minSimilarity) throws ParseException
@@ -892,11 +1428,10 @@ public class LuceneQueryParser extends QueryParser
             // cross language matching for each entry
             BooleanQuery booleanQuery = new BooleanQuery();
             List<Locale> locales = searchParameters.getLocales();
-            for (Locale locale : (((locales == null) || (locales.size() == 0)) ? Collections.singletonList(I18NUtil
+            for (Locale locale : (((locales == null) || (locales.size() == 0)) ? Collections.singletonList(I18NUtil.getLocale()) : locales))
-                    .getLocale()) : locales))
             {
-                
+
-                if(isAnalysed)
+                if (isAnalysed)
                 {
                     StringBuilder builder = new StringBuilder(queryText.length() + 10);
                     builder.append("\u0000").append(locale.toString()).append("\u0000").append(queryText);
@@ -913,15 +1448,14 @@ public class LuceneQueryParser extends QueryParser
                 else
                 {
                     // analyse ml text
-                    MLAnalysisMode analysisMode = searchParameters.getMlAnalaysisMode() == null ? config
+                    MLAnalysisMode analysisMode = searchParameters.getMlAnalaysisMode() == null ? config.getDefaultMLSearchAnalysisMode() : searchParameters.getMlAnalaysisMode();
-                            .getDefaultMLSearchAnalysisMode() : searchParameters.getMlAnalaysisMode();
                     // Do the analysis here
-                    VerbatimAnalyser vba = new VerbatimAnalyser(false);        
+                    VerbatimAnalyser vba = new VerbatimAnalyser(false);
                     MLTokenDuplicator duplicator = new MLTokenDuplicator(vba.tokenStream(field, new StringReader(queryText)), locale, null, analysisMode);
                     Token t;
                     try
                     {
-                        while( (t = duplicator.next()) != null)
+                        while ((t = duplicator.next()) != null)
                         {
                             Query subQuery = subQueryBuilder.getQuery(expandedFieldName, t.termText());
                             booleanQuery.add(subQuery, Occur.SHOULD);
@@ -929,15 +1463,15 @@ public class LuceneQueryParser extends QueryParser
                     }
                     catch (IOException e)
                     {
-                       
+
                     }
-                    if(booleanQuery.getClauses().length == 0)
+                    if (booleanQuery.getClauses().length == 0)
                     {
                         booleanQuery.add(new TermQuery(new Term("NO_TOKENS", "__")), Occur.SHOULD);
                     }
-                    
+
                 }
-                
+
             }
             return booleanQuery;
         }
@@ -947,8 +1481,7 @@ public class LuceneQueryParser extends QueryParser
             // Build a sub query for each locale and or the results together -
             // - add an explicit condition for the locale
 
-            MLAnalysisMode analysisMode = searchParameters.getMlAnalaysisMode() == null ? config
+            MLAnalysisMode analysisMode = searchParameters.getMlAnalaysisMode() == null ? config.getDefaultMLSearchAnalysisMode() : searchParameters.getMlAnalaysisMode();
-                    .getDefaultMLSearchAnalysisMode() : searchParameters.getMlAnalaysisMode();
 
             if (analysisMode.includesAll())
             {
@@ -957,8 +1490,7 @@ public class LuceneQueryParser extends QueryParser
 
             List<Locale> locales = searchParameters.getLocales();
             List<Locale> expandedLocales = new ArrayList<Locale>();
-            for (Locale locale : (((locales == null) || (locales.size() == 0)) ? Collections.singletonList(I18NUtil
+            for (Locale locale : (((locales == null) || (locales.size() == 0)) ? Collections.singletonList(I18NUtil.getLocale()) : locales))
-                    .getLocale()) : locales))
             {
                 expandedLocales.addAll(MLAnalysisMode.getLocales(analysisMode, locale, true));
             }
@@ -1008,7 +1540,7 @@ public class LuceneQueryParser extends QueryParser
             else
             {
                 Query query = subQueryBuilder.getQuery(expandedFieldName, queryText);
-                if(query != null)
+                if (query != null)
                 {
                     return query;
                 }
@@ -1022,7 +1554,7 @@ public class LuceneQueryParser extends QueryParser
         else
         {
             Query query = subQueryBuilder.getQuery(expandedFieldName, queryText);
-            if(query != null)
+            if (query != null)
             {
                 return query;
             }

source/java/org/alfresco/repo/search/impl/lucene/analysis/AlfrescoStandardAnalyser.java

@@ -57,7 +57,7 @@ public class AlfrescoStandardAnalyser extends Analyzer
     {
         stopSet = StopFilter.makeStopSet(stopWords);
     }
-
+    
     /**
      * Constructs a {@link StandardTokenizer} filtered by a {@link StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}.
      */

source/java/org/alfresco/repo/search/impl/lucene/analysis/MLTokenDuplicator.java

@@ -61,6 +61,11 @@ public class MLTokenDuplicator extends Tokenizer
         }
 
     }
+    
+    public MLTokenDuplicator(Locale locale, MLAnalysisMode mlAnalaysisMode)
+    {
+        this(null, locale, null, mlAnalaysisMode);
+    }
 
     @Override
     public Token next() throws IOException
@@ -87,6 +92,13 @@ public class MLTokenDuplicator extends Tokenizer
     private Iterator<Token> buildIterator() throws IOException
     {
         Token token = source.next();
+        return buildIterator(token);
+
+    }
+
+
+    public Iterator<Token> buildIterator(Token token)
+    {
         if (token == null)
         {
             return null;
@@ -110,4 +122,5 @@ public class MLTokenDuplicator extends Tokenizer
 
     }
 
+    
 }

source/java/org/alfresco/repo/search/impl/lucene/analysis/NumericEncoder.java

@@ -122,6 +122,12 @@ public class NumericEncoder
         return decodeFromHex(hex) ^ LONG_SIGN_MASK;
     }
     
+    
+    public static int decodeInt(String hex)
+    {
+        return decodeIntFromHex(hex) ^ INTEGER_SIGN_MASK;
+    }
+    
     /**
      * Encode a float into a string that orders correctly according to string
      * comparison. Note that there is no negative NaN but there are codings that
@@ -208,6 +214,19 @@ public class NumericEncoder
         return l;
     }
 
+    private static int decodeIntFromHex(String hex)
+    {
+        int l = 0;
+        int factor = 1;
+        for(int i = 7; i >= 0; i--, factor <<= 4)
+        {
+            int digit = Character.digit(hex.charAt(i), 16);
+            l += digit*factor;
+        }
+        return l;
+    }
+
+    
     private static final char[] DIGITS = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e',
             'f' };