ACS-11243 Field Tokenisation Override (#2244)

search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java

@@ -253,6 +253,7 @@ public class AlfrescoSolrDataModel implements QueryConstants
     private final Set<QName> crossLocaleSearchDataTypes = new HashSet<>();
     private final Set<QName> crossLocaleSearchProperties = new HashSet<>();
     private final Set<QName> identifierProperties = new HashSet<>();
+    private final Set<QName> tokeniseProperties = new HashSet<>();
     private final ThreadPoolExecutor threadPool;
 
     public void close() {
@@ -318,6 +319,11 @@ public class AlfrescoSolrDataModel implements QueryConstants
                 QName qName = QName.createQName(props.getProperty(stringKey));
                 identifierProperties.add(qName);
             }
+            else if (stringKey.startsWith("alfresco.tokenise.property."))
+            {
+                QName qName = QName.createQName(props.getProperty(stringKey));
+                tokeniseProperties.add(qName);
+            }
         }
 
         if(props.isEmpty())
@@ -788,7 +794,8 @@ public class AlfrescoSolrDataModel implements QueryConstants
      */
     private void addCompletionFields(PropertyDefinition propertyDefinition, IndexedField indexedField)
     {
-        if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH))
+        IndexTokenisationMode tokenisationMode = getIndexTokenisationMode(propertyDefinition);
+        if ((tokenisationMode == IndexTokenisationMode.BOTH))
         {
             if(crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) || crossLocaleSearchProperties.contains(propertyDefinition.getName()))
             {
@@ -799,7 +806,7 @@ public class AlfrescoSolrDataModel implements QueryConstants
                 indexedField.addField(getFieldForText(true, true, false, propertyDefinition), false, false);
             }
         }
-        else if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.TRUE))
+        else if ((tokenisationMode == IndexTokenisationMode.TRUE))
         {
             if(crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) || crossLocaleSearchProperties.contains(propertyDefinition.getName()))
             {
@@ -810,7 +817,7 @@ public class AlfrescoSolrDataModel implements QueryConstants
                 indexedField.addField(getFieldForText(true, true, false, propertyDefinition), false, false);
             }
         }
-        else if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE))
+        else if ((tokenisationMode == IndexTokenisationMode.FALSE))
         {
             indexedField.addField(getFieldForText(false, false, false, propertyDefinition), false, false);
         }
@@ -822,8 +829,9 @@ public class AlfrescoSolrDataModel implements QueryConstants
 
     private void addFullTextSearchFields( PropertyDefinition propertyDefinition , IndexedField indexedField)
     {
-        if (((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.TRUE)
-                || (propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH))
+        IndexTokenisationMode tokenisationMode = getIndexTokenisationMode(propertyDefinition);
+        if (((tokenisationMode == IndexTokenisationMode.TRUE)
+                || (tokenisationMode == IndexTokenisationMode.BOTH))
                 && !isIdentifierTextProperty(propertyDefinition.getName()))
         {
             indexedField.addField(getFieldForText(true, true, false, propertyDefinition), true, false);
@@ -867,8 +875,8 @@ public class AlfrescoSolrDataModel implements QueryConstants
      */
     private void addIdentifierSearchFields( PropertyDefinition propertyDefinition , IndexedField indexedField)
     {
-        if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE)
-                || (propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH))
+        IndexTokenisationMode tokenisationMode = getIndexTokenisationMode(propertyDefinition);
+        if ((tokenisationMode == IndexTokenisationMode.FALSE) || (tokenisationMode == IndexTokenisationMode.BOTH))
         {
 
             indexedField.addField(getFieldForText(true, false, false, propertyDefinition), true, false);
@@ -890,7 +898,8 @@ public class AlfrescoSolrDataModel implements QueryConstants
      */
     private void addExactSearchFields(PropertyDefinition propertyDefinition, IndexedField indexedField)
     {
-        if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE))
+        IndexTokenisationMode tokenisationMode = getIndexTokenisationMode(propertyDefinition);
+        if ((tokenisationMode == IndexTokenisationMode.FALSE))
         {
 
             indexedField.addField(getFieldForText(true, false, false, propertyDefinition), true, false);
@@ -917,6 +926,7 @@ public class AlfrescoSolrDataModel implements QueryConstants
      */
     private void addFacetSearchFields(PropertyDefinition propertyDefinition, IndexedField indexedField)
     {
+        IndexTokenisationMode tokenisationMode = getIndexTokenisationMode(propertyDefinition);
         if(propertyDefinition.getDataType().getName().equals(DataTypeDefinition.TEXT))
         {
             if (!isIdentifierTextProperty(propertyDefinition.getName()))
@@ -929,8 +939,8 @@ public class AlfrescoSolrDataModel implements QueryConstants
         }
 
 
-        if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE)
-                || (propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH)
+        if ((tokenisationMode == IndexTokenisationMode.FALSE)
+                || (tokenisationMode == IndexTokenisationMode.BOTH)
                 || isIdentifierTextProperty(propertyDefinition.getName()))
         {
 
@@ -951,8 +961,9 @@ public class AlfrescoSolrDataModel implements QueryConstants
 
     private void addMultiSearchFields( PropertyDefinition propertyDefinition , IndexedField indexedField)
     {
-        if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE)
-                || (propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH)
+        IndexTokenisationMode tokenisationMode = getIndexTokenisationMode(propertyDefinition);
+        if ((tokenisationMode == IndexTokenisationMode.FALSE)
+                || (tokenisationMode == IndexTokenisationMode.BOTH)
                 || isIdentifierTextProperty(propertyDefinition.getName()))
         {
 
@@ -978,14 +989,15 @@ public class AlfrescoSolrDataModel implements QueryConstants
 
     private void addSortSearchFields( PropertyDefinition propertyDefinition , IndexedField indexedField)
     {
+        IndexTokenisationMode tokenisationMode = getIndexTokenisationMode(propertyDefinition);
         // Can only order on single valued fields
         DataTypeDefinition dataTypeDefinition = propertyDefinition.getDataType();
         if(dataTypeDefinition.getName().equals(DataTypeDefinition.TEXT))
         {
             if(!propertyDefinition.isMultiValued())
             {
-                if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE)
-                        || (propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH))
+                if ((tokenisationMode == IndexTokenisationMode.FALSE)
+                        || (tokenisationMode == IndexTokenisationMode.BOTH))
                 {
                     indexedField.addField(getFieldForText(false, false, true, propertyDefinition), false, true);
                 }
@@ -1011,8 +1023,7 @@ public class AlfrescoSolrDataModel implements QueryConstants
         {
             if(!propertyDefinition.isMultiValued())
             {
-                if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE)
-                        || (propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH))
+                if ((tokenisationMode == IndexTokenisationMode.FALSE) || (tokenisationMode == IndexTokenisationMode.BOTH))
                 {
                     indexedField.addField(getFieldForText(false, false, true, propertyDefinition), false, true);
                 }
@@ -1040,22 +1051,23 @@ public class AlfrescoSolrDataModel implements QueryConstants
     public String getStoredTextField(QName propertyQName, String suffix)
     {
         PropertyDefinition propertyDefinition = getPropertyDefinition(propertyQName);
+        IndexTokenisationMode tokenisationMode = getIndexTokenisationMode(propertyDefinition);
 
         StringBuilder sb = new StringBuilder();
         sb.append("text@" + (propertyDefinition.isMultiValued()? "m" : "s") + "_stored_");
 
-        sb.append((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.TRUE ||
-            propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH)? "t" : "_");
+        sb.append((tokenisationMode == IndexTokenisationMode.TRUE ||
+            tokenisationMode == IndexTokenisationMode.BOTH)? "t" : "_");
 
-        sb.append((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE ||
-            propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH ||
+        sb.append((tokenisationMode == IndexTokenisationMode.FALSE ||
+            tokenisationMode == IndexTokenisationMode.BOTH ||
             isIdentifierTextProperty(propertyDefinition.getName()))? "s" : "_");
 
         sb.append((crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) ||
             crossLocaleSearchProperties.contains(propertyDefinition.getName())) ? "c" : "_");
 
-        sb.append((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE ||
-            propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH ||
+        sb.append((tokenisationMode == IndexTokenisationMode.FALSE ||
+            tokenisationMode == IndexTokenisationMode.BOTH ||
             isIdentifierTextProperty(propertyDefinition.getName())) && !propertyDefinition.isMultiValued()? "s" : "_");
 
         sb.append(isSuggestable(propertyQName)? "s": "_");
@@ -1080,15 +1092,16 @@ public class AlfrescoSolrDataModel implements QueryConstants
     public String getStoredMLTextField(QName propertyQName, String suffix)
     {
         PropertyDefinition propertyDefinition = getPropertyDefinition(propertyQName);
+        IndexTokenisationMode tokenisationMode = getIndexTokenisationMode(propertyDefinition);
 
         StringBuilder sb = new StringBuilder();
         sb.append("mltext@m_stored_");
 
-        sb.append((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.TRUE ||
-            propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH)? "t" : "_");
+        sb.append((tokenisationMode == IndexTokenisationMode.TRUE ||
+            tokenisationMode == IndexTokenisationMode.BOTH)? "t" : "_");
 
-        sb.append((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE ||
-            propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH ||
+        sb.append((tokenisationMode == IndexTokenisationMode.FALSE ||
+            tokenisationMode == IndexTokenisationMode.BOTH ||
             isIdentifierTextProperty(propertyDefinition.getName()))? "s" : "_");
 
         sb.append((crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) ||
@@ -1118,15 +1131,16 @@ public class AlfrescoSolrDataModel implements QueryConstants
     public String getStoredContentField(QName propertyQName, String suffix)
     {
         PropertyDefinition propertyDefinition = getPropertyDefinition(propertyQName);
+        IndexTokenisationMode tokenisationMode = getIndexTokenisationMode(propertyDefinition);
 
         StringBuilder sb = new StringBuilder();
         sb.append("content@s_stored_");
 
-        sb.append((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.TRUE ||
-            propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH)? "t" : "_");
+        sb.append((tokenisationMode == IndexTokenisationMode.TRUE ||
+            tokenisationMode == IndexTokenisationMode.BOTH)? "t" : "_");
 
-        sb.append((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE ||
-            propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH ||
+        sb.append((tokenisationMode == IndexTokenisationMode.FALSE ||
+            tokenisationMode == IndexTokenisationMode.BOTH ||
             isIdentifierTextProperty(propertyDefinition.getName()))? "s" : "_");
 
         sb.append((crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) ||
@@ -1194,6 +1208,7 @@ public class AlfrescoSolrDataModel implements QueryConstants
 
         IndexedField indexedField = new IndexedField();
         PropertyDefinition propertyDefinition = getPropertyDefinition(propertyQName);
+
         if((propertyDefinition == null))
         {
             return indexedField;
@@ -1204,10 +1219,11 @@ public class AlfrescoSolrDataModel implements QueryConstants
         }
 
         DataTypeDefinition dataTypeDefinition = propertyDefinition.getDataType();
+        IndexTokenisationMode tokenisationMode = getIndexTokenisationMode(propertyDefinition);
         if(isTextField(propertyDefinition))
         {
-            if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.TRUE)
-                    || (propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH))
+            if ((tokenisationMode == IndexTokenisationMode.TRUE)
+                    || (tokenisationMode == IndexTokenisationMode.BOTH))
             {
                 indexedField.addField(getFieldForText(true, true, false, propertyDefinition), true, false);
                 if(crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) || crossLocaleSearchProperties.contains(propertyDefinition.getName()))
@@ -1216,8 +1232,8 @@ public class AlfrescoSolrDataModel implements QueryConstants
                 }
             }
 
-            if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE)
-                    || (propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH
+            if ((tokenisationMode == IndexTokenisationMode.FALSE)
+                    || (tokenisationMode == IndexTokenisationMode.BOTH
                     || isIdentifierTextProperty(propertyDefinition.getName())))
             {
                 indexedField.addField(getFieldForText(true, false, false, propertyDefinition), true, false);
@@ -1226,8 +1242,8 @@ public class AlfrescoSolrDataModel implements QueryConstants
 
             if(dataTypeDefinition.getName().equals(DataTypeDefinition.TEXT))
             {
-                if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE)
-                        || (propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH))
+                if ((tokenisationMode == IndexTokenisationMode.FALSE)
+                        || (tokenisationMode == IndexTokenisationMode.BOTH))
                 {
                     if(!propertyDefinition.isMultiValued())
                     {
@@ -1245,8 +1261,8 @@ public class AlfrescoSolrDataModel implements QueryConstants
 
             if(dataTypeDefinition.getName().equals(DataTypeDefinition.MLTEXT))
             {
-                if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE)
-                        || (propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH))
+                if ((tokenisationMode == IndexTokenisationMode.FALSE)
+                        || (tokenisationMode == IndexTokenisationMode.BOTH))
                 {
                     if(!propertyDefinition.isMultiValued())
                     {
@@ -1273,6 +1289,19 @@ public class AlfrescoSolrDataModel implements QueryConstants
         return identifierProperties.contains(propertyQName);
     }
 
+    /*
+     * Override the model index tokenization mode as tokenised=TRUE if the property is configured in alfresco.tokenise.property in the
+     * shared properties file.
+     */
+    private IndexTokenisationMode getIndexTokenisationMode(PropertyDefinition propertyDefinition)
+    {
+        if (tokeniseProperties.contains(propertyDefinition.getName()))
+        {
+            return IndexTokenisationMode.TRUE;
+        }
+        return propertyDefinition.getIndexTokenisationMode();
+    }
+
     public boolean isTextField(PropertyDefinition propertyDefinition)
     {
         return ofNullable(propertyDefinition)
@@ -1298,7 +1327,6 @@ public class AlfrescoSolrDataModel implements QueryConstants
 
     private boolean hasDocValues(PropertyDefinition propertyDefinition)
     {
-
         if(isTextField(propertyDefinition))
         {
             // We only call this if text is untokenised and localised
@@ -1316,7 +1344,7 @@ public class AlfrescoSolrDataModel implements QueryConstants
             }
             else
             {
-                if(propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.TRUE)
+                if(getIndexTokenisationMode(propertyDefinition) == IndexTokenisationMode.TRUE)
                 {
                     return false;
                 }

search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerOfResources.java

@@ -46,6 +46,7 @@ import org.apache.solr.common.params.SolrParams;
 public class HandlerOfResources {
 
     protected static final List<String> DISALLOWED_SHARED_UPDATES = Arrays.asList("alfresco.identifier.property.",
+            "alfresco.tokenise.property.",
             "alfresco.suggestable.property.",
             "alfresco.cross.locale.property.",
             "alfresco.cross.locale.datatype.");

search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties

@@ -15,6 +15,10 @@ alfresco.identifier.property.2={http://www.alfresco.org/model/content/1.0}userNa
 alfresco.identifier.property.3={http://www.alfresco.org/model/content/1.0}authorityName
 alfresco.identifier.property.4={http://www.alfresco.org/model/content/1.0}lockOwner
 
+# Properties treated as tokenised=true when indexed (model override)
+#alfresco.tokenise.property.0={http://www.alfresco.org/model/content/1.0}creator
+#alfresco.tokenise.property.1={http://www.alfresco.org/model/content/1.0}modifier
+
 # Suggestable Propeties
 #alfresco.suggestable.property.0={http://www.alfresco.org/model/content/1.0}name
 #alfresco.suggestable.property.1={http://www.alfresco.org/model/content/1.0}title

search-services/alfresco-search/src/test/java/org/alfresco/solr/HandlerOfResourcesTest.java

@@ -62,6 +62,10 @@ public class HandlerOfResourcesTest
         assertFalse(HandlerOfResources.allowedProperties(props, HandlerOfResources.DISALLOWED_SHARED_UPDATES));
         props.remove("alfresco.identifier.property.0");
 
+        props.setProperty("alfresco.tokenise.property.0", "xy");
+        assertFalse(HandlerOfResources.allowedProperties(props, HandlerOfResources.DISALLOWED_SHARED_UPDATES));
+        props.remove("alfresco.tokenise.property.0");
+
         props.setProperty("alfresco.suggestable.property.1", "xy");
         assertFalse(HandlerOfResources.allowedProperties(props, HandlerOfResources.DISALLOWED_SHARED_UPDATES));
         props.remove("alfresco.suggestable.property.1");

search-services/alfresco-search/src/test/java/org/alfresco/solr/TokenisationOverrideIT.java

@@ -0,0 +1,152 @@
+/*
+ * #%L
+ * Alfresco Search Services
+ * %%
+ * Copyright (C) 2005 - 2020 Alfresco Software Limited
+ * %%
+ * This file is part of the Alfresco software.
+ * If the software was purchased under a paid Alfresco license, the terms of
+ * the paid license agreement will prevail.  Otherwise, the software is
+ * provided under the following open source license terms:
+ *
+ * Alfresco is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Alfresco is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
+ * #L%
+ */
+
+package org.alfresco.solr;
+
+import static java.util.Collections.singletonList;
+import static org.alfresco.solr.AlfrescoSolrUtils.ancestors;
+import static org.alfresco.solr.AlfrescoSolrUtils.getAcl;
+import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet;
+import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders;
+import static org.alfresco.solr.AlfrescoSolrUtils.getNode;
+import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData;
+import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction;
+import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet;
+import static org.carrot2.shaded.guava.common.collect.ImmutableList.of;
+
+import java.lang.reflect.Field;
+import java.util.HashSet;
+
+import org.alfresco.service.namespace.QName;
+import org.alfresco.solr.client.Acl;
+import org.alfresco.solr.client.AclChangeSet;
+import org.alfresco.solr.client.AclReaders;
+import org.alfresco.solr.client.Node;
+import org.alfresco.solr.client.NodeMetaData;
+import org.alfresco.solr.client.StringPropertyValue;
+import org.alfresco.solr.client.Transaction;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.TermQuery;
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Verifies that the alfresco.tokenise.property.* override forces
+ * tokenised=TRUE at index time, regardless of the model's own tokenisation setting.
+ */
+@SolrTestCaseJ4.SuppressSSL
+public class TokenisationOverrideIT extends AbstractAlfrescoDistributedIT
+{
+    private static final QName PROP = QName.createQName("custom", "untokenisedProp");
+    private static final QName PROP_OVERRIDE = QName.createQName("custom", "untokenisedPropOverride");
+    private static final QName ASPECT = QName.createQName("custom", "tokenisationOverrideTestAspect");
+
+    private static final String TOKENISED_FIELD_OVERRIDE = "text@s__lt@{custom}untokenisedPropOverride";
+    private static final String TOKENISED_FIELD_BASE     = "text@s__lt@{custom}untokenisedProp";
+
+    static final String PROP_VALUE = "The sky is very blue";
+    private static final int TIMEOUT = 100000;
+
+    private static HashSet<QName> savedTokeniseProperties;
+
+    @BeforeClass
+    public static void initData() throws Throwable
+    {
+        initSolrServers(1, TokenisationOverrideIT.class.getSimpleName(), null);
+        modifyAlfrescoSolrDataModel();
+        indexNodes();
+    }
+
+    private static void modifyAlfrescoSolrDataModel() throws NoSuchFieldException, IllegalAccessException
+    {
+        Field tokenisePropertiesField = AlfrescoSolrDataModel
+                .getInstance()
+                .getClass()
+                .getDeclaredField("tokeniseProperties");
+
+        tokenisePropertiesField.setAccessible(true);
+
+        HashSet<QName> tokeniseProperties = (HashSet<QName>) tokenisePropertiesField.get(AlfrescoSolrDataModel.getInstance());
+        savedTokeniseProperties = (HashSet<QName>) tokeniseProperties.clone();
+        tokeniseProperties.add(PROP_OVERRIDE);
+    }
+
+    private static void indexNodes() throws Exception
+    {
+        AclChangeSet aclChangeSet = getAclChangeSet(1);
+
+        Acl acl = getAcl(aclChangeSet);
+        AclReaders aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null);
+
+        indexAclChangeSet(aclChangeSet,
+                of(acl),
+                of(aclReaders));
+
+        Transaction txn = getTransaction(0, 2);
+
+        Node parentFolder = getNode(0, txn, acl, Node.SolrApiNodeStatus.UPDATED);
+        NodeMetaData parentFolderMetadata = getNodeMetaData(parentFolder, txn, acl, "joel", null, false);
+
+        Node testNode = getNode(1, txn, acl, Node.SolrApiNodeStatus.UPDATED);
+        NodeMetaData testNodeMetadata = getNodeMetaData(testNode, txn, acl, "joel",
+                ancestors(parentFolderMetadata.getNodeRef()), false);
+
+        testNodeMetadata.getAspects().add(ASPECT);
+        testNodeMetadata.getProperties().put(PROP, new StringPropertyValue(PROP_VALUE));
+        testNodeMetadata.getProperties().put(PROP_OVERRIDE, new StringPropertyValue(PROP_VALUE));
+
+        indexTransaction(txn,
+                of(parentFolder, testNode),
+                of(parentFolderMetadata, testNodeMetadata));
+    }
+
+    @AfterClass
+    public static void destroyData() throws NoSuchFieldException, IllegalAccessException
+    {
+        restoreAlfrescoSolrDataModel();
+        dismissSolrServers();
+    }
+
+    private static void restoreAlfrescoSolrDataModel() throws NoSuchFieldException, IllegalAccessException
+    {
+        Field tokenisePropertiesField = AlfrescoSolrDataModel
+                .getInstance()
+                .getClass()
+                .getDeclaredField("tokeniseProperties");
+
+        tokenisePropertiesField.setAccessible(true);
+        tokenisePropertiesField.set(AlfrescoSolrDataModel.getInstance(), savedTokeniseProperties);
+    }
+
+    @Test
+    public void tokenisationOverride_overriddenPropertyIsTokenised_basePropertyIsNot() throws Exception
+    {
+        waitForDocCount(new TermQuery(new Term(TOKENISED_FIELD_OVERRIDE, "blue")), 1, TIMEOUT);
+        waitForDocCount(new TermQuery(new Term(TOKENISED_FIELD_BASE, "blue")), 0, TIMEOUT);
+    }
+}

search-services/alfresco-search/src/test/resources/test-files/alfrescoModels/tokenisation-override-test.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<model xmlns="http://www.alfresco.org/model/dictionary/1.0" name="custom:tokenisationOverrideTest">
+    <imports>
+        <import uri="http://www.alfresco.org/model/dictionary/1.0" prefix="d"/>
+    </imports>
+    <namespaces>
+        <namespace uri="custom" prefix="custom"/>
+    </namespaces>
+    <data-types/>
+    <constraints/>
+    <types/>
+    <aspects>
+        <aspect name="custom:tokenisationOverrideTestAspect">
+            <title>Tokenisation Override Test Aspect</title>
+            <properties>
+                <property name="custom:untokenisedProp">
+                    <title>Untokenised Property</title>
+                    <type>d:text</type>
+                    <mandatory>false</mandatory>
+                    <index enabled="true">
+                        <tokenised>FALSE</tokenised>
+                    </index>
+                </property>
+                <property name="custom:untokenisedPropOverride">
+                    <title>Untokenised Property (to be overridden)</title>
+                    <type>d:text</type>
+                    <mandatory>false</mandatory>
+                    <index enabled="true">
+                        <tokenised>FALSE</tokenised>
+                    </index>
+                </property>
+            </properties>
+            <associations/>
+            <overrides/>
+            <mandatory-aspects/>
+        </aspect>
+    </aspects>
+</model>

search-services/alfresco-search/src/test/resources/test-files/conf/shared.properties

@@ -15,6 +15,10 @@ alfresco.identifier.property.2={http://www.alfresco.org/model/content/1.0}userNa
 alfresco.identifier.property.3={http://www.alfresco.org/model/content/1.0}authorityName
 alfresco.identifier.property.4={http://www.alfresco.org/model/content/1.0}lockOwner
 
+# Properties treated as tokenised=true when indexed (model override)
+#alfresco.tokenise.property.0={http://www.alfresco.org/model/content/1.0}creator
+#alfresco.tokenise.property.1={http://www.alfresco.org/model/content/1.0}modifier
+
 # Suggestable Propeties
 alfresco.suggestable.property.0={http://www.alfresco.org/model/content/1.0}name
 alfresco.suggestable.property.1={http://www.alfresco.org/model/content/1.0}title