[maven-release-plugin] prepare release 1.4.1-RC2

Feature/search 2012 null docs with field 14x

See merge request search_discovery/insightengine!280

README.md

@@ -40,4 +40,4 @@ More details are available at [search-services](/search-services) folder.
 
 **Following resources will not be available for Community users**
 
-More details are available at [insight-engine](/insight-engine) folder.
+More details are available at [insight-engine](/insight-engine) folder.

e2e-test/pom.xml

@@ -1,10 +1,9 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
       <groupId>org.alfresco</groupId>
       <artifactId>alfresco-search-and-insight-parent</artifactId>
-      <version>1.5.0-SNAPSHOT</version>
+      <version>1.4.1-RC2</version>
    </parent>
    <groupId>search-analytics-e2e-test</groupId>
    <artifactId>search-analytics-e2e-test</artifactId>
@@ -16,8 +15,8 @@
       <tas.utility.version>3.0.16</tas.utility.version>
       <rm.version>3.2.0</rm.version>
       <suiteXmlFile>src/test/resources/SearchSuite.xml</suiteXmlFile>
-      <test.exclude></test.exclude>
-      <test.include></test.include>
+      <test.exclude />
+      <test.include />
       <jackson.databind.version>2.7.7</jackson.databind.version>
    </properties>
    <build>
@@ -128,4 +127,4 @@
          <version>12-ea+10</version>
       </dependency>
    </dependencies>
-</project>
+</project>

pom.xml

@@ -7,7 +7,7 @@
       <version>10</version>
    </parent>
    <artifactId>alfresco-search-and-insight-parent</artifactId>
-   <version>1.5.0-SNAPSHOT</version>
+   <version>1.4.1-RC2</version>
    <packaging>pom</packaging>
    <name>Alfresco Search And Insight Parent</name>
    <distributionManagement>
@@ -24,7 +24,7 @@
       <connection>scm:git:https://git.alfresco.com/search_discovery/insightengine.git</connection>
       <developerConnection>scm:git:https://git.alfresco.com/search_discovery/insightengine.git</developerConnection>
       <url>https://git.alfresco.com/search_discovery/insightengine.git</url>
-      <tag>HEAD</tag>
+      <tag>1.4.1-RC2</tag>
    </scm>
    <properties>
       <java.version>11</java.version>

search-services/README.md

@@ -338,6 +338,22 @@ This Docker Image is available at Alfresco Docker Hub:
 To use the public image instead of the local one (`searchservices:develop`) just use `alfresco/alfresco-search-services:1.3.x.x` labels.
 
 
+## Docker Master-Slave setup
+### Enable Search Slave Replica config
+
+To enable slave node specify environment value `REPLICATION_TYPE=slave`, by default Master config is enabled and slave is disabled. 
+
+During deployment time whenever Search Services or Insight Engine image starts, it will execute the script [search_config_setup.sh](/packaging/src/docker) which will configure the slave config setup based on the value specified in the script.
+
+To run the docker image:
+
+```bash
+$ docker run -p 8984:8983 -e REPLICATION_TYPE=slave -e ALFRESCO_SECURE_COMMS=none -e SOLR_CREATE_ALFRESCO_DEFAULTS=alfresco,archive searchservices:develop
+```
+Solr-slave End point: [http://localhost:8984/solr](http://localhost:8984/solr)
+
+To generate your own Docker-compose file please follow [generator-alfresco-docker-compose](../e2e-test/generator-alfresco-docker-compose/README.md)
+
 ### Use Alfresco Search Services Docker Image with Docker Compose
 
 Sample configuration in a Docker Compose file using **Plain HTTP** protocol to communicate with Alfresco Repository.

search-services/alfresco-search/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.alfresco</groupId>
         <artifactId>alfresco-search-parent</artifactId>
-        <version>1.5.0-SNAPSHOT</version>
+        <version>1.4.1-RC2</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -15,7 +15,7 @@
         <dependency>
             <groupId>org.alfresco</groupId>
             <artifactId>alfresco-solrclient-lib</artifactId>
-            <version>1.5.0-SNAPSHOT</version>
+            <version>1.4.1-RC2</version>
             <exclusions>
                 <exclusion>
                     <artifactId>servlet-api</artifactId>

search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCollatableMLTextFieldType.java

@@ -39,12 +39,9 @@ import org.springframework.extensions.surf.util.I18NUtil;
 
 /**
  * @author Andy
- *
  */
 public class AlfrescoCollatableMLTextFieldType extends StrField
 {
-    
-
     /* (non-Javadoc)
      * @see org.apache.solr.schema.StrField#getSortField(org.apache.solr.schema.SchemaField, boolean)
      */
@@ -75,7 +72,6 @@ public class AlfrescoCollatableMLTextFieldType extends StrField
     }
 
 
-    
     public static class MLTextSortFieldComparatorSource extends FieldComparatorSource
     {
 
@@ -101,16 +97,20 @@ public class AlfrescoCollatableMLTextFieldType extends StrField
         private final String[] values;
 
         private BinaryDocValues docTerms;
-        
-        private Bits docsWithField;
+
+        /**
+         * An array of flags - one for each document in the segment. Each bit is set to true if the document has the
+         * field or false otherwise. If this is set to null then all docs in the segment have the field.
+         */
+        Bits docsWithField;
 
         private final String field;
 
-        final Collator collator;
+        Collator collator;
 
-        private String bottom;
-        
-        private String top;
+        String bottom;
+
+        String top;
 
         Locale collatorLocale;
 
@@ -138,7 +138,7 @@ public class AlfrescoCollatableMLTextFieldType extends StrField
         {
             final String comparableString = findBestValue(doc, docTerms.get(doc));
             return compareValues(bottom, comparableString);
-           
+
         }
 
         public void copy(int slot, int doc)
@@ -153,13 +153,14 @@ public class AlfrescoCollatableMLTextFieldType extends StrField
 
         private String findBestValue(int doc, BytesRef term)
         {
-            if (term.length == 0 && docsWithField.get(doc) == false) {
+            if (term.length == 0 && docsWithField != null && docsWithField.get(doc) == false)
+            {
                 return null;
             }
-            
+
             String withLocale = term.utf8ToString();
-            
-            // split strin into MLText object
+
+            // split string into MLText object
             if (withLocale == null)
             {
                 return withLocale;
@@ -231,14 +232,15 @@ public class AlfrescoCollatableMLTextFieldType extends StrField
         {
             docTerms = DocValues.getBinary(context.reader(), field);
             docsWithField = DocValues.getDocsWithField(context.reader(), field);
-            if (docsWithField instanceof Bits.MatchAllBits) {
-              docsWithField = null;
+            if (docsWithField instanceof Bits.MatchAllBits)
+            {
+                docsWithField = null;
             }
             return this;
         }
-        
+
         @Override
-        public int compareValues(String val1, String val2) 
+        public int compareValues(String val1, String val2)
         {
             if (val1 == null)
             {
@@ -254,9 +256,10 @@ public class AlfrescoCollatableMLTextFieldType extends StrField
             }
             return collator.compare(val1, val2);
         }
-        
-        @Override
-        public void setScorer(Scorer scorer) {}
-    }
 
+        @Override
+        public void setScorer(Scorer scorer)
+        {
+        }
+    }
 }

search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCollatableTextFieldType.java

@@ -104,16 +104,20 @@ public class AlfrescoCollatableTextFieldType extends StrField
         private final String[] values;
 
         private BinaryDocValues docTerms;
-        
-        private Bits docsWithField;
+
+        /**
+         * An array of flags - one for each document in the segment. Each bit is set to true if the document has the
+         * field or false otherwise. If this is set to null then all docs in the segment have the field.
+         */
+        Bits docsWithField;
 
         private final String field;
 
-        final Collator collator;
+        Collator collator;
 
-        private String bottom;
+        String bottom;
         
-        private String top;
+        String top;
 
         Locale collatorLocale;
 
@@ -141,7 +145,6 @@ public class AlfrescoCollatableTextFieldType extends StrField
         {
             final String comparableString = findBestValue(doc, docTerms.get(doc));
             return compareValues(bottom, comparableString);
-           
         }
 
         public void copy(int slot, int doc)
@@ -156,7 +159,8 @@ public class AlfrescoCollatableTextFieldType extends StrField
 
         private String findBestValue(int doc, BytesRef term)
         {
-            if (term.length == 0 && docsWithField.get(doc) == false) {
+            if (term.length == 0 && docsWithField != null && docsWithField.get(doc) == false)
+            {
                 return null;
             }
             

search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java

@@ -43,12 +43,14 @@ import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.function.Predicate;
+import java.util.stream.Stream;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
@@ -236,7 +238,8 @@ public final class SolrContentStore implements Closeable, AccessMode
         {
             try
             {
-                return Files.lines(Paths.get(root, ".version"))
+                return Files.readAllLines(Paths.get(root, ".version"))
+                        .stream()
                         .map(Long::parseLong)
                         .findFirst()
                         .orElse(NO_VERSION_AVAILABLE);
@@ -250,18 +253,29 @@ public final class SolrContentStore implements Closeable, AccessMode
         @Override
         public void setLastCommittedVersion(long version)
         {
+
+            File tmpFile = new File(root, ".version-" + new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date()));
             try
             {
-                File tmpFile = new File(root, ".version-" + new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date()));
                 FileWriter wr = new FileWriter(tmpFile);
                 wr.write(Long.toString(version));
                 wr.close();
 
-                tmpFile.renameTo(new File(root, ".version"));
+                // file.renameTo(..) does not work on windows. Use Files.move instead.
+                Files.move(tmpFile.toPath(), new File(root, ".version").toPath(), StandardCopyOption.ATOMIC_MOVE);
+
             }
             catch (IOException exception)
             {
                 logger.error("Unable to persist the last committed content store version {}. See the stacktrace below for furtger details.", version, exception);
+                try
+                {
+                    Files.delete(tmpFile.toPath());
+                }
+                catch (IOException e)
+                {
+                    logger.error("Unable to delete tmp contentstore version file {}.", version);
+                }
             }
         }
 

search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java

@@ -37,6 +37,7 @@ package org.alfresco.solr.handler;
 import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 import org.alfresco.solr.content.SolrContentStore;
+import org.apache.commons.io.FilenameUtils;
 import org.apache.http.client.HttpClient;
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.index.IndexCommit;
@@ -188,6 +189,13 @@ class AlfrescoIndexFetcher
     private final Map<String, FileInfo> confFileInfoCache = new HashMap<>();
     private volatile Date replicationStartTimeStamp;
     private RTimer replicationTimer;
+
+    /**
+     * The map<String, object> contains the following fields:
+     *      NAME : String -> file name(with path for contentstore files)
+     *      SIZE : long -> file size
+     *      CHECKSUM : long -> checksum
+     */
     private volatile List<Map<String, Object>> filesToDownload;
     private volatile List<Map<String, Object>> confFilesToDownload;
     private volatile List<Map<String, Object>> tlogFilesToDownload;
@@ -197,6 +205,7 @@ class AlfrescoIndexFetcher
     private volatile List<Map<String, Object>> confFilesDownloaded;
     private volatile List<Map<String, Object>> tlogFilesDownloaded;
     private volatile List<Map<String, Object>> contentStoreFilesDownloaded;
+
     private volatile Map<String, Object> currentFile;
     private volatile DirectoryFileFetcher dirFileFetcher;
     private volatile LocalFsFileFetcher localFileFetcher;
@@ -467,6 +476,8 @@ class AlfrescoIndexFetcher
             Map<String, List<Map<String, Object>>> contentStoreMap = (Map<String, List<Map<String, Object>>>) response
                     .get(CONTENT_STORE_FILES);
 
+            fullContentStoreReplication = false;
+
             if (contentStoreMap != null)
             {
                 contentStoreFilesToDownload = Collections.synchronizedList(contentStoreMap.get(SolrContentStore.ADDS));
@@ -1703,7 +1714,7 @@ class AlfrescoIndexFetcher
                 try
                 {
                     Files.createDirectories(Paths.get(csFile.getParent()));
-                    Files.copy(tmpFile.toPath(), csFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+                    Files.move(tmpFile.toPath(), csFile.toPath(), StandardCopyOption.ATOMIC_MOVE);
                 }
                 catch (IOException e)
                 {
@@ -1736,13 +1747,20 @@ class AlfrescoIndexFetcher
     private void cleanUpContentStore(String contentStorePath) throws Exception
     {
         AtomicInteger fileDeleted = new AtomicInteger();
-        Set<String> fileNames = contentStoreFilesToDownload.stream().map(e -> (String) e.get(NAME))
+
+        // This is the set of the ONLY files that should be in contentStore.
+        // This set is computed from the information got from master. After a full replication, only the files
+        // that have been downloaded from master (contentStoreFilesToDownload) should be in contentStore.
+        // The file paths are translated in the current OS path notation.
+        Set<String> contentStoreFiles = contentStoreFilesToDownload.stream()
+                .map(e -> (String) e.get(NAME))
+                .map(FilenameUtils::separatorsToSystem)
                 .collect(Collectors.toSet());
         try
         {
             Files.walk(Paths.get(contentStorePath)).forEach(p -> {
                 File f = new File(p.toUri());
-                if (!f.isDirectory() && !fileNames.contains(p.toString().replace(contentStorePath, "")))
+                if (!f.isDirectory() && !contentStoreFiles.contains(p.toString().replace(contentStorePath, "")))
                 {
                     try
                     {

search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoReplicationHandler.java

@@ -2122,6 +2122,7 @@ public class AlfrescoReplicationHandler extends RequestHandlerBase implements So
                             e.printStackTrace();
                         }
                     }
+
                 }
             }
             catch (Exception e)
@@ -2169,6 +2170,7 @@ public class AlfrescoReplicationHandler extends RequestHandlerBase implements So
                         if (bytesRead <= 0)
                         {
                             writeNothingAndFlush();
+                            inputStream.close();
                             break;
                         }
 
@@ -2185,6 +2187,7 @@ public class AlfrescoReplicationHandler extends RequestHandlerBase implements So
                         fos.flush();
                     }
                 }
+
             }
         }
     }

search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCollatableMLTextFieldTypeTest.java

@@ -0,0 +1,200 @@
+/*
+ * Copyright (C) 2005-2014 Alfresco Software Limited.
+ *
+ * This file is part of Alfresco
+ *
+ * Alfresco is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Alfresco is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.alfresco.solr;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.MockitoAnnotations.initMocks;
+
+import java.text.Collator;
+import java.util.Locale;
+
+import org.alfresco.solr.AlfrescoCollatableMLTextFieldType.MLTextSortFieldComparator;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+
+/** Unit tests for {@link AlfrescoCollatableMLTextFieldType}. */
+public class AlfrescoCollatableMLTextFieldTypeTest
+{
+    private static final int NUM_HITS = 3;
+    private static final String FIELD = "field";
+    private static final Locale LOCALE = Locale.getDefault();
+    /** A document id. */
+    private static final int DOC = 0;
+    /** A value for the current bottom document. */
+    private static final String BOTTOM_STRING = "Bottom";
+
+    @InjectMocks
+    MLTextSortFieldComparator textSortFieldComparator = new MLTextSortFieldComparator(NUM_HITS, FIELD, LOCALE);
+    @Mock
+    BinaryDocValues mockDocTerms;
+    @Mock
+    Bits mockDocsWithField;
+    @Mock
+    Collator mockCollator;
+
+    @Before
+    public void setUp()
+    {
+        initMocks(this);
+        reset(mockDocTerms, mockDocsWithField);
+        textSortFieldComparator.bottom = BOTTOM_STRING;
+    }
+
+    /** Check that a zero length term is sorted before a populated field. */
+    @Test
+    public void testCompareBottom_termLengthZeroAndDocDoesntHaveField()
+    {
+        // Set up the document to have an empty term.
+        when(mockDocTerms.get(DOC)).thenReturn(new BytesRef());
+        when(mockDocsWithField.get(DOC)).thenReturn(false);
+
+        // Call the method under test.
+        int result = textSortFieldComparator.compareBottom(DOC);
+
+        assertEquals("Expected value for doc to be null, and so it to be sorted before BOTTOM_TERM", 1, result);
+    }
+
+    /**
+     * Check the behaviour of compareBottom when docsWithField is null (this happens when all documents contain the
+     * field).
+     */
+    @Test
+    public void testCompareBottom_nullDocsWithField()
+    {
+        // Set docsWithField to null to simulate all documents containing the field.
+        Bits oldValue = textSortFieldComparator.docsWithField;
+        textSortFieldComparator.docsWithField = null;
+
+        // Set up the document to have an empty term.
+        when(mockDocTerms.get(DOC)).thenReturn(new BytesRef());
+
+        // Call the method under test.
+        textSortFieldComparator.compareBottom(DOC);
+
+        // Expect the EMPTY_TERM to be compared
+        verify(mockCollator).compare(BOTTOM_STRING, "");
+
+        // Reset docsWithField with the mock after the test.
+        textSortFieldComparator.docsWithField = oldValue;
+    }
+
+    /** Check that if the doc has a value then it is compared with the existing value. */
+    @Test
+    public void testCompareBottom_populatedTerm()
+    {
+        // Set up the document to have "Some value" for the field.
+        when(mockDocTerms.get(DOC)).thenReturn(new BytesRef("Some value"));
+        when(mockDocsWithField.get(DOC)).thenReturn(false);
+
+        // Call the method under test.
+        textSortFieldComparator.compareBottom(DOC);
+
+        verify(mockCollator).compare(BOTTOM_STRING, "Some value");
+    }
+
+    /** Check the behaviour if the multilanguage term is encoded. */
+    @Test
+    public void testCompareBottom_encodedTerm_localeFound()
+    {
+        // Create an encoded multilanguage string with Russian, US English and Thai with Thai digits.
+        String mlText = "\u0000ru\u0000First\u0000Ignored" +
+                "\u0000en_US\u0000Second\u0000IgnoredToo" +
+                "\u0000th_TH_TH\u0000Third\u0000AlsoIgnored";
+        // Set up the document to have an encoded value for the field.
+        when(mockDocTerms.get(DOC)).thenReturn(new BytesRef(mlText));
+        when(mockDocsWithField.get(DOC)).thenReturn(false);
+
+        // Check that the Russian text can be extracted.
+        textSortFieldComparator.collatorLocale = Locale.forLanguageTag("ru");
+        textSortFieldComparator.compareBottom(DOC);
+        verify(mockCollator).compare(BOTTOM_STRING, "First");
+
+        // Check that the English text can be extracted.
+        textSortFieldComparator.collatorLocale = Locale.forLanguageTag("en");
+        textSortFieldComparator.compareBottom(DOC);
+        verify(mockCollator).compare(BOTTOM_STRING, "Second");
+
+        // Check that the Thai text can be extracted.
+        textSortFieldComparator.collatorLocale = Locale.forLanguageTag("th");
+        textSortFieldComparator.compareBottom(DOC);
+        verify(mockCollator).compare(BOTTOM_STRING, "Third");
+
+        // Reset the locale for other tests.
+        textSortFieldComparator.collatorLocale = LOCALE;
+    }
+
+    /** Check the behaviour if the term has a locale but no text. */
+    @Test
+    public void testCompareBottom_badlyEncodedTerm()
+    {
+        // Set the value to have a locale but no text.
+        String mlText = "\u0000ru";
+        when(mockDocTerms.get(DOC)).thenReturn(new BytesRef(mlText));
+
+        // Call the method under test.
+        textSortFieldComparator.compareBottom(DOC);
+
+        // Check that an empty string is assumed.
+        verify(mockCollator).compare(BOTTOM_STRING, "");
+    }
+
+    @Test
+    public void testCompareValues_nullLessThanString()
+    {
+        int result = textSortFieldComparator.compareValues(null, "NotNull");
+        assertEquals("Expected null to be 'less' than string.", -1, result);
+    }
+
+    @Test
+    public void testCompareValues_stringGreaterThanNull()
+    {
+        int result = textSortFieldComparator.compareValues("NotNull", null);
+        assertEquals("Expected string to be 'greater' than null.", 1, result);
+    }
+
+    @Test
+    public void testCompareValues_nullEqualToNull()
+    {
+        int result = textSortFieldComparator.compareValues(null, null);
+        assertEquals("Expected two null values to be equal.", 0, result);
+    }
+
+    /** Check that when two non-null strings are compared then the underlying collator is used to get the result. */
+    @Test
+    public void testCompareValues_twoStringsCompared()
+    {
+        // An arbitrary value to be returned by the collator.
+        int comparisonResult = 10;
+        when(mockCollator.compare("NotNull1", "NotNull2")).thenReturn(comparisonResult);
+
+        // Call the method under test.
+        int result = textSortFieldComparator.compareValues("NotNull1", "NotNull2");
+
+        verify(mockCollator).compare("NotNull1", "NotNull2");
+        assertEquals("Expected result to be obtained from collator.", comparisonResult, result);
+    }
+}

search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCollatableTextFieldTypeTest.java

@@ -0,0 +1,168 @@
+/*
+ * Copyright (C) 2005-2014 Alfresco Software Limited.
+ *
+ * This file is part of Alfresco
+ *
+ * Alfresco is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Alfresco is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.alfresco.solr;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.MockitoAnnotations.initMocks;
+
+import java.text.Collator;
+import java.util.Locale;
+
+import org.alfresco.solr.AlfrescoCollatableTextFieldType.TextSortFieldComparator;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+
+/** Unit tests for {@link AlfrescoCollatableTextFieldType}. */
+public class AlfrescoCollatableTextFieldTypeTest
+{
+    private static final int NUM_HITS = 3;
+    private static final String FIELD = "field";
+    private static final Locale LOCALE = Locale.getDefault();
+    /** A document id. */
+    private static final int DOC = 0;
+    /** A value for the current bottom document. */
+    private static final String BOTTOM_STRING = "Bottom";
+
+    @InjectMocks
+    TextSortFieldComparator textSortFieldComparator = new TextSortFieldComparator(NUM_HITS, FIELD, LOCALE);
+    @Mock
+    BinaryDocValues mockDocTerms;
+    @Mock
+    Bits mockDocsWithField;
+    @Mock
+    Collator mockCollator;
+
+    @Before
+    public void setUp()
+    {
+        initMocks(this);
+        reset(mockDocTerms, mockDocsWithField);
+        textSortFieldComparator.bottom = BOTTOM_STRING;
+    }
+
+    /** Check that a zero length term is sorted before a populated field. */
+    @Test
+    public void testCompareBottom_termLengthZeroAndDocDoesntHaveField()
+    {
+        // Set up the document to have an empty term.
+        when(mockDocTerms.get(DOC)).thenReturn(new BytesRef());
+        when(mockDocsWithField.get(DOC)).thenReturn(false);
+
+        // Call the method under test.
+        int result = textSortFieldComparator.compareBottom(DOC);
+
+        assertEquals("Expected value for doc to be null, and so it to be sorted before BOTTOM_TERM", 1, result);
+    }
+
+    /**
+     * Check the behaviour of compareBottom when docsWithField is null (this happens when all documents contain the
+     * field).
+     */
+    @Test
+    public void testCompareBottom_nullDocsWithField()
+    {
+        // Set docsWithField to null to simulate all documents containing the field.
+        Bits oldValue = textSortFieldComparator.docsWithField;
+        textSortFieldComparator.docsWithField = null;
+
+        // Set up the document to have an empty term.
+        when(mockDocTerms.get(DOC)).thenReturn(new BytesRef());
+
+        // Call the method under test.
+        textSortFieldComparator.compareBottom(DOC);
+
+        // Expect the EMPTY_TERM to be compared
+        verify(mockCollator).compare(BOTTOM_STRING, "");
+
+        // Reset docsWithField with the mock after the test.
+        textSortFieldComparator.docsWithField = oldValue;
+    }
+
+    /** Check that if the doc has a value then it is compared with the existing value. */
+    @Test
+    public void testCompareBottom_populatedTerm()
+    {
+        // Set up the document to have "Some value" for the field.
+        when(mockDocTerms.get(DOC)).thenReturn(new BytesRef("Some value"));
+        when(mockDocsWithField.get(DOC)).thenReturn(false);
+
+        // Call the method under test.
+        textSortFieldComparator.compareBottom(DOC);
+
+        verify(mockCollator).compare(BOTTOM_STRING, "Some value");
+    }
+
+    /** Check the behaviour if the term is encoded. */
+    @Test
+    public void testCompareBottom_encodedTerm()
+    {
+        // Set up the document to have an encoded value for the field.
+        when(mockDocTerms.get(DOC)).thenReturn(new BytesRef("\u0000Value\u0000Ignored"));
+        when(mockDocsWithField.get(DOC)).thenReturn(false);
+
+        // Call the method under test.
+        textSortFieldComparator.compareBottom(DOC);
+
+        verify(mockCollator).compare(BOTTOM_STRING, "Value");
+    }
+
+    @Test
+    public void testCompareValues_nullLessThanString()
+    {
+        int result = textSortFieldComparator.compareValues(null, "NotNull");
+        assertEquals("Expected null to be 'less' than string.", -1, result);
+    }
+
+    @Test
+    public void testCompareValues_stringGreaterThanNull()
+    {
+        int result = textSortFieldComparator.compareValues("NotNull", null);
+        assertEquals("Expected string to be 'greater' than null.", 1, result);
+    }
+
+    @Test
+    public void testCompareValues_nullEqualToNull()
+    {
+        int result = textSortFieldComparator.compareValues(null, null);
+        assertEquals("Expected two null values to be equal.", 0, result);
+    }
+
+    /** Check that when two non-null strings are compared then the underlying collator is used to get the result. */
+    @Test
+    public void testCompareValues_twoStringsCompared()
+    {
+        // An arbitrary value to be returned by the collator.
+        int comparisonResult = 10;
+        when(mockCollator.compare("NotNull1", "NotNull2")).thenReturn(comparisonResult);
+
+        // Call the method under test.
+        int result = textSortFieldComparator.compareValues("NotNull1", "NotNull2");
+
+        verify(mockCollator).compare("NotNull1", "NotNull2");
+        assertEquals("Expected result to be obtained from collator.", comparisonResult, result);
+    }
+}

search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrSortIT.java

@@ -0,0 +1,108 @@
+/*
+ * Copyright (C) 2005-2014 Alfresco Software Limited.
+ *
+ * This file is part of Alfresco
+ *
+ * Alfresco is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Alfresco is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.alfresco.solr.query;
+
+import org.alfresco.solr.AbstractAlfrescoDistributedIT;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.util.NamedList;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.hamcrest.core.Is.is;
+
+/**
+ * https://issues.alfresco.com/jira/browse/SEARCH-2012
+ */
+@SolrTestCaseJ4.SuppressSSL
+@LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"})
+public class AlfrescoSolrSortIT extends AbstractAlfrescoDistributedIT
+{
+    @BeforeClass
+    private static void initData() throws Throwable
+    {
+        initSolrServers(1, getClassName(), null);
+    }
+
+    @AfterClass
+    private static void destroyData()
+    {
+        dismissSolrServers();
+    }
+
+    @After
+    public void clearData() throws Exception
+    {
+        deleteByQueryAllClients("*:*");
+    }
+
+    @Test
+    public void AlfrescoCollatableFieldType_emptyValuesSortingAsc__shouldBeRankedFirst() throws Exception {
+        prepareIndexSegmentWithAllNonNullFieldValues("text@s__sort@{http://www.alfresco.org/model/content/1.0}title");
+        putHandleDefaults();
+        // Docs with id 1, 3, 5 and 6 should be first (note that these will be sorted by indexing time).
+        String[] expectedRanking = new String[]{"1","3","5","6","4","2"};
+        
+        QueryResponse response = query(getDefaultTestClient(), true,
+                "{\"query\":\"(id:(1 2 3 4 5 6))\",\"locales\":[\"en\"], \"templates\": [{\"name\":\"t1\", \"template\":\"%cm:content\"}], \"authorities\": [\"joel\"], \"tenants\": []}",
+                params("qt", "/afts", "shards.qt", "/afts", "start", "0", "rows", "100", "sort", "text@s__sort@{http://www.alfresco.org/model/content/1.0}title asc"));
+
+        NamedList res = response.getResponse();
+        SolrDocumentList searchResults = (SolrDocumentList)res.get("response");
+        for(int i=0;i<searchResults.size();i++){
+            assertThat(searchResults.get(i).get("id"),is(expectedRanking[i]));
+        }
+    }
+
+    @Test
+    public void AlfrescoMLCollatableFieldType_emptyValuesSortingDesc_shouldRankThemLast() throws Exception {
+        prepareIndexSegmentWithAllNonNullFieldValues("mltext@m__sort@{http://www.alfresco.org/model/content/1.0}title");
+
+        putHandleDefaults();
+        // Docs with id 1, 3, 5 and 6 should be last (note that these will be sorted by indexing time).
+        String[] expectedRanking = new String[]{"2","4","1","3","5","6"};
+
+        QueryResponse response = query(getDefaultTestClient(), true,
+                "{\"query\":\"(id:(1 2 3 4 5 6))\",\"locales\":[\"en\"], \"templates\": [{\"name\":\"t1\", \"template\":\"%cm:content\"}], \"authorities\": [\"joel\"], \"tenants\": []}",
+                params("qt", "/afts", "shards.qt", "/afts", "start", "0", "rows", "100", "sort", "mltext@m__sort@{http://www.alfresco.org/model/content/1.0}title desc"));
+
+        NamedList res = response.getResponse();
+        SolrDocumentList searchResults = (SolrDocumentList)res.get("response");
+        for(int i=0;i<searchResults.size();i++){
+            assertThat(searchResults.get(i).get("id"),is(expectedRanking[i]));
+        }
+    }
+
+    /** Create six documents where the field is null for docs with id 1, 3, 5 and 6, and populated for docs with id 2 and 4. */
+    private void prepareIndexSegmentWithAllNonNullFieldValues(String field) throws Exception {
+        index(getDefaultTestClient(), true, "id", "1", "_version_", "0", field, "");
+        index(getDefaultTestClient(), true, "id", "2", "_version_", "0", field, "B");
+        index(getDefaultTestClient(), true, "id", "3", "_version_", "0", field, "");
+        index(getDefaultTestClient(), true, "id", "4", "_version_", "0", field, "A");
+        index(getDefaultTestClient(), true, "id", "5", "_version_", "0", field, "");
+        index(getDefaultTestClient(), true, "id", "6", "_version_", "0", field, "");
+        commit(getDefaultTestClient(), true);
+    }
+}
+

search-services/alfresco-solrclient-lib/pom.xml

@@ -7,7 +7,7 @@
     <parent>
         <groupId>org.alfresco</groupId>
         <artifactId>alfresco-search-parent</artifactId>
-        <version>1.5.0-SNAPSHOT</version>
+        <version>1.4.1-RC2</version>
     </parent>
 
     <distributionManagement>

search-services/packaging/pom.xml

@@ -13,7 +13,7 @@
     <parent>
         <groupId>org.alfresco</groupId>
         <artifactId>alfresco-search-parent</artifactId>
-        <version>1.5.0-SNAPSHOT</version>
+        <version>1.4.1-RC2</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <dependencies>

search-services/packaging/src/docker/6.x/docker-compose.yml

@@ -37,6 +37,10 @@ services:
   search:
     image: quay.io/alfresco/search-services:${SEARCH_TAG}
     environment:
+      #Replication properties
+      - REPLICATION_TYPE=master
+      #- REPLICATION_AFTER=commit,startup- SOLR_ALFRESCO_HOST=alfresco
+      #- REPLICATION_CONFIG_FILES=schema.xml,stopwords.txt- SOLR_ALFRESCO_PORT=8080
       #Solr needs to know how to register itself with Alfresco
       - SOLR_ALFRESCO_HOST=alfresco
       - SOLR_ALFRESCO_PORT=8080
@@ -51,6 +55,27 @@ services:
       - ENABLE_SPELLCHECK=${SEARCH_ENABLE_SPELLCHECK}
     ports:
       - 8083:8983 #Browser port
+  #search_slave:
+  #  image: quay.io/alfresco/search-services:${SEARCH_TAG}
+  #  environment:
+  #    #Replication properties
+  #    - REPLICATION_TYPE=slave
+  #    - REPLICATION_MASTER_HOST=search
+  #    - REPLICATION_MASTER_PORT=8983
+  #    #- REPLICATION_MASTER_PROTOCOL=http
+  #    #- REPLICATION_CORE_NAME=alfresco
+  #    #- REPLICATION_POLL_INTERVAL=00:00:60
+  #    #Solr needs to know how to register itself with Alfresco
+  #    - SOLR_ALFRESCO_HOST=alfresco
+  #    - SOLR_ALFRESCO_PORT=8080
+  #    #Alfresco needs to know how to call solr
+  #    - SOLR_SOLR_HOST=search
+  #    - SOLR_SOLR_PORT=8983
+  #    #Create the default alfresco and archive cores
+  #    - SOLR_CREATE_ALFRESCO_DEFAULTS=alfresco,archive
+  #  ports:
+  #    - 8084:8983 #Browser port
+
   activemq:
     image: alfresco/alfresco-activemq:5.15.6
     ports:

search-services/packaging/src/docker/search_config_setup.sh

@@ -1,5 +1,48 @@
 #!/bin/bash
 set -e
+# By default its going to deploy "Master" setup configuration with "REPLICATION_TYPE=master".
+# Slave replica service can be enabled using "REPLICATION_TYPE=slave" environment value.
+
+SOLR_CONFIG_FILE=$PWD/solrhome/templates/rerank/conf/solrconfig.xml
+if [[ $REPLICATION_TYPE == "master" ]]; then
+
+   findStringMaster='<requestHandler name="\/replication" class="org\.alfresco\.solr\.handler\.AlfrescoReplicationHandler">/<requestHandler name="\/replication" class="org\.alfresco\.solr\.handler\.AlfrescoReplicationHandler">'
+   replaceStringMaster="\n\t<lst name=\"master\"> \n"
+   if [[ $REPLICATION_AFTER == "" ]]; then
+      REPLICATION_AFTER=commit
+   fi
+   for i in $(echo $REPLICATION_AFTER | sed "s/,/ /g")
+   do
+      replaceStringMaster+="\t\t<str name=\"replicateAfter\">"$i"<\/str> \n"
+   done
+   if [[ ! -z "$REPLICATION_CONFIG_FILES" ]]; then
+      replaceStringMaster+="\t\t<str name=\"confFiles\">$REPLICATION_CONFIG_FILES<\/str> \n"
+   fi
+   replaceStringMaster+="\t<\/lst>"
+   sed -i "s/$findStringMaster/$findStringMaster$replaceStringMaster/g" $SOLR_CONFIG_FILE
+fi
+if [[ $REPLICATION_TYPE == "slave" ]]; then
+   if [[ $REPLICATION_MASTER_PROTOCOL == "" ]]; then
+      REPLICATION_MASTER_PROTOCOL=http
+   fi
+   if [[ $REPLICATION_MASTER_HOST == "" ]]; then
+      REPLICATION_MASTER_HOST=localhost
+   fi
+   if [[ $REPLICATION_MASTER_PORT == "" ]]; then
+      REPLICATION_MASTER_PORT=8083
+   fi
+   if [[ $REPLICATION_CORE_NAME == "" ]]; then
+      REPLICATION_CORE_NAME=alfresco
+   fi
+   if [[ $REPLICATION_POLL_INTERVAL == "" ]]; then
+      REPLICATION_POLL_INTERVAL=00:00:30
+   fi
+   sed -i 's/<requestHandler name="\/replication" class="org\.alfresco\.solr\.handler\.AlfrescoReplicationHandler">/<requestHandler name="\/replication" class="org\.alfresco\.solr\.handler\.AlfrescoReplicationHandler">\
+      <lst name="slave">\
+         <str name="masterUrl">'$REPLICATION_MASTER_PROTOCOL':\/\/'$REPLICATION_MASTER_HOST':'$REPLICATION_MASTER_PORT'\/solr\/'$REPLICATION_CORE_NAME'<\/str>\
+         <str name="pollInterval">'$REPLICATION_POLL_INTERVAL'<\/str>\
+      <\/lst>/g' $SOLR_CONFIG_FILE
+fi
 
 SOLR_IN_FILE=$PWD/solr.in.sh
 

search-services/pom.xml

@@ -4,13 +4,13 @@
     <parent>
         <groupId>org.alfresco</groupId>
         <artifactId>alfresco-search-and-insight-parent</artifactId>
-        <version>1.5.0-SNAPSHOT</version>
+        <version>1.4.1-RC2</version>
     </parent>
     <!-- The groupId and version are required by the maven pom extractor plugin on Bamboo - more details in this issue:
     https://bitbucket.org/dehringer/bamboo-maven-pom-extractor-plugin/issues/18/groupid-not-populated-if-using-parent-pom -->
     <groupId>org.alfresco</groupId>
     <artifactId>alfresco-search-parent</artifactId>
-    <version>1.5.0-SNAPSHOT</version>
+    <version>1.4.1-RC2</version>
     <packaging>pom</packaging>
     <name>Alfresco Solr Search parent</name>
     <properties>