Merged HEAD-BUG-FIX (5.0/Cloud) to HEAD (5.0/Cloud)

85082: Merged PLATFORM1 (5.0/Cloud) to HEAD-BUG-FIX (5.0/Cloud)
      84493: ACE-2637: Initial implementation of spell-check feature.


git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@85397 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

source/java/org/alfresco/opencmis/search/CMISResultSet.java

@@ -39,6 +39,7 @@ import org.alfresco.service.cmr.search.LimitBy;
 import org.alfresco.service.cmr.search.ResultSet;
 import org.alfresco.service.cmr.search.ResultSetRow;
 import org.alfresco.service.cmr.search.ResultSetSPI;
+import org.alfresco.service.cmr.search.SpellCheckResult;
 import org.alfresco.util.Pair;
 
 /**
@@ -359,4 +360,10 @@ public class CMISResultSet implements ResultSetSPI<CMISResultSetRow, CMISResultS
     {
         return Collections.emptyMap();
     }
+    
+    @Override
+    public SpellCheckResult getSpellCheckResult()
+    {
+        return new SpellCheckResult(null, null, false);
+    }
 }

source/java/org/alfresco/repo/jscript/ScriptSpellCheckResult.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2005-2014 Alfresco Software Limited.
+ *
+ * This file is part of Alfresco
+ *
+ * Alfresco is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Alfresco is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package org.alfresco.repo.jscript;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * @author Jamal Kaabi-Mofrad
+ * @since 5.0
+ */
+public class ScriptSpellCheckResult implements Serializable
+{
+    private static final long serialVersionUID = -5947320933438147267L;
+
+    private final String originalSearchTerm;
+    private final String resultName;
+    private final List<String> results;
+    private final boolean searchedFor;
+    private final boolean spellCheckExist;
+
+    public ScriptSpellCheckResult(String originalSearchTerm, String resultName, boolean searchedFor,
+                List<String> results, boolean spellCheckExist)
+    {
+        this.originalSearchTerm = originalSearchTerm;
+        this.resultName = resultName;
+        this.results = results;
+        this.searchedFor = searchedFor;
+        this.spellCheckExist = spellCheckExist;
+    }
+
+    /**
+     * @return the originalSearchTerm
+     */
+    public String getOriginalSearchTerm()
+    {
+        return this.originalSearchTerm;
+    }
+
+    /**
+     * @return the resultName
+     */
+    public String getResultName()
+    {
+        return this.resultName;
+    }
+
+    /**
+     * @return the results
+     */
+    public List<String> getResults()
+    {
+        return this.results;
+    }
+
+    /**
+     * @return the searchedFor
+     */
+    public boolean isSearchedFor()
+    {
+        return this.searchedFor;
+    }
+
+    /**
+     * @return the spellCheckExist
+     */
+    public boolean isSpellCheckExist()
+    {
+        return this.spellCheckExist;
+    }
+}

source/java/org/alfresco/repo/jscript/Search.java

@@ -48,6 +48,7 @@ import org.alfresco.service.cmr.search.SearchParameters;
 import org.alfresco.service.cmr.search.SearchParameters.FieldFacet;
 import org.alfresco.service.cmr.search.SearchParameters.Operator;
 import org.alfresco.service.cmr.search.SearchService;
+import org.alfresco.service.cmr.search.SpellCheckResult;
 import org.alfresco.service.cmr.security.AccessStatus;
 import org.alfresco.service.cmr.security.PermissionService;
 import org.alfresco.util.ISO9075;
@@ -585,6 +586,8 @@ public class Search extends BaseScopableProcessorExtension implements Initializi
                 String onerror = (String)def.get("onerror");
                 String defaultField = (String)def.get("defaultField");
                 String defaultOperator = (String)def.get("defaultOperator");
+                String searchTerm = (String) def.get("searchTerm");
+                boolean spellCheck = (boolean) def.get("spellCheck");
                 
                 // extract supplied values
                 
@@ -666,6 +669,8 @@ public class Search extends BaseScopableProcessorExtension implements Initializi
                 sp.addStore(store != null ? new StoreRef(store) : this.storeRef);
                 sp.setLanguage(language != null ? language : SearchService.LANGUAGE_LUCENE);
                 sp.setQuery(query);
+                sp.setSearchTerm(searchTerm);
+                sp.setSpellCheck(spellCheck);
                 if (defaultField != null)
                 {
                     sp.setDefaultFieldName(defaultField);
@@ -985,6 +990,13 @@ public class Search extends BaseScopableProcessorExtension implements Initializi
                 }
             }// End of bucketing
             meta.put("facets", facetMeta);
+            SpellCheckResult spellCheckResult = results.getSpellCheckResult();
+            meta.put("spellcheck", new ScriptSpellCheckResult(
+                                    sp.getSearchTerm(),
+                                    spellCheckResult.getResultName(),
+                                    spellCheckResult.isSearchedFor(),
+                                    spellCheckResult.getResults(),
+                                    spellCheckResult.isSpellCheckExist()));
         }
         catch (Throwable err)
         {

source/java/org/alfresco/repo/search/AbstractResultSet.java

@@ -27,6 +27,7 @@ import org.alfresco.service.cmr.repository.ChildAssociationRef;
 import org.alfresco.service.cmr.repository.NodeRef;
 import org.alfresco.service.cmr.search.ResultSet;
 import org.alfresco.service.cmr.search.ResultSetRow;
+import org.alfresco.service.cmr.search.SpellCheckResult;
 import org.alfresco.util.Pair;
 
 /**
@@ -128,4 +129,9 @@ public abstract class AbstractResultSet implements ResultSet
         return Collections.emptyMap();
     }    
     
+    @Override
+    public SpellCheckResult getSpellCheckResult()
+    {
+        return new SpellCheckResult(null, null, false);
+    }
 }

source/java/org/alfresco/repo/search/EmptyResultSet.java

@@ -32,6 +32,7 @@ import org.alfresco.service.cmr.search.ResultSet;
 import org.alfresco.service.cmr.search.ResultSetMetaData;
 import org.alfresco.service.cmr.search.ResultSetRow;
 import org.alfresco.service.cmr.search.SearchParameters;
+import org.alfresco.service.cmr.search.SpellCheckResult;
 import org.alfresco.util.Pair;
 
 /**
@@ -171,4 +172,10 @@ public class EmptyResultSet implements ResultSet
     {
         return Collections.emptyMap();
     }
+    
+    @Override
+    public SpellCheckResult getSpellCheckResult()
+    {
+        return new SpellCheckResult(null, null, false);
+    }
 }

source/java/org/alfresco/repo/search/impl/lucene/PagingLuceneResultSet.java

@@ -35,6 +35,7 @@ import org.alfresco.service.cmr.search.ResultSet;
 import org.alfresco.service.cmr.search.ResultSetMetaData;
 import org.alfresco.service.cmr.search.ResultSetRow;
 import org.alfresco.service.cmr.search.SearchParameters;
+import org.alfresco.service.cmr.search.SpellCheckResult;
 import org.alfresco.util.Pair;
 
 /**
@@ -257,4 +258,10 @@ public class PagingLuceneResultSet implements ResultSet, Serializable
     {
         return wrapped.getFacetQueries();
     }
+    
+    @Override
+    public SpellCheckResult getSpellCheckResult()
+    {
+        return wrapped.getSpellCheckResult();
+    }
 }

source/java/org/alfresco/repo/search/impl/lucene/SolrJSONResultSet.java

@@ -37,6 +37,7 @@ import org.alfresco.service.cmr.search.ResultSet;
 import org.alfresco.service.cmr.search.ResultSetMetaData;
 import org.alfresco.service.cmr.search.ResultSetRow;
 import org.alfresco.service.cmr.search.SearchParameters;
+import org.alfresco.service.cmr.search.SpellCheckResult;
 import org.alfresco.util.Pair;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -79,6 +80,7 @@ public class SolrJSONResultSet implements ResultSet, JSONResult
     
     private long lastIndexedTxId;
     
+    private SpellCheckResult spellCheckResult;
     /**
      * Detached result set based on that provided
      * @param resultSet
@@ -193,8 +195,38 @@ public class SolrJSONResultSet implements ResultSet, JSONResult
                     }
                 }
             }
+            // process Spell check 
+            JSONObject spellCheckJson = (JSONObject) json.opt("spellcheck");
+            if (spellCheckJson != null)
+            {
+                List<String> list = new ArrayList<>(3);
+                String flag = "";
+                boolean searchedFor = false;
+                if (spellCheckJson.has("searchInsteadFor"))
+                {
+                    flag = "searchInsteadFor";
+                    searchedFor = true;
+                    list.add(spellCheckJson.getString(flag));
 
                 }
+                else if (spellCheckJson.has("didYouMean"))
+                {
+                    flag = "didYouMean";
+                    JSONArray suggestions = spellCheckJson.getJSONArray(flag);
+                    for (int i = 0, lenght = suggestions.length(); i < lenght; i++)
+                    {
+                        list.add(suggestions.getString(i));
+                    }
+                }
+
+                spellCheckResult = new SpellCheckResult(flag, list, searchedFor);
+
+            }
+            else
+            {
+                spellCheckResult = new SpellCheckResult(null, null, false);
+            }
+        }
         catch (JSONException e)
         {
            logger.info(e.getMessage());
@@ -427,4 +459,11 @@ public class SolrJSONResultSet implements ResultSet, JSONResult
     {
         return Collections.unmodifiableMap(facetQueries);
     }
+
+
+    @Override
+    public SpellCheckResult getSpellCheckResult()
+    {
+        return this.spellCheckResult;
+    }
 }

source/java/org/alfresco/repo/search/impl/solr/SolrQueryHTTPClient.java

@@ -428,8 +428,18 @@ public class SolrQueryHTTPClient implements BeanFactoryAware
                     url.append("&facet.query=").append(encoder.encode("{!afts}"+facetQuery, "UTF-8"));
                 }                
             }
+            // end of field facets
 
-            // end of field factes
+            final String searchTerm = searchParameters.getSearchTerm();
+            String spellCheckQueryStr = null;
+            if (searchTerm != null && searchParameters.isSpellCheck())
+            {
+                StringBuilder builder = new StringBuilder();
+                builder.append("&spellcheck.q=").append(encoder.encode(searchTerm, "UTF-8"));
+                builder.append("&spellcheck=").append(encoder.encode("true", "UTF-8"));
+                spellCheckQueryStr = builder.toString();
+                url.append(spellCheckQueryStr);
+            }
 
             JSONObject body = new JSONObject();
             body.put("query", searchParameters.getQuery());
@@ -515,7 +525,7 @@ public class SolrQueryHTTPClient implements BeanFactoryAware
                     return new SolrJSONResultSet(json, searchParameters, nodeService, nodeDAO, limitBy, maximumResults);
                 }
                 
-            });
+            }, spellCheckQueryStr);
         }
         catch (UnsupportedEncodingException e)
         {
@@ -538,6 +548,39 @@ public class SolrQueryHTTPClient implements BeanFactoryAware
     protected JSONResult postSolrQuery(StoreRef store, String url, JSONObject body, SolrJsonProcessor<?> jsonProcessor)
                 throws UnsupportedEncodingException, IOException, HttpException, URIException,
                 JSONException
+    {
+        return postSolrQuery(store, url, body, jsonProcessor, null);
+    }
+
+    protected JSONResult postSolrQuery(StoreRef store, String url, JSONObject body, SolrJsonProcessor<?> jsonProcessor, String spellCheckParams)
+                throws UnsupportedEncodingException, IOException, HttpException, URIException,
+                JSONException
+    {
+        JSONObject json = postQuery(store, url, body);
+        if (spellCheckParams != null)
+        {
+            SpellCheckDecisionManager manager = new SpellCheckDecisionManager(json, url, body, spellCheckParams);
+            if (manager.isCollate())
+            {
+                json = postQuery(store, manager.getUrl(), body);
+            }
+            json.put("spellcheck", manager.getSpellCheckJsonValue());
+        }
+
+            JSONResult results = jsonProcessor.getResult(json);
+
+            if (s_logger.isDebugEnabled())
+            {
+                s_logger.debug("Sent :" + url);
+                s_logger.debug("   with: " + body.toString());
+                s_logger.debug("Got: " + results.getNumberFound() + " in " + results.getQueryTime() + " ms");
+            }
+            
+            return results;
+    }
+
+    protected JSONObject postQuery(StoreRef store, String url, JSONObject body) throws UnsupportedEncodingException,
+                IOException, HttpException, URIException, JSONException
     {
         PostMethod post = new PostMethod(url);
         if (body.toString().length() > DEFAULT_SAVEPOST_BUFFER)
@@ -585,17 +628,7 @@ public class SolrQueryHTTPClient implements BeanFactoryAware
                     throw new LuceneQueryParserException("SOLR side error: " + status.getString("message"));
                 }
             }
-
+            return json;
-            JSONResult results = jsonProcessor.getResult(json);
-
-            if (s_logger.isDebugEnabled())
-            {
-                s_logger.debug("Sent :" + url);
-                s_logger.debug("   with: " + body.toString());
-                s_logger.debug("Got: " + results.getNumberFound() + " in " + results.getQueryTime() + " ms");
-            }
-            
-            return results;
         }
         finally
         {

source/java/org/alfresco/repo/search/impl/solr/SpellCheckDecisionManager.java

@@ -0,0 +1,123 @@
+/*
+ * Copyright (C) 2005-2014 Alfresco Software Limited.
+ *
+ * This file is part of Alfresco
+ *
+ * Alfresco is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Alfresco is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package org.alfresco.repo.search.impl.solr;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+/**
+ * @author Jamal Kaabi-Mofrad
+ * @since 5.0
+ */
+public class SpellCheckDecisionManager
+{
+    private static final Log logger = LogFactory.getLog(SpellCheckDecisionManager.class);
+
+    private static final String COLLATION = "collation";
+    private boolean collate;
+    private String url;
+    private JSONObject spellCheckJsonValue;
+
+    public SpellCheckDecisionManager(JSONObject resultJson, String origURL, JSONObject reguestJsonBody,
+                String spellCheckParams)
+    {
+        try
+        {
+            List<String> collationQueriesList = new ArrayList<>();
+            JSONObject response = resultJson.getJSONObject("response");
+            long numberFound = response.getLong("numFound");
+            this.url = origURL;
+
+            JSONObject spellcheck = resultJson.getJSONObject("spellcheck");
+            JSONArray suggestions = spellcheck.getJSONArray("suggestions");
+
+            for (int key = 0, value = 1, length = suggestions.length(); value < length; key += 2, value += 2)
+            {
+                String jsonName = suggestions.getString(key);
+
+                if (COLLATION.equals(jsonName))
+                {
+                    JSONObject valueJsonObject = suggestions.getJSONObject(value);
+                    long collationHit = valueJsonObject.getLong("hits");
+                    this.collate = numberFound == 0 && collationHit > 0;
+                    if (collate)
+                    {
+                        reguestJsonBody.put("query", valueJsonObject.getString("collationQuery"));
+                        spellCheckJsonValue = new JSONObject();
+                        spellCheckJsonValue.put("searchInsteadFor", valueJsonObject.getString("collationQueryString"));
+                        break;
+                    }
+                    else if (collationHit > numberFound)
+                    {
+                        collationQueriesList.add(valueJsonObject.getString("collationQueryString"));
+
+                    }
+                }
+            }
+            if (collate)
+            {
+                this.url = origURL.replace(spellCheckParams, "");
+            }
+            else if (collationQueriesList.size() > 0)
+            {
+                spellCheckJsonValue = new JSONObject();
+                JSONArray jsonArray = new JSONArray(collationQueriesList);
+                spellCheckJsonValue.put("didYouMean", jsonArray);
+            }
+            else
+            {
+                spellCheckJsonValue = new JSONObject();
+            }
+        }
+        catch (Exception e)
+        {
+            logger.info(e.getMessage());
+        }
+    }
+
+    /**
+     * @return the collate
+     */
+    public boolean isCollate()
+    {
+        return this.collate;
+    }
+
+    /**
+     * @return the url
+     */
+    public String getUrl()
+    {
+        return this.url;
+    }
+
+    /**
+     * @return the spellCheckJsonValue
+     */
+    public JSONObject getSpellCheckJsonValue()
+    {
+        return this.spellCheckJsonValue;
+    }
+}

source/java/org/alfresco/repo/search/results/ResultSetSPIWrapper.java

@@ -28,6 +28,7 @@ import org.alfresco.service.cmr.search.ResultSet;
 import org.alfresco.service.cmr.search.ResultSetMetaData;
 import org.alfresco.service.cmr.search.ResultSetRow;
 import org.alfresco.service.cmr.search.ResultSetSPI;
+import org.alfresco.service.cmr.search.SpellCheckResult;
 import org.alfresco.util.Pair;
 
 /**
@@ -163,6 +164,12 @@ public class ResultSetSPIWrapper<ROW extends ResultSetRow, MD extends ResultSetM
         return wrapped.getFacetQueries();
     }
     
+    @Override
+    public SpellCheckResult getSpellCheckResult()
+    {
+        return wrapped.getSpellCheckResult();
+    }
+    
     private static class WrappedIterator<ROW extends ResultSetRow> implements Iterator<ResultSetRow>
     {
         private Iterator<ROW> wrapped;

source/java/org/alfresco/repo/search/results/SortedResultSet.java

@@ -45,6 +45,7 @@ import org.alfresco.service.cmr.search.ResultSetMetaData;
 import org.alfresco.service.cmr.search.ResultSetRow;
 import org.alfresco.service.cmr.search.SearchParameters;
 import org.alfresco.service.cmr.search.SearchParameters.SortDefinition;
+import org.alfresco.service.cmr.search.SpellCheckResult;
 import org.alfresco.service.namespace.NamespacePrefixResolver;
 import org.alfresco.service.namespace.NamespaceService;
 import org.alfresco.service.namespace.QName;
@@ -940,4 +941,10 @@ public class SortedResultSet implements ResultSet
         return resultSet.getFacetQueries();
     }
     
+    @Override
+    public SpellCheckResult getSpellCheckResult()
+    {
+        return resultSet.getSpellCheckResult();
+    }
+
 }

source/java/org/alfresco/repo/security/permissions/impl/acegi/FilteringResultSet.java

@@ -32,6 +32,7 @@ import org.alfresco.service.cmr.search.LimitBy;
 import org.alfresco.service.cmr.search.ResultSet;
 import org.alfresco.service.cmr.search.ResultSetMetaData;
 import org.alfresco.service.cmr.search.ResultSetRow;
+import org.alfresco.service.cmr.search.SpellCheckResult;
 import org.alfresco.util.Pair;
 
 /**
@@ -358,4 +359,10 @@ public class FilteringResultSet extends ACLEntryAfterInvocationProvider implemen
     {
         return unfiltered.getFacetQueries();
     }
+    
+    @Override
+    public SpellCheckResult getSpellCheckResult()
+    {
+        return unfiltered.getSpellCheckResult();
+    }
 }