ALF-9611 RSOLR 049: Share works against SOLR

- mostly working with bugs raised or the issues found
- ALF-9627 	RSOLR 049: Support for TAG list - simple field facettting
  - was not required. Facetting alpha is there (limits not set and need to go into JSON) - added to result set
- ALF-9628 	RSOLR 049: HTTPClient pooling and sharing by query
  - pooled for query, one reused instance for each tracker (could be shared)
- ALF-9629 	RSOLR 049: Support for PARENT field and start of PATH rebuilding other index tracking (group and site search)
  - done - also started PATH change and returns other data as not yet finished (QNAME is the most obvious)
  - fixed up and tidied query generation for index tokenisation mode form the model and what the query would like

git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@29585 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261
This commit is contained in:
Andrew Hind
2011-08-05 19:43:05 +00:00
parent 4a8f8b7c2f
commit 37f19ecf45
17 changed files with 304 additions and 22 deletions

View File

@@ -219,7 +219,7 @@
</property>
</bean>
<bean id="search.luceneCategoryService" class="org.alfresco.repo.search.impl.lucene.LuceneCategoryServiceImpl">
<bean id="search.luceneCategoryService" class="org.alfresco.repo.search.impl.solr.SolrCategoryServiceImpl">
<property name="nodeService">
<ref bean="nodeService" />
</property>

View File

@@ -20,6 +20,7 @@ package org.alfresco.cmis.search;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -40,6 +41,7 @@ import org.alfresco.service.cmr.repository.NodeService;
import org.alfresco.service.cmr.search.LimitBy;
import org.alfresco.service.cmr.search.ResultSet;
import org.alfresco.service.cmr.search.ResultSetRow;
import org.alfresco.util.Pair;
/**
* @author andyh
@@ -327,4 +329,10 @@ public class CMISResultSetImpl implements CMISResultSet, Serializable
{
return 0;
}
@Override
public List<Pair<String, Integer>> getFieldFacet(String field)
{
return Collections.<Pair<String, Integer>>emptyList();
}
}

View File

@@ -20,6 +20,7 @@ package org.alfresco.opencmis.search;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -37,6 +38,7 @@ import org.alfresco.service.cmr.search.LimitBy;
import org.alfresco.service.cmr.search.ResultSet;
import org.alfresco.service.cmr.search.ResultSetRow;
import org.alfresco.service.cmr.search.ResultSetSPI;
import org.alfresco.util.Pair;
/**
* @author andyh
@@ -328,4 +330,10 @@ public class CMISResultSet implements ResultSetSPI<CMISResultSetRow, CMISResultS
{
return 0;
}
@Override
public List<Pair<String, Integer>> getFieldFacet(String field)
{
return Collections.<Pair<String, Integer>>emptyList();
}
}

View File

@@ -19,12 +19,14 @@
package org.alfresco.repo.search;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.alfresco.service.cmr.repository.ChildAssociationRef;
import org.alfresco.service.cmr.repository.NodeRef;
import org.alfresco.service.cmr.search.ResultSet;
import org.alfresco.service.cmr.search.ResultSetRow;
import org.alfresco.util.Pair;
import org.apache.lucene.search.HitCollector;
/**
@@ -113,4 +115,12 @@ public abstract class AbstractResultSet implements ResultSet
{
return 0;
}
@Override
public List<Pair<String, Integer>> getFieldFacet(String field)
{
return Collections.<Pair<String, Integer>>emptyList();
}
}

View File

@@ -31,6 +31,7 @@ import org.alfresco.service.cmr.search.ResultSet;
import org.alfresco.service.cmr.search.ResultSetMetaData;
import org.alfresco.service.cmr.search.ResultSetRow;
import org.alfresco.service.cmr.search.SearchParameters;
import org.alfresco.util.Pair;
/**
* An empty result set
@@ -148,4 +149,10 @@ public class EmptyResultSet implements ResultSet
{
return false;
}
@Override
public List<Pair<String, Integer>> getFieldFacet(String field)
{
return Collections.<Pair<String, Integer>>emptyList();
}
}

View File

@@ -59,17 +59,17 @@ import org.alfresco.util.Pair;
*/
public class LuceneCategoryServiceImpl implements CategoryService
{
private NodeService nodeService;
protected NodeService nodeService;
private NodeService publicNodeService;
protected NodeService publicNodeService;
private TenantService tenantService;
protected TenantService tenantService;
private NamespacePrefixResolver namespacePrefixResolver;
protected NamespacePrefixResolver namespacePrefixResolver;
private DictionaryService dictionaryService;
protected DictionaryService dictionaryService;
private IndexerAndSearcher indexerAndSearcher;
protected IndexerAndSearcher indexerAndSearcher;
/**
*

View File

@@ -20,6 +20,7 @@ package org.alfresco.repo.search.impl.lucene;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -33,6 +34,7 @@ import org.alfresco.service.cmr.search.ResultSet;
import org.alfresco.service.cmr.search.ResultSetMetaData;
import org.alfresco.service.cmr.search.ResultSetRow;
import org.alfresco.service.cmr.search.SearchParameters;
import org.alfresco.util.Pair;
/**
* @author andyh
@@ -233,4 +235,10 @@ public class PagingLuceneResultSet implements ResultSet, Serializable
{
return wrapped.getBulkFetchSize();
}
@Override
public List<Pair<String, Integer>> getFieldFacet(String field)
{
return wrapped.getFieldFacet(field);
}
}

View File

@@ -19,6 +19,8 @@
package org.alfresco.repo.search.impl.lucene;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -26,6 +28,7 @@ import org.alfresco.repo.domain.node.NodeDAO;
import org.alfresco.repo.search.SimpleResultSetMetaData;
import org.alfresco.service.cmr.repository.ChildAssociationRef;
import org.alfresco.service.cmr.repository.NodeRef;
import org.alfresco.service.cmr.repository.StoreRef;
import org.alfresco.service.cmr.search.LimitBy;
import org.alfresco.service.cmr.search.PermissionEvaluationMode;
import org.alfresco.service.cmr.search.ResultSet;
@@ -33,6 +36,7 @@ import org.alfresco.service.cmr.search.ResultSetMetaData;
import org.alfresco.service.cmr.search.ResultSetRow;
import org.alfresco.service.cmr.search.SearchParameters;
import org.alfresco.util.Pair;
import org.apache.solr.client.solrj.response.FacetField;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
@@ -60,6 +64,8 @@ public class SolrJSONResultSet implements ResultSet
private SimpleResultSetMetaData resultSetMetaData;
private HashMap<String, List<Pair<String, Integer>>> fieldFacets = new HashMap<String, List<Pair<String, Integer>>>(1);
/**
* Detached result set based on that provided
* @param resultSet
@@ -98,6 +104,30 @@ public class SolrJSONResultSet implements ResultSet
}
}
if(json.has("facet_counts"))
{
JSONObject facet_counts = json.getJSONObject("facet_counts");
if(facet_counts.has("facet_fields"))
{
JSONObject facet_fields = facet_counts.getJSONObject("facet_fields");
for(Iterator it = facet_fields.keys(); it.hasNext(); /**/)
{
String fieldName = (String)it.next();
JSONArray facets = facet_fields.getJSONArray(fieldName);
int facetArraySize = facets.length();
ArrayList<Pair<String, Integer>> facetValues = new ArrayList<Pair<String, Integer>>(facetArraySize/2);
for(int i = 0; i < facetArraySize; i+=2)
{
String facetEntryName = facets.getString(i);
Integer facetEntryCount = Integer.parseInt(facets.getString(i+1));
Pair<String, Integer> pair = new Pair<String, Integer>(facetEntryName, facetEntryCount);
facetValues.add(pair);
}
fieldFacets.put(fieldName, facetValues);
}
}
}
}
catch (JSONException e)
{
@@ -147,7 +177,7 @@ public class SolrJSONResultSet implements ResultSet
Pair<Long, ChildAssociationRef> primaryParentAssoc = nodeDAO.getPrimaryParentAssoc(page.get(n).getFirst());
if(primaryParentAssoc != null)
{
return nodeDAO.getPrimaryParentAssoc(page.get(n).getFirst()).getSecond();
return primaryParentAssoc.getSecond();
}
else
{
@@ -179,7 +209,15 @@ public class SolrJSONResultSet implements ResultSet
public NodeRef getNodeRef(int n)
{
// TODO: lost nodes?
return nodeDAO.getNodePair(page.get(n).getFirst()).getSecond();
Pair<Long, NodeRef> nodePair = nodeDAO.getNodePair(page.get(n).getFirst());
if(nodePair != null)
{
return nodePair.getSecond();
}
else
{
return new NodeRef(new StoreRef("missing", "missing"), "missing");
}
}
/*
@@ -305,5 +343,17 @@ public class SolrJSONResultSet implements ResultSet
return numberFound;
}
@Override
public List<Pair<String, Integer>> getFieldFacet(String field)
{
List<Pair<String, Integer>> answer = fieldFacets.get(field);
if(answer != null)
{
return answer;
}
else
{
return Collections.<Pair<String, Integer>>emptyList();
}
}
}

View File

@@ -0,0 +1,118 @@
/*
* Copyright (C) 2005-2010 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.repo.search.impl.solr;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import org.alfresco.repo.search.impl.lucene.LuceneCategoryServiceImpl;
import org.alfresco.repo.search.impl.lucene.LuceneIndexerAndSearcher;
import org.alfresco.repo.search.impl.lucene.LuceneSearcher;
import org.alfresco.service.cmr.dictionary.AspectDefinition;
import org.alfresco.service.cmr.dictionary.DataTypeDefinition;
import org.alfresco.service.cmr.dictionary.PropertyDefinition;
import org.alfresco.service.cmr.repository.NodeRef;
import org.alfresco.service.cmr.repository.StoreRef;
import org.alfresco.service.cmr.search.ResultSet;
import org.alfresco.service.cmr.search.SearchParameters;
import org.alfresco.service.cmr.search.SearchParameters.FieldFacet;
import org.alfresco.service.cmr.search.SearchService;
import org.alfresco.service.namespace.QName;
import org.alfresco.util.Pair;
/**
* @author Andy
*
*/
public class SolrCategoryServiceImpl extends LuceneCategoryServiceImpl
{
@Override
public List<Pair<NodeRef, Integer>> getTopCategories(StoreRef storeRef, QName aspectName, int count)
{
AspectDefinition definition = dictionaryService.getAspect(aspectName);
if(definition == null)
{
throw new IllegalStateException("Unknown aspect");
}
QName catProperty = null;
Map<QName, PropertyDefinition> properties = definition.getProperties();
for(QName pName : properties.keySet())
{
if(pName.getNamespaceURI().equals(aspectName.getNamespaceURI()))
{
if(pName.getLocalName().equalsIgnoreCase(aspectName.getLocalName()))
{
PropertyDefinition def = properties.get(pName);
if(def.getDataType().getName().equals(DataTypeDefinition.CATEGORY))
{
catProperty = pName;
}
}
}
}
if(catProperty == null)
{
throw new IllegalStateException("Aspect does not have category property mirroring the aspect name");
}
String field = "@" + catProperty;
SearchParameters sp = new SearchParameters();
sp.setLanguage(SearchService.LANGUAGE_SOLR_FTS_ALFRESCO);
sp.addStore(storeRef);
sp.setQuery(catProperty+":*");
FieldFacet ff = new FieldFacet(field);
ff.setLimit(count);
sp.addFieldFacet(ff);
ResultSet resultSet = null;
try
{
resultSet = indexerAndSearcher.getSearcher(storeRef, false).query(sp);
List<Pair<String, Integer>> facetCounts = resultSet.getFieldFacet(field);
List<Pair<NodeRef, Integer>> answer = new LinkedList<Pair<NodeRef, Integer>>();
for (Pair<String, Integer> term : facetCounts)
{
Pair<NodeRef, Integer> toAdd;
NodeRef nodeRef = new NodeRef(term.getFirst());
if (nodeService.exists(nodeRef))
{
toAdd = new Pair<NodeRef, Integer>(nodeRef, term.getSecond());
}
else
{
toAdd = new Pair<NodeRef, Integer>(null, term.getSecond());
}
answer.add(toAdd);
}
return answer;
}
finally
{
if (resultSet != null)
{
resultSet.close();
}
}
}
}

View File

@@ -29,12 +29,14 @@ import java.util.Map;
import javax.servlet.http.HttpServletResponse;
import org.alfresco.error.AlfrescoRuntimeException;
import org.alfresco.model.ContentModel;
import org.alfresco.repo.domain.node.NodeDAO;
import org.alfresco.repo.search.impl.lucene.LuceneQueryParserException;
import org.alfresco.repo.search.impl.lucene.SolrJSONResultSet;
import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter;
import org.alfresco.service.cmr.search.ResultSet;
import org.alfresco.service.cmr.search.SearchParameters;
import org.alfresco.service.cmr.search.SearchParameters.FieldFacet;
import org.alfresco.service.cmr.search.SearchParameters.SortDefinition;
import org.alfresco.service.cmr.security.PermissionService;
import org.apache.commons.codec.net.URLCodec;
@@ -189,7 +191,7 @@ public class SolrQueryHTTPClient
{
authQuery.append(" ");
}
authQuery.append("AUTHORITY:\"").append(authority).append("\"");
authQuery.append("|AUTHORITY:\"").append(authority).append("\"");
}
// url.append("&fq=");
@@ -200,6 +202,18 @@ public class SolrQueryHTTPClient
// facets would go on url?
if(searchParameters.getFieldFacets().size() > 0)
{
url.append("&facet=").append(encoder.encode("true", "UTF-8"));
for(FieldFacet facet : searchParameters.getFieldFacets())
{
url.append("&facet.field=").append(encoder.encode(facet.getField(), "UTF-8"));
url.append("&").append(encoder.encode("f."+facet.getField()+".limit", "UTF-8")).append("=").append(encoder.encode(""+facet.getLimit(), "UTF-8"));
}
}
// end of field factes
JSONObject body = new JSONObject();
body.put("query", searchParameters.getQuery());
// body.put("defaultField", searchParameters.getDefaultFieldName());

View File

@@ -18,6 +18,7 @@
*/
package org.alfresco.repo.search.results;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -27,6 +28,7 @@ import org.alfresco.service.cmr.search.ResultSet;
import org.alfresco.service.cmr.search.ResultSetMetaData;
import org.alfresco.service.cmr.search.ResultSetRow;
import org.alfresco.service.cmr.search.ResultSetSPI;
import org.alfresco.util.Pair;
/**
* Wrap an SPI result set with the basic interface
@@ -149,6 +151,12 @@ public class ResultSetSPIWrapper<ROW extends ResultSetRow, MD extends ResultSetM
return wrapped.getBulkFetchSize();
}
@Override
public List<Pair<String, Integer>> getFieldFacet(String field)
{
return wrapped.getFieldFacet(field);
}
private static class WrappedIterator<ROW extends ResultSetRow> implements Iterator<ResultSetRow>
{
private Iterator<ROW> wrapped;

View File

@@ -38,6 +38,7 @@ import org.alfresco.service.cmr.search.SearchParameters.SortDefinition;
import org.alfresco.service.namespace.NamespacePrefixResolver;
import org.alfresco.service.namespace.NamespaceService;
import org.alfresco.service.namespace.QName;
import org.alfresco.util.Pair;
/**
* Sorted results
@@ -344,4 +345,10 @@ public class SortedResultSet implements ResultSet
return resultSet.getBulkFetchSize();
}
@Override
public List<Pair<String, Integer>> getFieldFacet(String field)
{
return resultSet.getFieldFacet(field);
}
}

View File

@@ -20,6 +20,7 @@ package org.alfresco.repo.security.permissions.impl.acegi;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collections;
import java.util.List;
import java.util.ListIterator;
@@ -29,6 +30,7 @@ import org.alfresco.service.cmr.repository.NodeRef;
import org.alfresco.service.cmr.search.ResultSet;
import org.alfresco.service.cmr.search.ResultSetMetaData;
import org.alfresco.service.cmr.search.ResultSetRow;
import org.alfresco.util.Pair;
/**
* Filtering result set to support permission checks
@@ -319,4 +321,10 @@ public class FilteringResultSet extends ACLEntryAfterInvocationProvider implemen
{
return unfiltered.getBulkFetchSize();
}
@Override
public List<Pair<String, Integer>> getFieldFacet(String field)
{
return unfiltered.getFieldFacet(field);
}
}

View File

@@ -32,7 +32,7 @@ public class MetaDataResultsFilter
private boolean includeAclId = true;
private boolean includeOwner = true;
private boolean includePaths = true;
private boolean includeAssociations = false;
private boolean includeParentAssociations = true;
private boolean includeChildAssociations = true;
private boolean includeNodeRef = true;
@@ -52,13 +52,13 @@ public class MetaDataResultsFilter
{
this.includeNodeRef = includeNodeRef;
}
public boolean getIncludeAssociations()
public boolean getIncludeParentAssociations()
{
return includeAssociations;
return includeParentAssociations;
}
public void setIncludeAssociations(boolean includeAssociations)
public void setIncludeParentAssociations(boolean includeParentAssociations)
{
this.includeAssociations = includeAssociations;
this.includeParentAssociations = includeParentAssociations;
}
public boolean getIncludeProperties()
{

View File

@@ -23,6 +23,7 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.zip.CRC32;
import org.alfresco.service.cmr.repository.ChildAssociationRef;
import org.alfresco.service.cmr.repository.NodeRef;
@@ -43,9 +44,10 @@ public class NodeMetaData
private Long aclId;
private Map<QName, Serializable> properties;
private Set<QName> aspects;
// private List<Path> paths;
private Collection<Pair<Path, QName>> paths;
private List<ChildAssociationRef> childAssocs;
private List<ChildAssociationRef> parentAssocs;
private Long parentAssocsCrc;
public String getOwner()
{
@@ -119,6 +121,23 @@ public class NodeMetaData
{
this.childAssocs = childAssocs;
}
/**
* @param parentAssocs
* @param crc
*/
public void setParentAssocs(List<ChildAssociationRef> parentAssocs, Long parentAssocsCrc)
{
this.parentAssocs = parentAssocs;
this.parentAssocsCrc = parentAssocsCrc;
}
public List<ChildAssociationRef> getParentAssocs()
{
return parentAssocs;
}
public Long getParentAssocsCrc()
{
return parentAssocsCrc;
}
}

View File

@@ -23,6 +23,7 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -31,11 +32,13 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.zip.CRC32;
import org.alfresco.model.ContentModel;
import org.alfresco.repo.dictionary.CompiledModel;
import org.alfresco.repo.dictionary.DictionaryDAO;
import org.alfresco.repo.dictionary.DictionaryDAOImpl;
import org.alfresco.repo.domain.CrcHelper;
import org.alfresco.repo.domain.node.Node;
import org.alfresco.repo.domain.node.NodeDAO;
import org.alfresco.repo.domain.node.NodeDAO.ChildAssocRefQueryCallback;
@@ -465,7 +468,7 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent
boolean includeAspects = (resultFilter == null ? true : resultFilter.getIncludeAspects());
boolean includePaths = (resultFilter == null ? true : resultFilter.getIncludePaths());
boolean includeNodeRef = (resultFilter == null ? true : resultFilter.getIncludeNodeRef());
boolean includeAssociations = (resultFilter == null ? true : resultFilter.getIncludeAssociations());
boolean includeParentAssociations = (resultFilter == null ? true : resultFilter.getIncludeParentAssociations());
boolean includeChildAssociations = (resultFilter == null ? true : resultFilter.getIncludeChildAssociations());
boolean includeOwner = (resultFilter == null ? true : resultFilter.getIncludeOwner());
@@ -554,7 +557,7 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent
nodeMetaData.setChildAssocs(childAssocs);
}
if(includeAssociations)
if(includeParentAssociations)
{
final List<ChildAssociationRef> parentAssocs = new ArrayList<ChildAssociationRef>(100);
nodeDAO.getParentAssocs(nodeId, null, null, null, new ChildAssocRefQueryCallback()
@@ -579,6 +582,20 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent
}
});
CRC32 crc = new CRC32();
for(ChildAssociationRef car : parentAssocs)
{
try
{
crc.update(car.toString().getBytes("UTF-8"));
}
catch (UnsupportedEncodingException e)
{
throw new RuntimeException("UTF-8 encoding is not supported");
}
}
nodeMetaData.setParentAssocs(parentAssocs, crc.getValue());
// TODO non-child associations
// Collection<Pair<Long, AssociationRef>> sourceAssocs = nodeDAO.getSourceNodeAssocs(nodeId);
// Collection<Pair<Long, AssociationRef>> targetAssocs = nodeDAO.getTargetNodeAssocs(nodeId);

View File

@@ -267,7 +267,7 @@ public class SOLRTrackingComponentTest extends TestCase
NodeMetaDataParameters nodeMetaDataParams = new NodeMetaDataParameters();
nodeMetaDataParams.setNodeIds(st.getNodeIds());
MetaDataResultsFilter filter = new MetaDataResultsFilter();
filter.setIncludeAssociations(false);
filter.setIncludeParentAssociations(false);
//filter.setIncludePaths(false);
filter.setIncludeChildAssociations(false);
getNodeMetaData(nodeMetaDataParams, filter, st);