inteligr8/SearchServices
1
0
Fork 0
mirror of https://github.com/Alfresco/SearchServices.git synced 2026-09-16 18:12:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' into feature/SEARCH-68_update_solr

Browse Source
This commit is contained in:
Gethin James
2016-09-08 15:17:34 +02:00
parent d303463f0c 771c8d5b1e
commit c47da98ab1
14 changed files with 411 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
search-services/alfresco-solr/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java
+6 -1
View File
@@ -469,7 +469,12 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler
Properties properties = new Properties();
//Set defaults
properties.setProperty(DATA_DIR_ROOT, newCore.getCanonicalPath()+"/data");
properties.setProperty("data.dir.store", storeRef.getProtocol() + "/" + storeRef.getIdentifier());
String shardRef = "";
if(shardCount > 0)
{
shardRef = "-"+shardInstance;
}
properties.setProperty("data.dir.store", storeRef.getProtocol() + "-" + storeRef.getIdentifier()+shardRef);
properties.setProperty("alfresco.stores", storeRef.toString());
//Potentially override the defaults
search-services/alfresco-solr/src/main/java/org/alfresco/solr/tracker/DateMonthRouter.java
+53
View File
@@ -0,0 +1,53 @@
/*
* Copyright (C) 2005-2014 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.tracker;
import org.alfresco.util.ISO8601DateFormat;
import org.apache.solr.common.util.Hash;
import org.alfresco.solr.client.Node;
import org.alfresco.solr.client.Acl;
import java.util.Date;
import java.util.GregorianCalendar;
/*
* @author Joel
*/
public class DateMonthRouter implements DocRouter
{
public boolean routeAcl(int numShards, int shardInstance, Acl acl) {
return true;
}
public boolean routeNode(int numShards, int shardInstance, Node node) {
if(numShards <= 1) {
return true;
}
String ISO8601Date = node.getShardPropertyValue();
//TODO: we can parse the string to make this more efficient rather then creating a calendar.
Date date = ISO8601DateFormat.parse(ISO8601Date);
GregorianCalendar cal = new GregorianCalendar();
cal.setTime(date);
int month = cal.get(cal.MONTH);
int year = cal.get(cal.YEAR);
String s = year+"_"+month;
return (Math.abs(Hash.murmurhash3_x86_32(s, 0, s.length(), 77)) % numShards) == shardInstance;
}
}
search-services/alfresco-solr/src/main/java/org/alfresco/solr/tracker/DocRouterFactory.java
+2
View File
@@ -34,6 +34,8 @@ public class DocRouterFactory
return new ACLIDMurmurRouter();
case MOD_ACL_ID:
return new ACLIDModRouter();
case DATE_MONTH:
return new DateMonthRouter();
default:
return new DBIDRouter();
}
search-services/alfresco-solr/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java
+29 -2
View File
@@ -24,9 +24,15 @@ import java.util.concurrent.ConcurrentLinkedQueue;
import org.alfresco.error.AlfrescoRuntimeException;
import org.alfresco.httpclient.AuthenticationException;
import org.alfresco.opencmis.dictionary.CMISStrictDictionaryService;
import org.alfresco.repo.dictionary.NamespaceDAO;
import org.alfresco.repo.index.shard.ShardMethodEnum;
import org.alfresco.repo.index.shard.ShardState;
import org.alfresco.repo.index.shard.ShardStateBuilder;
import org.alfresco.repo.search.impl.QueryParserUtils;
import org.alfresco.service.cmr.dictionary.DictionaryService;
import org.alfresco.service.cmr.dictionary.PropertyDefinition;
import org.alfresco.service.namespace.QName;
import org.alfresco.solr.AlfrescoSolrDataModel;
import org.alfresco.solr.BoundedDeque;
import org.alfresco.solr.InformationServer;
@@ -65,8 +71,7 @@ public class MetadataTracker extends AbstractTracker implements Tracker
private ConcurrentLinkedQueue<Long> nodesToPurge = new ConcurrentLinkedQueue<Long>();
private ConcurrentLinkedQueue<String> queriesToReindex = new ConcurrentLinkedQueue<String>();
private DocRouter docRouter;
private QName shardProperty;
public MetadataTracker(Properties p, SOLRAPIClient client, String coreName,
InformationServer informationServer)
@@ -75,6 +80,11 @@ public class MetadataTracker extends AbstractTracker implements Tracker
//System.out.println("####### MetadatTracker() ########");
transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", "100"));
shardMethod = p.getProperty("shard.method", SHARD_METHOD_DBID);
String shardKey = p.getProperty("shard.key");
if(shardKey != null) {
shardProperty = getShardProperty(shardKey);
}
docRouter = DocRouterFactory.getRouter(ShardMethodEnum.getShardMethod(shardMethod));
nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10"));
threadHandler = new ThreadHandler(p, coreName, "MetadataTracker");
@@ -289,6 +299,8 @@ public class MetadataTracker extends AbstractTracker implements Tracker
gnp.setTransactionIds(txs);
gnp.setStoreProtocol(storeRef.getProtocol());
gnp.setStoreIdentifier(storeRef.getIdentifier());
gnp.setShardProperty(shardProperty);
List<Node> nodes = client.getNodes(gnp, (int) info.getUpdates());
for (Node node : nodes)
{
@@ -1060,7 +1072,22 @@ public class MetadataTracker extends AbstractTracker implements Tracker
return false;
}
}
}
public static QName getShardProperty(String field) {
AlfrescoSolrDataModel dataModel = AlfrescoSolrDataModel.getInstance();
NamespaceDAO namespaceDAO = dataModel.getNamespaceDAO();
DictionaryService dictionaryService = dataModel.getDictionaryService(CMISStrictDictionaryService.DEFAULT);
PropertyDefinition propertyDef = QueryParserUtils.matchPropertyDefinition("http://www.alfresco.org/model/content/1.0",
namespaceDAO,
dictionaryService,
field);
return propertyDef.getName();
}
}
search-services/alfresco-solr/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedTest.java
+80 -36
View File
@@ -1,8 +1,6 @@
package org.alfresco.solr;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_DOC_TYPE;
import static org.alfresco.solr.AlfrescoSolrUtils.*;
import static org.alfresco.solr.AlfrescoSolrUtils.list;
import java.io.File;
import java.io.FileInputStream;
@@ -26,15 +24,14 @@ import java.util.Properties;
import java.util.Random;
import java.util.Set;
import java.util.SortedMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import javax.servlet.Filter;
import org.alfresco.repo.index.shard.ShardMethodEnum;
import org.alfresco.solr.AlfrescoCoreAdminHandler;
import org.alfresco.solr.SolrInformationServer;
import org.alfresco.solr.client.*;
import org.alfresco.solr.client.Node;
import org.alfresco.solr.client.NodeMetaData;
import org.alfresco.solr.client.SOLRAPIQueueClient;
import org.alfresco.solr.client.Transaction;
import org.apache.commons.io.FileUtils;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
@@ -62,7 +59,6 @@ import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
@@ -256,7 +252,8 @@ public abstract class AbstractAlfrescoDistributedTest extends SolrTestCaseJ4
* @param waitMillis
* @throws Exception
*/
public void waitForDocCountAllCores(Query query, int count, long waitMillis) throws Exception {
public void waitForDocCountAllCores(Query query, int count, long waitMillis) throws Exception
{
List<SolrCore> cores = getJettyCores(jettyContainers.values());
cores.addAll(getJettyCores(jettyShards));
@@ -273,10 +270,9 @@ public abstract class AbstractAlfrescoDistributedTest extends SolrTestCaseJ4
* @param waitMillis
* @throws Exception
*/
public void waitForDocCount(Query query, int count, long waitMillis) throws Exception {
public void waitForDocCount(Query query, int count, long waitMillis) throws Exception
{
long begin = System.currentTimeMillis();
List<SolrCore> cores = getJettyCores(jettyContainers.values());
//TODO: Support multiple cores per jetty
SolrCore controlCore = cores.get(0); //Get the first one
@@ -292,7 +288,8 @@ public abstract class AbstractAlfrescoDistributedTest extends SolrTestCaseJ4
* @param start
* @throws Exception
*/
public void waitForShardsCount(Query query, int count, long waitMillis, long start) throws Exception {
public void waitForShardsCount(Query query, int count, long waitMillis, long start) throws Exception
{
List<SolrCore> cores = getJettyCores(jettyShards);
long timeOut = start+waitMillis;
int totalCount = 0;
@@ -323,7 +320,8 @@ public abstract class AbstractAlfrescoDistributedTest extends SolrTestCaseJ4
protected List<SolrCore> getJettyCores(Collection<JettySolrRunner> runners)
{
List<SolrCore> cores = new ArrayList();
for (JettySolrRunner jettySolrRunner : runners) {
for (JettySolrRunner jettySolrRunner : runners)
{
jettySolrRunner.getCoreContainer().getCores().forEach(aCore -> cores.add(aCore));
}
return cores;
@@ -338,22 +336,63 @@ public abstract class AbstractAlfrescoDistributedTest extends SolrTestCaseJ4
return jettyClients.get(DEFAULT_TEST_CORENAME);
}
public void assertNodesPerShardGreaterThan(int count) throws Exception {
public void assertNodesPerShardGreaterThan(int count) throws Exception
{
List<SolrCore> cores = getJettyCores(jettyShards);
Query query = new TermQuery(new Term(FIELD_DOC_TYPE, SolrInformationServer.DOC_TYPE_NODE));
for (SolrCore core : cores) {
RefCounted<SolrIndexSearcher> refCounted = null;
try {
refCounted = core.getSearcher();
SolrIndexSearcher searcher = refCounted.get();
TopDocs topDocs = searcher.search(query, 10);
if(topDocs.totalHits < count) {
throw new Exception("Expected nodes per shard greater than "+count+" found "+topDocs.totalHits+" : "+query.toString());
}
} finally {
refCounted.decref();
for (SolrCore core : cores)
{
RefCounted<SolrIndexSearcher> refCounted = null;
try
{
refCounted = core.getSearcher();
SolrIndexSearcher searcher = refCounted.get();
TopDocs topDocs = searcher.search(query, 10);
if(topDocs.totalHits < count)
{
throw new Exception("Expected nodes per shard greater than "+count+" found "+topDocs.totalHits+" : "+query.toString());
}
}
finally
{
refCounted.decref();
}
}
}
public void assertCountAndColocation(Query query, int count) throws Exception
{
List<SolrCore> cores = getJettyCores(jettyShards);
int shardHit = 0;
int totalCount = 0;
for (SolrCore core : cores)
{
RefCounted<SolrIndexSearcher> refCounted = null;
try
{
refCounted = core.getSearcher();
SolrIndexSearcher searcher = refCounted.get();
TopDocs topDocs = searcher.search(query, 10);
totalCount += topDocs.totalHits;
if(topDocs.totalHits > 0)
{
shardHit++;
}
}
finally
{
refCounted.decref();
}
}
if(totalCount != count) {
throw new Exception(totalCount+" docs found for query: "+query.toString()+" expecting "+count);
}
if(shardHit > 1) {
throw new Exception(shardHit+" shards found data for query: "+query.toString()+" expecting 1");
}
}
private void waitForDocCountCore(SolrCore core,
@@ -469,7 +508,8 @@ public abstract class AbstractAlfrescoDistributedTest extends SolrTestCaseJ4
JettySolrRunner jsr = createJetty(jettyKey);
jettyContainers.put(jettyKey, jsr);
for (int i = 0; i < coreNames.length; i++) {
for (int i = 0; i < coreNames.length; i++)
{
addCoreToJetty(jettyKey, coreNames[i], coreNames[i], additionalProperties);
}
@@ -477,7 +517,8 @@ public abstract class AbstractAlfrescoDistributedTest extends SolrTestCaseJ4
startJetty(jsr);
int jettyPort = jsr.getLocalPort();
for (int i = 0; i < coreNames.length; i++) {
for (int i = 0; i < coreNames.length; i++)
{
String url = buildUrl(jettyPort) + "/" + coreNames[i];
log.info(url);
jettyClients.put(coreNames[i], createNewSolrClient(url));
@@ -485,8 +526,6 @@ public abstract class AbstractAlfrescoDistributedTest extends SolrTestCaseJ4
shardsArr = new String[numShards];
StringBuilder sb = new StringBuilder();
String shardMethod = getShardMethod().toString();
log.info("################# shardMethod:"+shardMethod);
for (int i = 0; i < numShards; i++)
{
@@ -494,7 +533,6 @@ public abstract class AbstractAlfrescoDistributedTest extends SolrTestCaseJ4
final String shardname = "shard" + i;
if (additionalProperties == null) additionalProperties = new Properties();
additionalProperties.put("shard.instance", Integer.toString(i));
additionalProperties.put("shard.method", shardMethod);
additionalProperties.put("shard.count", Integer.toString(numShards));
String shardKey = jettyKey+"_shard_"+i;
@@ -513,11 +551,6 @@ public abstract class AbstractAlfrescoDistributedTest extends SolrTestCaseJ4
shards = sb.toString();
}
protected ShardMethodEnum getShardMethod() {
return ShardMethodEnum.DB_ID;
}
protected void setDistributedParams(ModifiableSolrParams params)
{
params.set("shards", getShardsString());
@@ -1490,6 +1523,17 @@ public abstract class AbstractAlfrescoDistributedTest extends SolrTestCaseJ4
this.numShards = numShards;
this.solrcoreProperties = new Properties();
}
/**
* Creates the jetty servers with the specified number of shards and sensible defaults.
* @param numShards
*/
public JettyServerRule(int numShards, Properties solrcoreProperties)
{
this.serverName = DEFAULT_TEST_CORENAME;
coreNames = new String[]{DEFAULT_TEST_CORENAME};
this.numShards = numShards;
this.solrcoreProperties = solrcoreProperties;
}
@Override
protected void before() throws Throwable
search-services/alfresco-solr/src/test/java/org/alfresco/solr/test/AlfrescoSolrReloadTest.java → search-services/alfresco-solr/src/test/java/org/alfresco/solr/AlfrescoSolrReloadTest.java
+1 -2
View File
@@ -1,4 +1,4 @@
package org.alfresco.solr.test;
package org.alfresco.solr;
import org.alfresco.model.ContentModel;
import org.alfresco.repo.search.adaptor.lucene.QueryConstants;
@@ -8,7 +8,6 @@ import org.alfresco.service.cmr.repository.StoreRef;
import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter;
import org.alfresco.service.namespace.NamespaceService;
import org.alfresco.service.namespace.QName;
import org.alfresco.solr.AbstractAlfrescoSolrTests;
import org.alfresco.solr.AlfrescoCoreAdminHandler;
import org.alfresco.solr.AlfrescoSolrDataModel;
import org.alfresco.solr.SolrInformationServer;
search-services/alfresco-solr/src/test/java/org/alfresco/solr/test/CMISDataCreatorTest.java → search-services/alfresco-solr/src/test/java/org/alfresco/solr/CMISDataCreatorTest.java
+1 -1
View File
@@ -16,7 +16,7 @@
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.test;
package org.alfresco.solr;
import java.io.ByteArrayInputStream;
import java.io.UnsupportedEncodingException;
search-services/alfresco-solr/src/test/java/org/alfresco/solr/SolrDataModelTest.java
+1 -1
View File
@@ -97,7 +97,7 @@ public class SolrDataModelTest
assertEquals(4, dataModel.getIndexedFieldNamesForProperty(NAME).getFields().size());
assertEquals(1, dataModel.getIndexedFieldNamesForProperty(CREATION_DATE).getFields().size());
assertEquals(0, dataModel.getIndexedFieldNamesForProperty(IS_IMMUTABLE).getFields().size());
assertEquals(1, dataModel.getIndexedFieldNamesForProperty(IS_PRIVATE_WOKING_COPY).getFields().size());
assertEquals(0, dataModel.getIndexedFieldNamesForProperty(IS_PRIVATE_WOKING_COPY).getFields().size());
assertEquals(1, dataModel.getIndexedFieldNamesForProperty(CONTENT_STREAM_LENGTH).getFields().size());
assertEquals(1, dataModel .getQueryableFields(OBJECT_ID, null, FieldUse.FACET).getFields().size());
search-services/alfresco-solr/src/test/java/org/alfresco/solr/query/LoadAFTSTestData.java
+45 -46
View File
@@ -134,12 +134,13 @@ public class LoadAFTSTestData extends AbstractAlfrescoSolrTests implements Alfre
@BeforeClass
public static void loadTestSet() throws Exception
{
initAlfrescoCore("solrconfig-afts.xml", "schema-afts.xml");
Thread.sleep(1000);
// Root
SolrCore core = h.getCore();
AlfrescoSolrDataModel dataModel = AlfrescoSolrDataModel.getInstance();
dataModel.getNamespaceDAO().removePrefix("");
dataModel.setCMDefaultUri();
rootNodeRef = new NodeRef(new StoreRef("workspace", "SpacesStore"), createGUID());
@@ -151,9 +152,9 @@ public class LoadAFTSTestData extends AbstractAlfrescoSolrTests implements Alfre
n01CAR = new ChildAssociationRef(ContentModel.ASSOC_CHILDREN, rootNodeRef, n01QName,
n01NodeRef, true, 0);
addNode(core, dataModel, 1, 2, 1, testSuperType, null, getOrderProperties(), null, "andy",
new ChildAssociationRef[] { n01CAR }, new NodeRef[] { rootNodeRef }, new String[] { "/"
+ n01QName.toString() }, n01NodeRef, true);
new ChildAssociationRef[]{n01CAR}, new NodeRef[]{rootNodeRef}, new String[]{"/"
+ n01QName.toString()}, n01NodeRef, true);
testNodeRef = n01NodeRef;
// 2
@@ -163,8 +164,8 @@ public class LoadAFTSTestData extends AbstractAlfrescoSolrTests implements Alfre
n02CAR = new ChildAssociationRef(ContentModel.ASSOC_CHILDREN, rootNodeRef, n02QName,
n02NodeRef, true, 0);
addNode(core, dataModel, 1, 3, 1, testSuperType, null, getOrderProperties(), null, "bob",
new ChildAssociationRef[] { n02CAR }, new NodeRef[] { rootNodeRef }, new String[] { "/"
+ n02QName.toString() }, n02NodeRef, true);
new ChildAssociationRef[]{n02CAR}, new NodeRef[]{rootNodeRef}, new String[]{"/"
+ n02QName.toString()}, n02NodeRef, true);
// 3
@@ -173,8 +174,8 @@ public class LoadAFTSTestData extends AbstractAlfrescoSolrTests implements Alfre
n03CAR = new ChildAssociationRef(ContentModel.ASSOC_CHILDREN, rootNodeRef, n03QName,
n03NodeRef, true, 0);
addNode(core, dataModel, 1, 4, 1, testSuperType, null, getOrderProperties(), null, "cid",
new ChildAssociationRef[] { n03CAR }, new NodeRef[] { rootNodeRef }, new String[] { "/"
+ n03QName.toString() }, n03NodeRef, true);
new ChildAssociationRef[]{n03CAR}, new NodeRef[]{rootNodeRef}, new String[]{"/"
+ n03QName.toString()}, n03NodeRef, true);
// 4
@@ -268,9 +269,9 @@ public class LoadAFTSTestData extends AbstractAlfrescoSolrTests implements Alfre
n04NodeRef, true, 0);
properties04.put(QName.createQName(TEST_NAMESPACE, "aspectProperty"), new StringPropertyValue(""));
addNode(core, dataModel, 1, 5, 1, testType, new QName[] { testAspect }, properties04, content04, "dave",
new ChildAssociationRef[] { n04CAR }, new NodeRef[] { rootNodeRef }, new String[] { "/"
+ n04QName.toString() }, n04NodeRef, true);
addNode(core, dataModel, 1, 5, 1, testType, new QName[]{testAspect}, properties04, content04, "dave",
new ChildAssociationRef[]{n04CAR}, new NodeRef[]{rootNodeRef}, new String[]{"/"
+ n04QName.toString()}, n04NodeRef, true);
// 5
@@ -279,8 +280,8 @@ public class LoadAFTSTestData extends AbstractAlfrescoSolrTests implements Alfre
n05CAR = new ChildAssociationRef(ContentModel.ASSOC_CONTAINS, n01NodeRef, n05QName,
n05NodeRef, true, 0);
addNode(core, dataModel, 1, 6, 1, testSuperType, null, getOrderProperties(), null, "eoin",
new ChildAssociationRef[] { n05CAR }, new NodeRef[] { rootNodeRef, n01NodeRef },
new String[] { "/" + n01QName.toString() + "/" + n05QName.toString() }, n05NodeRef, true);
new ChildAssociationRef[]{n05CAR}, new NodeRef[]{rootNodeRef, n01NodeRef},
new String[]{"/" + n01QName.toString() + "/" + n05QName.toString()}, n05NodeRef, true);
// 6
@@ -289,8 +290,8 @@ public class LoadAFTSTestData extends AbstractAlfrescoSolrTests implements Alfre
n06CAR = new ChildAssociationRef(ContentModel.ASSOC_CONTAINS, n01NodeRef, n06QName,
n06NodeRef, true, 0);
addNode(core, dataModel, 1, 7, 1, testSuperType, null, getOrderProperties(), null, "fred",
new ChildAssociationRef[] { n06CAR }, new NodeRef[] { rootNodeRef, n01NodeRef },
new String[] { "/" + n01QName.toString() + "/" + n06QName.toString() }, n06NodeRef, true);
new ChildAssociationRef[]{n06CAR}, new NodeRef[]{rootNodeRef, n01NodeRef},
new String[]{"/" + n01QName.toString() + "/" + n06QName.toString()}, n06NodeRef, true);
// 7
@@ -299,8 +300,8 @@ public class LoadAFTSTestData extends AbstractAlfrescoSolrTests implements Alfre
n07CAR = new ChildAssociationRef(ContentModel.ASSOC_CONTAINS, n02NodeRef, n07QName,
n07NodeRef, true, 0);
addNode(core, dataModel, 1, 8, 1, testSuperType, null, getOrderProperties(), null, "gail",
new ChildAssociationRef[] { n07CAR }, new NodeRef[] { rootNodeRef, n02NodeRef },
new String[] { "/" + n02QName.toString() + "/" + n07QName.toString() }, n07NodeRef, true);
new ChildAssociationRef[]{n07CAR}, new NodeRef[]{rootNodeRef, n02NodeRef},
new String[]{"/" + n02QName.toString() + "/" + n07QName.toString()}, n07NodeRef, true);
// 8
@@ -316,10 +317,10 @@ public class LoadAFTSTestData extends AbstractAlfrescoSolrTests implements Alfre
n08NodeRef, true, 0);
addNode(core, dataModel, 1, 9, 1, testSuperType, null, getOrderProperties(), null, "hal",
new ChildAssociationRef[] { n08CAR_0, n08CAR_1, n08CAR_2 }, new NodeRef[] { rootNodeRef,
rootNodeRef, n01NodeRef, rootNodeRef, n02NodeRef }, new String[] {
new ChildAssociationRef[]{n08CAR_0, n08CAR_1, n08CAR_2}, new NodeRef[]{rootNodeRef,
rootNodeRef, n01NodeRef, rootNodeRef, n02NodeRef}, new String[]{
"/" + n08QName_0, "/" + n01QName.toString() + "/" + n08QName_1.toString(),
"/" + n02QName.toString() + "/" + n08QName_2.toString() }, n08NodeRef, true);
"/" + n02QName.toString() + "/" + n08QName_2.toString()}, n08NodeRef, true);
// 9
@@ -328,8 +329,8 @@ public class LoadAFTSTestData extends AbstractAlfrescoSolrTests implements Alfre
n09CAR = new ChildAssociationRef(ContentModel.ASSOC_CONTAINS, n05NodeRef, n09QName,
n09NodeRef, true, 0);
addNode(core, dataModel, 1, 10, 1, testSuperType, null, getOrderProperties(), null, "ian",
new ChildAssociationRef[] { n09CAR }, new NodeRef[] { rootNodeRef, n01NodeRef, n05NodeRef },
new String[] { "/" + n01QName.toString() + "/" + n05QName.toString() + "/" + n09QName },
new ChildAssociationRef[]{n09CAR}, new NodeRef[]{rootNodeRef, n01NodeRef, n05NodeRef},
new String[]{"/" + n01QName.toString() + "/" + n05QName.toString() + "/" + n09QName},
n09NodeRef, true);
// 10
@@ -339,8 +340,8 @@ public class LoadAFTSTestData extends AbstractAlfrescoSolrTests implements Alfre
n10CAR = new ChildAssociationRef(ContentModel.ASSOC_CONTAINS, n05NodeRef, n10QName,
n10NodeRef, true, 0);
addNode(core, dataModel, 1, 11, 1, testSuperType, null, getOrderProperties(), null, "jake",
new ChildAssociationRef[] { n10CAR }, new NodeRef[] { rootNodeRef, n01NodeRef, n05NodeRef },
new String[] { "/" + n01QName.toString() + "/" + n05QName.toString() + "/" + n10QName },
new ChildAssociationRef[]{n10CAR}, new NodeRef[]{rootNodeRef, n01NodeRef, n05NodeRef},
new String[]{"/" + n01QName.toString() + "/" + n05QName.toString() + "/" + n10QName},
n10NodeRef, true);
// 11
@@ -350,8 +351,8 @@ public class LoadAFTSTestData extends AbstractAlfrescoSolrTests implements Alfre
n11CAR = new ChildAssociationRef(ContentModel.ASSOC_CONTAINS, n05NodeRef, n11QName,
n11NodeRef, true, 0);
addNode(core, dataModel, 1, 12, 1, testSuperType, null, getOrderProperties(), null, "kara",
new ChildAssociationRef[] { n11CAR }, new NodeRef[] { rootNodeRef, n01NodeRef, n05NodeRef },
new String[] { "/" + n01QName.toString() + "/" + n05QName.toString() + "/" + n11QName },
new ChildAssociationRef[]{n11CAR}, new NodeRef[]{rootNodeRef, n01NodeRef, n05NodeRef},
new String[]{"/" + n01QName.toString() + "/" + n05QName.toString() + "/" + n11QName},
n11NodeRef, true);
// 12
@@ -361,8 +362,8 @@ public class LoadAFTSTestData extends AbstractAlfrescoSolrTests implements Alfre
n12CAR = new ChildAssociationRef(ContentModel.ASSOC_CONTAINS, n05NodeRef, n12QName,
n12NodeRef, true, 0);
addNode(core, dataModel, 1, 13, 1, testSuperType, null, getOrderProperties(), null, "loon",
new ChildAssociationRef[] { n12CAR }, new NodeRef[] { rootNodeRef, n01NodeRef, n05NodeRef },
new String[] { "/" + n01QName.toString() + "/" + n05QName.toString() + "/" + n12QName },
new ChildAssociationRef[]{n12CAR}, new NodeRef[]{rootNodeRef, n01NodeRef, n05NodeRef},
new String[]{"/" + n01QName.toString() + "/" + n05QName.toString() + "/" + n12QName},
n12NodeRef, true);
// 13
@@ -375,11 +376,11 @@ public class LoadAFTSTestData extends AbstractAlfrescoSolrTests implements Alfre
n13CARLink = new ChildAssociationRef(ContentModel.ASSOC_CONTAINS, n02NodeRef, n13QName,
n13NodeRef, false, 0);
addNode(core, dataModel, 1, 14, 1, testSuperType, null, getOrderProperties(), null, "mike",
new ChildAssociationRef[] { n13CAR, n13CARLink }, new NodeRef[] { rootNodeRef, n01NodeRef,
n05NodeRef, n12NodeRef, rootNodeRef, n02NodeRef },
new String[] {
new ChildAssociationRef[]{n13CAR, n13CARLink}, new NodeRef[]{rootNodeRef, n01NodeRef,
n05NodeRef, n12NodeRef, rootNodeRef, n02NodeRef},
new String[]{
"/" + n01QName.toString() + "/" + n05QName.toString() + "/" + n12QName + "/"
+ n13QName, "/" + n02QName.toString() + "/" + n13QNameLink },
+ n13QName, "/" + n02QName.toString() + "/" + n13QNameLink},
n13NodeRef, true);
// 14
@@ -392,12 +393,9 @@ public class LoadAFTSTestData extends AbstractAlfrescoSolrTests implements Alfre
desc1.addValue(Locale.US, "Alfresco tutorial");
Date explicitCreatedDate = new Date();
try
{
try {
Thread.sleep(2000);
}
catch (InterruptedException e)
{
} catch (InterruptedException e) {
e.printStackTrace();
}
@@ -438,10 +436,10 @@ public class LoadAFTSTestData extends AbstractAlfrescoSolrTests implements Alfre
n14QNameCommon, n14NodeRef, false, 0);
n14CAR_13 = new ChildAssociationRef(ContentModel.ASSOC_CONTAINS, n13NodeRef,
n14QNameCommon, n14NodeRef, false, 0);
addNode(core, dataModel, 1, 15, 1, ContentModel.TYPE_CONTENT, new QName[] {ContentModel.ASPECT_TITLED }, properties14, content14, "noodle",
new ChildAssociationRef[] { n14CAR, n14CAR_1, n14CAR_2, n14CAR_5, n14CAR_6, n14CAR_12,
n14CAR_13 }, new NodeRef[] { rootNodeRef, n01NodeRef, n05NodeRef, n12NodeRef,
n13NodeRef }, new String[] {
addNode(core, dataModel, 1, 15, 1, ContentModel.TYPE_CONTENT, new QName[]{ContentModel.ASPECT_TITLED}, properties14, content14, "noodle",
new ChildAssociationRef[]{n14CAR, n14CAR_1, n14CAR_2, n14CAR_5, n14CAR_6, n14CAR_12,
n14CAR_13}, new NodeRef[]{rootNodeRef, n01NodeRef, n05NodeRef, n12NodeRef,
n13NodeRef}, new String[]{
"/" + n01QName.toString() + "/" + n05QName.toString() + "/" + n12QName + "/"
+ n13QName + "/" + n14QName,
"/" + n02QName.toString() + "/" + n13QNameLink + "/" + n14QName,
@@ -451,7 +449,7 @@ public class LoadAFTSTestData extends AbstractAlfrescoSolrTests implements Alfre
"/" + n01QName + "/" + n06QName + "/" + n14QNameCommon,
"/" + n01QName + "/" + n05QName + "/" + n12QName + "/" + n14QNameCommon,
"/" + n01QName + "/" + n05QName + "/" + n12QName + "/" + n13QName + "/"
+ n14QNameCommon }, n14NodeRef, true);
+ n14QNameCommon}, n14NodeRef, true);
// 15
@@ -468,12 +466,13 @@ public class LoadAFTSTestData extends AbstractAlfrescoSolrTests implements Alfre
n15CAR = new ChildAssociationRef(ContentModel.ASSOC_CONTAINS, n13NodeRef, n15QName,
n15NodeRef, true, 0);
addNode(core, dataModel, 1, 16, 1, ContentModel.TYPE_THUMBNAIL, null, properties15, content15, "ood",
new ChildAssociationRef[] { n15CAR }, new NodeRef[] { rootNodeRef, n01NodeRef, n05NodeRef,
n12NodeRef, n13NodeRef },
new String[] {
new ChildAssociationRef[]{n15CAR}, new NodeRef[]{rootNodeRef, n01NodeRef, n05NodeRef,
n12NodeRef, n13NodeRef},
new String[]{
"/" + n01QName.toString() + "/" + n05QName.toString() + "/" + n12QName + "/"
+ n13QName + "/" + n15QName,
"/" + n02QName.toString() + "/" + n13QNameLink + "/" + n14QName }, n15NodeRef, true);
"/" + n02QName.toString() + "/" + n13QNameLink + "/" + n14QName}, n15NodeRef, true);
}
private static Map<QName, PropertyValue> getOrderProperties()
search-services/alfresco-solr/src/test/java/org/alfresco/solr/query/cmis/LoadCMISData.java
+7 -6
View File
@@ -197,6 +197,7 @@ public class LoadCMISData extends AbstractAlfrescoSolrTests
initAlfrescoCore("solrconfig-afts.xml", "schema-afts.xml");
SolrCore core = h.getCore();
AlfrescoSolrDataModel dataModel = AlfrescoSolrDataModel.getInstance();
dataModel.getNamespaceDAO().removePrefix("");
dataModel.setCMDefaultUri();
NodeRef rootNodeRef = new NodeRef(new StoreRef("workspace", "SpacesStore"), createGUID());
testCMISRootNodeRef = rootNodeRef;
@@ -862,13 +863,13 @@ public class LoadCMISData extends AbstractAlfrescoSolrTests
NodeRef content00NodeRef = new NodeRef(new StoreRef("workspace", "SpacesStore"), createGUID());
QName content00QName = QName.createQName(NamespaceService.CONTENT_MODEL_1_0_URI, "Test " + position);
ChildAssociationRef content00CAR = new ChildAssociationRef(ContentModel.ASSOC_CONTAINS, folder00NodeRef,
content00QName, content00NodeRef, true, 0);
content00QName, content00NodeRef, true, 0);
addNode(h.getCore(),
dataModel, 1, 1000 + position, 1, extendedContent, new QName[] { ContentModel.ASPECT_OWNABLE,
ContentModel.ASPECT_TITLED }, content00Properties, null, "andy",
new ChildAssociationRef[] { content00CAR }, new NodeRef[] { baseFolderNodeRef, rootNodeRef,
folder00NodeRef }, new String[] { "/" + baseFolderQName.toString() + "/"
+ folder00QName.toString() + "/" + content00QName.toString() }, content00NodeRef, true);
dataModel, 1, 1000 + position, 1, extendedContent, new QName[]{ContentModel.ASPECT_OWNABLE,
ContentModel.ASPECT_TITLED}, content00Properties, null, "andy",
new ChildAssociationRef[]{content00CAR}, new NodeRef[]{baseFolderNodeRef, rootNodeRef,
folder00NodeRef}, new String[]{"/" + baseFolderQName.toString() + "/"
+ folder00QName.toString() + "/" + content00QName.toString() }, content00NodeRef, true);
}
private static String[] orderable = new String[] { "zero loons", "one banana", "two apples", "three fruit",
search-services/alfresco-solr/src/test/java/org/alfresco/solr/tracker/DistributedAclIdAlfrescoSolrTrackerTest.java
+29 -16
View File
@@ -18,10 +18,31 @@
*/
package org.alfresco.solr.tracker;
import org.alfresco.solr.AbstractAlfrescoDistributedTest;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_DOC_TYPE;
import static org.alfresco.solr.AlfrescoSolrUtils.getAcl;
import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet;
import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders;
import static org.alfresco.solr.AlfrescoSolrUtils.getNode;
import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData;
import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction;
import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet;
import static org.alfresco.solr.AlfrescoSolrUtils.list;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import java.util.Random;
import org.alfresco.repo.index.shard.ShardMethodEnum;
import org.alfresco.solr.AbstractAlfrescoDistributedTest;
import org.alfresco.solr.SolrInformationServer;
import org.alfresco.solr.client.*;
import org.alfresco.solr.client.Acl;
import org.alfresco.solr.client.AclChangeSet;
import org.alfresco.solr.client.AclReaders;
import org.alfresco.solr.client.Node;
import org.alfresco.solr.client.NodeMetaData;
import org.alfresco.solr.client.Transaction;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.LuceneTestCase;
@@ -30,16 +51,6 @@ import org.apache.solr.client.solrj.response.QueryResponse;
import org.junit.Rule;
import org.junit.Test;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_DOC_TYPE;
import static org.alfresco.solr.AlfrescoSolrUtils.*;
import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders;
import static org.alfresco.solr.AlfrescoSolrUtils.list;
import java.util.Collections;
import java.util.List;
import java.util.ArrayList;
import java.util.Random;
/**
* @author Joel
*/
@@ -48,7 +59,7 @@ import java.util.Random;
public class DistributedAclIdAlfrescoSolrTrackerTest extends AbstractAlfrescoDistributedTest
{
@Rule
public JettyServerRule jetty = new JettyServerRule(2);
public JettyServerRule jetty = new JettyServerRule(2,getShardMethod());
@Test
public void testAclId() throws Exception
@@ -116,14 +127,16 @@ public class DistributedAclIdAlfrescoSolrTrackerTest extends AbstractAlfrescoDis
}
}
protected ShardMethodEnum getShardMethod() {
protected Properties getShardMethod()
{
Random random = random();
List<ShardMethodEnum> methods = new ArrayList();
methods.add(ShardMethodEnum.ACL_ID);
methods.add(ShardMethodEnum.MOD_ACL_ID);
Collections.shuffle(methods, random);
return methods.get(0);
Properties prop = new Properties();
prop.put("shard.method", methods.get(0).toString());
return prop;
}
}
search-services/alfresco-solr/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java
+156
View File
@@ -0,0 +1,156 @@
/*
* Copyright (C) 2005-2014 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.tracker;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_DOC_TYPE;
import static org.alfresco.solr.AlfrescoSolrUtils.getAcl;
import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet;
import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders;
import static org.alfresco.solr.AlfrescoSolrUtils.getNode;
import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData;
import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction;
import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet;
import static org.alfresco.solr.AlfrescoSolrUtils.list;
import java.util.*;
import org.alfresco.model.ContentModel;
import org.alfresco.repo.index.shard.ShardMethodEnum;
import org.alfresco.repo.search.adaptor.lucene.QueryConstants;
import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter;
import org.alfresco.solr.AbstractAlfrescoDistributedTest;
import org.alfresco.solr.AlfrescoSolrDataModel;
import org.alfresco.solr.SolrInformationServer;
import org.alfresco.solr.client.*;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.LegacyNumericRangeQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.schema.TrieDateField;
import org.junit.Rule;
import org.junit.Test;
/**
* @author Joel
*
*
*
*
*/
@SolrTestCaseJ4.SuppressSSL
@LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"})
public class DistributedDateMonthAlfrescoSolrTrackerTest extends AbstractAlfrescoDistributedTest
{
@Rule
public JettyServerRule jetty = new JettyServerRule(2,getShardMethod());
//@Test
public void testDateMonth() throws Exception
{
handle.put("explain", SKIPVAL);
handle.put("timestamp", SKIPVAL);
handle.put("score", SKIPVAL);
handle.put("wt", SKIP);
handle.put("distrib", SKIP);
handle.put("shards.qt", SKIP);
handle.put("shards", SKIP);
handle.put("q", SKIP);
handle.put("maxScore", SKIPVAL);
handle.put("_version_", SKIP);
handle.put("_original_parameters_", SKIP);
int numAcls = 25;
AclChangeSet bulkAclChangeSet = getAclChangeSet(numAcls);
List<Acl> bulkAcls = new ArrayList();
List<AclReaders> bulkAclReaders = new ArrayList();
for (int i = 0; i < numAcls; i++) {
Acl bulkAcl = getAcl(bulkAclChangeSet);
bulkAcls.add(bulkAcl);
bulkAclReaders.add(getAclReaders(bulkAclChangeSet,
bulkAcl,
list("joel" + bulkAcl.getId()),
list("phil" + bulkAcl.getId()),
null));
}
indexAclChangeSet(bulkAclChangeSet,
bulkAcls,
bulkAclReaders);
int numNodes = 1000;
List<Node> nodes = new ArrayList();
List<NodeMetaData> nodeMetaDatas = new ArrayList();
Transaction bigTxn = getTransaction(0, numNodes);
Date[] dates = new Date[10];
Calendar cal = new GregorianCalendar();
for (int i = 0; i < dates.length; i++) {
cal.add(cal.MONTH, -i);
dates[i] = cal.getTime();
}
int[] counts = new int[dates.length];
for (int i = 0; i < numNodes; i++) {
int aclIndex = i % numAcls;
int dateIndex = i % dates.length;
String dateString = DefaultTypeConverter.INSTANCE.convert(String.class, dates[dateIndex]);
counts[dateIndex]++;
Node node = getNode(bigTxn, bulkAcls.get(aclIndex), Node.SolrApiNodeStatus.UPDATED);
node.setShardPropertyValue(dateString);
nodes.add(node);
NodeMetaData nodeMetaData = getNodeMetaData(node, bigTxn, bulkAcls.get(aclIndex), "mike", null, false);
nodeMetaData.getProperties().put(ContentModel.PROP_CREATED,
new StringPropertyValue(dateString));
nodeMetaDatas.add(nodeMetaData);
}
indexTransaction(bigTxn, nodes, nodeMetaDatas);
waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), numNodes, 100000);
waitForDocCountAllCores(new TermQuery(new Term(FIELD_DOC_TYPE, SolrInformationServer.DOC_TYPE_ACL)), numAcls, 100000);
List<AlfrescoSolrDataModel.FieldInstance> fieldInstanceList = AlfrescoSolrDataModel.getInstance().getIndexedFieldNamesForProperty(MetadataTracker.getShardProperty("created")).getFields();
AlfrescoSolrDataModel.FieldInstance fieldInstance = fieldInstanceList.get(0);
String fieldName = fieldInstance.getField();
for (int i = 0; i < dates.length; i++) {
LegacyNumericRangeQuery query = LegacyNumericRangeQuery.newLongRange(fieldName, dates[i].getTime(), dates[i].getTime() + 1, true, false);
assertCountAndColocation(query, counts[i]);
}
}
protected Properties getShardMethod()
{
Properties prop = new Properties();
prop.put("shard.method", ShardMethodEnum.DATE_MONTH.toString());
return prop;
}
}
search-services/packaging/Dockerfile
-1
View File
@@ -20,7 +20,6 @@ WORKDIR $DIST_DIR
RUN unzip *.zip -d $DIST_DIR/ && rm *.zip && \
mkdir -p $DIST_DIR/data/content && \
mkdir -p $DIST_DIR/data/indexroot && \
mv $DIST_DIR/solrhome/alfrescoModels $DIST_DIR/data/ && \
mv $DIST_DIR/solrhome $DIST_DIR/data/ && \
chown -R solr:solr $DIST_DIR
search-services/packaging/src/test/resources/solr.in.sh
+1 -1
View File
@@ -2,4 +2,4 @@
# Path to a directory for Solr to store cores and their data. By default, Solr will use server/solr
# This directory needs to contain solr.xml
SOLR_HOME=$DIST_DIR/data/solrhome
SOLR_OPTS="$SOLR_OPTS -Dsolr.data.dir.root=/opt/alfresco-solr/data/indexroot -Dsolr.solr.content.dir=/opt/alfresco-solr/data/content -Dsolr.solr.model.dir=/opt/alfresco-solr/data/alfrescoModels"
SOLR_OPTS="$SOLR_OPTS -Dsolr.data.dir.root=/opt/alfresco-solr/data -Dsolr.solr.content.dir=/opt/alfresco-solr/data/content -Dsolr.solr.model.dir=/opt/alfresco-solr/data/alfrescoModels"