diff --git a/config/alfresco/caches.properties b/config/alfresco/caches.properties index 89f42a117c..520c7c8d9e 100644 --- a/config/alfresco/caches.properties +++ b/config/alfresco/caches.properties @@ -658,7 +658,7 @@ cache.shardStateSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} cache.shardStateSharedCache.maxItems=500 cache.shardStateSharedCache.timeToLiveSeconds=1800 cache.shardStateSharedCache.maxIdleSeconds=0 -cache.shardStateSharedCache.cluster.type=fully-distributed +cache.shardStateSharedCache.cluster.type=invalidating cache.shardStateSharedCache.backup-count=1 cache.shardStateSharedCache.eviction-policy=LRU cache.shardStateSharedCache.eviction-percentage=25 @@ -673,7 +673,7 @@ cache.shardToGuidSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} cache.shardToGuidSharedCache.maxItems=500 cache.shardToGuidSharedCache.timeToLiveSeconds=0 cache.shardToGuidSharedCache.maxIdleSeconds=0 -cache.shardToGuidSharedCache.cluster.type=fully-distributed +cache.shardToGuidSharedCache.cluster.type=invalidating cache.shardToGuidSharedCache.backup-count=1 cache.shardToGuidSharedCache.eviction-policy=LRU cache.shardToGuidSharedCache.eviction-percentage=25 diff --git a/config/alfresco/subsystems/Search/common-search-context.xml b/config/alfresco/subsystems/Search/common-search-context.xml index 4f91f55163..c2b1d5c974 100644 --- a/config/alfresco/subsystems/Search/common-search-context.xml +++ b/config/alfresco/subsystems/Search/common-search-context.xml @@ -20,56 +20,6 @@ - - - - - - - - org.alfresco.repo.index.shard.ShardRegistry - - - - - - - - - - - - - - - - - - - - - - ${server.transaction.mode.readOnly} - ${server.transaction.mode.readOnly} - ${server.transaction.mode.default} - - - - - - - - - - - ${search.solrShardRegistry.purgeOnInit} - - - ${search.solrShardRegistry.shardInstanceTimeoutInSeconds} - - - ${search.solrShardRegistry.maxAllowedReplicaTxCountDifference} - diff --git a/config/alfresco/subsystems/Search/solr4/solr-search-context.xml b/config/alfresco/subsystems/Search/solr4/solr-search-context.xml index e7201c271b..dc8b63e7a3 100644 --- a/config/alfresco/subsystems/Search/solr4/solr-search-context.xml +++ b/config/alfresco/subsystems/Search/solr4/solr-search-context.xml @@ -43,9 +43,6 @@ - - - @@ -78,6 +75,7 @@ + diff --git a/config/alfresco/tx-cache-context.xml b/config/alfresco/tx-cache-context.xml index cb7ed3a24c..e059e691a6 100644 --- a/config/alfresco/tx-cache-context.xml +++ b/config/alfresco/tx-cache-context.xml @@ -700,7 +700,7 @@ org.alfresco.cache.shardStateTransactionalCache - + @@ -716,7 +716,7 @@ org.alfresco.cache.shardToGuidTransactionalCache - + diff --git a/source/java/org/alfresco/repo/index/shard/ShardRegistryImpl.java b/source/java/org/alfresco/repo/index/shard/ShardRegistryImpl.java deleted file mode 100644 index 76ee440b7d..0000000000 --- a/source/java/org/alfresco/repo/index/shard/ShardRegistryImpl.java +++ /dev/null @@ -1,583 +0,0 @@ -/* - * Copyright (C) 2005-2015 Alfresco Software Limited. - * - * This file is part of Alfresco - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.repo.index.shard; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Random; -import java.util.Set; - -import org.alfresco.repo.cache.SimpleCache; -import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback; -import org.alfresco.service.cmr.attributes.AttributeService; -import org.alfresco.service.cmr.attributes.AttributeService.AttributeQueryCallback; -import org.alfresco.service.cmr.search.SearchParameters; -import org.alfresco.service.transaction.TransactionService; -import org.alfresco.util.GUID; -import org.alfresco.util.Pair; - -import com.hazelcast.util.ConcurrentHashSet; - -/** - * @author Andy - */ -public class ShardRegistryImpl implements ShardRegistry -{ - /** - * - */ - public static final String INSTANCE_STATE = "instance.state"; - - public enum ReplicaState {ACTIVE, LAGGING, SILENT }; - - /** - * The best shard sould be at the top; - * @author Andy - * - */ - public static class FlocComparator implements Comparator>>> - { - public FlocComparator() - { - - } - - @Override - public int compare(Pair>> left, Pair>> right) - { - double leftTxCount = 0; - for(HashSet states : left.getSecond().values()) - { - long shardMaxTxCount = 0; - for(ShardState state : states) - { - shardMaxTxCount = Math.max(shardMaxTxCount, state.getLastIndexedTxId()); - } - leftTxCount += ((double)shardMaxTxCount)/left.getFirst().getNumberOfShards(); - } - - double rightTxCount = 0; - for(HashSet states : right.getSecond().values()) - { - long shardMaxTxCount = 0; - for(ShardState state : states) - { - shardMaxTxCount = Math.max(shardMaxTxCount, state.getLastIndexedTxId()); - } - rightTxCount += ((double)shardMaxTxCount)/right.getFirst().getNumberOfShards(); - } - return (int)(rightTxCount - leftTxCount); - } - - } - - private static String SHARD_STATE_KEY = ".SHARD_STATE"; - - private AttributeService attributeService; - - private SimpleCache shardStateCache; - - private SimpleCache shardToGuidCache; - - private ConcurrentHashSet knownFlocks = new ConcurrentHashSet(); - - private Random random = new Random(123); - - private boolean purgeOnInit = false; - - TransactionService transactionService; - - private long shardInstanceTimeoutInSeconds = 300; - - private long maxAllowedReplicaTxCountDifference = 1000; - - public ShardRegistryImpl() - { - } - - public void init() - { - if(purgeOnInit && (transactionService != null)) - { - transactionService.getRetryingTransactionHelper().doInTransaction(new RetryingTransactionCallback() - { - - @Override - public Object execute() throws Throwable - { - purge(); - return null; - } - }, false, true); - - } - } - - public void setAttributeService(AttributeService attributeService) - { - this.attributeService = attributeService; - } - - public void setShardStateCache(SimpleCache shardStateCache) - { - this.shardStateCache = shardStateCache; - } - - public void setShardToGuidCache(SimpleCache shardToGuidCache) - { - this.shardToGuidCache = shardToGuidCache; - } - - public void setPurgeOnInit(boolean purgeOnInit) - { - this.purgeOnInit = purgeOnInit; - } - - public void setShardInstanceTimeoutInSeconds(int shardInstanceTimeoutInSeconds) - { - this.shardInstanceTimeoutInSeconds = shardInstanceTimeoutInSeconds; - } - - - - /** - * @param maxAllowedReplicaTxCountDifference the maxAllowedReplicaTxCountDifference to set - */ - public void setMaxAllowedReplicaTxCountDifference(long maxAllowedReplicaTxCountDifference) - { - this.maxAllowedReplicaTxCountDifference = maxAllowedReplicaTxCountDifference; - } - - /** - * @param transactionService the transactionService to set - */ - public void setTransactionService(TransactionService transactionService) - { - this.transactionService = transactionService; - } - - public void purge() - { - ShardStateCollector shardStates = getPersistedShardStates(); - for(String guid : shardStates.shardGuids.values()) - { - DeleteCallBack dcb = new DeleteCallBack(attributeService, guid); - transactionService.getRetryingTransactionHelper().doInTransaction(dcb, false, true); - } - } - - private static class DeleteCallBack implements RetryingTransactionCallback - { - AttributeService attributeService; - - String guid; - - DeleteCallBack(AttributeService attributeService, String guid) - { - this.attributeService = attributeService; - this.guid = guid; - } - - /* (non-Javadoc) - * @see org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback#execute() - */ - @Override - public Object execute() throws Throwable - { - attributeService.removeAttributes(SHARD_STATE_KEY, guid); - return null; - } - - } - - /** - * @param shardState - */ - public void registerShardState(final ShardState shardState) - { - - transactionService.getRetryingTransactionHelper().doInTransaction(new RetryingTransactionCallback() - { - - @Override - public Object execute() throws Throwable - { - String guid = getPersistedShardStatusGuid(shardState.getShardInstance()); - attributeService.setAttribute(shardState, SHARD_STATE_KEY, guid); - shardStateCache.put(shardState.getShardInstance(), shardState); - knownFlocks.add(shardState.getShardInstance().getShard().getFloc()); - return null; - } - }, false, true); - - - } - - public List getIndexSlice(SearchParameters searchParameters) - { - Set flocs = findFlocsFromKnown(searchParameters); - if (flocs.size() == 0) - { - updateKnownFlocs(); - flocs = findFlocsFromKnown(searchParameters); - } - return selectShardInstancesForBestFlock(flocs); - - } - - private ArrayList>>> buildIndexes(Set flocs, boolean excludeTimedOut) - { - ArrayList>>> indexes = new ArrayList>>> (); - - for(Floc floc : flocs) - { - HashMap> index = new HashMap>(); - getShardStatesFromCache(floc, index, excludeTimedOut); - if (index.size() < floc.getNumberOfShards()) - { - updateShardStateCache(floc); - getShardStatesFromCache(floc, index, excludeTimedOut); - } - indexes.add(new Pair>>(floc, index)); - } - return indexes; - } - - - private List selectShardInstancesForBestFlock(Set flocs) - { - ArrayList>>> indexes = buildIndexes(flocs, true); - - Collections.sort(indexes, new FlocComparator()); - - Pair>> best = indexes.get(0); - ArrayList slice = new ArrayList(best.getFirst().getNumberOfShards()); - for (Shard shard : best.getSecond().keySet()) - { - // Only allow replicas within some fraction of the max TxId - ShardState[] allowedInstances = getAllowedInstances(best.getSecond().get(shard)); - int position = random.nextInt(allowedInstances.length); - ShardInstance instance = allowedInstances[position].getShardInstance(); - slice.add(instance); - } - return slice; - } - - - /** - * @param hashSet - * @return - */ - private ShardState[] getAllowedInstances(HashSet states) - { - HashSet allowed = new HashSet(); - - long maxTxId = 0; - for(ShardState state :states) - { - maxTxId = Math.max(maxTxId, state.getLastIndexedTxId()); - } - - for(ShardState state :states) - { - if( isShardAllowed(maxTxId, state)) - { - allowed.add(state); - } - } - - - return allowed.toArray(new ShardState[] {}); - } - - /** - * @param maxTxId - * @param state - * @return - */ - private boolean isShardAllowed(long maxTxId, ShardState state) - { - return (maxTxId - state.getLastIndexedTxId()) <= maxAllowedReplicaTxCountDifference; - } - - /** - * @param floc - */ - private void updateShardStateCache(Floc floc) - { - ShardStateCollector shardStates = getPersistedShardStates(); - HashMap> shards = shardStates.getIndexes().get(floc); - if(shards != null) - { - for (HashSet set : shards.values()) - { - for (ShardState instance : set) - { - shardStateCache.put(instance.getShardInstance(), instance); - } - } - } - } - - /** - * @param floc - * @param index - */ - private void getShardStatesFromCache(Floc floc, HashMap> index, boolean excludeTimedOut) - { - long now = System.currentTimeMillis(); - for (ShardInstance instance : shardStateCache.getKeys()) - { - ShardState state = shardStateCache.get(instance); - if(excludeTimedOut && isShardTimedOut(now, state) ) - { - continue; - } - - if (instance.getShard().getFloc().equals(floc)) - { - HashSet replicas = index.get(instance.getShard()); - if (replicas == null) - { - replicas = new HashSet(); - index.put(instance.getShard(), replicas); - } - replicas.add(state); - } - } - } - - /** - * @param now - * @param state - * @return - */ - private boolean isShardTimedOut(long now, ShardState state) - { - return (now - state.getLastUpdated()) > (shardInstanceTimeoutInSeconds * 1000); - } - - private void updateKnownFlocs() - { - ShardStateCollector shardStates = getPersistedShardStates(); - knownFlocks.addAll(shardStates.getIndexes().keySet()); - } - - private HashSet findFlocsFromKnown(SearchParameters searchParameters) - { - HashSet flocs = new HashSet(); - for (Floc floc : knownFlocks) - { - if (floc.getStoreRefs().containsAll(searchParameters.getStores())) - { - flocs.add(floc); - } - } - return flocs; - } - - private Floc getBestFloc(Floc best, Floc floc) - { - if (best == null) - { - return floc; - } - - - if (best.getNumberOfShards() >= floc.getNumberOfShards()) - { - return best; - } - else - { - return floc; - } - } - - private String getPersistedShardStatusGuid(ShardInstance shardInstance) - { - String guid = shardToGuidCache.get(shardInstance); - if (guid == null) - { - ShardStateCollector shardStates = getPersistedShardStates(); - for (ShardInstance instance : shardStates.getShardGuids().keySet()) - { - if (!shardToGuidCache.contains(instance)) - { - shardToGuidCache.put(instance, shardStates.getShardGuids().get(instance)); - } - } - guid = shardToGuidCache.get(shardInstance); - } - if(guid == null) - { - guid = GUID.generate(); - shardToGuidCache.put(shardInstance, guid); - } - return guid; - } - - private ShardStateCollector getPersistedShardStates() - { - ShardStateCollector shardStateCollector = new ShardStateCollector(); - attributeService.getAttributes(shardStateCollector, SHARD_STATE_KEY); - knownFlocks.addAll(shardStateCollector.getIndexes().keySet()); - return shardStateCollector; - } - - protected static class ShardStateCollector implements AttributeQueryCallback - { - HashMap shardGuids = new HashMap(); - - HashMap>> indexes = new HashMap>>(); - - public ShardStateCollector() - { - } - - @Override - public boolean handleAttribute(Long id, Serializable value, Serializable[] keys) - { - if(value == null) - { - return true; - } - - - String shardInstanceGuid = (String) keys[1]; - - ShardState shardState = (ShardState) value; - - shardGuids.put(shardState.getShardInstance(), shardInstanceGuid); - - HashMap> shards = indexes.get(shardState.getShardInstance().getShard().getFloc()); - if (shards == null) - { - shards = new HashMap>(); - indexes.put(shardState.getShardInstance().getShard().getFloc(), shards); - } - HashSet shardInstances = shards.get(shardState.getShardInstance().getShard()); - if (shardInstances == null) - { - shardInstances = new HashSet(); - shards.put(shardState.getShardInstance().getShard(), shardInstances); - } - shardInstances.add(shardState); - - return true; - } - - /** - * @return the shardGuids - */ - public HashMap getShardGuids() - { - return shardGuids; - } - - /** - * @return the indexes - */ - public HashMap>> getIndexes() - { - return indexes; - } - - } - - /* (non-Javadoc) - * @see org.alfresco.repo.index.shard.ShardRegistry#getFlocs() - */ - @Override - public HashMap>> getFlocs() - { - ShardStateCollector shardStatesCollector = getPersistedShardStates(); - HashMap>> flocs = shardStatesCollector.getIndexes(); - - long now = System.currentTimeMillis(); - for (Floc floc : flocs.keySet()) - { - HashMap> shards = flocs.get(floc); - for(Shard shard : shards.keySet()) - { - HashSet instances = shards.get(shard); - - - long minTxId = Long.MAX_VALUE; - long maxTxId = 0; - for(ShardState state : instances) - { - minTxId = Math.min(minTxId, state.getLastIndexedTxId()); - maxTxId = Math.max(maxTxId, state.getLastIndexedTxId()); - } - - for(ShardState state : instances) - { - if(isShardTimedOut(now, state)) - { - state.getPropertyBag().put(INSTANCE_STATE, ReplicaState.SILENT.toString()); - } - else if(isShardAllowed(maxTxId, state)) - { - state.getPropertyBag().put(INSTANCE_STATE, ReplicaState.ACTIVE.toString()); - } - else - { - state.getPropertyBag().put(INSTANCE_STATE, ReplicaState.LAGGING.toString()); - } - } - - } - } - return flocs; - } - - /* (non-Javadoc) - * @see org.alfresco.repo.index.shard.ShardRegistry#purgeAgedOutShards() - */ - @Override - public void purgeAgedOutShards() - { - long now = System.currentTimeMillis(); - ShardStateCollector shardStates = getPersistedShardStates(); - - for(Floc floc : shardStates.indexes.keySet()) - { - HashMap> shards = shardStates.indexes.get(floc); - for(Shard shard : shards.keySet()) - { - HashSet states = shards.get(shard); - for(ShardState state : states) - { - if(isShardTimedOut(now, state)) - { - String guid = shardStates.shardGuids.get(state.getShardInstance()); - DeleteCallBack dcb = new DeleteCallBack(attributeService, guid); - transactionService.getRetryingTransactionHelper().doInTransaction(dcb, false, true); - } - } - } - } - } -} diff --git a/source/java/org/alfresco/repo/management/subsystems/ChildApplicationContextFactory.java b/source/java/org/alfresco/repo/management/subsystems/ChildApplicationContextFactory.java index 5004f6f0ea..830d044eba 100644 --- a/source/java/org/alfresco/repo/management/subsystems/ChildApplicationContextFactory.java +++ b/source/java/org/alfresco/repo/management/subsystems/ChildApplicationContextFactory.java @@ -140,6 +140,9 @@ public class ChildApplicationContextFactory extends AbstractPropertyBackedBean i /** The suffix to the context file search path. */ private static final String CONTEXT_SUFFIX = "/*-context.xml"; + + /** The suffix to the enterprise context file search path. */ + private static final String ENTERPRISE_CONTEXT_SUFFIX = "/*-enterprise-context.xml"; /** The prefix to default file search paths. */ private static final String CLASSPATH_PREFIX = "classpath*:alfresco/subsystems/"; @@ -448,6 +451,8 @@ public class ChildApplicationContextFactory extends AbstractPropertyBackedBean i { ChildApplicationContextFactory.CLASSPATH_PREFIX + getCategory() + '/' + getTypeName() + ChildApplicationContextFactory.CONTEXT_SUFFIX, + ChildApplicationContextFactory.CLASSPATH_PREFIX + getCategory() + '/' + getTypeName() + + ChildApplicationContextFactory.ENTERPRISE_CONTEXT_SUFFIX, ChildApplicationContextFactory.EXTENSION_CLASSPATH_PREFIX + getCategory() + '/' diff --git a/source/java/org/alfresco/repo/search/impl/solr/SolrQueryHTTPClient.java b/source/java/org/alfresco/repo/search/impl/solr/SolrQueryHTTPClient.java index 9315da015b..d4b42fdcc2 100644 --- a/source/java/org/alfresco/repo/search/impl/solr/SolrQueryHTTPClient.java +++ b/source/java/org/alfresco/repo/search/impl/solr/SolrQueryHTTPClient.java @@ -86,12 +86,17 @@ import org.json.JSONTokener; import org.springframework.beans.BeansException; import org.springframework.beans.factory.BeanFactory; import org.springframework.beans.factory.BeanFactoryAware; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.config.BeanFactoryPostProcessor; +import org.springframework.beans.factory.config.ListFactoryBean; +import org.springframework.context.ApplicationListener; +import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.extensions.surf.util.I18NUtil; /** * @author Andy */ -public class SolrQueryHTTPClient implements BeanFactoryAware +public class SolrQueryHTTPClient implements BeanFactoryAware, InitializingBean { static Log s_logger = LogFactory.getLog(SolrQueryHTTPClient.class); @@ -144,10 +149,6 @@ public class SolrQueryHTTPClient implements BeanFactoryAware PropertyCheck.mandatory(this, "StoreMappings", storeMappings); PropertyCheck.mandatory(this, "RepositoryState", repositoryState); - for(SolrStoreMapping mapping : storeMappings) - { - mappingLookup.put(mapping.getStoreRef(), new ExplicitSolrStoreMappingWrapper(mapping, beanFactory)); - } } public void setAlternativeDictionary(String alternativeDictionary) @@ -781,7 +782,7 @@ public class SolrQueryHTTPClient implements BeanFactoryAware private SolrStoreMappingWrapper extractMapping(StoreRef store) { - if(useDynamicShardRegistration) + if((shardRegistry != null) && useDynamicShardRegistration) { SearchParameters sp = new SearchParameters(); sp.addStore(store); @@ -820,4 +821,17 @@ public class SolrQueryHTTPClient implements BeanFactoryAware this.beanFactory = beanFactory; } + /* (non-Javadoc) + * @see org.springframework.beans.factory.InitializingBean#afterPropertiesSet() + */ + @Override + public void afterPropertiesSet() throws Exception + { + mappingLookup.clear(); + for(SolrStoreMapping mapping : storeMappings) + { + mappingLookup.put(mapping.getStoreRef(), new ExplicitSolrStoreMappingWrapper(mapping, beanFactory)); + } + } + } diff --git a/source/java/org/alfresco/repo/solr/SOLRTrackingComponentImpl.java b/source/java/org/alfresco/repo/solr/SOLRTrackingComponentImpl.java index ae19ee2ed0..6e2280ed95 100644 --- a/source/java/org/alfresco/repo/solr/SOLRTrackingComponentImpl.java +++ b/source/java/org/alfresco/repo/solr/SOLRTrackingComponentImpl.java @@ -43,7 +43,6 @@ import org.alfresco.repo.domain.permissions.AclDAO; import org.alfresco.repo.domain.qname.QNameDAO; import org.alfresco.repo.domain.solr.SOLRDAO; import org.alfresco.repo.index.shard.ShardRegistry; -import org.alfresco.repo.index.shard.ShardRegistryImpl; import org.alfresco.repo.index.shard.ShardState; import org.alfresco.repo.search.AspectIndexFilter; import org.alfresco.repo.search.TypeIndexFilter; diff --git a/source/test-java/org/alfresco/AllUnitTestsSuite.java b/source/test-java/org/alfresco/AllUnitTestsSuite.java index bdcc0cd02a..9da4397a3b 100644 --- a/source/test-java/org/alfresco/AllUnitTestsSuite.java +++ b/source/test-java/org/alfresco/AllUnitTestsSuite.java @@ -101,6 +101,5 @@ public class AllUnitTestsSuite extends TestSuite suite.addTest(new JUnit4TestAdapter(org.alfresco.util.BeanExtenderUnitTest.class)); suite.addTest(new JUnit4TestAdapter(org.alfresco.repo.search.impl.solr.SpellCheckDecisionManagerTest.class)); suite.addTest(new JUnit4TestAdapter(org.alfresco.repo.search.impl.solr.SolrStoreMappingWrapperTest.class)); - suite.addTest(new JUnit4TestAdapter(org.alfresco.repo.index.shard.ShardRegistryTest.class)); } } diff --git a/source/test-java/org/alfresco/repo/index/shard/ShardRegistryTest.java b/source/test-java/org/alfresco/repo/index/shard/ShardRegistryTest.java deleted file mode 100644 index 5db3c473c9..0000000000 --- a/source/test-java/org/alfresco/repo/index/shard/ShardRegistryTest.java +++ /dev/null @@ -1,879 +0,0 @@ -/* - * Copyright (C) 2005-2010 Alfresco Software Limited. - * - * This file is part of Alfresco - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.repo.index.shard; - -import static org.junit.Assert.*; -import static org.mockito.Matchers.*; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.io.Serializable; -import java.net.UnknownHostException; -import java.util.HashSet; -import java.util.List; - -import org.alfresco.repo.cache.DefaultSimpleCache; -import org.alfresco.repo.index.shard.ShardRegistryImpl.ShardStateCollector; -import org.alfresco.repo.transaction.RetryingTransactionHelper; -import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback; -import org.alfresco.service.cmr.attributes.AttributeService; -import org.alfresco.service.cmr.attributes.AttributeService.AttributeQueryCallback; -import org.alfresco.service.cmr.repository.StoreRef; -import org.alfresco.service.cmr.search.SearchParameters; -import org.alfresco.service.transaction.TransactionService; -import org.alfresco.util.GUID; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.runners.MockitoJUnitRunner; -import org.mockito.stubbing.Answer; -import org.mockito.Matchers; - -/** - * @author Andy - * - */ -@RunWith(MockitoJUnitRunner.class) -public class ShardRegistryTest -{ - private ShardRegistryImpl shardRegistry; - - private DefaultSimpleCache shardStateCache = new DefaultSimpleCache(); - - private DefaultSimpleCache shardToGuidCache = new DefaultSimpleCache(); - - private @Mock AttributeService attributeService; - - private @Mock TransactionService transactionService; - - private @Mock RetryingTransactionHelper retryingTransactionHelper; - - /** - * - */ - public ShardRegistryTest() - { - // TODO Auto-generated constructor stub - } - - @Before - public void setUp() throws UnknownHostException - { - shardRegistry = new ShardRegistryImpl(); - shardRegistry.setAttributeService(attributeService); - shardRegistry.setShardStateCache(shardStateCache); - shardRegistry.setShardToGuidCache(shardToGuidCache); - shardRegistry.setTransactionService(transactionService); - shardRegistry.setPurgeOnInit(false); - shardRegistry.setShardInstanceTimeoutInSeconds(30); - shardRegistry.setMaxAllowedReplicaTxCountDifference(10); - shardRegistry.init(); - - when(transactionService.getRetryingTransactionHelper()).thenReturn(retryingTransactionHelper); - - final ArgumentCaptor argument = ArgumentCaptor.forClass(RetryingTransactionCallback.class); - when(retryingTransactionHelper.doInTransaction(argument.capture(), anyBoolean(), anyBoolean())).thenAnswer( - new Answer() { - public Object answer(InvocationOnMock invocation) { - try - { - argument.getValue().execute(); - } - catch (Throwable e) - { - // TODO Auto-generated catch block - e.printStackTrace(); - } - return null; - } - }); - - } - - @Test - public void registerLocalShardState() - { - - ShardState shardState1 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()) - .withShardInstance().withBaseUrl("/solr4/shard1").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - shardRegistry.registerShardState(shardState1); - - assertEquals(1, shardToGuidCache.getKeys().size()); - assertEquals(1, shardStateCache.getKeys().size()); - - assertEquals(shardState1, shardStateCache.get(shardState1.getShardInstance())); - - String guid1 = shardToGuidCache.get(shardState1.getShardInstance()); - verify(attributeService).setAttribute(shardState1, ".SHARD_STATE", guid1); - - - ShardState shardState2 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()) - .withShardInstance().withBaseUrl("/solr4/shard2").withHostName("meep").withPort(1234) - .withShard().withInstance(2) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - shardRegistry.registerShardState(shardState2); - - assertEquals(2, shardToGuidCache.getKeys().size()); - assertEquals(2, shardStateCache.getKeys().size()); - - assertEquals(shardState2, shardStateCache.get(shardState2.getShardInstance())); - - String guid2 = shardToGuidCache.get(shardState2.getShardInstance()); - verify(attributeService).setAttribute(shardState2, ".SHARD_STATE", guid2); - - // and again - - shardRegistry.registerShardState(shardState2); - assertEquals(2, shardToGuidCache.getKeys().size()); - assertEquals(2, shardStateCache.getKeys().size()); - - assertEquals(shardState2, shardStateCache.get(shardState2.getShardInstance())); - verify(attributeService, times(2)).setAttribute(shardState2, ".SHARD_STATE", guid2); - - SearchParameters sp = new SearchParameters(); - sp.addStore(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE); - List slice = shardRegistry.getIndexSlice(sp); - assertEquals(2, slice.size()); - assertTrue(slice.contains(shardState1.getShardInstance())); - assertTrue(slice.contains(shardState2.getShardInstance())); - } - - @Test - public void testShardsExpire() - { - - ShardState shardState1 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()) - .withShardInstance().withBaseUrl("/solr4/shard1").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - shardRegistry.registerShardState(shardState1); - - assertEquals(1, shardToGuidCache.getKeys().size()); - assertEquals(1, shardStateCache.getKeys().size()); - - assertEquals(shardState1, shardStateCache.get(shardState1.getShardInstance())); - - String guid1 = shardToGuidCache.get(shardState1.getShardInstance()); - verify(attributeService).setAttribute(shardState1, ".SHARD_STATE", guid1); - - - ShardState shardState2 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()) - .withShardInstance().withBaseUrl("/solr4/shard2").withHostName("meep").withPort(1234) - .withShard().withInstance(2) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - shardRegistry.registerShardState(shardState2); - - assertEquals(2, shardToGuidCache.getKeys().size()); - assertEquals(2, shardStateCache.getKeys().size()); - - assertEquals(shardState2, shardStateCache.get(shardState2.getShardInstance())); - - String guid2 = shardToGuidCache.get(shardState2.getShardInstance()); - verify(attributeService).setAttribute(shardState2, ".SHARD_STATE", guid2); - - // and again - - shardRegistry.registerShardState(shardState2); - assertEquals(2, shardToGuidCache.getKeys().size()); - assertEquals(2, shardStateCache.getKeys().size()); - - assertEquals(shardState2, shardStateCache.get(shardState2.getShardInstance())); - verify(attributeService, times(2)).setAttribute(shardState2, ".SHARD_STATE", guid2); - - SearchParameters sp = new SearchParameters(); - sp.addStore(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE); - List slice = shardRegistry.getIndexSlice(sp); - assertEquals(2, slice.size()); - assertTrue(slice.contains(shardState1.getShardInstance())); - assertTrue(slice.contains(shardState2.getShardInstance())); - - - // expire check - - synchronized(this) - { - try - { - wait(40000); - } - catch (InterruptedException e) - { - fail("Failed to wait"); - } - } - - slice = shardRegistry.getIndexSlice(sp); - assertEquals(0, slice.size()); - - shardState1.setLastUpdated(System.currentTimeMillis()); - shardRegistry.registerShardState(shardState1); - - slice = shardRegistry.getIndexSlice(sp); - assertEquals(1, slice.size()); - - shardState2.setLastUpdated(System.currentTimeMillis()); - shardRegistry.registerShardState(shardState2); - - slice = shardRegistry.getIndexSlice(sp); - assertEquals(2, slice.size()); - - } - - @Test - public void registerHalfPersisted() - { - final String guid1 = GUID.generate(); - - final ShardState shardState1 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()) - .withShardInstance().withBaseUrl("/solr4/shard1").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - doAnswer(new Answer() - { - long id = 0; - ShardStateCollector callback; - void handle(Serializable value, String... keys) - { - callback.handleAttribute(id++, value, keys); - } - - @Override - public Object answer(InvocationOnMock invocation) throws Throwable - { - callback = (ShardStateCollector) invocation.getArguments()[0]; - handle(shardState1, ".SHARD_STATE", guid1); - return null; - } - }).when(attributeService).getAttributes(any(AttributeQueryCallback.class), eq(".SHARD_STATE")); - - ShardState shardState2 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()) - .withShardInstance().withBaseUrl("/solr4/shard2").withHostName("meep").withPort(1234) - .withShard().withInstance(2) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - shardRegistry.registerShardState(shardState2); - - assertEquals(2, shardToGuidCache.getKeys().size()); - assertEquals(1, shardStateCache.getKeys().size()); - - assertEquals(shardState2, shardStateCache.get(shardState2.getShardInstance())); - - String guid2 = shardToGuidCache.get(shardState2.getShardInstance()); - verify(attributeService).setAttribute(shardState2, ".SHARD_STATE", guid2); - - // and again - - shardRegistry.registerShardState(shardState2); - assertEquals(2, shardToGuidCache.getKeys().size()); - assertEquals(1, shardStateCache.getKeys().size()); - - assertEquals(shardState2, shardStateCache.get(shardState2.getShardInstance())); - verify(attributeService, times(2)).setAttribute(shardState2, ".SHARD_STATE", guid2); - - SearchParameters sp = new SearchParameters(); - sp.addStore(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE); - List slice = shardRegistry.getIndexSlice(sp); - assertEquals(2, slice.size()); - assertTrue(slice.contains(shardState1.getShardInstance())); - assertTrue(slice.contains(shardState2.getShardInstance())); - - } - - @Test - public void registerAllPersisted() - { - final String guid1 = GUID.generate(); - final String guid2 = GUID.generate(); - - final ShardState shardState1 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()) - .withShardInstance().withBaseUrl("/solr4/shard1").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState2 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()) - .withShardInstance().withBaseUrl("/solr4/shard2").withHostName("meep").withPort(1234) - .withShard().withInstance(2) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - doAnswer(new Answer() - { - long id = 0; - ShardStateCollector callback; - void handle(Serializable value, String... keys) - { - callback.handleAttribute(id++, value, keys); - } - - @Override - public Object answer(InvocationOnMock invocation) throws Throwable - { - callback = (ShardStateCollector) invocation.getArguments()[0]; - handle(shardState1, ".SHARD_STATE", guid1); - handle(shardState2, ".SHARD_STATE", guid2); - return null; - } - }).when(attributeService).getAttributes(any(AttributeQueryCallback.class), eq(".SHARD_STATE")); - - - - - shardRegistry.registerShardState(shardState2); - - assertEquals(2, shardToGuidCache.getKeys().size()); - assertEquals(1, shardStateCache.getKeys().size()); - - assertEquals(shardState2, shardStateCache.get(shardState2.getShardInstance())); - - verify(attributeService).setAttribute(shardState2, ".SHARD_STATE", guid2); - - // and again - - shardRegistry.registerShardState(shardState2); - assertEquals(2, shardToGuidCache.getKeys().size()); - assertEquals(1, shardStateCache.getKeys().size()); - - assertEquals(shardState2, shardStateCache.get(shardState2.getShardInstance())); - verify(attributeService, times(2)).setAttribute(shardState2, ".SHARD_STATE", guid2); - - SearchParameters sp = new SearchParameters(); - sp.addStore(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE); - List slice = shardRegistry.getIndexSlice(sp); - assertEquals(2, slice.size()); - assertTrue(slice.contains(shardState1.getShardInstance())); - assertTrue(slice.contains(shardState2.getShardInstance())); - - } - - - - @Test - public void testFromPersisted() - { - final String guid1 = GUID.generate(); - final String guid2 = GUID.generate(); - - final ShardState shardState1 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()) - .withShardInstance().withBaseUrl("/solr4/shard1").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState2 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()) - .withShardInstance().withBaseUrl("/solr4/shard2").withHostName("meep").withPort(1234) - .withShard().withInstance(2) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - doAnswer(new Answer() - { - long id = 0; - ShardStateCollector callback; - void handle(Serializable value, String... keys) - { - callback.handleAttribute(id++, value, keys); - } - - @Override - public Object answer(InvocationOnMock invocation) throws Throwable - { - callback = (ShardStateCollector) invocation.getArguments()[0]; - handle(shardState1, ".SHARD_STATE", guid1); - handle(shardState2, ".SHARD_STATE", guid2); - return null; - } - }).when(attributeService).getAttributes(any(AttributeQueryCallback.class), eq(".SHARD_STATE")); - - SearchParameters sp = new SearchParameters(); - sp.addStore(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE); - List slice = shardRegistry.getIndexSlice(sp); - assertEquals(2, slice.size()); - assertTrue(slice.contains(shardState1.getShardInstance())); - assertTrue(slice.contains(shardState2.getShardInstance())); - } - - @Test - public void testComplexFromPersisted() - { - final String guid1 = GUID.generate(); - final String guid2 = GUID.generate(); - final String guid3 = GUID.generate(); - final String guid4 = GUID.generate(); - final String guid5 = GUID.generate(); - final String guid6 = GUID.generate(); - final String guid7 = GUID.generate(); - - final ShardState shardState1 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()) - .withShardInstance().withBaseUrl("/solr4/shard1").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState2 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()) - .withShardInstance().withBaseUrl("/solr4/shard2").withHostName("meep").withPort(1234) - .withShard().withInstance(2) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState3 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()) - .withShardInstance().withBaseUrl("/solr4/rep1").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState4 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()) - .withShardInstance().withBaseUrl("/solr4/rep2").withHostName("meep").withPort(1234) - .withShard().withInstance(2) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState5 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()) - .withShardInstance().withBaseUrl("/solr4/shard1").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_ARCHIVE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState6 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()) - .withShardInstance().withBaseUrl("/solr4/shard2").withHostName("meep").withPort(1234) - .withShard().withInstance(2) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_ARCHIVE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState7 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()) - .withShardInstance().withBaseUrl("/solr4/shard2").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(1).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - doAnswer(new Answer() - { - long id = 0; - ShardStateCollector callback; - void handle(Serializable value, String... keys) - { - callback.handleAttribute(id++, value, keys); - } - - @Override - public Object answer(InvocationOnMock invocation) throws Throwable - { - callback = (ShardStateCollector) invocation.getArguments()[0]; - handle(shardState1, ".SHARD_STATE", guid1); - handle(shardState2, ".SHARD_STATE", guid2); - handle(shardState3, ".SHARD_STATE", guid3); - handle(shardState4, ".SHARD_STATE", guid4); - handle(shardState5, ".SHARD_STATE", guid5); - handle(shardState6, ".SHARD_STATE", guid6); - return null; - } - }).when(attributeService).getAttributes(any(AttributeQueryCallback.class), eq(".SHARD_STATE")); - - SearchParameters sp = new SearchParameters(); - sp.addStore(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE); - List slice = shardRegistry.getIndexSlice(sp); - assertEquals(2, slice.size()); - assertTrue(slice.contains(shardState1.getShardInstance()) || slice.contains(shardState3.getShardInstance())); - assertTrue(slice.contains(shardState2.getShardInstance()) || slice.contains(shardState4.getShardInstance())); - - - sp = new SearchParameters(); - sp.addStore(StoreRef.STORE_REF_ARCHIVE_SPACESSTORE); - slice = shardRegistry.getIndexSlice(sp); - assertEquals(2, slice.size()); - assertTrue(slice.contains(shardState5.getShardInstance())); - assertTrue(slice.contains(shardState6.getShardInstance())); - } - - @Test - public void testSelectFlocBasedInTxCount1() - { - final String guid1 = GUID.generate(); - - final String guid2 = GUID.generate(); - final String guid3 = GUID.generate(); - - final String guid4 = GUID.generate(); - final String guid5 = GUID.generate(); - final String guid6 = GUID.generate(); - final String guid7 = GUID.generate(); - - final ShardState shardState1 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(10) - .withShardInstance().withBaseUrl("/solr4/single").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(1).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - - final ShardState shardState2 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(9) - .withShardInstance().withBaseUrl("/solr4/2-1").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState3 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(9) - .withShardInstance().withBaseUrl("/solr4/2-2").withHostName("meep").withPort(1234) - .withShard().withInstance(2) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState4 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(8) - .withShardInstance().withBaseUrl("/solr4/4-1").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(4).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState5 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(8) - .withShardInstance().withBaseUrl("/solr4/4-2").withHostName("meep").withPort(1234) - .withShard().withInstance(2) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(4).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState6 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(8) - .withShardInstance().withBaseUrl("/solr4/4-3").withHostName("meep").withPort(1234) - .withShard().withInstance(3) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(4).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState7 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(8) - .withShardInstance().withBaseUrl("/solr4/4-4").withHostName("meep").withPort(1234) - .withShard().withInstance(4) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(4).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - - doAnswer(new Answer() - { - long id = 0; - ShardStateCollector callback; - void handle(Serializable value, String... keys) - { - callback.handleAttribute(id++, value, keys); - } - - @Override - public Object answer(InvocationOnMock invocation) throws Throwable - { - callback = (ShardStateCollector) invocation.getArguments()[0]; - handle(shardState1, ".SHARD_STATE", guid1); - handle(shardState2, ".SHARD_STATE", guid2); - handle(shardState3, ".SHARD_STATE", guid3); - handle(shardState4, ".SHARD_STATE", guid4); - handle(shardState5, ".SHARD_STATE", guid5); - handle(shardState6, ".SHARD_STATE", guid6); - handle(shardState7, ".SHARD_STATE", guid7); - return null; - } - }).when(attributeService).getAttributes(any(AttributeQueryCallback.class), eq(".SHARD_STATE")); - - SearchParameters sp = new SearchParameters(); - sp.addStore(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE); - List slice = shardRegistry.getIndexSlice(sp); - assertEquals(1, slice.size()); - assertTrue(slice.contains(shardState1.getShardInstance())); - } - - @Test - public void testSelectFlocBasedInTxCount2() - { - final String guid1 = GUID.generate(); - - final String guid2 = GUID.generate(); - final String guid3 = GUID.generate(); - - final String guid4 = GUID.generate(); - final String guid5 = GUID.generate(); - final String guid6 = GUID.generate(); - final String guid7 = GUID.generate(); - - final ShardState shardState1 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(8) - .withShardInstance().withBaseUrl("/solr4/single").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(1).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - - final ShardState shardState2 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(10) - .withShardInstance().withBaseUrl("/solr4/2-1").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState3 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(10) - .withShardInstance().withBaseUrl("/solr4/2-2").withHostName("meep").withPort(1234) - .withShard().withInstance(2) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState4 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(9) - .withShardInstance().withBaseUrl("/solr4/4-1").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(4).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState5 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(9) - .withShardInstance().withBaseUrl("/solr4/4-2").withHostName("meep").withPort(1234) - .withShard().withInstance(2) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(4).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState6 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(9) - .withShardInstance().withBaseUrl("/solr4/4-3").withHostName("meep").withPort(1234) - .withShard().withInstance(3) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(4).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState7 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(9) - .withShardInstance().withBaseUrl("/solr4/4-4").withHostName("meep").withPort(1234) - .withShard().withInstance(4) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(4).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - - doAnswer(new Answer() - { - long id = 0; - ShardStateCollector callback; - void handle(Serializable value, String... keys) - { - callback.handleAttribute(id++, value, keys); - } - - @Override - public Object answer(InvocationOnMock invocation) throws Throwable - { - callback = (ShardStateCollector) invocation.getArguments()[0]; - handle(shardState1, ".SHARD_STATE", guid1); - handle(shardState2, ".SHARD_STATE", guid2); - handle(shardState3, ".SHARD_STATE", guid3); - handle(shardState4, ".SHARD_STATE", guid4); - handle(shardState5, ".SHARD_STATE", guid5); - handle(shardState6, ".SHARD_STATE", guid6); - handle(shardState7, ".SHARD_STATE", guid7); - return null; - } - }).when(attributeService).getAttributes(any(AttributeQueryCallback.class), eq(".SHARD_STATE")); - - SearchParameters sp = new SearchParameters(); - sp.addStore(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE); - List slice = shardRegistry.getIndexSlice(sp); - assertEquals(2, slice.size()); - assertTrue(slice.contains(shardState2.getShardInstance())); - assertTrue(slice.contains(shardState3.getShardInstance())); - } - - - @Test - public void testSelectFlocBasedInTxCount3() - { - final String guid1 = GUID.generate(); - - final String guid2 = GUID.generate(); - final String guid3 = GUID.generate(); - - final String guid4 = GUID.generate(); - final String guid5 = GUID.generate(); - final String guid6 = GUID.generate(); - final String guid7 = GUID.generate(); - - final ShardState shardState1 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(9) - .withShardInstance().withBaseUrl("/solr4/single").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(1).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - - final ShardState shardState2 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(8) - .withShardInstance().withBaseUrl("/solr4/2-1").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState3 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(8) - .withShardInstance().withBaseUrl("/solr4/2-2").withHostName("meep").withPort(1234) - .withShard().withInstance(2) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState4 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(10) - .withShardInstance().withBaseUrl("/solr4/4-1").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(4).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState5 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(10) - .withShardInstance().withBaseUrl("/solr4/4-2").withHostName("meep").withPort(1234) - .withShard().withInstance(2) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(4).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState6 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(10) - .withShardInstance().withBaseUrl("/solr4/4-3").withHostName("meep").withPort(1234) - .withShard().withInstance(3) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(4).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState7 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(10) - .withShardInstance().withBaseUrl("/solr4/4-4").withHostName("meep").withPort(1234) - .withShard().withInstance(4) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(4).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - - doAnswer(new Answer() - { - long id = 0; - ShardStateCollector callback; - void handle(Serializable value, String... keys) - { - callback.handleAttribute(id++, value, keys); - } - - @Override - public Object answer(InvocationOnMock invocation) throws Throwable - { - callback = (ShardStateCollector) invocation.getArguments()[0]; - handle(shardState1, ".SHARD_STATE", guid1); - handle(shardState2, ".SHARD_STATE", guid2); - handle(shardState3, ".SHARD_STATE", guid3); - handle(shardState4, ".SHARD_STATE", guid4); - handle(shardState5, ".SHARD_STATE", guid5); - handle(shardState6, ".SHARD_STATE", guid6); - handle(shardState7, ".SHARD_STATE", guid7); - return null; - } - }).when(attributeService).getAttributes(any(AttributeQueryCallback.class), eq(".SHARD_STATE")); - - SearchParameters sp = new SearchParameters(); - sp.addStore(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE); - List slice = shardRegistry.getIndexSlice(sp); - assertEquals(4, slice.size()); - assertTrue(slice.contains(shardState4.getShardInstance())); - assertTrue(slice.contains(shardState5.getShardInstance())); - assertTrue(slice.contains(shardState6.getShardInstance())); - assertTrue(slice.contains(shardState7.getShardInstance())); - } - - @Test - public void testReplicaSelection() - { - final String guid1 = GUID.generate(); - final String guid2 = GUID.generate(); - - final String guid3 = GUID.generate(); - final String guid4 = GUID.generate(); - - final String guid5 = GUID.generate(); - final String guid6 = GUID.generate(); - - final ShardState shardState1 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(100) - .withShardInstance().withBaseUrl("/solr4/1-1").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState2 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(100) - .withShardInstance().withBaseUrl("/solr4/1-2").withHostName("meep").withPort(1234) - .withShard().withInstance(2) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState3 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(90) - .withShardInstance().withBaseUrl("/solr4/2-1").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState4 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(90) - .withShardInstance().withBaseUrl("/solr4/2-2").withHostName("meep").withPort(1234) - .withShard().withInstance(2) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState5 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(80) - .withShardInstance().withBaseUrl("/solr4/3-1").withHostName("meep").withPort(1234) - .withShard().withInstance(1) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - final ShardState shardState6 = ShardStateBuilder.shardState().withMaster(true).withLastUpdated(System.currentTimeMillis()).withLastIndexedTxId(80) - .withShardInstance().withBaseUrl("/solr4/3-2").withHostName("meep").withPort(1234) - .withShard().withInstance(2) - .withFloc().withAddedStoreRef(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE).withHasContent(true).withNumberOfShards(2).withShardMethod(ShardMethodEnum.MOD_ACL_ID).withTemplate("default") - .endFloc().endShard().endShardInstance().build(); - - - doAnswer(new Answer() - { - long id = 0; - ShardStateCollector callback; - void handle(Serializable value, String... keys) - { - callback.handleAttribute(id++, value, keys); - } - - @Override - public Object answer(InvocationOnMock invocation) throws Throwable - { - callback = (ShardStateCollector) invocation.getArguments()[0]; - handle(shardState1, ".SHARD_STATE", guid1); - handle(shardState2, ".SHARD_STATE", guid2); - handle(shardState3, ".SHARD_STATE", guid3); - handle(shardState4, ".SHARD_STATE", guid4); - handle(shardState5, ".SHARD_STATE", guid5); - handle(shardState6, ".SHARD_STATE", guid6); - return null; - } - }).when(attributeService).getAttributes(any(AttributeQueryCallback.class), eq(".SHARD_STATE")); - - HashSet found = new HashSet(); - for(int i = 0; i < 1000; i++) - { - SearchParameters sp = new SearchParameters(); - sp.addStore(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE); - List slice = shardRegistry.getIndexSlice(sp); - assertEquals(2, slice.size()); - assertTrue(slice.contains(shardState1.getShardInstance()) || slice.contains(shardState3.getShardInstance())); - assertTrue(slice.contains(shardState2.getShardInstance()) || slice.contains(shardState4.getShardInstance())); - - found.add(slice.get(0)); - found.add(slice.get(1)); - } - - assertEquals(4, found.size()); - } -}