diff --git a/config/alfresco/extension/new-indexer-context.xml.sample b/config/alfresco/extension/new-indexer-context.xml.sample new file mode 100644 index 0000000000..b34616efed --- /dev/null +++ b/config/alfresco/extension/new-indexer-context.xml.sample @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + + + + + + + + + + ${dir.indexes} + + + + + + + + + ${lucene.maxAtomicTransformationTime} + + + ${lucene.query.maxClauses} + + + ${lucene.indexer.batchSize} + + + ${lucene.indexer.minMergeDocs} + + + ${lucene.indexer.mergeFactor} + + + ${lucene.indexer.maxMergeDocs} + + + ${dir.indexes.lock} + + + ${lucene.indexer.maxFieldLength} + + + ${lucene.write.lock.timeout} + + + ${lucene.commit.lock.timeout} + + + ${lucene.lock.poll.interval} + + + + + + + + + + + + + + + + + + ${dir.root}/backup-lucene-indexes + + + + \ No newline at end of file diff --git a/source/java/org/alfresco/repo/search/Indexer.java b/source/java/org/alfresco/repo/search/Indexer.java index 71a38fa741..1de19baf08 100644 --- a/source/java/org/alfresco/repo/search/Indexer.java +++ b/source/java/org/alfresco/repo/search/Indexer.java @@ -16,6 +16,7 @@ */ package org.alfresco.repo.search; +import org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexerImpl; import org.alfresco.service.cmr.repository.ChildAssociationRef; import org.alfresco.service.cmr.repository.NodeRef; @@ -99,7 +100,8 @@ public interface Indexer * @param relationshipRef */ public void deleteChildRelationship(ChildAssociationRef relationshipRef); - + + } diff --git a/source/java/org/alfresco/repo/search/IndexerAndSearcher.java b/source/java/org/alfresco/repo/search/IndexerAndSearcher.java index 16f0094dd2..b92657aad5 100644 --- a/source/java/org/alfresco/repo/search/IndexerAndSearcher.java +++ b/source/java/org/alfresco/repo/search/IndexerAndSearcher.java @@ -34,7 +34,7 @@ public interface IndexerAndSearcher * @return * @throws IndexerException */ - public abstract Indexer getIndexer(StoreRef storeRef) throws IndexerException; + public abstract IndexerSPI getIndexer(StoreRef storeRef) throws IndexerException; /** * Get a searcher for a store diff --git a/source/java/org/alfresco/repo/search/IndexerSPI.java b/source/java/org/alfresco/repo/search/IndexerSPI.java new file mode 100644 index 0000000000..82a6e311ee --- /dev/null +++ b/source/java/org/alfresco/repo/search/IndexerSPI.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2005 Alfresco, Inc. + * + * Licensed under the Mozilla Public License version 1.1 + * with a permitted attribution clause. You may obtain a + * copy of the License at + * + * http://www.alfresco.org/legal/license.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.alfresco.repo.search; + +import org.alfresco.repo.search.impl.lucene.fts.FTSIndexerAware; + +public interface IndexerSPI extends Indexer +{ + public void registerCallBack(FTSIndexerAware callBack); + + public void updateFullTextSearch(int i); + +} diff --git a/source/java/org/alfresco/repo/search/impl/NodeSearcher.java b/source/java/org/alfresco/repo/search/impl/NodeSearcher.java index 4c7bfda98f..c244a08ff0 100644 --- a/source/java/org/alfresco/repo/search/impl/NodeSearcher.java +++ b/source/java/org/alfresco/repo/search/impl/NodeSearcher.java @@ -66,7 +66,7 @@ public class NodeSearcher /** * @see NodeServiceXPath */ - public synchronized List selectNodes(NodeRef contextNodeRef, String xpathIn, + public List selectNodes(NodeRef contextNodeRef, String xpathIn, QueryParameterDefinition[] paramDefs, NamespacePrefixResolver namespacePrefixResolver, boolean followAllParentLinks, String language) { diff --git a/source/java/org/alfresco/repo/search/impl/lucene/FilterIndexReaderByNodeRefs2.java b/source/java/org/alfresco/repo/search/impl/lucene/FilterIndexReaderByNodeRefs2.java new file mode 100644 index 0000000000..d19e58a239 --- /dev/null +++ b/source/java/org/alfresco/repo/search/impl/lucene/FilterIndexReaderByNodeRefs2.java @@ -0,0 +1,239 @@ +/* + * Copyright (C) 2005 Alfresco, Inc. + * + * Licensed under the Mozilla Public License version 1.1 + * with a permitted attribution clause. You may obtain a + * copy of the License at + * + * http://www.alfresco.org/legal/license.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.alfresco.repo.search.impl.lucene; + +import java.io.IOException; +import java.util.BitSet; +import java.util.Set; + +import org.alfresco.error.AlfrescoRuntimeException; +import org.alfresco.service.cmr.repository.NodeRef; +import org.apache.lucene.index.FilterIndexReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermDocs; +import org.apache.lucene.index.TermEnum; +import org.apache.lucene.index.TermPositions; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Hits; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MultiSearcher; +import org.apache.lucene.search.Searcher; +import org.apache.lucene.search.TermQuery; + +public class FilterIndexReaderByNodeRefs2 extends FilterIndexReader +{ + BitSet deletedDocuments; + + public FilterIndexReaderByNodeRefs2(IndexReader reader, Set deletions, boolean deleteNodesOnly) + { + super(reader); + deletedDocuments = new BitSet(reader.maxDoc()); + + try + { + if (!deleteNodesOnly) + { + for (NodeRef nodeRef : deletions) + { + TermDocs td = reader.termDocs(new Term("ID", nodeRef.toString())); + while (td.next()) + { + deletedDocuments.set(td.doc(), true); + } + } + } + else + { + + Searcher searcher = new IndexSearcher(reader); + for (NodeRef nodeRef : deletions) + { + BooleanQuery query = new BooleanQuery(); + query.add(new TermQuery(new Term("ID", nodeRef.toString())), true, false); + query.add(new TermQuery(new Term("ISNODE", "T")), false, false); + Hits hits = searcher.search(query); + if (hits.length() > 0) + { + for (int i = 0; i < hits.length(); i++) + { + deletedDocuments.set(hits.id(i), true); + } + } + } + + } + } + catch (IOException e) + { + throw new AlfrescoRuntimeException("Failed to construct filtering index reader", e); + } + } + + public static class FilterTermDocs implements TermDocs + { + BitSet deletedDocuments; + + protected TermDocs in; + + public FilterTermDocs(TermDocs in, BitSet deletedDocuments) + { + this.in = in; + this.deletedDocuments = deletedDocuments; + } + + public void seek(Term term) throws IOException + { + // Seek is left to the base implementation + in.seek(term); + } + + public void seek(TermEnum termEnum) throws IOException + { + // Seek is left to the base implementation + in.seek(termEnum); + } + + public int doc() + { + // The current document info is valid in the base implementation + return in.doc(); + } + + public int freq() + { + // The frequency is valid in the base implementation + return in.freq(); + } + + public boolean next() throws IOException + { + while (in.next()) + { + if (!deletedDocuments.get(in.doc())) + { + // Not masked + return true; + } + } + return false; + } + + public int read(int[] docs, int[] freqs) throws IOException + { + int[] innerDocs = new int[docs.length]; + int[] innerFreq = new int[docs.length]; + int count = in.read(innerDocs, innerFreq); + + // Is the stream exhausted + if (count == 0) + { + return 0; + } + + if (allDeleted(innerDocs, count)) + { + // Did not find anything - try again + return read(docs, freqs); + } + + // Add non deleted + + int insertPosition = 0; + for (int i = 0; i < count; i++) + { + if (!deletedDocuments.get(innerDocs[i])) + { + docs[insertPosition] = innerDocs[i]; + freqs[insertPosition] = innerFreq[i]; + insertPosition++; + } + } + + return insertPosition; + } + + private boolean allDeleted(int[] docs, int fillSize) + { + for (int i = 0; i < fillSize; i++) + { + if (!deletedDocuments.get(docs[i])) + { + return false; + } + } + return true; + } + + public boolean skipTo(int i) throws IOException + { + boolean result = in.skipTo(i); + if (result == false) + { + return false; + } + + if (deletedDocuments.get(in.doc())) + { + return skipTo(i); + } + else + { + return true; + } + } + + public void close() throws IOException + { + // Leave to internal implementation + in.close(); + } + } + + /** Base class for filtering {@link TermPositions} implementations. */ + public static class FilterTermPositions extends FilterTermDocs implements TermPositions + { + + public FilterTermPositions(TermPositions in, BitSet deletedDocuements) + { + super(in, deletedDocuements); + } + + public int nextPosition() throws IOException + { + return ((TermPositions) this.in).nextPosition(); + } + } + + @Override + public int numDocs() + { + return super.numDocs() - deletedDocuments.cardinality(); + } + + @Override + public TermDocs termDocs() throws IOException + { + return new FilterTermDocs(super.termDocs(), deletedDocuments); + } + + @Override + public TermPositions termPositions() throws IOException + { + return new FilterTermPositions(super.termPositions(), deletedDocuments); + } +} diff --git a/source/java/org/alfresco/repo/search/impl/lucene/LuceneBase2.java b/source/java/org/alfresco/repo/search/impl/lucene/LuceneBase2.java new file mode 100644 index 0000000000..4db47d16c4 --- /dev/null +++ b/source/java/org/alfresco/repo/search/impl/lucene/LuceneBase2.java @@ -0,0 +1,283 @@ +/* + * Copyright (C) 2005 Alfresco, Inc. + * + * Licensed under the Mozilla Public License version 1.1 + * with a permitted attribution clause. You may obtain a + * copy of the License at + * + * http://www.alfresco.org/legal/license.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.alfresco.repo.search.impl.lucene; + +import java.io.File; +import java.io.IOException; +import java.util.Set; + +import org.alfresco.repo.search.IndexerException; +import org.alfresco.repo.search.impl.lucene.index.IndexInfo; +import org.alfresco.repo.search.impl.lucene.index.TransactionStatus; +import org.alfresco.service.cmr.dictionary.DictionaryService; +import org.alfresco.service.cmr.repository.NodeRef; +import org.alfresco.service.cmr.repository.StoreRef; +import org.apache.log4j.Logger; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Searcher; + +/** + * Common support for abstracting the lucene indexer from its configuration and management requirements. + * + *

+ * This class defines where the indexes are stored. This should be via a configurable Bean property in Spring. + * + *

+ * The default file structure is + *

    + *
  1. "base"/"protocol"/"name"/ for the main index + *
  2. "base"/"protocol"/"name"/deltas/"id" for transactional updates + *
  3. "base"/"protocol"/"name"/undo/"id" undo information + *
+ * + *

+ * The IndexWriter and IndexReader for a given index are toggled (one should be used for delete and the other for write). These are reused/closed/initialised as required. + * + *

+ * The index deltas are buffered to memory and persisted in the file system as required. + * + * @author Andy Hind + * + */ + +public abstract class LuceneBase2 +{ + private static Logger s_logger = Logger.getLogger(LuceneBase2.class); + + private IndexInfo indexInfo; + + /** + * The identifier for the store + */ + + protected StoreRef store; + + /** + * The identifier for the delta + */ + + protected String deltaId; + + private LuceneConfig config; + + // "lucene-indexes"; + + /** + * Initialise the configuration elements of the lucene store indexers and searchers. + * + * @param store + * @param deltaId + * @throws IOException + */ + protected void initialise(StoreRef store, String deltaId, boolean createMain, boolean createDelta) + throws LuceneIndexException + { + this.store = store; + this.deltaId = deltaId; + + String basePath = getBasePath(); + File baseDir = new File(basePath); + indexInfo = IndexInfo.getIndexInfo(baseDir); + try + { + if (deltaId != null) + { + indexInfo.setStatus(deltaId, TransactionStatus.ACTIVE, null, null); + } + } + catch (IOException e) + { + throw new IndexerException("Filed to set delta as active"); + } + + } + + /** + * Utility method to find the path to the base index + * + * @return + */ + private String getBasePath() + { + if (config.getIndexRootLocation() == null) + { + throw new IndexerException("No configuration for index location"); + } + String basePath = config.getIndexRootLocation() + + File.separator + store.getProtocol() + File.separator + store.getIdentifier() + File.separator; + return basePath; + } + + /** + * Get a searcher for the main index TODO: Split out support for the main index. We really only need this if we want to search over the changing index before it is committed + * + * @return + * @throws IOException + */ + + protected IndexSearcher getSearcher() throws LuceneIndexException + { + try + { + return new ClosingIndexSearcher(indexInfo.getMainIndexReferenceCountingReadOnlyIndexReader()); + } + catch (IOException e) + { + s_logger.error("Error", e); + throw new LuceneIndexException("Failed to open IndexSarcher for " + getBasePath(), e); + } + } + + protected Searcher getSearcher(LuceneIndexer2 luceneIndexer) throws LuceneIndexException + { + // If we know the delta id we should do better + + try + { + if (luceneIndexer == null) + { + return new ClosingIndexSearcher(indexInfo.getMainIndexReferenceCountingReadOnlyIndexReader()); + } + else + { + // TODO: Create appropriate reader that lies about deletions + // from the first + // + luceneIndexer.flushPending(); + return new ClosingIndexSearcher(indexInfo.getMainIndexReferenceCountingReadOnlyIndexReader(deltaId, + luceneIndexer.getDeletions(), luceneIndexer.getDeleteOnlyNodes())); + } + + } + catch (IOException e) + { + s_logger.error("Error", e); + throw new LuceneIndexException("Failed to open IndexSarcher for " + getBasePath(), e); + } + } + + /** + * Get a reader for the on file portion of the delta + * + * @return + * @throws IOException + * @throws IOException + */ + + protected IndexReader getDeltaReader() throws LuceneIndexException, IOException + { + return indexInfo.getDeltaIndexReader(deltaId); + } + + /** + * Close the on file reader for the delta if it is open + * + * @throws IOException + * + * @throws IOException + */ + + protected void closeDeltaReader() throws LuceneIndexException, IOException + { + indexInfo.closeDeltaIndexReader(deltaId); + } + + /** + * Get the on file writer for the delta + * + * @return + * @throws IOException + * @throws IOException + */ + protected IndexWriter getDeltaWriter() throws LuceneIndexException, IOException + { + return indexInfo.getDeltaIndexWriter(deltaId, new LuceneAnalyser(dictionaryService)); + } + + /** + * Close the on disk delta writer + * + * @throws IOException + * + * @throws IOException + */ + + protected void closeDeltaWriter() throws LuceneIndexException, IOException + { + indexInfo.closeDeltaIndexWriter(deltaId); + } + + /** + * Save the in memory delta to the disk, make sure there is nothing held in memory + * + * @throws IOException + * + * @throws IOException + */ + protected void saveDelta() throws LuceneIndexException, IOException + { + // Only one should exist so we do not need error trapping to execute the + // other + closeDeltaReader(); + closeDeltaWriter(); + } + + protected void setInfo(long docs, Set deletions, boolean deleteNodesOnly) throws IOException + { + indexInfo.setPreparedState(deltaId, deletions, docs, deleteNodesOnly); + } + + protected void setStatus(TransactionStatus status) throws IOException + { + indexInfo.setStatus(deltaId, status, null, null); + } + + private DictionaryService dictionaryService; + + protected IndexReader getReader() throws LuceneIndexException, IOException + { + return indexInfo.getMainIndexReferenceCountingReadOnlyIndexReader(); + } + + public void setDictionaryService(DictionaryService dictionaryService) + { + this.dictionaryService = dictionaryService; + } + + public DictionaryService getDictionaryService() + { + return dictionaryService; + } + + public void setLuceneConfig(LuceneConfig config) + { + this.config = config; + } + + public LuceneConfig getLuceneConfig() + { + return config; + } + + public String getDeltaId() + { + return deltaId; + } + +} diff --git a/source/java/org/alfresco/repo/search/impl/lucene/LuceneCategoryTest2.java b/source/java/org/alfresco/repo/search/impl/lucene/LuceneCategoryTest2.java new file mode 100644 index 0000000000..2341ec5820 --- /dev/null +++ b/source/java/org/alfresco/repo/search/impl/lucene/LuceneCategoryTest2.java @@ -0,0 +1,672 @@ +/* + * Copyright (C) 2005 Alfresco, Inc. + * + * Licensed under the Mozilla Public License version 1.1 + * with a permitted attribution clause. You may obtain a + * copy of the License at + * + * http://www.alfresco.org/legal/license.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.alfresco.repo.search.impl.lucene; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Random; + +import javax.transaction.UserTransaction; + +import junit.framework.TestCase; + +import org.alfresco.model.ContentModel; +import org.alfresco.repo.dictionary.DictionaryDAO; +import org.alfresco.repo.dictionary.M2Aspect; +import org.alfresco.repo.dictionary.M2Model; +import org.alfresco.repo.dictionary.M2Property; +import org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer; +import org.alfresco.repo.search.transaction.LuceneIndexLock; +import org.alfresco.service.ServiceRegistry; +import org.alfresco.service.cmr.dictionary.DictionaryService; +import org.alfresco.service.cmr.dictionary.DataTypeDefinition; +import org.alfresco.service.cmr.repository.ChildAssociationRef; +import org.alfresco.service.cmr.repository.NodeRef; +import org.alfresco.service.cmr.repository.NodeService; +import org.alfresco.service.cmr.repository.StoreRef; +import org.alfresco.service.cmr.search.CategoryService; +import org.alfresco.service.cmr.search.ResultSet; +import org.alfresco.service.cmr.search.ResultSetRow; +import org.alfresco.service.cmr.search.SearchService; +import org.alfresco.service.namespace.DynamicNamespacePrefixResolver; +import org.alfresco.service.namespace.NamespacePrefixResolver; +import org.alfresco.service.namespace.NamespaceService; +import org.alfresco.service.namespace.QName; +import org.alfresco.service.transaction.TransactionService; +import org.alfresco.util.ApplicationContextHelper; +import org.springframework.context.ApplicationContext; + +public class LuceneCategoryTest2 extends TestCase +{ + private ServiceRegistry serviceRegistry; + + static ApplicationContext ctx = ApplicationContextHelper.getApplicationContext(); + NodeService nodeService; + DictionaryService dictionaryService; + LuceneIndexLock luceneIndexLock; + private NodeRef rootNodeRef; + private NodeRef n1; + private NodeRef n2; + private NodeRef n3; + private NodeRef n4; + private NodeRef n6; + private NodeRef n5; + private NodeRef n7; + private NodeRef n8; + private NodeRef n9; + private NodeRef n10; + private NodeRef n11; + private NodeRef n12; + private NodeRef n13; + private NodeRef n14; + + private NodeRef catContainer; + private NodeRef catRoot; + private NodeRef catACBase; + private NodeRef catACOne; + private NodeRef catACTwo; + private NodeRef catACThree; + private FullTextSearchIndexer luceneFTS; + private DictionaryDAO dictionaryDAO; + private String TEST_NAMESPACE = "http://www.alfresco.org/test/lucenecategorytest"; + private QName regionCategorisationQName; + private QName assetClassCategorisationQName; + private QName investmentRegionCategorisationQName; + private QName marketingRegionCategorisationQName; + private NodeRef catRBase; + private NodeRef catROne; + private NodeRef catRTwo; + private NodeRef catRThree; + private SearchService searcher; + private LuceneIndexerAndSearcher indexerAndSearcher; + + private CategoryService categoryService; + + public LuceneCategoryTest2() + { + super(); + } + + public LuceneCategoryTest2(String arg0) + { + super(arg0); + } + + public void setUp() throws Exception + { + nodeService = (NodeService)ctx.getBean("dbNodeService"); + luceneIndexLock = (LuceneIndexLock)ctx.getBean("luceneIndexLock"); + dictionaryService = (DictionaryService)ctx.getBean("dictionaryService"); + luceneFTS = (FullTextSearchIndexer) ctx.getBean("LuceneFullTextSearchIndexer"); + dictionaryDAO = (DictionaryDAO) ctx.getBean("dictionaryDAO"); + searcher = (SearchService) ctx.getBean("searchService"); + indexerAndSearcher = (LuceneIndexerAndSearcher) ctx.getBean("luceneIndexerAndSearcherFactory"); + categoryService = (CategoryService) ctx.getBean("categoryService"); + serviceRegistry = (ServiceRegistry) ctx.getBean(ServiceRegistry.SERVICE_REGISTRY); + + createTestTypes(); + + TransactionService transactionService = serviceRegistry.getTransactionService(); + UserTransaction tx = transactionService.getUserTransaction(); + tx.begin(); + + StoreRef storeRef = nodeService.createStore( + StoreRef.PROTOCOL_WORKSPACE, + "Test_" + System.currentTimeMillis()); + rootNodeRef = nodeService.getRootNode(storeRef); + + n1 = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}one"), ContentModel.TYPE_CONTAINER).getChildRef(); + nodeService.setProperty(n1, QName.createQName("{namespace}property-1"), "value-1"); + n2 = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}two"), ContentModel.TYPE_CONTAINER).getChildRef(); + nodeService.setProperty(n2, QName.createQName("{namespace}property-1"), "value-1"); + nodeService.setProperty(n2, QName.createQName("{namespace}property-2"), "value-2"); + n3 = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}three"), ContentModel.TYPE_CONTAINER).getChildRef(); + n4 = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}four"), ContentModel.TYPE_CONTAINER).getChildRef(); + n5 = nodeService.createNode(n1, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}five"), ContentModel.TYPE_CONTAINER).getChildRef(); + n6 = nodeService.createNode(n1, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}six"), ContentModel.TYPE_CONTAINER).getChildRef(); + n7 = nodeService.createNode(n2, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}seven"), ContentModel.TYPE_CONTAINER).getChildRef(); + n8 = nodeService.createNode(n2, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}eight-2"), ContentModel.TYPE_CONTAINER).getChildRef(); + n9 = nodeService.createNode(n5, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}nine"), ContentModel.TYPE_CONTAINER).getChildRef(); + n10 = nodeService.createNode(n5, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}ten"), ContentModel.TYPE_CONTAINER).getChildRef(); + n11 = nodeService.createNode(n5, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}eleven"), ContentModel.TYPE_CONTAINER).getChildRef(); + n12 = nodeService.createNode(n5, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}twelve"), ContentModel.TYPE_CONTAINER).getChildRef(); + n13 = nodeService.createNode(n12, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}thirteen"), ContentModel.TYPE_CONTAINER).getChildRef(); + n14 = nodeService.createNode(n13, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}fourteen"), ContentModel.TYPE_CONTAINER).getChildRef(); + + nodeService.addChild(rootNodeRef, n8, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}eight-0")); + nodeService.addChild(n1, n8, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}eight-1")); + nodeService.addChild(n2, n13, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}link")); + + nodeService.addChild(n1, n14, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}common")); + nodeService.addChild(n2, n14, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}common")); + nodeService.addChild(n5, n14, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}common")); + nodeService.addChild(n6, n14, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}common")); + nodeService.addChild(n12, n14, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}common")); + nodeService.addChild(n13, n14, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}common")); + + // Categories + + catContainer = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName.createQName(NamespaceService.CONTENT_MODEL_1_0_URI, "categoryContainer"), ContentModel.TYPE_CONTAINER).getChildRef(); + catRoot = nodeService.createNode(catContainer, ContentModel.ASSOC_CHILDREN, QName.createQName(NamespaceService.CONTENT_MODEL_1_0_URI, "categoryRoot"), ContentModel.TYPE_CATEGORYROOT).getChildRef(); + + + + catRBase = nodeService.createNode(catRoot, ContentModel.ASSOC_CATEGORIES, QName.createQName(TEST_NAMESPACE, "Region"), ContentModel.TYPE_CATEGORY).getChildRef(); + catROne = nodeService.createNode(catRBase, ContentModel.ASSOC_SUBCATEGORIES, QName.createQName(TEST_NAMESPACE, "Europe"), ContentModel.TYPE_CATEGORY).getChildRef(); + catRTwo = nodeService.createNode(catRBase, ContentModel.ASSOC_SUBCATEGORIES, QName.createQName(TEST_NAMESPACE, "RestOfWorld"), ContentModel.TYPE_CATEGORY).getChildRef(); + catRThree = nodeService.createNode(catRTwo, ContentModel.ASSOC_SUBCATEGORIES, QName.createQName(TEST_NAMESPACE, "US"), ContentModel.TYPE_CATEGORY).getChildRef(); + + nodeService.addChild(catRoot, catRBase, ContentModel.ASSOC_CATEGORIES, QName.createQName(TEST_NAMESPACE, "InvestmentRegion")); + nodeService.addChild(catRoot, catRBase, ContentModel.ASSOC_CATEGORIES, QName.createQName(TEST_NAMESPACE, "MarketingRegion")); + + + catACBase = nodeService.createNode(catRoot, ContentModel.ASSOC_CATEGORIES, QName.createQName(TEST_NAMESPACE, "AssetClass"), ContentModel.TYPE_CATEGORY).getChildRef(); + catACOne = nodeService.createNode(catACBase, ContentModel.ASSOC_SUBCATEGORIES, QName.createQName(TEST_NAMESPACE, "Fixed"), ContentModel.TYPE_CATEGORY).getChildRef(); + catACTwo = nodeService.createNode(catACBase, ContentModel.ASSOC_SUBCATEGORIES, QName.createQName(TEST_NAMESPACE, "Equity"), ContentModel.TYPE_CATEGORY).getChildRef(); + catACThree = nodeService.createNode(catACTwo, ContentModel.ASSOC_SUBCATEGORIES, QName.createQName(TEST_NAMESPACE, "SpecialEquity"), ContentModel.TYPE_CATEGORY).getChildRef(); + + + + nodeService.addAspect(n1, assetClassCategorisationQName, createMap("assetClass", catACBase)); + nodeService.addAspect(n1, regionCategorisationQName, createMap("region", catRBase)); + + nodeService.addAspect(n2, assetClassCategorisationQName, createMap("assetClass", catACOne)); + nodeService.addAspect(n3, assetClassCategorisationQName, createMap("assetClass", catACOne)); + nodeService.addAspect(n4, assetClassCategorisationQName, createMap("assetClass", catACOne)); + nodeService.addAspect(n5, assetClassCategorisationQName, createMap("assetClass", catACOne)); + nodeService.addAspect(n6, assetClassCategorisationQName, createMap("assetClass", catACOne)); + + nodeService.addAspect(n7, assetClassCategorisationQName, createMap("assetClass", catACTwo)); + nodeService.addAspect(n8, assetClassCategorisationQName, createMap("assetClass", catACTwo)); + nodeService.addAspect(n9, assetClassCategorisationQName, createMap("assetClass", catACTwo)); + nodeService.addAspect(n10, assetClassCategorisationQName, createMap("assetClass", catACTwo)); + nodeService.addAspect(n11, assetClassCategorisationQName, createMap("assetClass", catACTwo)); + + nodeService.addAspect(n12, assetClassCategorisationQName, createMap("assetClass", catACOne, catACTwo)); + nodeService.addAspect(n13, assetClassCategorisationQName, createMap("assetClass", catACOne, catACTwo, catACThree)); + nodeService.addAspect(n14, assetClassCategorisationQName, createMap("assetClass", catACOne, catACTwo)); + + nodeService.addAspect(n2, regionCategorisationQName, createMap("region", catROne)); + nodeService.addAspect(n3, regionCategorisationQName, createMap("region", catROne)); + nodeService.addAspect(n4, regionCategorisationQName, createMap("region", catRTwo)); + nodeService.addAspect(n5, regionCategorisationQName, createMap("region", catRTwo)); + + nodeService.addAspect(n5, investmentRegionCategorisationQName, createMap("investmentRegion", catRBase)); + nodeService.addAspect(n5, marketingRegionCategorisationQName, createMap("marketingRegion", catRBase)); + nodeService.addAspect(n6, investmentRegionCategorisationQName, createMap("investmentRegion", catRBase)); + nodeService.addAspect(n7, investmentRegionCategorisationQName, createMap("investmentRegion", catRBase)); + nodeService.addAspect(n8, investmentRegionCategorisationQName, createMap("investmentRegion", catRBase)); + nodeService.addAspect(n9, investmentRegionCategorisationQName, createMap("investmentRegion", catRBase)); + nodeService.addAspect(n10, marketingRegionCategorisationQName, createMap("marketingRegion", catRBase)); + nodeService.addAspect(n11, marketingRegionCategorisationQName, createMap("marketingRegion", catRBase)); + nodeService.addAspect(n12, marketingRegionCategorisationQName, createMap("marketingRegion", catRBase)); + nodeService.addAspect(n13, marketingRegionCategorisationQName, createMap("marketingRegion", catRBase)); + nodeService.addAspect(n14, marketingRegionCategorisationQName, createMap("marketingRegion", catRBase)); + + tx.commit(); + } + + private HashMap createMap(String name, NodeRef[] nodeRefs) + { + HashMap map = new HashMap(); + Serializable value = (Serializable) Arrays.asList(nodeRefs); + map.put(QName.createQName(TEST_NAMESPACE, name), value); + return map; + } + + private HashMap createMap(String name, NodeRef nodeRef) + { + return createMap(name, new NodeRef[]{nodeRef}); + } + + private HashMap createMap(String name, NodeRef nodeRef1, NodeRef nodeRef2) + { + return createMap(name, new NodeRef[]{nodeRef1, nodeRef2}); + } + + private HashMap createMap(String name, NodeRef nodeRef1, NodeRef nodeRef2, NodeRef nodeRef3) + { + return createMap(name, new NodeRef[]{nodeRef1, nodeRef2, nodeRef3}); + } + + private void createTestTypes() + { + M2Model model = M2Model.createModel("test:lucenecategory"); + model.createNamespace(TEST_NAMESPACE, "test"); + model.createImport(NamespaceService.DICTIONARY_MODEL_1_0_URI, NamespaceService.DICTIONARY_MODEL_PREFIX); + model.createImport(NamespaceService.CONTENT_MODEL_1_0_URI, NamespaceService.CONTENT_MODEL_PREFIX); + + regionCategorisationQName = QName.createQName(TEST_NAMESPACE, "Region"); + M2Aspect generalCategorisation = model.createAspect("test:" + regionCategorisationQName.getLocalName()); + generalCategorisation.setParentName("cm:" + ContentModel.ASPECT_CLASSIFIABLE.getLocalName()); + M2Property genCatProp = generalCategorisation.createProperty("test:region"); + genCatProp.setIndexed(true); + genCatProp.setIndexedAtomically(true); + genCatProp.setMandatory(true); + genCatProp.setMultiValued(true); + genCatProp.setStoredInIndex(true); + genCatProp.setTokenisedInIndex(true); + genCatProp.setType("d:" + DataTypeDefinition.CATEGORY.getLocalName()); + + assetClassCategorisationQName = QName.createQName(TEST_NAMESPACE, "AssetClass"); + M2Aspect assetClassCategorisation = model.createAspect("test:" + assetClassCategorisationQName.getLocalName()); + assetClassCategorisation.setParentName("cm:" + ContentModel.ASPECT_CLASSIFIABLE.getLocalName()); + M2Property acProp = assetClassCategorisation.createProperty("test:assetClass"); + acProp.setIndexed(true); + acProp.setIndexedAtomically(true); + acProp.setMandatory(true); + acProp.setMultiValued(true); + acProp.setStoredInIndex(true); + acProp.setTokenisedInIndex(true); + acProp.setType("d:" + DataTypeDefinition.CATEGORY.getLocalName()); + + investmentRegionCategorisationQName = QName.createQName(TEST_NAMESPACE, "InvestmentRegion"); + M2Aspect investmentRegionCategorisation = model.createAspect("test:" + investmentRegionCategorisationQName.getLocalName()); + investmentRegionCategorisation.setParentName("cm:" + ContentModel.ASPECT_CLASSIFIABLE.getLocalName()); + M2Property irProp = investmentRegionCategorisation.createProperty("test:investmentRegion"); + irProp.setIndexed(true); + irProp.setIndexedAtomically(true); + irProp.setMandatory(true); + irProp.setMultiValued(true); + irProp.setStoredInIndex(true); + irProp.setTokenisedInIndex(true); + irProp.setType("d:" + DataTypeDefinition.CATEGORY.getLocalName()); + + marketingRegionCategorisationQName = QName.createQName(TEST_NAMESPACE, "MarketingRegion"); + M2Aspect marketingRegionCategorisation = model.createAspect("test:" + marketingRegionCategorisationQName.getLocalName()); + marketingRegionCategorisation.setParentName("cm:" + ContentModel.ASPECT_CLASSIFIABLE.getLocalName()); + M2Property mrProp = marketingRegionCategorisation.createProperty("test:marketingRegion"); + mrProp.setIndexed(true); + mrProp.setIndexedAtomically(true); + mrProp.setMandatory(true); + mrProp.setMultiValued(true); + mrProp.setStoredInIndex(true); + mrProp.setTokenisedInIndex(true); + mrProp.setType("d:" + DataTypeDefinition.CATEGORY.getLocalName()); + + dictionaryDAO.putModel(model); + } + + private void buildBaseIndex() + { + LuceneIndexerImpl2 indexer = LuceneIndexerImpl2.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + System.currentTimeMillis() + "_" + (new Random().nextInt()), indexerAndSearcher); + indexer.setNodeService(nodeService); + //indexer.setLuceneIndexLock(luceneIndexLock); + indexer.setDictionaryService(dictionaryService); + indexer.setLuceneFullTextSearchIndexer(luceneFTS); + //indexer.clearIndex(); + indexer.createNode(new ChildAssociationRef(null, null, null, rootNodeRef)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CHILDREN, rootNodeRef, QName.createQName("{namespace}one"), n1)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CHILDREN, rootNodeRef, QName.createQName("{namespace}two"), n2)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CHILDREN, rootNodeRef, QName.createQName("{namespace}three"), n3)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CHILDREN, rootNodeRef, QName.createQName("{namespace}four"), n4)); + + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CHILDREN, rootNodeRef, QName.createQName("{namespace}categoryContainer"), catContainer)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CHILDREN, catContainer, QName.createQName("{cat}categoryRoot"), catRoot)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CATEGORIES, catRoot, QName.createQName(TEST_NAMESPACE, "AssetClass"), catACBase)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_SUBCATEGORIES, catACBase, QName.createQName(TEST_NAMESPACE, "Fixed"), catACOne)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_SUBCATEGORIES, catACBase, QName.createQName(TEST_NAMESPACE, "Equity"), catACTwo)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_SUBCATEGORIES, catACTwo, QName.createQName(TEST_NAMESPACE, "SpecialEquity"), catACThree)); + + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CATEGORIES, catRoot, QName.createQName(TEST_NAMESPACE, "Region"), catRBase)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_SUBCATEGORIES, catRBase, QName.createQName(TEST_NAMESPACE, "Europe"), catROne)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_SUBCATEGORIES, catRBase, QName.createQName(TEST_NAMESPACE, "RestOfWorld"), catRTwo)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_SUBCATEGORIES, catRTwo, QName.createQName(TEST_NAMESPACE, "US"), catRThree)); + + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CATEGORIES, n1, QName.createQName("{namespace}five"), n5)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CATEGORIES, n1, QName.createQName("{namespace}six"), n6)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CATEGORIES, n2, QName.createQName("{namespace}seven"), n7)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CATEGORIES, n2, QName.createQName("{namespace}eight"), n8)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CATEGORIES, n5, QName.createQName("{namespace}nine"), n9)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CATEGORIES, n5, QName.createQName("{namespace}ten"), n10)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CATEGORIES, n5, QName.createQName("{namespace}eleven"), n11)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CATEGORIES, n5, QName.createQName("{namespace}twelve"), n12)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CATEGORIES, n12, QName.createQName("{namespace}thirteen"), n13)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CATEGORIES, n13, QName.createQName("{namespace}fourteen"), n14)); + indexer.prepare(); + indexer.commit(); + } + + + public void testMulti() throws Exception + { + TransactionService transactionService = serviceRegistry.getTransactionService(); + UserTransaction tx = transactionService.getUserTransaction(); + tx.begin(); + buildBaseIndex(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("")); + ResultSet results; + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//*\" AND (PATH:\"/test:AssetClass/test:Equity/member\" PATH:\"/test:MarketingRegion/member\")", null, null); + //printPaths(results); + assertEquals(9, results.length()); + results.close(); + tx.rollback(); + } + + public void testBasic() throws Exception + { + TransactionService transactionService = serviceRegistry.getTransactionService(); + UserTransaction tx = transactionService.getUserTransaction(); + tx.begin(); + buildBaseIndex(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("")); + ResultSet results; + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:MarketingRegion\"", null, null); + //printPaths(results); + assertEquals(1, results.length()); + results.close(); + + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:MarketingRegion//member\"", null, null); + //printPaths(results); + assertEquals(6, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/cm:categoryContainer\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/cm:categoryContainer/cm:categoryRoot\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/cm:categoryContainer/cm:categoryRoot\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/cm:categoryContainer/cm:categoryRoot/test:AssetClass\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/cm:categoryContainer/cm:categoryRoot/test:AssetClass/member\" ", null, null); + assertEquals(1, results.length()); + results.close(); + + + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/cm:categoryContainer/cm:categoryRoot/test:AssetClass/test:Fixed\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/cm:categoryContainer/cm:categoryRoot/test:AssetClass/test:Equity\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:AssetClass\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:AssetClass/test:Fixed\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:AssetClass/test:Equity\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:AssetClass/test:*\"", null, null); + assertEquals(2, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:AssetClass//test:*\"", null, null); + assertEquals(3, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:AssetClass/test:Fixed/member\"", null, null); + //printPaths(results); + assertEquals(8, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:AssetClass/test:Equity/member\"", null, null); + //printPaths(results); + assertEquals(8, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:AssetClass/test:Equity/test:SpecialEquity/member//.\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:AssetClass/test:Equity/test:SpecialEquity/member//*\"", null, null); + assertEquals(0, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:AssetClass/test:Equity/test:SpecialEquity/member\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "+PATH:\"/test:AssetClass/test:Equity/member\" AND +PATH:\"/test:AssetClass/test:Fixed/member\"", null, null); + //printPaths(results); + assertEquals(3, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:AssetClass/test:Equity/member\" PATH:\"/test:AssetClass/test:Fixed/member\"", null, null); + //printPaths(results); + assertEquals(13, results.length()); + results.close(); + + // Region + + assertEquals(4, nodeService.getChildAssocs(catRoot).size()); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:Region\"", null, null); + //printPaths(results); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:Region/member\"", null, null); + //printPaths(results); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:Region/test:Europe/member\"", null, null); + //printPaths(results); + assertEquals(2, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:Region/test:RestOfWorld/member\"", null, null); + //printPaths(results); + assertEquals(2, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:Region//member\"", null, null); + //printPaths(results); + assertEquals(5, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:InvestmentRegion//member\"", null, null); + //printPaths(results); + assertEquals(5, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:MarketingRegion//member\"", null, null); + //printPaths(results); + assertEquals(6, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "+PATH:\"/test:AssetClass/test:Fixed/member\" AND +PATH:\"/test:Region/test:Europe/member\"", null, null); + //printPaths(results); + assertEquals(2, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "+PATH:\"/cm:categoryContainer/cm:categoryRoot/test:AssetClass/test:Fixed/member\" AND +PATH:\"/cm:categoryContainer/cm:categoryRoot/test:Region/test:Europe/member\"", null, null); + //printPaths(results); + assertEquals(2, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/test:AssetClass/test:Equity/member\" PATH:\"/test:MarketingRegion/member\"", null, null); + //printPaths(results); + assertEquals(9, results.length()); + results.close(); + tx.rollback(); + } + + public void testCategoryServiceImpl() throws Exception + { + TransactionService transactionService = serviceRegistry.getTransactionService(); + UserTransaction tx = transactionService.getUserTransaction(); + tx.begin(); + buildBaseIndex(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("")); + + ResultSet + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/cm:categoryContainer/cm:categoryRoot/test:AssetClass/*\" ", null, null); + assertEquals(3, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/cm:categoryContainer/cm:categoryRoot/test:AssetClass/member\" ", null, null); + assertEquals(1, results.length()); + results.close(); + + LuceneCategoryServiceImpl impl = new LuceneCategoryServiceImpl(); + impl.setNodeService(nodeService); + impl.setNamespacePrefixResolver(getNamespacePrefixReolsver("")); + impl.setIndexerAndSearcher(indexerAndSearcher); + impl.setDictionaryService(dictionaryService); + + Collection + result = impl.getChildren(catACBase , CategoryService.Mode.MEMBERS, CategoryService.Depth.IMMEDIATE); + assertEquals(1, result.size()); + + + result = impl.getChildren(catACBase , CategoryService.Mode.ALL, CategoryService.Depth.IMMEDIATE); + assertEquals(3, result.size()); + + + result = impl.getChildren(catACBase , CategoryService.Mode.SUB_CATEGORIES, CategoryService.Depth.IMMEDIATE); + assertEquals(2, result.size()); + + + result = impl.getChildren(catACBase , CategoryService.Mode.MEMBERS, CategoryService.Depth.ANY); + assertEquals(18, result.size()); + + + result = impl.getChildren(catACBase , CategoryService.Mode.ALL, CategoryService.Depth.ANY); + assertEquals(21, result.size()); + + + result = impl.getChildren(catACBase , CategoryService.Mode.SUB_CATEGORIES, CategoryService.Depth.ANY); + assertEquals(3, result.size()); + + + result = impl.getClassifications(rootNodeRef.getStoreRef()); + assertEquals(4, result.size()); + + + result = impl.getCategories(rootNodeRef.getStoreRef(), QName.createQName(TEST_NAMESPACE, "AssetClass"), CategoryService.Depth.IMMEDIATE); + assertEquals(2, result.size()); + + + Collection aspects = impl.getClassificationAspects(); + assertEquals(6, aspects.size()); + + tx.rollback(); + } + + private NamespacePrefixResolver getNamespacePrefixReolsver(String defaultURI) + { + DynamicNamespacePrefixResolver nspr = new DynamicNamespacePrefixResolver(null); + nspr.registerNamespace(NamespaceService.CONTENT_MODEL_PREFIX, NamespaceService.CONTENT_MODEL_1_0_URI); + nspr.registerNamespace("namespace", "namespace"); + nspr.registerNamespace("test", TEST_NAMESPACE); + nspr.registerNamespace(NamespaceService.DEFAULT_PREFIX, defaultURI); + return nspr; + } + + public void testCategoryService() throws Exception + { + TransactionService transactionService = serviceRegistry.getTransactionService(); + UserTransaction tx = transactionService.getUserTransaction(); + tx.begin(); + buildBaseIndex(); + assertEquals(1, categoryService.getChildren(catACBase , CategoryService.Mode.MEMBERS, CategoryService.Depth.IMMEDIATE).size()); + assertEquals(2, categoryService.getChildren(catACBase , CategoryService.Mode.SUB_CATEGORIES, CategoryService.Depth.IMMEDIATE).size()); + assertEquals(3, categoryService.getChildren(catACBase , CategoryService.Mode.ALL, CategoryService.Depth.IMMEDIATE).size()); + assertEquals(18, categoryService.getChildren(catACBase , CategoryService.Mode.MEMBERS, CategoryService.Depth.ANY).size()); + assertEquals(3, categoryService.getChildren(catACBase , CategoryService.Mode.SUB_CATEGORIES, CategoryService.Depth.ANY).size()); + assertEquals(21, categoryService.getChildren(catACBase , CategoryService.Mode.ALL, CategoryService.Depth.ANY).size()); + assertEquals(4, categoryService.getClassifications(rootNodeRef.getStoreRef()).size()); + assertEquals(2, categoryService.getCategories(rootNodeRef.getStoreRef(), QName.createQName(TEST_NAMESPACE, "AssetClass"), CategoryService.Depth.IMMEDIATE).size()); + assertEquals(3, categoryService.getCategories(rootNodeRef.getStoreRef(), QName.createQName(TEST_NAMESPACE, "AssetClass"), CategoryService.Depth.ANY).size()); + assertEquals(6, categoryService.getClassificationAspects().size()); + assertEquals(2, categoryService.getRootCategories(rootNodeRef.getStoreRef(), QName.createQName(TEST_NAMESPACE, "AssetClass")).size()); + + NodeRef newRoot = categoryService.createRootCategory(rootNodeRef.getStoreRef(),QName.createQName(TEST_NAMESPACE, "AssetClass"), "Fruit"); + tx.commit(); + tx = transactionService.getUserTransaction(); + tx.begin(); + assertEquals(3, categoryService.getRootCategories(rootNodeRef.getStoreRef(), QName.createQName(TEST_NAMESPACE, "AssetClass")).size()); + assertEquals(3, categoryService.getCategories(rootNodeRef.getStoreRef(), QName.createQName(TEST_NAMESPACE, "AssetClass"), CategoryService.Depth.IMMEDIATE).size()); + assertEquals(4, categoryService.getCategories(rootNodeRef.getStoreRef(), QName.createQName(TEST_NAMESPACE, "AssetClass"), CategoryService.Depth.ANY).size()); + + NodeRef newCat = categoryService.createCategory(newRoot, "Banana"); + tx.commit(); + tx = transactionService.getUserTransaction(); + tx.begin(); + assertEquals(3, categoryService.getRootCategories(rootNodeRef.getStoreRef(), QName.createQName(TEST_NAMESPACE, "AssetClass")).size()); + assertEquals(3, categoryService.getCategories(rootNodeRef.getStoreRef(), QName.createQName(TEST_NAMESPACE, "AssetClass"), CategoryService.Depth.IMMEDIATE).size()); + assertEquals(5, categoryService.getCategories(rootNodeRef.getStoreRef(), QName.createQName(TEST_NAMESPACE, "AssetClass"), CategoryService.Depth.ANY).size()); + + categoryService.deleteCategory(newCat); + tx.commit(); + tx = transactionService.getUserTransaction(); + tx.begin(); + assertEquals(3, categoryService.getRootCategories(rootNodeRef.getStoreRef(), QName.createQName(TEST_NAMESPACE, "AssetClass")).size()); + assertEquals(3, categoryService.getCategories(rootNodeRef.getStoreRef(), QName.createQName(TEST_NAMESPACE, "AssetClass"), CategoryService.Depth.IMMEDIATE).size()); + assertEquals(4, categoryService.getCategories(rootNodeRef.getStoreRef(), QName.createQName(TEST_NAMESPACE, "AssetClass"), CategoryService.Depth.ANY).size()); + + categoryService.deleteCategory(newRoot); + tx.commit(); + tx = transactionService.getUserTransaction(); + tx.begin(); + assertEquals(2, categoryService.getRootCategories(rootNodeRef.getStoreRef(), QName.createQName(TEST_NAMESPACE, "AssetClass")).size()); + assertEquals(2, categoryService.getCategories(rootNodeRef.getStoreRef(), QName.createQName(TEST_NAMESPACE, "AssetClass"), CategoryService.Depth.IMMEDIATE).size()); + assertEquals(3, categoryService.getCategories(rootNodeRef.getStoreRef(), QName.createQName(TEST_NAMESPACE, "AssetClass"), CategoryService.Depth.ANY).size()); + + + tx.rollback(); + } + + private int getTotalScore(ResultSet results) + { + int totalScore = 0; + for(ResultSetRow row: results) + { + totalScore += row.getScore(); + } + return totalScore; + } +} diff --git a/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexer.java b/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexer.java index 763ea20996..68c8763abe 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexer.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexer.java @@ -19,6 +19,7 @@ package org.alfresco.repo.search.impl.lucene; import java.util.Set; import org.alfresco.repo.search.Indexer; +import org.alfresco.repo.search.IndexerSPI; import org.alfresco.repo.search.impl.lucene.fts.FTSIndexerAware; import org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer; import org.alfresco.service.cmr.dictionary.DictionaryService; @@ -28,7 +29,7 @@ import org.alfresco.service.cmr.repository.NodeService; /** * @author Andy Hind */ -public interface LuceneIndexer extends Indexer, Lockable +public interface LuceneIndexer extends IndexerSPI, Lockable { public void commit(); @@ -39,9 +40,6 @@ public interface LuceneIndexer extends Indexer, Lockable public void setDictionaryService(DictionaryService dictionaryService); public void setLuceneFullTextSearchIndexer(FullTextSearchIndexer luceneFullTextSearchIndexer); - public void updateFullTextSearch(int size); - public void registerCallBack(FTSIndexerAware indexer); - public String getDeltaId(); public void flushPending() throws LuceneIndexException; public Set getDeletions(); diff --git a/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexer2.java b/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexer2.java new file mode 100644 index 0000000000..a7924ed311 --- /dev/null +++ b/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexer2.java @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2005 Alfresco, Inc. + * + * Licensed under the Mozilla Public License version 1.1 + * with a permitted attribution clause. You may obtain a + * copy of the License at + * + * http://www.alfresco.org/legal/license.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.alfresco.repo.search.impl.lucene; + +import java.util.Set; + +import org.alfresco.repo.search.Indexer; +import org.alfresco.repo.search.IndexerSPI; +import org.alfresco.repo.search.impl.lucene.fts.FTSIndexerAware; +import org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer; +import org.alfresco.service.cmr.dictionary.DictionaryService; +import org.alfresco.service.cmr.repository.NodeRef; +import org.alfresco.service.cmr.repository.NodeService; + +/** + * @author Andy Hind + */ +public interface LuceneIndexer2 extends IndexerSPI +{ + + public void commit(); + public void rollback(); + public int prepare(); + public boolean isModified(); + public void setNodeService(NodeService nodeService); + public void setDictionaryService(DictionaryService dictionaryService); + public void setLuceneFullTextSearchIndexer(FullTextSearchIndexer luceneFullTextSearchIndexer); + + public String getDeltaId(); + public void flushPending() throws LuceneIndexException; + public Set getDeletions(); + public boolean getDeleteOnlyNodes(); +} diff --git a/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexerAndSearcherFactory2.java b/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexerAndSearcherFactory2.java new file mode 100644 index 0000000000..c46a0e3add --- /dev/null +++ b/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexerAndSearcherFactory2.java @@ -0,0 +1,1105 @@ +/* + * Copyright (C) 2005 Alfresco, Inc. + * + * Licensed under the Mozilla Public License version 1.1 + * with a permitted attribution clause. You may obtain a + * copy of the License at + * + * http://www.alfresco.org/legal/license.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.alfresco.repo.search.impl.lucene; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.transaction.RollbackException; +import javax.transaction.SystemException; +import javax.transaction.Transaction; +import javax.transaction.xa.XAException; +import javax.transaction.xa.XAResource; +import javax.transaction.xa.Xid; + +import org.alfresco.error.AlfrescoRuntimeException; +import org.alfresco.repo.search.IndexerException; +import org.alfresco.repo.search.QueryRegisterComponent; +import org.alfresco.repo.search.SearcherException; +import org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer; +import org.alfresco.repo.search.transaction.LuceneIndexLock; +import org.alfresco.repo.search.transaction.SimpleTransaction; +import org.alfresco.repo.search.transaction.SimpleTransactionManager; +import org.alfresco.repo.transaction.AlfrescoTransactionSupport; +import org.alfresco.repo.transaction.TransactionUtil; +import org.alfresco.repo.transaction.TransactionUtil.TransactionWork; +import org.alfresco.service.cmr.dictionary.DictionaryService; +import org.alfresco.service.cmr.repository.ContentService; +import org.alfresco.service.cmr.repository.NodeService; +import org.alfresco.service.cmr.repository.StoreRef; +import org.alfresco.service.namespace.NamespaceService; +import org.alfresco.service.transaction.TransactionService; +import org.alfresco.util.GUID; +import org.apache.commons.io.FileUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.store.Lock; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +/** + * This class is resource manager LuceneIndexers and LuceneSearchers. + * + * It supports two phase commit inside XA transactions and outside transactions + * it provides thread local transaction support. + * + * TODO: Provide pluggable support for a transaction manager TODO: Integrate + * with Spring transactions + * + * @author andyh + * + */ + +public class LuceneIndexerAndSearcherFactory2 implements LuceneIndexerAndSearcher, XAResource +{ + private DictionaryService dictionaryService; + + private NamespaceService nameSpaceService; + + private int queryMaxClauses; + + private int indexerBatchSize; + + private int indexerMinMergeDocs; + + private int indexerMergeFactor; + + private int indexerMaxMergeDocs; + + private String lockDirectory; + + /** + * A map of active global transactions . It contains all the indexers a + * transaction has used, with at most one indexer for each store within a + * transaction + */ + + private static Map> activeIndexersInGlobalTx = new HashMap>(); + + /** + * Suspended global transactions. + */ + private static Map> suspendedIndexersInGlobalTx = new HashMap>(); + + /** + * Thread local indexers - used outside a global transaction + */ + + private static ThreadLocal> threadLocalIndexers = new ThreadLocal>(); + + /** + * The dafault timeout for transactions TODO: Respect this + */ + + private int timeout = DEFAULT_TIMEOUT; + + /** + * Default time out value set to 10 minutes. + */ + private static final int DEFAULT_TIMEOUT = 600000; + + /** + * The node service we use to get information about nodes + */ + + private NodeService nodeService; + + private LuceneIndexLock luceneIndexLock; + + private FullTextSearchIndexer luceneFullTextSearchIndexer; + + private String indexRootLocation; + + private ContentService contentService; + + private QueryRegisterComponent queryRegister; + + /** the maximum transformation time to allow atomically, defaulting to 20ms */ + private long maxAtomicTransformationTime = 20; + + private int indexerMaxFieldLength; + + /** + * Private constructor for the singleton TODO: FIt in with IOC + */ + + public LuceneIndexerAndSearcherFactory2() + { + super(); + } + + /** + * Setter for getting the node service via IOC Used in the Spring container + * + * @param nodeService + */ + + public void setNodeService(NodeService nodeService) + { + this.nodeService = nodeService; + } + + public void setDictionaryService(DictionaryService dictionaryService) + { + this.dictionaryService = dictionaryService; + } + + public void setNameSpaceService(NamespaceService nameSpaceService) + { + this.nameSpaceService = nameSpaceService; + } + + public void setLuceneIndexLock(LuceneIndexLock luceneIndexLock) + { + this.luceneIndexLock = luceneIndexLock; + } + + public void setLuceneFullTextSearchIndexer(FullTextSearchIndexer luceneFullTextSearchIndexer) + { + this.luceneFullTextSearchIndexer = luceneFullTextSearchIndexer; + } + + public void setIndexRootLocation(String indexRootLocation) + { + this.indexRootLocation = indexRootLocation; + } + + public void setQueryRegister(QueryRegisterComponent queryRegister) + { + this.queryRegister = queryRegister; + } + + /** + * Set the maximum average transformation time allowed to a transformer in order to have + * the transformation performed in the current transaction. The default is 20ms. + * + * @param maxAtomicTransformationTime the maximum average time that a text transformation may + * take in order to be performed atomically. + */ + public void setMaxAtomicTransformationTime(long maxAtomicTransformationTime) + { + this.maxAtomicTransformationTime = maxAtomicTransformationTime; + } + + /** + * Check if we are in a global transactoin according to the transaction + * manager + * + * @return + */ + + private boolean inGlobalTransaction() + { + try + { + return SimpleTransactionManager.getInstance().getTransaction() != null; + } + catch (SystemException e) + { + return false; + } + } + + /** + * Get the local transaction - may be null oif we are outside a transaction. + * + * @return + * @throws IndexerException + */ + private SimpleTransaction getTransaction() throws IndexerException + { + try + { + return SimpleTransactionManager.getInstance().getTransaction(); + } + catch (SystemException e) + { + throw new IndexerException("Failed to get transaction", e); + } + } + + /** + * Get an indexer for the store to use in the current transaction for this + * thread of control. + * + * @param storeRef - + * the id of the store + */ + public LuceneIndexer2 getIndexer(StoreRef storeRef) throws IndexerException + { + // register to receive txn callbacks + // TODO: make this conditional on whether the XA stuff is being used + // directly on not + AlfrescoTransactionSupport.bindLucene(this); + + if (inGlobalTransaction()) + { + SimpleTransaction tx = getTransaction(); + // Only find indexers in the active list + Map indexers = activeIndexersInGlobalTx.get(tx); + if (indexers == null) + { + if (suspendedIndexersInGlobalTx.containsKey(tx)) + { + throw new IndexerException("Trying to obtain an index for a suspended transaction."); + } + indexers = new HashMap(); + activeIndexersInGlobalTx.put(tx, indexers); + try + { + tx.enlistResource(this); + } + // TODO: what to do in each case? + catch (IllegalStateException e) + { + throw new IndexerException("", e); + } + catch (RollbackException e) + { + throw new IndexerException("", e); + } + catch (SystemException e) + { + throw new IndexerException("", e); + } + } + LuceneIndexer2 indexer = indexers.get(storeRef); + if (indexer == null) + { + indexer = createIndexer(storeRef, getTransactionId(tx, storeRef)); + indexers.put(storeRef, indexer); + } + return indexer; + } + else + // A thread local transaction + { + return getThreadLocalIndexer(storeRef); + } + + } + + private LuceneIndexer2 getThreadLocalIndexer(StoreRef storeRef) + { + Map indexers = threadLocalIndexers.get(); + if (indexers == null) + { + indexers = new HashMap(); + threadLocalIndexers.set(indexers); + } + LuceneIndexer2 indexer = indexers.get(storeRef); + if (indexer == null) + { + indexer = createIndexer(storeRef, GUID.generate()); + indexers.put(storeRef, indexer); + } + return indexer; + } + + /** + * Get the transaction identifier uised to store it in the transaction map. + * + * @param tx + * @return + */ + private static String getTransactionId(Transaction tx, StoreRef storeRef) + { + if (tx instanceof SimpleTransaction) + { + SimpleTransaction simpleTx = (SimpleTransaction) tx; + return simpleTx.getGUID(); + } + else + { + Map indexers = threadLocalIndexers.get(); + if (indexers != null) + { + LuceneIndexer2 indexer = indexers.get(storeRef); + if (indexer != null) + { + return indexer.getDeltaId(); + } + } + return null; + } + } + + /** + * Encapsulate creating an indexer + * + * @param storeRef + * @param deltaId + * @return + */ + private LuceneIndexerImpl2 createIndexer(StoreRef storeRef, String deltaId) + { + LuceneIndexerImpl2 indexer = LuceneIndexerImpl2.getUpdateIndexer(storeRef, deltaId, this); + indexer.setNodeService(nodeService); + indexer.setDictionaryService(dictionaryService); + //indexer.setLuceneIndexLock(luceneIndexLock); + indexer.setLuceneFullTextSearchIndexer(luceneFullTextSearchIndexer); + indexer.setContentService(contentService); + indexer.setMaxAtomicTransformationTime(maxAtomicTransformationTime); + return indexer; + } + + /** + * Encapsulate creating a searcher over the main index + */ + public LuceneSearcher2 getSearcher(StoreRef storeRef, boolean searchDelta) throws SearcherException + { + String deltaId = null; + LuceneIndexer2 indexer = null; + if (searchDelta) + { + deltaId = getTransactionId(getTransaction(), storeRef); + if (deltaId != null) + { + indexer = getIndexer(storeRef); + } + } + LuceneSearcher2 searcher = getSearcher(storeRef, indexer); + return searcher; + } + + /** + * Get a searcher over the index and the current delta + * + * @param storeRef + * @param deltaId + * @return + * @throws SearcherException + */ + private LuceneSearcher2 getSearcher(StoreRef storeRef, LuceneIndexer2 indexer) throws SearcherException + { + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(storeRef, indexer, this); + searcher.setNamespacePrefixResolver(nameSpaceService); + //searcher.setLuceneIndexLock(luceneIndexLock); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setQueryRegister(queryRegister); + return searcher; + } + + /* + * XAResource implementation + */ + + public void commit(Xid xid, boolean onePhase) throws XAException + { + try + { + // TODO: Should be remembering overall state + // TODO: Keep track of prepare responses + Map indexers = activeIndexersInGlobalTx.get(xid); + if (indexers == null) + { + if (suspendedIndexersInGlobalTx.containsKey(xid)) + { + throw new XAException("Trying to commit indexes for a suspended transaction."); + } + else + { + // nothing to do + return; + } + } + + if (onePhase) + { + if (indexers.size() == 0) + { + return; + } + else if (indexers.size() == 1) + { + for (LuceneIndexer2 indexer : indexers.values()) + { + indexer.commit(); + } + return; + } + else + { + throw new XAException("Trying to do one phase commit on more than one index"); + } + } + else + // two phase + { + for (LuceneIndexer2 indexer : indexers.values()) + { + indexer.commit(); + } + return; + } + } finally + { + activeIndexersInGlobalTx.remove(xid); + } + } + + public void end(Xid xid, int flag) throws XAException + { + Map indexers = activeIndexersInGlobalTx.get(xid); + if (indexers == null) + { + if (suspendedIndexersInGlobalTx.containsKey(xid)) + { + throw new XAException("Trying to commit indexes for a suspended transaction."); + } + else + { + // nothing to do + return; + } + } + if (flag == XAResource.TMSUSPEND) + { + activeIndexersInGlobalTx.remove(xid); + suspendedIndexersInGlobalTx.put(xid, indexers); + } + else if (flag == TMFAIL) + { + activeIndexersInGlobalTx.remove(xid); + suspendedIndexersInGlobalTx.remove(xid); + } + else if (flag == TMSUCCESS) + { + activeIndexersInGlobalTx.remove(xid); + } + } + + public void forget(Xid xid) throws XAException + { + activeIndexersInGlobalTx.remove(xid); + suspendedIndexersInGlobalTx.remove(xid); + } + + public int getTransactionTimeout() throws XAException + { + return timeout; + } + + public boolean isSameRM(XAResource xar) throws XAException + { + return (xar instanceof LuceneIndexerAndSearcherFactory2); + } + + public int prepare(Xid xid) throws XAException + { + // TODO: Track state OK, ReadOnly, Exception (=> rolled back?) + Map indexers = activeIndexersInGlobalTx.get(xid); + if (indexers == null) + { + if (suspendedIndexersInGlobalTx.containsKey(xid)) + { + throw new XAException("Trying to commit indexes for a suspended transaction."); + } + else + { + // nothing to do + return XAResource.XA_OK; + } + } + boolean isPrepared = true; + boolean isModified = false; + for (LuceneIndexer2 indexer : indexers.values()) + { + try + { + isModified |= indexer.isModified(); + indexer.prepare(); + } + catch (IndexerException e) + { + isPrepared = false; + } + } + if (isPrepared) + { + if (isModified) + { + return XAResource.XA_OK; + } + else + { + return XAResource.XA_RDONLY; + } + } + else + { + throw new XAException("Failed to prepare: requires rollback"); + } + } + + public Xid[] recover(int arg0) throws XAException + { + // We can not rely on being able to recover at the moment + // Avoiding for performance benefits at the moment + // Assume roll back and no recovery - in the worst case we get an unused + // delta + // This should be there to avoid recovery of partial commits. + // It is difficult to see how we can mandate the same conditions. + return new Xid[0]; + } + + public void rollback(Xid xid) throws XAException + { + // TODO: What to do if all do not roll back? + try + { + Map indexers = activeIndexersInGlobalTx.get(xid); + if (indexers == null) + { + if (suspendedIndexersInGlobalTx.containsKey(xid)) + { + throw new XAException("Trying to commit indexes for a suspended transaction."); + } + else + { + // nothing to do + return; + } + } + for (LuceneIndexer2 indexer : indexers.values()) + { + indexer.rollback(); + } + } finally + { + activeIndexersInGlobalTx.remove(xid); + } + } + + public boolean setTransactionTimeout(int timeout) throws XAException + { + this.timeout = timeout; + return true; + } + + public void start(Xid xid, int flag) throws XAException + { + Map active = activeIndexersInGlobalTx.get(xid); + Map suspended = suspendedIndexersInGlobalTx.get(xid); + if (flag == XAResource.TMJOIN) + { + // must be active + if ((active != null) && (suspended == null)) + { + return; + } + else + { + throw new XAException("Trying to rejoin transaction in an invalid state"); + } + + } + else if (flag == XAResource.TMRESUME) + { + // must be suspended + if ((active == null) && (suspended != null)) + { + suspendedIndexersInGlobalTx.remove(xid); + activeIndexersInGlobalTx.put(xid, suspended); + return; + } + else + { + throw new XAException("Trying to rejoin transaction in an invalid state"); + } + + } + else if (flag == XAResource.TMNOFLAGS) + { + if ((active == null) && (suspended == null)) + { + return; + } + else + { + throw new XAException("Trying to start an existing or suspended transaction"); + } + } + else + { + throw new XAException("Unkown flags for start " + flag); + } + + } + + /* + * Thread local support for transactions + */ + + /** + * Commit the transaction + */ + + public void commit() throws IndexerException + { + try + { + Map indexers = threadLocalIndexers.get(); + if (indexers != null) + { + for (LuceneIndexer2 indexer : indexers.values()) + { + try + { + indexer.commit(); + } + catch (IndexerException e) + { + rollback(); + throw e; + } + } + } + } finally + { + if (threadLocalIndexers.get() != null) + { + threadLocalIndexers.get().clear(); + threadLocalIndexers.set(null); + } + } + } + + /** + * Prepare the transaction TODO: Store prepare results + * + * @return + */ + public int prepare() throws IndexerException + { + boolean isPrepared = true; + boolean isModified = false; + Map indexers = threadLocalIndexers.get(); + if (indexers != null) + { + for (LuceneIndexer2 indexer : indexers.values()) + { + try + { + isModified |= indexer.isModified(); + indexer.prepare(); + } + catch (IndexerException e) + { + isPrepared = false; + throw new IndexerException("Failed to prepare: requires rollback", e); + } + } + } + if (isPrepared) + { + if (isModified) + { + return XAResource.XA_OK; + } + else + { + return XAResource.XA_RDONLY; + } + } + else + { + throw new IndexerException("Failed to prepare: requires rollback"); + } + } + + /** + * Roll back the transaction + */ + public void rollback() + { + Map indexers = threadLocalIndexers.get(); + + if (indexers != null) + { + for (LuceneIndexer2 indexer : indexers.values()) + { + try + { + indexer.rollback(); + } + catch (IndexerException e) + { + + } + } + } + + if (threadLocalIndexers.get() != null) + { + threadLocalIndexers.get().clear(); + threadLocalIndexers.set(null); + } + + } + + public void flush() + { + // TODO: Needs fixing if we expose the indexer in JTA + Map indexers = threadLocalIndexers.get(); + + if (indexers != null) + { + for (LuceneIndexer2 indexer : indexers.values()) + { + indexer.flushPending(); + } + } + } + + public void setContentService(ContentService contentService) + { + this.contentService = contentService; + } + + public String getIndexRootLocation() + { + return indexRootLocation; + } + + public int getIndexerBatchSize() + { + return indexerBatchSize; + } + + public void setIndexerBatchSize(int indexerBatchSize) + { + this.indexerBatchSize = indexerBatchSize; + } + + public int getIndexerMaxMergeDocs() + { + return indexerMaxMergeDocs; + } + + public void setIndexerMaxMergeDocs(int indexerMaxMergeDocs) + { + this.indexerMaxMergeDocs = indexerMaxMergeDocs; + } + + public int getIndexerMergeFactor() + { + return indexerMergeFactor; + } + + public void setIndexerMergeFactor(int indexerMergeFactor) + { + this.indexerMergeFactor = indexerMergeFactor; + } + + public int getIndexerMinMergeDocs() + { + return indexerMinMergeDocs; + } + + public void setIndexerMinMergeDocs(int indexerMinMergeDocs) + { + this.indexerMinMergeDocs = indexerMinMergeDocs; + } + + public String getLockDirectory() + { + return lockDirectory; + } + + public void setLockDirectory(String lockDirectory) + { + this.lockDirectory = lockDirectory; + // Set the lucene lock file via System property + // org.apache.lucene.lockdir + System.setProperty("org.apache.lucene.lockdir", lockDirectory); + // Make sure the lock directory exists + File lockDir = new File(lockDirectory); + if (!lockDir.exists()) + { + lockDir.mkdirs(); + } + // clean out any existing locks when we start up + + File[] children = lockDir.listFiles(); + if (children != null) + { + for (int i = 0; i < children.length; i++) + { + File child = children[i]; + if (child.isFile()) + { + if (child.exists() && !child.delete() && child.exists()) + { + throw new IllegalStateException("Failed to delete " + child); + } + } + } + } + } + + public int getQueryMaxClauses() + { + return queryMaxClauses; + } + + public void setQueryMaxClauses(int queryMaxClauses) + { + this.queryMaxClauses = queryMaxClauses; + BooleanQuery.setMaxClauseCount(this.queryMaxClauses); + } + + public void setWriteLockTimeout(long timeout) + { + IndexWriter.WRITE_LOCK_TIMEOUT = timeout; + } + + public void setCommitLockTimeout(long timeout) + { + IndexWriter.COMMIT_LOCK_TIMEOUT = timeout; + } + + public void setLockPollInterval(long time) + { + Lock.LOCK_POLL_INTERVAL = time; + } + + public int getIndexerMaxFieldLength() + { + return indexerMaxFieldLength; + } + + public void setIndexerMaxFieldLength(int indexerMaxFieldLength) + { + this.indexerMaxFieldLength = indexerMaxFieldLength; + System.setProperty("org.apache.lucene.maxFieldLength", "" + indexerMaxFieldLength); + } + + /** + * This component is able to safely perform backups of the Lucene indexes while + * the server is running. + *

+ * It can be run directly by calling the {@link #backup() } method, but the convenience + * {@link LuceneIndexBackupJob} can be used to call it as well. + * + * @author Derek Hulley + */ + public static class LuceneIndexBackupComponent + { + private static Log logger = LogFactory.getLog(LuceneIndexerAndSearcherFactory2.class); + + private TransactionService transactionService; + private LuceneIndexerAndSearcherFactory2 factory; + private NodeService nodeService; + private String targetLocation; + + public LuceneIndexBackupComponent() + { + } + + /** + * Provides transactions in which to perform the work + * + * @param transactionService + */ + public void setTransactionService(TransactionService transactionService) + { + this.transactionService = transactionService; + } + + /** + * Set the Lucene index factory that will be used to control the index locks + * + * @param factory the index factory + */ + public void setFactory(LuceneIndexerAndSearcherFactory2 factory) + { + this.factory = factory; + } + + /** + * Used to retrieve the stores + * + * @param nodeService the node service + */ + public void setNodeService(NodeService nodeService) + { + this.nodeService = nodeService; + } + + /** + * Set the directory to which the backup will be copied + * + * @param targetLocation the backup directory + */ + public void setTargetLocation(String targetLocation) + { + this.targetLocation = targetLocation; + } + + /** + * Backup the Lucene indexes + */ + public void backup() + { + TransactionWork backupWork = new TransactionWork() + { + public Object doWork() throws Exception + { + backupImpl(); + return null; + } + }; + TransactionUtil.executeInUserTransaction(transactionService, backupWork); + } + + private void backupImpl() + { + // create the location to copy to + File targetDir = new File(targetLocation); + if (targetDir.exists() && !targetDir.isDirectory()) + { + throw new AlfrescoRuntimeException("Target location is a file and not a directory: " + targetDir); + } + File targetParentDir = targetDir.getParentFile(); + if (targetParentDir == null) + { + throw new AlfrescoRuntimeException("Target location may not be a root directory: " + targetDir); + } + File tempDir = new File(targetParentDir, "indexbackup_temp"); + + // get all the available stores + List storeRefs = nodeService.getStores(); + + // lock all the stores + List lockedStores = new ArrayList(storeRefs.size()); + try + { + for (StoreRef storeRef : storeRefs) + { + factory.luceneIndexLock.getWriteLock(storeRef); + lockedStores.add(storeRef); + } + File indexRootDir = new File(factory.indexRootLocation); + // perform the copy + backupDirectory(indexRootDir, tempDir, targetDir); + } + catch (Throwable e) + { + throw new AlfrescoRuntimeException("Failed to copy Lucene index root: \n" + + " Index root: " + factory.indexRootLocation + "\n" + + " Target: " + targetDir, + e); + } + finally + { + for (StoreRef storeRef : lockedStores) + { + try + { + factory.luceneIndexLock.releaseWriteLock(storeRef); + } + catch (Throwable e) + { + logger.error("Failed to release index lock for store " + storeRef, e); + } + } + } + if (logger.isDebugEnabled()) + { + logger.debug("Backed up Lucene indexes: \n" + + " Target directory: " + targetDir); + } + } + + /** + * Makes a backup of the source directory via a temporary folder + * @param storeRef + */ + private void backupDirectory(File sourceDir, File tempDir, File targetDir) throws Exception + { + if (!sourceDir.exists()) + { + // there is nothing to copy + return; + } + // delete the files from the temp directory + if (tempDir.exists()) + { + FileUtils.deleteDirectory(tempDir); + if (tempDir.exists()) + { + throw new AlfrescoRuntimeException("Temp directory exists and cannot be deleted: " + tempDir); + } + } + // copy to the temp directory + FileUtils.copyDirectory(sourceDir, tempDir, true); + // check that the temp directory was created + if (!tempDir.exists()) + { + throw new AlfrescoRuntimeException("Copy to temp location failed"); + } + // delete the target directory + FileUtils.deleteDirectory(targetDir); + if (targetDir.exists()) + { + throw new AlfrescoRuntimeException("Failed to delete older files from target location"); + } + // rename the temp to be the target + tempDir.renameTo(targetDir); + // make sure the rename worked + if (!targetDir.exists()) + { + throw new AlfrescoRuntimeException("Failed to rename temporary directory to target backup directory"); + } + } + } + + /** + * Job that lock uses the {@link LuceneIndexBackupComponent} to perform safe backups of the Lucene indexes. + * + * @author Derek Hulley + */ + public static class LuceneIndexBackupJob implements Job + { + /** KEY_LUCENE_INDEX_BACKUP_COMPONENT = 'luceneIndexBackupComponent' */ + public static final String KEY_LUCENE_INDEX_BACKUP_COMPONENT = "luceneIndexBackupComponent"; + + /** + * Locks the Lucene indexes and copies them to a backup location + */ + public void execute(JobExecutionContext context) throws JobExecutionException + { + JobDataMap jobData = context.getJobDetail().getJobDataMap(); + LuceneIndexBackupComponent backupComponent = (LuceneIndexBackupComponent) jobData.get(KEY_LUCENE_INDEX_BACKUP_COMPONENT); + if (backupComponent == null) + { + throw new JobExecutionException("Missing job data: " + KEY_LUCENE_INDEX_BACKUP_COMPONENT); + } + // perform the backup + backupComponent.backup(); + } + } +} diff --git a/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexerImpl2.java b/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexerImpl2.java new file mode 100644 index 0000000000..858326e4da --- /dev/null +++ b/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexerImpl2.java @@ -0,0 +1,1974 @@ +/* + * Copyright (C) 2005 Alfresco, Inc. + * + * Licensed under the Mozilla Public License version 1.1 + * with a permitted attribution clause. You may obtain a + * copy of the License at + * + * http://www.alfresco.org/legal/license.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.alfresco.repo.search.impl.lucene; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Serializable; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; + +import javax.transaction.Status; +import javax.transaction.xa.XAResource; + +import org.alfresco.model.ContentModel; +import org.alfresco.repo.content.MimetypeMap; +import org.alfresco.repo.content.transform.ContentTransformer; +import org.alfresco.repo.search.IndexerException; +import org.alfresco.repo.search.impl.lucene.fts.FTSIndexerAware; +import org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer; +import org.alfresco.repo.search.impl.lucene.index.TransactionStatus; +import org.alfresco.service.cmr.dictionary.AspectDefinition; +import org.alfresco.service.cmr.dictionary.DataTypeDefinition; +import org.alfresco.service.cmr.dictionary.DictionaryService; +import org.alfresco.service.cmr.dictionary.PropertyDefinition; +import org.alfresco.service.cmr.dictionary.TypeDefinition; +import org.alfresco.service.cmr.repository.ChildAssociationRef; +import org.alfresco.service.cmr.repository.ContentData; +import org.alfresco.service.cmr.repository.ContentIOException; +import org.alfresco.service.cmr.repository.ContentReader; +import org.alfresco.service.cmr.repository.ContentService; +import org.alfresco.service.cmr.repository.ContentWriter; +import org.alfresco.service.cmr.repository.InvalidNodeRefException; +import org.alfresco.service.cmr.repository.NodeRef; +import org.alfresco.service.cmr.repository.NodeService; +import org.alfresco.service.cmr.repository.Path; +import org.alfresco.service.cmr.repository.StoreRef; +import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter; +import org.alfresco.service.cmr.search.ResultSetRow; +import org.alfresco.service.cmr.search.SearchParameters; +import org.alfresco.service.namespace.QName; +import org.alfresco.util.EqualsHelper; +import org.alfresco.util.ISO9075; +import org.apache.log4j.Logger; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermDocs; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Hits; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Searcher; +import org.apache.lucene.search.TermQuery; + +/** + * The implementation of the lucene based indexer. Supports basic transactional behaviour if used on its own. + * + * @author andyh + * + */ +public class LuceneIndexerImpl2 extends LuceneBase2 implements LuceneIndexer2 +{ + public static final String NOT_INDEXED_NO_TRANSFORMATION = "nint"; + + public static final String NOT_INDEXED_TRANSFORMATION_FAILED = "nitf"; + + public static final String NOT_INDEXED_CONTENT_MISSING = "nicm"; + + private static Logger s_logger = Logger.getLogger(LuceneIndexerImpl2.class); + + /** + * Enum for indexing actions against a node + */ + private enum Action + { + INDEX, REINDEX, DELETE, CASCADEREINDEX + }; + + /** + * The node service we use to get information about nodes + */ + private NodeService nodeService; + + /** + * Content service to get content for indexing. + */ + private ContentService contentService; + + /** the maximum transformation time to allow atomically, defaulting to 20ms */ + private long maxAtomicTransformationTime = 20; + + /** + * A list of all deletions we have made - at merge these deletions need to be made against the main index. + * + * TODO: Consider if this information needs to be persisted for recovery + */ + private Set deletions = new LinkedHashSet(); + + private long docs; + + /** + * The status of this index - follows javax.transaction.Status + */ + + private int status = Status.STATUS_UNKNOWN; + + /** + * Has this index been modified? + */ + + private boolean isModified = false; + + /** + * Flag to indicte if we are doing an in transactional delta or a batch update to the index. If true, we are just fixing up non atomically indexed things from one or more other + * updates. + */ + + private Boolean isFTSUpdate = null; + + /** + * List of pending indexing commands. + */ + private List commandList = new ArrayList(10000); + + /** + * Call back to make after doing non atomic indexing + */ + private FTSIndexerAware callBack; + + /** + * Count of remaining items to index non atomically + */ + private int remainingCount = 0; + + /** + * A list of stuff that requires non atomic indexing + */ + private ArrayList toFTSIndex = new ArrayList(); + + /** + * Default construction + * + */ + LuceneIndexerImpl2() + { + super(); + } + + /** + * IOC setting of dictionary service + */ + + public void setDictionaryService(DictionaryService dictionaryService) + { + super.setDictionaryService(dictionaryService); + } + + /** + * Setter for getting the node service via IOC Used in the Spring container + * + * @param nodeService + */ + + public void setNodeService(NodeService nodeService) + { + this.nodeService = nodeService; + } + + /** + * IOC setting of the content service + * + * @param contentService + */ + public void setContentService(ContentService contentService) + { + this.contentService = contentService; + } + + public void setMaxAtomicTransformationTime(long maxAtomicTransformationTime) + { + this.maxAtomicTransformationTime = maxAtomicTransformationTime; + } + + /* + * =========================== Indexer Implementation ============================ + */ + + /** + * Utility method to check we are in the correct state to do work Also keeps track of the dirty flag. + * + */ + + private void checkAbleToDoWork(boolean isFTS, boolean isModified) + { + if (isFTSUpdate == null) + { + isFTSUpdate = Boolean.valueOf(isFTS); + } + else + { + if (isFTS != isFTSUpdate.booleanValue()) + { + throw new IndexerException("Can not mix FTS and transactional updates"); + } + } + + switch (status) + { + case Status.STATUS_UNKNOWN: + status = Status.STATUS_ACTIVE; + break; + case Status.STATUS_ACTIVE: + // OK + break; + default: + // All other states are a problem + throw new IndexerException(buildErrorString()); + } + this.isModified = isModified; + } + + /** + * Utility method to report errors about invalid state. + * + * @return + */ + private String buildErrorString() + { + StringBuilder buffer = new StringBuilder(128); + buffer.append("The indexer is unable to accept more work: "); + switch (status) + { + case Status.STATUS_COMMITTED: + buffer.append("The indexer has been committed"); + break; + case Status.STATUS_COMMITTING: + buffer.append("The indexer is committing"); + break; + case Status.STATUS_MARKED_ROLLBACK: + buffer.append("The indexer is marked for rollback"); + break; + case Status.STATUS_PREPARED: + buffer.append("The indexer is prepared to commit"); + break; + case Status.STATUS_PREPARING: + buffer.append("The indexer is preparing to commit"); + break; + case Status.STATUS_ROLLEDBACK: + buffer.append("The indexer has been rolled back"); + break; + case Status.STATUS_ROLLING_BACK: + buffer.append("The indexer is rolling back"); + break; + case Status.STATUS_UNKNOWN: + buffer.append("The indexer is in an unknown state"); + break; + default: + break; + } + return buffer.toString(); + } + + /* + * Indexer Implementation + */ + + public void createNode(ChildAssociationRef relationshipRef) throws LuceneIndexException + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Create node " + relationshipRef.getChildRef()); + } + checkAbleToDoWork(false, true); + try + { + NodeRef childRef = relationshipRef.getChildRef(); + // If we have the root node we delete all other root nodes first + if ((relationshipRef.getParentRef() == null) + && childRef.equals(nodeService.getRootNode(childRef.getStoreRef()))) + { + addRootNodesToDeletionList(); + s_logger.warn("Detected root node addition: deleting all nodes from the index"); + } + index(childRef); + } + catch (LuceneIndexException e) + { + setRollbackOnly(); + throw new LuceneIndexException("Create node failed", e); + } + } + + private void addRootNodesToDeletionList() + { + IndexReader mainReader = null; + try + { + try + { + mainReader = getReader(); + TermDocs td = mainReader.termDocs(new Term("ISROOT", "T")); + while (td.next()) + { + int doc = td.doc(); + Document document = mainReader.document(doc); + String id = document.get("ID"); + NodeRef ref = new NodeRef(id); + deleteImpl(ref, false, true, mainReader); + } + } + catch (IOException e) + { + throw new LuceneIndexException("Failed to delete all primary nodes", e); + } + } + finally + { + if (mainReader != null) + { + try + { + mainReader.close(); + } + catch (IOException e) + { + throw new LuceneIndexException("Filed to close main reader", e); + } + } + } + } + + public void updateNode(NodeRef nodeRef) throws LuceneIndexException + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Update node " + nodeRef); + } + checkAbleToDoWork(false, true); + try + { + reindex(nodeRef, false); + } + catch (LuceneIndexException e) + { + setRollbackOnly(); + throw new LuceneIndexException("Update node failed", e); + } + } + + public void deleteNode(ChildAssociationRef relationshipRef) throws LuceneIndexException + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Delete node " + relationshipRef.getChildRef()); + } + checkAbleToDoWork(false, true); + try + { + delete(relationshipRef.getChildRef()); + } + catch (LuceneIndexException e) + { + setRollbackOnly(); + throw new LuceneIndexException("Delete node failed", e); + } + } + + public void createChildRelationship(ChildAssociationRef relationshipRef) throws LuceneIndexException + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Create child " + relationshipRef); + } + checkAbleToDoWork(false, true); + try + { + // TODO: Optimise + // reindex(relationshipRef.getParentRef()); + reindex(relationshipRef.getChildRef(), true); + } + catch (LuceneIndexException e) + { + setRollbackOnly(); + throw new LuceneIndexException("Failed to create child relationship", e); + } + } + + public void updateChildRelationship(ChildAssociationRef relationshipBeforeRef, + ChildAssociationRef relationshipAfterRef) throws LuceneIndexException + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Update child " + relationshipBeforeRef + " to " + relationshipAfterRef); + } + checkAbleToDoWork(false, true); + try + { + // TODO: Optimise + if (relationshipBeforeRef.getParentRef() != null) + { + // reindex(relationshipBeforeRef.getParentRef()); + } + reindex(relationshipBeforeRef.getChildRef(), true); + } + catch (LuceneIndexException e) + { + setRollbackOnly(); + throw new LuceneIndexException("Failed to update child relationship", e); + } + } + + public void deleteChildRelationship(ChildAssociationRef relationshipRef) throws LuceneIndexException + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Delete child " + relationshipRef); + } + checkAbleToDoWork(false, true); + try + { + // TODO: Optimise + if (relationshipRef.getParentRef() != null) + { + // reindex(relationshipRef.getParentRef()); + } + reindex(relationshipRef.getChildRef(), true); + } + catch (LuceneIndexException e) + { + setRollbackOnly(); + throw new LuceneIndexException("Failed to delete child relationship", e); + } + } + + /** + * Generate an indexer + * + * @param storeRef + * @param deltaId + * @return + */ + public static LuceneIndexerImpl2 getUpdateIndexer(StoreRef storeRef, String deltaId, LuceneConfig config) + throws LuceneIndexException + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Creating indexer"); + } + LuceneIndexerImpl2 indexer = new LuceneIndexerImpl2(); + indexer.setLuceneConfig(config); + indexer.initialise(storeRef, deltaId, false, true); + return indexer; + } + + /* + * Transactional support Used by the resource manager for indexers. + */ + + /** + * Commit this index + */ + + public void commit() throws LuceneIndexException + { + switch (status) + { + case Status.STATUS_COMMITTING: + throw new LuceneIndexException("Unable to commit: Transaction is committing"); + case Status.STATUS_COMMITTED: + throw new LuceneIndexException("Unable to commit: Transaction is commited "); + case Status.STATUS_ROLLING_BACK: + throw new LuceneIndexException("Unable to commit: Transaction is rolling back"); + case Status.STATUS_ROLLEDBACK: + throw new LuceneIndexException("Unable to commit: Transaction is aleady rolled back"); + case Status.STATUS_MARKED_ROLLBACK: + throw new LuceneIndexException("Unable to commit: Transaction is marked for roll back"); + case Status.STATUS_PREPARING: + throw new LuceneIndexException("Unable to commit: Transaction is preparing"); + case Status.STATUS_ACTIVE: + // special case - commit from active + prepare(); + // drop through to do the commit; + default: + if (status != Status.STATUS_PREPARED) + { + throw new LuceneIndexException("Index must be prepared to commit"); + } + status = Status.STATUS_COMMITTING; + try + { + setStatus(TransactionStatus.COMMITTING); + if (isModified()) + { + if (isFTSUpdate.booleanValue()) + { + doFTSIndexCommit(); + // FTS does not trigger indexing request + } + else + { + // Build the deletion terms + // Set terms = new LinkedHashSet(); + // for (NodeRef nodeRef : deletions) + // { + // terms.add(new Term("ID", nodeRef.toString())); + // } + // Merge + // mergeDeltaIntoMain(terms); + setInfo(docs, deletions, false); + luceneFullTextSearchIndexer.requiresIndex(store); + } + } + status = Status.STATUS_COMMITTED; + if (callBack != null) + { + callBack.indexCompleted(store, remainingCount, null); + } + setStatus(TransactionStatus.COMMITTED); + } + catch (IOException e) + { + // If anything goes wrong we try and do a roll back + rollback(); + throw new LuceneIndexException("Commit failed", e); + } + catch (LuceneIndexException e) + { + // If anything goes wrong we try and do a roll back + rollback(); + throw new LuceneIndexException("Commit failed", e); + } + finally + { + // Make sure we tidy up + // deleteDelta(); + } + break; + } + } + + private void doFTSIndexCommit() throws LuceneIndexException + { + IndexReader mainReader = null; + IndexReader deltaReader = null; + IndexSearcher mainSearcher = null; + IndexSearcher deltaSearcher = null; + + try + { + try + { + mainReader = getReader(); + deltaReader = getDeltaReader(); + mainSearcher = new IndexSearcher(mainReader); + deltaSearcher = new IndexSearcher(deltaReader); + + for (Helper helper : toFTSIndex) + { + //BooleanQuery query = new BooleanQuery(); + //query.add(new TermQuery(new Term("ID", helper.nodeRef.toString())), true, false); + //query.add(new TermQuery(new Term("TX", helper.tx)), true, false); + //query.add(new TermQuery(new Term("ISNODE", "T")), false, false); + + deletions.add(helper.nodeRef); + + +// try +// { +// Hits hits = mainSearcher.search(query); +// if (hits.length() > 0) +// { +// for (int i = 0; i < hits.length(); i++) +// { +// mainReader.delete(hits.id(i)); +// } +// } +// else +// { +// hits = deltaSearcher.search(query); +// for (int i = 0; i < hits.length(); i++) +// { +// deltaReader.delete(hits.id(i)); +// } +// } +// } +// catch (IOException e) +// { +// throw new LuceneIndexException("Failed to delete an FTS update from the original index", e); +// } + } + + } + finally + { + if (deltaSearcher != null) + { + try + { + deltaSearcher.close(); + } + catch (IOException e) + { + s_logger.warn("Failed to close delta searcher", e); + } + } + if (mainSearcher != null) + { + try + { + mainSearcher.close(); + } + catch (IOException e) + { + s_logger.warn("Failed to close main searcher", e); + } + } + try + { + closeDeltaReader(); + } + catch (LuceneIndexException e) + { + s_logger.warn("Failed to close delta reader", e); + } + if (mainReader != null) + { + try + { + mainReader.close(); + } + catch (IOException e) + { + s_logger.warn("Failed to close main reader", e); + } + } + } + + setInfo(docs, deletions, true); + // mergeDeltaIntoMain(new LinkedHashSet()); + } + catch (IOException e) + { + // If anything goes wrong we try and do a roll back + rollback(); + throw new LuceneIndexException("Commit failed", e); + } + catch (LuceneIndexException e) + { + // If anything goes wrong we try and do a roll back + rollback(); + throw new LuceneIndexException("Commit failed", e); + } + finally + { + // Make sure we tidy up + // deleteDelta(); + } + + } + + /** + * Prepare to commit + * + * At the moment this makes sure we have all the locks + * + * TODO: This is not doing proper serialisation against the index as would a data base transaction. + * + * @return + */ + public int prepare() throws LuceneIndexException + { + + switch (status) + { + case Status.STATUS_COMMITTING: + throw new IndexerException("Unable to prepare: Transaction is committing"); + case Status.STATUS_COMMITTED: + throw new IndexerException("Unable to prepare: Transaction is commited "); + case Status.STATUS_ROLLING_BACK: + throw new IndexerException("Unable to prepare: Transaction is rolling back"); + case Status.STATUS_ROLLEDBACK: + throw new IndexerException("Unable to prepare: Transaction is aleady rolled back"); + case Status.STATUS_MARKED_ROLLBACK: + throw new IndexerException("Unable to prepare: Transaction is marked for roll back"); + case Status.STATUS_PREPARING: + throw new IndexerException("Unable to prepare: Transaction is already preparing"); + case Status.STATUS_PREPARED: + throw new IndexerException("Unable to prepare: Transaction is already prepared"); + default: + status = Status.STATUS_PREPARING; + try + { + setStatus(TransactionStatus.PREPARING); + if (isModified()) + { + saveDelta(); + flushPending(); + // prepareToMergeIntoMain(); + } + status = Status.STATUS_PREPARED; + setStatus(TransactionStatus.PREPARED); + return isModified ? XAResource.XA_OK : XAResource.XA_RDONLY; + } + catch (IOException e) + { + // If anything goes wrong we try and do a roll back + rollback(); + throw new LuceneIndexException("Commit failed", e); + } + catch (LuceneIndexException e) + { + setRollbackOnly(); + throw new LuceneIndexException("Index failed to prepare", e); + } + } + } + + /** + * Has this index been modified? + * + * @return + */ + public boolean isModified() + { + return isModified; + } + + /** + * Return the javax.transaction.Status integer status code + * + * @return + */ + public int getStatus() + { + return status; + } + + /** + * Roll back the index changes (this just means they are never added) + * + */ + + public void rollback() throws LuceneIndexException + { + switch (status) + { + + case Status.STATUS_COMMITTED: + throw new IndexerException("Unable to roll back: Transaction is committed "); + case Status.STATUS_ROLLING_BACK: + throw new IndexerException("Unable to roll back: Transaction is rolling back"); + case Status.STATUS_ROLLEDBACK: + throw new IndexerException("Unable to roll back: Transaction is already rolled back"); + case Status.STATUS_COMMITTING: + // Can roll back during commit + default: + status = Status.STATUS_ROLLING_BACK; + // if (isModified()) + // { + // deleteDelta(); + // } + try + { + setStatus(TransactionStatus.ROLLINGBACK); + setStatus(TransactionStatus.ROLLEDBACK); + } + catch (IOException e) + { + throw new LuceneIndexException("roolback failed ", e); + } + + if (callBack != null) + { + callBack.indexCompleted(store, 0, null); + } + break; + } + } + + /** + * Mark this index for roll back only. This action can not be reversed. It will reject all other work and only allow roll back. + * + */ + + public void setRollbackOnly() + { + switch (status) + { + case Status.STATUS_COMMITTING: + throw new IndexerException("Unable to mark for rollback: Transaction is committing"); + case Status.STATUS_COMMITTED: + throw new IndexerException("Unable to mark for rollback: Transaction is committed"); + default: + status = Status.STATUS_MARKED_ROLLBACK; + break; + } + } + + /* + * Implementation + */ + + private void index(NodeRef nodeRef) throws LuceneIndexException + { + addCommand(new Command(nodeRef, Action.INDEX)); + } + + private void reindex(NodeRef nodeRef, boolean cascadeReindexDirectories) throws LuceneIndexException + { + addCommand(new Command(nodeRef, cascadeReindexDirectories ? Action.CASCADEREINDEX : Action.REINDEX)); + } + + private void delete(NodeRef nodeRef) throws LuceneIndexException + { + addCommand(new Command(nodeRef, Action.DELETE)); + } + + private void addCommand(Command command) + { + if (commandList.size() > 0) + { + Command last = commandList.get(commandList.size() - 1); + if ((last.action == command.action) && (last.nodeRef.equals(command.nodeRef))) + { + return; + } + } + purgeCommandList(command); + commandList.add(command); + + if (commandList.size() > getLuceneConfig().getIndexerBatchSize()) + { + flushPending(); + } + } + + private void purgeCommandList(Command command) + { + if (command.action == Action.DELETE) + { + removeFromCommandList(command, false); + } + else if (command.action == Action.REINDEX) + { + removeFromCommandList(command, true); + } + else if (command.action == Action.INDEX) + { + removeFromCommandList(command, true); + } + else if (command.action == Action.CASCADEREINDEX) + { + removeFromCommandList(command, true); + } + } + + private void removeFromCommandList(Command command, boolean matchExact) + { + for (ListIterator it = commandList.listIterator(commandList.size()); it.hasPrevious(); /**/) + { + Command current = it.previous(); + if (matchExact) + { + if ((current.action == command.action) && (current.nodeRef.equals(command.nodeRef))) + { + it.remove(); + return; + } + } + else + { + if (current.nodeRef.equals(command.nodeRef)) + { + it.remove(); + } + } + } + } + + public void flushPending() throws LuceneIndexException + { + IndexReader mainReader = null; + try + { + mainReader = getReader(); + Set forIndex = new LinkedHashSet(); + + for (Command command : commandList) + { + if (command.action == Action.INDEX) + { + // Indexing just requires the node to be added to the list + forIndex.add(command.nodeRef); + } + else if (command.action == Action.REINDEX) + { + // Reindex is a delete and then and index + Set set = deleteImpl(command.nodeRef, true, false, mainReader); + + // Deleting any pending index actions + // - make sure we only do at most one index + forIndex.removeAll(set); + // Add the nodes for index + forIndex.addAll(set); + } + else if (command.action == Action.CASCADEREINDEX) + { + // Reindex is a delete and then and index + Set set = deleteImpl(command.nodeRef, true, true, mainReader); + + // Deleting any pending index actions + // - make sure we only do at most one index + forIndex.removeAll(set); + // Add the nodes for index + forIndex.addAll(set); + } + else if (command.action == Action.DELETE) + { + // Delete the nodes + Set set = deleteImpl(command.nodeRef, false, true, mainReader); + // Remove any pending indexes + forIndex.removeAll(set); + } + } + commandList.clear(); + indexImpl(forIndex, false); + docs = getDeltaWriter().docCount(); + } + catch (IOException e) + { + // If anything goes wrong we try and do a roll back + throw new LuceneIndexException("Failed to flush index", e); + } + finally + { + if (mainReader != null) + { + try + { + mainReader.close(); + } + catch (IOException e) + { + throw new LuceneIndexException("Filed to close main reader", e); + } + } + // Make sure deletes are sent + try + { + closeDeltaReader(); + } + catch (IOException e) + { + + } + // Make sure writes and updates are sent. + try + { + closeDeltaWriter(); + } + catch (IOException e) + { + + } + } + } + + private Set deleteImpl(NodeRef nodeRef, boolean forReindex, boolean cascade, IndexReader mainReader) + throws LuceneIndexException, IOException + + { + // startTimer(); + getDeltaReader(); + // outputTime("Delete "+nodeRef+" size = "+getDeltaWriter().docCount()); + Set refs = new LinkedHashSet(); + + refs.addAll(deleteContainerAndBelow(nodeRef, getDeltaReader(), true, cascade)); + refs.addAll(deleteContainerAndBelow(nodeRef, mainReader, false, cascade)); + + if (!forReindex) + { + Set leafrefs = new LinkedHashSet(); + + leafrefs.addAll(deletePrimary(refs, getDeltaReader(), true)); + leafrefs.addAll(deletePrimary(refs, mainReader, false)); + + leafrefs.addAll(deleteReference(refs, getDeltaReader(), true)); + leafrefs.addAll(deleteReference(refs, mainReader, false)); + + refs.addAll(leafrefs); + } + + deletions.addAll(refs); + + return refs; + + } + + private Set deletePrimary(Collection nodeRefs, IndexReader reader, boolean delete) + throws LuceneIndexException + { + + Set refs = new LinkedHashSet(); + + for (NodeRef nodeRef : nodeRefs) + { + + try + { + TermDocs td = reader.termDocs(new Term("PRIMARYPARENT", nodeRef.toString())); + while (td.next()) + { + int doc = td.doc(); + Document document = reader.document(doc); + String id = document.get("ID"); + NodeRef ref = new NodeRef(id); + refs.add(ref); + if (delete) + { + reader.delete(doc); + } + } + } + catch (IOException e) + { + throw new LuceneIndexException("Failed to delete node by primary parent for " + nodeRef.toString(), e); + } + } + + return refs; + + } + + private Set deleteReference(Collection nodeRefs, IndexReader reader, boolean delete) + throws LuceneIndexException + { + + Set refs = new LinkedHashSet(); + + for (NodeRef nodeRef : nodeRefs) + { + + try + { + TermDocs td = reader.termDocs(new Term("PARENT", nodeRef.toString())); + while (td.next()) + { + int doc = td.doc(); + Document document = reader.document(doc); + String id = document.get("ID"); + NodeRef ref = new NodeRef(id); + refs.add(ref); + if (delete) + { + reader.delete(doc); + } + } + } + catch (IOException e) + { + throw new LuceneIndexException("Failed to delete node by parent for " + nodeRef.toString(), e); + } + } + + return refs; + + } + + private Set deleteContainerAndBelow(NodeRef nodeRef, IndexReader reader, boolean delete, boolean cascade) + throws LuceneIndexException + { + Set refs = new LinkedHashSet(); + + try + { + if (delete) + { + reader.delete(new Term("ID", nodeRef.toString())); + } + refs.add(nodeRef); + if (cascade) + { + TermDocs td = reader.termDocs(new Term("ANCESTOR", nodeRef.toString())); + while (td.next()) + { + int doc = td.doc(); + Document document = reader.document(doc); + String id = document.get("ID"); + NodeRef ref = new NodeRef(id); + refs.add(ref); + if (delete) + { + reader.delete(doc); + } + } + } + } + catch (IOException e) + { + throw new LuceneIndexException("Failed to delete container and below for " + nodeRef.toString(), e); + } + return refs; + } + + private void indexImpl(Set nodeRefs, boolean isNew) throws LuceneIndexException, IOException + { + for (NodeRef ref : nodeRefs) + { + indexImpl(ref, isNew); + } + } + + private void indexImpl(NodeRef nodeRef, boolean isNew) throws LuceneIndexException, IOException + { + IndexWriter writer = getDeltaWriter(); + + // avoid attempting to index nodes that don't exist + + try + { + List docs = createDocuments(nodeRef, isNew, false, true); + for (Document doc : docs) + { + try + { + writer.addDocument(doc /* + * TODO: Select the language based analyser + */); + } + catch (IOException e) + { + throw new LuceneIndexException("Failed to add document to index", e); + } + } + } + catch (InvalidNodeRefException e) + { + // The node does not exist + return; + } + + } + + static class Counter + { + int countInParent = 0; + + int count = -1; + + int getCountInParent() + { + return countInParent; + } + + int getRepeat() + { + return (count / countInParent) + 1; + } + + void incrementParentCount() + { + countInParent++; + } + + void increment() + { + count++; + } + + } + + private class Pair + { + private F first; + + private S second; + + public Pair(F first, S second) + { + this.first = first; + this.second = second; + } + + public F getFirst() + { + return first; + } + + public S getSecond() + { + return second; + } + } + + private List createDocuments(NodeRef nodeRef, boolean isNew, boolean indexAllProperties, + boolean includeDirectoryDocuments) + { + Map nodeCounts = getNodeCounts(nodeRef); + List docs = new ArrayList(); + ChildAssociationRef qNameRef = null; + Map properties = nodeService.getProperties(nodeRef); + NodeRef.Status nodeStatus = nodeService.getNodeStatus(nodeRef); + + Collection directPaths = nodeService.getPaths(nodeRef, false); + Collection> categoryPaths = getCategoryPaths(nodeRef, properties); + Collection> paths = new ArrayList>(directPaths.size() + + categoryPaths.size()); + for (Path path : directPaths) + { + paths.add(new Pair(path, null)); + } + paths.addAll(categoryPaths); + + Document xdoc = new Document(); + xdoc.add(new Field("ID", nodeRef.toString(), true, true, false)); + xdoc.add(new Field("TX", nodeStatus.getChangeTxnId(), true, true, false)); + boolean isAtomic = true; + for (QName propertyName : properties.keySet()) + { + Serializable value = properties.get(propertyName); + if (indexAllProperties) + { + indexProperty(nodeRef, propertyName, value, xdoc, false); + } + else + { + isAtomic &= indexProperty(nodeRef, propertyName, value, xdoc, true); + } + } + + boolean isRoot = nodeRef.equals(nodeService.getRootNode(nodeRef.getStoreRef())); + + StringBuilder qNameBuffer = new StringBuilder(64); + + for (Iterator> it = paths.iterator(); it.hasNext(); /**/) + { + Pair pair = it.next(); + // Lucene flags in order are: Stored, indexed, tokenised + + qNameRef = getLastRefOrNull(pair.getFirst()); + + String pathString = pair.getFirst().toString(); + if ((pathString.length() > 0) && (pathString.charAt(0) == '/')) + { + pathString = pathString.substring(1); + } + + if (isRoot) + { + // Root node + } + else if (pair.getFirst().size() == 1) + { + // Pseudo root node ignore + } + else + // not a root node + { + Counter counter = nodeCounts.get(qNameRef); + // If we have something in a container with root aspect we will + // not find it + + if ((counter == null) || (counter.getRepeat() < counter.getCountInParent())) + { + if ((qNameRef != null) && (qNameRef.getParentRef() != null) && (qNameRef.getQName() != null)) + { + if (qNameBuffer.length() > 0) + { + qNameBuffer.append(";/"); + } + qNameBuffer.append(ISO9075.getXPathName(qNameRef.getQName())); + xdoc.add(new Field("PARENT", qNameRef.getParentRef().toString(), true, true, false)); + xdoc.add(new Field("ASSOCTYPEQNAME", ISO9075.getXPathName(qNameRef.getTypeQName()), true, + false, false)); + xdoc.add(new Field("LINKASPECT", (pair.getSecond() == null) ? "" : ISO9075.getXPathName(pair + .getSecond()), true, true, false)); + } + } + + if (counter != null) + { + counter.increment(); + } + + // TODO: DC: Should this also include aspect child definitions? + QName nodeTypeRef = nodeService.getType(nodeRef); + TypeDefinition nodeTypeDef = getDictionaryService().getType(nodeTypeRef); + // check for child associations + + if (includeDirectoryDocuments) + { + if (nodeTypeDef.getChildAssociations().size() > 0) + { + if (directPaths.contains(pair.getFirst())) + { + Document directoryEntry = new Document(); + directoryEntry.add(new Field("ID", nodeRef.toString(), true, true, false)); + directoryEntry.add(new Field("PATH", pathString, true, true, true)); + for (NodeRef parent : getParents(pair.getFirst())) + { + directoryEntry.add(new Field("ANCESTOR", parent.toString(), false, true, false)); + } + directoryEntry.add(new Field("ISCONTAINER", "T", true, true, false)); + + if (isCategory(getDictionaryService().getType(nodeService.getType(nodeRef)))) + { + directoryEntry.add(new Field("ISCATEGORY", "T", true, true, false)); + } + + docs.add(directoryEntry); + } + } + } + } + } + + // Root Node + if (isRoot) + { + // TODO: Does the root element have a QName? + xdoc.add(new Field("ISCONTAINER", "T", true, true, false)); + xdoc.add(new Field("PATH", "", true, true, true)); + xdoc.add(new Field("QNAME", "", true, true, true)); + xdoc.add(new Field("ISROOT", "T", false, true, false)); + xdoc.add(new Field("PRIMARYASSOCTYPEQNAME", ISO9075.getXPathName(ContentModel.ASSOC_CHILDREN), true, false, + false)); + xdoc.add(new Field("ISNODE", "T", false, true, false)); + docs.add(xdoc); + + } + else + // not a root node + { + xdoc.add(new Field("QNAME", qNameBuffer.toString(), true, true, true)); + // xdoc.add(new Field("PARENT", parentBuffer.toString(), true, true, + // true)); + + ChildAssociationRef primary = nodeService.getPrimaryParent(nodeRef); + xdoc.add(new Field("PRIMARYPARENT", primary.getParentRef().toString(), true, true, false)); + xdoc.add(new Field("PRIMARYASSOCTYPEQNAME", ISO9075.getXPathName(primary.getTypeQName()), true, false, + false)); + QName typeQName = nodeService.getType(nodeRef); + + xdoc.add(new Field("TYPE", ISO9075.getXPathName(typeQName), true, true, false)); + for (QName classRef : nodeService.getAspects(nodeRef)) + { + xdoc.add(new Field("ASPECT", ISO9075.getXPathName(classRef), true, true, false)); + } + + xdoc.add(new Field("ISROOT", "F", false, true, false)); + xdoc.add(new Field("ISNODE", "T", false, true, false)); + if (isAtomic || indexAllProperties) + { + xdoc.add(new Field("FTSSTATUS", "Clean", false, true, false)); + } + else + { + if (isNew) + { + xdoc.add(new Field("FTSSTATUS", "New", false, true, false)); + } + else + { + xdoc.add(new Field("FTSSTATUS", "Dirty", false, true, false)); + } + } + + // { + docs.add(xdoc); + // } + } + + return docs; + } + + private ArrayList getParents(Path path) + { + ArrayList parentsInDepthOrderStartingWithSelf = new ArrayList(8); + for (Iterator elit = path.iterator(); elit.hasNext(); /**/) + { + Path.Element element = elit.next(); + if (!(element instanceof Path.ChildAssocElement)) + { + throw new IndexerException("Confused path: " + path); + } + Path.ChildAssocElement cae = (Path.ChildAssocElement) element; + parentsInDepthOrderStartingWithSelf.add(0, cae.getRef().getChildRef()); + + } + return parentsInDepthOrderStartingWithSelf; + } + + private ChildAssociationRef getLastRefOrNull(Path path) + { + if (path.last() instanceof Path.ChildAssocElement) + { + Path.ChildAssocElement cae = (Path.ChildAssocElement) path.last(); + return cae.getRef(); + } + else + { + return null; + } + } + + /** + * @param indexAtomicPropertiesOnly + * true to ignore all properties that must be indexed non-atomically + * @return Returns true if the property was indexed atomically, or false if it should be done asynchronously + */ + private boolean indexProperty(NodeRef nodeRef, QName propertyName, Serializable value, Document doc, + boolean indexAtomicPropertiesOnly) + { + String attributeName = "@" + + QName.createQName(propertyName.getNamespaceURI(), ISO9075.encode(propertyName.getLocalName())); + + boolean store = true; + boolean index = true; + boolean tokenise = true; + boolean atomic = true; + boolean isContent = false; + + PropertyDefinition propertyDef = getDictionaryService().getProperty(propertyName); + if (propertyDef != null) + { + index = propertyDef.isIndexed(); + store = propertyDef.isStoredInIndex(); + tokenise = propertyDef.isTokenisedInIndex(); + atomic = propertyDef.isIndexedAtomically(); + isContent = propertyDef.getDataType().getName().equals(DataTypeDefinition.CONTENT); + } + if (value == null) + { + // the value is null + return true; + } + else if (indexAtomicPropertiesOnly && !atomic) + { + // we are only doing atomic properties and the property is definitely non-atomic + return false; + } + + if (!indexAtomicPropertiesOnly) + { + doc.removeFields(propertyName.toString()); + } + boolean wereAllAtomic = true; + // convert value to String + for (String strValue : DefaultTypeConverter.INSTANCE.getCollection(String.class, value)) + { + if (strValue == null) + { + // nothing to index + continue; + } + // String strValue = ValueConverter.convert(String.class, value); + // TODO: Need to add with the correct language based analyser + + if (isContent) + { + ContentData contentData = DefaultTypeConverter.INSTANCE.convert(ContentData.class, value); + if (!index || contentData.getMimetype() == null) + { + // no mimetype or property not indexed + continue; + } + // store mimetype in index - even if content does not index it is useful + doc.add(new Field(attributeName + ".mimetype", contentData.getMimetype(), false, true, false)); + + ContentReader reader = contentService.getReader(nodeRef, propertyName); + if (reader != null && reader.exists()) + { + boolean readerReady = true; + // transform if necessary (it is not a UTF-8 text document) + if (!EqualsHelper.nullSafeEquals(reader.getMimetype(), MimetypeMap.MIMETYPE_TEXT_PLAIN) + || !EqualsHelper.nullSafeEquals(reader.getEncoding(), "UTF-8")) + { + // get the transformer + ContentTransformer transformer = contentService.getTransformer(reader.getMimetype(), + MimetypeMap.MIMETYPE_TEXT_PLAIN); + // is this transformer good enough? + if (transformer == null) + { + // log it + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Not indexed: No transformation: \n" + + " source: " + reader + "\n" + " target: " + + MimetypeMap.MIMETYPE_TEXT_PLAIN); + } + // don't index from the reader + readerReady = false; + // not indexed: no transformation + doc.add(Field.Text("TEXT", NOT_INDEXED_NO_TRANSFORMATION)); + doc.add(Field.Text(attributeName, NOT_INDEXED_NO_TRANSFORMATION)); + } + else if (indexAtomicPropertiesOnly + && transformer.getTransformationTime() > maxAtomicTransformationTime) + { + // only indexing atomic properties + // indexing will take too long, so push it to the background + wereAllAtomic = false; + } + else + { + // We have a transformer that is fast enough + ContentWriter writer = contentService.getTempWriter(); + writer.setMimetype(MimetypeMap.MIMETYPE_TEXT_PLAIN); + // this is what the analyzers expect on the stream + writer.setEncoding("UTF-8"); + try + { + + transformer.transform(reader, writer); + // point the reader to the new-written content + reader = writer.getReader(); + } + catch (ContentIOException e) + { + // log it + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Not indexed: Transformation failed", e); + } + // don't index from the reader + readerReady = false; + // not indexed: transformation + // failed + doc.add(Field.Text("TEXT", NOT_INDEXED_TRANSFORMATION_FAILED)); + doc.add(Field.Text(attributeName, NOT_INDEXED_TRANSFORMATION_FAILED)); + } + } + } + // add the text field using the stream from the + // reader, but only if the reader is valid + if (readerReady) + { + InputStreamReader isr = null; + InputStream ris = reader.getContentInputStream(); + try + { + isr = new InputStreamReader(ris, "UTF-8"); + } + catch (UnsupportedEncodingException e) + { + isr = new InputStreamReader(ris); + } + doc.add(Field.Text("TEXT", isr)); + + ris = reader.getReader().getContentInputStream(); + try + { + isr = new InputStreamReader(ris, "UTF-8"); + } + catch (UnsupportedEncodingException e) + { + isr = new InputStreamReader(ris); + } + + doc.add(Field.Text("@" + + QName.createQName(propertyName.getNamespaceURI(), ISO9075.encode(propertyName + .getLocalName())), isr)); + } + } + else + // URL not present (null reader) or no content at the URL (file missing) + { + // log it + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Not indexed: Content Missing \n" + + " node: " + nodeRef + "\n" + " reader: " + reader + "\n" + " content exists: " + + (reader == null ? " --- " : Boolean.toString(reader.exists()))); + } + // not indexed: content missing + doc.add(Field.Text("TEXT", NOT_INDEXED_CONTENT_MISSING)); + doc.add(Field.Text(attributeName, NOT_INDEXED_CONTENT_MISSING)); + } + } + else + { + doc.add(new Field(attributeName, strValue, store, index, tokenise)); + } + } + + return wereAllAtomic; + } + + private Map getNodeCounts(NodeRef nodeRef) + { + Map nodeCounts = new HashMap(5); + List parentAssocs = nodeService.getParentAssocs(nodeRef); + // count the number of times the association is duplicated + for (ChildAssociationRef assoc : parentAssocs) + { + Counter counter = nodeCounts.get(assoc); + if (counter == null) + { + counter = new Counter(); + nodeCounts.put(assoc, counter); + } + counter.incrementParentCount(); + + } + return nodeCounts; + } + + private Collection> getCategoryPaths(NodeRef nodeRef, Map properties) + { + ArrayList> categoryPaths = new ArrayList>(); + Set aspects = nodeService.getAspects(nodeRef); + + for (QName classRef : aspects) + { + AspectDefinition aspDef = getDictionaryService().getAspect(classRef); + if (isCategorised(aspDef)) + { + LinkedList> aspectPaths = new LinkedList>(); + for (PropertyDefinition propDef : aspDef.getProperties().values()) + { + if (propDef.getDataType().getName().equals(DataTypeDefinition.CATEGORY)) + { + for (NodeRef catRef : DefaultTypeConverter.INSTANCE.getCollection(NodeRef.class, properties + .get(propDef.getName()))) + { + if (catRef != null) + { + for (Path path : nodeService.getPaths(catRef, false)) + { + if ((path.size() > 1) && (path.get(1) instanceof Path.ChildAssocElement)) + { + Path.ChildAssocElement cae = (Path.ChildAssocElement) path.get(1); + boolean isFakeRoot = true; + for (ChildAssociationRef car : nodeService.getParentAssocs(cae.getRef() + .getChildRef())) + { + if (cae.getRef().equals(car)) + { + isFakeRoot = false; + break; + } + } + if (isFakeRoot) + { + if (path.toString().indexOf(aspDef.getName().toString()) != -1) + { + aspectPaths.add(new Pair(path, aspDef.getName())); + } + } + } + } + + } + } + } + } + categoryPaths.addAll(aspectPaths); + } + } + // Add member final element + for (Pair pair : categoryPaths) + { + if (pair.getFirst().last() instanceof Path.ChildAssocElement) + { + Path.ChildAssocElement cae = (Path.ChildAssocElement) pair.getFirst().last(); + ChildAssociationRef assocRef = cae.getRef(); + pair.getFirst().append( + new Path.ChildAssocElement(new ChildAssociationRef(assocRef.getTypeQName(), assocRef + .getChildRef(), QName.createQName("member"), nodeRef))); + } + } + + return categoryPaths; + } + + private boolean isCategorised(AspectDefinition aspDef) + { + AspectDefinition current = aspDef; + while (current != null) + { + if (current.getName().equals(ContentModel.ASPECT_CLASSIFIABLE)) + { + return true; + } + else + { + QName parentName = current.getParentName(); + if (parentName == null) + { + break; + } + current = getDictionaryService().getAspect(parentName); + } + } + return false; + } + + private boolean isCategory(TypeDefinition typeDef) + { + if (typeDef == null) + { + return false; + } + TypeDefinition current = typeDef; + while (current != null) + { + if (current.getName().equals(ContentModel.TYPE_CATEGORY)) + { + return true; + } + else + { + QName parentName = current.getParentName(); + if (parentName == null) + { + break; + } + current = getDictionaryService().getType(parentName); + } + } + return false; + } + + public void updateFullTextSearch(int size) throws LuceneIndexException + { + checkAbleToDoWork(true, false); + // if (!mainIndexExists()) + // { + // remainingCount = size; + // return; + // } + try + { + NodeRef lastId = null; + + toFTSIndex = new ArrayList(size); + BooleanQuery booleanQuery = new BooleanQuery(); + booleanQuery.add(new TermQuery(new Term("FTSSTATUS", "Dirty")), false, false); + booleanQuery.add(new TermQuery(new Term("FTSSTATUS", "New")), false, false); + + int count = 0; + Searcher searcher = null; + LuceneResultSet results = null; + try + { + searcher = getSearcher(null); + // commit on another thread - appears like there is no index ...try later + if (searcher == null) + { + remainingCount = size; + return; + } + Hits hits; + try + { + hits = searcher.search(booleanQuery); + } + catch (IOException e) + { + throw new LuceneIndexException( + "Failed to execute query to find content which needs updating in the index", e); + } + results = new LuceneResultSet(hits, searcher, nodeService, null, new SearchParameters()); + + for (ResultSetRow row : results) + { + LuceneResultSetRow lrow = (LuceneResultSetRow) row; + Helper helper = new Helper(lrow.getNodeRef(), lrow.getDocument().getField("TX").stringValue()); + toFTSIndex.add(helper); + if (++count >= size) + { + break; + } + } + count = results.length(); + } + finally + { + if (results != null) + { + results.close(); // closes the searcher + } + else if (searcher != null) + { + try + { + searcher.close(); + } + catch (IOException e) + { + throw new LuceneIndexException("Failed to close searcher", e); + } + } + } + + if (toFTSIndex.size() > 0) + { + checkAbleToDoWork(true, true); + + IndexWriter writer = null; + try + { + writer = getDeltaWriter(); + for (Helper helper : toFTSIndex) + { + // Document document = helper.document; + NodeRef ref = helper.nodeRef; + // bypass nodes that have disappeared + if (!nodeService.exists(ref)) + { + continue; + } + + List docs = createDocuments(ref, false, true, false); + for (Document doc : docs) + { + try + { + writer.addDocument(doc /* + * TODO: Select the language based analyser + */); + } + catch (IOException e) + { + throw new LuceneIndexException("Failed to add document while updating fts index", e); + } + } + + // Need to do all the current id in the TX - should all + // be + // together so skip until id changes + if (writer.docCount() > size) + { + if (lastId == null) + { + lastId = ref; + } + if (!lastId.equals(ref)) + { + break; + } + } + } + + remainingCount = count - writer.docCount(); + } + catch (LuceneIndexException e) + { + if (writer != null) + { + closeDeltaWriter(); + } + } + } + } + catch (IOException e) + { + setRollbackOnly(); + throw new LuceneIndexException("Failed FTS update", e); + } + catch (LuceneIndexException e) + { + setRollbackOnly(); + throw new LuceneIndexException("Failed FTS update", e); + } + } + + public void registerCallBack(FTSIndexerAware callBack) + { + this.callBack = callBack; + } + + private static class Helper + { + NodeRef nodeRef; + + String tx; + + Helper(NodeRef nodeRef, String tx) + { + this.nodeRef = nodeRef; + this.tx = tx; + } + } + + private static class Command + { + NodeRef nodeRef; + + Action action; + + Command(NodeRef nodeRef, Action action) + { + this.nodeRef = nodeRef; + this.action = action; + } + + public String toString() + { + StringBuffer buffer = new StringBuffer(); + if (action == Action.INDEX) + { + buffer.append("Index "); + } + else if (action == Action.DELETE) + { + buffer.append("Delete "); + } + else if (action == Action.REINDEX) + { + buffer.append("Reindex "); + } + else + { + buffer.append("Unknown ... "); + } + buffer.append(nodeRef); + return buffer.toString(); + } + + } + + private FullTextSearchIndexer luceneFullTextSearchIndexer; + + public void setLuceneFullTextSearchIndexer(FullTextSearchIndexer luceneFullTextSearchIndexer) + { + this.luceneFullTextSearchIndexer = luceneFullTextSearchIndexer; + } + + public Set getDeletions() + { + return Collections.unmodifiableSet(deletions); + } + + public boolean getDeleteOnlyNodes() + { + if(isFTSUpdate != null) + { + return isFTSUpdate.booleanValue(); + } + else + { + return false; + } + } + + +} diff --git a/source/java/org/alfresco/repo/search/impl/lucene/LuceneSearcher2.java b/source/java/org/alfresco/repo/search/impl/lucene/LuceneSearcher2.java new file mode 100644 index 0000000000..4c4d6d6ddf --- /dev/null +++ b/source/java/org/alfresco/repo/search/impl/lucene/LuceneSearcher2.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2005 Alfresco, Inc. + * + * Licensed under the Mozilla Public License version 1.1 + * with a permitted attribution clause. You may obtain a + * copy of the License at + * + * http://www.alfresco.org/legal/license.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.alfresco.repo.search.impl.lucene; + +import org.alfresco.service.cmr.repository.NodeService; +import org.alfresco.service.cmr.search.SearchService; +import org.alfresco.service.namespace.NamespacePrefixResolver; + +public interface LuceneSearcher2 extends SearchService +{ + public boolean indexExists(); + public void setNodeService(NodeService nodeService); + public void setNamespacePrefixResolver(NamespacePrefixResolver namespacePrefixResolver); +} diff --git a/source/java/org/alfresco/repo/search/impl/lucene/LuceneSearcherImpl2.java b/source/java/org/alfresco/repo/search/impl/lucene/LuceneSearcherImpl2.java new file mode 100644 index 0000000000..193226c743 --- /dev/null +++ b/source/java/org/alfresco/repo/search/impl/lucene/LuceneSearcherImpl2.java @@ -0,0 +1,654 @@ +/* + * Copyright (C) 2005 Alfresco, Inc. + * + * Licensed under the Mozilla Public License version 1.1 + * with a permitted attribution clause. You may obtain a + * copy of the License at + * + * http://www.alfresco.org/legal/license.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.alfresco.repo.search.impl.lucene; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; + +import org.alfresco.repo.search.CannedQueryDef; +import org.alfresco.repo.search.EmptyResultSet; +import org.alfresco.repo.search.Indexer; +import org.alfresco.repo.search.QueryRegisterComponent; +import org.alfresco.repo.search.SearcherException; +import org.alfresco.repo.search.impl.NodeSearcher; +import org.alfresco.service.cmr.dictionary.DictionaryService; +import org.alfresco.service.cmr.repository.InvalidNodeRefException; +import org.alfresco.service.cmr.repository.NodeRef; +import org.alfresco.service.cmr.repository.NodeService; +import org.alfresco.service.cmr.repository.Path; +import org.alfresco.service.cmr.repository.StoreRef; +import org.alfresco.service.cmr.repository.XPathException; +import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter; +import org.alfresco.service.cmr.search.QueryParameter; +import org.alfresco.service.cmr.search.QueryParameterDefinition; +import org.alfresco.service.cmr.search.ResultSet; +import org.alfresco.service.cmr.search.SearchParameters; +import org.alfresco.service.cmr.search.SearchService; +import org.alfresco.service.namespace.NamespacePrefixResolver; +import org.alfresco.service.namespace.QName; +import org.alfresco.util.ISO9075; +import org.alfresco.util.SearchLanguageConversion; +import org.apache.lucene.search.Hits; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Searcher; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.saxpath.SAXPathException; + +import com.werken.saxpath.XPathReader; + +/** + * The Lucene implementation of Searcher At the moment we support only lucene + * based queries. + * + * TODO: Support for other query languages + * + * @author andyh + * + */ +public class LuceneSearcherImpl2 extends LuceneBase2 implements LuceneSearcher2 +{ + + /** + * Default field name + */ + private static final String DEFAULT_FIELD = "TEXT"; + + private NamespacePrefixResolver namespacePrefixResolver; + + private NodeService nodeService; + + private DictionaryService dictionaryService; + + private QueryRegisterComponent queryRegister; + + private LuceneIndexer2 indexer; + + /* + * Searcher implementation + */ + + /** + * Get an initialised searcher for the store and transaction Normally we do + * not search against a a store and delta. Currently only gets the searcher + * against the main index. + * + * @param storeRef + * @param deltaId + * @return + */ + public static LuceneSearcherImpl2 getSearcher(StoreRef storeRef, LuceneIndexer2 indexer, LuceneConfig config) + { + LuceneSearcherImpl2 searcher = new LuceneSearcherImpl2(); + searcher.setLuceneConfig(config); + try + { + searcher.initialise(storeRef, indexer == null ? null : indexer.getDeltaId(), false, false); + searcher.indexer = indexer; + } + catch (LuceneIndexException e) + { + throw new SearcherException(e); + } + return searcher; + } + + /** + * Get an intialised searcher for the store. No transactional ammendsmends + * are searched. + * + * + * @param storeRef + * @return + */ + public static LuceneSearcherImpl2 getSearcher(StoreRef storeRef, LuceneConfig config) + { + return getSearcher(storeRef, null, config); + } + + public void setNamespacePrefixResolver(NamespacePrefixResolver namespacePrefixResolver) + { + this.namespacePrefixResolver = namespacePrefixResolver; + } + + public boolean indexExists() + { + //return mainIndexExists(); + return true; + } + + public void setNodeService(NodeService nodeService) + { + this.nodeService = nodeService; + } + + public void setDictionaryService(DictionaryService dictionaryService) + { + this.dictionaryService = dictionaryService; + } + + public void setQueryRegister(QueryRegisterComponent queryRegister) + { + this.queryRegister = queryRegister; + } + + public ResultSet query(StoreRef store, String language, String queryString, Path[] queryOptions, + QueryParameterDefinition[] queryParameterDefinitions) throws SearcherException + { + SearchParameters sp = new SearchParameters(); + sp.addStore(store); + sp.setLanguage(language); + sp.setQuery(queryString); + if (queryOptions != null) + { + for (Path path : queryOptions) + { + sp.addAttrbutePath(path); + } + } + if (queryParameterDefinitions != null) + { + for (QueryParameterDefinition qpd : queryParameterDefinitions) + { + sp.addQueryParameterDefinition(qpd); + } + } + sp.excludeDataInTheCurrentTransaction(true); + + return query(sp); + } + + public ResultSet query(SearchParameters searchParameters) + { + if (searchParameters.getStores().size() != 1) + { + throw new IllegalStateException("Only one store can be searched at present"); + } + + String parameterisedQueryString; + if (searchParameters.getQueryParameterDefinitions().size() > 0) + { + Map map = new HashMap(); + + for (QueryParameterDefinition qpd : searchParameters.getQueryParameterDefinitions()) + { + map.put(qpd.getQName(), qpd); + } + + parameterisedQueryString = parameterise(searchParameters.getQuery(), map, null, namespacePrefixResolver); + } + else + { + parameterisedQueryString = searchParameters.getQuery(); + } + + if (searchParameters.getLanguage().equalsIgnoreCase(SearchService.LANGUAGE_LUCENE)) + { + try + { + + int defaultOperator; + if (searchParameters.getDefaultOperator() == SearchParameters.AND) + { + defaultOperator = LuceneQueryParser.DEFAULT_OPERATOR_AND; + } + else + { + defaultOperator = LuceneQueryParser.DEFAULT_OPERATOR_OR; + } + + Query query = LuceneQueryParser.parse(parameterisedQueryString, DEFAULT_FIELD, new LuceneAnalyser( + dictionaryService), namespacePrefixResolver, dictionaryService, defaultOperator); + Searcher searcher = getSearcher(indexer); + if (searcher == null) + { + // no index return an empty result set + return new EmptyResultSet(); + } + + Hits hits; + + if (searchParameters.getSortDefinitions().size() > 0) + { + int index = 0; + SortField[] fields = new SortField[searchParameters.getSortDefinitions().size()]; + for (SearchParameters.SortDefinition sd : searchParameters.getSortDefinitions()) + { + switch (sd.getSortType()) + { + case FIELD: + fields[index++] = new SortField(sd.getField(), !sd.isAscending()); + break; + case DOCUMENT: + fields[index++] = new SortField(null, SortField.DOC, !sd.isAscending()); + break; + case SCORE: + fields[index++] = new SortField(null, SortField.SCORE, !sd.isAscending()); + break; + } + + } + hits = searcher.search(query, new Sort(fields)); + } + else + { + hits = searcher.search(query); + } + + return new LuceneResultSet(hits, searcher, nodeService, searchParameters.getAttributePaths().toArray( + new Path[0]), searchParameters); + + } + catch (ParseException e) + { + throw new SearcherException("Failed to parse query: " + parameterisedQueryString, e); + } + catch (IOException e) + { + throw new SearcherException("IO exception during search", e); + } + } + else if (searchParameters.getLanguage().equalsIgnoreCase(SearchService.LANGUAGE_XPATH)) + { + try + { + XPathReader reader = new XPathReader(); + LuceneXPathHandler handler = new LuceneXPathHandler(); + handler.setNamespacePrefixResolver(namespacePrefixResolver); + handler.setDictionaryService(dictionaryService); + // TODO: Handler should have the query parameters to use in + // building its lucene query + // At the moment xpath style parameters in the PATH + // expression are not supported. + reader.setXPathHandler(handler); + reader.parse(parameterisedQueryString); + Query query = handler.getQuery(); + Searcher searcher = getSearcher(null); + if (searcher == null) + { + // no index return an empty result set + return new EmptyResultSet(); + } + Hits hits = searcher.search(query); + return new LuceneResultSet(hits, searcher, nodeService, searchParameters.getAttributePaths().toArray( + new Path[0]), searchParameters); + } + catch (SAXPathException e) + { + throw new SearcherException("Failed to parse query: " + searchParameters.getQuery(), e); + } + catch (IOException e) + { + throw new SearcherException("IO exception during search", e); + } + } + else + { + throw new SearcherException("Unknown query language: " + searchParameters.getLanguage()); + } + } + + public ResultSet query(StoreRef store, String language, String query) + { + return query(store, language, query, null, null); + } + + public ResultSet query(StoreRef store, String language, String query, + QueryParameterDefinition[] queryParameterDefintions) + { + return query(store, language, query, null, queryParameterDefintions); + } + + public ResultSet query(StoreRef store, String language, String query, Path[] attributePaths) + { + return query(store, language, query, attributePaths, null); + } + + public ResultSet query(StoreRef store, QName queryId, QueryParameter[] queryParameters) + { + CannedQueryDef definition = queryRegister.getQueryDefinition(queryId); + + // Do parameter replacement + // As lucene phrases are tokensied it is correct to just do straight + // string replacement. + // The string will be formatted by the tokeniser. + // + // For non phrase queries this is incorrect but string replacement is + // probably the best we can do. + // As numbers and text are indexed specially, direct term queries only + // make sense against textual data + + checkParameters(definition, queryParameters); + + String queryString = parameterise(definition.getQuery(), definition.getQueryParameterMap(), queryParameters, + definition.getNamespacePrefixResolver()); + + return query(store, definition.getLanguage(), queryString, null, null); + } + + /** + * The definitions must provide a default value, or of not there must be a + * parameter to provide the value + * + * @param definition + * @param queryParameters + * @throws QueryParameterisationException + */ + private void checkParameters(CannedQueryDef definition, QueryParameter[] queryParameters) + throws QueryParameterisationException + { + List missing = new ArrayList(); + + Set parameterQNameSet = new HashSet(); + if (queryParameters != null) + { + for (QueryParameter parameter : queryParameters) + { + parameterQNameSet.add(parameter.getQName()); + } + } + + for (QueryParameterDefinition parameterDefinition : definition.getQueryParameterDefs()) + { + if (!parameterDefinition.hasDefaultValue()) + { + if (!parameterQNameSet.contains(parameterDefinition.getQName())) + { + missing.add(parameterDefinition.getQName()); + } + } + } + + if (missing.size() > 0) + { + StringBuilder buffer = new StringBuilder(128); + buffer.append("The query is missing values for the following parameters: "); + for (QName qName : missing) + { + buffer.append(qName); + buffer.append(", "); + } + buffer.delete(buffer.length() - 1, buffer.length() - 1); + buffer.delete(buffer.length() - 1, buffer.length() - 1); + throw new QueryParameterisationException(buffer.toString()); + } + } + + /* + * Parameterise the query string - not sure if it is required to escape + * lucence spacials chars The parameters could be used to build the query - + * the contents of parameters should alread have been escaped if required. + * ... mush better to provide the parameters and work out what to do TODO: + * conditional query escapement - may be we should have a parameter type + * that is not escaped + */ + private String parameterise(String unparameterised, Map map, + QueryParameter[] queryParameters, NamespacePrefixResolver nspr) throws QueryParameterisationException + { + + Map> valueMap = new HashMap>(); + + if (queryParameters != null) + { + for (QueryParameter parameter : queryParameters) + { + List list = valueMap.get(parameter.getQName()); + if (list == null) + { + list = new ArrayList(); + valueMap.put(parameter.getQName(), list); + } + list.add(parameter.getValue()); + } + } + + Map> iteratorMap = new HashMap>(); + + List missing = new ArrayList(1); + StringBuilder buffer = new StringBuilder(unparameterised); + int index = 0; + while ((index = buffer.indexOf("${", index)) != -1) + { + int endIndex = buffer.indexOf("}", index); + String qNameString = buffer.substring(index + 2, endIndex); + QName key = QName.createQName(qNameString, nspr); + QueryParameterDefinition parameterDefinition = map.get(key); + if (parameterDefinition == null) + { + missing.add(key); + buffer.replace(index, endIndex + 1, ""); + } + else + { + ListIterator it = iteratorMap.get(key); + if ((it == null) || (!it.hasNext())) + { + List list = valueMap.get(key); + if ((list != null) && (list.size() > 0)) + { + it = list.listIterator(); + } + if (it != null) + { + iteratorMap.put(key, it); + } + } + String value; + if (it == null) + { + value = parameterDefinition.getDefault(); + } + else + { + value = DefaultTypeConverter.INSTANCE.convert(String.class, it.next()); + } + buffer.replace(index, endIndex + 1, value); + } + } + if (missing.size() > 0) + { + StringBuilder error = new StringBuilder(); + error.append("The query uses the following parameters which are not defined: "); + for (QName qName : missing) + { + error.append(qName); + error.append(", "); + } + error.delete(error.length() - 1, error.length() - 1); + error.delete(error.length() - 1, error.length() - 1); + throw new QueryParameterisationException(error.toString()); + } + return buffer.toString(); + } + + /** + * @see org.alfresco.repo.search.impl.NodeSearcher + */ + public List selectNodes(NodeRef contextNodeRef, String xpath, QueryParameterDefinition[] parameters, + NamespacePrefixResolver namespacePrefixResolver, boolean followAllParentLinks, String language) + throws InvalidNodeRefException, XPathException + { + NodeSearcher nodeSearcher = new NodeSearcher(nodeService, dictionaryService, this); + return nodeSearcher.selectNodes(contextNodeRef, xpath, parameters, namespacePrefixResolver, + followAllParentLinks, language); + } + + /** + * @see org.alfresco.repo.search.impl.NodeSearcher + */ + public List selectProperties(NodeRef contextNodeRef, String xpath, + QueryParameterDefinition[] parameters, NamespacePrefixResolver namespacePrefixResolver, + boolean followAllParentLinks, String language) throws InvalidNodeRefException, XPathException + { + NodeSearcher nodeSearcher = new NodeSearcher(nodeService, dictionaryService, this); + return nodeSearcher.selectProperties(contextNodeRef, xpath, parameters, namespacePrefixResolver, + followAllParentLinks, language); + } + + /** + * @return Returns true if the pattern is present, otherwise false. + */ + public boolean contains(NodeRef nodeRef, QName propertyQName, String googleLikePattern) + { + return contains(nodeRef, propertyQName, googleLikePattern, SearchParameters.Operator.OR); + } + + /** + * @return Returns true if the pattern is present, otherwise false. + */ + public boolean contains(NodeRef nodeRef, QName propertyQName, String googleLikePattern, + SearchParameters.Operator defaultOperator) + { + ResultSet resultSet = null; + try + { + // build Lucene search string specific to the node + StringBuilder sb = new StringBuilder(); + sb.append("+ID:\"").append(nodeRef.toString()).append("\" +(TEXT:(") + .append(googleLikePattern.toLowerCase()).append(") "); + if (propertyQName != null) + { + sb.append(" OR @").append( + LuceneQueryParser.escape(QName.createQName(propertyQName.getNamespaceURI(), + ISO9075.encode(propertyQName.getLocalName())).toString())); + sb.append(":(").append(googleLikePattern.toLowerCase()).append(")"); + } + else + { + for (QName key : nodeService.getProperties(nodeRef).keySet()) + { + sb.append(" OR @").append( + LuceneQueryParser.escape(QName.createQName(key.getNamespaceURI(), + ISO9075.encode(key.getLocalName())).toString())); + sb.append(":(").append(googleLikePattern.toLowerCase()).append(")"); + } + } + sb.append(")"); + + SearchParameters sp = new SearchParameters(); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery(sb.toString()); + sp.setDefaultOperator(defaultOperator); + sp.addStore(nodeRef.getStoreRef()); + + resultSet = this.query(sp); + boolean answer = resultSet.length() > 0; + return answer; + } + finally + { + if (resultSet != null) + { + resultSet.close(); + } + } + } + + /** + * @return Returns true if the pattern is present, otherwise false. + * + * @see #setIndexer(Indexer) + * @see #setSearcher(SearchService) + */ + public boolean like(NodeRef nodeRef, QName propertyQName, String sqlLikePattern, boolean includeFTS) + { + if (propertyQName == null) + { + throw new IllegalArgumentException("Property QName is mandatory for the like expression"); + } + + StringBuilder sb = new StringBuilder(sqlLikePattern.length() * 3); + + if (includeFTS) + { + // convert the SQL-like pattern into a Lucene-compatible string + String pattern = SearchLanguageConversion.convertXPathLikeToLucene(sqlLikePattern.toLowerCase()); + + // build Lucene search string specific to the node + sb = new StringBuilder(); + sb.append("+ID:\"").append(nodeRef.toString()).append("\" +("); + // FTS or attribute matches + if (includeFTS) + { + sb.append("TEXT:(").append(pattern).append(") "); + } + if (propertyQName != null) + { + sb.append(" @").append( + LuceneQueryParser.escape(QName.createQName(propertyQName.getNamespaceURI(), + ISO9075.encode(propertyQName.getLocalName())).toString())).append(":(").append(pattern) + .append(")"); + } + sb.append(")"); + + ResultSet resultSet = null; + try + { + resultSet = this.query(nodeRef.getStoreRef(), "lucene", sb.toString()); + boolean answer = resultSet.length() > 0; + return answer; + } + finally + { + if (resultSet != null) + { + resultSet.close(); + } + } + } + else + { + // convert the SQL-like pattern into a Lucene-compatible string + String pattern = SearchLanguageConversion.convertXPathLikeToRegex(sqlLikePattern.toLowerCase()); + + Serializable property = nodeService.getProperty(nodeRef, propertyQName); + if (property == null) + { + return false; + } + else + { + String propertyString = DefaultTypeConverter.INSTANCE.convert(String.class, nodeService.getProperty( + nodeRef, propertyQName)); + return propertyString.toLowerCase().matches(pattern); + } + } + } + + public List selectNodes(NodeRef contextNodeRef, String xpath, QueryParameterDefinition[] parameters, + NamespacePrefixResolver namespacePrefixResolver, boolean followAllParentLinks) + throws InvalidNodeRefException, XPathException + { + return selectNodes(contextNodeRef, xpath, parameters, namespacePrefixResolver, followAllParentLinks, + SearchService.LANGUAGE_XPATH); + } + + public List selectProperties(NodeRef contextNodeRef, String xpath, + QueryParameterDefinition[] parameters, NamespacePrefixResolver namespacePrefixResolver, + boolean followAllParentLinks) throws InvalidNodeRefException, XPathException + { + return selectProperties(contextNodeRef, xpath, parameters, namespacePrefixResolver, followAllParentLinks, + SearchService.LANGUAGE_XPATH); + } +} diff --git a/source/java/org/alfresco/repo/search/impl/lucene/LuceneTest.java b/source/java/org/alfresco/repo/search/impl/lucene/LuceneTest.java index 25af3ad3a9..12ec9baf4b 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/LuceneTest.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/LuceneTest.java @@ -46,6 +46,7 @@ import org.alfresco.repo.search.results.ChildAssocRefResultSet; import org.alfresco.repo.search.results.DetachedResultSet; import org.alfresco.repo.search.transaction.LuceneIndexLock; import org.alfresco.repo.security.authentication.AuthenticationComponent; +import org.alfresco.repo.security.authentication.AuthenticationUtil; import org.alfresco.service.ServiceRegistry; import org.alfresco.service.cmr.dictionary.DataTypeDefinition; import org.alfresco.service.cmr.dictionary.DictionaryService; @@ -178,7 +179,7 @@ public class LuceneTest extends TestCase serviceRegistry = (ServiceRegistry) ctx.getBean(ServiceRegistry.SERVICE_REGISTRY); this.authenticationComponent = (AuthenticationComponent) ctx.getBean("authenticationComponent"); - this.authenticationComponent.setSystemUserAsCurrentUser(); + queryRegisterComponent.loadQueryCollection("testQueryRegister.xml"); @@ -187,7 +188,8 @@ public class LuceneTest extends TestCase testTX = transactionService.getUserTransaction(); testTX.begin(); - + this.authenticationComponent.setSystemUserAsCurrentUser(); + // load in the test model ClassLoader cl = BaseNodeServiceTest.class.getClassLoader(); InputStream modelStream = cl.getResourceAsStream("org/alfresco/repo/search/impl/lucene/LuceneTest_model.xml"); @@ -323,7 +325,7 @@ public class LuceneTest extends TestCase { testTX.rollback(); } - authenticationComponent.clearCurrentSecurityContext(); + AuthenticationUtil.clearCurrentSecurityContext(); super.tearDown(); } @@ -391,7 +393,7 @@ public class LuceneTest extends TestCase public void testMTDeleteIssue() throws Exception { - + luceneFTS.pause(); testTX.commit(); UserTransaction tx = transactionService.getUserTransaction(); @@ -505,6 +507,7 @@ public class LuceneTest extends TestCase public void testDeltaIssue() throws Exception { + luceneFTS.pause(); final NodeService pns = (NodeService) ctx.getBean("NodeService"); testTX.commit(); @@ -855,7 +858,7 @@ public class LuceneTest extends TestCase + System.currentTimeMillis() + "_1", indexerAndSearcher); indexer.setNodeService(nodeService); - indexer.setLuceneIndexLock(luceneIndexLock); + //indexer.setLuceneIndexLock(luceneIndexLock); indexer.setDictionaryService(dictionaryService); indexer.setLuceneFullTextSearchIndexer(luceneFTS); indexer.setContentService(contentService); @@ -879,7 +882,7 @@ public class LuceneTest extends TestCase + System.currentTimeMillis() + "_1", indexerAndSearcher); indexer.setNodeService(nodeService); - indexer.setLuceneIndexLock(luceneIndexLock); + //indexer.setLuceneIndexLock(luceneIndexLock); indexer.setDictionaryService(dictionaryService); indexer.setLuceneFullTextSearchIndexer(luceneFTS); indexer.setContentService(contentService); @@ -1070,7 +1073,7 @@ public class LuceneTest extends TestCase LuceneIndexerImpl indexer = LuceneIndexerImpl.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + System.currentTimeMillis() + "_" + (new Random().nextInt()), indexerAndSearcher); indexer.setNodeService(nodeService); - indexer.setLuceneIndexLock(luceneIndexLock); + //indexer.setLuceneIndexLock(luceneIndexLock); indexer.setDictionaryService(dictionaryService); indexer.setLuceneFullTextSearchIndexer(luceneFTS); indexer.setContentService(contentService); @@ -1782,7 +1785,7 @@ public class LuceneTest extends TestCase LuceneIndexerImpl indexer = LuceneIndexerImpl.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + System.currentTimeMillis(), indexerAndSearcher); indexer.setNodeService(nodeService); - indexer.setLuceneIndexLock(luceneIndexLock); + //indexer.setLuceneIndexLock(luceneIndexLock); indexer.setDictionaryService(dictionaryService); indexer.setLuceneFullTextSearchIndexer(luceneFTS); indexer.setContentService(contentService); @@ -1818,7 +1821,7 @@ public class LuceneTest extends TestCase LuceneIndexerImpl indexer = LuceneIndexerImpl.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + System.currentTimeMillis(), indexerAndSearcher); indexer.setNodeService(nodeService); - indexer.setLuceneIndexLock(luceneIndexLock); + //indexer.setLuceneIndexLock(luceneIndexLock); indexer.setDictionaryService(dictionaryService); indexer.setLuceneFullTextSearchIndexer(luceneFTS); indexer.setContentService(contentService); @@ -2055,7 +2058,39 @@ public class LuceneTest extends TestCase LuceneIndexerImpl indexer = LuceneIndexerImpl.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + System.currentTimeMillis(), indexerAndSearcher); indexer.setNodeService(nodeService); - indexer.setLuceneIndexLock(luceneIndexLock); + //indexer.setLuceneIndexLock(luceneIndexLock); + indexer.setDictionaryService(dictionaryService); + indexer.setLuceneFullTextSearchIndexer(luceneFTS); + indexer.setContentService(contentService); + + ChildAssociationRef car = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName + .createQName("{namespace}" + COMPLEX_LOCAL_NAME), testSuperType); + indexer.createNode(car); + + indexer.commit(); + + LuceneSearcherImpl searcher = LuceneSearcherImpl.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + ResultSet results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:" + + ISO9075.encode(COMPLEX_LOCAL_NAME) + "\"", null, null); + assertEquals(1, results.length()); + results.close(); + } + + public void testNumericInPath() throws Exception + { + String COMPLEX_LOCAL_NAME = "Woof12"; + + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + + LuceneIndexerImpl indexer = LuceneIndexerImpl.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + + System.currentTimeMillis(), indexerAndSearcher); + indexer.setNodeService(nodeService); + //indexer.setLuceneIndexLock(luceneIndexLock); indexer.setDictionaryService(dictionaryService); indexer.setLuceneFullTextSearchIndexer(luceneFTS); indexer.setContentService(contentService); @@ -2085,7 +2120,7 @@ public class LuceneTest extends TestCase LuceneIndexerImpl indexer = LuceneIndexerImpl.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + System.currentTimeMillis(), indexerAndSearcher); indexer.setNodeService(nodeService); - indexer.setLuceneIndexLock(luceneIndexLock); + //indexer.setLuceneIndexLock(luceneIndexLock); indexer.setDictionaryService(dictionaryService); indexer.setLuceneFullTextSearchIndexer(luceneFTS); indexer.setContentService(contentService); @@ -2323,7 +2358,7 @@ public class LuceneTest extends TestCase LuceneIndexerImpl indexer = LuceneIndexerImpl.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + System.currentTimeMillis(), indexerAndSearcher); indexer.setNodeService(nodeService); - indexer.setLuceneIndexLock(luceneIndexLock); + //indexer.setLuceneIndexLock(luceneIndexLock); indexer.setDictionaryService(dictionaryService); indexer.setLuceneFullTextSearchIndexer(luceneFTS); indexer.setContentService(contentService); @@ -2566,7 +2601,7 @@ public class LuceneTest extends TestCase indexer = LuceneIndexerImpl.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + System.currentTimeMillis(), indexerAndSearcher); indexer.setNodeService(nodeService); - indexer.setLuceneIndexLock(luceneIndexLock); + //indexer.setLuceneIndexLock(luceneIndexLock); indexer.setDictionaryService(dictionaryService); indexer.setLuceneFullTextSearchIndexer(luceneFTS); indexer.setContentService(contentService); @@ -2610,7 +2645,7 @@ public class LuceneTest extends TestCase LuceneIndexerImpl indexer = LuceneIndexerImpl.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + System.currentTimeMillis(), indexerAndSearcher); indexer.setNodeService(nodeService); - indexer.setLuceneIndexLock(luceneIndexLock); + //indexer.setLuceneIndexLock(luceneIndexLock); indexer.setDictionaryService(dictionaryService); indexer.setLuceneFullTextSearchIndexer(luceneFTS); indexer.setContentService(contentService); @@ -2679,7 +2714,7 @@ public class LuceneTest extends TestCase LuceneIndexerImpl indexer = LuceneIndexerImpl.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + System.currentTimeMillis() + "_" + (new Random().nextInt()), indexerAndSearcher); indexer.setNodeService(nodeService); - indexer.setLuceneIndexLock(luceneIndexLock); + //indexer.setLuceneIndexLock(luceneIndexLock); indexer.setDictionaryService(dictionaryService); indexer.setLuceneFullTextSearchIndexer(luceneFTS); indexer.setContentService(contentService); @@ -2758,7 +2793,7 @@ public class LuceneTest extends TestCase LuceneIndexerImpl indexer = LuceneIndexerImpl.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + System.currentTimeMillis() + "_" + (new Random().nextInt()), indexerAndSearcher); indexer.setNodeService(nodeService); - indexer.setLuceneIndexLock(luceneIndexLock); + //indexer.setLuceneIndexLock(luceneIndexLock); indexer.setDictionaryService(dictionaryService); indexer.setLuceneFullTextSearchIndexer(luceneFTS); indexer.setContentService(contentService); @@ -3091,7 +3126,7 @@ public class LuceneTest extends TestCase LuceneIndexerImpl indexer = LuceneIndexerImpl.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + System.currentTimeMillis() + "_" + (new Random().nextInt()), indexerAndSearcher); indexer.setNodeService(nodeService); - indexer.setLuceneIndexLock(luceneIndexLock); + //indexer.setLuceneIndexLock(luceneIndexLock); indexer.setDictionaryService(dictionaryService); indexer.setLuceneFullTextSearchIndexer(luceneFTS); indexer.setContentService(contentService); diff --git a/source/java/org/alfresco/repo/search/impl/lucene/LuceneTest2.java b/source/java/org/alfresco/repo/search/impl/lucene/LuceneTest2.java new file mode 100644 index 0000000000..26a00b8604 --- /dev/null +++ b/source/java/org/alfresco/repo/search/impl/lucene/LuceneTest2.java @@ -0,0 +1,3197 @@ +/* + * Copyright (C) 2005 Alfresco, Inc. + * + * Licensed under the Mozilla Public License version 1.1 + * with a permitted attribution clause. You may obtain a + * copy of the License at + * + * http://www.alfresco.org/legal/license.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.alfresco.repo.search.impl.lucene; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Random; + +import javax.transaction.Status; +import javax.transaction.UserTransaction; + +import junit.framework.TestCase; + +import org.alfresco.model.ContentModel; +import org.alfresco.repo.dictionary.DictionaryDAO; +import org.alfresco.repo.dictionary.M2Model; +import org.alfresco.repo.node.BaseNodeServiceTest; +import org.alfresco.repo.search.QueryParameterDefImpl; +import org.alfresco.repo.search.QueryRegisterComponent; +import org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer; +import org.alfresco.repo.search.results.ChildAssocRefResultSet; +import org.alfresco.repo.search.results.DetachedResultSet; +import org.alfresco.repo.search.transaction.LuceneIndexLock; +import org.alfresco.repo.security.authentication.AuthenticationComponent; +import org.alfresco.repo.security.authentication.AuthenticationUtil; +import org.alfresco.service.ServiceRegistry; +import org.alfresco.service.cmr.dictionary.DataTypeDefinition; +import org.alfresco.service.cmr.dictionary.DictionaryService; +import org.alfresco.service.cmr.repository.ChildAssociationRef; +import org.alfresco.service.cmr.repository.ContentData; +import org.alfresco.service.cmr.repository.ContentService; +import org.alfresco.service.cmr.repository.ContentWriter; +import org.alfresco.service.cmr.repository.NodeRef; +import org.alfresco.service.cmr.repository.NodeService; +import org.alfresco.service.cmr.repository.Path; +import org.alfresco.service.cmr.repository.StoreRef; +import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter; +import org.alfresco.service.cmr.search.QueryParameter; +import org.alfresco.service.cmr.search.QueryParameterDefinition; +import org.alfresco.service.cmr.search.ResultSet; +import org.alfresco.service.cmr.search.ResultSetRow; +import org.alfresco.service.cmr.search.SearchParameters; +import org.alfresco.service.cmr.search.SearchService; +import org.alfresco.service.namespace.DynamicNamespacePrefixResolver; +import org.alfresco.service.namespace.NamespacePrefixResolver; +import org.alfresco.service.namespace.NamespaceService; +import org.alfresco.service.namespace.QName; +import org.alfresco.service.transaction.TransactionService; +import org.alfresco.util.ApplicationContextHelper; +import org.alfresco.util.CachingDateFormat; +import org.alfresco.util.ISO9075; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.context.ApplicationContext; + +/** + * @author andyh + * + */ +@SuppressWarnings("unused") +public class LuceneTest2 extends TestCase +{ + + private static final String TEST_NAMESPACE = "http://www.alfresco.org/test/lucenetest"; + + private static final QName ASSOC_TYPE_QNAME = QName.createQName(TEST_NAMESPACE, "assoc"); + + private static ApplicationContext ctx = ApplicationContextHelper.getApplicationContext(); + + private static Log logger = LogFactory.getLog(LuceneTest2.class); + + TransactionService transactionService; + + NodeService nodeService; + + DictionaryService dictionaryService; + + LuceneIndexLock luceneIndexLock; + + private NodeRef rootNodeRef; + + private NodeRef n1; + + private NodeRef n2; + + private NodeRef n3; + + private NodeRef n4; + + private NodeRef n5; + + private NodeRef n6; + + private NodeRef n7; + + private NodeRef n8; + + private NodeRef n9; + + private NodeRef n10; + + private NodeRef n11; + + private NodeRef n12; + + private NodeRef n13; + + private NodeRef n14; + + private DictionaryDAO dictionaryDAO; + + private FullTextSearchIndexer luceneFTS; + + private QName testType = QName.createQName(TEST_NAMESPACE, "testType"); + + private QName testSuperType = QName.createQName(TEST_NAMESPACE, "testSuperType"); + + private QName testAspect = QName.createQName(TEST_NAMESPACE, "testAspect"); + + private QName testSuperAspect = QName.createQName(TEST_NAMESPACE, "testSuperAspect"); + + private ContentService contentService; + + private QueryRegisterComponent queryRegisterComponent; + + private NamespacePrefixResolver namespacePrefixResolver; + + private LuceneIndexerAndSearcher indexerAndSearcher; + + private ServiceRegistry serviceRegistry; + + private UserTransaction testTX; + + private AuthenticationComponent authenticationComponent; + + private NodeRef[] documentOrder; + + public LuceneTest2() + { + super(); + } + + public void setUp() throws Exception + { + nodeService = (NodeService) ctx.getBean("dbNodeService"); + luceneIndexLock = (LuceneIndexLock) ctx.getBean("luceneIndexLock"); + dictionaryService = (DictionaryService) ctx.getBean("dictionaryService"); + dictionaryDAO = (DictionaryDAO) ctx.getBean("dictionaryDAO"); + luceneFTS = (FullTextSearchIndexer) ctx.getBean("LuceneFullTextSearchIndexer"); + contentService = (ContentService) ctx.getBean("contentService"); + queryRegisterComponent = (QueryRegisterComponent) ctx.getBean("queryRegisterComponent"); + namespacePrefixResolver = (NamespacePrefixResolver) ctx.getBean("namespaceService"); + indexerAndSearcher = (LuceneIndexerAndSearcher) ctx.getBean("luceneIndexerAndSearcherFactory"); + transactionService = (TransactionService) ctx.getBean("transactionComponent"); + serviceRegistry = (ServiceRegistry) ctx.getBean(ServiceRegistry.SERVICE_REGISTRY); + + this.authenticationComponent = (AuthenticationComponent) ctx.getBean("authenticationComponent"); + + + queryRegisterComponent.loadQueryCollection("testQueryRegister.xml"); + + assertEquals(true, ctx.isSingleton("luceneIndexLock")); + assertEquals(true, ctx.isSingleton("LuceneFullTextSearchIndexer")); + + testTX = transactionService.getUserTransaction(); + testTX.begin(); + this.authenticationComponent.setSystemUserAsCurrentUser(); + + // load in the test model + ClassLoader cl = BaseNodeServiceTest.class.getClassLoader(); + InputStream modelStream = cl.getResourceAsStream("org/alfresco/repo/search/impl/lucene/LuceneTest_model.xml"); + assertNotNull(modelStream); + M2Model model = M2Model.createModel(modelStream); + dictionaryDAO.putModel(model); + + StoreRef storeRef = nodeService.createStore(StoreRef.PROTOCOL_WORKSPACE, "Test_" + System.currentTimeMillis()); + rootNodeRef = nodeService.getRootNode(storeRef); + + n1 = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}one"), + testSuperType).getChildRef(); + nodeService.setProperty(n1, QName.createQName("{namespace}property-1"), "ValueOne"); + + n2 = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}two"), + testSuperType).getChildRef(); + nodeService.setProperty(n2, QName.createQName("{namespace}property-1"), "valueone"); + nodeService.setProperty(n2, QName.createQName("{namespace}property-2"), "valuetwo"); + + n3 = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}three"), + testSuperType).getChildRef(); + + ObjectOutputStream oos; + try + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + oos = new ObjectOutputStream(baos); + oos.writeObject(n3); + oos.close(); + + ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(baos.toByteArray())); + Object o = ois.readObject(); + ois.close(); + } + catch (IOException e) + { + // TODO Auto-generated catch block + e.printStackTrace(); + } + catch (ClassNotFoundException e) + { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + Map testProperties = new HashMap(); + testProperties.put(QName.createQName(TEST_NAMESPACE, "text-indexed-stored-tokenised-atomic"), + "TEXT THAT IS INDEXED STORED AND TOKENISED ATOMICALLY KEYONE"); + testProperties.put(QName.createQName(TEST_NAMESPACE, "text-indexed-unstored-tokenised-atomic"), + "TEXT THAT IS INDEXED STORED AND TOKENISED ATOMICALLY KEYUNSTORED"); + testProperties.put(QName.createQName(TEST_NAMESPACE, "text-indexed-stored-tokenised-nonatomic"), + "TEXT THAT IS INDEXED STORED AND TOKENISED BUT NOT ATOMICALLY KEYTWO"); + testProperties.put(QName.createQName(TEST_NAMESPACE, "int-ista"), Integer.valueOf(1)); + testProperties.put(QName.createQName(TEST_NAMESPACE, "long-ista"), Long.valueOf(2)); + testProperties.put(QName.createQName(TEST_NAMESPACE, "float-ista"), Float.valueOf(3.4f)); + testProperties.put(QName.createQName(TEST_NAMESPACE, "double-ista"), Double.valueOf(5.6)); + testProperties.put(QName.createQName(TEST_NAMESPACE, "date-ista"), new Date()); + testProperties.put(QName.createQName(TEST_NAMESPACE, "datetime-ista"), new Date()); + testProperties.put(QName.createQName(TEST_NAMESPACE, "boolean-ista"), Boolean.valueOf(true)); + testProperties.put(QName.createQName(TEST_NAMESPACE, "qname-ista"), QName.createQName("{wibble}wobble")); + testProperties.put(QName.createQName(TEST_NAMESPACE, "category-ista"), new NodeRef(storeRef, "CategoryId")); + testProperties.put(QName.createQName(TEST_NAMESPACE, "noderef-ista"), n1); + testProperties.put(QName.createQName(TEST_NAMESPACE, "path-ista"), nodeService.getPath(n3)); + testProperties.put(QName.createQName(TEST_NAMESPACE, "null"), null); + testProperties.put(QName.createQName(TEST_NAMESPACE, "list"), new ArrayList()); + ArrayList testList = new ArrayList(); + testList.add(null); + testProperties.put(QName.createQName(TEST_NAMESPACE, "nullList"), testList); + ArrayList testList2 = new ArrayList(); + testList2.add("woof"); + testList2.add(null); + + n4 = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}four"), + testType, testProperties).getChildRef(); + + nodeService.getProperties(n1); + nodeService.getProperties(n2); + nodeService.getProperties(n3); + nodeService.getProperties(n4); + + n5 = nodeService.createNode(n1, ASSOC_TYPE_QNAME, QName.createQName("{namespace}five"), testSuperType) + .getChildRef(); + n6 = nodeService.createNode(n1, ASSOC_TYPE_QNAME, QName.createQName("{namespace}six"), testSuperType) + .getChildRef(); + n7 = nodeService.createNode(n2, ASSOC_TYPE_QNAME, QName.createQName("{namespace}seven"), testSuperType) + .getChildRef(); + n8 = nodeService.createNode(n2, ASSOC_TYPE_QNAME, QName.createQName("{namespace}eight-2"), testSuperType) + .getChildRef(); + n9 = nodeService.createNode(n5, ASSOC_TYPE_QNAME, QName.createQName("{namespace}nine"), testSuperType) + .getChildRef(); + n10 = nodeService.createNode(n5, ASSOC_TYPE_QNAME, QName.createQName("{namespace}ten"), testSuperType) + .getChildRef(); + n11 = nodeService.createNode(n5, ASSOC_TYPE_QNAME, QName.createQName("{namespace}eleven"), testSuperType) + .getChildRef(); + n12 = nodeService.createNode(n5, ASSOC_TYPE_QNAME, QName.createQName("{namespace}twelve"), testSuperType) + .getChildRef(); + n13 = nodeService.createNode(n12, ASSOC_TYPE_QNAME, QName.createQName("{namespace}thirteen"), testSuperType) + .getChildRef(); + + Map properties = new HashMap(); + properties.put(ContentModel.PROP_CONTENT, new ContentData(null, "text/plain", 0L, "UTF-16")); + n14 = nodeService.createNode(n13, ASSOC_TYPE_QNAME, QName.createQName("{namespace}fourteen"), + ContentModel.TYPE_CONTENT, properties).getChildRef(); + // nodeService.addAspect(n14, DictionaryBootstrap.ASPECT_QNAME_CONTENT, + // properties); + + ContentWriter writer = contentService.getWriter(n14, ContentModel.PROP_CONTENT, true); + // InputStream is = + // this.getClass().getClassLoader().getResourceAsStream("test.doc"); + // writer.putContent(is); + writer.putContent("The quick brown fox jumped over the lazy dog"); + + nodeService.addChild(rootNodeRef, n8, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}eight-0")); + nodeService.addChild(n1, n8, ASSOC_TYPE_QNAME, QName.createQName("{namespace}eight-1")); + nodeService.addChild(n2, n13, ASSOC_TYPE_QNAME, QName.createQName("{namespace}link")); + + nodeService.addChild(n1, n14, ASSOC_TYPE_QNAME, QName.createQName("{namespace}common")); + nodeService.addChild(n2, n14, ASSOC_TYPE_QNAME, QName.createQName("{namespace}common")); + nodeService.addChild(n5, n14, ASSOC_TYPE_QNAME, QName.createQName("{namespace}common")); + nodeService.addChild(n6, n14, ASSOC_TYPE_QNAME, QName.createQName("{namespace}common")); + nodeService.addChild(n12, n14, ASSOC_TYPE_QNAME, QName.createQName("{namespace}common")); + nodeService.addChild(n13, n14, ASSOC_TYPE_QNAME, QName.createQName("{namespace}common")); + + documentOrder = new NodeRef[] { rootNodeRef, n1, n2, n3, n4, n5, n6, n7, n8, n9, n10, n11, n12, n13, n14 }; + + } + + @Override + protected void tearDown() throws Exception + { + + if (testTX.getStatus() == Status.STATUS_ACTIVE) + { + testTX.rollback(); + } + AuthenticationUtil.clearCurrentSecurityContext(); + super.tearDown(); + } + + public LuceneTest2(String arg0) + { + super(arg0); + } + + public void test0() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + luceneFTS.resume(); + } + + public void testDeleteIssue() throws Exception + { + + testTX.commit(); + + UserTransaction tx = transactionService.getUserTransaction(); + tx.begin(); + ChildAssociationRef testFind = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName + .createQName("{namespace}testFind"), testSuperType); + tx.commit(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + searcher.setQueryRegister(queryRegisterComponent); + + ResultSet results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "QNAME:\"namespace:testFind\""); + assertEquals(1, results.length()); + results.close(); + + UserTransaction tx1 = transactionService.getUserTransaction(); + tx1.begin(); + for (int i = 0; i < 100; i++) + { + HashSet refs = new HashSet(); + for (int j = 0; j < i; j++) + { + ChildAssociationRef test = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName + .createQName("{namespace}test"), testSuperType); + refs.add(test); + } + + for (ChildAssociationRef car : refs) + { + nodeService.deleteNode(car.getChildRef()); + } + + } + tx1.commit(); + + UserTransaction tx3 = transactionService.getUserTransaction(); + tx3.begin(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "QNAME:\"namespace:testFind\""); + assertEquals(1, results.length()); + results.close(); + tx3.commit(); + } + + public void testMTDeleteIssue() throws Exception + { + luceneFTS.pause(); + testTX.commit(); + + UserTransaction tx = transactionService.getUserTransaction(); + tx.begin(); + ChildAssociationRef testFind = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName + .createQName("{namespace}testFind"), testSuperType); + tx.commit(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + searcher.setQueryRegister(queryRegisterComponent); + + ResultSet results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "QNAME:\"namespace:testFind\""); + assertEquals(1, results.length()); + results.close(); + + Thread runner = null; + + for (int i = 0; i < 20; i++) + { + runner = new Nester("Concurrent-" + i, runner); + } + if (runner != null) + { + runner.start(); + + try + { + runner.join(); + } + catch (InterruptedException e) + { + e.printStackTrace(); + } + } + + UserTransaction tx3 = transactionService.getUserTransaction(); + tx3.begin(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "QNAME:\"namespace:testFind\""); + assertEquals(1, results.length()); + results.close(); + tx3.commit(); + } + + class Nester extends Thread + { + Thread waiter; + + Nester(String name, Thread waiter) + { + super(name); + this.setDaemon(true); + this.waiter = waiter; + } + + public void run() + { + authenticationComponent.setSystemUserAsCurrentUser(); + if (waiter != null) + { + waiter.start(); + } + try + { + System.out.println("Start " + this.getName()); + UserTransaction tx1 = transactionService.getUserTransaction(); + tx1.begin(); + for (int i = 0; i < 20; i++) + { + HashSet refs = new HashSet(); + for (int j = 0; j < i; j++) + { + ChildAssociationRef test = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, + QName.createQName("{namespace}test"), testSuperType); + refs.add(test); + } + + for (ChildAssociationRef car : refs) + { + nodeService.deleteNode(car.getChildRef()); + } + + } + tx1.commit(); + System.out.println("End " + this.getName()); + } + catch (Exception e) + { + e.printStackTrace(); + System.exit(12); + } + finally + { + authenticationComponent.clearCurrentSecurityContext(); + } + if (waiter != null) + { + try + { + waiter.join(); + } + catch (InterruptedException e) + { + } + } + } + + } + + public void testDeltaIssue() throws Exception + { + luceneFTS.pause(); + final NodeService pns = (NodeService) ctx.getBean("NodeService"); + + testTX.commit(); + UserTransaction tx = transactionService.getUserTransaction(); + tx.begin(); + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + tx.commit(); + + Thread thread = new Thread(new Runnable() + { + + public void run() + { + try + { + authenticationComponent.setSystemUserAsCurrentUser(); + UserTransaction tx = transactionService.getUserTransaction(); + tx = transactionService.getUserTransaction(); + tx.begin(); + + SearchParameters sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.excludeDataInTheCurrentTransaction(false); + ResultSet results = serviceRegistry.getSearchService().query(sp); + assertEquals(15, results.length()); + results.close(); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.excludeDataInTheCurrentTransaction(false); + results = serviceRegistry.getSearchService().query(sp); + assertEquals(15, results.length()); + results.close(); + + Map props = new HashMap(); + props.put(ContentModel.PROP_TITLE, "woof"); + pns.addAspect(n1, ContentModel.ASPECT_TITLED, props); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.excludeDataInTheCurrentTransaction(false); + results = serviceRegistry.getSearchService().query(sp); + assertEquals(15, results.length()); + results.close(); + + tx.rollback(); + } + catch (Throwable e) + { + throw new RuntimeException(e); + } + + } + + }); + + thread.start(); + thread.join(); + + tx = transactionService.getUserTransaction(); + tx.begin(); + + SearchParameters sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.excludeDataInTheCurrentTransaction(false); + ResultSet results = serviceRegistry.getSearchService().query(sp); + assertEquals(15, results.length()); + results.close(); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.excludeDataInTheCurrentTransaction(false); + results = serviceRegistry.getSearchService().query(sp); + assertEquals(15, results.length()); + results.close(); + + runBaseTests(); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.excludeDataInTheCurrentTransaction(false); + results = serviceRegistry.getSearchService().query(sp); + assertEquals(15, results.length()); + results.close(); + + Map props = new HashMap(); + props.put(ContentModel.PROP_TITLE, "woof"); + pns.addAspect(n1, ContentModel.ASPECT_TITLED, props); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.excludeDataInTheCurrentTransaction(false); + results = serviceRegistry.getSearchService().query(sp); + assertEquals(15, results.length()); + results.close(); + + pns.setProperty(n1, ContentModel.PROP_TITLE, "cube"); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.excludeDataInTheCurrentTransaction(false); + results = serviceRegistry.getSearchService().query(sp); + assertEquals(15, results.length()); + results.close(); + + tx.rollback(); + + } + + public void testRepeatPerformance() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + + String query = "ID:\"" + rootNodeRef + "\""; + // check that we get the result + SearchParameters sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery(query); + ResultSet results = searcher.query(sp); + assertEquals("No results found from query", 1, results.length()); + + long start = System.nanoTime(); + int count = 1000; + // repeat + for (int i = 0; i < count; i++) + { + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery(query); + results = searcher.query(sp); + } + long end = System.nanoTime(); + // dump results + double duration = ((double) (end - start)) / 1E6; // duration in ms + double average = duration / (double) count; + System.out.println("Searched for identifier: \n" + + " count: " + count + "\n" + " average: " + average + " ms/search \n" + + " a million searches could take: " + (1E6 * average) / 1E3 / 60D + " minutes"); + // anything over 10ms is dire + if (average > 10.0) + { + logger.error("Search taking longer than 10ms: " + query); + } + } + + public void testSort() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + + SearchParameters sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.addSort("ID", true); + ResultSet results = searcher.query(sp); + + String current = null; + for (ResultSetRow row : results) + { + String id = row.getNodeRef().getId(); + + if (current != null) + { + if (current.compareTo(id) > 0) + { + fail(); + } + } + current = id; + } + results.close(); + + SearchParameters sp2 = new SearchParameters(); + sp2.addStore(rootNodeRef.getStoreRef()); + sp2.setLanguage(SearchService.LANGUAGE_LUCENE); + sp2.setQuery("PATH:\"//.\""); + sp2.addSort("ID", false); + results = searcher.query(sp2); + + current = null; + for (ResultSetRow row : results) + { + String id = row.getNodeRef().getId(); + if (current != null) + { + if (current.compareTo(id) < 0) + { + fail(); + } + } + current = id; + } + results.close(); + + luceneFTS.resume(); + + SearchParameters sp3 = new SearchParameters(); + sp3.addStore(rootNodeRef.getStoreRef()); + sp3.setLanguage(SearchService.LANGUAGE_LUCENE); + sp3.setQuery("PATH:\"//.\""); + sp3.addSort(SearchParameters.SORT_IN_DOCUMENT_ORDER_ASCENDING); + results = searcher.query(sp3); + + int count = 0; + for (ResultSetRow row : results) + { + assertEquals(documentOrder[count++], row.getNodeRef()); + } + results.close(); + + SearchParameters sp4 = new SearchParameters(); + sp4.addStore(rootNodeRef.getStoreRef()); + sp4.setLanguage(SearchService.LANGUAGE_LUCENE); + sp4.setQuery("PATH:\"//.\""); + sp4.addSort(SearchParameters.SORT_IN_DOCUMENT_ORDER_DESCENDING); + results = searcher.query(sp4); + + count = 1; + for (ResultSetRow row : results) + { + assertEquals(documentOrder[documentOrder.length - (count++)], row.getNodeRef()); + } + results.close(); + + SearchParameters sp5 = new SearchParameters(); + sp5.addStore(rootNodeRef.getStoreRef()); + sp5.setLanguage(SearchService.LANGUAGE_LUCENE); + sp5.setQuery("PATH:\"//.\""); + sp5.addSort(SearchParameters.SORT_IN_SCORE_ORDER_ASCENDING); + results = searcher.query(sp5); + + float score = 0; + for (ResultSetRow row : results) + { + assertTrue(score <= row.getScore()); + score = row.getScore(); + } + results.close(); + + SearchParameters sp6 = new SearchParameters(); + sp6.addStore(rootNodeRef.getStoreRef()); + sp6.setLanguage(SearchService.LANGUAGE_LUCENE); + sp6.setQuery("PATH:\"//.\""); + sp6.addSort(SearchParameters.SORT_IN_SCORE_ORDER_DESCENDING); + results = searcher.query(sp6); + + score = 1.0f; + for (ResultSetRow row : results) + { + assertTrue(score >= row.getScore()); + score = row.getScore(); + } + results.close(); + + luceneFTS.resume(); + } + + public void test1() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + luceneFTS.resume(); + } + + public void test2() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + luceneFTS.resume(); + } + + public void test3() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + luceneFTS.resume(); + } + + public void test4() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + + LuceneSearcherImpl searcher = LuceneSearcherImpl.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setDictionaryService(dictionaryService); + + ResultSet results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "\\@\\{namespace\\}property\\-2:\"valuetwo\"", null, null); + results.close(); + luceneFTS.resume(); + } + + public void test5() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + luceneFTS.resume(); + } + + public void test6() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + luceneFTS.resume(); + } + + public void testNoOp() throws Exception + { + luceneFTS.pause(); + LuceneIndexerImpl indexer = LuceneIndexerImpl.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + + System.currentTimeMillis() + "_1", indexerAndSearcher); + + indexer.setNodeService(nodeService); + //indexer.setLuceneIndexLock(luceneIndexLock); + indexer.setDictionaryService(dictionaryService); + indexer.setLuceneFullTextSearchIndexer(luceneFTS); + indexer.setContentService(contentService); + + indexer.prepare(); + indexer.commit(); + luceneFTS.resume(); + } + + /** + * Test basic index and search + * + * @throws InterruptedException + * + */ + + public void testStandAloneIndexerCommit() throws Exception + { + luceneFTS.pause(); + LuceneIndexerImpl2 indexer = LuceneIndexerImpl2.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + + System.currentTimeMillis() + "_1", indexerAndSearcher); + + indexer.setNodeService(nodeService); + //indexer.setLuceneIndexLock(luceneIndexLock); + indexer.setDictionaryService(dictionaryService); + indexer.setLuceneFullTextSearchIndexer(luceneFTS); + indexer.setContentService(contentService); + + // //indexer.clearIndex(); + + indexer.createNode(new ChildAssociationRef(null, null, null, rootNodeRef)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CHILDREN, rootNodeRef, QName + .createQName("{namespace}one"), n1)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CHILDREN, rootNodeRef, QName + .createQName("{namespace}two"), n2)); + indexer.updateNode(n1); + // indexer.deleteNode(new ChildRelationshipRef(rootNode, "path", + // newNode)); + + indexer.prepare(); + indexer.commit(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + + ResultSet results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "\\@\\{namespace\\}property\\-2:\"valuetwo\"", null, null); + simpleResultSetTest(results); + + ChildAssocRefResultSet r2 = new ChildAssocRefResultSet(nodeService, results.getNodeRefs(), null, false); + simpleResultSetTest(r2); + + ChildAssocRefResultSet r3 = new ChildAssocRefResultSet(nodeService, results.getNodeRefs(), null, true); + simpleResultSetTest(r3); + + ChildAssocRefResultSet r4 = new ChildAssocRefResultSet(nodeService, results.getChildAssocRefs(), null); + simpleResultSetTest(r4); + + DetachedResultSet r5 = new DetachedResultSet(results, null); + simpleResultSetTest(r5); + + DetachedResultSet r6 = new DetachedResultSet(r2, null); + simpleResultSetTest(r6); + + DetachedResultSet r7 = new DetachedResultSet(r3, null); + simpleResultSetTest(r7); + + DetachedResultSet r8 = new DetachedResultSet(r4, null); + simpleResultSetTest(r8); + + DetachedResultSet r9 = new DetachedResultSet(r5, null); + simpleResultSetTest(r9); + + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@\\{namespace\\}property\\-1:\"valueone\"", + null, null); + assertEquals(2, results.length()); + assertEquals(n2.getId(), results.getNodeRef(0).getId()); + assertEquals(n1.getId(), results.getNodeRef(1).getId()); + assertEquals(1.0f, results.getScore(0)); + assertEquals(1.0f, results.getScore(1)); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@namespace\\:property\\-1:\"valueone\"", null, + null); + assertEquals(2, results.length()); + assertEquals(n2.getId(), results.getNodeRef(0).getId()); + assertEquals(n1.getId(), results.getNodeRef(1).getId()); + assertEquals(1.0f, results.getScore(0)); + assertEquals(1.0f, results.getScore(1)); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@property\\-1:\"valueone\"", null, null); + assertEquals(2, results.length()); + assertEquals(n2.getId(), results.getNodeRef(0).getId()); + assertEquals(n1.getId(), results.getNodeRef(1).getId()); + assertEquals(1.0f, results.getScore(0)); + assertEquals(1.0f, results.getScore(1)); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@property\\-1:\"Valueone\"", null, null); + assertEquals(2, results.length()); + assertEquals(n2.getId(), results.getNodeRef(0).getId()); + assertEquals(n1.getId(), results.getNodeRef(1).getId()); + assertEquals(1.0f, results.getScore(0)); + assertEquals(1.0f, results.getScore(1)); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@property\\-1:ValueOne", null, null); + assertEquals(2, results.length()); + assertEquals(n2.getId(), results.getNodeRef(0).getId()); + assertEquals(n1.getId(), results.getNodeRef(1).getId()); + assertEquals(1.0f, results.getScore(0)); + assertEquals(1.0f, results.getScore(1)); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@property\\-1:valueone", null, null); + assertEquals(2, results.length()); + assertEquals(n2.getId(), results.getNodeRef(0).getId()); + assertEquals(n1.getId(), results.getNodeRef(1).getId()); + assertEquals(1.0f, results.getScore(0)); + assertEquals(1.0f, results.getScore(1)); + results.close(); + + QName qname = QName.createQName("", "property-1"); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "ID:\"" + n1.toString() + "\"", null, null); + + assertEquals(2, results.length()); + + results.close(); + luceneFTS.resume(); + } + + private void simpleResultSetTest(ResultSet results) + { + assertEquals(1, results.length()); + assertEquals(n2.getId(), results.getNodeRef(0).getId()); + assertEquals(n2, results.getNodeRef(0)); + assertEquals(new ChildAssociationRef(ContentModel.ASSOC_CHILDREN, rootNodeRef, QName + .createQName("{namespace}two"), n2), results.getChildAssocRef(0)); + assertEquals(1, results.getChildAssocRefs().size()); + assertNotNull(results.getChildAssocRefs()); + assertEquals(0, results.getRow(0).getIndex()); + assertEquals(1.0f, results.getRow(0).getScore()); + assertEquals(new ChildAssociationRef(ContentModel.ASSOC_CHILDREN, rootNodeRef, QName + .createQName("{namespace}two"), n2), results.getRow(0).getChildAssocRef()); + assertEquals(n2, results.getRow(0).getNodeRef()); + assertEquals(QName.createQName("{namespace}two"), results.getRow(0).getQName()); + assertEquals("valuetwo", results.getRow(0).getValue(QName.createQName("{namespace}property-2"))); + for (ResultSetRow row : results) + { + assertNotNull(row); + } + } + + public void testStandAlonePathIndexer() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + + ResultSet results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "@\\{namespace\\}property-1:valueone", + null, null); + try + { + assertEquals(2, results.length()); + assertEquals(n1.getId(), results.getNodeRef(0).getId()); + assertEquals(n2.getId(), results.getNodeRef(1).getId()); + // assertEquals(1.0f, results.getScore(0)); + // assertEquals(1.0f, results.getScore(1)); + + QName qname = QName.createQName("", "property-1"); + + } + finally + { + results.close(); + } + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "+ID:\"" + n1.toString() + "\"", null, null); + try + { + assertEquals(2, results.length()); + } + finally + { + results.close(); + } + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "ID:\"" + rootNodeRef.toString() + "\"", null, + null); + try + { + assertEquals(1, results.length()); + } + finally + { + results.close(); + } + luceneFTS.resume(); + } + + private void buildBaseIndex() + { + LuceneIndexerImpl2 indexer = LuceneIndexerImpl2.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + + System.currentTimeMillis() + "_" + (new Random().nextInt()), indexerAndSearcher); + indexer.setNodeService(nodeService); + //indexer.setLuceneIndexLock(luceneIndexLock); + indexer.setDictionaryService(dictionaryService); + indexer.setLuceneFullTextSearchIndexer(luceneFTS); + indexer.setContentService(contentService); + // indexer.clearIndex(); + indexer.createNode(new ChildAssociationRef(null, null, null, rootNodeRef)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CHILDREN, rootNodeRef, QName + .createQName("{namespace}one"), n1)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CHILDREN, rootNodeRef, QName + .createQName("{namespace}two"), n2)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CHILDREN, rootNodeRef, QName + .createQName("{namespace}three"), n3)); + indexer.createNode(new ChildAssociationRef(ContentModel.ASSOC_CHILDREN, rootNodeRef, QName + .createQName("{namespace}four"), n4)); + indexer.createNode(new ChildAssociationRef(ASSOC_TYPE_QNAME, n1, QName.createQName("{namespace}five"), n5)); + indexer.createNode(new ChildAssociationRef(ASSOC_TYPE_QNAME, n1, QName.createQName("{namespace}six"), n6)); + indexer.createNode(new ChildAssociationRef(ASSOC_TYPE_QNAME, n2, QName.createQName("{namespace}seven"), n7)); + indexer.createNode(new ChildAssociationRef(ASSOC_TYPE_QNAME, n2, QName.createQName("{namespace}eight"), n8)); + indexer.createNode(new ChildAssociationRef(ASSOC_TYPE_QNAME, n5, QName.createQName("{namespace}nine"), n9)); + indexer.createNode(new ChildAssociationRef(ASSOC_TYPE_QNAME, n5, QName.createQName("{namespace}ten"), n10)); + indexer.createNode(new ChildAssociationRef(ASSOC_TYPE_QNAME, n5, QName.createQName("{namespace}eleven"), n11)); + indexer.createNode(new ChildAssociationRef(ASSOC_TYPE_QNAME, n5, QName.createQName("{namespace}twelve"), n12)); + indexer + .createNode(new ChildAssociationRef(ASSOC_TYPE_QNAME, n12, QName.createQName("{namespace}thirteen"), + n13)); + indexer + .createNode(new ChildAssociationRef(ASSOC_TYPE_QNAME, n13, QName.createQName("{namespace}fourteen"), + n14)); + indexer.prepare(); + indexer.commit(); + } + + public void testAllPathSearch() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + + runBaseTests(); + luceneFTS.resume(); + } + + private void runBaseTests() + { + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + searcher.setQueryRegister(queryRegisterComponent); + ResultSet results; + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/.\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:three\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:four\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:eight-0\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:five\"", null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:one\"", null, + null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:two\"", null, + null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:one\"", null, + null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:two\"", null, + null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:five\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:six\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:seven\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:eight-1\"", + null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:eight-2\"", + null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:eight-2\"", + null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:eight-1\"", + null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:eight-0\"", + null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:eight-0\"", + null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:nine\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:ten\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:eleven\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:twelve\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:twelve/namespace:thirteen\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:twelve/namespace:thirteen/namespace:fourteen\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/namespace:*/namespace:*\"", + null, null); + assertEquals(8, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:*/namespace:*\"", null, null); + assertEquals(6, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:*/namespace:five\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH_WITH_REPEATS:\"/namespace:*/namespace:*/namespace:*\"", null, null); + assertEquals(8, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:*/namespace:*/namespace:*\"", + null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher + .query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:*\"", null, null); + assertEquals(4, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:*/namespace:five/namespace:*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:*/namespace:nine\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/*/*\"", null, null); + assertEquals(8, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*/*\"", null, null); + assertEquals(6, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*/namespace:five\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/*/*/*\"", null, null); + assertEquals(8, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*/*/*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/*\"", null, null); + assertEquals(4, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*/namespace:five/*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/*/namespace:nine\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//.\"", null, null); + assertEquals(26, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//.\"", null, null); + assertEquals(15, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//*\"", null, null); + assertEquals(14, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//*\"", null, null); + assertEquals(25, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//*/.\"", null, null); + assertEquals(14, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//*/.\"", null, null); + assertEquals(25, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//*/./.\"", null, null); + assertEquals(14, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//*/./.\"", null, null); + assertEquals(25, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//./*\"", null, null); + assertEquals(25, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//./*\"", null, null); + assertEquals(14, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//././*/././.\"", null, null); + assertEquals(14, results.length()); + results.close(); + results = searcher + .query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//././*/././.\"", null, null); + assertEquals(25, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//common\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//common\"", null, null); + assertEquals(7, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one//common\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/one//common\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one/five//*\"", null, null); + assertEquals(6, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/one/five//*\"", null, null); + assertEquals(9, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one/five//.\"", null, null); + assertEquals(7, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/one/five//.\"", null, null); + assertEquals(10, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one//five/nine\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one//thirteen/fourteen\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher + .query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one//thirteen/fourteen//.\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one//thirteen/fourteen//.//.\"", null, + null); + assertEquals(1, results.length()); + results.close(); + + // Type search tests + + QName qname = QName.createQName(TEST_NAMESPACE, "int-ista"); + results = searcher + .query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":\"1\"", null, null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "int-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":1", null, null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "int-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":\"01\"", null, + null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "int-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":01", null, null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "int-ista"); + results = searcher + .query(rootNodeRef.getStoreRef(), "lucene", "@" + escapeQName(qname) + ":\"001\"", null, null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "int-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@test\\:int\\-ista:\"0001\"", null, null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "int-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":[0 TO 2]", null, + null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "int-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":{0 TO 1}", null, + null); + assertEquals(0, results.length()); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "int-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":{1 TO 2}", null, + null); + assertEquals(0, results.length()); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "long-ista"); + results = searcher + .query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":\"2\"", null, null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "long-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":\"02\"", null, + null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "long-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":\"002\"", null, + null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "long-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":\"0002\"", null, + null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "long-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":[0 TO 2]", null, + null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "long-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":{0 TO 2}", null, + null); + assertEquals(0, results.length()); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "long-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":{2 TO 3}", null, + null); + assertEquals(0, results.length()); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "float-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":\"3.4\"", null, + null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "float-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":[3 TO 4]", null, + null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "float-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":[3.3 TO 3.4]", + null, null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "float-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":{3.3 TO 3.4}", + null, null); + assertEquals(0, results.length()); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "float-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":\"3.40\"", null, + null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "float-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":\"03.4\"", null, + null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "float-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":\"03.40\"", null, + null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "double-ista")) + ":\"5.6\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "double-ista")) + ":\"05.6\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "double-ista")) + ":\"5.60\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "double-ista")) + ":\"05.60\"", null, null); + assertEquals(1, results.length()); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "double-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":[5.5 TO 5.7]", + null, null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "double-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":{5.5 TO 5.6}", + null, null); + assertEquals(0, results.length()); + results.close(); + + qname = QName.createQName(TEST_NAMESPACE, "double-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(qname) + ":{5.6 TO 5.7}", + null, null); + assertEquals(0, results.length()); + results.close(); + + Date date = new Date(); + String sDate = CachingDateFormat.getDateFormat().format(date); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "date-ista")) + ":\"" + sDate + "\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "datetime-ista")) + ":\"" + sDate + "\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "boolean-ista")) + ":\"true\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "qname-ista")) + ":\"{wibble}wobble\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "category-ista")) + + ":\"" + + DefaultTypeConverter.INSTANCE.convert(String.class, new NodeRef(rootNodeRef.getStoreRef(), + "CategoryId")) + "\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "noderef-ista")) + ":\"" + n1 + "\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "path-ista")) + ":\"" + nodeService.getPath(n3) + "\"", + null, null); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(QName.createQName(TEST_NAMESPACE, "path-ista"))); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "TYPE:\"" + testType.toString() + "\"", null, + null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "TYPE:\"" + testSuperType.toString() + "\"", + null, null); + assertEquals(13, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "ASPECT:\"" + + ISO9075.getXPathName(testAspect) + "\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "ASPECT:\"" + + ISO9075.getXPathName(testSuperAspect) + "\"", null, null); + assertEquals(1, results.length()); + results.close(); + + // FTS test + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "TEXT:\"fox\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "@" + + LuceneQueryParser.escape(ContentModel.PROP_CONTENT.toString()) + ":\"fox\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "@" + + LuceneQueryParser.escape(ContentModel.PROP_CONTENT.toString()) + ".mimetype:\"text/plain\"", null, + null); + assertEquals(1, results.length()); + results.close(); + + QName queryQName = QName.createQName("alf:test1", namespacePrefixResolver); + results = searcher.query(rootNodeRef.getStoreRef(), queryQName, null); + assertEquals(1, results.length()); + results.close(); + + // Parameters + + queryQName = QName.createQName("alf:test2", namespacePrefixResolver); + results = searcher.query(rootNodeRef.getStoreRef(), queryQName, null); + assertEquals(1, results.length()); + results.close(); + + queryQName = QName.createQName("alf:test2", namespacePrefixResolver); + QueryParameter qp = new QueryParameter(QName.createQName("alf:banana", namespacePrefixResolver), "woof"); + results = searcher.query(rootNodeRef.getStoreRef(), queryQName, new QueryParameter[] { qp }); + assertEquals(0, results.length()); + results.close(); + + queryQName = QName.createQName("alf:test3", namespacePrefixResolver); + qp = new QueryParameter(QName.createQName("alf:banana", namespacePrefixResolver), "/one/five//*"); + results = searcher.query(rootNodeRef.getStoreRef(), queryQName, new QueryParameter[] { qp }); + assertEquals(6, results.length()); + results.close(); + + // TODO: should not have a null property type definition + QueryParameterDefImpl paramDef = new QueryParameterDefImpl(QName.createQName("alf:lemur", + namespacePrefixResolver), (DataTypeDefinition) null, true, "fox"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "TEXT:\"${alf:lemur}\"", null, + new QueryParameterDefinition[] { paramDef }); + assertEquals(1, results.length()); + results.close(); + + paramDef = new QueryParameterDefImpl(QName.createQName("alf:intvalue", namespacePrefixResolver), + (DataTypeDefinition) null, true, "1"); + qname = QName.createQName(TEST_NAMESPACE, "int-ista"); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(qname) + ":\"${alf:intvalue}\"", null, new QueryParameterDefinition[] { paramDef }); + assertEquals(1, results.length()); + assertNotNull(results.getRow(0).getValue(qname)); + results.close(); + + } + + public void testPathSearch() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + + // //* + + ResultSet + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//common\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//common\"", null, null); + assertEquals(7, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one//common\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/one//common\"", null, null); + assertEquals(5, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one/five//*\"", null, null); + assertEquals(6, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/one/five//*\"", null, null); + assertEquals(9, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one/five//.\"", null, null); + assertEquals(7, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/one/five//.\"", null, null); + assertEquals(10, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one//five/nine\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one//thirteen/fourteen\"", null, null); + assertEquals(1, results.length()); + results.close(); + luceneFTS.resume(); + } + + public void testXPathSearch() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + + // //* + + ResultSet + + results = searcher.query(rootNodeRef.getStoreRef(), "xpath", "//./*", null, null); + assertEquals(14, results.length()); + results.close(); + luceneFTS.resume(); + + QueryParameterDefinition paramDef = new QueryParameterDefImpl(QName.createQName("alf:query", + namespacePrefixResolver), (DataTypeDefinition) null, true, "//./*"); + results = searcher.query(rootNodeRef.getStoreRef(), "xpath", "${alf:query}", null, + new QueryParameterDefinition[] { paramDef }); + assertEquals(14, results.length()); + results.close(); + } + + public void testMissingIndex() throws Exception + { + luceneFTS.pause(); + StoreRef storeRef = new StoreRef(StoreRef.PROTOCOL_WORKSPACE, "_missing_"); + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(storeRef, indexerAndSearcher); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + + // //* + + ResultSet + + results = searcher.query(storeRef, "xpath", "//./*", null, null); + assertEquals(0, results.length()); + luceneFTS.resume(); + } + + public void testUpdateIndex() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + + runBaseTests(); + + LuceneIndexerImpl2 indexer = LuceneIndexerImpl2.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + + System.currentTimeMillis(), indexerAndSearcher); + indexer.setNodeService(nodeService); + //indexer.setLuceneIndexLock(luceneIndexLock); + indexer.setDictionaryService(dictionaryService); + indexer.setLuceneFullTextSearchIndexer(luceneFTS); + indexer.setContentService(contentService); + + indexer.updateNode(rootNodeRef); + indexer.updateNode(n1); + indexer.updateNode(n2); + indexer.updateNode(n3); + indexer.updateNode(n4); + indexer.updateNode(n5); + indexer.updateNode(n6); + indexer.updateNode(n7); + indexer.updateNode(n8); + indexer.updateNode(n9); + indexer.updateNode(n10); + indexer.updateNode(n11); + indexer.updateNode(n12); + indexer.updateNode(n13); + indexer.updateNode(n14); + + indexer.commit(); + + runBaseTests(); + luceneFTS.resume(); + } + + public void testDeleteLeaf() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + + LuceneIndexerImpl2 indexer = LuceneIndexerImpl2.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + + System.currentTimeMillis(), indexerAndSearcher); + indexer.setNodeService(nodeService); + //indexer.setLuceneIndexLock(luceneIndexLock); + indexer.setDictionaryService(dictionaryService); + indexer.setLuceneFullTextSearchIndexer(luceneFTS); + indexer.setContentService(contentService); + + indexer + .deleteNode(new ChildAssociationRef(ASSOC_TYPE_QNAME, n13, QName.createQName("{namespace}fourteen"), + n14)); + + indexer.commit(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + ResultSet results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:three\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:four\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:eight-0\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:five\"", null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:one\"", null, + null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:two\"", null, + null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:one\"", null, + null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:two\"", null, + null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:five\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:six\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:seven\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:eight-1\"", + null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:eight-2\"", + null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:eight-2\"", + null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:eight-1\"", + null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:eight-0\"", + null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:eight-0\"", + null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:nine\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:ten\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:eleven\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:twelve\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:twelve/namespace:thirteen\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:twelve/namespace:thirteen/namespace:fourteen\"", null, + null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:*/namespace:*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/namespace:*/namespace:*\"", + null, null); + assertEquals(6, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:*/namespace:five\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:*/namespace:*/namespace:*\"", + null, null); + assertEquals(4, results.length()); + results.close(); + results = searcher + .query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:*\"", null, null); + assertEquals(3, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:*/namespace:five/namespace:*\"", null, null); + assertEquals(4, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:*/namespace:nine\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*/*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/*/*\"", null, null); + assertEquals(6, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*/namespace:five\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*/*/*\"", null, null); + assertEquals(4, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/*\"", null, null); + assertEquals(3, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*/namespace:five/*\"", null, null); + assertEquals(4, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/*/namespace:nine\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//.\"", null, null); + assertEquals(14, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//.\"", null, null); + assertEquals(17, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//*\"", null, null); + assertEquals(13, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//*\"", null, null); + assertEquals(16, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//*/.\"", null, null); + assertEquals(13, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//*/.\"", null, null); + assertEquals(16, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//*/./.\"", null, null); + assertEquals(13, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//*/./.\"", null, null); + assertEquals(16, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//./*\"", null, null); + assertEquals(13, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//./*\"", null, null); + assertEquals(16, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//././*/././.\"", null, null); + assertEquals(13, results.length()); + results.close(); + results = searcher + .query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//././*/././.\"", null, null); + assertEquals(16, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//common\"", null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one//common\"", null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one/five//*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/one/five//*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one/five//.\"", null, null); + assertEquals(6, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one//five/nine\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one//thirteen/fourteen\"", null, null); + assertEquals(0, results.length()); + results.close(); + luceneFTS.resume(); + } + + public void testAddEscapedChild() throws Exception + { + String COMPLEX_LOCAL_NAME = " `¬¦!\"£$%^&*()-_=+\t\n\\\u0000[]{};'#:@~,./<>?\\|\u0123\u4567\u8900\uabcd\uefff_xT65A_"; + + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + + LuceneIndexerImpl2 indexer = LuceneIndexerImpl2.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + + System.currentTimeMillis(), indexerAndSearcher); + indexer.setNodeService(nodeService); + //indexer.setLuceneIndexLock(luceneIndexLock); + indexer.setDictionaryService(dictionaryService); + indexer.setLuceneFullTextSearchIndexer(luceneFTS); + indexer.setContentService(contentService); + + ChildAssociationRef car = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName + .createQName("{namespace}" + COMPLEX_LOCAL_NAME), testSuperType); + indexer.createNode(car); + + indexer.commit(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + ResultSet results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:" + + ISO9075.encode(COMPLEX_LOCAL_NAME) + "\"", null, null); + assertEquals(1, results.length()); + results.close(); + } + + public void testNumericInPath() throws Exception + { + String COMPLEX_LOCAL_NAME = "Woof12"; + + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + + LuceneIndexerImpl2 indexer = LuceneIndexerImpl2.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + + System.currentTimeMillis(), indexerAndSearcher); + indexer.setNodeService(nodeService); + //indexer.setLuceneIndexLock(luceneIndexLock); + indexer.setDictionaryService(dictionaryService); + indexer.setLuceneFullTextSearchIndexer(luceneFTS); + indexer.setContentService(contentService); + + ChildAssociationRef car = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName + .createQName("{namespace}" + COMPLEX_LOCAL_NAME), testSuperType); + indexer.createNode(car); + + indexer.commit(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + ResultSet results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:" + + ISO9075.encode(COMPLEX_LOCAL_NAME) + "\"", null, null); + assertEquals(1, results.length()); + results.close(); + } + + public void testDeleteContainer() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + + LuceneIndexerImpl2 indexer = LuceneIndexerImpl2.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + + System.currentTimeMillis(), indexerAndSearcher); + indexer.setNodeService(nodeService); + //indexer.setLuceneIndexLock(luceneIndexLock); + indexer.setDictionaryService(dictionaryService); + indexer.setLuceneFullTextSearchIndexer(luceneFTS); + indexer.setContentService(contentService); + + indexer + .deleteNode(new ChildAssociationRef(ASSOC_TYPE_QNAME, n12, QName.createQName("{namespace}thirteen"), + n13)); + + indexer.commit(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + ResultSet results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:three\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:four\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:eight-0\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:five\"", null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:one\"", null, + null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:two\"", null, + null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:one\"", null, + null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:two\"", null, + null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:five\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:six\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:seven\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:eight-1\"", + null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:eight-2\"", + null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:eight-2\"", + null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:eight-1\"", + null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:eight-0\"", + null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:eight-0\"", + null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:nine\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:ten\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:eleven\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:twelve\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:twelve/namespace:thirteen\"", null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:twelve/namespace:thirteen/namespace:fourteen\"", null, + null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/namespace:*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:*/namespace:*\"", null, null); + assertEquals(4, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/namespace:*/namespace:*\"", + null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:*/namespace:five\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:*/namespace:*/namespace:*\"", + null, null); + assertEquals(4, results.length()); + results.close(); + results = searcher + .query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:*\"", null, null); + assertEquals(3, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:*/namespace:five/namespace:*\"", null, null); + assertEquals(4, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:*/namespace:nine\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*/*\"", null, null); + assertEquals(4, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/*/*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*/namespace:five\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*/*/*\"", null, null); + assertEquals(4, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/*/*/*\"", null, null); + assertEquals(4, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/*\"", null, null); + assertEquals(3, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*/namespace:five/*\"", null, null); + assertEquals(4, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/*/namespace:nine\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//.\"", null, null); + assertEquals(13, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//.\"", null, null); + assertEquals(15, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//*\"", null, null); + assertEquals(12, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//*\"", null, null); + assertEquals(14, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//*/.\"", null, null); + assertEquals(12, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//*/.\"", null, null); + assertEquals(14, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//*/./.\"", null, null); + assertEquals(12, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//*/./.\"", null, null); + assertEquals(14, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//./*\"", null, null); + assertEquals(12, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//./*\"", null, null); + assertEquals(14, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//././*/././.\"", null, null); + assertEquals(12, results.length()); + results.close(); + results = searcher + .query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//././*/././.\"", null, null); + assertEquals(14, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//common\"", null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one//common\"", null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one/five//*\"", null, null); + assertEquals(4, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one/five//.\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one//five/nine\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one//thirteen/fourteen\"", null, null); + assertEquals(0, results.length()); + results.close(); + luceneFTS.resume(); + } + + public void testDeleteAndAddReference() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + + LuceneIndexerImpl2 indexer = LuceneIndexerImpl2.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + + System.currentTimeMillis(), indexerAndSearcher); + indexer.setNodeService(nodeService); + //indexer.setLuceneIndexLock(luceneIndexLock); + indexer.setDictionaryService(dictionaryService); + indexer.setLuceneFullTextSearchIndexer(luceneFTS); + indexer.setContentService(contentService); + + nodeService.removeChild(n2, n13); + indexer.deleteChildRelationship(new ChildAssociationRef(ASSOC_TYPE_QNAME, n2, QName + .createQName("{namespace}link"), n13)); + + indexer.commit(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + ResultSet results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:three\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:four\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:eight-0\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:five\"", null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:one\"", null, + null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:two\"", null, + null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:one\"", null, + null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:two\"", null, + null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:five\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:six\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:seven\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:eight-1\"", + null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:eight-2\"", + null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:eight-2\"", + null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:eight-1\"", + null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:two/namespace:eight-0\"", + null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:eight-0\"", + null, null); + assertEquals(0, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:nine\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:ten\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:eleven\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:twelve\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:twelve/namespace:thirteen\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:five/namespace:twelve/namespace:thirteen/namespace:fourteen\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:*/namespace:*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/namespace:*/namespace:*\"", + null, null); + assertEquals(7, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:*/namespace:five\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:*/namespace:*/namespace:*\"", + null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH_WITH_REPEATS:\"/namespace:*/namespace:*/namespace:*\"", null, null); + assertEquals(6, results.length()); + results.close(); + results = searcher + .query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/namespace:*\"", null, null); + assertEquals(4, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:*/namespace:five/namespace:*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH:\"/namespace:one/namespace:*/namespace:nine\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*/*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/*/*\"", null, null); + assertEquals(7, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*/namespace:five\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*/*/*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/*/*/*\"", null, null); + assertEquals(6, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/*\"", null, null); + assertEquals(4, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/*/namespace:five/*\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/namespace:one/*/namespace:nine\"", null, + null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//.\"", null, null); + assertEquals(15, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//.\"", null, null); + assertEquals(23, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//*\"", null, null); + assertEquals(14, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//*\"", null, null); + assertEquals(22, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//*/.\"", null, null); + assertEquals(14, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//*/.\"", null, null); + assertEquals(22, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//*/./.\"", null, null); + assertEquals(14, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//*/./.\"", null, null); + assertEquals(22, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//./*\"", null, null); + assertEquals(14, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//./*\"", null, null); + assertEquals(22, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//././*/././.\"", null, null); + assertEquals(14, results.length()); + results.close(); + results = searcher + .query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//././*/././.\"", null, null); + assertEquals(22, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//common\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//common\"", null, null); + assertEquals(6, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one//common\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/one//common\"", null, null); + assertEquals(5, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one/five//*\"", null, null); + assertEquals(6, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/one/five//*\"", null, null); + assertEquals(9, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one/five//.\"", null, null); + assertEquals(7, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"/one/five//.\"", null, null); + assertEquals(10, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one//five/nine\"", null, null); + assertEquals(1, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"/one//thirteen/fourteen\"", null, null); + assertEquals(1, results.length()); + results.close(); + + indexer = LuceneIndexerImpl2.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + System.currentTimeMillis(), + indexerAndSearcher); + indexer.setNodeService(nodeService); + //indexer.setLuceneIndexLock(luceneIndexLock); + indexer.setDictionaryService(dictionaryService); + indexer.setLuceneFullTextSearchIndexer(luceneFTS); + indexer.setContentService(contentService); + + nodeService.addChild(n2, n13, ASSOC_TYPE_QNAME, QName.createQName("{namespace}link")); + indexer.createChildRelationship(new ChildAssociationRef(ASSOC_TYPE_QNAME, n2, QName + .createQName("{namespace}link"), n13)); + + indexer.commit(); + + runBaseTests(); + luceneFTS.resume(); + } + + public void testRenameReference() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + + ResultSet results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//namespace:link//.\"", null, + null); + assertEquals(2, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH_WITH_REPEATS:\"//namespace:link//.\"", + null, null); + assertEquals(3, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//namespace:renamed_link//.\"", null, + null); + assertEquals(0, results.length()); + results.close(); + + LuceneIndexerImpl2 indexer = LuceneIndexerImpl2.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + + System.currentTimeMillis(), indexerAndSearcher); + indexer.setNodeService(nodeService); + //indexer.setLuceneIndexLock(luceneIndexLock); + indexer.setDictionaryService(dictionaryService); + indexer.setLuceneFullTextSearchIndexer(luceneFTS); + indexer.setContentService(contentService); + + nodeService.removeChild(n2, n13); + nodeService.addChild(n2, n13, ASSOC_TYPE_QNAME, QName.createQName("{namespace}renamed_link")); + + indexer.updateChildRelationship(new ChildAssociationRef(ASSOC_TYPE_QNAME, n2, QName.createQName("namespace", + "link"), n13), new ChildAssociationRef(ASSOC_TYPE_QNAME, n2, QName.createQName("namespace", + "renamed_link"), n13)); + + indexer.commit(); + + runBaseTests(); + + searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + searcher.setDictionaryService(dictionaryService); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//namespace:link//.\"", null, null); + assertEquals(0, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//namespace:renamed_link//.\"", null, + null); + assertEquals(2, results.length()); + results.close(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", + "PATH_WITH_REPEATS:\"//namespace:renamed_link//.\"", null, null); + assertEquals(3, results.length()); + results.close(); + luceneFTS.resume(); + } + + public void testDelayIndex() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + + ResultSet results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "text-indexed-stored-tokenised-atomic")) + + ":\"KEYONE\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "text-indexed-unstored-tokenised-atomic")) + + ":\"KEYUNSTORED\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "text-indexed-stored-tokenised-nonatomic")) + + ":\"KEYTWO\"", null, null); + assertEquals(0, results.length()); + results.close(); + + // Do index + + LuceneIndexerImpl2 indexer = LuceneIndexerImpl2.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + + System.currentTimeMillis() + "_" + (new Random().nextInt()), indexerAndSearcher); + indexer.setNodeService(nodeService); + //indexer.setLuceneIndexLock(luceneIndexLock); + indexer.setDictionaryService(dictionaryService); + indexer.setLuceneFullTextSearchIndexer(luceneFTS); + indexer.setContentService(contentService); + indexer.updateFullTextSearch(1000); + indexer.prepare(); + indexer.commit(); + + searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + searcher.setDictionaryService(dictionaryService); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "text-indexed-stored-tokenised-atomic")) + + ":\"keyone\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "text-indexed-stored-tokenised-nonatomic")) + + ":\"keytwo\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "text-indexed-unstored-tokenised-atomic")) + + ":\"keyunstored\"", null, null); + assertEquals(1, results.length()); + results.close(); + + runBaseTests(); + luceneFTS.resume(); + } + + public void testWaitForIndex() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + + ResultSet results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "text-indexed-stored-tokenised-atomic")) + + ":\"KEYONE\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "text-indexed-unstored-tokenised-atomic")) + + ":\"KEYUNSTORED\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "text-indexed-stored-tokenised-nonatomic")) + + ":\"KEYTWO\"", null, null); + assertEquals(0, results.length()); + results.close(); + + // Do index + + searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "text-indexed-stored-tokenised-atomic")) + + ":\"keyone\"", null, null); + assertEquals(1, results.length()); + results.close(); + + LuceneIndexerImpl2 indexer = LuceneIndexerImpl2.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + + System.currentTimeMillis() + "_" + (new Random().nextInt()), indexerAndSearcher); + indexer.setNodeService(nodeService); + //indexer.setLuceneIndexLock(luceneIndexLock); + indexer.setDictionaryService(dictionaryService); + indexer.setLuceneFullTextSearchIndexer(luceneFTS); + indexer.setContentService(contentService); + indexer.updateFullTextSearch(1000); + indexer.prepare(); + indexer.commit(); + + luceneFTS.resume(); + // luceneFTS.requiresIndex(rootNodeRef.getStoreRef()); + // luceneFTS.index(); + // luceneFTS.index(); + // luceneFTS.index(); + + Thread.sleep(35000); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "text-indexed-stored-tokenised-nonatomic")) + + ":\"keytwo\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + + escapeQName(QName.createQName(TEST_NAMESPACE, "text-indexed-unstored-tokenised-atomic")) + + ":\"KEYUNSTORED\"", null, null); + assertEquals(1, results.length()); + results.close(); + + runBaseTests(); + } + + private String escapeQName(QName qname) + { + return LuceneQueryParser.escape(qname.toString()); + } + + public void testForKev() throws Exception + { + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + + ResultSet results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PARENT:\"" + + rootNodeRef.toString() + "\"", null, null); + assertEquals(5, results.length()); + results.close(); + + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "+PARENT:\"" + + rootNodeRef.toString() + "\" +QNAME:\"one\"", null, null); + assertEquals(1, results.length()); + results.close(); + + results = searcher + .query( + rootNodeRef.getStoreRef(), + "lucene", + "( +TYPE:\"{http://www.alfresco.org/model/content/1.0}content\" +@\\{http\\://www.alfresco.org/model/content/1.0\\}name:\"content woof\") OR TEXT:\"content\"", + null, null); + + luceneFTS.resume(); + } + + public void testIssueAR47() throws Exception + { + // This bug arose from repeated deletes and adds creating empty index + // segments. + // Two segements each containing one deletyed entry were merged together + // producing a single empty entry. + // This seemed to be bad for lucene - I am not sure why + + // So we add something, add and delete someting repeatedly and then + // check we can still do the search. + + // Running in autocommit against the index + testTX.commit(); + UserTransaction tx = transactionService.getUserTransaction(); + tx.begin(); + ChildAssociationRef testFind = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName + .createQName("{namespace}testFind"), testSuperType); + tx.commit(); + + LuceneSearcherImpl2 searcher = LuceneSearcherImpl2.getSearcher(rootNodeRef.getStoreRef(), indexerAndSearcher); + searcher.setNodeService(nodeService); + searcher.setDictionaryService(dictionaryService); + searcher.setNamespacePrefixResolver(getNamespacePrefixReolsver("namespace")); + searcher.setQueryRegister(queryRegisterComponent); + + ResultSet results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "QNAME:\"namespace:testFind\""); + assertEquals(1, results.length()); + results.close(); + + for (int i = 0; i < 100; i++) + { + UserTransaction tx1 = transactionService.getUserTransaction(); + tx1.begin(); + ChildAssociationRef test = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName + .createQName("{namespace}test"), testSuperType); + tx1.commit(); + + UserTransaction tx2 = transactionService.getUserTransaction(); + tx2.begin(); + nodeService.deleteNode(test.getChildRef()); + tx2.commit(); + } + + UserTransaction tx3 = transactionService.getUserTransaction(); + tx3.begin(); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "QNAME:\"namespace:testFind\""); + assertEquals(1, results.length()); + results.close(); + tx3.commit(); + } + + // Ignore the following test until implementation is completed + + public void testReadAgainstDelta() throws Exception + { + testTX.commit(); + UserTransaction tx = transactionService.getUserTransaction(); + tx.begin(); + luceneFTS.pause(); + buildBaseIndex(); + runBaseTests(); + tx.commit(); + + // Delete + + tx = transactionService.getUserTransaction(); + tx.begin(); + + runBaseTests(); + + serviceRegistry.getNodeService().deleteNode(n1); + + SearchParameters sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.excludeDataInTheCurrentTransaction(false); + ResultSet results = serviceRegistry.getSearchService().query(sp); + assertEquals(5, results.length()); + results.close(); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.excludeDataInTheCurrentTransaction(true); + results = serviceRegistry.getSearchService().query(sp); + assertEquals(15, results.length()); + results.close(); + + tx.rollback(); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.addSort("ID", true); + sp.excludeDataInTheCurrentTransaction(false); + results = serviceRegistry.getSearchService().query(sp); + assertEquals(15, results.length()); + results.close(); + + // Create + + tx = transactionService.getUserTransaction(); + tx.begin(); + + runBaseTests(); + + assertEquals(5, serviceRegistry.getNodeService().getChildAssocs(rootNodeRef).size()); + serviceRegistry.getNodeService().createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, + QName.createQName("{namespace}texas"), testSuperType).getChildRef(); + assertEquals(6, serviceRegistry.getNodeService().getChildAssocs(rootNodeRef).size()); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.excludeDataInTheCurrentTransaction(false); + results = serviceRegistry.getSearchService().query(sp); + assertEquals(16, results.length()); + results.close(); + + tx.rollback(); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.addSort("ID", true); + sp.excludeDataInTheCurrentTransaction(false); + results = serviceRegistry.getSearchService().query(sp); + assertEquals(15, results.length()); + results.close(); + + // update property + + tx = transactionService.getUserTransaction(); + tx.begin(); + + runBaseTests(); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("\\@\\{namespace\\}property\\-1:\"valueone\""); + sp.addSort("ID", true); + sp.excludeDataInTheCurrentTransaction(false); + results = serviceRegistry.getSearchService().query(sp); + + assertEquals(2, results.length()); + results.close(); + + nodeService.setProperty(n1, QName.createQName("{namespace}property-1"), "Different"); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("\\@\\{namespace\\}property\\-1:\"valueone\""); + sp.addSort("ID", true); + sp.excludeDataInTheCurrentTransaction(false); + results = serviceRegistry.getSearchService().query(sp); + + assertEquals(1, results.length()); + results.close(); + + tx.rollback(); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("\\@\\{namespace\\}property\\-1:\"valueone\""); + sp.excludeDataInTheCurrentTransaction(false); + sp.addSort("ID", true); + results = serviceRegistry.getSearchService().query(sp); + + assertEquals(2, results.length()); + results.close(); + + // Add and delete + + tx = transactionService.getUserTransaction(); + tx.begin(); + + runBaseTests(); + + serviceRegistry.getNodeService().deleteNode(n1); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.excludeDataInTheCurrentTransaction(false); + results = serviceRegistry.getSearchService().query(sp); + assertEquals(5, results.length()); + results.close(); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.excludeDataInTheCurrentTransaction(true); + results = serviceRegistry.getSearchService().query(sp); + assertEquals(15, results.length()); + results.close(); + + NodeRef created = serviceRegistry.getNodeService().createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, + QName.createQName("{namespace}texas"), testSuperType).getChildRef(); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.excludeDataInTheCurrentTransaction(false); + results = serviceRegistry.getSearchService().query(sp); + assertEquals(6, results.length()); + results.close(); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.excludeDataInTheCurrentTransaction(true); + results = serviceRegistry.getSearchService().query(sp); + assertEquals(15, results.length()); + results.close(); + + serviceRegistry.getNodeService().deleteNode(created); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.excludeDataInTheCurrentTransaction(false); + results = serviceRegistry.getSearchService().query(sp); + assertEquals(5, results.length()); + results.close(); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.excludeDataInTheCurrentTransaction(true); + results = serviceRegistry.getSearchService().query(sp); + assertEquals(15, results.length()); + results.close(); + + tx.rollback(); + + sp = new SearchParameters(); + sp.addStore(rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"//.\""); + sp.addSort("ID", true); + sp.excludeDataInTheCurrentTransaction(false); + results = serviceRegistry.getSearchService().query(sp); + assertEquals(15, results.length()); + results.close(); + + } + + private void runPerformanceTest(double time, boolean clear) + { + LuceneIndexerImpl2 indexer = LuceneIndexerImpl2.getUpdateIndexer(rootNodeRef.getStoreRef(), "delta" + + System.currentTimeMillis() + "_" + (new Random().nextInt()), indexerAndSearcher); + indexer.setNodeService(nodeService); + //indexer.setLuceneIndexLock(luceneIndexLock); + indexer.setDictionaryService(dictionaryService); + indexer.setLuceneFullTextSearchIndexer(luceneFTS); + indexer.setContentService(contentService); + if (clear) + { + // indexer.clearIndex(); + } + indexer.createNode(new ChildAssociationRef(null, null, null, rootNodeRef)); + + long startTime = System.currentTimeMillis(); + int count = 0; + for (int i = 0; i < 10000000; i++) + { + if (i % 10 == 0) + { + if (System.currentTimeMillis() - startTime > time) + { + count = i; + break; + } + } + + QName qname = QName.createQName("{namespace}a_" + i); + NodeRef ref = nodeService.createNode(rootNodeRef, ASSOC_TYPE_QNAME, qname, ContentModel.TYPE_CONTAINER) + .getChildRef(); + indexer.createNode(new ChildAssociationRef(ASSOC_TYPE_QNAME, rootNodeRef, qname, ref)); + + } + indexer.commit(); + float delta = ((System.currentTimeMillis() - startTime) / 1000.0f); + // System.out.println("\tCreated " + count + " in " + delta + " = " + + // (count / delta)); + } + + private NamespacePrefixResolver getNamespacePrefixReolsver(String defaultURI) + { + DynamicNamespacePrefixResolver nspr = new DynamicNamespacePrefixResolver(null); + nspr.registerNamespace(NamespaceService.ALFRESCO_PREFIX, NamespaceService.ALFRESCO_URI); + nspr.registerNamespace(NamespaceService.CONTENT_MODEL_PREFIX, NamespaceService.CONTENT_MODEL_1_0_URI); + nspr.registerNamespace("namespace", "namespace"); + nspr.registerNamespace("test", TEST_NAMESPACE); + nspr.registerNamespace(NamespaceService.DEFAULT_PREFIX, defaultURI); + return nspr; + } + + public static void main(String[] args) throws Exception + { + LuceneTest2 test = new LuceneTest2(); + test.setUp(); + // test.testForKev(); + // test.testDeleteContainer(); + + // test.testReadAgainstDelta(); + + NodeRef targetNode = test.rootNodeRef; + Path path = test.serviceRegistry.getNodeService().getPath(targetNode); + + SearchParameters sp = new SearchParameters(); + sp.addStore(test.rootNodeRef.getStoreRef()); + sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setQuery("PATH:\"" + path + "//." + "\""); + ResultSet results = test.serviceRegistry.getSearchService().query(sp); + + results.close(); + + // test.dictionaryService.getType(test.nodeService.getType(test.rootNodeRef)).getDefaultAspects(); + } +} diff --git a/source/java/org/alfresco/repo/search/impl/lucene/fts/FullTextSearchIndexerImpl.java b/source/java/org/alfresco/repo/search/impl/lucene/fts/FullTextSearchIndexerImpl.java index 83eccdb198..def19125b7 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/fts/FullTextSearchIndexerImpl.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/fts/FullTextSearchIndexerImpl.java @@ -20,7 +20,10 @@ import java.util.HashSet; import java.util.LinkedHashSet; import java.util.Set; +import org.alfresco.repo.search.Indexer; +import org.alfresco.repo.search.IndexerSPI; import org.alfresco.repo.search.impl.lucene.LuceneIndexer; +import org.alfresco.repo.search.impl.lucene.LuceneIndexerAndSearcher; import org.alfresco.repo.search.impl.lucene.LuceneIndexerAndSearcherFactory; import org.alfresco.service.cmr.repository.StoreRef; import org.springframework.context.ApplicationContext; @@ -36,7 +39,7 @@ public class FullTextSearchIndexerImpl implements FTSIndexerAware, FullTextSearc private static Set indexing = new HashSet(); - LuceneIndexerAndSearcherFactory luceneIndexerAndSearcherFactory; + LuceneIndexerAndSearcher luceneIndexerAndSearcherFactory; private int pauseCount = 0; @@ -161,7 +164,7 @@ public class FullTextSearchIndexerImpl implements FTSIndexerAware, FullTextSearc if (toIndex != null) { //System.out.println("Indexing "+toIndex+" at "+(new java.util.Date())); - LuceneIndexer indexer = luceneIndexerAndSearcherFactory.getIndexer(toIndex); + IndexerSPI indexer = luceneIndexerAndSearcherFactory.getIndexer(toIndex); indexer.registerCallBack(this); indexer.updateFullTextSearch(1000); } @@ -198,7 +201,7 @@ public class FullTextSearchIndexerImpl implements FTSIndexerAware, FullTextSearc return nextStoreRef; } - public void setLuceneIndexerAndSearcherFactory(LuceneIndexerAndSearcherFactory luceneIndexerAndSearcherFactory) + public void setLuceneIndexerAndSearcherFactory(LuceneIndexerAndSearcher luceneIndexerAndSearcherFactory) { this.luceneIndexerAndSearcherFactory = luceneIndexerAndSearcherFactory; } diff --git a/source/java/org/alfresco/repo/search/impl/lucene/index/IndexEntry.java b/source/java/org/alfresco/repo/search/impl/lucene/index/IndexEntry.java index 180d4a2468..9f468d3678 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/index/IndexEntry.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/index/IndexEntry.java @@ -53,7 +53,9 @@ class IndexEntry private long deletions; - IndexEntry(IndexType type, String name, String parentName, TransactionStatus status, String mergeId, long documentCount, long deletions) + private boolean deletOnlyNodes; + + IndexEntry(IndexType type, String name, String parentName, TransactionStatus status, String mergeId, long documentCount, long deletions, boolean deletOnlyNodes) { this.type = type; this.name = name; @@ -62,6 +64,7 @@ class IndexEntry this.mergeId = mergeId; this.documentCount = documentCount; this.deletions = deletions; + this.deletOnlyNodes = deletOnlyNodes; } public String getMergeId() @@ -134,6 +137,16 @@ class IndexEntry this.deletions = deletions; } + public boolean isDeletOnlyNodes() + { + return deletOnlyNodes; + } + + public void setDeletOnlyNodes(boolean deletOnlyNodes) + { + this.deletOnlyNodes = deletOnlyNodes; + } + public String toString() { StringBuilder builder = new StringBuilder(); diff --git a/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java b/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java index e86cc1ca5b..148621084d 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java @@ -48,6 +48,7 @@ import java.util.zip.CRC32; import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.repo.search.IndexerException; import org.alfresco.repo.search.impl.lucene.FilterIndexReaderByNodeRefs; +import org.alfresco.repo.search.impl.lucene.FilterIndexReaderByNodeRefs2; import org.alfresco.service.cmr.repository.NodeRef; import org.alfresco.service.cmr.repository.StoreRef; import org.alfresco.util.GUID; @@ -60,9 +61,13 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Hits; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Searcher; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; -import org.hibernate.criterion.Order; /** * The information that makes up an index. @@ -112,6 +117,8 @@ public class IndexInfo private static String INDEX_INFO_DELETIONS = "IndexInfoDeletions"; + private static String OLD_INDEX = "index"; + /** * Is this index shared by more than one repository? We can make many lock optimisations if the index is not shared. */ @@ -196,12 +203,25 @@ public class IndexInfo private Directory emptyIndex = new RAMDirectory(); + private static HashMap indexInfos = new HashMap(); + + public static synchronized IndexInfo getIndexInfo(File file) + { + IndexInfo indexInfo = indexInfos.get(file); + if (indexInfo == null) + { + indexInfo = new IndexInfo(file); + indexInfos.put(file, indexInfo); + } + return indexInfo; + } + /** * Construct an index in the given directory. * * @param indexDirectory */ - public IndexInfo(File indexDirectory) + private IndexInfo(File indexDirectory) { super(); initialiseTransitions(); @@ -228,6 +248,8 @@ public class IndexInfo { // a spanking new index version = 0; + + } // Open the files and channels @@ -238,7 +260,50 @@ public class IndexInfo this.indexInfoBackupChannel = this.indexInfoBackupRAF.getChannel(); // Read info from disk if this is not a new index. - if (version == -1) + if (version == 0) + { + // Check if an old style index exists + + final File oldIndex = new File(this.indexDirectory, OLD_INDEX); + if (IndexReader.indexExists(oldIndex)) + { + getWriteLock(); + try + { + doWithFileLock(new LockWork() + { + public Object doWork() throws Exception + { + IndexWriter writer; + try + { + writer = new IndexWriter(oldIndex, new StandardAnalyzer(), false); + writer.optimize(); + long docs = writer.docCount(); + writer.close(); + + indexEntries.put(OLD_INDEX, new IndexEntry(IndexType.INDEX, OLD_INDEX, "", + TransactionStatus.COMMITTED, "", docs, 0, false)); + + writeStatus(); + } + catch (IOException e) + { + throw new IndexerException("Failed to optimise old index"); + } + return null; + } + }); + } + finally + { + releaseWriteLock(); + } + + } + } + + else if (version == -1) { getWriteLock(); try @@ -358,6 +423,11 @@ public class IndexInfo */ public IndexReader getDeltaIndexReader(String id) throws IOException { + if (id == null) + { + throw new IndexerException("\"null\" is not a valid identifier for a transaction"); + } + // No read lock required as the delta should be bound to one thread only // Index readers are simply thread safe IndexReader reader = indexReaders.get(id); @@ -390,6 +460,11 @@ public class IndexInfo */ private File ensureDeltaIsRegistered(String id) throws IOException { + if (id == null) + { + throw new IndexerException("\"null\" is not a valid identifier for a transaction"); + } + // A write lock is required if we have to update the local index entries. // There should only be one thread trying to access this delta. File location = new File(indexDirectory, id); @@ -406,8 +481,8 @@ public class IndexInfo // Make sure the index exists if (!indexEntries.containsKey(id)) { - indexEntries.put(id, - new IndexEntry(IndexType.DELTA, id, "", TransactionStatus.ACTIVE, "", 0, 0)); + indexEntries.put(id, new IndexEntry(IndexType.DELTA, id, "", TransactionStatus.ACTIVE, "", 0, + 0, false)); } } finally @@ -426,11 +501,11 @@ public class IndexInfo return location; } - private IndexWriter makeDeltaIndexWriter(File location) throws IOException + private IndexWriter makeDeltaIndexWriter(File location, Analyzer analyzer) throws IOException { if (!IndexReader.indexExists(location)) { - IndexWriter creator = new IndexWriter(location, new StandardAnalyzer(), true); + IndexWriter creator = new IndexWriter(location, analyzer, true); creator.setUseCompoundFile(true); creator.minMergeDocs = 1000; creator.mergeFactor = 5; @@ -442,6 +517,11 @@ public class IndexInfo public IndexWriter getDeltaIndexWriter(String id, Analyzer analyzer) throws IOException { + if (id == null) + { + throw new IndexerException("\"null\" is not a valid identifier for a transaction"); + } + // No read lock required as the delta should be bound to one thread only IndexWriter writer = indexWriters.get(id); if (writer == null) @@ -449,7 +529,7 @@ public class IndexInfo // close index writer if required closeDeltaIndexReader(id); File location = ensureDeltaIsRegistered(id); - writer = makeDeltaIndexWriter(location); + writer = makeDeltaIndexWriter(location, analyzer); if (writer == null) { writer = new IndexWriter(location, analyzer, false); @@ -465,34 +545,50 @@ public class IndexInfo public void closeDeltaIndexReader(String id) throws IOException { + if (id == null) + { + throw new IndexerException("\"null\" is not a valid identifier for a transaction"); + } + // No lock required as the delta applied to one thread. The delta is still active. - IndexReader reader = indexReaders.get(id); + IndexReader reader = indexReaders.remove(id); if (reader != null) { reader.close(); - indexReaders.remove(id); } } public void closeDeltaIndexWriter(String id) throws IOException { + if (id == null) + { + throw new IndexerException("\"null\" is not a valid identifier for a transaction"); + } + // No lock required as the delta applied to one thread. The delta is still active. - IndexWriter writer = indexWriters.get(id); + IndexWriter writer = indexWriters.remove(id); if (writer != null) { writer.close(); - indexWriters.remove(id); } } public void closeDelta(String id) throws IOException { + if (id == null) + { + throw new IndexerException("\"null\" is not a valid identifier for a transaction"); + } closeDeltaIndexReader(id); closeDeltaIndexWriter(id); } public Set getDeletions(String id) throws IOException { + if (id == null) + { + throw new IndexerException("\"null\" is not a valid identifier for a transaction"); + } // Check state Set deletions = new HashSet(); File location = new File(indexDirectory, id); @@ -513,8 +609,13 @@ public class IndexInfo } - public void setPreparedState(String id, Set toDelete, long documents) throws IOException + public void setPreparedState(String id, Set toDelete, long documents, boolean deleteNodesOnly) + throws IOException { + if (id == null) + { + throw new IndexerException("\"null\" is not a valid identifier for a transaction"); + } // Check state if (toDelete.size() > 0) { @@ -544,12 +645,14 @@ public class IndexInfo { throw new IndexerException("Invalid index delta id " + id); } - if (entry.getStatus() != TransactionStatus.PREPARING) + if ((entry.getStatus() != TransactionStatus.PREPARING) + && (entry.getStatus() != TransactionStatus.COMMITTING)) { throw new IndexerException("Deletes and doc count can only be set on a preparing index"); } entry.setDocumentCount(documents); entry.setDeletions(toDelete.size()); + entry.setDeletOnlyNodes(deleteNodesOnly); } finally { @@ -618,9 +721,13 @@ public class IndexInfo } } - public IndexReader getMainIndexReferenceCountingReadOnlyIndexReader(String id, Set deletions) - throws IOException + public IndexReader getMainIndexReferenceCountingReadOnlyIndexReader(String id, Set deletions, + boolean deleteOnlyNodes) throws IOException { + if (id == null) + { + throw new IndexerException("\"null\" is not a valid identifier for a transaction"); + } getReadLock(); try { @@ -677,9 +784,12 @@ public class IndexInfo // TODO: Should use the in memory index but we often end up forcing to disk anyway. // Is it worth it? // luceneIndexer.flushPending(); - IndexReader deltaReader = getDeltaIndexReader(id); + IndexReader deltaReader = ReferenceCountingReadOnlyIndexReaderFactory.createReader(id, + getDeltaIndexReader(id)); + ReferenceCounting deltaRefCount = (ReferenceCounting) deltaReader; + deltaRefCount.incrementReferenceCount(); IndexReader reader = new MultiReader(new IndexReader[] { - new FilterIndexReaderByNodeRefs(mainIndexReader, deletions), deltaReader }); + new FilterIndexReaderByNodeRefs2(mainIndexReader, deletions, deleteOnlyNodes), deltaReader }); return reader; } finally @@ -691,6 +801,10 @@ public class IndexInfo public void setStatus(final String id, final TransactionStatus state, final Set toDelete, final Set read) throws IOException { + if (id == null) + { + throw new IndexerException("\"null\" is not a valid identifier for a transaction"); + } final Transition transition = getTransition(state); getWriteLock(); try @@ -1043,12 +1157,16 @@ public class IndexInfo IndexEntry entry = indexEntries.get(id); if (entry != null) { - throw new IndexerException("TX Already active " + id); + if (entry.getStatus() != TransactionStatus.ACTIVE) + { + throw new IndexerException("TX Already active " + id); + } } if (TransactionStatus.ACTIVE.follows(null)) { - indexEntries.put(id, new IndexEntry(IndexType.DELTA, id, "", TransactionStatus.ACTIVE, "", 0, 0)); + indexEntries + .put(id, new IndexEntry(IndexType.DELTA, id, "", TransactionStatus.ACTIVE, "", 0, 0, false)); } else { @@ -1199,7 +1317,8 @@ public class IndexInfo else if (entry.getType() == IndexType.DELTA) { reader = new MultiReader(new IndexReader[] { - new FilterIndexReaderByNodeRefs(reader, getDeletions(entry.getName())), subReader }); + new FilterIndexReaderByNodeRefs2(reader, getDeletions(entry.getName()), entry + .isDeletOnlyNodes()), subReader }); } } } @@ -1351,10 +1470,14 @@ public class IndexInfo crc32.update((int) (deletions >>> 32) & 0xFFFFFFFF); crc32.update((int) (deletions >>> 0) & 0xFFFFFFFF); + byte deleteOnlyNodesFlag = buffer.get(); + crc32.update(deleteOnlyNodesFlag); + boolean isDeletOnlyNodes = deleteOnlyNodesFlag == 1; + if (!status.isTransient()) { newIndexEntries.put(name, new IndexEntry(indexType, name, parentName, status, mergeId, - documentCount, deletions)); + documentCount, deletions, isDeletOnlyNodes)); } } long onDiskCRC32 = buffer.getLong(); @@ -1465,6 +1588,9 @@ public class IndexInfo buffer.putLong(entry.getDeletions()); crc32.update((int) (entry.getDeletions() >>> 32) & 0xFFFFFFFF); crc32.update((int) (entry.getDeletions() >>> 0) & 0xFFFFFFFF); + + buffer.put(entry.isDeletOnlyNodes() ? (byte) 1 : (byte) 0); + crc32.update(entry.isDeletOnlyNodes() ? new byte[] { (byte) 1 } : new byte[] { (byte) 0 }); } buffer.putLong(crc32.getValue()); @@ -1496,6 +1622,7 @@ public class IndexInfo size += (entry.getMergeId().length()) + 4; size += 8; size += 8; + size += 1; } size += 8; return size; @@ -1589,7 +1716,7 @@ public class IndexInfo ii.closeDeltaIndexWriter(guid); ii.setStatus(guid, TransactionStatus.PREPARING, null, null); - ii.setPreparedState(guid, deletions, docs); + ii.setPreparedState(guid, deletions, docs, false); ii.getDeletions(guid); ii.setStatus(guid, TransactionStatus.PREPARED, null, null); ii.setStatus(guid, TransactionStatus.COMMITTING, null, null); @@ -1937,7 +2064,7 @@ public class IndexInfo final HashSet invalidIndexes = new HashSet(); final HashMap newIndexCounts = new HashMap(); - + try { LinkedHashMap readers = new LinkedHashMap(); @@ -1964,9 +2091,31 @@ public class IndexInfo IndexReader reader = readers.get(key); for (NodeRef nodeRef : deletions) { - if (reader.delete(new Term("ID", nodeRef.toString())) > 0) + if (currentDelete.isDeletOnlyNodes()) { - invalidIndexes.add(key); + Searcher searcher = new IndexSearcher(reader); + + BooleanQuery query = new BooleanQuery(); + query.add(new TermQuery(new Term("ID", nodeRef.toString())), true, false); + query.add(new TermQuery(new Term("ISNODE", "T")), false, false); + Hits hits = searcher.search(query); + if (hits.length() > 0) + { + for (int i = 0; i < hits.length(); i++) + { + reader.delete(hits.id(i)); + invalidIndexes.add(key); + } + } + searcher.close(); + + } + else + { + if (reader.delete(new Term("ID", nodeRef.toString())) > 0) + { + invalidIndexes.add(key); + } } } @@ -2016,14 +2165,14 @@ public class IndexInfo } } - - for(String key : newIndexCounts.keySet()) + + for (String key : newIndexCounts.keySet()) { Long newCount = newIndexCounts.get(key); IndexEntry entry = indexEntries.get(key); entry.setDocumentCount(newCount); } - + writeStatus(); for (String id : invalidIndexes) @@ -2051,7 +2200,7 @@ public class IndexInfo } mainIndexReader = null; } - + if (s_logger.isDebugEnabled()) { for (String id : toDelete.keySet()) @@ -2064,13 +2213,12 @@ public class IndexInfo } s_logger.debug("...deleting done"); } - + dumpInfo(); - + return null; } - }); } finally @@ -2078,8 +2226,6 @@ public class IndexInfo releaseWriteLock(); } - - // TODO: Flush readers etc } @@ -2147,7 +2293,7 @@ public class IndexInfo if (set.size() > 0) { IndexEntry target = new IndexEntry(IndexType.INDEX, guid, "", - TransactionStatus.MERGE_TARGET, guid, count, 0); + TransactionStatus.MERGE_TARGET, guid, count, 0, false); set.put(guid, target); // rebuild merged index elements LinkedHashMap reordered = new LinkedHashMap(); @@ -2287,7 +2433,7 @@ public class IndexInfo } dumpInfo(); - + writeStatus(); clearOldReaders(); diff --git a/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfoTest.java b/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfoTest.java index e834c21c83..eefa22a92d 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfoTest.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfoTest.java @@ -85,7 +85,7 @@ public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", " File tempLocation = TempFileProvider.getTempDir(); File testArea = new File(tempLocation, "IndexInfoTest"); File testDir = new File(testArea, "" + System.currentTimeMillis()); - final IndexInfo ii = new IndexInfo(testDir); + final IndexInfo ii = IndexInfo.getIndexInfo(testDir); for (int i = 0; i < WORD_LIST.length; i++) { @@ -107,7 +107,7 @@ public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", " ii.closeDeltaIndexWriter(guid); ii.setStatus(guid, TransactionStatus.PREPARING, null, null); - ii.setPreparedState(guid, deletions, 1); + ii.setPreparedState(guid, deletions, 1, false); ii.getDeletions(guid); ii.setStatus(guid, TransactionStatus.PREPARED, null, null); @@ -128,7 +128,7 @@ public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", " } reader.close(); - reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, deletions); + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, deletions, false); assertEquals(reader.numDocs(), i + 1); for (int j = 0; j < WORD_LIST.length; j++) { @@ -183,7 +183,7 @@ public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", " File tempLocation = TempFileProvider.getTempDir(); File testArea = new File(tempLocation, "IndexInfoTest"); File testDir = new File(testArea, "" + System.currentTimeMillis()); - final IndexInfo ii = new IndexInfo(testDir); + final IndexInfo ii = IndexInfo.getIndexInfo(testDir); for (int i = 0; i < CREATE_LIST.length; i++) { @@ -208,7 +208,7 @@ public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", " ii.closeDeltaIndexWriter(guid); ii.setStatus(guid, TransactionStatus.PREPARING, null, null); - ii.setPreparedState(guid, new HashSet(), 1); + ii.setPreparedState(guid, new HashSet(), 1, false); ii.getDeletions(guid); ii.setStatus(guid, TransactionStatus.PREPARED, null, null); @@ -229,7 +229,7 @@ public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", " } reader.close(); - reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, new HashSet()); + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, new HashSet(), false); assertEquals(reader.numDocs(), i + 1); for (int j = 0; j < CREATE_LIST.length; j++) { @@ -281,7 +281,7 @@ public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", " ii.setStatus(guid, TransactionStatus.ACTIVE, null, null); ii.closeDeltaIndexWriter(guid); ii.setStatus(guid, TransactionStatus.PREPARING, null, null); - ii.setPreparedState(guid, deletions, 1); + ii.setPreparedState(guid, deletions, 1, false); ii.getDeletions(guid); ii.setStatus(guid, TransactionStatus.PREPARED, null, null); @@ -304,7 +304,7 @@ public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", " } reader.close(); - reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, deletions); + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, deletions, false); assertEquals(reader.numDocs(), UPDATE_LIST.length - i - 1); lastDoc = -1; for (int j = 0; j < CREATE_LIST.length; j++) @@ -372,7 +372,7 @@ public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", " File tempLocation = TempFileProvider.getTempDir(); File testArea = new File(tempLocation, "IndexInfoTest"); File testDir = new File(testArea, "" + System.currentTimeMillis()); - final IndexInfo ii = new IndexInfo(testDir); + final IndexInfo ii = IndexInfo.getIndexInfo(testDir); for (int i = 0; i < CREATE_LIST.length; i++) { @@ -397,7 +397,7 @@ public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", " ii.closeDeltaIndexWriter(guid); ii.setStatus(guid, TransactionStatus.PREPARING, null, null); - ii.setPreparedState(guid, new HashSet(), 1); + ii.setPreparedState(guid, new HashSet(), 1, false); ii.getDeletions(guid); ii.setStatus(guid, TransactionStatus.PREPARED, null, null); @@ -418,7 +418,7 @@ public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", " } reader.close(); - reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, new HashSet()); + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, new HashSet(), false); assertEquals(reader.numDocs(), i + 1); for (int j = 0; j < CREATE_LIST.length; j++) { @@ -480,7 +480,7 @@ public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", " ii.closeDeltaIndexWriter(guid); ii.setStatus(guid, TransactionStatus.PREPARING, null, null); - ii.setPreparedState(guid, deletions, 1); + ii.setPreparedState(guid, deletions, 1, false); ii.getDeletions(guid); ii.setStatus(guid, TransactionStatus.PREPARED, null, null); @@ -517,7 +517,7 @@ public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", " } reader.close(); - reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, deletions); + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, deletions, false); assertEquals(reader.numDocs(), UPDATE_LIST.length); lastDoc = -1; for (int j = 0; j < CREATE_LIST.length; j++) @@ -599,7 +599,7 @@ public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", " File tempLocation = TempFileProvider.getTempDir(); File testArea = new File(tempLocation, "IndexInfoTest"); File testDir = new File(testArea, "" + System.currentTimeMillis()); - final IndexInfo ii = new IndexInfo(testDir); + final IndexInfo ii = IndexInfo.getIndexInfo(testDir); Thread thread1 = new Thread(new Test(ii, CREATE_LIST, UPDATE_LIST)); Thread thread2 = new Thread(new Test(ii, CREATE_LIST_2, UPDATE_LIST_2)); @@ -663,7 +663,7 @@ public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", " ii.closeDeltaIndexWriter(guid); ii.setStatus(guid, TransactionStatus.PREPARING, null, null); - ii.setPreparedState(guid, new HashSet(), 1); + ii.setPreparedState(guid, new HashSet(), 1, false); ii.getDeletions(guid); ii.setStatus(guid, TransactionStatus.PREPARED, null, null); @@ -687,7 +687,7 @@ public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", " } reader.close(); - reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, new HashSet()); + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, new HashSet(), false); lastDoc = -1; for (int j = 0; j < create.length; j++) { @@ -751,7 +751,7 @@ public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", " ii.closeDeltaIndexWriter(guid); ii.setStatus(guid, TransactionStatus.PREPARING, null, null); - ii.setPreparedState(guid, deletions, 1); + ii.setPreparedState(guid, deletions, 1, false); ii.getDeletions(guid); ii.setStatus(guid, TransactionStatus.PREPARED, null, null); @@ -788,7 +788,7 @@ public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", " } reader.close(); - reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, deletions); + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, deletions, false); lastDoc = -1; for (int j = 0; j < create.length; j++) diff --git a/source/java/org/alfresco/repo/transaction/AlfrescoTransactionSupport.java b/source/java/org/alfresco/repo/transaction/AlfrescoTransactionSupport.java index 1e7885a43a..c2df1a06a9 100644 --- a/source/java/org/alfresco/repo/transaction/AlfrescoTransactionSupport.java +++ b/source/java/org/alfresco/repo/transaction/AlfrescoTransactionSupport.java @@ -26,6 +26,7 @@ import java.util.Set; import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.repo.node.db.NodeDaoService; import org.alfresco.repo.node.integrity.IntegrityChecker; +import org.alfresco.repo.search.impl.lucene.LuceneIndexerAndSearcher; import org.alfresco.repo.search.impl.lucene.LuceneIndexerAndSearcherFactory; import org.alfresco.service.cmr.rule.RuleService; import org.alfresco.util.GUID; @@ -250,7 +251,7 @@ public abstract class AlfrescoTransactionSupport * @param indexerAndSearcher the Lucene indexer to perform transaction completion * tasks on */ - public static void bindLucene(LuceneIndexerAndSearcherFactory indexerAndSearcher) + public static void bindLucene(LuceneIndexerAndSearcher indexerAndSearcher) { // get transaction-local synchronization TransactionSynchronizationImpl synch = getSynchronization(); @@ -426,7 +427,7 @@ public abstract class AlfrescoTransactionSupport private final String txnId; private final Set nodeDaoServices; private final Set integrityCheckers; - private final Set lucenes; + private final Set lucenes; private final Set listeners; private final Map resources; @@ -440,7 +441,7 @@ public abstract class AlfrescoTransactionSupport this.txnId = txnId; nodeDaoServices = new HashSet(3); integrityCheckers = new HashSet(3); - lucenes = new HashSet(3); + lucenes = new HashSet(3); listeners = new HashSet(5); resources = new HashMap(17); } @@ -472,7 +473,7 @@ public abstract class AlfrescoTransactionSupport * @return Returns a set of LuceneIndexerAndSearcherFactory that will be called * during end-of-transaction processing */ - public Set getLucenes() + public Set getLucenes() { return lucenes; } @@ -589,7 +590,7 @@ public abstract class AlfrescoTransactionSupport // flush flush(); // prepare the indexes - for (LuceneIndexerAndSearcherFactory lucene : lucenes) + for (LuceneIndexerAndSearcher lucene : lucenes) { lucene.prepare(); } @@ -630,7 +631,7 @@ public abstract class AlfrescoTransactionSupport } // commit/rollback Lucene - for (LuceneIndexerAndSearcherFactory lucene : lucenes) + for (LuceneIndexerAndSearcher lucene : lucenes) { try {