inteligr8/alfresco-community-repo
1
0
Fork 0
mirror of https://github.com/Alfresco/alfresco-community-repo.git synced 2025-07-31 17:39:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add sorting support for text and mltext

Browse Source 
git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@12395 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261
This commit is contained in:
Andrew Hind
2008-12-15 16:08:30 +00:00
parent 41f9e4a380
commit 6bda204c4d
6 changed files with 462 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
config/alfresco/model/contentModel.xml
View File

@@ -32,6 +32,11 @@
<title>Name</title> <title>Name</title>
<type>d:text</type> <type>d:text</type>
<mandatory enforced="true">true</mandatory> <mandatory enforced="true">true</mandatory>
<index enabled="true">
<atomic>true</atomic>
<stored>false</stored>
<tokenised>both</tokenised>
</index>
<constraints> <constraints>
<constraint ref="cm:filename" /> <constraint ref="cm:filename" />
</constraints> </constraints>
@@ -373,10 +378,20 @@
<property name="cm:title"> <property name="cm:title">
<title>Title</title> <title>Title</title>
<type>d:mltext</type> <type>d:mltext</type>
<index enabled="true">
<atomic>true</atomic>
<stored>false</stored>
<tokenised>both</tokenised>
</index>
</property> </property>
<property name="cm:description"> <property name="cm:description">
<title>Description</title> <title>Description</title>
<type>d:mltext</type> <type>d:mltext</type>
<index enabled="true">
<atomic>true</atomic>
<stored>false</stored>
<tokenised>both</tokenised>
</index>
</property> </property>
</properties> </properties>
</aspect> </aspect>

3
source/java/org/alfresco/repo/search/MLAnalysisMode.java
View File

@@ -27,6 +27,7 @@ package org.alfresco.repo.search;
import java.util.Collection; import java.util.Collection;
import java.util.HashSet; import java.util.HashSet;
import java.util.Locale; import java.util.Locale;
import java.util.Set;
import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.error.AlfrescoRuntimeException;
@@ -720,7 +721,7 @@ public enum MLAnalysisMode
public abstract boolean includesExactCountryMatch(); public abstract boolean includesExactCountryMatch();
public static Collection<Locale> getLocales(MLAnalysisMode mlAnalaysisMode, Locale locale, boolean withWildcards) public static Set<Locale> getLocales(MLAnalysisMode mlAnalaysisMode, Locale locale, boolean withWildcards)
{ {
HashSet<Locale> locales = new HashSet<Locale>(); HashSet<Locale> locales = new HashSet<Locale>();

207
source/java/org/alfresco/repo/search/impl/lucene/ADMLuceneIndexerImpl.java
View File

@@ -50,7 +50,10 @@ import org.alfresco.repo.content.MimetypeMap;
import org.alfresco.repo.content.transform.ContentTransformer; import org.alfresco.repo.content.transform.ContentTransformer;
import org.alfresco.repo.dictionary.IndexTokenisationMode; import org.alfresco.repo.dictionary.IndexTokenisationMode;
import org.alfresco.repo.search.IndexerException; import org.alfresco.repo.search.IndexerException;
import org.alfresco.repo.search.MLAnalysisMode;
import org.alfresco.repo.search.impl.lucene.analysis.DateTimeAnalyser; import org.alfresco.repo.search.impl.lucene.analysis.DateTimeAnalyser;
import org.alfresco.repo.search.impl.lucene.analysis.MLTokenDuplicator;
import org.alfresco.repo.search.impl.lucene.analysis.VerbatimAnalyser;
import org.alfresco.repo.search.impl.lucene.fts.FTSIndexerAware; import org.alfresco.repo.search.impl.lucene.fts.FTSIndexerAware;
import org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer; import org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer;
import org.alfresco.repo.tenant.TenantService; import org.alfresco.repo.tenant.TenantService;
@@ -89,6 +92,7 @@ import org.apache.lucene.index.TermDocs;
import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Hits; import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Searcher; import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanClause.Occur;
@@ -393,7 +397,7 @@ public class ADMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<NodeRef> imp
indexer.initialise(storeRef, deltaId); indexer.initialise(storeRef, deltaId);
return indexer; return indexer;
} }
public static ADMLuceneNoActionIndexerImpl getNoActionIndexer(StoreRef storeRef, String deltaId, LuceneConfig config) throws LuceneIndexException public static ADMLuceneNoActionIndexerImpl getNoActionIndexer(StoreRef storeRef, String deltaId, LuceneConfig config) throws LuceneIndexException
{ {
if (s_logger.isDebugEnabled()) if (s_logger.isDebugEnabled())
@@ -814,6 +818,8 @@ public class ADMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<NodeRef> imp
if (isContent) if (isContent)
{ {
// Content is always tokenised
ContentData contentData = DefaultTypeConverter.INSTANCE.convert(ContentData.class, serializableValue); ContentData contentData = DefaultTypeConverter.INSTANCE.convert(ContentData.class, serializableValue);
if (!index || contentData.getMimetype() == null) if (!index || contentData.getMimetype() == null)
{ {
@@ -909,17 +915,6 @@ public class ADMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<NodeRef> imp
if (readerReady) if (readerReady)
{ {
InputStreamReader isr = null; InputStreamReader isr = null;
// InputStream ris = reader.getContentInputStream();
// try
// {
// isr = new InputStreamReader(ris, "UTF-8");
// }
// catch (UnsupportedEncodingException e)
// {
// isr = new InputStreamReader(ris);
// }
// doc.add(new Field("TEXT", isr, Field.TermVector.NO));
InputStream ris = reader.getReader().getContentInputStream(); InputStream ris = reader.getReader().getContentInputStream();
try try
{ {
@@ -984,9 +979,85 @@ public class ADMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<NodeRef> imp
for (Locale locale : mlText.getLocales()) for (Locale locale : mlText.getLocales())
{ {
String localeString = mlText.getValue(locale); String localeString = mlText.getValue(locale);
StringBuilder builder = new StringBuilder(); StringBuilder builder;
builder.append("\u0000").append(locale.toString()).append("\u0000").append(localeString); MLAnalysisMode analysisMode;
doc.add(new Field(attributeName, builder.toString(), fieldStore, fieldIndex, Field.TermVector.NO)); VerbatimAnalyser vba;
MLTokenDuplicator duplicator;
Token t;
switch (tokenise)
{
case TRUE:
builder = new StringBuilder();
builder.append("\u0000").append(locale.toString()).append("\u0000").append(localeString);
doc.add(new Field(attributeName, builder.toString(), fieldStore, fieldIndex, Field.TermVector.NO));
break;
case FALSE:
// analyse ml text
analysisMode = getLuceneConfig().getDefaultMLIndexAnalysisMode();
// Do the analysis here
vba = new VerbatimAnalyser(false);
duplicator = new MLTokenDuplicator(vba.tokenStream(attributeName, new StringReader(localeString)), locale, null, analysisMode);
try
{
while ((t = duplicator.next()) != null)
{
String localeText = "";
if (t.termText().indexOf('{') == 0)
{
int end = t.termText().indexOf('}', 1);
if (end != -1)
{
localeText = t.termText().substring(1, end);
}
}
if (localeText.length() > 0)
{
doc.add(new Field(attributeName + "." + localeText + ".sort", t.termText(), Field.Store.NO, Field.Index.NO_NORMS, Field.TermVector.NO));
}
}
}
catch (IOException e)
{
// TODO ??
}
break;
case BOTH:
builder = new StringBuilder();
builder.append("\u0000").append(locale.toString()).append("\u0000").append(localeString);
doc.add(new Field(attributeName, builder.toString(), fieldStore, fieldIndex, Field.TermVector.NO));
// analyse ml text
analysisMode = getLuceneConfig().getDefaultMLIndexAnalysisMode();
// Do the analysis here
vba = new VerbatimAnalyser(false);
duplicator = new MLTokenDuplicator(vba.tokenStream(attributeName, new StringReader(localeString)), locale, null, analysisMode);
try
{
while ((t = duplicator.next()) != null)
{
String localeText = "";
if (t.termText().indexOf('{') == 0)
{
int end = t.termText().indexOf('}', 1);
if (end != -1)
{
localeText = t.termText().substring(1, end);
}
}
if (localeText.length() > 0)
{
doc.add(new Field(attributeName + "." + localeText + ".sort", t.termText(), Field.Store.NO, Field.Index.NO_NORMS, Field.TermVector.NO));
}
}
}
catch (IOException e)
{
// TODO ??
}
break;
}
} }
} }
else if (isText) else if (isText)
@@ -1013,6 +1084,10 @@ public class ADMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<NodeRef> imp
locale = I18NUtil.getLocale(); locale = I18NUtil.getLocale();
} }
StringBuilder builder; StringBuilder builder;
MLAnalysisMode analysisMode;
VerbatimAnalyser vba;
MLTokenDuplicator duplicator;
Token t;
switch (tokenise) switch (tokenise)
{ {
default: default:
@@ -1020,37 +1095,109 @@ public class ADMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<NodeRef> imp
builder = new StringBuilder(); builder = new StringBuilder();
builder.append("\u0000").append(locale.toString()).append("\u0000").append(strValue); builder.append("\u0000").append(locale.toString()).append("\u0000").append(strValue);
doc.add(new Field(attributeName, builder.toString(), fieldStore, fieldIndex, Field.TermVector.NO)); doc.add(new Field(attributeName, builder.toString(), fieldStore, fieldIndex, Field.TermVector.NO));
break; break;
case FALSE: case FALSE:
doc.add(new Field(attributeName, strValue, fieldStore, fieldIndex, Field.TermVector.NO)); analysisMode = getLuceneConfig().getDefaultMLIndexAnalysisMode();
// Do the analysis here
vba = new VerbatimAnalyser(false);
duplicator = new MLTokenDuplicator(vba.tokenStream(attributeName, new StringReader(strValue)), locale, null, analysisMode);
try
{
while ((t = duplicator.next()) != null)
{
String localeText = "";
if (t.termText().indexOf('{') == 0)
{
int end = t.termText().indexOf('}', 1);
if (end != -1)
{
localeText = t.termText().substring(1, end);
}
}
if (localeText.length() > 0)
{
doc.add(new Field(attributeName + "." + localeText + ".sort", t.termText(), Field.Store.NO, Field.Index.NO_NORMS, Field.TermVector.NO));
}
}
}
catch (IOException e)
{
// TODO ??
}
break; break;
case BOTH: case BOTH:
builder = new StringBuilder(); builder = new StringBuilder();
builder.append("\u0000").append(locale.toString()).append("\u0000").append(strValue); builder.append("\u0000").append(locale.toString()).append("\u0000").append(strValue);
doc.add(new Field(attributeName, builder.toString(), fieldStore, fieldIndex, Field.TermVector.NO)); doc.add(new Field(attributeName, builder.toString(), fieldStore, fieldIndex, Field.TermVector.NO));
doc.add(new Field(attributeName, strValue, fieldStore, fieldIndex, Field.TermVector.NO)); analysisMode = getLuceneConfig().getDefaultMLIndexAnalysisMode();
// Do the analysis here
vba = new VerbatimAnalyser(false);
duplicator = new MLTokenDuplicator(vba.tokenStream(attributeName, new StringReader(strValue)), locale, null, analysisMode);
try
{
while ((t = duplicator.next()) != null)
{
String localeText = "";
if (t.termText().indexOf('{') == 0)
{
int end = t.termText().indexOf('}', 1);
if (end != -1)
{
localeText = t.termText().substring(1, end);
}
}
if (localeText.length() > 0)
{
doc.add(new Field(attributeName + "." + localeText + ".sort", t.termText(), Field.Store.NO, Field.Index.NO_NORMS, Field.TermVector.NO));
}
}
}
catch (IOException e)
{
// TODO ??
}
break; break;
} }
} }
else if (isDateTime) else if (isDateTime)
{ {
doc.add(new Field(attributeName, strValue, fieldStore, fieldIndex, Field.TermVector.NO)); SimpleDateFormat df;
SimpleDateFormat df = CachingDateFormat.getDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS", true);
Date date; Date date;
try switch (tokenise)
{ {
date = df.parse(strValue); default:
doc.add(new Field(attributeName + ".sort", df.format(date), Field.Store.NO, Field.Index.NO_NORMS, Field.TermVector.NO)); case TRUE:
doc.add(new Field(attributeName, strValue, fieldStore, fieldIndex, Field.TermVector.NO));
break;
case FALSE:
df = CachingDateFormat.getDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS", true);
try
{
date = df.parse(strValue);
doc.add(new Field(attributeName + ".sort", df.format(date), Field.Store.NO, Field.Index.NO_NORMS, Field.TermVector.NO));
}
catch (ParseException e)
{
// ignore for ordering
}
break;
case BOTH:
doc.add(new Field(attributeName, strValue, fieldStore, fieldIndex, Field.TermVector.NO));
df = CachingDateFormat.getDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS", true);
try
{
date = df.parse(strValue);
doc.add(new Field(attributeName + ".sort", df.format(date), Field.Store.NO, Field.Index.NO_NORMS, Field.TermVector.NO));
}
catch (ParseException e)
{
// ignore for ordering
}
break;
} }
catch (ParseException e)
{
// ignore for ordering
}
} }
else else
{ {

133
source/java/org/alfresco/repo/search/impl/lucene/ADMLuceneSearcherImpl.java
View File

@@ -26,22 +26,30 @@ package org.alfresco.repo.search.impl.lucene;
import java.io.IOException; import java.io.IOException;
import java.io.Serializable; import java.io.Serializable;
import java.io.StringReader;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.ListIterator; import java.util.ListIterator;
import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import org.alfresco.i18n.I18NUtil;
import org.alfresco.repo.search.CannedQueryDef; import org.alfresco.repo.search.CannedQueryDef;
import org.alfresco.repo.search.EmptyResultSet; import org.alfresco.repo.search.EmptyResultSet;
import org.alfresco.repo.search.MLAnalysisMode;
import org.alfresco.repo.search.QueryRegisterComponent; import org.alfresco.repo.search.QueryRegisterComponent;
import org.alfresco.repo.search.SearcherException; import org.alfresco.repo.search.SearcherException;
import org.alfresco.repo.search.impl.NodeSearcher; import org.alfresco.repo.search.impl.NodeSearcher;
import org.alfresco.repo.search.impl.lucene.QueryParser.Operator; import org.alfresco.repo.search.impl.lucene.QueryParser.Operator;
import org.alfresco.repo.search.impl.lucene.analysis.DateTimeAnalyser; import org.alfresco.repo.search.impl.lucene.analysis.DateTimeAnalyser;
import org.alfresco.repo.search.impl.lucene.analysis.MLTokenDuplicator;
import org.alfresco.repo.search.impl.lucene.analysis.VerbatimAnalyser;
import org.alfresco.repo.search.results.SortedResultSet; import org.alfresco.repo.search.results.SortedResultSet;
import org.alfresco.repo.tenant.TenantService; import org.alfresco.repo.tenant.TenantService;
import org.alfresco.service.cmr.dictionary.DataTypeDefinition; import org.alfresco.service.cmr.dictionary.DataTypeDefinition;
@@ -68,6 +76,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term; import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermEnum; import org.apache.lucene.index.TermEnum;
import org.apache.lucene.index.IndexReader.FieldOption;
import org.apache.lucene.search.Hits; import org.apache.lucene.search.Hits;
import org.apache.lucene.search.Query; import org.apache.lucene.search.Query;
import org.apache.lucene.search.Searcher; import org.apache.lucene.search.Searcher;
@@ -269,13 +278,77 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
switch (sd.getSortType()) switch (sd.getSortType())
{ {
case FIELD: case FIELD:
Locale sortLocale = null;
String field = sd.getField(); String field = sd.getField();
if (field.startsWith("@")) if (field.startsWith("@"))
{ {
field = expandAttributeFieldName(field); field = expandAttributeFieldName(field);
PropertyDefinition propertyDef = getDictionaryService().getProperty(QName.createQName(field.substring(1))); PropertyDefinition propertyDef = getDictionaryService().getProperty(QName.createQName(field.substring(1)));
if (propertyDef.getDataType().getName().equals(DataTypeDefinition.DATETIME)) if (propertyDef.getDataType().getName().equals(DataTypeDefinition.CONTENT))
{
throw new SearcherException("Order on content properties is not curently supported");
}
else if ((propertyDef.getDataType().getName().equals(DataTypeDefinition.MLTEXT)) || (propertyDef.getDataType().getName().equals(DataTypeDefinition.TEXT)))
{
List<Locale> locales = searchParameters.getLocales();
if (((locales == null) || (locales.size() == 0)))
{
locales = Collections.singletonList(I18NUtil.getLocale());
}
if (locales.size() > 1)
{
throw new SearcherException("Order on text/mltext properties with more than one locale is not curently supported");
}
sortLocale = locales.get(0);
// find best field match
MLAnalysisMode analysisMode = getLuceneConfig().getDefaultMLSearchAnalysisMode();
HashSet<String> allowableLocales = new HashSet<String>();
for (Locale l : MLAnalysisMode.getLocales(analysisMode, sortLocale, false))
{
allowableLocales.add(l.toString());
}
String sortField = field;
for (Object current : searcher.getReader().getFieldNames(FieldOption.INDEXED))
{
String currentString = (String) current;
if (currentString.startsWith(field) && currentString.endsWith(".sort"))
{
String fieldLocale = currentString.substring(field.length() + 1, currentString.length() - 5);
if (allowableLocales.contains(fieldLocale))
{
if (fieldLocale.equals(sortLocale.toString()))
{
sortField = currentString;
break;
}
else if (sortLocale.toString().startsWith(fieldLocale))
{
if (sortField.equals(field) || (currentString.length() < sortField.length()))
{
sortField = currentString;
}
}
else if (fieldLocale.startsWith(sortLocale.toString()))
{
if (sortField.equals(field) || (currentString.length() < sortField.length()))
{
sortField = currentString;
}
}
}
}
}
field = sortField;
}
else if (propertyDef.getDataType().getName().equals(DataTypeDefinition.DATETIME))
{ {
DataTypeDefinition dataType = propertyDef.getDataType(); DataTypeDefinition dataType = propertyDef.getDataType();
String analyserClassName = dataType.getAnalyserClassName(); String analyserClassName = dataType.getAnalyserClassName();
@@ -292,7 +365,7 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
} }
if (fieldHasTerm(searcher.getReader(), field)) if (fieldHasTerm(searcher.getReader(), field))
{ {
fields[index++] = new SortField(field, !sd.isAscending()); fields[index++] = new SortField(field, sortLocale, !sd.isAscending());
} }
else else
{ {
@@ -309,7 +382,7 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
} }
hits = searcher.search(query, new Sort(fields)); hits = searcher.search(query, new Sort(fields));
} }
else else
{ {
@@ -318,7 +391,7 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
Path[] paths = searchParameters.getAttributePaths().toArray(new Path[0]); Path[] paths = searchParameters.getAttributePaths().toArray(new Path[0]);
ResultSet rs = new LuceneResultSet(hits, searcher, nodeService, tenantService, paths, searchParameters, getLuceneConfig()); ResultSet rs = new LuceneResultSet(hits, searcher, nodeService, tenantService, paths, searchParameters, getLuceneConfig());
if(requiresPostSort) if (requiresPostSort)
{ {
ResultSet sorted = new SortedResultSet(rs, nodeService, searchParameters, namespacePrefixResolver); ResultSet sorted = new SortedResultSet(rs, nodeService, searchParameters, namespacePrefixResolver);
return sorted; return sorted;
@@ -359,7 +432,8 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
return new EmptyResultSet(); return new EmptyResultSet();
} }
Hits hits = searcher.search(query); Hits hits = searcher.search(query);
return new LuceneResultSet(hits, searcher, nodeService, tenantService, searchParameters.getAttributePaths().toArray(new Path[0]), searchParameters, getLuceneConfig()); return new LuceneResultSet(hits, searcher, nodeService, tenantService, searchParameters.getAttributePaths().toArray(new Path[0]), searchParameters,
getLuceneConfig());
} }
catch (SAXPathException e) catch (SAXPathException e)
{ {
@@ -757,7 +831,7 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
Term term = terms.term(); Term term = terms.term();
if (term != null) if (term != null)
{ {
if(!term.field().equals(field)) if (!term.field().equals(field))
{ {
break; break;
} }
@@ -765,26 +839,26 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
Pair<String, Integer> pair = new Pair<String, Integer>(term.text(), Integer.valueOf(freq)); Pair<String, Integer> pair = new Pair<String, Integer>(term.text(), Integer.valueOf(freq));
if (answer.size() < count) if (answer.size() < count)
{ {
if (answer.size() == 0) if (answer.size() == 0)
{ {
answer.add(pair); answer.add(pair);
} }
else if (answer.get(answer.size() - 1).getSecond().compareTo(pair.getSecond()) >= 0) else if (answer.get(answer.size() - 1).getSecond().compareTo(pair.getSecond()) >= 0)
{ {
answer.add(pair); answer.add(pair);
} }
else else
{ {
for (ListIterator<Pair<String, Integer>> it = answer.listIterator(); it.hasNext(); /**/) for (ListIterator<Pair<String, Integer>> it = answer.listIterator(); it.hasNext(); /**/)
{ {
Pair<String, Integer> test = it.next(); Pair<String, Integer> test = it.next();
if (test.getSecond().compareTo(pair.getSecond()) < 0) if (test.getSecond().compareTo(pair.getSecond()) < 0)
{ {
it.previous(); it.previous();
it.add(pair); it.add(pair);
break; break;
} }
} }
} }
} }
else if (answer.get(count - 1).getSecond().compareTo(pair.getSecond()) < 0) else if (answer.get(count - 1).getSecond().compareTo(pair.getSecond()) < 0)
@@ -806,7 +880,8 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
// off the end // off the end
} }
} }
} while (terms.next()); }
while (terms.next());
terms.close(); terms.close();
return answer; return answer;
@@ -832,7 +907,9 @@ public class ADMLuceneSearcherImpl extends AbstractLuceneBase implements LuceneS
} }
/* (non-Javadoc) /*
* (non-Javadoc)
*
* @see org.alfresco.repo.search.impl.lucene.LuceneSearcher#getClosingIndexSearcher() * @see org.alfresco.repo.search.impl.lucene.LuceneSearcher#getClosingIndexSearcher()
*/ */
public ClosingIndexSearcher getClosingIndexSearcher() public ClosingIndexSearcher getClosingIndexSearcher()

171
source/java/org/alfresco/repo/search/impl/lucene/ADMLuceneTest.java
View File

@@ -128,6 +128,10 @@ public class ADMLuceneTest extends TestCase
QName orderInt = QName.createQName(TEST_NAMESPACE, "orderInt"); QName orderInt = QName.createQName(TEST_NAMESPACE, "orderInt");
QName orderText = QName.createQName(TEST_NAMESPACE, "orderText");
QName orderMLText = QName.createQName(TEST_NAMESPACE, "orderMLText");
QName aspectWithChildren = QName.createQName(TEST_NAMESPACE, "aspectWithChildren"); QName aspectWithChildren = QName.createQName(TEST_NAMESPACE, "aspectWithChildren");
TransactionService transactionService; TransactionService transactionService;
@@ -422,12 +426,12 @@ public class ADMLuceneTest extends TestCase
documentOrder = new NodeRef[] { rootNodeRef, n4, n5, n6, n7, n8, n9, n10, n11, n12, n13, n14, n3, n1, n2 }; documentOrder = new NodeRef[] { rootNodeRef, n4, n5, n6, n7, n8, n9, n10, n11, n12, n13, n14, n3, n1, n2 };
// TODO: Why was the cm:auditable aspect added here? // TODO: Why was the cm:auditable aspect added here?
// By adding it, the auditable properties were set automatically. // By adding it, the auditable properties were set automatically.
// nodeService.addAspect(n3, ContentModel.ASPECT_AUDITABLE, null); // nodeService.addAspect(n3, ContentModel.ASPECT_AUDITABLE, null);
// nodeService.addAspect(n1, ContentModel.ASPECT_AUDITABLE, null); // nodeService.addAspect(n1, ContentModel.ASPECT_AUDITABLE, null);
nodeService.setProperty(n1, ContentModel.PROP_MODIFIED, new Date(new Date().getTime() - 1000*60*60)); nodeService.setProperty(n1, ContentModel.PROP_MODIFIED, new Date(new Date().getTime() - 1000 * 60 * 60));
// nodeService.addAspect(n2, ContentModel.ASPECT_AUDITABLE, null); // nodeService.addAspect(n2, ContentModel.ASPECT_AUDITABLE, null);
} }
private double orderDoubleCount = -0.11d; private double orderDoubleCount = -0.11d;
@@ -440,6 +444,8 @@ public class ADMLuceneTest extends TestCase
private int orderIntCount = -45764576; private int orderIntCount = -45764576;
private int orderTextCount = 0;
/** /**
* @return properties * @return properties
*/ */
@@ -451,11 +457,20 @@ public class ADMLuceneTest extends TestCase
testProperties.put(orderFloat, orderFloatCount); testProperties.put(orderFloat, orderFloatCount);
testProperties.put(orderLong, orderLongCount); testProperties.put(orderLong, orderLongCount);
testProperties.put(orderInt, orderIntCount); testProperties.put(orderInt, orderIntCount);
testProperties.put(orderText, new String(new char[] { (char) ('a' + orderTextCount) }) + " cabbage");
MLText mlText = new MLText();
mlText.addValue(Locale.ENGLISH, new String(new char[] { (char) ('a' + orderTextCount) }) + " banana");
mlText.addValue(Locale.FRENCH, new String(new char[] { (char) ('z' - orderTextCount) }) + " banane");
mlText.addValue(Locale.CHINESE, new String(new char[] { (char) ('香' + orderTextCount) }) + " 香蕉");
testProperties.put(orderMLText, mlText);
orderDate = Duration.subtract(orderDate, new Duration("P1D")); orderDate = Duration.subtract(orderDate, new Duration("P1D"));
orderDoubleCount += 0.1d; orderDoubleCount += 0.1d;
orderFloatCount += 0.82f; orderFloatCount += 0.82f;
orderLongCount += 299999999999999l; orderLongCount += 299999999999999l;
orderIntCount += 8576457; orderIntCount += 8576457;
orderTextCount++;
return testProperties; return testProperties;
} }
@@ -691,7 +706,7 @@ public class ADMLuceneTest extends TestCase
private void doBulkTest(int n) throws Exception private void doBulkTest(int n) throws Exception
{ {
SessionSizeResourceManager.setDisableInTransaction(); SessionSizeResourceManager.setDisableInTransaction();
Map<QName, Serializable> testProperties = new HashMap<QName, Serializable>(); Map<QName, Serializable> testProperties = new HashMap<QName, Serializable>();
testProperties.put(QName.createQName(TEST_NAMESPACE, "text-indexed-stored-tokenised-atomic"), "BULK"); testProperties.put(QName.createQName(TEST_NAMESPACE, "text-indexed-stored-tokenised-atomic"), "BULK");
for (int i = 0; i < n; i++) for (int i = 0; i < n; i++)
@@ -734,7 +749,7 @@ public class ADMLuceneTest extends TestCase
private void getCold(ADMLuceneSearcherImpl searcher, int n) private void getCold(ADMLuceneSearcherImpl searcher, int n)
{ {
hibernateL1CacheBulkLoader.clear(); hibernateL1CacheBulkLoader.clear();
long start; long start;
long end; long end;
@@ -759,10 +774,10 @@ public class ADMLuceneTest extends TestCase
System.out.println(n + " Cold in " + ((end - start) / 10e9)); System.out.println(n + " Cold in " + ((end - start) / 10e9));
} }
private void getWarm(ADMLuceneSearcherImpl searcher, int n) private void getWarm(ADMLuceneSearcherImpl searcher, int n)
{ {
long start; long start;
long end; long end;
@@ -787,11 +802,11 @@ public class ADMLuceneTest extends TestCase
System.out.println(n + " Warm in " + ((end - start) / 10e9)); System.out.println(n + " Warm in " + ((end - start) / 10e9));
} }
private void getCold10(ADMLuceneSearcherImpl searcher, int n) private void getCold10(ADMLuceneSearcherImpl searcher, int n)
{ {
hibernateL1CacheBulkLoader.clear(); hibernateL1CacheBulkLoader.clear();
long start; long start;
long end; long end;
@@ -816,11 +831,11 @@ public class ADMLuceneTest extends TestCase
System.out.println(n + " Prefetch 10 in " + ((end - start) / 10e9)); System.out.println(n + " Prefetch 10 in " + ((end - start) / 10e9));
} }
private void getCold100(ADMLuceneSearcherImpl searcher, int n) private void getCold100(ADMLuceneSearcherImpl searcher, int n)
{ {
hibernateL1CacheBulkLoader.clear(); hibernateL1CacheBulkLoader.clear();
long start; long start;
long end; long end;
@@ -845,11 +860,11 @@ public class ADMLuceneTest extends TestCase
System.out.println(n + " Prefetch 100 in " + ((end - start) / 10e9)); System.out.println(n + " Prefetch 100 in " + ((end - start) / 10e9));
} }
private void getCold1000(ADMLuceneSearcherImpl searcher, int n) private void getCold1000(ADMLuceneSearcherImpl searcher, int n)
{ {
hibernateL1CacheBulkLoader.clear(); hibernateL1CacheBulkLoader.clear();
long start; long start;
long end; long end;
@@ -874,11 +889,11 @@ public class ADMLuceneTest extends TestCase
System.out.println(n + " Prefetch 1000 in " + ((end - start) / 10e9)); System.out.println(n + " Prefetch 1000 in " + ((end - start) / 10e9));
} }
private void getCold10000(ADMLuceneSearcherImpl searcher, int n) private void getCold10000(ADMLuceneSearcherImpl searcher, int n)
{ {
hibernateL1CacheBulkLoader.clear(); hibernateL1CacheBulkLoader.clear();
long start; long start;
long end; long end;
@@ -2016,7 +2031,7 @@ public class ADMLuceneTest extends TestCase
date = currentBun; date = currentBun;
} }
results.close(); results.close();
SearchParameters sp_7 = new SearchParameters(); SearchParameters sp_7 = new SearchParameters();
sp_7.addStore(rootNodeRef.getStoreRef()); sp_7.addStore(rootNodeRef.getStoreRef());
sp_7.setLanguage(SearchService.LANGUAGE_LUCENE); sp_7.setLanguage(SearchService.LANGUAGE_LUCENE);
@@ -2038,7 +2053,7 @@ public class ADMLuceneTest extends TestCase
c.set(Calendar.HOUR_OF_DAY, 0); c.set(Calendar.HOUR_OF_DAY, 0);
currentBun = c.getTime(); currentBun = c.getTime();
} }
if (date != null) if ((date != null) && (currentBun != null))
{ {
assertTrue(date.compareTo(currentBun) <= 0); assertTrue(date.compareTo(currentBun) <= 0);
} }
@@ -2245,16 +2260,116 @@ public class ADMLuceneTest extends TestCase
} }
results.close(); results.close();
luceneFTS.resume(); // sort by text
SearchParameters sp17 = new SearchParameters(); SearchParameters sp17 = new SearchParameters();
sp17.addStore(rootNodeRef.getStoreRef()); sp17.addStore(rootNodeRef.getStoreRef());
sp17.setLanguage(SearchService.LANGUAGE_LUCENE); sp17.setLanguage(SearchService.LANGUAGE_LUCENE);
sp17.setQuery("PATH:\"//.\""); sp17.setQuery("PATH:\"//.\"");
sp17.addSort("cabbage", false); sp17.addSort("@" + orderText, true);
results = searcher.query(sp17); results = searcher.query(sp17);
String text = null;
for (ResultSetRow row : results)
{
String currentBun = DefaultTypeConverter.INSTANCE.convert(String.class, nodeService.getProperty(row.getNodeRef(), orderText));
// System.out.println( (currentBun == null ? "null" : NumericEncoder.encode(currentBun))+ " "+currentBun);
if ((text != null) && (currentBun != null))
{
assertTrue(text.compareTo(currentBun) <= 0);
}
text = currentBun;
}
results.close(); results.close();
SearchParameters sp18 = new SearchParameters();
sp18.addStore(rootNodeRef.getStoreRef());
sp18.setLanguage(SearchService.LANGUAGE_LUCENE);
sp18.setQuery("PATH:\"//.\"");
sp18.addSort("@" + orderText, false);
results = searcher.query(sp18);
text = null;
for (ResultSetRow row : results)
{
String currentBun = DefaultTypeConverter.INSTANCE.convert(String.class, nodeService.getProperty(row.getNodeRef(), orderText));
// System.out.println(currentBun);
if ((text != null) && (currentBun != null))
{
assertTrue(text.compareTo(currentBun) >= 0);
}
text = currentBun;
}
results.close();
// sort by ML text
Locale[] testLocales = new Locale[] { I18NUtil.getLocale(), Locale.ENGLISH, Locale.FRENCH, Locale.CHINESE };
for (Locale testLocale : testLocales)
{
SearchParameters sp19 = new SearchParameters();
sp19.addStore(rootNodeRef.getStoreRef());
sp19.setLanguage(SearchService.LANGUAGE_LUCENE);
sp19.setQuery("PATH:\"//.\"");
sp19.addSort("@" + orderMLText, true);
sp19.addLocale(testLocale);
results = searcher.query(sp19);
text = null;
for (ResultSetRow row : results)
{
MLText mltext = DefaultTypeConverter.INSTANCE.convert(MLText.class, nodeService.getProperty(row.getNodeRef(), orderMLText));
if (mltext != null)
{
String currentBun = mltext.getValue(testLocale);
// System.out.println( (currentBun == null ? "null" : NumericEncoder.encode(currentBun))+ "
// "+currentBun);
if ((text != null) && (currentBun != null))
{
assertTrue(text.compareTo(currentBun) <= 0);
}
text = currentBun;
}
}
results.close();
SearchParameters sp20 = new SearchParameters();
sp20.addStore(rootNodeRef.getStoreRef());
sp20.setLanguage(SearchService.LANGUAGE_LUCENE);
sp20.setQuery("PATH:\"//.\"");
sp20.addSort("@" + orderMLText, false);
sp20.addLocale(testLocale);
results = searcher.query(sp20);
text = null;
for (ResultSetRow row : results)
{
MLText mltext = DefaultTypeConverter.INSTANCE.convert(MLText.class, nodeService.getProperty(row.getNodeRef(), orderMLText));
if (mltext != null)
{
String currentBun = mltext.getValue(testLocale);
if ((text != null) && (currentBun != null))
{
assertTrue(text.compareTo(currentBun) >= 0);
}
text = currentBun;
}
}
results.close();
}
luceneFTS.resume();
SearchParameters spN = new SearchParameters();
spN.addStore(rootNodeRef.getStoreRef());
spN.setLanguage(SearchService.LANGUAGE_LUCENE);
spN.setQuery("PATH:\"//.\"");
spN.addSort("cabbage", false);
results = searcher.query(spN);
results.close();
luceneFTS.resume(); luceneFTS.resume();
} }
@@ -4480,13 +4595,13 @@ public class ADMLuceneTest extends TestCase
results = searcher.query(sp); results = searcher.query(sp);
assertEquals(1, results.length()); assertEquals(1, results.length());
results.close(); results.close();
sp = new SearchParameters(); sp = new SearchParameters();
sp.addStore(rootNodeRef.getStoreRef()); sp.addStore(rootNodeRef.getStoreRef());
sp.setLanguage("lucene"); sp.setLanguage("lucene");
sp.setQuery("ISNULL:\"" + QName.createQName(TEST_NAMESPACE, "null").toString() + "\""); sp.setQuery("ISNULL:\"" + QName.createQName(TEST_NAMESPACE, "null").toString() + "\"");
results = searcher.query(sp); results = searcher.query(sp);
//assertEquals(62, results.length()); // assertEquals(62, results.length());
results.close(); results.close();
sp = new SearchParameters(); sp = new SearchParameters();
@@ -4496,13 +4611,13 @@ public class ADMLuceneTest extends TestCase
results = searcher.query(sp); results = searcher.query(sp);
assertEquals(0, results.length()); assertEquals(0, results.length());
results.close(); results.close();
sp = new SearchParameters(); sp = new SearchParameters();
sp.addStore(rootNodeRef.getStoreRef()); sp.addStore(rootNodeRef.getStoreRef());
sp.setLanguage("lucene"); sp.setLanguage("lucene");
sp.setQuery("ISNULL:\"" + QName.createQName(TEST_NAMESPACE, "path-ista").toString() + "\""); sp.setQuery("ISNULL:\"" + QName.createQName(TEST_NAMESPACE, "path-ista").toString() + "\"");
results = searcher.query(sp); results = searcher.query(sp);
//assertEquals(61, results.length()); // assertEquals(61, results.length());
results.close(); results.close();
sp = new SearchParameters(); sp = new SearchParameters();
@@ -4534,7 +4649,7 @@ public class ADMLuceneTest extends TestCase
sp.setLanguage("lucene"); sp.setLanguage("lucene");
sp.setQuery("ISNULL:\"" + QName.createQName(TEST_NAMESPACE, "aspectProperty").toString() + "\""); sp.setQuery("ISNULL:\"" + QName.createQName(TEST_NAMESPACE, "aspectProperty").toString() + "\"");
results = searcher.query(sp); results = searcher.query(sp);
//assertEquals(62, results.length()); // assertEquals(62, results.length());
results.close(); results.close();
sp = new SearchParameters(); sp = new SearchParameters();

20
source/java/org/alfresco/repo/search/impl/lucene/LuceneTest_model.xml
View File

@@ -68,6 +68,26 @@
<stored>true</stored> <stored>true</stored>
<tokenised>true</tokenised> <tokenised>true</tokenised>
</index> </index>
</property>
<property name="test:orderText">
<type>d:text</type>
<mandatory>true</mandatory>
<multiple>false</multiple>
<index enabled="true">
<atomic>true</atomic>
<stored>true</stored>
<tokenised>both</tokenised>
</index>
</property>
<property name="test:orderMLText">
<type>d:mltext</type>
<mandatory>true</mandatory>
<multiple>false</multiple>
<index enabled="true">
<atomic>true</atomic>
<stored>true</stored>
<tokenised>both</tokenised>
</index>
</property> </property>
</properties> </properties>
<associations> <associations>