inteligr8/SearchServices
1
0
Fork 0
mirror of https://github.com/Alfresco/SearchServices.git synced 2026-09-16 18:12:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'fix/SEARCH-2202_MoreSonarFixesMaster' into 'master'

Browse Source 
Fix/search 2202 more sonar fixes master

See merge request search_discovery/insightengine!477
This commit is contained in:
Tom Page
2020-04-28 09:15:12 +01:00
parent a61ba7d04d 1aa959c186
commit cfb670302c
11 changed files with 94 additions and 934 deletions
Download Patch File Download Diff File Expand all files Collapse all files
search-services/alfresco-search/src/main/java/org/alfresco/solr/ResizeableArrayList.java
-168
View File
@@ -1,168 +0,0 @@
/*
* Copyright (C) 2005-2013 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr;
import java.io.Serializable;
import java.util.AbstractList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.RandomAccess;
/**
* A {@link List} implementation, backed by an array, taht supports resizing.
*
* This class supports the reuse of arrays across cache instances, reducing the number of
* temporary objects created by the Alfresco Solr indexing service.
*
* @author Alex Miller
*/
public class ResizeableArrayList<E> extends AbstractList<E> implements List<E>, RandomAccess, Cloneable, Serializable
{
private static final long serialVersionUID = 1L;
private Object[] values;
private int size;
boolean active = false;
public ResizeableArrayList()
{
this(10);
}
public ResizeableArrayList(int initialSize)
{
values = new Object[initialSize];
size = initialSize;
}
@SuppressWarnings("unchecked")
@Override
public E get(int index)
{
checkSize(index);
return (E)values[index];
}
private void checkSize(int index)
{
if (index >= size)
{
throw new ArrayIndexOutOfBoundsException("Index: " + index + ", Size: " + values.length);
}
}
/**
* Resize the underlying to at least minSize.
*
* @param minSize
* @throws IllegalStateException if the instance is not active.
*/
public void resize(int minSize)
{
isActive();
int oldSize = values.length;
if (minSize > oldSize)
{
int newSize = (oldSize * 3)/2 + 1;
if (newSize < minSize)
{
newSize = minSize;
}
// minCapacity is usually close to size, so this is a win:
values = Arrays.copyOf(values, newSize);
}
size = minSize;
}
@Override
public int size()
{
return size;
}
/**
* Copy elements from from into this instance
*/
public <T> void copyFrom(ResizeableArrayList<T> from)
{
isActive();
values = Arrays.copyOf(from.values, from.values.length);
size = from.size;
}
@SuppressWarnings("unchecked")
@Override
public E set(int index, E value)
{
checkSize(index);
isActive();
E oldValue = (E)values[index];
values[index] = value;
return oldValue;
}
@SuppressWarnings("unchecked")
@Override
protected Object clone() throws CloneNotSupportedException
{
try {
ResizeableArrayList<E> v = (ResizeableArrayList<E>) super.clone();
v.values = Arrays.copyOf(values, values.length);
v.size = size;
return v;
} catch (CloneNotSupportedException e) {
throw new InternalError();
}
}
private void isActive()
{
if (!active)
{
throw new IllegalStateException("Not active");
}
}
void activate()
{
active = true;
}
void deactivate()
{
active = false;
for (int i = 0 ; i < size ; i++)
{
values[i] = null;
}
}
/**
* Sort the elements, in-place, contained in this list using {@link Comparator#}
*
* @param comparator The {@link Comparator} to sort with
*/
@SuppressWarnings("unchecked")
public void sort(Comparator<? super E> comparator)
{
DualPivotQuickSort.sort((E[])values, 0, size -1 , comparator);
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java
+6 -10
View File
@@ -69,6 +69,9 @@ import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_TXCOM
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_TXID;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_TYPE;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_VERSION;
import static org.alfresco.service.cmr.security.AuthorityType.EVERYONE;
import static org.alfresco.service.cmr.security.AuthorityType.GROUP;
import static org.alfresco.service.cmr.security.AuthorityType.GUEST;
import static org.alfresco.solr.utils.Utils.notNullOrEmpty;
import java.io.File;
@@ -3216,17 +3219,10 @@ public class SolrInformationServer implements InformationServer
*/
private String addTenantToAuthority(String authority, String tenant)
{
switch (AuthorityType.getAuthorityType(authority))
AuthorityType authorityType = AuthorityType.getAuthorityType(authority);
if ((authorityType == GROUP || authorityType == EVERYONE || authorityType == GUEST) && !tenant.isEmpty())
{
case GROUP:
case EVERYONE:
case GUEST:
if (tenant.length() != 0)
{
return authority + "@" + tenant;
}
default:
break;
return authority + "@" + tenant;
}
return authority;
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java
+2 -4
View File
@@ -5496,8 +5496,6 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
throw new RuntimeException("Error analyzing multiTerm term: " + part, e);
}
}
private boolean analyzeRangeTerms = true;
protected Query newRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) {
final BytesRef start;
@@ -5506,13 +5504,13 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
if (part1 == null) {
start = null;
} else {
start = analyzeRangeTerms ? analyzeMultitermTerm(field, part1) : new BytesRef(part1);
start = getAnalyzeRangeTerms() ? analyzeMultitermTerm(field, part1) : new BytesRef(part1);
}
if (part2 == null) {
end = null;
} else {
end = analyzeRangeTerms ? analyzeMultitermTerm(field, part2) : new BytesRef(part2);
end = getAnalyzeRangeTerms() ? analyzeMultitermTerm(field, part2) : new BytesRef(part2);
}
final TermRangeQuery query = new TermRangeQuery(field, start, end, startInclusive, endInclusive);
search-services/alfresco-search/src/main/java/org/alfresco/solr/query/SolrContainerScorer.java
-4
View File
@@ -37,10 +37,6 @@ import org.apache.lucene.search.Weight;
*/
public class SolrContainerScorer extends Scorer
{
// Unused
Weight weight;
SolrContainerScorerDocIdSetIterator iterator;
search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java
-2
View File
@@ -47,8 +47,6 @@ public class CachedDocTransformer extends DocTransformer
{
protected final static Logger log = LoggerFactory.getLogger(CachedDocTransformer.class);
private ResultContext context;
/* (non-Javadoc)
* @see org.apache.solr.response.transform.DocTransformer#getName()
*/
search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/DocValueDocTransformer.java
-4
View File
@@ -42,11 +42,7 @@ import org.slf4j.LoggerFactory;
public class DocValueDocTransformer extends DocTransformer
{
protected final static Logger log = LoggerFactory.getLogger(DocValueDocTransformer.class);
ResultContext context;
/* (non-Javadoc)
* @see org.apache.solr.response.transform.DocTransformer#getName()
*/
search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerIT.java
+25 -9
View File
@@ -1,8 +1,10 @@
package org.alfresco.solr;
import static org.junit.Assert.fail;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.common.SolrException;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.handler.admin.CoreAdminHandler;
import org.apache.solr.response.SolrQueryResponse;
@@ -32,19 +34,33 @@ public class AdminHandlerIT extends AbstractAlfrescoSolrIT
}
@Test
public void testhandledCores() throws Exception
public void testHandledCores()
{
requestAction("newCore");
requestAction("updateCore");
requestAction("removeCore");
try
{
requestAction("newCore");
requestAction("updateCore");
requestAction("removeCore");
}
catch (Exception e)
{
fail("Expected to receive no exception, but got " + e);
}
}
@Test
public void testhandledReports() throws Exception
public void testHandledReports()
{
requestAction("CHECK");
requestAction("REPORT");
requestAction("SUMMARY");
try
{
requestAction("CHECK");
requestAction("REPORT");
requestAction("SUMMARY");
}
catch (Exception e)
{
fail("Expected to receive no exception, but got " + e);
}
}
private void requestAction(String actionName) throws Exception {
search-services/alfresco-search/src/test/java/org/alfresco/solr/CloudTest.java
-109
View File
@@ -1,109 +0,0 @@
/*
* Copyright (C) 2005-2014 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr;
public class CloudTest
{
/*
public final static String QUERY = "a query";
private Cloud cloud = new Cloud();
@Mock SolrQueryRequest request;
@SuppressWarnings({ "rawtypes" })
@Test
public void testGetQueryWithZeroValues()
{
String fieldName = FIELD_DBID;
String operator = SolrInformationServer.AND;
Collection[] valueLists = new Collection[1];
valueLists[0] = new ArrayList();
String query = cloud.getQuery(fieldName, operator, valueLists);
assertEquals("", query);
valueLists = new Collection[0];
query = cloud.getQuery(fieldName, operator, valueLists);
assertEquals("", query);
}
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testGetQueryWithOneValue()
{
String fieldName = FIELD_DBID;
String operator = SolrInformationServer.AND;
Collection[] valueLists = new Collection[1];
valueLists[0] = new ArrayList();
Object value = "value";
valueLists[0].add(value);
String query = cloud.getQuery(fieldName, operator, valueLists);
assertEquals(FIELD_DBID + ":" + value, query);
}
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testGetQueryWithManyValues()
{
String fieldName = FIELD_DBID;
String operator = SolrInformationServer.AND;
Collection[] valueLists = new Collection[2];
valueLists[0] = new ArrayList();
Object value1 = "value1";
valueLists[0].add(value1);
Object value2 = "value2";
valueLists[0].add(value2);
Object value3 = "value3";
valueLists[0].add(value3);
valueLists[1] = new ArrayList();
Object value4 = "value4";
valueLists[1].add(value4);
String query = cloud.getQuery(fieldName, operator, valueLists);
assertEquals(FIELD_DBID + ":" + value1 + operator + FIELD_DBID + ":" + value2 + operator
+ FIELD_DBID + ":" + value3 + operator + FIELD_DBID + ":" + value4, query);
}
@Test
public void testExists()
{
boolean exists = cloud.exists(super.selectRequestHandler, request, QUERY);
assertFalse(exists);
}
@Test
public void testGetDocList()
{
// The response is created in the getDocList method, and the aftsRequestHandler is simply mocked.
// Therefore nothing is expected to be on the response for tests, so this verifies behavior.
DocList docList = cloud.getDocList(super.aftsRequestHandler, request, QUERY);
assertNull(docList);
ArgumentCaptor<SolrQueryResponse> response = ArgumentCaptor.forClass(SolrQueryResponse.class);
verify(super.aftsRequestHandler).handleRequest(eq(request), response.capture());
assertNotNull(response.getValue());
}
@Test
public void testSelect()
{
SolrParams params = new ModifiableSolrParams(request.getParams());
ResultContext rc = cloud.getResultContext(super.aftsRequestHandler, request, params);
assertNull(rc);
}
*/
}
search-services/alfresco-search/src/test/java/org/apache/lucene/analysis/minhash/MinHashFilterIT.java
-567
View File
@@ -1,567 +0,0 @@
package org.apache.lucene.analysis.minhash;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory;
import org.apache.lucene.analysis.minhash.MinHashFilter.FixedSizeTreeSet;
import org.apache.lucene.analysis.minhash.MinHashFilter.LongPair;
import org.apache.lucene.analysis.shingle.ShingleFilterFactory;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharFilterFactory;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
import org.apache.lucene.util.automaton.RegExp;
import org.junit.Test;
public class MinHashFilterIT extends BaseTokenStreamTestCase
{
@Test
public void testIntHash() {
LongPair hash = new LongPair();
MinHashFilter.murmurhash3_x64_128(MinHashFilter.getBytes(0), 0, 4, 0, hash);
assertEquals(-3485513579396041028L, hash.val1);
assertEquals(6383328099726337777L, hash.val2);
}
@Test
public void testStringHash() throws UnsupportedEncodingException {
LongPair hash = new LongPair();
byte[] bytes = "woof woof woof woof woof".getBytes("UTF-16LE");
MinHashFilter.murmurhash3_x64_128(bytes, 0, bytes.length, 0, hash);
assertEquals(7638079586852243959L, hash.val1);
assertEquals(4378804943379391304L, hash.val2);
}
@Test
public void testSimpleOrder() throws UnsupportedEncodingException {
LongPair hash1 = new LongPair();
hash1.val1 = 1;
hash1.val2 = 2;
LongPair hash2 = new LongPair();
hash2.val1 = 2;
hash2.val2 = 1;
assert (hash1.compareTo(hash2) > 0);
}
@Test
public void testHashOrder() {
assertTrue(!MinHashFilter.isLessThanUnsigned(0l, 0l));
assertTrue(MinHashFilter.isLessThanUnsigned(0l, -1l));
assertTrue(MinHashFilter.isLessThanUnsigned(1l, -1l));
assertTrue(MinHashFilter.isLessThanUnsigned(-2l, -1l));
assertTrue(MinHashFilter.isLessThanUnsigned(1l, 2l));
assertTrue(MinHashFilter.isLessThanUnsigned(Long.MAX_VALUE, Long.MIN_VALUE));
FixedSizeTreeSet<LongPair> minSet = new FixedSizeTreeSet<LongPair>(500);
HashSet<LongPair> unadded = new HashSet<LongPair>();
for (int i = 0; i < 100; i++) {
LongPair hash = new LongPair();
MinHashFilter.murmurhash3_x64_128(MinHashFilter.getBytes(i), 0, 4, 0, hash);
LongPair peek = null;
if (minSet.size() > 0) {
peek = minSet.last();
}
if (!minSet.add(hash)) {
unadded.add(hash);
} else {
if (peek != null) {
if ((minSet.size() == 500) && !peek.equals(minSet.last())) {
unadded.add(peek);
}
}
}
}
assertEquals(100, minSet.size());
assertEquals(0, unadded.size());
HashSet<LongPair> collisionDetection = new HashSet<LongPair>();
unadded = new HashSet<LongPair>();
minSet = new FixedSizeTreeSet<LongPair>(500);
for (int i = 0; i < 1000000; i++) {
LongPair hash = new LongPair();
MinHashFilter.murmurhash3_x64_128(MinHashFilter.getBytes(i), 0, 4, 0, hash);
collisionDetection.add(hash);
LongPair peek = null;
if (minSet.size() > 0) {
peek = minSet.last();
}
if (!minSet.add(hash)) {
unadded.add(hash);
} else {
if (peek != null) {
if ((minSet.size() == 500) && !peek.equals(minSet.last())) {
unadded.add(peek);
}
}
}
}
assertEquals(1000000, collisionDetection.size());
assertEquals(500, minSet.size());
assertEquals(999500, unadded.size());
LongPair last = null;
LongPair current = null;
while ((current = minSet.pollLast()) != null) {
if (last != null) {
assertTrue(isLessThan(current, last));
} else {
}
last = current;
}
}
@Test
public void testHashNotRepeated() {
FixedSizeTreeSet<LongPair> minSet = new FixedSizeTreeSet<LongPair>(500);
HashSet<LongPair> unadded = new HashSet<LongPair>();
for (int i = 0; i < 10000; i++) {
LongPair hash = new LongPair();
MinHashFilter.murmurhash3_x64_128(MinHashFilter.getBytes(i), 0, 4, 0, hash);
LongPair peek = null;
if (minSet.size() > 0) {
peek = minSet.last();
}
if (!minSet.add(hash)) {
unadded.add(hash);
} else {
if (peek != null) {
if ((minSet.size() == 500) && !peek.equals(minSet.last())) {
unadded.add(peek);
}
}
}
}
assertEquals(500, minSet.size());
LongPair last = null;
LongPair current = null;
while ((current = minSet.pollLast()) != null) {
if (last != null) {
assertTrue(isLessThan(current, last));
} else {
}
last = current;
}
}
@Test
public void testMockShingleTokenizer() throws IOException {
Tokenizer mockShingleTokenizer = createMockShingleTokenizer(5,
"woof woof woof woof woof" + " " + "woof woof woof woof puff");
assertTokenStreamContents(mockShingleTokenizer,
new String[] {"woof woof woof woof woof", "woof woof woof woof puff"});
}
@Test
public void testTokenStreamSingleInput() throws IOException {
String[] hashes = new String[] {"℁팽徭聙↝ꇁ홱杯"};
TokenStream ts = createTokenStream(5, "woof woof woof woof woof", 1, 1, 100, false);
assertTokenStreamContents(ts, hashes, new int[] {0},
new int[] {24}, new String[] {MinHashFilter.MIN_HASH_TYPE}, new int[] {1}, new int[] {1}, 24, 0, null,
true);
ts = createTokenStream(5, "woof woof woof woof woof", 2, 1, 1, false);
assertTokenStreamContents(ts, new String[] {new String(new char[] {0, 0, 8449, 54077, 64133, 32857, 8605, 41409}),
new String(new char[] {0, 1, 16887, 58164, 39536, 14926, 6529, 17276})}, new int[] {0, 0},
new int[] {24, 24}, new String[] {MinHashFilter.MIN_HASH_TYPE, MinHashFilter.MIN_HASH_TYPE}, new int[] {1, 0}, new int[] {1, 1}, 24, 0, null,
true);
}
@Test
public void testTokenStream1() throws IOException {
String[] hashes = new String[] {"℁팽徭聙↝ꇁ홱杯",
new String(new char[] {36347, 63457, 43013, 56843, 52284, 34231, 57934, 42302})};
TokenStream ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 1, 1, 100,false);
assertTokenStreamContents(ts, hashes, new int[] {0, 0},
new int[] {49, 49}, new String[] {MinHashFilter.MIN_HASH_TYPE, MinHashFilter.MIN_HASH_TYPE}, new int[] {1, 0},
new int[] {1, 1}, 49, 0, null, true);
}
private ArrayList<String> getTokens(TokenStream ts) throws IOException {
ArrayList<String> tokens = new ArrayList<String>();
ts.reset();
while (ts.incrementToken()) {
CharTermAttribute termAttribute = ts.getAttribute(CharTermAttribute.class);
String token = new String(termAttribute.buffer(), 0, termAttribute.length());
tokens.add(token);
}
ts.end();
ts.close();
return tokens;
}
private ArrayList<String> getTokens(Analyzer analyzer, String field, String value) throws IOException
{
ArrayList<String> tokens = new ArrayList<String>();
TokenStream ts = analyzer.tokenStream(field, value);
ts.reset();
while(ts.incrementToken())
{
CharTermAttribute termAttribute = ts.getAttribute(CharTermAttribute.class);
String token = new String(termAttribute.buffer(), 0, termAttribute.length());
tokens.add(token);
}
ts.end();
ts.close();
return tokens;
}
@Test
public void testTokenStream2() throws IOException {
TokenStream ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 100, 1, 1, false);
ArrayList<String> tokens = getTokens(ts);
ts.close();
assertEquals(100, tokens.size());
}
@Test
public void testTokenStream3() throws IOException {
TokenStream ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 10, 1, 10, false);
ArrayList<String> tokens = getTokens(ts);
ts.close();
assertEquals(20, tokens.size());
}
@Test
public void testTokenStream4() throws IOException {
TokenStream ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 10, 10, 1, false);
ArrayList<String> tokens = getTokens(ts);
ts.close();
assertEquals(20, tokens.size());
ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 10, 10, 1, true);
tokens = getTokens(ts);
ts.close();
assertEquals(100, tokens.size());
}
@Test
public void testLSHQuery() throws IOException
{
Analyzer analyzer = createMinHashAnalyzer(5, 1, 100);
IndexWriterConfig config = new IndexWriterConfig(analyzer);
RAMDirectory directory = new RAMDirectory();
IndexWriter writer = new IndexWriter(directory, config);
Document doc = new Document();
doc.add(new TextField("text", "woof woof woof woof woof", Store.NO));
writer.addDocument(doc);
doc = new Document();
doc.add(new TextField("text", "woof woof woof woof woof puff", Store.NO));
writer.addDocument(doc);
doc = new Document();
doc.add(new TextField("text", "woof woof woof woof puff", Store.NO));
writer.addDocument(doc);
writer.commit();
writer.close();
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(directory));
BooleanQuery.Builder builder = new BooleanQuery.Builder();
builder.add(new ConstantScoreQuery(new TermQuery(new Term("text", "℁팽徭聙↝ꇁ홱杯"))), Occur.SHOULD);
builder.add(new ConstantScoreQuery(new TermQuery(new Term("text", new String(new char[] {36347, 63457, 43013, 56843, 52284, 34231, 57934, 42302})))), Occur.SHOULD);
builder.setDisableCoord(true);
TopDocs topDocs = searcher.search(builder.build(), 10);
assertEquals(3, topDocs.totalHits);
float score = topDocs.scoreDocs[0].score;
assertEquals(topDocs.scoreDocs[1].score, score/2, 0f);
assertEquals(topDocs.scoreDocs[2].score, score/2, 0f);
}
@Test
public void testLSHQuery2() throws IOException
{
String[] parts = new String[]{"one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"};
int min = 5;
Analyzer analyzer = createMinHashAnalyzer(min, 1, 100);
IndexWriterConfig config = new IndexWriterConfig(analyzer);
RAMDirectory directory = new RAMDirectory();
IndexWriter writer = new IndexWriter(directory, config);
for(int i = 0; i < parts.length; i++)
{
StringBuilder builder = new StringBuilder();
for(int j = 0; j < parts.length -i; j++)
{
if(builder.length() > 0)
{
builder.append(" ");
}
builder.append(parts[i+j]);
if(j >= min -1)
{
Document doc = new Document();
doc.add(new TextField("text", builder.toString(), Store.NO));
writer.addDocument(doc);
}
}
}
writer.commit();
writer.close();
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(directory));
TopDocs topDocs = searcher.search(buildQuery("text", "one two three four five", min, 1, 100), 100);
assertEquals(6, topDocs.totalHits);
assertAllScores(topDocs, 1.0f);
topDocs = searcher.search(buildQuery("text", "two three four five six", min, 1, 100), 100);
assertEquals(10, topDocs.totalHits);
assertAllScores(topDocs, 1.0f);
topDocs = searcher.search(buildQuery("text", "three four five six seven", min, 1, 100), 100);
assertEquals(12, topDocs.totalHits);
assertAllScores(topDocs, 1.0f);
topDocs = searcher.search(buildQuery("text", "four five six seven eight", min, 1, 100), 100);
assertEquals(12, topDocs.totalHits);
assertAllScores(topDocs, 1.0f);
topDocs = searcher.search(buildQuery("text", "five six seven eight nine", min, 1, 100), 100);
assertEquals(10, topDocs.totalHits);
assertAllScores(topDocs, 1.0f);
topDocs = searcher.search(buildQuery("text", "six seven eight nine ten", min, 1, 100), 100);
assertEquals(6, topDocs.totalHits);
assertAllScores(topDocs, 1.0f);
topDocs = searcher.search(buildQuery("text", "one two three four five six", min, 1, 100), 100);
assertEquals(11, topDocs.totalHits);
topDocs = searcher.search(buildQuery("text", "one two three four five six seven eight nine ten", min, 1, 100), 100);
assertEquals(21, topDocs.totalHits);
for(int i = 0; i < topDocs.totalHits; i++)
{
System.out.println(i+" = "+topDocs.scoreDocs[i]);
}
float topScore = 6.0f;
assertEquals(topDocs.scoreDocs[0].score, topScore, 0.001f);
assertEquals(topDocs.scoreDocs[1].score, topScore * 5/6, 0.001f);
assertEquals(topDocs.scoreDocs[2].score, topScore * 5/6, 0.001f);
assertEquals(topDocs.scoreDocs[3].score, topScore * 4/6, 0.001f);
assertEquals(topDocs.scoreDocs[4].score, topScore * 4/6, 0.001f);
assertEquals(topDocs.scoreDocs[5].score, topScore * 4/6, 0.001f);
assertEquals(topDocs.scoreDocs[6].score, topScore * 3/6, 0.001f);
assertEquals(topDocs.scoreDocs[7].score, topScore * 3/6, 0.001f);
assertEquals(topDocs.scoreDocs[8].score, topScore * 3/6, 0.001f);
assertEquals(topDocs.scoreDocs[9].score, topScore * 3/6, 0.001f);
assertEquals(topDocs.scoreDocs[10].score, topScore * 2/6, 0.001f);
assertEquals(topDocs.scoreDocs[11].score, topScore * 2/6, 0.001f);
assertEquals(topDocs.scoreDocs[12].score, topScore * 2/6, 0.001f);
assertEquals(topDocs.scoreDocs[13].score, topScore * 2/6, 0.001f);
assertEquals(topDocs.scoreDocs[14].score, topScore * 2/6, 0.001f);
assertEquals(topDocs.scoreDocs[15].score, topScore * 1/6, 0.001f);
assertEquals(topDocs.scoreDocs[16].score, topScore * 1/6, 0.001f);
assertEquals(topDocs.scoreDocs[17].score, topScore * 1/6, 0.001f);
assertEquals(topDocs.scoreDocs[18].score, topScore * 1/6, 0.001f);
assertEquals(topDocs.scoreDocs[19].score, topScore * 1/6, 0.001f);
assertEquals(topDocs.scoreDocs[20].score, topScore * 1/6, 0.001f);
}
private void assertAllScores(TopDocs topDocs, float score)
{
for(int i = 0; i < topDocs.totalHits; i++)
{
assertEquals(topDocs.scoreDocs[i].score, score, 0f);
}
}
private Query buildQuery(String field, String query, int min, int hashCount, int hashSetSize) throws IOException
{
TokenizerChain chain = createMinHashAnalyzer(min, hashCount, hashSetSize);
ArrayList<String> tokens = getTokens(chain, field, query);
chain.close();
BooleanQuery.Builder builder = new BooleanQuery.Builder();
for(String token : tokens)
{
builder.add(new ConstantScoreQuery(new TermQuery(new Term("text", token))), Occur.SHOULD);
}
builder.setDisableCoord(true);
return builder.build();
}
public static TokenStream createTokenStream(int shingleSize, String shingles, int hashCount, int bucketCount, int hashSetSize, boolean withRotation) {
Tokenizer tokenizer = createMockShingleTokenizer(shingleSize, shingles);
HashMap<String,String> lshffargs = new HashMap<String,String>();
lshffargs.put("hashCount", "" + hashCount);
lshffargs.put("bucketCount", "" + bucketCount);
lshffargs.put("hashSetSize", "" + hashSetSize);
lshffargs.put("withRotation", "" + withRotation);
MinHashFilterFactory lshff = new MinHashFilterFactory(lshffargs);
return lshff.create(tokenizer);
}
public static TokenizerChain createMinHashAnalyzer(int min, int hashCount, int hashSetSize)
{
WhitespaceTokenizerFactory icutf = new WhitespaceTokenizerFactory(Collections.<String, String>emptyMap());
HashMap<String, String> sffargs = new HashMap<String, String>();
sffargs.put("minShingleSize", ""+min);
sffargs.put("maxShingleSize", ""+min);
sffargs.put("outputUnigrams", "false");
sffargs.put("outputUnigramsIfNoShingles", "false");
sffargs.put("tokenSeparator", " ");
ShingleFilterFactory sff = new ShingleFilterFactory(sffargs);
HashMap<String, String> lshffargs = new HashMap<String, String>();
lshffargs.put("hashCount", ""+hashCount);
lshffargs.put("hashSetSize", ""+hashSetSize);
MinHashFilterFactory lshff = new MinHashFilterFactory(lshffargs);
TokenizerChain chain = new TokenizerChain(new CharFilterFactory[]{}, icutf, new TokenFilterFactory[]{sff, lshff});
return chain;
}
public static Tokenizer createMockShingleTokenizer(int shingleSize, String shingles) {
MockTokenizer tokenizer = new MockTokenizer(
new CharacterRunAutomaton(new RegExp("[^ \t\r\n]+([ \t\r\n]+[^ \t\r\n]+){4}").toAutomaton()),
true);
tokenizer.setEnableChecks(true);
if (shingles != null) {
tokenizer.setReader(new StringReader(shingles));
}
return tokenizer;
}
private boolean isLessThan(LongPair hash1, LongPair hash2) {
if (MinHashFilter.isLessThanUnsigned(hash1.val2, hash2.val2)) {
return true;
} else if (hash1.val2 == hash2.val2) {
return (MinHashFilter.isLessThanUnsigned(hash1.val1, hash2.val1));
} else {
return false;
}
}
/**
* An analyzer that uses a tokenizer and a list of token filters to
* create a TokenStream - lifted from SOLR to make this analyzer test lucene only.
*/
public static class TokenizerChain extends Analyzer {
final private CharFilterFactory[] charFilters;
final private TokenizerFactory tokenizer;
final private TokenFilterFactory[] filters;
/**
* Creates a new TokenizerChain.
*
* @param charFilters Factories for the CharFilters to use, if any - if null, will be treated as if empty.
* @param tokenizer Factory for the Tokenizer to use, must not be null.
* @param filters Factories for the TokenFilters to use if any- if null, will be treated as if empty.
*/
public TokenizerChain(CharFilterFactory[] charFilters, TokenizerFactory tokenizer, TokenFilterFactory[] filters) {
this.charFilters = charFilters;
this.tokenizer = tokenizer;
this.filters = filters;
}
@Override
public Reader initReader(String fieldName, Reader reader) {
if (charFilters != null && charFilters.length > 0) {
Reader cs = reader;
for (CharFilterFactory charFilter : charFilters) {
cs = charFilter.create(cs);
}
reader = cs;
}
return reader;
}
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer tk = tokenizer.create();
TokenStream ts = tk;
for (TokenFilterFactory filter : filters) {
ts = filter.create(ts);
}
return new TokenStreamComponents(tk, ts);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder("TokenizerChain(");
for (CharFilterFactory filter: charFilters) {
sb.append(filter);
sb.append(", ");
}
sb.append(tokenizer);
for (TokenFilterFactory filter: filters) {
sb.append(", ");
sb.append(filter);
}
sb.append(')');
return sb.toString();
}
}
}
search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AlfrescoModel.java
+29 -27
View File
@@ -1,28 +1,28 @@
/*
* #%L
* Alfresco Solr Client
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
/*
* #%L
* Alfresco Solr Client
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.solr.client;
import org.alfresco.repo.dictionary.M2Model;
@@ -52,7 +52,8 @@ public class AlfrescoModel
{
return checksum;
}
@Override
public boolean equals(Object other)
{
if (this == other)
@@ -70,7 +71,8 @@ public class AlfrescoModel
checksum.equals(model.getChecksum()));
}
public int hashcode()
@Override
public int hashCode()
{
int result = 17;
result = 31 * result + model.hashCode();
search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AlfrescoModelDiff.java
+32 -30
View File
@@ -1,28 +1,28 @@
/*
* #%L
* Alfresco Solr Client
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
/*
* #%L
* Alfresco Solr Client
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.solr.client;
import org.alfresco.service.namespace.QName;
@@ -36,10 +36,10 @@ import org.alfresco.service.namespace.QName;
*/
public class AlfrescoModelDiff
{
public static enum TYPE
public enum TYPE
{
NEW, CHANGED, REMOVED;
};
}
private QName modelName;
private TYPE type;
@@ -74,7 +74,8 @@ public class AlfrescoModelDiff
{
return newChecksum;
}
@Override
public boolean equals(Object other)
{
if(this == other)
@@ -91,8 +92,9 @@ public class AlfrescoModelDiff
diff.getOldChecksum().equals(getOldChecksum()) &&
diff.getNewChecksum().equals(getNewChecksum()));
}
public int hashcode()
@Override
public int hashCode()
{
int result = 17;
result = 31 * result + getModelName().hashCode();