diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 2faf7684a..eb27c4090 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -17,6 +17,7 @@ src/test/resources/SearchSuite.xml + 23.0 2.9.10.8 community @@ -79,6 +80,10 @@ + + com.google.guava + guava + org.alfresco alfresco-governance-services-automation-enterprise-rest-api @@ -94,7 +99,7 @@ com.fasterxml.jackson.core jackson-databind - ${jackson.databind.version} + ${dependency.jackson.version} org.alfresco diff --git a/pom.xml b/pom.xml index a30632ebf..012d0a8d3 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ alfresco-search-and-insight-parent 2.0.6-SNAPSHOT pom - Alfresco Search And Insight Parent + Alfresco Search And Insight Engine alfresco-enterprise-releases @@ -53,73 +53,207 @@ enterprise true false + + 1.3 + 13.4 + + 2.13.4 + 2.13.4.2 + + 31.1-jre 1.21 + 3.11 + 2.6 3.0.1 - 1.27 - 2.7.7 2.3.12 - 2.0.6.1 - 5.3.18 4.5.13 1.9.14-atlassian-6 + 2.7.1 + + 1.1.4c + 1.2.0 + 2.3.3 + + 1.32.0 + 1.7.36 + 3.4.8 + + 3.1.0 + + 4.13.2 + 4.6.1 + 2.7.8 + 1.1.0 + 3.19.6 + 2.7.0 + 3.1.8 + + org.codehaus.janino + commons-compiler + ${dependency.janino.version} + + + org.codehaus.janino + janino + ${dependency.janino.version} + + + com.jayway.jsonpath + json-path + ${dependency.jayway.jsonpath.version} + + + com.google.protobuf + protobuf-java + ${dependency.protobuf.version} + + + + com.fasterxml.jackson.core + jackson-core + ${dependency.jackson.version} + + + + com.fasterxml.jackson.core + jackson-annotations + ${dependency.jackson.version} + + + + com.fasterxml.jackson.core + jackson-databind + ${dependency.jackson-databind.version} + + + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + ${dependency.jackson.version} + + + javax.servlet + javax.servlet-api + ${dependency.javax.servlet.api.version} + + + com.carrotsearch.thirdparty + simple-xml-safe + ${dependency.carrotsearch.thirdpaty.simple-xml-safe.version} + + + org.apache.calcite + calcite-core + ${dependency.calcite.version} + + + com.sun.xml.bind + jaxb-xjc + ${dependency.jaxb-xjc.version} + + + jaxen + jaxen + ${dependency.jaxen.version} + + + xpp3 + xpp3 + ${dependency.xpp3.version} + + + com.google.guava + guava + ${dependency.google.guava.version} + + + org.alfresco + alfresco-xmlfactory + ${dependency.alfresco.xml-factory.version} + + + commons-lang + commons-lang + ${dependency.apache-commons-lang.version} + + + org.apache.cxf + cxf-core + ${dependency.cxf.version} + + + org.apache.cxf + cxf-rt-bindings-soap + ${dependency.cxf.version} + + + org.apache.cxf + cxf-rt-bindings-xml + ${dependency.cxf.version} + + + org.apache.cxf + cxf-rt-databinding-jaxb + ${dependency.cxf.version} + + + org.apache.cxf + cxf-rt-frontend-jaxws + ${dependency.cxf.version} + + + org.apache.cxf + cxf-rt-frontend-simple + ${dependency.cxf.version} + + + org.apache.cxf + cxf-rt-transports-http + ${dependency.cxf.version} + + + org.apache.cxf + cxf-rt-ws-addr + ${dependency.cxf.version} + + + org.apache.cxf + cxf-rt-ws-policy + ${dependency.cxf.version} + + + org.apache.cxf + cxf-rt-wsdl + ${dependency.cxf.version} + + + org.apache.commons + commons-lang3 + ${dependency.apache-commons-lang3.version} + + + org.slf4j + slf4j-api + ${dependency.slf4j.version} + + + org.slf4j + slf4j-reload4j + ${dependency.slf4j.version} + org.apache.commons commons-compress ${dependency.apache-commons-compress.version} - - org.apache.tika - tika-core - ${dependency.tika.version} - - - org.apache.tika - tika-java7 - ${dependency.tika.version} - - - org.apache.tika - tika-parsers - ${dependency.tika.version} - - - org.apache.tika - tika-xmp - ${dependency.tika.version} - jakarta.xml.bind jakarta.xml.bind-api ${dependency.jakarta.xml.bind-api.version} - - com.adobe.xmp - xmpcore - 6.1.11 - - - org.apache.hadoop - hadoop-annotations - ${dependency.hadoop.version} - - - org.apache.hadoop - hadoop-auth - ${dependency.hadoop.version} - - - org.apache.hadoop - hadoop-common - ${dependency.hadoop.version} - - - org.apache.hadoop - hadoop-hdfs - ${dependency.hadoop.version} - org.restlet.jee org.restlet @@ -151,6 +285,26 @@ jackson-mapper-asl ${dependency.codehaus.jackson.version} + + junit + junit + ${dependency.junit.version} + + + org.mockito + mockito-core + ${dependency.mockito.version} + + + com.carrotsearch.randomizedtesting + randomizedtesting-runner + ${dependency.carrotsearch.randomizedtesting.version} + + + org.apache.chemistry.opencmis + chemistry-opencmis-client-impl + ${dependency.chemistry.opencmis.version} + diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index 874830fbc..9b036bca1 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -42,6 +42,38 @@ ${solr.version} provided + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + + + org.codehaus.janino + * + + + org.apache.calcite + * + + + org.apache.calcite.avatica + * + + + org.apache.hadoop + * + jdk.tools jdk.tools @@ -52,26 +84,92 @@ - org.apache.solr solr-analysis-extras ${solr.version} provided + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + + + org.codehaus.janino + * + + + org.apache.calcite + * + + + org.apache.calcite.avatica + * + + + org.apache.calcite.avatica + * + log4j log4j + + org.apache.hadoop + * + - org.apache.solr solr-langid ${solr.version} provided + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + + + org.codehaus.janino + * + + + com.adobe.xmp + * + + + org.apache.calcite + * + + + org.apache.calcite.avatica + * + xercesImpl xerces @@ -88,10 +186,18 @@ org.bouncycastle bcprov-jdk15on + + org.apache.hadoop + * + log4j log4j + + org.apache.tika + * + org.apache.poi * @@ -102,13 +208,44 @@ - org.apache.solr solr-clustering ${solr.version} provided + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + + + org.codehaus.janino + * + + + org.apache.calcite + * + + + org.apache.calcite.avatica + * + + + org.apache.hadoop + * + org.simpleframework simple-xml @@ -123,119 +260,103 @@ com.carrotsearch.thirdparty simple-xml-safe - 2.7.1 provided - org.slf4j slf4j-api - ${slf4j.version} - org.slf4j slf4j-reload4j - ${slf4j.version} - jaxen jaxen - 1.2.0 - com.sun.xml.bind jaxb-xjc - 2.3.3 - - - - org.apache.commons - commons-lang3 - 3.11 org.apache.cxf cxf-core - ${cxf.version} org.apache.cxf cxf-rt-bindings-soap - ${cxf.version} org.apache.cxf cxf-rt-bindings-xml - ${cxf.version} org.apache.cxf cxf-rt-databinding-jaxb - ${cxf.version} org.apache.cxf cxf-rt-frontend-jaxws - ${cxf.version} org.apache.cxf cxf-rt-frontend-simple - ${cxf.version} org.apache.cxf cxf-rt-transports-http - ${cxf.version} org.apache.cxf cxf-rt-ws-addr - ${cxf.version} org.apache.cxf cxf-rt-ws-policy - ${cxf.version} org.apache.cxf cxf-rt-wsdl - ${cxf.version} xpp3 xpp3 - 1.1.4c - org.alfresco alfresco-xmlfactory - 1.3 - - - - org.apache.calcite - calcite-core - 1.13.0 - - - org.apache.calcite - calcite-linq4j - 1.13.0 - - org.apache.calcite.avatica - avatica-core - 1.13.0 + com.google.guava + guava + + + org.apache.calcite + calcite-core + + + org.codehaus.janino + commons-compiler + + + org.codehaus.janino + janino + + + com.jayway.jsonpath + json-path + + + com.google.protobuf + protobuf-java + + + org.apache.commons + commons-lang3 + + + commons-lang + commons-lang org.apache.httpcomponents @@ -246,14 +367,12 @@ junit junit - 4.13 test org.mockito mockito-core - 4.6.1 test @@ -263,6 +382,38 @@ ${solr.version} test + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + + + org.codehaus.janino + * + + + org.apache.calcite + * + + + org.apache.calcite.avatica + * + + + org.apache.hadoop + * + log4j log4j @@ -272,13 +423,11 @@ com.carrotsearch.randomizedtesting randomizedtesting-runner - 2.7.8 test org.apache.chemistry.opencmis chemistry-opencmis-client-impl - 1.1.0 test diff --git a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/HighlightComponent.java b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/HighlightComponent.java new file mode 100644 index 000000000..976a1d039 --- /dev/null +++ b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/HighlightComponent.java @@ -0,0 +1,297 @@ +/* + * #%L + * Alfresco Search Services + * %% + * Copyright (C) 2005 - 2020 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.apache.solr.handler.component; + +import java.io.IOException; +import java.net.URL; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Stream; + +import com.google.common.base.Objects; +import org.apache.lucene.search.Query; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.HighlightParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.core.PluginInfo; +import org.apache.solr.core.SolrCore; +import org.apache.solr.highlight.DefaultSolrHighlighter; +import org.apache.solr.highlight.PostingsSolrHighlighter; +import org.apache.solr.highlight.SolrHighlighter; +import org.apache.solr.highlight.UnifiedSolrHighlighter; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.search.QParser; +import org.apache.solr.search.QParserPlugin; +import org.apache.solr.search.QueryParsing; +import org.apache.solr.search.SyntaxError; +import org.apache.solr.util.SolrPluginUtils; +import org.apache.solr.util.plugin.PluginInfoInitialized; +import org.apache.solr.util.plugin.SolrCoreAware; + +import static java.util.Optional.ofNullable; +import static java.util.stream.Collectors.toMap; + +/** + * TODO! + * + * + * @since solr 1.3 + */ +public class HighlightComponent extends SearchComponent implements PluginInfoInitialized, SolrCoreAware +{ + public enum HighlightMethod { + UNIFIED("unified"), + FAST_VECTOR("fastVector"), + POSTINGS("postings"), + ORIGINAL("original"); + + private static final Map METHODS = Collections.unmodifiableMap(Stream.of(values()) + .collect(toMap(HighlightMethod::getMethodName, Function.identity()))); + + private final String methodName; + + HighlightMethod(String method) { + this.methodName = method; + } + + public String getMethodName() { + return methodName; + } + + public static HighlightMethod parse(String method) { + return METHODS.get(method); + } + } + + public static final String COMPONENT_NAME = "highlight"; + + private PluginInfo info = PluginInfo.EMPTY_INFO; + + @Deprecated // DWS: in 7.0 lets restructure the abstractions/relationships + private SolrHighlighter solrConfigHighlighter; + + /** + * @deprecated instead depend on {@link #process(ResponseBuilder)} to choose the highlighter based on + * {@link HighlightParams#METHOD} + */ + @Deprecated + public static SolrHighlighter getHighlighter(SolrCore core) { + HighlightComponent hl = (HighlightComponent) core.getSearchComponents().get(HighlightComponent.COMPONENT_NAME); + return hl==null ? null: hl.getHighlighter(); + } + + @Deprecated + public SolrHighlighter getHighlighter() { + return solrConfigHighlighter; + } + + @Override + public void init(PluginInfo info) { + this.info = info; + } + + @Override + public void prepare(ResponseBuilder rb) throws IOException { + SolrParams params = rb.req.getParams(); + rb.doHighlights = solrConfigHighlighter.isHighlightingEnabled(params); + if(rb.doHighlights){ + rb.setNeedDocList(true); + String hlq = params.get(HighlightParams.Q); + String hlparser = + ofNullable(params.get(HighlightParams.QPARSER)) + .orElseGet( () -> params.get(QueryParsing.DEFTYPE, QParserPlugin.DEFAULT_QTYPE)); + if(hlq != null){ + try { + QParser parser = QParser.getParser(hlq, hlparser, rb.req); + rb.setHighlightQuery(parser.getHighlightQuery()); + } catch (SyntaxError e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); + } + } + } + } + + @Override + public void inform(SolrCore core) { + List children = info.getChildren("highlighting"); + if(children.isEmpty()) { + PluginInfo pluginInfo = core.getSolrConfig().getPluginInfo(SolrHighlighter.class.getName()); //TODO deprecated configuration remove later + if (pluginInfo != null) { + solrConfigHighlighter = core.createInitInstance(pluginInfo, SolrHighlighter.class, null, DefaultSolrHighlighter.class.getName()); + } else { + DefaultSolrHighlighter defHighlighter = new DefaultSolrHighlighter(core); + defHighlighter.init(PluginInfo.EMPTY_INFO); + solrConfigHighlighter = defHighlighter; + } + } else { + solrConfigHighlighter = core.createInitInstance(children.get(0),SolrHighlighter.class,null, DefaultSolrHighlighter.class.getName()); + } + + } + + @Override + public void process(ResponseBuilder rb) throws IOException { + + if (rb.doHighlights) { + SolrQueryRequest req = rb.req; + SolrParams params = req.getParams(); + + SolrHighlighter highlighter = getHighlighter(params); + + String[] defaultHighlightFields; //TODO: get from builder by default? + if (rb.getQparser() != null) { + defaultHighlightFields = rb.getQparser().getDefaultHighlightFields(); + } else { + defaultHighlightFields = params.getParams(CommonParams.DF); + } + + Query highlightQuery = rb.getHighlightQuery(); + if(highlightQuery==null) { + if (rb.getQparser() != null) { + try { + highlightQuery = rb.getQparser().getHighlightQuery(); + rb.setHighlightQuery( highlightQuery ); + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); + } + } else { + highlightQuery = rb.getQuery(); + rb.setHighlightQuery( highlightQuery ); + } + } + + // No highlighting if there is no query -- consider q.alt=*:* + if( highlightQuery != null ) { + NamedList sumData = highlighter.doHighlighting( + rb.getResults().docList, + highlightQuery, + req, defaultHighlightFields ); + + if(sumData != null) { + // TODO ???? add this directly to the response? + rb.rsp.add("highlighting", sumData); + } + } + } + } + + protected SolrHighlighter getHighlighter(SolrParams params) { + HighlightMethod method = HighlightMethod.parse(params.get(HighlightParams.METHOD)); + if (method == null) { + return solrConfigHighlighter; + } + + switch (method) { + case UNIFIED: + if (solrConfigHighlighter instanceof UnifiedSolrHighlighter) { + return solrConfigHighlighter; + } + return new UnifiedSolrHighlighter(); // TODO cache one? + case POSTINGS: + if (solrConfigHighlighter instanceof PostingsSolrHighlighter) { + return solrConfigHighlighter; + } + return new PostingsSolrHighlighter(); // TODO cache one? + case FAST_VECTOR: // fall-through + case ORIGINAL: + if (solrConfigHighlighter instanceof DefaultSolrHighlighter) { + return solrConfigHighlighter; + } else { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "In order to use " + HighlightParams.METHOD + "=" + method.getMethodName() + " the configured" + + " highlighter in solrconfig must be " + DefaultSolrHighlighter.class); + } + default: throw new AssertionError(); + } + } + + @Override + public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) { + if (!rb.doHighlights) return; + + // Turn on highlighting only only when retrieving fields + if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) { + sreq.purpose |= ShardRequest.PURPOSE_GET_HIGHLIGHTS; + // should already be true... + sreq.params.set(HighlightParams.HIGHLIGHT, "true"); + } else { + sreq.params.set(HighlightParams.HIGHLIGHT, "false"); + } + } + + @Override + public void handleResponses(ResponseBuilder rb, ShardRequest sreq) { + } + + @Override + public void finishStage(ResponseBuilder rb) { + if (rb.doHighlights && rb.stage == ResponseBuilder.STAGE_GET_FIELDS) { + + NamedList.NamedListEntry[] arr = new NamedList.NamedListEntry[rb.resultIds.size()]; + + // TODO: make a generic routine to do automatic merging of id keyed data + for (ShardRequest sreq : rb.finished) { + if ((sreq.purpose & ShardRequest.PURPOSE_GET_HIGHLIGHTS) == 0) continue; + for (ShardResponse srsp : sreq.responses) { + if (srsp.getException() != null) { + // can't expect the highlight content if there was an exception for this request + // this should only happen when using shards.tolerant=true + continue; + } + NamedList hl = (NamedList)srsp.getSolrResponse().getResponse().get("highlighting"); + SolrPluginUtils.copyNamedListIntoArrayByDocPosInResponse(hl, rb.resultIds, arr); + } + } + + // remove nulls in case not all docs were able to be retrieved + rb.rsp.add("highlighting", SolrPluginUtils.removeNulls(arr, new SimpleOrderedMap<>())); + } + } + + //////////////////////////////////////////// + /// SolrInfoMBean + //////////////////////////////////////////// + + @Override + public String getDescription() { + return "Highlighting"; + } + + @Override + public Category getCategory() { + return Category.HIGHLIGHTER; + } + + @Override + public URL[] getDocs() { + return null; + } +} diff --git a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/StatsValuesFactory.java b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/StatsValuesFactory.java new file mode 100644 index 000000000..e82dedbea --- /dev/null +++ b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/StatsValuesFactory.java @@ -0,0 +1,935 @@ +/* + * #%L + * Alfresco Search Services + * %% + * Copyright (C) 2005 - 2020 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.apache.solr.handler.component; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.*; +import java.nio.ByteBuffer; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.util.BytesRef; +import org.apache.solr.common.EnumFieldValue; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.handler.component.StatsField.Stat; +import org.apache.solr.schema.*; + +import com.tdunning.math.stats.AVLTreeDigest; +import com.google.common.hash.HashFunction; + +import org.apache.solr.util.hll.HLL; +import org.apache.solr.util.hll.HLLType; + +/** + * Factory class for creating instance of + * {@link org.apache.solr.handler.component.StatsValues} + */ +public class StatsValuesFactory { + + /** + * Creates an instance of StatsValues which supports values from the specified + * {@link StatsField} + * + * @param statsField + * {@link StatsField} whose statistics will be created by the + * resulting {@link StatsValues} + * @return Instance of {@link StatsValues} that will create statistics from + * values from the specified {@link StatsField} + */ + public static StatsValues createStatsValues(StatsField statsField) { + + final SchemaField sf = statsField.getSchemaField(); + + if (null == sf) { + // function stats + return new NumericStatsValues(statsField); + } + + final FieldType fieldType = sf.getType(); // TODO: allow FieldType to provide impl. + + if (TrieDateField.class.isInstance(fieldType) || DatePointField.class.isInstance(fieldType)) { + DateStatsValues statsValues = new DateStatsValues(statsField); + if (sf.multiValued()) { + return new SortedDateStatsValues(statsValues, statsField); + } + return statsValues; + } else if (TrieField.class.isInstance(fieldType) || PointField.class.isInstance(fieldType)) { + + NumericStatsValues statsValue = new NumericStatsValues(statsField); + if (sf.multiValued()) { + return new SortedNumericStatsValues(statsValue, statsField); + } + return statsValue; + } else if (StrField.class.isInstance(fieldType)) { + return new StringStatsValues(statsField); + } else if (sf.getType().getClass().equals(EnumField.class)) { + return new EnumStatsValues(statsField); + } else { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Field type " + fieldType + " is not currently supported"); + } + } +} + +/** + * Abstract implementation of + * {@link org.apache.solr.handler.component.StatsValues} that provides the + * default behavior for most StatsValues implementations. + * + * There are very few requirements placed on what statistics concrete + * implementations should collect, with the only required statistics being the + * minimum and maximum values. + */ +abstract class AbstractStatsValues implements StatsValues { + private static final String FACETS = "facets"; + + /** Tracks all data about tthe stats we need to collect */ + final protected StatsField statsField; + + /** may be null if we are collecting stats directly from a function ValueSource */ + final protected SchemaField sf; + /** + * may be null if we are collecting stats directly from a function ValueSource + */ + final protected FieldType ft; + + // final booleans from StatsField to allow better inlining & JIT optimizing + final protected boolean computeCount; + final protected boolean computeMissing; + final protected boolean computeCalcDistinct; // needed for either countDistinct or distinctValues + final protected boolean computeMin; + final protected boolean computeMax; + final protected boolean computeMinOrMax; + final protected boolean computeCardinality; + + /** + * Either a function value source to collect from, or the ValueSource associated + * with a single valued field we are collecting from. Will be null until/unless + * {@link #setNextReader} is called at least once + */ + private ValueSource valueSource; + /** + * Context to use when retrieving FunctionValues, will be null until/unless + * {@link #setNextReader} is called at least once + */ + private Map vsContext; + /** + * Values to collect, will be null until/unless {@link #setNextReader} is + * called at least once + */ + protected FunctionValues values; + + protected T max; + protected T min; + protected long missing; + protected long count; + protected long countDistinct; + protected final Set distinctValues; + + /** + * Hash function that must be used by implementations of {@link #hash} + */ + protected final HashFunction hasher; + // if null, no HLL logic can be computed; not final because of "union" optimization (see below) + private HLL hll; + + // facetField facetValue + protected Map> facets = new HashMap<>(); + + protected AbstractStatsValues(StatsField statsField) { + this.statsField = statsField; + this.computeCount = statsField.calculateStats(Stat.count); + this.computeMissing = statsField.calculateStats(Stat.missing); + this.computeCalcDistinct = statsField.calculateStats(Stat.countDistinct) + || statsField.calculateStats(Stat.distinctValues); + this.computeMin = statsField.calculateStats(Stat.min); + this.computeMax = statsField.calculateStats(Stat.max); + this.computeMinOrMax = computeMin || computeMax; + + this.distinctValues = computeCalcDistinct ? new TreeSet<>() : null; + + this.computeCardinality = statsField.calculateStats(Stat.cardinality); + if ( computeCardinality ) { + + hasher = statsField.getHllOptions().getHasher(); + hll = statsField.getHllOptions().newHLL(); + assert null != hll : "Cardinality requires an HLL"; + } else { + hll = null; + hasher = null; + } + + // alternatively, we could refactor a common base class that doesn't know/care + // about either SchemaField or ValueSource - but then there would be a lot of + // duplicate code between "NumericSchemaFieldStatsValues" and + // "NumericValueSourceStatsValues" which would have diff parent classes + // + // part of the complexity here being that the StatsValues API serves two + // masters: collecting concrete Values from things like DocValuesStats and + // the distributed aggregation logic, but also collecting docIds which it + // then + // uses to go out and pull concreate values from the ValueSource + // (from a func, or single valued field) + if (null != statsField.getSchemaField()) { + assert null == statsField.getValueSource(); + this.sf = statsField.getSchemaField(); + this.ft = sf.getType(); + } else { + assert null != statsField.getValueSource(); + assert null == statsField.getSchemaField(); + this.sf = null; + this.ft = null; + } + } + + /** + * {@inheritDoc} + */ + @Override + public void accumulate(NamedList stv) { + if (computeCount) { + count += (Long) stv.get("count"); + } + if (computeMissing) { + missing += (Long) stv.get("missing"); + } + if (computeCalcDistinct) { + distinctValues.addAll((Collection) stv.get("distinctValues")); + countDistinct = distinctValues.size(); + } + + if (computeMinOrMax) { + updateMinMax((T) stv.get("min"), (T) stv.get("max")); + } + + if (computeCardinality) { + byte[] data = (byte[]) stv.get("cardinality"); + HLL other = HLL.fromBytes(data); + if (hll.getType().equals(HLLType.EMPTY)) { + // The HLL.union method goes out of it's way not to modify the "other" HLL. + // Which means in the case of merging into an "EMPTY" HLL (garunteed to happen at + // least once in every coordination of shard requests) it always clones all + // of the internal storage -- but since we're going to throw "other" away after + // the merge, this just means a short term doubling of RAM that we can skip. + hll = other; + } else { + hll.union(other); + } + } + + updateTypeSpecificStats(stv); + + NamedList f = (NamedList) stv.get(FACETS); + if (f == null) { + return; + } + + for (int i = 0; i < f.size(); i++) { + String field = f.getName(i); + NamedList vals = (NamedList) f.getVal(i); + Map addTo = facets.get(field); + if (addTo == null) { + addTo = new HashMap<>(); + facets.put(field, addTo); + } + for (int j = 0; j < vals.size(); j++) { + String val = vals.getName(j); + StatsValues vvals = addTo.get(val); + if (vvals == null) { + vvals = StatsValuesFactory.createStatsValues(statsField); + addTo.put(val, vvals); + } + vvals.accumulate((NamedList) vals.getVal(j)); + } + } + } + + /** + * {@inheritDoc} + */ + @Override + public void accumulate(BytesRef value, int count) { + if (null == ft) { + throw new IllegalStateException( + "Can't collect & convert BytesRefs on stats that do't use a a FieldType: " + + statsField); + } + T typedValue = (T) ft.toObject(sf, value); + accumulate(typedValue, count); + } + + public void accumulate(T value, int count) { + assert null != value : "Can't accumulate null"; + + if (computeCount) { + this.count += count; + } + if (computeCalcDistinct) { + distinctValues.add(value); + countDistinct = distinctValues.size(); + } + if (computeMinOrMax) { + updateMinMax(value, value); + } + if (computeCardinality) { + if (null == hasher) { + assert value instanceof Number : "pre-hashed value support only works with numeric longs"; + hll.addRaw(((Number)value).longValue()); + } else { + hll.addRaw(hash(value)); + } + } + updateTypeSpecificStats(value, count); + } + + /** + * {@inheritDoc} + */ + @Override + public void missing() { + if (computeMissing) { + missing++; + } + } + + /** + * {@inheritDoc} + */ + @Override + public void addMissing(int count) { + missing += count; + } + + /** + * {@inheritDoc} + */ + @Override + public void addFacet(String facetName, Map facetValues) { + facets.put(facetName, facetValues); + } + + /** + * {@inheritDoc} + */ + @Override + public NamedList getStatsValues() { + NamedList res = new SimpleOrderedMap<>(); + + if (statsField.includeInResponse(Stat.min)) { + res.add("min", min); + } + if (statsField.includeInResponse(Stat.max)) { + res.add("max", max); + } + if (statsField.includeInResponse(Stat.count)) { + res.add("count", count); + } + if (statsField.includeInResponse(Stat.missing)) { + res.add("missing", missing); + } + if (statsField.includeInResponse(Stat.distinctValues)) { + res.add("distinctValues", distinctValues); + } + if (statsField.includeInResponse(Stat.countDistinct)) { + res.add("countDistinct", countDistinct); + } + if (statsField.includeInResponse(Stat.cardinality)) { + if (statsField.getIsShard()) { + res.add("cardinality", hll.toBytes()); + } else { + res.add("cardinality", hll.cardinality()); + } + } + + addTypeSpecificStats(res); + + if (!facets.isEmpty()) { + + // add the facet stats + NamedList> nl = new SimpleOrderedMap<>(); + for (Map.Entry> entry : facets.entrySet()) { + NamedList> nl2 = new SimpleOrderedMap<>(); + nl.add(entry.getKey(), nl2); + for (Map.Entry e2 : entry.getValue().entrySet()) { + nl2.add(e2.getKey(), e2.getValue().getStatsValues()); + } + } + + res.add(FACETS, nl); + } + + return res; + } + + /** + * {@inheritDoc} + */ + public void setNextReader(LeafReaderContext ctx) throws IOException { + if (valueSource == null) { + // first time we've collected local values, get the right ValueSource + valueSource = (null == ft) + ? statsField.getValueSource() + : ft.getValueSource(sf, null); + vsContext = ValueSource.newContext(statsField.getSearcher()); + } + values = valueSource.getValues(vsContext, ctx); + } + + /** + * Hash function to be used for computing cardinality. + * + * This method will not be called in cases where the user has indicated the values + * are already hashed. If this method is called, then {@link #hasher} will be non-null, + * and should be used to generate the appropriate hash value. + * + * @see Stat#cardinality + * @see #hasher + */ + protected abstract long hash(T value); + + /** + * Updates the minimum and maximum statistics based on the given values + * + * @param min + * Value that the current minimum should be updated against + * @param max + * Value that the current maximum should be updated against + */ + protected abstract void updateMinMax(T min, T max); + + /** + * Updates the type specific statistics based on the given value + * + * @param value + * Value the statistics should be updated against + * @param count + * Number of times the value is being accumulated + */ + protected abstract void updateTypeSpecificStats(T value, int count); + + /** + * Updates the type specific statistics based on the values in the given list + * + * @param stv + * List containing values the current statistics should be updated + * against + */ + protected abstract void updateTypeSpecificStats(NamedList stv); + + /** + * Add any type specific statistics to the given NamedList + * + * @param res + * NamedList to add the type specific statistics too + */ + protected abstract void addTypeSpecificStats(NamedList res); +} + +/** + * Implementation of StatsValues that supports Double values + */ +class NumericStatsValues extends AbstractStatsValues { + + double sum; + double sumOfSquares; + + AVLTreeDigest tdigest; + + double minD; // perf optimization, only valid if (null != this.min) + double maxD; // perf optimization, only valid if (null != this.max) + + final protected boolean computeSum; + final protected boolean computeSumOfSquares; + final protected boolean computePercentiles; + + public NumericStatsValues(StatsField statsField) { + super(statsField); + + this.computeSum = statsField.calculateStats(Stat.sum); + this.computeSumOfSquares = statsField.calculateStats(Stat.sumOfSquares); + + this.computePercentiles = statsField.calculateStats(Stat.percentiles); + if ( computePercentiles ) { + tdigest = new AVLTreeDigest(statsField.getTdigestCompression()); + } + + } + + @Override + public long hash(Number v) { + // have to use a bit of reflection to ensure good hash values since + // we don't have truely type specific stats + if (v instanceof Long) { + return hasher.hashLong(v.longValue()).asLong(); + } else if (v instanceof Integer) { + return hasher.hashInt(v.intValue()).asLong(); + } else if (v instanceof Double) { + return hasher.hashLong(Double.doubleToRawLongBits(v.doubleValue())).asLong(); + } else if (v instanceof Float) { + return hasher.hashInt(Float.floatToRawIntBits(v.floatValue())).asLong(); + } else if (v instanceof Byte) { + return hasher.newHasher().putByte(v.byteValue()).hash().asLong(); + } else if (v instanceof Short) { + return hasher.newHasher().putShort(v.shortValue()).hash().asLong(); + } + // else... + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Unsupported Numeric Type ("+v.getClass()+") for hashing: " +statsField); + } + + @Override + public void accumulate(int docID) { + if (values.exists(docID)) { + Number value = (Number) values.objectVal(docID); + accumulate(value, 1); + } else { + missing(); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void updateTypeSpecificStats(NamedList stv) { + if (computeSum) { + sum += ((Number) stv.get("sum")).doubleValue(); + } + if (computeSumOfSquares) { + sumOfSquares += ((Number) stv.get("sumOfSquares")).doubleValue(); + } + + if (computePercentiles) { + byte[] data = (byte[]) stv.get("percentiles"); + ByteBuffer buf = ByteBuffer.wrap(data); + tdigest.add(AVLTreeDigest.fromBytes(buf)); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void updateTypeSpecificStats(Number v, int count) { + double value = v.doubleValue(); + if (computeSumOfSquares) { + sumOfSquares += (value * value * count); // for std deviation + } + if (computeSum) { + sum += value * count; + } + if (computePercentiles) { + tdigest.add(value, count); + } + } + + /** + * {@inheritDoc} + */ + @Override + protected void updateMinMax(Number min, Number max) { + // we always use the double values, because that way the response Object class is + // consistent regardless of whether we only have 1 value or many that we min/max + // + // TODO: would be nice to have subclasses for each type of Number ... breaks backcompat + + if (computeMin) { // nested if to encourage JIT to optimize aware final var? + if (null != min) { + double minD = min.doubleValue(); + if (null == this.min || minD < this.minD) { + // Double for result & cached primitive double to minimize unboxing in future comparisons + this.min = this.minD = minD; + } + } + } + if (computeMax) { // nested if to encourage JIT to optimize aware final var? + if (null != max) { + double maxD = max.doubleValue(); + if (null == this.max || this.maxD < maxD) { + // Double for result & cached primitive double to minimize unboxing in future comparisons + this.max = this.maxD = maxD; + } + } + } + } + + /** + * Adds sum, sumOfSquares, mean, stddev, and percentiles to the given + * NamedList + * + * @param res + * NamedList to add the type specific statistics too + */ + @Override + protected void addTypeSpecificStats(NamedList res) { + if (statsField.includeInResponse(Stat.sum)) { + res.add("sum", sum); + } + if (statsField.includeInResponse(Stat.sumOfSquares)) { + res.add("sumOfSquares", sumOfSquares); + } + if (statsField.includeInResponse(Stat.mean)) { + res.add("mean", sum / count); + } + if (statsField.includeInResponse(Stat.stddev)) { + res.add("stddev", getStandardDeviation()); + } + if (statsField.includeInResponse(Stat.percentiles)) { + if (statsField.getIsShard()) { + // as of current t-digest version, smallByteSize() internally does a full conversion in + // order to determine what the size is (can't be precomputed?) .. so rather then + // serialize to a ByteBuffer twice, allocate the max possible size buffer, + // serialize once, and then copy only the byte[] subset that we need, and free up the buffer + ByteBuffer buf = ByteBuffer.allocate(tdigest.byteSize()); // upper bound + tdigest.asSmallBytes(buf); + res.add("percentiles", Arrays.copyOf(buf.array(), buf.position()) ); + } else { + NamedList percentileNameList = new NamedList(); + for (Double percentile : statsField.getPercentilesList()) { + // Empty document set case + if (tdigest.size() == 0) { + percentileNameList.add(percentile.toString(), null); + } else { + Double cutoff = tdigest.quantile(percentile / 100); + percentileNameList.add(percentile.toString(), cutoff); + } + } + res.add("percentiles", percentileNameList); + } + } + } + + + /** + * Calculates the standard deviation statistic + * + * @return Standard deviation statistic + */ + private double getStandardDeviation() { + if (count <= 1.0D) { + return 0.0D; + } + + return Math.sqrt(((count * sumOfSquares) - (sum * sum)) / (count * (count - 1.0D))); + + } +} + +/** + * Implementation of StatsValues that supports EnumField values + */ +class EnumStatsValues extends AbstractStatsValues { + + public EnumStatsValues(StatsField statsField) { + super(statsField); + } + + @Override + public long hash(EnumFieldValue v) { + return hasher.hashInt(v.toInt().intValue()).asLong(); + } + + /** + * {@inheritDoc} + */ + @Override + public void accumulate(int docID) { + if (values.exists(docID)) { + Integer intValue = (Integer) values.objectVal(docID); + String stringValue = values.strVal(docID); + EnumFieldValue enumFieldValue = new EnumFieldValue(intValue, stringValue); + accumulate(enumFieldValue, 1); + } else { + missing(); + } + } + + /** + * {@inheritDoc} + */ + protected void updateMinMax(EnumFieldValue min, EnumFieldValue max) { + if (computeMin) { // nested if to encourage JIT to optimize aware final var? + if (null != min) { + if (null == this.min || (min.compareTo(this.min) < 0)) { + this.min = min; + } + } + } + if (computeMax) { // nested if to encourage JIT to optimize aware final var? + if (null != max) { + if (null == this.max || (max.compareTo(this.max) > 0)) { + this.max = max; + } + } + } + } + + /** + * {@inheritDoc} + */ + @Override + protected void updateTypeSpecificStats(NamedList stv) { + // No type specific stats + } + + /** + * {@inheritDoc} + */ + @Override + protected void updateTypeSpecificStats(EnumFieldValue value, int count) { + // No type specific stats + } + + /** + * Adds no type specific statistics + */ + @Override + protected void addTypeSpecificStats(NamedList res) { + // Add no statistics + } + +} + +/** + * /** Implementation of StatsValues that supports Date values + */ +class DateStatsValues extends AbstractStatsValues { + + private double sum = 0.0; + double sumOfSquares = 0; + + final protected boolean computeSum; + final protected boolean computeSumOfSquares; + + public DateStatsValues(StatsField statsField) { + super(statsField); + this.computeSum = statsField.calculateStats(Stat.sum); + this.computeSumOfSquares = statsField.calculateStats(Stat.sumOfSquares); + } + + @Override + public long hash(Date v) { + return hasher.hashLong(v.getTime()).asLong(); + } + + @Override + public void accumulate(int docID) { + if (values.exists(docID)) { + accumulate((Date) values.objectVal(docID), 1); + } else { + missing(); + } + } + + /** + * {@inheritDoc} + */ + @Override + protected void updateTypeSpecificStats(NamedList stv) { + if (computeSum) { + sum += ((Number) stv.get("sum")).doubleValue(); + } + if (computeSumOfSquares) { + sumOfSquares += ((Number) stv.get("sumOfSquares")).doubleValue(); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void updateTypeSpecificStats(Date v, int count) { + long value = v.getTime(); + if (computeSumOfSquares) { + sumOfSquares += ((double)value * value * count); // for std deviation + } + if (computeSum) { + sum += value * count; + } + } + + /** + * {@inheritDoc} + */ + @Override + protected void updateMinMax(Date min, Date max) { + if (computeMin) { // nested if to encourage JIT to optimize aware final var? + if (null != min && (this.min==null || this.min.after(min))) { + this.min = min; + } + } + if (computeMax) { // nested if to encourage JIT to optimize aware final var? + if (null != max && (this.max==null || this.max.before(max))) { + this.max = max; + } + } + } + + /** + * Adds sum and mean statistics to the given NamedList + * + * @param res + * NamedList to add the type specific statistics too + */ + @Override + protected void addTypeSpecificStats(NamedList res) { + if (statsField.includeInResponse(Stat.sum)) { + res.add("sum", sum); + } + if (statsField.includeInResponse(Stat.mean)) { + res.add("mean", (count > 0) ? new Date((long)(sum / count)) : null); + } + if (statsField.includeInResponse(Stat.sumOfSquares)) { + res.add("sumOfSquares", sumOfSquares); + } + if (statsField.includeInResponse(Stat.stddev)) { + res.add("stddev", getStandardDeviation()); + } + } + + /** + * Calculates the standard deviation. For dates, this is really the MS + * deviation + * + * @return Standard deviation statistic + */ + private double getStandardDeviation() { + if (count <= 1) { + return 0.0D; + } + return Math.sqrt(((count * sumOfSquares) - (sum * sum)) + / (count * (count - 1.0D))); + } +} + +/** + * Implementation of StatsValues that supports String values + */ +class StringStatsValues extends AbstractStatsValues { + + public StringStatsValues(StatsField statsField) { + super(statsField); + } + + @Override + public long hash(String v) { + return hasher.hashString(v, Charset.defaultCharset()).asLong(); + } + + @Override + public void accumulate(int docID) { + if (values.exists(docID)) { + String value = values.strVal(docID); + if (value != null) { + accumulate(value, 1); + } else { + missing(); + } + } else { + missing(); + } + } + + /** + * {@inheritDoc} + */ + @Override + protected void updateTypeSpecificStats(NamedList stv) { + // No type specific stats + } + + /** + * {@inheritDoc} + */ + @Override + protected void updateTypeSpecificStats(String value, int count) { + // No type specific stats + } + + /** + * {@inheritDoc} + */ + @Override + protected void updateMinMax(String min, String max) { + if (computeMin) { // nested if to encourage JIT to optimize aware final var? + this.min = min(this.min, min); + } + if (computeMax) { // nested if to encourage JIT to optimize aware final var? + this.max = max(this.max, max); + } + } + + /** + * Adds no type specific statistics + */ + @Override + protected void addTypeSpecificStats(NamedList res) { + // Add no statistics + } + + /** + * Determines which of the given Strings is the maximum, as computed by + * {@link String#compareTo(String)} + * + * @param str1 + * String to compare against b + * @param str2 + * String compared against a + * @return str1 if it is considered greater by + * {@link String#compareTo(String)}, str2 otherwise + */ + private static String max(String str1, String str2) { + if (str1 == null) { + return str2; + } else if (str2 == null) { + return str1; + } + return (str1.compareTo(str2) > 0) ? str1 : str2; + } + + /** + * Determines which of the given Strings is the minimum, as computed by + * {@link String#compareTo(String)} + * + * @param str1 + * String to compare against b + * @param str2 + * String compared against a + * @return str1 if it is considered less by {@link String#compareTo(String)}, + * str2 otherwise + */ + private static String min(String str1, String str2) { + if (str1 == null) { + return str2; + } else if (str2 == null) { + return str1; + } + return (str1.compareTo(str2) < 0) ? str1 : str2; + } +} diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrconfig.xml b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrconfig.xml index 10d546ba9..38d47cf45 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrconfig.xml +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrconfig.xml @@ -921,103 +921,6 @@ - - - - - explicit - - - velocity - browse - layout - Solritas - - - edismax - - text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 - title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0 - - text - 100% - *:* - 10 - *,score - - - text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 - title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0 - - text,features,name,sku,id,manu,cat,title,description,keywords,author,resourcename - 3 - - - on - cat - manu_exact - content_type - author_s - ipod - GB - 1 - cat,inStock - after - price - 0 - 600 - 50 - popularity - 0 - 10 - 3 - manufacturedate_dt - NOW/YEAR-10YEARS - NOW - +1YEAR - before - after - - - on - content features title name - html - <b> - </b> - 0 - title - 0 - name - 3 - 200 - content - 750 - - - on - false - 5 - 2 - 5 - true - true - 5 - 3 - - - - - spellcheck - - text/plain; charset=UTF-8 - - - - - - - explicit - - - velocity - browse - layout - Solritas - - - edismax - - text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 - title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0 - - text - 100% - *:* - 10 - *,score - - - text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 - title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0 - - text,features,name,sku,id,manu,cat,title,description,keywords,author,resourcename - 3 - - - on - cat - manu_exact - content_type - author_s - ipod - GB - 1 - cat,inStock - after - price - 0 - 600 - 50 - popularity - 0 - 10 - 3 - manufacturedate_dt - NOW/YEAR-10YEARS - NOW - +1YEAR - before - after - - - on - content features title name - html - <b> - </b> - 0 - title - 0 - name - 3 - 200 - content - 750 - - - on - false - 5 - 2 - 5 - true - true - 5 - 3 - - - - - spellcheck - - - - - maven-jar-plugin @@ -106,7 +100,7 @@ **/SOLRAPIClientTest.java diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/tracker/TrackerStats.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/tracker/TrackerStats.java index a13c92011..81ec7039d 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/tracker/TrackerStats.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/tracker/TrackerStats.java @@ -1,41 +1,39 @@ -/* - * #%L - * Alfresco Search Services - * %% - * Copyright (C) 2005 - 2020 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ - +/* + * #%L + * Alfresco Search Services + * %% + * Copyright (C) 2005 - 2020 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ + package org.alfresco.solr.tracker; import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.HashMap; -import java.util.List; -import java.util.Map.Entry; -import java.util.concurrent.ConcurrentHashMap; - -import javax.annotation.concurrent.NotThreadSafe; - +import java.util.List; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; + import org.alfresco.solr.InformationServerCollectionProvider; import org.alfresco.solr.adapters.ISimpleOrderedMap; import org.alfresco.util.Pair; @@ -287,10 +285,10 @@ public class TrackerStats map.add("StdDev", getStandardDeviation()); if (incdludeDetail) { - for (Entry copy : copies.entrySet()) - { - map.add(copy.getKey(), copy.getValue().getNamedList(includeHist, includeValues)); - } + for (Entry copy : copies.entrySet()) + { + map.add(copy.getKey(), copy.getValue().getNamedList(includeHist, includeValues)); + } } return map; @@ -385,7 +383,6 @@ public class TrackerStats } - @NotThreadSafe public static class IncrementalStats { Date start = new Date(); diff --git a/search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/client/SOLRAPIClientFactoryTest.java b/search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/client/SOLRAPIClientFactoryTest.java index fccc36ed6..7df0ece15 100644 --- a/search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/client/SOLRAPIClientFactoryTest.java +++ b/search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/client/SOLRAPIClientFactoryTest.java @@ -26,23 +26,24 @@ package org.alfresco.solr.client; -import static org.mockito.Mockito.*; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertSame; - -import java.util.Properties; - import org.alfresco.encryption.KeyResourceLoader; import org.alfresco.repo.dictionary.NamespaceDAO; import org.alfresco.service.cmr.dictionary.DictionaryService; -import org.alfresco.solr.client.SOLRAPIClient; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; +import java.util.Properties; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.when; + @RunWith(MockitoJUnitRunner.class) public class SOLRAPIClientFactoryTest { diff --git a/search-services/packaging/pom.xml b/search-services/packaging/pom.xml index bc4757ad2..f1492067b 100644 --- a/search-services/packaging/pom.xml +++ b/search-services/packaging/pom.xml @@ -32,7 +32,6 @@ junit junit - 4.13 test @@ -159,23 +158,30 @@ + + + + + - - - - + + + + + + diff --git a/search-services/packaging/src/main/resources/licenses/notice.txt b/search-services/packaging/src/main/resources/licenses/notice.txt index 59d913bb0..e3daee6dc 100644 --- a/search-services/packaging/src/main/resources/licenses/notice.txt +++ b/search-services/packaging/src/main/resources/licenses/notice.txt @@ -14,53 +14,57 @@ The Apache Lucene project page is https://lucene.apache.org/ === Common Public 1.0 === wsdl4j-1.6.3.jar http://sourceforge.net/projects/wsdl4j - === BSD 3 === antlr-runtime-3.5.2.jar http://www.antlr.org/ ST4-4.0.8.jar https://www.stringtemplate.org/license.html antlr-3.5.2.jar http://www.antlr.org/ jaxen-1.2.0.jar http://www.cafeconleche.org/jaxen/ -jsr305-3.0.1.jar http://code.google.com/p/jsr-305/ - === BSD variant License === xpp3-1.1.4c.jar http://www.extreme.indiana.edu/dist/java-repository/xpp3/licenses/LICENSE.txt - === JSON === json-20210307.jar https://github.com/stleary/JSON-java - === Apache 2.0 === +error_prone_annotations-2.11.0.jar https://github.com/google/error-prone +jsr305-3.0.2.jar https://mvnrepository.com/artifact/com.google.code.findbugs/jsr305/1.3.9 +geronimo-jta_1.1_spec-1.1.1.jar https://mvnrepository.com/artifact/org.apache.geronimo.specs/geronimo-jta_1.1_spec/1.1.1 +proj4j-1.1.5.jar https://github.com/locationtech/proj4j +j2objc-annotations-1.3.jar https://github.com/google/j2objc +accessors-smart-2.4.7.jar https://github.com/netplex/json-smart-v2 +json-path-2.7.0.jar https://github.com/json-path/JsonPath +json-smart-2.4.7.jar https://github.com/netplex/json-smart-v2 +json-simple-1.1.1.jar https://github.com/fangyidong/json-simple +apiguardian-api-1.1.2.jar https://github.com/apiguardian-team/apiguardian xml-resolver-1.2.jar https://github.com/FasterXML/jackson neethi-3.1.1.jar http://ws.apache.org/commons/neethi/ -commons-dbcp-1.4.jar http://jakarta.apache.org/commons/ commons-logging-1.2.jar http://jakarta.apache.org/commons/ commons-lang3-3.11.jar http://jakarta.apache.org/commons/ -commons-pool-1.5.4.jar http://jakarta.apache.org/commons/ commons-compress-1.21.jar https://commons.apache.org/proper/commons-compress/ chemistry-opencmis-commons-impl-1.1.0.jar http://chemistry.apache.org/ chemistry-opencmis-commons-api-1.1.0.jar http://chemistry.apache.org/ xmlschema-core-2.2.5.jar http://ws.apache.org/commons/XmlSchema/ HikariCP-java7-2.4.13.jar https://github.com/brettwooldridge/HikariCP -cxf-core-3.2.14.jar https://cxf.apache.org/ -cxf-rt-bindings-soap-3.2.14.jar https://cxf.apache.org/ -cxf-rt-bindings-xml-3.2.14.jar https://cxf.apache.org/ -cxf-rt-databinding-jaxb-3.2.14.jar https://cxf.apache.org/ -cxf-rt-frontend-jaxws-3.2.14.jar https://cxf.apache.org/ -cxf-rt-frontend-simple-3.2.14.jar https://cxf.apache.org/ -cxf-rt-transports-http-3.2.14.jar https://cxf.apache.org/ -cxf-rt-ws-addr-3.2.14.jar https://cxf.apache.org/ -cxf-rt-ws-policy-3.2.14.jar https://cxf.apache.org/ -cxf-rt-wsdl-3.2.14.jar https://cxf.apache.org/ +cxf-core-3.4.8.jar https://cxf.apache.org/ +cxf-rt-bindings-soap-3.4.8.jar https://cxf.apache.org/ +cxf-rt-bindings-xml-3.4.8.jar https://cxf.apache.org/ +cxf-rt-databinding-jaxb-3.4.8.jar https://cxf.apache.org/ +cxf-rt-frontend-jaxws-3.4.8.jar https://cxf.apache.org/ +cxf-rt-frontend-simple-3.4.8.jar https://cxf.apache.org/ +cxf-rt-transports-http-3.4.8.jar https://cxf.apache.org/ +cxf-rt-ws-addr-3.4.8.jar https://cxf.apache.org/ +cxf-rt-ws-policy-3.4.8.jar https://cxf.apache.org/ +cxf-rt-wsdl-3.4.8.jar https://cxf.apache.org/ chemistry-opencmis-server-support-1.0.0.jar http://chemistry.apache.org/ chemistry-opencmis-server-bindings-1.0.0.jar http://chemistry.apache.org/ +failureaccess-1.0.1.jar https://mvnrepository.com/artifact/com.google.guava/failureaccess/1.0.1 +listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar https://mvnrepository.com/artifact/com.google.guava/listenablefuture quartz-2.3.2.jar http://quartz-scheduler.org/ -jackson-core-2.12.2.jar https://github.com/FasterXML/jackson -jackson-annotations-2.12.2.jar https://github.com/FasterXML/jackson -jackson-databind-2.12.2.jar https://github.com/FasterXML/jackson -jackson-dataformat-smile-2.12.2.jar https://github.com/FasterXML/jackson -commons-httpclient-3.1.jar http://jakarta.apache.org/commons/ +jackson-core-2.13.4.jar https://github.com/FasterXML/jackson +jackson-annotations-2.13.4.jar https://github.com/FasterXML/jackson +jackson-databind-2.13.4.2.jar https://github.com/FasterXML/jackson +commons-httpclient-3.1-HTTPCLIENT-1265.jar http://jakarta.apache.org/commons/ spring-aop-5.3.18.jar http://projects.spring.io/spring-framework/ spring-beans-5.3.18.jar http://projects.spring.io/spring-framework/ spring-context-5.3.18.jar http://projects.spring.io/spring-framework/ @@ -85,24 +89,7 @@ jetty-servlets-9.3.30.v20211001.jar https://www.eclipse.org/jetty/licenses.html jetty-util-9.3.30.v20211001.jar https://www.eclipse.org/jetty/licenses.html jetty-webapp-9.3.30.v20211001.jar https://www.eclipse.org/jetty/licenses.html jetty-xml-9.3.30.v20211001.jar https://www.eclipse.org/jetty/licenses.html -woodstox-core-5.0.3.jar https://github.com/FasterXML/woodstox -aggdesigner-algorithm-6.0.jar https://github.com/julianhyde/aggdesigner -tika-core-1.27.jar https://github.com/apache/tika/blob/1.27/LICENSE.txt -tika-java7-1.27.jar https://github.com/apache/tika/blob/1.27/LICENSE.txt -tika-parsers-1.27.jar https://github.com/apache/tika/blob/1.27/LICENSE.txt -tika-xmp-1.27.jar https://github.com/apache/tika/blob/1.27/LICENSE.txt -jcommander-1.81.jar https://github.com/cbeust/jcommander/blob/1.81/license.txt -isoparser-1.9.41.7.jar http://www.apache.org/licenses/LICENSE-2.0.txt -metadata-extractor-2.15.0.1.jar http://www.apache.org/licenses/LICENSE-2.0.txt -hadoop-annotations-2.7.7.jar https://github.com/apache/hadoop/blob/rel/release-2.7.7/LICENSE.txt -hadoop-auth-2.7.7.jar https://github.com/apache/hadoop/blob/rel/release-2.7.7/LICENSE.txt -hadoop-common-2.7.7.jar https://github.com/apache/hadoop/blob/rel/release-2.7.7/LICENSE.txt -hadoop-hdfs-2.7.7.jar https://github.com/apache/hadoop/blob/rel/release-2.7.7/LICENSE.txt -hadoop-annotations-2.10.1.jar https://github.com/apache/hadoop/blob/rel/release-2.10.1/LICENSE.txt -hadoop-auth-2.10.1.jar https://github.com/apache/hadoop/blob/rel/release-2.10.1/LICENSE.txt -hadoop-common-2.10.1.jar https://github.com/apache/hadoop/blob/rel/release-2.10.1/LICENSE.txt -hadoop-hdfs-2.10.1.jar https://github.com/apache/hadoop/blob/rel/release-2.10.1/LICENSE.txt -hadoop-hdfs-client-2.10.1.jar https://github.com/apache/hadoop/blob/rel/release-2.10.1/LICENSE.txt +woodstox-core-6.2.8.jar https://github.com/FasterXML/woodstox org.restlet-2.3.12.jar https://github.com/restlet/restlet-framework-java/blob/2.3.12/README.md org.restlet.ext.servlet-2.3.12.jar https://github.com/restlet/restlet-framework-java/blob/2.3.12/README.md xercesImpl-2.11.0-alfresco-patched-20180402.jar http://www.apache.org/licenses/LICENSE-2.0.txt @@ -113,10 +100,6 @@ jackson-core-asl-1.9.14-atlassian-6.jar http://www.apache.org/licenses/LICENSE-2 jackson-mapper-asl-1.9.14-atlassian-6.jar http://www.apache.org/licenses/LICENSE-2.0.txt -=== Creative Commons Public Domain === -jcip-annotations-1.0.jar https://jcip.net/listings.html - - === CDDL 1.1 === jaxb-impl-2.3.3.jar http://jaxb.java.net/ jaxb-xjc-2.3.3.jar http://jaxb.java.net/ @@ -127,10 +110,11 @@ jakarta.activation-1.2.2.jar https://eclipse-ee4j.github.io/jaf jakarta.activation-api-1.2.2.jar https://eclipse-ee4j.github.io/jaf jakarta.jws-api-2.1.0.jar https://projects.eclipse.org/projects/ee4j.websocket/releases/1.1.1 jakarta.xml.bind-api-3.0.1.jar https://projects.eclipse.org/projects/ee4j.jaxb -istack-commons-runtime-3.0.11.jar https://projects.eclipse.org/projects/ee4j.jaxb-impl -txw2-2.3.3.jar https://eclipse-ee4j.github.io/jaxb-ri -jaxb-runtime-2.3.3.jar https://github.com/eclipse-ee4j/jaxb-ri/blob/2.3.4-RI/LICENSE.md - +jakarta.xml.soap-api-1.4.2.jar https://projects.eclipse.org/projects/ee4j.jaxb +jakarta.xml.ws-api-2.3.3.jar https://projects.eclipse.org/projects/ee4j.jaxb +istack-commons-runtime-3.0.12.jar https://github.com/eclipse-ee4j/jaxb-istack-commons +txw2-2.3.4.jar https://eclipse-ee4j.github.io/jaxb-ri +jaxb-runtime-2.3.4.jar https://github.com/eclipse-ee4j/jaxb-ri/blob/2.3.4-RI/LICENSE.md === Eclipse Public License 1.0 === @@ -138,18 +122,15 @@ jaxb-runtime-2.3.3.jar https://github.com/eclipse-ee4j/jaxb-ri/blob/2.3.4-RI/LI === Eclipse Public License 2.0 === jakarta.annotation-api-1.3.5.jar https://projects.eclipse.org/projects/ee4j.ca jakarta.transaction-api-1.3.3.jar https://projects.eclipse.org/projects/ee4j.jta - +jts-io-common-1.19.0.jar https://locationtech.github.io/jts/ +jts-core-1.19.0.jar https://locationtech.github.io/jts/ === BSD === jibx-run-1.3.3.jar http://jibx.sourceforge.net/ dom4j-2.1.3.jar https://github.com/dom4j/dom4j/blob/master/LICENSE -xmpcore-6.1.11.jar https://github.com/adobe/XMP-Toolkit-SDK/blob/main/LICENSE -xmpcore-shaded-6.1.11.jar https://github.com/tballison/xmpcore-shaded/blob/6.1.11/src/main/resources/META-INF/LICENSE - === MIT === -dd-plist-1.23.jar https://github.com/3breadt/dd-plist/blob/dd-plist-1.23/LICENSE.txt - +checker-qual-3.12.0.jar https://checkerframework.org/ === Part of Apache Solr (Licenses listed separately) === antlr4-runtime-4.5.1-1.jar @@ -161,10 +142,6 @@ aspectjrt-1.8.0.jar attributes-binder-1.3.1.jar boilerpipe-1.1.0.jar caffeine-2.4.0.jar -calcite-core-1.13.0.jar -calcite-linq4j-1.13.0.jar -avatica-core-1.13.0.jar -avatica-metrics-1.13.0.jar carrot2-guava-18.0.jar carrot2-mini-3.15.0.jar commons-cli-1.2.jar @@ -176,15 +153,21 @@ commons-configuration-1.6.jar commons-exec-1.3.jar commons-fileupload-1.3.3.jar commons-io-2.5.jar +calcite-core-1.32.0.jar +calcite-linq4j-1.32.0.jar +avatica-core-1.22.0.jar +avatica-metrics-1.22.0.jar commons-lang-2.6.jar commons-math3-3.4.1.jar curator-client-2.8.0.jar curator-framework-2.8.0.jar curator-recipes-2.8.0.jar +commons-compiler-3.1.8.jar +janino-3.1.8.jar curvesapi-1.04.jar eigenbase-properties-1.1.5.jar gmetric4j-1.0.7.jar -guava-14.0.1.jar +guava-31.1-jre.jar hppc-0.7.1.jar htrace-core-3.2.0-incubating.jar httpclient-4.5.13.jar @@ -192,7 +175,6 @@ httpcore-4.4.1.jar httpmime-4.4.1.jar icu4j-56.1.jar jackcess-2.1.8.jar -janino-2.7.6.jar java-libpst-0.8.1.jar javax.servlet-api-3.1.0.jar jcl-over-slf4j-1.7.7.jar @@ -235,6 +217,7 @@ morfologik-polish-2.1.1.jar morfologik-stemming-2.1.1.jar noggit-0.6.jar protobuf-java-3.6.1.jar +protobuf-java-3.19.6.jar rome-1.5.1.jar slf4j-api-1.7.7.jar slf4j-log4j12-1.7.7.jar diff --git a/search-services/pom.xml b/search-services/pom.xml index aff716b0b..cbdd17cdb 100644 --- a/search-services/pom.xml +++ b/search-services/pom.xml @@ -6,16 +6,11 @@ alfresco-search-and-insight-parent 2.0.6-SNAPSHOT - - org.alfresco alfresco-search-parent 2.0.6-SNAPSHOT pom Alfresco Solr Search parent - 1.7.36 - 3.2.14 community @@ -71,30 +66,4 @@ http://maven.restlet.talend.com - - - - - - org.springframework - spring-orm - ${dependency.spring.version} - - - org.springframework - spring-context - ${dependency.spring.version} - - - org.springframework - spring-context-support - ${dependency.spring.version} - - - org.springframework - spring-web - ${dependency.spring.version} - - -