diff --git a/source/java/org/alfresco/rest/api/search/impl/ResultMapper.java b/source/java/org/alfresco/rest/api/search/impl/ResultMapper.java index 9dcaa9a29f..1d7a362bb2 100644 --- a/source/java/org/alfresco/rest/api/search/impl/ResultMapper.java +++ b/source/java/org/alfresco/rest/api/search/impl/ResultMapper.java @@ -29,11 +29,14 @@ package org.alfresco.rest.api.search.impl; import static org.alfresco.rest.api.search.impl.StoreMapper.DELETED; import static org.alfresco.rest.api.search.impl.StoreMapper.LIVE_NODES; import static org.alfresco.rest.api.search.impl.StoreMapper.VERSIONS; +import com.sun.javafx.font.Metrics; import org.alfresco.repo.search.impl.lucene.SolrJSONResultSet; import org.alfresco.repo.search.impl.solr.facet.facetsresponse.GenericBucket; import org.alfresco.repo.search.impl.solr.facet.facetsresponse.GenericFacetResponse; import org.alfresco.repo.search.impl.solr.facet.facetsresponse.GenericFacetResponse.FACET_TYPE; -import org.alfresco.repo.search.impl.solr.facet.facetsresponse.MetricCount; +import org.alfresco.repo.search.impl.solr.facet.facetsresponse.Metric; +import org.alfresco.repo.search.impl.solr.facet.facetsresponse.Metric.METRIC_TYPE; +import org.alfresco.repo.search.impl.solr.facet.facetsresponse.SimpleMetric; import org.alfresco.repo.security.permissions.impl.acegi.FilteringResultSet; import org.alfresco.repo.version.Version2Model; import org.alfresco.rest.api.DeletedNodes; @@ -318,8 +321,10 @@ public class ResultMapper Map>> facetInterval = solrResultSet.getFacetIntervals(); facets.addAll(getGenericFacetsForIntervals(facetInterval, searchQuery)); - List pimped = getPivots(searchRequestContext, solrResultSet.getPivotFacets()); + List stats = getFieldStats(searchRequestContext, solrResultSet.getStats()); + List pimped = getPivots(searchRequestContext, solrResultSet.getPivotFacets(), stats); facets.addAll(pimped); + facets.addAll(stats); //Spelling SpellCheckResult spell = solrResultSet.getSpellCheckResult(); @@ -332,6 +337,7 @@ public class ResultMapper context = new SearchContext(solrResultSet.getLastIndexedTxId(), facets, facetResults, ffcs, spellCheckContext, searchRequestContext.includeRequest()?searchQuery:null); return isNullContext(context)?null:context; } + /** * Builds a facet field from facet queries. * @param facetQueries @@ -376,7 +382,23 @@ public class ResultMapper return facetResults; } - protected List getPivots(SearchRequestContext searchRequest, List pivots) + protected List getFieldStats(SearchRequestContext searchRequestContext, Map> stats) + { + if(stats != null && !stats.isEmpty()) + { + return stats.entrySet().stream().map(statsFieldEntry -> { + return new GenericFacetResponse(FACET_TYPE.stats, statsFieldEntry.getKey(), + Arrays.asList(new GenericBucket(null,null, null, + statsFieldEntry.getValue(), null)) ); + } + ).collect(Collectors.toList()); + } + + return Collections.emptyList(); + } + + protected List getPivots(SearchRequestContext searchRequest, List pivots, + List stats) { if(pivots != null && !pivots.isEmpty()) { @@ -386,11 +408,20 @@ public class ResultMapper String pivotLabel = pivotKeys.containsKey(aFacet.getLabel())?pivotKeys.get(aFacet.getLabel()):aFacet.getLabel(); - List bucks = aFacet.getBuckets().stream().map(genericBucket -> { + //can reference, facetfield, the last one can be rangefacet, facetquery or stats + List bucks = new ArrayList<>(); + Optional foundStat = stats.stream().filter( + aStat -> aStat.getLabel().equals(pivotLabel)).findFirst(); + if (foundStat.isPresent()) + { + bucks.add(foundStat.get().getBuckets().get(0)); + stats.remove(foundStat.get()); + } + bucks.addAll(aFacet.getBuckets().stream().map(genericBucket -> { Object display = propertyLookup.lookup(aFacet.getLabel(), genericBucket.getLabel()); return new GenericBucket(genericBucket.getLabel(), genericBucket.getFilterQuery(), - display,genericBucket.getMetrics(), getPivots(searchRequest, genericBucket.getFacets())); - }).collect(Collectors.toList()); + display,genericBucket.getMetrics(), getPivots(searchRequest, genericBucket.getFacets(), stats)); + }).collect(Collectors.toList())); return new GenericFacetResponse(aFacet.getType(), pivotLabel, bucks); }).collect(Collectors.toList()); @@ -476,7 +507,7 @@ public class ResultMapper } } } - GenericBucket bucket = new GenericBucket(buck.getFirst(), filterQuery, null , Arrays.asList(new MetricCount(buck.getSecond())), null); + GenericBucket bucket = new GenericBucket(buck.getFirst(), filterQuery, null , Arrays.asList(new SimpleMetric(METRIC_TYPE.count,String.valueOf(buck.getSecond()))), null); buckets.add(bucket); } ffcs.add(new GenericFacetResponse(FACET_TYPE.interval, facet.getKey(), buckets)); diff --git a/source/java/org/alfresco/rest/api/search/impl/SearchMapper.java b/source/java/org/alfresco/rest/api/search/impl/SearchMapper.java index 082e025ca6..656135a284 100644 --- a/source/java/org/alfresco/rest/api/search/impl/SearchMapper.java +++ b/source/java/org/alfresco/rest/api/search/impl/SearchMapper.java @@ -58,6 +58,7 @@ import org.alfresco.service.cmr.search.SearchParameters.FieldFacetSort; import org.alfresco.service.cmr.search.SearchParameters.Operator; import org.alfresco.service.cmr.search.SearchParameters.SortDefinition; import org.alfresco.service.cmr.search.SearchParameters.SortDefinition.SortType; +import org.alfresco.service.cmr.search.StatsRequestParameters; import org.alfresco.util.ParameterCheck; import static org.alfresco.rest.api.Nodes.PARAM_INCLUDE_ALLOWABLEOPERATIONS; @@ -70,6 +71,7 @@ import static org.alfresco.service.cmr.search.SearchService.*; import java.util.Arrays; import java.util.List; +import java.util.ListIterator; import java.util.Optional; import java.util.regex.Matcher; @@ -110,7 +112,8 @@ public class SearchMapper fromDefault(sp, searchQuery.getDefaults()); fromFilterQuery(sp, searchQuery.getFilterQueries()); fromFacetQuery(sp, searchQuery.getFacetQueries()); - fromPivot(sp, searchQuery.getFacetFields(), searchQuery.getPivots(), searchRequestContext); + fromPivot(sp, searchQuery.getStats(), searchQuery.getFacetFields(), searchQuery.getPivots(), searchRequestContext); + fromStats(sp, searchQuery.getStats()); fromFacetFields(sp, searchQuery.getFacetFields()); fromSpellCheck(sp, searchQuery.getSpellcheck()); fromHighlight(sp, searchQuery.getHighlight()); @@ -508,19 +511,21 @@ public class SearchMapper sp.setInterval(facetIntervals); } - public void fromPivot(SearchParameters sp, FacetFields facetFields, List pivots, SearchRequestContext searchRequestContext) + public void fromPivot(SearchParameters sp, List stats, FacetFields facetFields, List pivots, SearchRequestContext searchRequestContext) { if (facetFields != null && pivots != null && !pivots.isEmpty()) { ParameterCheck.mandatory("facetFields facets", facetFields.getFacets()); - if (facetFields.getFacets() != null && !facetFields.getFacets().isEmpty()) - { - for (Pivot pivot:pivots) - { - ParameterCheck.mandatoryString("pivot key", pivot.getKey()); + ListIterator piterator = pivots.listIterator(); - String pivotKey = pivot.getKey(); + while (piterator.hasNext()) { + + Pivot pivot = piterator.next(); + ParameterCheck.mandatoryString("pivot key", pivot.getKey()); + String pivotKey = pivot.getKey(); + if (facetFields.getFacets() != null && !facetFields.getFacets().isEmpty()) + { Optional found = facetFields.getFacets().stream().filter( queryable -> pivotKey.equals(queryable.getLabel()!=null?queryable.getLabel():queryable.getField())).findFirst(); @@ -532,14 +537,63 @@ public class SearchMapper } else { - throw new InvalidArgumentException(InvalidArgumentException.DEFAULT_MESSAGE_ID, - new Object[] { ": Pivot parameter "+pivotKey+" is does not reference a facet Field." }); + if (piterator.hasNext()) + { + //Its not the last one so lets complain + throw new InvalidArgumentException(InvalidArgumentException.DEFAULT_MESSAGE_ID, + new Object[] { ": Pivot parameter " + pivotKey + " is does not reference a facet Field." }); + } + else + { + //It is the last one so it can reference facetquery or stats + /** + Optional foundStat = stats.stream().filter(stas -> pivotKey.equals(stas.getLabel())).findFirst(); + if (foundStat.isPresent()) + { + stats.remove(foundStat.get()); + } + **/ + sp.addPivot(pivotKey); + searchRequestContext.getPivotKeys().put(pivotKey, pivotKey); + } } + } } } } + public void fromStats(SearchParameters sp, List stats) + { + if (stats != null && !stats.isEmpty()) + { + for (StatsRequestParameters aStat:stats) + { + ParameterCheck.mandatory("stats field", aStat.getField()); + + List perc = aStat.getPercentiles(); + if (perc != null && !perc.isEmpty()) + { + for (Float percentile:perc) + { + if (percentile == null || percentile < 0 || percentile > 100) + { + throw new IllegalArgumentException("Invalid percentile "+percentile); + } + } + } + + if (aStat.getCardinality() && (aStat.getCardinalityAccuracy() < 0 || aStat.getCardinalityAccuracy() > 1)) + { + throw new IllegalArgumentException("Invalid cardinality accuracy "+aStat.getCardinalityAccuracy() + " It must be between 0 and 1."); + } + } + + sp.setStats(stats); + } + + } + protected void validateSets(List intervalSets, String prefix) { if (intervalSets != null && !intervalSets.isEmpty()) diff --git a/source/java/org/alfresco/rest/api/search/model/SearchQuery.java b/source/java/org/alfresco/rest/api/search/model/SearchQuery.java index b81e019897..802766fb94 100644 --- a/source/java/org/alfresco/rest/api/search/model/SearchQuery.java +++ b/source/java/org/alfresco/rest/api/search/model/SearchQuery.java @@ -29,6 +29,7 @@ package org.alfresco.rest.api.search.model; import org.alfresco.rest.framework.resource.parameters.Paging; import org.alfresco.service.cmr.search.GeneralHighlightParameters; import org.alfresco.service.cmr.search.IntervalParameters; +import org.alfresco.service.cmr.search.StatsRequestParameters; import org.codehaus.jackson.annotate.JsonCreator; import org.codehaus.jackson.annotate.JsonProperty; @@ -58,8 +59,10 @@ public class SearchQuery private final IntervalParameters facetIntervals; private final boolean includeRequest; private final List pivots; + private final List stats; - public static final SearchQuery EMPTY = new SearchQuery(null, null, null, null, null, null,null, null, null, null,null, null, null, null, null, null, null); + public static final SearchQuery EMPTY = new SearchQuery(null, null, null, null, null, null, + null,null, null, null, null,null, null, null, null, null, null, null); @JsonCreator public SearchQuery(@JsonProperty("query") Query query, @@ -78,7 +81,8 @@ public class SearchQuery @JsonProperty("limits")Limits limits, @JsonProperty("highlight")GeneralHighlightParameters highlight, @JsonProperty("facetIntervals")IntervalParameters facetIntervals, - @JsonProperty("pivots") List pivots) + @JsonProperty("pivots") List pivots, + @JsonProperty("stats") List stats) { this.query = query; this.includeRequest = includeRequest==null?false:includeRequest; @@ -97,6 +101,7 @@ public class SearchQuery this.highlight = highlight; this.facetIntervals = facetIntervals; this.pivots = pivots; + this.stats = stats; } public Query getQuery() @@ -182,4 +187,9 @@ public class SearchQuery { return pivots; } + + public List getStats() + { + return stats; + } } diff --git a/source/test-java/org/alfresco/rest/api/search/ResultMapperTests.java b/source/test-java/org/alfresco/rest/api/search/ResultMapperTests.java index bd2a785abf..856312a6a1 100644 --- a/source/test-java/org/alfresco/rest/api/search/ResultMapperTests.java +++ b/source/test-java/org/alfresco/rest/api/search/ResultMapperTests.java @@ -120,6 +120,7 @@ public class ResultMapperTests + " \"_DEFAULT_!800001579e3d1964!800001579e3d1969\": {\"name\": [\"some very long name\"],\"title\": [\"title1 is very long\"], \"DBID\": \"521\"}," + " \"_DEFAULT_!800001579e3d1964!800001579e3d196a\": {\"name\": [\"this is some long text. It\", \" has the word long in many places\", \". In fact, it has long on some\", \" happens to long in this case.\"], \"DBID\": \"1475846153692\"}" + "}," + + "\"stats\":{\"stats_fields\":{\"creator\":{\"min\":\"System\",\"max\":\"mjackson\",\"count\":\"990\",\"missing\":\"290\"}, \"created\":{\"sumOfSquares\":2.1513045770343806E27,\"min\":\"2011-02-15T20:16:27.080Z\",\"max\":\"2017-04-10T15:06:30.143Z\",\"mean\":\"2016-09-05T04:20:12.898Z\",\"count\":990,\"missing\":290,\"sum\":1.458318720769983E15,\"stddev\":5.6250677994522545E10}}}," + "\"processedDenies\":true, \"lastIndexedTx\":34}"; public static final Params EMPTY_PARAMS = Params.valueOf((String)null,(String)null,(WebScriptRequest) null); public static final String FROZEN_ID = "frozen"; @@ -305,7 +306,7 @@ public class ResultMapperTests assertEquals("last",intervalFacets.get(0).getBuckets().get(0).getLabel()); assertEquals("cm:creator:",intervalFacets.get(1).getBuckets().get(0).getFilterQuery()); assertEquals(METRIC_TYPE.count,intervalFacets.get(1).getBuckets().get(0).getMetrics().get(0).getType()); - assertEquals(5,intervalFacets.get(1).getBuckets().get(0).getMetrics().get(0).getValue().get("count")); + assertEquals("5",intervalFacets.get(1).getBuckets().get(0).getMetrics().get(0).getValue().get("count")); assertEquals("lastYear",intervalFacets.get(1).getBuckets().get(1).getLabel()); assertEquals("cm:created:[2016 TO 2017>",intervalFacets.get(1).getBuckets().get(1).getFilterQuery()); assertEquals(METRIC_TYPE.count,intervalFacets.get(1).getBuckets().get(1).getMetrics().get(0).getType()); - assertEquals(0,intervalFacets.get(1).getBuckets().get(1).getMetrics().get(0).getValue().get("count")); + assertEquals("0",intervalFacets.get(1).getBuckets().get(1).getMetrics().get(0).getValue().get("count")); assertEquals("currentYear",intervalFacets.get(1).getBuckets().get(2).getLabel()); assertEquals("cm:created:[NOW/YEAR TO NOW/YEAR+1YEAR]",intervalFacets.get(1).getBuckets().get(2).getFilterQuery()); assertEquals(METRIC_TYPE.count,intervalFacets.get(1).getBuckets().get(2).getMetrics().get(0).getType()); - assertEquals(854,intervalFacets.get(1).getBuckets().get(2).getMetrics().get(0).getValue().get("count")); + assertEquals("854",intervalFacets.get(1).getBuckets().get(2).getMetrics().get(0).getValue().get("count")); } @Test diff --git a/source/test-java/org/alfresco/rest/api/search/SearchMapperTests.java b/source/test-java/org/alfresco/rest/api/search/SearchMapperTests.java index ede120e842..b81cb5d6b4 100644 --- a/source/test-java/org/alfresco/rest/api/search/SearchMapperTests.java +++ b/source/test-java/org/alfresco/rest/api/search/SearchMapperTests.java @@ -62,6 +62,7 @@ import org.alfresco.service.cmr.search.LimitBy; import org.alfresco.service.cmr.search.SearchParameters; import org.alfresco.service.cmr.search.SearchParameters.FieldFacet; import org.alfresco.service.cmr.search.SearchService; +import org.alfresco.service.cmr.search.StatsRequestParameters; import org.junit.BeforeClass; import org.junit.Test; @@ -584,17 +585,88 @@ public class SearchMapperTests assertEquals(searchParameters.getHighlight(), highlightParameters); } + @Test + public void fromStats() throws Exception + { + SearchParameters searchParameters = new SearchParameters(); + searchMapper.fromStats(searchParameters, null); + + List statsRequestParameters = new ArrayList<>(1); + statsRequestParameters.add(new StatsRequestParameters(null, null, null, null, null,null, null, null, null,null, null, null,null, null, null, null)); + + try + { + searchMapper.fromStats(searchParameters, statsRequestParameters); + } + catch (IllegalArgumentException iae) + { + assertTrue(iae.getLocalizedMessage().contains("field is a mandatory parameter")); + } + + statsRequestParameters.clear(); + statsRequestParameters.add(new StatsRequestParameters("cm:content", "myLabel", null, null,null, null, null, null,null, null, null, null,null, null, null, null)); + searchMapper.fromStats(searchParameters, statsRequestParameters); + assertEquals(1 ,searchParameters.getStats().size()); + + statsRequestParameters.clear(); + statsRequestParameters.add(new StatsRequestParameters("cm:content", "myLabel", Arrays.asList(3.4f, 12f, 10f), null, null,null, null, null, null,null, null, null,null, null, null, null)); + searchMapper.fromStats(searchParameters, statsRequestParameters); + assertEquals(1 ,searchParameters.getStats().size()); + + statsRequestParameters.clear(); + statsRequestParameters.add(new StatsRequestParameters("cm:content", "myLabel", Arrays.asList(-3.4f), null, null,null, null, null, null,null, null, null,null, null, null, null)); + + try + { + searchMapper.fromStats(searchParameters, statsRequestParameters); + } + catch (IllegalArgumentException iae) + { + assertTrue(iae.getLocalizedMessage().contains("Invalid percentile -3.4")); + } + + statsRequestParameters.clear(); + statsRequestParameters.add(new StatsRequestParameters("cm:content", "myLabel", Arrays.asList(101f),null, null,null, null, null, null,null, null, null,null, null, null, null)); + + try + { + searchMapper.fromStats(searchParameters, statsRequestParameters); + } + catch (IllegalArgumentException iae) + { + assertTrue(iae.getLocalizedMessage().contains("Invalid percentile 101")); + } + + statsRequestParameters.clear(); + statsRequestParameters.add(new StatsRequestParameters("cm:content", "myLabel", null, null,null, null, null, null,null, null, null, null,null, true, 12f, null)); + try + { + searchMapper.fromStats(searchParameters, statsRequestParameters); + } + catch (IllegalArgumentException iae) + { + assertTrue(iae.getLocalizedMessage().contains("Invalid cardinality accuracy 12.0")); + } + + statsRequestParameters.clear(); + statsRequestParameters.add(new StatsRequestParameters("cm:content", "myLabel", null, null,null, null, null, null,null, null, null, null,null, null, 12f, null)); + searchMapper.fromStats(searchParameters, statsRequestParameters); + //cardinality is ignored if not true + assertEquals(1 ,searchParameters.getStats().size()); + + } + @Test public void fromPivot() throws Exception { SearchParameters searchParameters = new SearchParameters(); - searchMapper.fromPivot(searchParameters,null, null, null); + searchMapper.fromPivot(searchParameters, null, null, null, null); List facets = new ArrayList<>(1); facets.add(new FacetField("myfield",null,null,null,null,null,null,null,null,null,null)); FacetFields ff = new FacetFields(facets); searchMapper.fromFacetFields(searchParameters,ff); - searchMapper.fromPivot(searchParameters,ff, null, null); + searchMapper.fromPivot(searchParameters, null, ff, null, null); assertEquals(1 ,searchParameters.getFieldFacets().size()); assertEquals(0 ,searchParameters.getPivots().size()); @@ -603,7 +675,7 @@ public class SearchMapperTests try { - searchMapper.fromPivot(searchParameters,ff, Arrays.asList(new Pivot(null)), null); + searchMapper.fromPivot(searchParameters, null, ff, Arrays.asList(new Pivot(null)), null); fail(); } catch (IllegalArgumentException iae) @@ -614,7 +686,7 @@ public class SearchMapperTests try { - searchMapper.fromPivot(searchParameters,ff, Arrays.asList(new Pivot("")), null); + searchMapper.fromPivot(searchParameters, null, ff, Arrays.asList(new Pivot("")), null); fail(); } catch (IllegalArgumentException iae) @@ -623,20 +695,25 @@ public class SearchMapperTests assertNotNull(iae); } + SearchRequestContext searchRequestContext = SearchRequestContext.from(minimalQuery()); + + //"bob" doesn't refer to a field facet but its the last one so lets be kind + searchMapper.fromPivot(searchParameters, null, ff, Arrays.asList(new Pivot("bob")), searchRequestContext); + try { - searchMapper.fromPivot(searchParameters,ff, Arrays.asList(new Pivot("bob")), null); + searchMapper.fromPivot(searchParameters, null, ff, Arrays.asList(new Pivot("ken"),new Pivot("bob")), searchRequestContext); fail(); } catch (InvalidArgumentException iae) { - //"bob" doesn't refer to a field facet + //"ken" doesn't refer to a field facet and its not the last one assertNotNull(iae); } searchParameters = new SearchParameters(); - SearchRequestContext searchRequestContext = SearchRequestContext.from(minimalQuery()); - searchMapper.fromPivot(searchParameters,ff, Arrays.asList(new Pivot("myfield")), searchRequestContext); + + searchMapper.fromPivot(searchParameters, null, ff, Arrays.asList(new Pivot("myfield")), searchRequestContext); searchMapper.fromFacetFields(searchParameters,ff); //Moved from a field facet to a pivot assertEquals(0 ,searchParameters.getFieldFacets().size()); @@ -717,7 +794,7 @@ public class SearchMapperTests private SearchQuery minimalQuery() { Query query = new Query("cmis", "foo", ""); - SearchQuery sq = new SearchQuery(query, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); + SearchQuery sq = new SearchQuery(query, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); return sq; } @Test diff --git a/source/test-java/org/alfresco/rest/api/search/SearchQuerySerializerTests.java b/source/test-java/org/alfresco/rest/api/search/SearchQuerySerializerTests.java index dda7b1ce8f..594322cd9e 100644 --- a/source/test-java/org/alfresco/rest/api/search/SearchQuerySerializerTests.java +++ b/source/test-java/org/alfresco/rest/api/search/SearchQuerySerializerTests.java @@ -166,6 +166,10 @@ public class SearchQuerySerializerTests assertEquals(1,searchQuery.getPivots().size()); assertEquals("mylabel",searchQuery.getPivots().get(0).getKey()); + + assertEquals(1,searchQuery.getStats().size()); + assertEquals("cm:creator",searchQuery.getStats().get(0).getField()); + assertEquals("mylabel",searchQuery.getStats().get(0).getLabel()); } @Test diff --git a/source/test-java/org/alfresco/rest/api/search/SerializerTestHelper.java b/source/test-java/org/alfresco/rest/api/search/SerializerTestHelper.java index d3c185996f..578baa9836 100644 --- a/source/test-java/org/alfresco/rest/api/search/SerializerTestHelper.java +++ b/source/test-java/org/alfresco/rest/api/search/SerializerTestHelper.java @@ -70,6 +70,7 @@ public class SerializerTestHelper implements RequestReader + "}," + "{\"label\":\"TheCreated\",\"field\":\"cm:created\",\"sets\":[{\"label\":\"lastYear\",\"start\":\"2016\",\"end\":\"2017\",\"endInclusive\":false},{\"label\":\"currentYear\",\"start\":\"NOW/YEAR\",\"end\":\"NOW/YEAR+1YEAR\"},{\"label\":\"earlier\",\"start\":\"*\",\"end\":\"2016\",\"endInclusive\":false}]}" + "]}," + + "\"stats\": [{\"field\": \"cm:creator\", \"label\": \"mylabel\"}]," + "\"spellcheck\": {\"query\": \"alfrezco\"}," + "\"limits\": {\"permissionEvaluationCount\": \"2000\",\"permissionEvaluationTime\": \"5000\"}," + "\"scope\": { \"locations\": [\"nodes\"]},"