diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java index 0c42a3e0a..0859c1ecc 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java @@ -152,6 +152,23 @@ public class AlfrescoSolrDataModel implements QueryConstants } public static final String CONTENT_S_LOCALE_PREFIX = "content@s__locale@"; + + static final String PART_FIELDNAME_PREFIX = "part@sd@"; + + /** + * Infix used for denoting a primitive single valued field with no doc values enabled + * + * @see #getFieldForText + */ + private static final String SINGLE_VALUE_WITHOUT_DOC_VALUES_MARKER = "@s_@"; + + /** + * Infix used for denoting a primitive single valued field with no doc values enabled + * + * @see #getFieldForText + */ + private static final String SINGLE_VALUE_WITH_DOC_VALUES_MARKER = "@sd@"; + static final String SHARED_PROPERTIES = "shared.properties"; protected final static Logger log = LoggerFactory.getLogger(AlfrescoSolrDataModel.class); @@ -760,7 +777,6 @@ public class AlfrescoSolrDataModel implements QueryConstants private void addHighlightSearchFields( PropertyDefinition propertyDefinition , IndexedField indexedField) { - QName propertyName = propertyDefinition.getName(); QName propertyDataTypeQName = propertyDefinition.getDataType().getName(); String fieldName; @@ -1416,7 +1432,7 @@ public class AlfrescoSolrDataModel implements QueryConstants public static class IndexedField { - private List fields = new LinkedList<>(); + private final List fields = new LinkedList<>(); public List getFields() { @@ -1747,4 +1763,27 @@ public class AlfrescoSolrDataModel implements QueryConstants getNamespaceDAO().addPrefix("", NamespaceService.CONTENT_MODEL_1_0_URI); } } + + /** + * Returns the prefix used for denoting a field which is meant to represent a constituent part of a + * date or datetime (e.g. year, month, second, minute). + * + * @param sourceFieldName the date/datetime source field name. + * @param sourceDataTypeDefinition the datatype of the source field name (date or datetime) + * @return the prefix that can be used for denoting a date or datetime part + */ + public String destructuredDateTimePartFieldNamePrefix(String sourceFieldName, DataTypeDefinition sourceDataTypeDefinition) { + // source field name example: datetime@sd@{http://www.alfresco.org/model/content/1.0}created + // prefix: datetime + String prefix = sourceFieldName.substring(0, sourceFieldName.indexOf("@")); + + // prefix, docValues disabled option: datetime@s_@ + String sourceFieldNamePrefixWithoutDocValues = prefix + SINGLE_VALUE_WITHOUT_DOC_VALUES_MARKER; + + // prefix, docValues enabled option: datetime@sd@ + String sourceFieldNamePrefixWithDocValues = prefix + SINGLE_VALUE_WITH_DOC_VALUES_MARKER; + + return sourceFieldName.replace(sourceFieldNamePrefixWithoutDocValues, PART_FIELDNAME_PREFIX) + .replace(sourceFieldNamePrefixWithDocValues, PART_FIELDNAME_PREFIX); + } } \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 0181296be..fda8d0b41 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -72,6 +72,7 @@ import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_VERSI import static org.alfresco.solr.AlfrescoSolrDataModel.getAclChangeSetDocumentId; import static org.alfresco.solr.AlfrescoSolrDataModel.getAclDocumentId; import static org.alfresco.solr.utils.Utils.notNullOrEmpty; +import static org.alfresco.util.ISO8601DateFormat.isTimeComponentDefined; import java.io.File; import java.io.IOException; @@ -80,6 +81,8 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.nio.charset.StandardCharsets; import java.text.DecimalFormat; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -118,6 +121,7 @@ import org.alfresco.repo.dictionary.M2Model; import org.alfresco.repo.dictionary.NamespaceDAO; import org.alfresco.repo.search.adaptor.lucene.QueryConstants; import org.alfresco.service.cmr.dictionary.AspectDefinition; +import org.alfresco.service.cmr.dictionary.DataTypeDefinition; import org.alfresco.service.cmr.dictionary.PropertyDefinition; import org.alfresco.service.cmr.dictionary.TypeDefinition; import org.alfresco.service.cmr.repository.NodeRef; @@ -363,6 +367,14 @@ public class SolrInformationServer implements InformationServer /** Shared property to determine if the cascade tracking is enabled. */ public static final String CASCADE_TRACKER_ENABLED = "alfresco.cascade.tracker.enabled"; + private static final String UNIT_OF_TIME_FIELD_INFIX = "_unit_of_time"; + static final String UNIT_OF_TIME_YEAR_FIELD_SUFFIX = UNIT_OF_TIME_FIELD_INFIX + "_year"; + static final String UNIT_OF_TIME_MONTH_FIELD_SUFFIX = UNIT_OF_TIME_FIELD_INFIX + "_month"; + static final String UNIT_OF_TIME_DAY_FIELD_SUFFIX = UNIT_OF_TIME_FIELD_INFIX + "_day"; + static final String UNIT_OF_TIME_HOUR_FIELD_SUFFIX = UNIT_OF_TIME_FIELD_INFIX + "_hour"; + static final String UNIT_OF_TIME_MINUTE_FIELD_SUFFIX = UNIT_OF_TIME_FIELD_INFIX + "_minute"; + static final String UNIT_OF_TIME_SECOND_FIELD_SUFFIX = UNIT_OF_TIME_FIELD_INFIX + "_second"; + private final static Function> LAZY_EMPTY_MUTABLE_LIST = key -> new ArrayList<>(); private final AlfrescoCoreAdminHandler adminHandler; @@ -403,6 +415,8 @@ public class SolrInformationServer implements InformationServer private String skippingDocsQueryString; private boolean isSkippingDocsInitialized; + private final boolean dateFieldDestructuringHasBeenEnabledOnThisInstance; + static class DocListCollector implements Collector, LeafCollector { private final IntArrayList docs = new IntArrayList(); @@ -580,6 +594,11 @@ public class SolrInformationServer implements InformationServer port = portNumber(props); baseUrl = baseUrl(props); + + dateFieldDestructuringHasBeenEnabledOnThisInstance = Boolean.parseBoolean(coreConfiguration.getProperty("alfresco.destructureDateFields", "true")); + LOGGER.info( + "Date fields destructuring has been {} on this instance.", + dateFieldDestructuringHasBeenEnabledOnThisInstance ? "enabled" : "disabled"); } @Override @@ -2191,7 +2210,7 @@ public class SolrInformationServer implements InformationServer } } - static void mltextProperty(QName propertyQName, MLTextPropertyValue value, final BiConsumer valueHolder) + void mltextProperty(QName propertyQName, MLTextPropertyValue value, final BiConsumer valueHolder) { AlfrescoSolrDataModel dataModel = AlfrescoSolrDataModel.getInstance(); List fields = dataModel.getIndexedFieldNamesForProperty(propertyQName).getFields(); @@ -2204,7 +2223,7 @@ public class SolrInformationServer implements InformationServer .forEach(field -> addMLTextProperty(valueHolder, field, value)); } - static void stringProperty(QName propertyQName, StringPropertyValue value, PropertyValue locale, final BiConsumer valueHolder) + void stringProperty(QName propertyQName, StringPropertyValue value, PropertyValue locale, final BiConsumer valueHolder) { AlfrescoSolrDataModel dataModel = AlfrescoSolrDataModel.getInstance(); PropertyDefinition definition = dataModel.getPropertyDefinition(propertyQName); @@ -2217,15 +2236,21 @@ public class SolrInformationServer implements InformationServer dataModel.getIndexedFieldNamesForProperty(propertyQName).getFields() .stream() .filter(field -> field.getField().startsWith("text@sd___@")) - .forEach(field -> addStringProperty(valueHolder, field, value, locale)); + .forEach(field -> addStringProperty(valueHolder, field, value, locale, definition)); } else { dataModel.getIndexedFieldNamesForProperty(propertyQName).getFields() - .forEach(field -> addStringProperty(valueHolder, field, value, locale)); + .forEach(field -> { + if (canBeDestructured(definition, field.getField())) + { + setUnitOfTimeFields(valueHolder, field.getField(), value.getValue(), definition.getDataType()); + } + addStringProperty(valueHolder, field, value, locale, definition); + }); } } - static void populateProperties( + void populateProperties( Map properties, boolean contentIndexingHasBeenRequestedForThisNode, SolrInputDocument document, @@ -2384,7 +2409,7 @@ public class SolrInformationServer implements InformationServer } } - private static void addContentPropertyMetadata( + private void addContentPropertyMetadata( BiConsumer consumer, QName propertyQName, ContentPropertyValue contentPropertyValue, @@ -2424,7 +2449,7 @@ public class SolrInformationServer implements InformationServer * * @see #insertContentUpdateMarker(SolrInputDocument, ContentPropertyValue) */ - private static void markAsContentInSynch(SolrInputDocument document) + private void markAsContentInSynch(SolrInputDocument document) { markAsContentInSynch(document, (ContentPropertyValue)null); } @@ -2435,7 +2460,7 @@ public class SolrInformationServer implements InformationServer * * @see #insertContentUpdateMarker(SolrInputDocument, ContentPropertyValue) */ - private static void markAsContentInSynch(SolrInputDocument document, ContentPropertyValue value) + private void markAsContentInSynch(SolrInputDocument document, ContentPropertyValue value) { ofNullable(value) .map(ContentPropertyValue::getId) @@ -2450,7 +2475,7 @@ public class SolrInformationServer implements InformationServer * * @see #insertContentUpdateMarker(SolrInputDocument, ContentPropertyValue) */ - private static void markAsContentInSynch(SolrInputDocument document, Long id) + private void markAsContentInSynch(SolrInputDocument document, Long id) { long contentVersionId = ofNullable(id).orElse(CONTENT_UPDATED_MARKER); @@ -2513,7 +2538,7 @@ public class SolrInformationServer implements InformationServer * * As a side note, please keep in mind the FTSSTATUS field has been removed. */ - private static void insertContentUpdateMarker(SolrInputDocument document, ContentPropertyValue value) + private void insertContentUpdateMarker(SolrInputDocument document, ContentPropertyValue value) { ofNullable(value) .map(ContentPropertyValue::getId) @@ -2524,7 +2549,7 @@ public class SolrInformationServer implements InformationServer () -> document.setField(LAST_INCOMING_CONTENT_VERSION_ID, CONTENT_OUTDATED_MARKER)); } - private static void addContentProperty( + private void addContentProperty( BiConsumer consumer, SolrInputDocument document, QName propertyName, @@ -2629,7 +2654,7 @@ public class SolrInformationServer implements InformationServer return indexOfSeparator == -1 ? locale : locale.substring(0, indexOfSeparator); } - private static List getLocalisedValues(MLTextPropertyValue mlTextPropertyValue) + private List getLocalisedValues(MLTextPropertyValue mlTextPropertyValue) { if (mlTextPropertyValue == null) { @@ -2650,7 +2675,7 @@ public class SolrInformationServer implements InformationServer return values; } - private static void addMLTextProperty(BiConsumer consumer, FieldInstance field, MLTextPropertyValue mlTextPropertyValue) + private void addMLTextProperty(BiConsumer consumer, FieldInstance field, MLTextPropertyValue mlTextPropertyValue) { if (mlTextPropertyValue == null) { @@ -3700,7 +3725,7 @@ public class SolrInformationServer implements InformationServer } } - private static String getLocalisedValue(StringPropertyValue property, PropertyValue localeProperty) + private String getLocalisedValue(StringPropertyValue property, PropertyValue localeProperty) { Locale locale = ofNullable(localeProperty) @@ -3712,12 +3737,80 @@ public class SolrInformationServer implements InformationServer return "\u0000" + locale.getLanguage() + "\u0000" + property.getValue(); } - private static void addStringProperty(BiConsumer consumer, FieldInstance field, StringPropertyValue property, PropertyValue localeProperty) + private void addStringProperty(BiConsumer consumer, FieldInstance field, StringPropertyValue property, PropertyValue localeProperty, PropertyDefinition definition) { consumer.accept(field.getField(), field.isLocalised() ? getLocalisedValue(property, localeProperty) : property.getValue()); } - private static void addFieldIfNotSet(SolrInputDocument doc, String name) + /** + * Checks if the given property definition refers to a field that can be destructured in parts. + * A field can be destructured only if: + * + *
    + *
  • It is not multivalued
    • + *
  • Destructuring is not disabled in configuration
    • + *
  • Its data type is {@link DataTypeDefinition#DATE} or {@link DataTypeDefinition#DATETIME}
    • + *
+ */ + boolean canBeDestructured(PropertyDefinition definition, String fieldName) + { + return !definition.isMultiValued() && + fieldName != null && + fieldName.contains("@") && // avoid static date/datetime fields + dateFieldDestructuringHasBeenEnabledOnThisInstance && + (definition.getDataType().getName().equals(DataTypeDefinition.DATETIME) || + definition.getDataType().getName().equals(DataTypeDefinition.DATE)); + } + + /** + * To add support for SQL date functions SEARCH-2171 introduced date and datetime fields destructuration. + * Other than being indexed as plain TrieDate fields, date and datetime fields are also indexed in separate fields that + * contain their constituent parts. + * + * Date fields: + * + *
    + *
  • YEAR
    • + *
  • MONTH (1-12)
    • + *
  • DAY (OF THE MONTH)
    • + *
+ * + * Datetime fields also add the following: + * + *
    + *
  • HOUR (0-23)
    • + *
  • MINUTE
    • + *
  • SECOND
    • + *
+ * + * Note the destructured parts are not localised to a specific Timezone: they are always expressed in UTC. + * + * @see SEARCH-2171 + */ + void setUnitOfTimeFields(BiConsumer consumer, String sourceFieldName, String value, DataTypeDefinition dataType) + { + try + { + String fieldNamePrefix = dataModel.destructuredDateTimePartFieldNamePrefix(sourceFieldName, dataType); + ZonedDateTime dateTime = ZonedDateTime.parse(value, DateTimeFormatter.ISO_ZONED_DATE_TIME); + consumer.accept(fieldNamePrefix + UNIT_OF_TIME_YEAR_FIELD_SUFFIX, dateTime.getYear()); + consumer.accept(fieldNamePrefix + UNIT_OF_TIME_MONTH_FIELD_SUFFIX, dateTime.getMonth().getValue()); + consumer.accept(fieldNamePrefix + UNIT_OF_TIME_DAY_FIELD_SUFFIX, dateTime.getDayOfMonth()); + + if (DataTypeDefinition.DATETIME.equals(dataType.getName()) && isTimeComponentDefined(value)) + { + consumer.accept(fieldNamePrefix + UNIT_OF_TIME_MINUTE_FIELD_SUFFIX, dateTime.getMinute()); + consumer.accept(fieldNamePrefix + UNIT_OF_TIME_HOUR_FIELD_SUFFIX, dateTime.getHour()); + consumer.accept(fieldNamePrefix + UNIT_OF_TIME_SECOND_FIELD_SUFFIX, dateTime.getSecond()); + } + } + catch (Exception exception) + { + LOGGER.error("Unable to destructure date/datetime value {} (Field was {})", value, sourceFieldName, exception); + } + } + + private void addFieldIfNotSet(SolrInputDocument doc, String name) { doc.addField(FIELD_FIELDS, name); } @@ -3796,12 +3889,12 @@ public class SolrInformationServer implements InformationServer return searchers; } - private static void clearFields(SolrInputDocument document, List fields) + private void clearFields(SolrInputDocument document, List fields) { notNullOrEmpty(fields).forEach(document::removeField); } - private static void clearFields(SolrInputDocument document, String... fields) + private void clearFields(SolrInputDocument document, String... fields) { stream(notNullOrEmpty(fields)).forEach(document::removeField); } diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/schema.xml b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/schema.xml index 285258b9a..89a202500 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/schema.xml +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/schema.xml @@ -660,7 +660,19 @@ - + + + + + diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties index 33b09117e..15058b2e1 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties @@ -168,6 +168,16 @@ alfresco.metadata.ignore.datatype.1=app:configurations alfresco.metadata.skipDescendantDocsForSpecificAspects=false #alfresco.metadata.ignore.aspect.0= +# Date/Datetime fields only: if this property is set to true (default value) each date/datetime field +# +# - will be indexed as a whole value +# - will generate additional fields corresponding to its constituent parts (year, month, day, hour, minute, second) +# +# If this property is set to false then the only the whole value is indexed. +# This will result in a smaller index, but date function support will be disabled. +# +# alfresco.destructureDateFields=true + # The number of matches from the index to include when rewriting wildcard search terms as an OR-ed list. alfresco.topTermSpanRewriteLimit=1000 diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/schema.xml b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/schema.xml index 377d927f4..f34ba1061 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/schema.xml +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/schema.xml @@ -1067,6 +1067,18 @@ + + + + diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties index 1d33062d9..4df0106b1 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties @@ -165,6 +165,16 @@ alfresco.metadata.ignore.datatype.1=app:configurations alfresco.metadata.skipDescendantDocsForSpecificAspects=false #alfresco.metadata.ignore.aspect.0= +# Date/Datetime fields only: if this property is set to true (default value) each date/datetime field +# +# - will be indexed as a whole value +# - will generate additional fields corresponding to its constituent parts (year, month, day, hour, minute, second) +# +# If this property is set to false then the only the whole value is indexed. +# This will result in a smaller index, but date function support will be disabled. +# +# alfresco.destructureDateFields=true + # The number of matches from the index to include when rewriting wildcard search terms as an OR-ed list. alfresco.topTermSpanRewriteLimit=1000 diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadIT.java index 22f82c167..aad132767 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadIT.java @@ -86,7 +86,7 @@ public class AlfrescoSolrReloadIT extends AbstractAlfrescoSolrIT waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), numNodes, 100000); Collection trackers = getTrackers(); - + int numOfTrackers = trackers.size(); int jobs = getJobsCount(); @@ -125,4 +125,4 @@ public class AlfrescoSolrReloadIT extends AbstractAlfrescoSolrIT { return admin.getScheduler().getJobsCount(); } -} +} \ No newline at end of file diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrUtils.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrUtils.java index 53b042ad8..7ded65d02 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrUtils.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrUtils.java @@ -477,11 +477,14 @@ public class AlfrescoSolrUtils SolrServletRequest solrQueryRequest = null; try { + AlfrescoCoreAdminHandler admin = (AlfrescoCoreAdminHandler) core.getCoreContainer().getMultiCoreHandler(); + SolrInformationServer solrInformationServer = (SolrInformationServer) admin.getInformationServers().get(core.getName()); + solrQueryRequest = new SolrServletRequest(core, null); AddUpdateCommand addDocCmd = new AddUpdateCommand(solrQueryRequest); addDocCmd.overwrite = true; addDocCmd.solrDoc = createDocument(dataModel, new Long(txid), new Long(dbid), nodeRef, type, aspects, - properties, content, new Long(aclid), paths, owner, parentAssocs, ancestors); + properties, content, new Long(aclid), paths, owner, parentAssocs, ancestors, solrInformationServer); core.getUpdateHandler().addDoc(addDocCmd); if (commit) { @@ -528,7 +531,8 @@ public class AlfrescoSolrUtils String[] paths, String owner, ChildAssociationRef[] parentAssocs, - NodeRef[] ancestors)throws IOException + NodeRef[] ancestors, + SolrInformationServer solrInformationServer) { SolrInputDocument doc = new SolrInputDocument(); String id = AlfrescoSolrDataModel.getNodeDocumentId(AlfrescoSolrDataModel.DEFAULT_TENANT, dbid); @@ -615,7 +619,7 @@ public class AlfrescoSolrUtils { final boolean isContentIndexedForNode = true; final boolean transformContentFlag = true; - SolrInformationServer.populateProperties(properties, isContentIndexedForNode, doc, transformContentFlag); + solrInformationServer.populateProperties(properties, isContentIndexedForNode, doc, transformContentFlag); if (content != null) { addContentToDoc(doc, content); @@ -740,12 +744,15 @@ public class AlfrescoSolrUtils SolrServletRequest solrQueryRequest = null; try { + AlfrescoCoreAdminHandler admin = (AlfrescoCoreAdminHandler) core.getCoreContainer().getMultiCoreHandler(); + SolrInformationServer solrInformationServer = (SolrInformationServer) admin.getInformationServers().get(core.getName()); + solrQueryRequest = new SolrServletRequest(core, null); AddUpdateCommand addDocCmd = new AddUpdateCommand(solrQueryRequest); addDocCmd.overwrite = true; addDocCmd.solrDoc = createDocument(dataModel, new Long(txid), new Long(dbid), rootNodeRef, ContentModel.TYPE_STOREROOT, new QName[]{ContentModel.ASPECT_ROOT}, null, null, new Long(aclid), - new String[]{"/"}, "system", null, null); + new String[]{"/"}, "system", null, null, solrInformationServer); core.getUpdateHandler().addDoc(addDocCmd); addAcl(solrQueryRequest, core, dataModel, acltxid, aclid, 0, 0); AddUpdateCommand txCmd = new AddUpdateCommand(solrQueryRequest); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrInformationServerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrInformationServerTest.java index 05a56e277..82525c195 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrInformationServerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrInformationServerTest.java @@ -20,9 +20,13 @@ package org.alfresco.solr; import java.util.Properties; +import java.util.stream.Stream; +import org.alfresco.service.cmr.dictionary.DataTypeDefinition; +import org.alfresco.service.cmr.dictionary.PropertyDefinition; import org.alfresco.solr.client.SOLRAPIClient; import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.SolrCore; @@ -34,11 +38,40 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; +import static java.util.Optional.ofNullable; +import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.ANY; +import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.ASSOC_REF; +import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.BOOLEAN; +import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.CATEGORY; +import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.CHILD_ASSOC_REF; +import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.CONTENT; +import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.DOUBLE; +import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.ENCRYPTED; +import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.FLOAT; +import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.INT; +import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.LOCALE; +import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.LONG; +import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.MLTEXT; +import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.NODE_REF; +import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.PATH; +import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.PERIOD; +import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.QNAME; +import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.TEXT; +import static org.alfresco.solr.SolrInformationServer.UNIT_OF_TIME_DAY_FIELD_SUFFIX; +import static org.alfresco.solr.SolrInformationServer.UNIT_OF_TIME_HOUR_FIELD_SUFFIX; +import static org.alfresco.solr.SolrInformationServer.UNIT_OF_TIME_MINUTE_FIELD_SUFFIX; +import static org.alfresco.solr.SolrInformationServer.UNIT_OF_TIME_MONTH_FIELD_SUFFIX; +import static org.alfresco.solr.SolrInformationServer.UNIT_OF_TIME_SECOND_FIELD_SUFFIX; +import static org.alfresco.solr.SolrInformationServer.UNIT_OF_TIME_YEAR_FIELD_SUFFIX; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -112,6 +145,147 @@ public class SolrInformationServerTest assertSame(state, document); } + @Test + public void setUnitOfTimeFieldsWithDatetimeField_shouldSetDateAndTimeFields() + { + DataTypeDefinition datatype = mock(DataTypeDefinition.class); + when(datatype.getName()).thenReturn(DataTypeDefinition.DATETIME); + + SolrInputDocument document = new SolrInputDocument(); + + String fieldPrefix = "datetime@sd@"; + String fieldSuffix = "{http://www.alfresco.org/model/content/1.0}created"; + String sourceFieldName = fieldPrefix + fieldSuffix; + + infoServer.setUnitOfTimeFields(document::setField, sourceFieldName, "1972-09-16T17:33:18Z", datatype); + + assertEquals(1972, + unitOfTimeFieldValue(document, AlfrescoSolrDataModel.PART_FIELDNAME_PREFIX + fieldSuffix + UNIT_OF_TIME_YEAR_FIELD_SUFFIX)); + assertEquals(9, + unitOfTimeFieldValue(document, AlfrescoSolrDataModel.PART_FIELDNAME_PREFIX + fieldSuffix + UNIT_OF_TIME_MONTH_FIELD_SUFFIX)); + assertEquals(16, + unitOfTimeFieldValue(document, AlfrescoSolrDataModel.PART_FIELDNAME_PREFIX + fieldSuffix + UNIT_OF_TIME_DAY_FIELD_SUFFIX)); + assertEquals(17, + unitOfTimeFieldValue(document, AlfrescoSolrDataModel.PART_FIELDNAME_PREFIX + fieldSuffix + UNIT_OF_TIME_HOUR_FIELD_SUFFIX)); + assertEquals(33, + unitOfTimeFieldValue(document, AlfrescoSolrDataModel.PART_FIELDNAME_PREFIX + fieldSuffix + UNIT_OF_TIME_MINUTE_FIELD_SUFFIX)); + assertEquals(18, + unitOfTimeFieldValue(document, AlfrescoSolrDataModel.PART_FIELDNAME_PREFIX + fieldSuffix + UNIT_OF_TIME_SECOND_FIELD_SUFFIX)); + } + + @Test + public void setUnitOfTimeFieldsWithDatetimeField_shouldSetOnlyDateFields() + { + DataTypeDefinition datatype = mock(DataTypeDefinition.class); + when(datatype.getName()).thenReturn(DataTypeDefinition.DATE); + + SolrInputDocument document = new SolrInputDocument(); + + String fieldPrefix = "datetime@sd@"; + String fieldSuffix = "{http://www.alfresco.org/model/content/1.0}created"; + String sourceFieldName = fieldPrefix + fieldSuffix; + + infoServer.setUnitOfTimeFields(document::setField, sourceFieldName, "1972-09-16T17:33:18Z", datatype); + + assertEquals(1972, + unitOfTimeFieldValue(document, AlfrescoSolrDataModel.PART_FIELDNAME_PREFIX + fieldSuffix + UNIT_OF_TIME_YEAR_FIELD_SUFFIX)); + assertEquals(9, + unitOfTimeFieldValue(document, AlfrescoSolrDataModel.PART_FIELDNAME_PREFIX + fieldSuffix + UNIT_OF_TIME_MONTH_FIELD_SUFFIX)); + assertEquals(16, + unitOfTimeFieldValue(document, AlfrescoSolrDataModel.PART_FIELDNAME_PREFIX + fieldSuffix + UNIT_OF_TIME_DAY_FIELD_SUFFIX)); + + assertFalse(document.containsKey(AlfrescoSolrDataModel.PART_FIELDNAME_PREFIX + fieldSuffix + UNIT_OF_TIME_HOUR_FIELD_SUFFIX)); + assertFalse(document.containsKey(AlfrescoSolrDataModel.PART_FIELDNAME_PREFIX + fieldSuffix + UNIT_OF_TIME_MINUTE_FIELD_SUFFIX)); + assertFalse(document.containsKey(AlfrescoSolrDataModel.PART_FIELDNAME_PREFIX + fieldSuffix + UNIT_OF_TIME_SECOND_FIELD_SUFFIX)); + } + + @Test + public void destructuringCannotBeAppliedToMultivaluedFields() + { + PropertyDefinition multiValuedProperty = mock(PropertyDefinition.class); + when(multiValuedProperty.isMultiValued()).thenReturn(true); + + assertFalse(infoServer.canBeDestructured(multiValuedProperty, "datetime@sd@{http://www.alfresco.org/model/content/1.0}created")); + } + + /** + * Destructuring cannot only be applied to those fields matching a dynamic field in the schema. + * The dynamic fields defined in the schema follow the naming: + * + * [datatype]@[options]@[namespace][local name] + * + * For example + * + * datetime@sd@{http://www.alfresco.org/model/content/1.0}created + * + * SearchServices calls those fields "PROPERTIES", while the static fields (even belonging to a Node) are called + * "FIELDS". + */ + @Test + public void destructuringCannotBeAppliedToProperties() + { + PropertyDefinition multiValuedProperty = mock(PropertyDefinition.class); + when(multiValuedProperty.isMultiValued()).thenReturn(false); + + assertFalse( + "Destructuring can only be applied to Node properties (e.g. dynamic fields in Solr schema)", + infoServer.canBeDestructured(multiValuedProperty, "this_is_a_field_that_doesnt_follow_the_dynamic_naming")); + } + + @Test + public void destructuringCanBeAppliedToDateFields() + { + PropertyDefinition propertyThatCanBeDestructured = mock(PropertyDefinition.class); + when(propertyThatCanBeDestructured.isMultiValued()).thenReturn(false); + + DataTypeDefinition date = mock(DataTypeDefinition.class); + when(date.getName()).thenReturn(DataTypeDefinition.DATE); + when(propertyThatCanBeDestructured.getDataType()).thenReturn(date); + + assertTrue( + "Destructuring must be supported in Date fields!", + infoServer.canBeDestructured(propertyThatCanBeDestructured,"date@sd@{http://www.alfresco.org/model/content/1.0}created")); + } + + @Test + public void destructuringCanBeAppliedToDateTimeFields() + { + PropertyDefinition propertyThatCanBeDestructured = mock(PropertyDefinition.class); + when(propertyThatCanBeDestructured.isMultiValued()).thenReturn(false); + + DataTypeDefinition datetime = mock(DataTypeDefinition.class); + when(datetime.getName()).thenReturn(DataTypeDefinition.DATETIME); + when(propertyThatCanBeDestructured.getDataType()).thenReturn(datetime); + + assertTrue( + "Destructuring must be supported in Datetime fields!", + infoServer.canBeDestructured(propertyThatCanBeDestructured, "datetime@sd@{http://www.alfresco.org/model/content/1.0}created")); + } + + @Test + public void destructuringCanBeAppliedOnlyToDateOrDatetimeFields() + { + Stream.of(ANY,ENCRYPTED,TEXT,MLTEXT,CONTENT,INT,LONG,FLOAT,DOUBLE, + BOOLEAN,QNAME,CATEGORY,NODE_REF,CHILD_ASSOC_REF,ASSOC_REF, + PATH,LOCALE,PERIOD) + .map(qname -> { + PropertyDefinition propertyThatCannotBeDestructured = mock(PropertyDefinition.class); + DataTypeDefinition def = Mockito.mock(DataTypeDefinition.class); + when(def.getName()).thenReturn(qname); + when(propertyThatCannotBeDestructured.getDataType()).thenReturn(def); + return propertyThatCannotBeDestructured;}) + .forEach(property -> assertFalse( + "Destructuring must be supported only on Date or Datetime fields!", + infoServer.canBeDestructured(property, "somedatatype@sd@{http://www.alfresco.org/model/content/1.0}somefield"))); + } + + private int unitOfTimeFieldValue(SolrInputDocument doc, String fieldName) { + return ofNullable(doc.getFieldValue(fieldName)) + .map(Number.class::cast) + .map(Number::intValue) + .orElseThrow(() -> new IllegalArgumentException(fieldName + " hasn't been set.")); + } + /** * GetState returns null in case the given id doesn't correspond to an existing state document. */ @@ -134,4 +308,4 @@ public class SolrInformationServerTest assertNull(document); } -} +} \ No newline at end of file